"""Tests for S3 Transfer Module. Tests cover S3Config validation, S3TransferManager, and transfer_to_s3_if_enabled using mocked boto3 calls. No real AWS connections are made. """ import pytest from pathlib import Path from unittest.mock import patch, MagicMock from s3_transfer import ( S3Config, S3TransferManager, S3UploadResult, S3Error, S3ConfigurationError, S3UploadError, transfer_to_s3_if_enabled, ) # ==================== S3Config Tests ==================== def test_s3_config_valid(): """Test S3Config with valid parameters.""" cfg = S3Config( bucket='my-bucket', folder='my-folder', region='us-west-2', retries=5, ) assert cfg.bucket == 'my-bucket' assert cfg.sanitized_folder == 'my-folder' assert cfg.region == 'us-west-2' assert cfg.retries == 5 def test_s3_config_empty_bucket_raises(): """Test that empty bucket raises S3ConfigurationError.""" with pytest.raises(S3ConfigurationError, match="bucket is required"): S3Config(bucket='', folder='test') def test_s3_config_folder_traversal_raises(): """Test that '..' in folder raises S3ConfigurationError.""" with pytest.raises(S3ConfigurationError, match="traversal"): S3Config(bucket='bucket', folder='../etc/passwd') def test_s3_config_sanitizes_folder(): """Test that folder is sanitized (leading/trailing slashes removed).""" cfg = S3Config(bucket='bucket', folder='/leading/trailing/') assert cfg.sanitized_folder == 'leading/trailing' def test_s3_config_defaults(): """Test S3Config default values.""" cfg = S3Config(bucket='bucket') assert cfg.folder == '' assert cfg.sanitized_folder == '' assert cfg.region == 'us-east-1' assert cfg.retries == 3 assert cfg.backoff_sec == 2.0 assert cfg.timeout == 60.0 assert cfg.continue_on_error is False # ==================== S3UploadResult Tests ==================== def test_s3_upload_result_success(): """Test S3UploadResult.success() returns True when all uploaded.""" result = S3UploadResult(uploaded=5, failed=[], total=5) assert result.success() is True def test_s3_upload_result_partial_failure(): """Test S3UploadResult.success() returns False on failures.""" result = S3UploadResult( uploaded=3, failed=[Path('/tmp/failed.txt')], total=4, ) assert result.success() is False def test_s3_upload_result_s3_keys(): """Test S3UploadResult stores S3 keys.""" result = S3UploadResult( uploaded=2, failed=[], total=2, s3_keys=['folder/file1.txt.gz', 'folder/file2.txt.gz'], ) assert len(result.s3_keys) == 2 assert 'folder/file1.txt.gz' in result.s3_keys # ==================== S3TransferManager Tests ==================== def test_s3_manager_connect_not_connected_error(): """Test upload_file raises error when not connected.""" cfg = S3Config(bucket='bucket', folder='folder') mgr = S3TransferManager(cfg) with pytest.raises(S3Error, match="not connected"): mgr.upload_file(Path('/tmp/test.txt')) @patch('s3_transfer.boto3.client') def test_s3_manager_connect_success(mock_boto_client): """Test successful S3 connection.""" mock_client = MagicMock() mock_boto_client.return_value = mock_client mock_client.head_bucket.return_value = {} cfg = S3Config(bucket='test-bucket', region='us-east-1') mgr = S3TransferManager(cfg) mgr.connect() mock_boto_client.assert_called_once_with('s3', region_name='us-east-1') mock_client.head_bucket.assert_called_once_with(Bucket='test-bucket') assert mgr._connected is True @patch('s3_transfer.boto3.client') def test_s3_manager_connect_bucket_not_found(mock_boto_client): """Test S3ConfigurationError when bucket doesn't exist.""" from botocore.exceptions import ClientError mock_client = MagicMock() mock_boto_client.return_value = mock_client mock_client.head_bucket.side_effect = ClientError( {'Error': {'Code': '404', 'Message': 'Not Found'}}, 'HeadBucket' ) cfg = S3Config(bucket='nonexistent-bucket') mgr = S3TransferManager(cfg) with pytest.raises(S3ConfigurationError, match="does not exist"): mgr.connect() @patch('s3_transfer.boto3.client') def test_s3_manager_connect_access_denied(mock_boto_client): """Test S3ConfigurationError when access is denied.""" from botocore.exceptions import ClientError mock_client = MagicMock() mock_boto_client.return_value = mock_client mock_client.head_bucket.side_effect = ClientError( {'Error': {'Code': '403', 'Message': 'Forbidden'}}, 'HeadBucket' ) cfg = S3Config(bucket='private-bucket') mgr = S3TransferManager(cfg) with pytest.raises(S3ConfigurationError, match="Access denied"): mgr.connect() def test_s3_manager_mock_mode(monkeypatch): """Test S3 mock mode creates dummy client.""" monkeypatch.setenv('S3_MOCK', 'True') cfg = S3Config(bucket='bucket', folder='folder') mgr = S3TransferManager(cfg) mgr.connect() assert mgr._connected is True assert hasattr(mgr._s3_client, 'uploads') # Mock client attribute @patch('s3_transfer.boto3.client') def test_s3_manager_upload_file_success(mock_boto_client, tmp_path): """Test successful file upload.""" mock_client = MagicMock() mock_boto_client.return_value = mock_client mock_client.head_bucket.return_value = {} # Create test file test_file = tmp_path / 'test.txt.gz' test_file.write_text('test content') cfg = S3Config(bucket='bucket', folder='my-prefix') mgr = S3TransferManager(cfg) mgr.connect() s3_key = mgr.upload_file(test_file) assert s3_key == 'my-prefix/test.txt.gz' mock_client.upload_file.assert_called_once_with( str(test_file), 'bucket', 'my-prefix/test.txt.gz', ) @patch('s3_transfer.boto3.client') def test_s3_manager_upload_file_with_retries(mock_boto_client, tmp_path): """Test upload retries on failure then succeeds.""" mock_client = MagicMock() mock_boto_client.return_value = mock_client mock_client.head_bucket.return_value = {} # Fail twice, then succeed mock_client.upload_file.side_effect = [ Exception('Timeout'), Exception('Connection reset'), None, # Success ] test_file = tmp_path / 'test.txt' test_file.write_text('content') cfg = S3Config( bucket='bucket', folder='prefix', retries=3, backoff_sec=0.01, # Fast for tests ) mgr = S3TransferManager(cfg) mgr.connect() s3_key = mgr.upload_file(test_file) assert s3_key == 'prefix/test.txt' assert mock_client.upload_file.call_count == 3 @patch('s3_transfer.boto3.client') def test_s3_manager_upload_file_all_retries_fail(mock_boto_client, tmp_path): """Test S3UploadError when all retries fail.""" mock_client = MagicMock() mock_boto_client.return_value = mock_client mock_client.head_bucket.return_value = {} mock_client.upload_file.side_effect = Exception('Persistent failure') test_file = tmp_path / 'test.txt' test_file.write_text('content') cfg = S3Config( bucket='bucket', retries=2, backoff_sec=0.01, ) mgr = S3TransferManager(cfg) mgr.connect() with pytest.raises(S3UploadError, match="Failed to upload"): mgr.upload_file(test_file) @patch('s3_transfer.boto3.client') def test_s3_manager_upload_files_batch(mock_boto_client, tmp_path): """Test batch upload of multiple files.""" mock_client = MagicMock() mock_boto_client.return_value = mock_client mock_client.head_bucket.return_value = {} # Create test files files = [] for i in range(3): f = tmp_path / f'file{i}.txt.gz' f.write_text(f'content {i}') files.append(f) cfg = S3Config(bucket='bucket', folder='batch') mgr = S3TransferManager(cfg) mgr.connect() result = mgr.upload_files(files) assert result.uploaded == 3 assert result.total == 3 assert result.success() is True assert len(result.s3_keys) == 3 @patch('s3_transfer.boto3.client') def test_s3_manager_upload_files_continue_on_error(mock_boto_client, tmp_path): """Test batch upload continues on error when configured.""" mock_client = MagicMock() mock_boto_client.return_value = mock_client mock_client.head_bucket.return_value = {} # Second file fails mock_client.upload_file.side_effect = [ None, # file0 success Exception('Failed'), # file1 fails (all retries) Exception('Failed'), Exception('Failed'), Exception('Failed'), None, # file2 success ] files = [] for i in range(3): f = tmp_path / f'file{i}.txt' f.write_text(f'content {i}') files.append(f) cfg = S3Config( bucket='bucket', folder='batch', continue_on_error=True, retries=3, backoff_sec=0.01, ) mgr = S3TransferManager(cfg) mgr.connect() result = mgr.upload_files(files) assert result.uploaded == 2 assert len(result.failed) == 1 assert result.failed[0].name == 'file1.txt' assert result.success() is False @patch('s3_transfer.boto3.client') def test_s3_manager_upload_files_abort_on_error(mock_boto_client, tmp_path): """Test batch upload aborts on first error when not continue_on_error.""" mock_client = MagicMock() mock_boto_client.return_value = mock_client mock_client.head_bucket.return_value = {} # First file fails (after retries) mock_client.upload_file.side_effect = Exception('Failed') files = [] for i in range(3): f = tmp_path / f'file{i}.txt' f.write_text(f'content {i}') files.append(f) cfg = S3Config( bucket='bucket', folder='batch', continue_on_error=False, retries=0, backoff_sec=0.01, ) mgr = S3TransferManager(cfg) mgr.connect() result = mgr.upload_files(files) # Should abort after first failure assert result.uploaded == 0 assert len(result.failed) == 1 assert result.total == 3 def test_s3_manager_close(): """Test close resets connection state.""" cfg = S3Config(bucket='bucket') mgr = S3TransferManager(cfg) mgr._connected = True mgr._s3_client = MagicMock() mgr.close() assert mgr._connected is False assert mgr._s3_client is None # ==================== transfer_to_s3_if_enabled Tests ==================== def test_transfer_to_s3_if_enabled_empty_files(): """Test transfer with empty file list returns None.""" cfg = S3Config(bucket='bucket') result = transfer_to_s3_if_enabled([], cfg) assert result is None @patch('s3_transfer.boto3.client') def test_transfer_to_s3_if_enabled_success(mock_boto_client, tmp_path): """Test transfer_to_s3_if_enabled with valid files.""" mock_client = MagicMock() mock_boto_client.return_value = mock_client mock_client.head_bucket.return_value = {} test_file = tmp_path / 'test.txt.gz' test_file.write_text('test') cfg = S3Config(bucket='bucket', folder='prefix') result = transfer_to_s3_if_enabled([str(test_file)], cfg) assert result is not None assert result.success() is True assert result.uploaded == 1 def test_transfer_to_s3_if_enabled_mock_mode(monkeypatch, tmp_path): """Test transfer with mock mode enabled.""" monkeypatch.setenv('S3_MOCK', 'True') test_file = tmp_path / 'test.txt.gz' test_file.write_text('test') cfg = S3Config(bucket='bucket', folder='mock-prefix') result = transfer_to_s3_if_enabled([str(test_file)], cfg) assert result is not None assert result.success() is True # ==================== S3 Key Construction Tests ==================== @patch('s3_transfer.boto3.client') def test_s3_key_with_folder(mock_boto_client, tmp_path): """Test S3 key includes folder prefix.""" mock_client = MagicMock() mock_boto_client.return_value = mock_client mock_client.head_bucket.return_value = {} test_file = tmp_path / 'data.txt.gz' test_file.write_text('data') cfg = S3Config(bucket='bucket', folder='settlement/2026') mgr = S3TransferManager(cfg) mgr.connect() s3_key = mgr.upload_file(test_file) assert s3_key == 'settlement/2026/data.txt.gz' @patch('s3_transfer.boto3.client') def test_s3_key_without_folder(mock_boto_client, tmp_path): """Test S3 key without folder prefix (root bucket).""" mock_client = MagicMock() mock_boto_client.return_value = mock_client mock_client.head_bucket.return_value = {} test_file = tmp_path / 'data.txt.gz' test_file.write_text('data') cfg = S3Config(bucket='bucket', folder='') mgr = S3TransferManager(cfg) mgr.connect() s3_key = mgr.upload_file(test_file) assert s3_key == 'data.txt.gz'