"""Unit tests for S3Downloader.""" import os import tempfile from unittest.mock import Mock, patch import pytest from src.errors import ( ChecksumMismatchError, FileIntegrityError, FileSizeExceededError, FileSystemError, S3FileNotFoundError, TransientError, ) from src.services.s3_downloader import S3Downloader class TestS3Downloader: """Tests for S3Downloader class.""" @pytest.fixture def mock_s3_connector(self): """Mock S3 connector.""" return Mock() @pytest.fixture def s3_metadata(self): """Sample S3 file metadata.""" metadata = Mock() metadata.size = 1024 * 1024 # 1 MB metadata.etag = 'abc123' # Simple ETag (not multipart) return metadata @pytest.fixture def temp_file_path(self): """Create a temporary file path.""" with tempfile.NamedTemporaryFile(delete=False) as f: file_path = f.name yield file_path # Cleanup if os.path.exists(file_path): os.remove(file_path) def test_init_sets_max_bytes(self, mock_s3_connector): """Test initialization sets max bytes.""" downloader = S3Downloader(mock_s3_connector, max_bytes=1073741824) assert downloader._s3_connection == mock_s3_connector assert downloader._max_bytes == 1073741824 def test_init_with_custom_max_bytes(self, mock_s3_connector): """Test initialization with custom max bytes.""" downloader = S3Downloader(mock_s3_connector, max_bytes=5000000) assert downloader._max_bytes == 5000000 @patch('src.services.s3_downloader.try_disk_space') @patch('src.services.s3_downloader.get_dir_path') @patch('src.services.s3_downloader.compute_md5') def test_download_success( self, mock_compute_md5, mock_get_dir_path, mock_try_disk_space, mock_s3_connector, s3_metadata, temp_file_path, ): """Test successful download with checksum verification.""" mock_s3_connector.get_file_metadata.return_value = s3_metadata mock_get_dir_path.return_value = '/tmp' mock_compute_md5.return_value = 'abc123' # Create a test file to simulate download with open(temp_file_path, 'wb') as f: f.write(b'x' * (1024 * 1024)) downloader = S3Downloader(mock_s3_connector, max_bytes=1073741824) file_size = downloader.download('test-bucket', 'test-key', temp_file_path) assert file_size == 1024 * 1024 mock_s3_connector.get_file_metadata.assert_called_once_with( 'test-bucket', 'test-key' ) mock_try_disk_space.assert_called_once() mock_s3_connector.download_file.assert_called_once_with( 'test-bucket', 'test-key', temp_file_path ) mock_compute_md5.assert_called_once_with(temp_file_path) @patch('src.services.s3_downloader.try_disk_space') @patch('src.services.s3_downloader.get_dir_path') @patch('src.services.s3_downloader.is_simple_etag') def test_download_success_multipart_etag( self, mock_is_simple_etag, mock_get_dir_path, mock_try_disk_space, mock_s3_connector, s3_metadata, temp_file_path, ): """Test successful download with multipart ETag (no checksum verification).""" s3_metadata.etag = 'abc123-2' # Multipart ETag mock_s3_connector.get_file_metadata.return_value = s3_metadata mock_get_dir_path.return_value = '/tmp' mock_is_simple_etag.return_value = False # Create a test file to simulate download with open(temp_file_path, 'wb') as f: f.write(b'x' * (1024 * 1024)) downloader = S3Downloader(mock_s3_connector, max_bytes=1073741824) file_size = downloader.download('test-bucket', 'test-key', temp_file_path) assert file_size == 1024 * 1024 # Checksum should be skipped for multipart uploads mock_is_simple_etag.assert_called_once_with('abc123-2') def test_download_file_not_found(self, mock_s3_connector): """Test download raises error when file not found.""" mock_s3_connector.get_file_metadata.return_value = None downloader = S3Downloader(mock_s3_connector, max_bytes=1073741824) with pytest.raises(S3FileNotFoundError, match='S3 file not found'): downloader.download('test-bucket', 'test-key', '/tmp/test.csv') def test_download_file_size_exceeded(self, mock_s3_connector, s3_metadata): """Test download raises error when file size exceeds limit.""" s3_metadata.size = 200 * 1024 * 1024 # 200 MB mock_s3_connector.get_file_metadata.return_value = s3_metadata downloader = S3Downloader(mock_s3_connector, max_bytes=100 * 1024 * 1024) with pytest.raises(FileSizeExceededError, match='File size exceeds maximum'): downloader.download('test-bucket', 'test-key', '/tmp/test.csv') @patch('src.services.s3_downloader.try_disk_space') @patch('src.services.s3_downloader.get_dir_path') def test_download_insufficient_disk_space( self, mock_get_dir_path, mock_try_disk_space, mock_s3_connector, s3_metadata, ): """Test download raises error when insufficient disk space.""" mock_s3_connector.get_file_metadata.return_value = s3_metadata mock_get_dir_path.return_value = '/tmp' mock_try_disk_space.side_effect = TransientError('Insufficient disk space') downloader = S3Downloader(mock_s3_connector, max_bytes=1073741824) with pytest.raises(TransientError, match='Insufficient disk space'): downloader.download('test-bucket', 'test-key', '/tmp/test.csv') @patch('src.services.s3_downloader.try_disk_space') @patch('src.services.s3_downloader.get_dir_path') @patch('os.path.getsize') def test_download_file_integrity_error( self, mock_getsize, mock_get_dir_path, mock_try_disk_space, mock_s3_connector, s3_metadata, temp_file_path, ): """Test download raises error when file size doesn't match.""" mock_s3_connector.get_file_metadata.return_value = s3_metadata mock_get_dir_path.return_value = '/tmp' # Downloaded file has wrong size mock_getsize.return_value = 512 * 1024 # Expected 1 MB, got 512 KB downloader = S3Downloader(mock_s3_connector, max_bytes=1073741824) with pytest.raises(FileIntegrityError, match='Download size mismatch'): downloader.download('test-bucket', 'test-key', temp_file_path) @patch('src.services.s3_downloader.try_disk_space') @patch('src.services.s3_downloader.get_dir_path') @patch('os.path.getsize') def test_download_file_integrity_error_os_error( self, mock_getsize, mock_get_dir_path, mock_try_disk_space, mock_s3_connector, s3_metadata, temp_file_path, ): """Test download raises error when unable to verify file size.""" mock_s3_connector.get_file_metadata.return_value = s3_metadata mock_get_dir_path.return_value = '/tmp' mock_getsize.side_effect = OSError('Cannot access file') downloader = S3Downloader(mock_s3_connector, max_bytes=1073741824) with pytest.raises(FileSystemError, match='Cannot verify download'): downloader.download('test-bucket', 'test-key', temp_file_path) @patch('src.services.s3_downloader.try_disk_space') @patch('src.services.s3_downloader.get_dir_path') @patch('src.services.s3_downloader.compute_md5') def test_download_checksum_mismatch( self, mock_compute_md5, mock_get_dir_path, mock_try_disk_space, mock_s3_connector, s3_metadata, temp_file_path, ): """Test download raises error when checksum doesn't match.""" mock_s3_connector.get_file_metadata.return_value = s3_metadata mock_get_dir_path.return_value = '/tmp' mock_compute_md5.return_value = 'wrong_checksum' # Create a test file to simulate download with open(temp_file_path, 'wb') as f: f.write(b'x' * (1024 * 1024)) downloader = S3Downloader(mock_s3_connector, max_bytes=1073741824) with pytest.raises(ChecksumMismatchError, match='MD5 checksum mismatch'): downloader.download('test-bucket', 'test-key', temp_file_path) @patch('src.services.s3_downloader.try_disk_space') @patch('src.services.s3_downloader.get_dir_path') @patch('src.services.s3_downloader.compute_md5') def test_download_checksum_computation_error( self, mock_compute_md5, mock_get_dir_path, mock_try_disk_space, mock_s3_connector, s3_metadata, temp_file_path, ): """Test download raises error when unable to compute checksum.""" mock_s3_connector.get_file_metadata.return_value = s3_metadata mock_get_dir_path.return_value = '/tmp' mock_compute_md5.side_effect = OSError('Cannot read file') # Create a test file to simulate download with open(temp_file_path, 'wb') as f: f.write(b'x' * (1024 * 1024)) downloader = S3Downloader(mock_s3_connector, max_bytes=1073741824) with pytest.raises(FileSystemError, match='Cannot compute checksum'): downloader.download('test-bucket', 'test-key', temp_file_path) @patch('os.remove') def test_cleanup_file_success(self, mock_remove, mock_s3_connector): """Test cleanup_file successfully removes file.""" downloader = S3Downloader(mock_s3_connector, max_bytes=1073741824) downloader._cleanup_file('/tmp/test.csv') mock_remove.assert_called_once_with('/tmp/test.csv') @patch('os.remove') def test_cleanup_file_file_not_found(self, mock_remove, mock_s3_connector): """Test cleanup_file handles FileNotFoundError gracefully.""" mock_remove.side_effect = FileNotFoundError('File does not exist') downloader = S3Downloader(mock_s3_connector, max_bytes=1073741824) # Should not raise exception downloader._cleanup_file('/tmp/test.csv') mock_remove.assert_called_once_with('/tmp/test.csv') @patch('os.remove') def test_cleanup_file_os_error(self, mock_remove, mock_s3_connector): """Test cleanup_file handles OS error gracefully.""" mock_remove.side_effect = OSError('Permission denied') downloader = S3Downloader(mock_s3_connector, max_bytes=1073741824) # Should not raise exception downloader._cleanup_file('/tmp/test.csv') mock_remove.assert_called_once_with('/tmp/test.csv') @patch('src.services.s3_downloader.is_simple_etag') def test_verify_file_checksum_simple_etag_success( self, mock_is_simple_etag, mock_s3_connector, temp_file_path ): """Test verify_file_checksum with simple ETag succeeds.""" mock_is_simple_etag.return_value = True with open(temp_file_path, 'wb') as f: f.write(b'test content') downloader = S3Downloader(mock_s3_connector, max_bytes=1073741824) with patch('src.services.s3_downloader.compute_md5', return_value='abc123'): # Should not raise exception downloader._verify_file_checksum(temp_file_path, 'abc123') @patch('src.services.s3_downloader.is_simple_etag') def test_verify_file_checksum_multipart_etag_skipped( self, mock_is_simple_etag, mock_s3_connector, temp_file_path ): """Test verify_file_checksum skips verification for multipart ETag.""" mock_is_simple_etag.return_value = False downloader = S3Downloader(mock_s3_connector, max_bytes=1073741824) # Should not raise exception and should skip checksum downloader._verify_file_checksum(temp_file_path, 'abc123-2') mock_is_simple_etag.assert_called_once_with('abc123-2') @patch('src.services.s3_downloader.is_simple_etag') @patch('src.services.s3_downloader.compute_md5') def test_verify_file_checksum_mismatch_cleans_up( self, mock_compute_md5, mock_is_simple_etag, mock_s3_connector, temp_file_path ): """Test verify_file_checksum cleans up file on checksum mismatch.""" mock_is_simple_etag.return_value = True mock_compute_md5.return_value = 'wrong_checksum' with open(temp_file_path, 'wb') as f: f.write(b'test content') downloader = S3Downloader(mock_s3_connector, max_bytes=1073741824) with patch.object(downloader, '_cleanup_file') as mock_cleanup: with pytest.raises(ChecksumMismatchError): downloader._verify_file_checksum(temp_file_path, 'abc123') mock_cleanup.assert_called_once_with(temp_file_path) @patch('src.services.s3_downloader.is_simple_etag') @patch('src.services.s3_downloader.compute_md5') def test_verify_file_checksum_computation_error_cleans_up( self, mock_compute_md5, mock_is_simple_etag, mock_s3_connector, temp_file_path ): """Test verify_file_checksum cleans up file on computation error.""" mock_is_simple_etag.return_value = True mock_compute_md5.side_effect = OSError('Read error') downloader = S3Downloader(mock_s3_connector, max_bytes=1073741824) with patch.object(downloader, '_cleanup_file') as mock_cleanup: with pytest.raises(FileSystemError): downloader._verify_file_checksum(temp_file_path, 'abc123') mock_cleanup.assert_called_once_with(temp_file_path) @patch('os.path.getsize') def test_verify_file_size_success( self, mock_getsize, mock_s3_connector, temp_file_path ): """Test verify_file_size succeeds with matching size.""" mock_getsize.return_value = 1024 downloader = S3Downloader(mock_s3_connector, max_bytes=1073741824) # Should not raise exception downloader._verify_file_size(temp_file_path, 1024) mock_getsize.assert_called_once_with(temp_file_path) @patch('os.path.getsize') def test_verify_file_size_mismatch_cleans_up( self, mock_getsize, mock_s3_connector, temp_file_path ): """Test verify_file_size cleans up file on size mismatch.""" mock_getsize.return_value = 512 downloader = S3Downloader(mock_s3_connector, max_bytes=1073741824) with patch.object(downloader, '_cleanup_file') as mock_cleanup: with pytest.raises(FileIntegrityError, match='Download size mismatch'): downloader._verify_file_size(temp_file_path, 1024) mock_cleanup.assert_called_once_with(temp_file_path) @patch('os.path.getsize') def test_verify_file_size_os_error_cleans_up( self, mock_getsize, mock_s3_connector, temp_file_path ): """Test verify_file_size cleans up file on OS error.""" mock_getsize.side_effect = OSError('File not found') downloader = S3Downloader(mock_s3_connector, max_bytes=1073741824) with patch.object(downloader, '_cleanup_file') as mock_cleanup: with pytest.raises(FileSystemError, match='Cannot verify download'): downloader._verify_file_size(temp_file_path, 1024) mock_cleanup.assert_called_once_with(temp_file_path) @patch('src.services.s3_downloader.try_disk_space') @patch('src.services.s3_downloader.get_dir_path') def test_download_at_max_size_boundary( self, mock_get_dir_path, mock_try_disk_space, mock_s3_connector, s3_metadata, temp_file_path, ): """Test download succeeds when file size equals max_bytes.""" s3_metadata.size = 100 * 1024 * 1024 # 100 MB s3_metadata.etag = 'abc123-2' # Multipart to skip checksum mock_s3_connector.get_file_metadata.return_value = s3_metadata mock_get_dir_path.return_value = '/tmp' # Create a test file to simulate download with open(temp_file_path, 'wb') as f: f.write(b'x' * (100 * 1024 * 1024)) downloader = S3Downloader(mock_s3_connector, max_bytes=100 * 1024 * 1024) with patch('src.services.s3_downloader.is_simple_etag', return_value=False): file_size = downloader.download('test-bucket', 'test-key', temp_file_path) assert file_size == 100 * 1024 * 1024