"""Unit tests for processor module.""" from unittest.mock import Mock, patch import pytest from src.constants import AdjustmentInputSchema from src.enums import BatchErrorCode, BatchStatus, EventType from src.errors import ( EmptyFileError, MissingHeadersError, RowCountExceededError, S3FileNotFoundError, ) from src.schemas import ( AdjustmentFilePrepareEvent, AdjustmentFilePrepareEventData, AdjustmentFilePrepareEventDetail, AdjustmentFilePrepareEventMetadata, ) from src.services.processor import AdjustmentFilePrepareProcessor class TestAdjustmentFilePrepareProcessor: """Tests for AdjustmentFilePrepareProcessor.""" @pytest.fixture def mock_royalty_accounting_client(self): """Mock RoyaltyAccountingClient.""" mock = Mock() mock.update_batch_status.return_value = 1 # Default return value mock.delete_batch_from_staging.return_value = 0 # Default return value mock_cursor = Mock() mock_cursor.rowcount = 100 mock_cursor.__enter__ = Mock(return_value=mock_cursor) mock_cursor.__exit__ = Mock(return_value=False) mock_conn = Mock() mock_conn.cursor.return_value = mock_cursor mock.conn = mock_conn return mock @pytest.fixture def mock_s3_connector(self): """Mock S3Connector.""" mock = Mock() # Default return value for file metadata with size (10 MB) metadata = Mock() metadata.size = 10 * 1024 * 1024 mock.get_file_metadata.return_value = metadata return mock @pytest.fixture def mock_duck_conn(self): """Mock DuckDBConnector.""" mock = Mock() # Default return values mock.get_file_columns.return_value = { c.display_name for c in AdjustmentInputSchema.COLUMNS } mock.get_table_columns.return_value = [ c.display_name for c in AdjustmentInputSchema.COLUMNS ] mock.get_table_row_count.return_value = 10 # Mock execute().fetchall() for get_file_columns execute_result = Mock() execute_result.fetchall.return_value = [ ('account_id',), ('contract_id',), ('amount',), ] mock.execute.return_value = execute_result # Mock cursor for context manager cursor = Mock() cursor.__enter__ = Mock(return_value=cursor) cursor.__exit__ = Mock(return_value=False) mock.cursor.return_value = cursor return mock @pytest.fixture def mock_s3_file_downloader(self): """Mock S3 file downloader.""" return Mock() @pytest.fixture def mock_adjustment_file_validator(self): """Mock DuckDBValidator.""" mock = Mock() mock.init.return_value = None # Mock the validation result validation_result = Mock() validation_result.valid_rows = 10 validation_result.invalid_rows = 0 validation_result.total_amount_raw = 1000.0 validation_result.total_amount = 1000.0 mock.validate.return_value = validation_result return mock @pytest.fixture def mock_adjustment_file_loader(self): """Mock AdjustmentFileLoader.""" mock = Mock() return mock @pytest.fixture def mock_reference_data_service(self): """Mock ReferenceDataService.""" mock = Mock() return mock @pytest.fixture def processor( self, mock_royalty_accounting_client, mock_s3_connector, mock_s3_file_downloader, mock_duck_conn, mock_adjustment_file_loader, mock_adjustment_file_validator, mock_reference_data_service, ): """Processor instance.""" return AdjustmentFilePrepareProcessor( adjustment_file_loader=mock_adjustment_file_loader, adjustment_file_validator=mock_adjustment_file_validator, reference_data_service=mock_reference_data_service, duck_conn=mock_duck_conn, royalty_accounting_client=mock_royalty_accounting_client, s3_file_downloader=mock_s3_file_downloader, ) @pytest.fixture def event(self): """Sample event.""" return AdjustmentFilePrepareEvent( detail_type='adjustment_batch.initialized', detail=AdjustmentFilePrepareEventDetail( metadata=AdjustmentFilePrepareEventMetadata( target_type='worksheet_flowthrough_batch', target_id=123, correlation_id='corr-123', ), data=AdjustmentFilePrepareEventData( s3_bucket='test-bucket', s3_key='test/file.csv', ), ), ) @patch('src.connectors.mysql.utils.load_from_s3') @patch('src.services.processor.config') @patch('src.utils.file_utils.create_temp_file') @patch('src.services.processor.os.remove') def test_process_success_csv( self, mock_remove, mock_gen_path, mock_config, mock_load_from_s3, processor, event, mock_s3_connector, mock_royalty_accounting_client, mock_duck_conn, mock_reference_data_service, ): """Test successful processing of a CSV file.""" # Setup mocks mock_config.storage.S3_STAGING_FORMAT.format.return_value = ( 'staging/123/prepared.csv' ) mock_config.env.is_local = False mock_gen_path.return_value = '/tmp/test' mock_load_from_s3.return_value = 100 # Execute response = processor.process(event) # Verify assert response.detail_type == EventType.ADJUSTMENT_BATCH_PREPARED assert response.detail.metadata.target_id == event.detail.metadata.target_id assert response.detail.data.valid_row_count == 10 assert response.detail.data.invalid_row_count == 0 # Verify calls mock_reference_data_service.load_all.assert_called_once() mock_royalty_accounting_client.update_batch_status.assert_called_with( 123, status=BatchStatus.VALIDATING, expected_status=BatchStatus.PENDING ) # Verify staging was called mock_royalty_accounting_client.delete_batch_from_staging.assert_called_once_with( 123 ) # Cleanup mock_remove.assert_called_once() @patch('src.services.processor.config') @patch('src.utils.file_utils.create_temp_file') @patch('src.services.processor.os.path.exists') @patch('src.services.processor.os.remove') def test_process_success_xlsx( self, mock_remove, mock_exists, mock_gen_path, mock_config, processor, event, ): """Test successful processing of an XLSX file.""" # Setup mocks mock_config.storage.S3_STAGING_FORMAT.format.return_value = ( 'staging/123/prepared.csv' ) mock_gen_path.return_value = '/tmp/test' mock_exists.return_value = True # Execute response = processor.process(event) # Verify response assert response.detail_type == EventType.ADJUSTMENT_BATCH_PREPARED assert response.detail.data.valid_row_count == 10 assert response.detail.data.invalid_row_count == 0 @patch('src.services.processor.config') @patch('src.utils.file_utils.create_temp_file') @patch('src.services.processor.os.path.exists') @patch('src.services.processor.os.remove') def test_process_success_xlsx_gzip( self, mock_remove, mock_exists, mock_gen_path, mock_config, processor, event, ): """Test successful processing of a gzipped XLSX file.""" # Setup mocks mock_config.storage.S3_STAGING_FORMAT.format.return_value = ( 'staging/123/prepared.csv' ) mock_gen_path.return_value = '/tmp/test' mock_exists.return_value = True # Execute response = processor.process(event) # Verify response assert response.detail_type == EventType.ADJUSTMENT_BATCH_PREPARED assert response.detail.data.valid_row_count == 10 assert response.detail.data.invalid_row_count == 0 @patch('src.utils.file_utils.create_temp_file') @patch('src.services.processor.os.path.exists') @patch('src.services.processor.os.remove') def test_process_missing_headers( self, mock_remove, mock_exists, mock_gen_path, processor, event, mock_royalty_accounting_client, mock_adjustment_file_validator, ): """Test processing with missing headers.""" mock_gen_path.return_value = '/tmp/test' mock_exists.return_value = True # Make validator raise MissingHeadersError mock_adjustment_file_validator.validate.side_effect = MissingHeadersError( 'Missing required headers' ) # Execute with pytest.raises(MissingHeadersError): processor.process(event) # Verify batch status update mock_royalty_accounting_client.update_batch_status.assert_called_with( 123, status=BatchStatus.ERROR, expected_status=BatchStatus.VALIDATING, errors=[BatchErrorCode.MISSING_HEADERS], ) @patch('src.utils.file_utils.create_temp_file') @patch('src.services.processor.os.path.exists') @patch('src.services.processor.os.remove') def test_process_empty_file( self, mock_remove, mock_exists, mock_gen_path, processor, event, mock_royalty_accounting_client, mock_adjustment_file_validator, ): """Test processing an empty file.""" mock_gen_path.return_value = '/tmp/test' mock_exists.return_value = True # Make validator raise EmptyFileError mock_adjustment_file_validator.validate.side_effect = EmptyFileError( 'File is empty' ) # Execute with pytest.raises(EmptyFileError): processor.process(event) # Verify batch status update mock_royalty_accounting_client.update_batch_status.assert_called_with( 123, status=BatchStatus.ERROR, expected_status=BatchStatus.VALIDATING, errors=[BatchErrorCode.EMPTY_FILE], ) @patch('src.utils.file_utils.create_temp_file') def test_process_s3_download_error( self, mock_gen_path, processor, event, mock_s3_file_downloader, mock_royalty_accounting_client, ): """Test handling of S3 download error.""" mock_gen_path.return_value = '/tmp/test.csv' mock_s3_file_downloader.download.side_effect = S3FileNotFoundError('Not found') # Execute with pytest.raises(S3FileNotFoundError): processor.process(event) # Verify batch status is updated to ERROR # S3FileNotFoundError is a PermanentError, so the catch block in process should run mock_royalty_accounting_client.update_batch_status.assert_any_call( 123, status=BatchStatus.ERROR, expected_status=BatchStatus.VALIDATING, errors=[BatchErrorCode.FILE_NOT_FOUND], ) @patch('src.utils.file_utils.create_temp_file') @patch('src.services.processor.os.path.exists') @patch('src.services.processor.os.remove') def test_process_row_count_exceeded( self, mock_remove, mock_exists, mock_gen_path, processor, event, mock_royalty_accounting_client, mock_adjustment_file_validator, ): """Test processing file that exceeds max row count.""" mock_gen_path.return_value = '/tmp/test' mock_exists.return_value = True # Make validator raise RowCountExceededError mock_adjustment_file_validator.validate.side_effect = RowCountExceededError( 'Too many rows' ) # Execute with pytest.raises(RowCountExceededError): processor.process(event) # Verify batch status update mock_royalty_accounting_client.update_batch_status.assert_called_with( 123, status=BatchStatus.ERROR, expected_status=BatchStatus.VALIDATING, errors=[BatchErrorCode.ROW_COUNT_EXCEEDED], ) @patch('src.services.processor.logger') @patch('src.utils.file_utils.create_temp_file') @patch('src.services.processor.os.path.exists') @patch('src.services.processor.os.remove') def test_process_database_update_failure_after_error( self, mock_remove, mock_exists, mock_gen_path, mock_logger, processor, event, mock_royalty_accounting_client, mock_adjustment_file_validator, ): """Test handling database update failure after error state.""" mock_gen_path.return_value = '/tmp/test' mock_exists.return_value = True # Make validator raise EmptyFileError mock_adjustment_file_validator.validate.side_effect = EmptyFileError( 'File is empty' ) # Make first call succeed (VALIDATING), second call fail (ERROR update) mock_royalty_accounting_client.update_batch_status.side_effect = [ 1, Exception('DB connection lost'), ] # Execute - should raise the original EmptyFileError with pytest.raises(EmptyFileError): processor.process(event) # Verify error was logged assert any( 'Failed to update batch status' in str(call) for call in mock_logger.error.call_args_list ) @patch('src.utils.file_utils.create_temp_file') def test_process_s3_file_not_found_metadata_none( self, mock_gen_path, processor, event, mock_s3_file_downloader, mock_royalty_accounting_client, ): """Test handling when S3 file not found during download.""" mock_gen_path.return_value = '/tmp/test.csv' mock_s3_file_downloader.download.side_effect = S3FileNotFoundError( 'File not found' ) # Execute with pytest.raises(S3FileNotFoundError): processor.process(event) @patch('src.utils.file_utils.create_temp_file') def test_process_file_size_exceeded( self, mock_gen_path, processor, event, mock_s3_file_downloader, mock_royalty_accounting_client, ): """Test handling when file size exceeds maximum.""" from src.errors import FileSizeExceededError mock_gen_path.return_value = '/tmp/test.csv' # Make file downloader raise FileSizeExceededError mock_s3_file_downloader.download.side_effect = FileSizeExceededError( 'File size exceeds maximum' ) # Execute with pytest.raises(FileSizeExceededError): processor.process(event) @patch('src.services.processor.config') @patch('src.utils.file_utils.create_temp_file') @patch('src.services.processor.os.path.exists') @patch('src.services.processor.os.remove') def test_process_os_error_getting_file_size( self, mock_remove, mock_exists, mock_gen_path, mock_config, processor, event, ): """Test successful processing (os.path.getsize not used in current implementation).""" mock_config.storage.S3_STAGING_FORMAT.format.return_value = ( 'staging/123/prepared.csv' ) mock_gen_path.return_value = '/tmp/test' mock_exists.return_value = True # Execute response = processor.process(event) # Should succeed assert response.detail_type == EventType.ADJUSTMENT_BATCH_PREPARED assert response.detail.data.valid_row_count == 10 assert response.detail.data.invalid_row_count == 0 @patch('src.utils.file_utils.create_temp_file') def test_process_disk_space_exceeded( self, mock_gen_path, processor, event, mock_s3_connector, mock_royalty_accounting_client, ): """Test disk space validation failure.""" mock_gen_path.return_value = '/tmp/test.csv' # Set file size large enough to exceed max disk space after multiplier s3_metadata = Mock() s3_metadata.size = 900 * 1024 * 1024 # 900MB (passes S3 1GB check) mock_s3_connector.get_file_metadata.return_value = s3_metadata @patch('src.utils.file_utils.create_temp_file') @patch('src.services.processor.os.path.exists') @patch('src.services.processor.os.remove') def test_process_missing_required_columns_only( self, mock_remove, mock_exists, mock_gen_path, processor, event, mock_royalty_accounting_client, mock_adjustment_file_validator, ): """Test processing with missing required columns specifically.""" mock_gen_path.return_value = '/tmp/test' mock_exists.return_value = True # Make validator raise MissingHeadersError for missing required columns mock_adjustment_file_validator.validate.side_effect = MissingHeadersError( 'Missing required columns' ) # Execute with pytest.raises(MissingHeadersError) as exc_info: processor.process(event) assert 'Missing required columns' in str(exc_info.value) @patch('src.services.processor.config') @patch('src.services.processor.logger') @patch('src.utils.file_utils.create_temp_file') @patch('src.services.processor.os.remove') def test_process_cleanup_oserror( self, mock_remove, mock_gen_path, mock_logger, mock_config, processor, event, mock_s3_connector, mock_duck_conn, ): """Test cleanup handles OSError during file deletion gracefully.""" mock_config.storage.S3_STAGING_FORMAT.format.return_value = ( 'staging/123/prepared.csv' ) mock_gen_path.return_value = '/tmp/test' # Make file deletion fail mock_remove.side_effect = OSError('Delete failed') # Execute response = processor.process(event) # Should still succeed, just log error assert response.detail_type == EventType.ADJUSTMENT_BATCH_PREPARED assert any( 'OS error while deleting file' in str(call) for call in mock_logger.error.call_args_list ) @patch('src.services.processor.config') @patch('src.services.processor.logger') @patch('src.utils.file_utils.create_temp_file') @patch('src.services.processor.os.path.exists') @patch('src.services.processor.os.remove') def test_process_cleanup_unexpected_exception( self, mock_remove, mock_exists, mock_gen_path, mock_logger, mock_config, processor, event, mock_s3_connector, mock_duck_conn, ): """Test cleanup handles unexpected exception during file deletion gracefully.""" mock_config.storage.S3_STAGING_FORMAT.format.return_value = ( 'staging/123/prepared.csv' ) mock_gen_path.return_value = '/tmp/test' mock_exists.return_value = True # Make file removal fail mock_remove.side_effect = Exception('Delete failed') # Execute response = processor.process(event) # Should still succeed, just log error assert response.detail_type == EventType.ADJUSTMENT_BATCH_PREPARED assert any( 'Unexpected error while deleting file' in str(call) for call in mock_logger.error.call_args_list ) def test_cleanup_file_with_none_path(self, processor): """Test _cleanup_file handles None path gracefully.""" # Should not raise any errors processor._cleanup_file(None) def test_set_batch_validating_failure( self, processor, event, mock_royalty_accounting_client ): """Test _set_batch_validating raises UpdateBatchError when update fails.""" from src.errors import UpdateBatchError # Make update return 0 rows (indicating failure) mock_royalty_accounting_client.update_batch_status.return_value = 0 # Execute & Verify with pytest.raises(UpdateBatchError) as exc_info: processor._set_batch_validating(123) assert 'Batch 123 could not be updated' in str(exc_info.value) assert 'Expected status=pending' in str(exc_info.value) def test_set_batch_error_failure( self, processor, event, mock_royalty_accounting_client ): """Test _set_batch_error raises UpdateBatchError when update fails.""" from src.errors import UpdateBatchError # Make update return 0 rows (indicating failure) mock_royalty_accounting_client.update_batch_status.return_value = 0 # Execute & Verify with pytest.raises(UpdateBatchError) as exc_info: processor._set_batch_error(123, [BatchErrorCode.EMPTY_FILE]) assert 'Batch 123 could not be updated' in str(exc_info.value) assert 'Expected status=validating' in str(exc_info.value)