"""Test processor base.""" import csv from csv import DictReader, DictWriter from datetime import datetime from io import StringIO import logging from unittest.mock import call, Mock, patch, PropertyMock from faker import Faker import pytest from src.models import TaxFormInfoDetailsItem from src.processors.base import LogEntry, Processor, ReportProcessor from tests.unit.factories import TaxFormInfoDetailsItemFactory class DummyProcessor(Processor): """Dummy processor.""" def process(self) -> None: return None class TestProcessor: """Processor test suite.""" @patch.object(DummyProcessor, '_report_logs') @patch.object(DummyProcessor, '_download_file') @patch.object(DummyProcessor, 'process') @patch.object(DummyProcessor, '__init__', return_value=None) def test_execute( self, mock_init: Mock, mock_process: Mock, mock_download_file: Mock, mock_report_logs: Mock, faker: Faker, ) -> None: """Test execute method.""" bucket_name = faker.pystr() file_path = faker.pystr() DummyProcessor.execute(bucket_name, file_path) mock_init.assert_called_once_with(bucket_name, file_path) mock_process.assert_called_once_with() mock_download_file.assert_called_once_with() mock_report_logs.assert_called_once_with() @patch.object(DummyProcessor, '_report_logs') @patch.object(DummyProcessor, '_download_file') @patch.object(DummyProcessor, 'process', side_effect=RuntimeError('boom')) @patch.object(DummyProcessor, '__init__', return_value=None) def test_execute_reports_logs_even_when_process_raises( self, _mock_init: Mock, _mock_process: Mock, _mock_download_file: Mock, mock_report_logs: Mock, faker: Faker, ) -> None: """_report_logs must run whether process() succeeds or raises. Otherwise every row-level _add_log accumulated before the failure is silently dropped — no Sentry, no logger output. This invariant is relied upon by every processor that fails late in process(). """ with pytest.raises(RuntimeError, match='boom'): DummyProcessor.execute(faker.pystr(), faker.pystr()) mock_report_logs.assert_called_once_with() @patch.object( DummyProcessor, '_report_logs', side_effect=RuntimeError('log flush crashed') ) @patch.object(DummyProcessor, '_download_file') @patch.object(DummyProcessor, 'process', side_effect=ValueError('process failed')) @patch.object(DummyProcessor, '__init__', return_value=None) def test_execute_does_not_mask_process_exception_with_report_logs_failure( self, _mock_init: Mock, _mock_process: Mock, _mock_download_file: Mock, _mock_report_logs: Mock, faker: Faker, ) -> None: """If both process() and _report_logs() raise, the process() exception is what the caller sees. A failed log flush must not swallow the original error — that would hide the real fault. """ with pytest.raises(ValueError, match='process failed'): DummyProcessor.execute(faker.pystr(), faker.pystr()) @patch.object(DummyProcessor, '_report_logs') @patch.object(DummyProcessor, '_download_file') @patch.object(DummyProcessor, 'process', side_effect=KeyboardInterrupt) @patch.object(DummyProcessor, '__init__', return_value=None) def test_execute_does_not_swallow_keyboard_interrupt( self, _mock_init: Mock, _mock_process: Mock, _mock_download_file: Mock, mock_report_logs: Mock, faker: Faker, ) -> None: """Interpreter-level signals (KeyboardInterrupt, SystemExit) must propagate without running _report_logs — flushing logs during shutdown has caused Sentry I/O deadlocks. Catching `Exception` (not `BaseException`) is the safety boundary here. """ with pytest.raises(KeyboardInterrupt): DummyProcessor.execute(faker.pystr(), faker.pystr()) mock_report_logs.assert_not_called() @patch('src.processors.base.s3') def test_download_file(self, mock_s3: Mock, faker: Faker) -> None: """Test _download_file method.""" test_value = faker.pystr() bucket_name = faker.pystr() file_path = faker.pystr() buffer = StringIO() buffer.write(test_value) buffer.seek(0) mock_s3.download_file.return_value = buffer processor = DummyProcessor(bucket_name, file_path) processor._download_file() mock_s3.download_file.assert_called_once_with(bucket_name, file_path) assert processor._input_file_buffer.read() == test_value @patch('src.processors.base.s3') def test_download_file_failure_bad_file(self, mock_s3: Mock, faker: Faker) -> None: """Test download file failure.""" bucket_name = faker.pystr() file_path = faker.pystr() mock_s3.download_file.side_effect = Exception('Test message') processor = DummyProcessor(bucket_name, file_path) with pytest.raises(Exception, match='Test message'): processor._download_file() mock_s3.download_file.assert_called_once_with(bucket_name, file_path) def test_csv_dict_reader(self, faker: Faker) -> None: """Test csv_dict_reader prop.""" bucket_name = faker.pystr() file_path = faker.pystr() column_name = faker.pystr() test_data = {column_name: faker.pystr()} buffer = StringIO() writer = DictWriter(buffer, fieldnames=[column_name]) writer.writeheader() writer.writerow(test_data) buffer.seek(0) processor = DummyProcessor(bucket_name, file_path) processor._input_file_buffer = buffer reader = processor.csv_dict_reader assert isinstance(reader, DictReader) assert list(reader) == [test_data] @pytest.mark.parametrize( 'input_args,expected_result', ( ( [ {'message': 'm1'}, {'message': 'm2', 'log_level': logging.WARNING}, {'message': 'm3', 'log_level': logging.ERROR}, { 'message': 'm4', 'log_level': logging.INFO, 'group_condition': '123', }, ], { logging.ERROR: [LogEntry('m1'), LogEntry('m3')], logging.WARNING: [LogEntry('m2')], logging.INFO: [LogEntry('m4', '123')], }, ), ( [{'message': 'm1'}, {'message': 'm2'}, {'message': 'm3', 'f1': 100}], { logging.ERROR: [ LogEntry('m1'), LogEntry('m2'), LogEntry('m3', additional_data={'f1': 100}), ] }, ), ( [ { 'message': 'm1', 'log_level': logging.WARNING, 'group_condition': 'gr1', 'a': 'b', 'f': 1, }, {'message': 'm2', 'log_level': logging.WARNING}, ], { logging.WARNING: [ LogEntry('m1', 'gr1', {'a': 'b', 'f': 1}), LogEntry('m2'), ] }, ), ([], None), ), ) def test_add_log( self, faker: Faker, input_args: list[dict[str, str | int]], expected_result: dict[int, list[LogEntry]] | None, ) -> None: """Test _add_log.""" processor = DummyProcessor(faker.pystr(), faker.pystr()) for item in input_args: processor._add_log(**item) # type: ignore assert processor._logs == expected_result @pytest.mark.parametrize( 'logs,expected_messages', ( ( { logging.ERROR: [LogEntry('m11'), LogEntry('m13')], logging.WARNING: [LogEntry('m12')], logging.INFO: [LogEntry('m14')], }, { logging.ERROR: ['m11', 'm13'], logging.WARNING: ['m12'], logging.INFO: ['m14'], }, ), ( {logging.ERROR: [LogEntry('m21'), LogEntry('m22'), LogEntry('m23')]}, {logging.ERROR: ['m21', 'm22', 'm23']}, ), ( {logging.WARNING: [LogEntry('m31'), LogEntry('m32')]}, {logging.WARNING: ['m31', 'm32']}, ), ( {logging.INFO: [LogEntry('m41'), LogEntry('m42')]}, {logging.INFO: ['m41', 'm42']}, ), (None, None), ( { logging.ERROR: [ LogEntry('m11'), LogEntry('m12', 'gr1', {'f': 'v1'}), LogEntry('m14', 'group2', {'f3': 'v23', 'f1': 'v21', 'f2': 22}), LogEntry('m12', 'gr1', {'f': 'v2'}), LogEntry('m12', 'gr1', {'f': 'v3'}), LogEntry('m14', 'group2', {'f1': 'v11', 'f2': 12, 'f3': 'v13'}), ], logging.WARNING: [ LogEntry('m21'), LogEntry('m22', 'gr1', {'f': 'v'}), ], }, { logging.ERROR: [ 'm11', 'm12 | gr1: f v1, v2, v3', 'm14 | group2: (f1=v11, f2=12, f3=v13); (f1=v21, f2=22, f3=v23)', ], logging.WARNING: ['m21', 'm22 | gr1: f v'], }, ), ), ) @patch('src.processors.base.logger') @patch('src.processors.base.capture_message') def test_report_errors( self, mock_capture_message: Mock, mock_logger: Mock, faker: Faker, logs: dict[int, list[LogEntry]] | None, expected_messages: dict[int, list[str]] | None, ) -> None: """Test _report_errors.""" level_name_map = {40: 'error', 30: 'warning', 20: 'info'} processor = DummyProcessor(faker.pystr(), faker.pystr()) processor._logs = logs processor._report_logs() if logs and expected_messages: expected_calls = [] for log_level, log_messages in expected_messages.items(): expected_calls.append( call( log_level, f'{level_name_map[log_level]}s count: {len(logs[log_level])}', ) ) for message in log_messages: expected_calls.append(call(log_level, message)) assert mock_logger.log.call_args_list == expected_calls assert mock_capture_message.call_args_list == [ call( '\n'.join(expected_messages[log_level]), level=level_name_map[log_level], ) for log_level, log_entries in logs.items() if log_level >= 30 ] else: mock_logger.error.assert_not_called() mock_logger.log.assert_not_called() mock_capture_message.assert_not_called() class DummyReportProcessor(ReportProcessor): """Dummy ReportProcessor.""" def process(self) -> None: return None class TestReportProcessor: """ReportProcessor test suite.""" @patch.object(DummyReportProcessor, '_report_logs') @patch.object(DummyReportProcessor, '_upload_file') @patch.object(DummyReportProcessor, '_generate_report') @patch.object(DummyReportProcessor, '_download_file') @patch.object(DummyReportProcessor, 'process') @patch.object(DummyReportProcessor, '__init__', return_value=None) def test_execute( self, mock_init: Mock, mock_process: Mock, mock_download_file: Mock, mock_generate_report: Mock, mock_upload_file: Mock, mock_report_logs: Mock, faker: Faker, ) -> None: """Test execute method.""" bucket_name = faker.pystr() file_path = faker.pystr() DummyReportProcessor.execute(bucket_name, file_path) mock_init.assert_called_once_with(bucket_name, file_path) mock_process.assert_called_once_with() mock_download_file.assert_called_once_with() mock_generate_report.assert_called_once_with() mock_upload_file.assert_called_once_with() mock_report_logs.assert_called_once_with() @patch.object(DummyReportProcessor, 'get_output_file_buffer') @patch.object(DummyReportProcessor, 'report_file_name', new_callable=PropertyMock) @patch('src.processors.base.s3') def test_upload_file( self, mock_s3: Mock, mock_report_file_name: PropertyMock, mock_get_output_file_buffer: Mock, faker: Faker, ) -> None: """Test _upload_file method.""" bucket_name = faker.pystr() file_path = '/home/abc/file_test.csv' report_file_name = faker.pystr() test_str = faker.pystr() report_type = faker.pystr() mock_report_file_name.return_value = report_file_name mock_get_output_file_buffer.return_value = test_str processor = DummyReportProcessor(bucket_name, file_path) processor._report_type = report_type processor._upload_file() mock_s3.upload_file.assert_called_once_with( bucket_name, f'reports/{report_file_name}', test_str, metadata={'original-file-name': 'file_test.csv'}, tags={'report-type': report_type}, ) mock_get_output_file_buffer.assert_called_once_with(True) mock_report_file_name.assert_called_once_with() def test_report_type(self, faker: Faker) -> None: """Test report_type prop.""" report_type = faker.pystr() processor = DummyReportProcessor(faker.pystr(), faker.pystr()) processor._report_type = report_type assert processor.report_type == report_type @patch('src.processors.base.datetime') def test_report_file_name(self, mock_datetime: Mock, faker: Faker) -> None: """Test report_file_name method.""" ts = datetime(2025, 2, 5, 6, 9, 11, 22222) file_prefix = faker.pystr() mock_datetime.now.return_value = ts processor = DummyReportProcessor(faker.pystr(), faker.pystr()) processor._report_type = file_prefix assert processor.report_file_name == f'{file_prefix}_report_20250205_060911.csv' def test_get_output_file_buffer(self, faker: Faker) -> None: """Test get_output_file_buffer method.""" data = 'test_str' processor = DummyReportProcessor(faker.pystr(), faker.pystr()) assert processor._output_file_buffer is None result_content = processor.get_output_file_buffer() result_content.write(data) assert processor._output_file_buffer == result_content assert processor.get_output_file_buffer() == result_content assert processor.get_output_file_buffer(False).read() == '' assert processor.get_output_file_buffer(True).read() == data def test_generate_report(self, faker: Faker) -> None: """Test _generate_report method.""" tax_form_data = TaxFormInfoDetailsItemFactory.batch(6) processor = DummyReportProcessor(faker.pystr(), faker.pystr()) processor._output_columns = TaxFormInfoDetailsItem.model_fields.keys() processor._output_data = tax_form_data processor._generate_report() reader = csv.DictReader(processor.get_output_file_buffer(True)) data = list(reader) expected_data = [] for item in tax_form_data: item_dict = item.model_dump(mode='json') for key, value in item_dict.items(): item_dict[key] = '' if value is None else str(value) expected_data.append(item_dict) assert data == expected_data