"""Tests for infra row reading pipeline — FileReader, DictRowIterator, RowReader.""" import csv import json import pytest from openpyxl import Workbook from infra.readers import CsvFileReader, JsonFileReader, XlsxFileReader from infra.row_reader import ( ContractRowIterator, DictRowIterator, MissingColumnsError, RowReader, RowValidationError, UnsupportedFormatError, ) from schemas import ContractRow, ContractType def _write_csv(tmp_path, rows, filename='input.csv'): path = tmp_path / filename with open(path, 'w', newline='') as f: writer = csv.DictWriter(f, fieldnames=rows[0].keys()) writer.writeheader() writer.writerows(rows) return str(path) MINIMAL_ROW = { 'Account ID': '123', 'Contract Name': 'Test', 'Contract Type': 'Distribution', 'Current Period Start Date': '01/01/2026', } class TestCsvFileReader: def test_yields_raw_rows(self, tmp_path): csv_file = _write_csv(tmp_path, [MINIMAL_ROW]) reader = CsvFileReader() rows = list(reader.read(csv_file)) assert len(rows) == 2 # header + 1 data row assert rows[0][0] == 'Account ID' # header assert rows[1][0] == '123' # data class TestJsonFileReader: def _write_json(self, tmp_path, data, filename='input.json'): path = tmp_path / filename path.write_text(json.dumps(data)) return str(path) def test_yields_header_and_rows(self, tmp_path): json_file = self._write_json(tmp_path, [MINIMAL_ROW]) reader = JsonFileReader() rows = list(reader.read(json_file)) assert len(rows) == 2 # header + 1 data row assert rows[0] == list(MINIMAL_ROW.keys()) assert rows[1] == list(MINIMAL_ROW.values()) def test_multiple_objects(self, tmp_path): data = [ MINIMAL_ROW, {**MINIMAL_ROW, 'Account ID': '456', 'Contract Name': 'Second'}, ] json_file = self._write_json(tmp_path, data) reader = JsonFileReader() rows = list(reader.read(json_file)) assert len(rows) == 3 # header + 2 data rows assert rows[1][0] == '123' assert rows[2][0] == '456' def test_empty_list_yields_nothing(self, tmp_path): json_file = self._write_json(tmp_path, []) reader = JsonFileReader() rows = list(reader.read(json_file)) assert rows == [] def test_non_list_yields_nothing(self, tmp_path): json_file = self._write_json(tmp_path, {'key': 'value'}) reader = JsonFileReader() rows = list(reader.read(json_file)) assert rows == [] def test_missing_keys_produce_empty_strings(self, tmp_path): data = [ {'A': '1', 'B': '2'}, {'A': '3'}, # missing B ] json_file = self._write_json(tmp_path, data) reader = JsonFileReader() rows = list(reader.read(json_file)) assert rows[2] == ['3', ''] def test_non_string_values_converted(self, tmp_path): data = [{'num': 42, 'flag': True, 'nil': None}] json_file = self._write_json(tmp_path, data) reader = JsonFileReader() rows = list(reader.read(json_file)) assert rows[1] == ['42', 'True', 'None'] class TestJsonFileReaderStreaming: """Tests that explicitly exercise the ijson streaming path.""" def _write_json(self, tmp_path, data, filename='input.json'): path = tmp_path / filename path.write_text(json.dumps(data)) return str(path) def test_streaming_matches_eager_output(self, tmp_path): data = [ {'A': '1', 'B': '2'}, {'A': '3', 'B': '4'}, ] json_file = self._write_json(tmp_path, data) reader = JsonFileReader() eager = list(reader._read_eager(json_file)) streaming = list(reader._read_streaming(json_file, __import__('ijson'))) assert eager == streaming def test_streaming_missing_keys(self, tmp_path): data = [ {'A': '1', 'B': '2'}, {'A': '3'}, ] json_file = self._write_json(tmp_path, data) reader = JsonFileReader() rows = list(reader._read_streaming(json_file, __import__('ijson'))) assert rows[0] == ['A', 'B'] assert rows[2] == ['3', ''] def test_streaming_empty_array(self, tmp_path): json_file = self._write_json(tmp_path, []) reader = JsonFileReader() rows = list(reader._read_streaming(json_file, __import__('ijson'))) assert rows == [] class TestXlsxFileReader: def _write_xlsx(self, tmp_path, header, data_rows, filename='input.xlsx'): path = tmp_path / filename wb = Workbook() ws = wb.active ws.append(header) for row in data_rows: ws.append(row) wb.save(str(path)) return str(path) def test_yields_header_and_rows(self, tmp_path): header = list(MINIMAL_ROW.keys()) data = [list(MINIMAL_ROW.values())] xlsx_file = self._write_xlsx(tmp_path, header, data) reader = XlsxFileReader() rows = list(reader.read(xlsx_file)) assert len(rows) == 2 assert rows[0] == header assert rows[1] == list(MINIMAL_ROW.values()) def test_none_cells_become_empty_strings(self, tmp_path): xlsx_file = self._write_xlsx( tmp_path, ['A', 'B', 'C'], [['val', None, 'end']], ) reader = XlsxFileReader() rows = list(reader.read(xlsx_file)) assert rows[1] == ['val', '', 'end'] def test_numeric_cells_converted_to_strings(self, tmp_path): xlsx_file = self._write_xlsx( tmp_path, ['ID', 'Amount'], [[123, 45.67]], ) reader = XlsxFileReader() rows = list(reader.read(xlsx_file)) assert rows[1] == ['123', '45.67'] def test_multiple_rows(self, tmp_path): xlsx_file = self._write_xlsx( tmp_path, ['Name'], [['Alice'], ['Bob'], ['Charlie']], ) reader = XlsxFileReader() rows = list(reader.read(xlsx_file)) assert len(rows) == 4 # header + 3 data rows assert rows[1] == ['Alice'] assert rows[3] == ['Charlie'] def test_empty_sheet_yields_nothing(self, tmp_path): path = tmp_path / 'empty.xlsx' wb = Workbook() wb.save(str(path)) reader = XlsxFileReader() rows = list(reader.read(str(path))) # openpyxl yields no rows for a blank sheet assert rows == [] class TestDictRowIterator: def test_uses_first_row_as_headers(self, tmp_path): csv_file = _write_csv(tmp_path, [MINIMAL_ROW]) raw = CsvFileReader().read(csv_file) dict_iter = DictRowIterator(raw) rows = list(dict_iter) assert len(rows) == 1 _, row = rows[0] assert row['Account ID'] == '123' assert row['Contract Name'] == 'Test' def test_indices_are_one_based(self, tmp_path): csv_file = _write_csv( tmp_path, [ MINIMAL_ROW, {**MINIMAL_ROW, 'Account ID': '456'}, ], ) raw = CsvFileReader().read(csv_file) indices = [idx for idx, _ in DictRowIterator(raw)] assert indices == [1, 2] def test_fieldnames_available_after_iteration(self, tmp_path): csv_file = _write_csv(tmp_path, [MINIMAL_ROW]) raw = CsvFileReader().read(csv_file) dict_iter = DictRowIterator(raw) list(dict_iter) assert 'Account ID' in dict_iter.fieldnames def test_fieldnames_available_immediately(self, tmp_path): csv_file = _write_csv(tmp_path, [MINIMAL_ROW]) raw = CsvFileReader().read(csv_file) dict_iter = DictRowIterator(raw) # fieldnames available before iteration (header consumed eagerly) assert 'Account ID' in dict_iter.fieldnames class TestRowReader: def _reader(self): return RowReader(readers={'.csv': CsvFileReader()}) def test_yields_contract_rows(self, tmp_path): csv_file = _write_csv(tmp_path, [MINIMAL_ROW]) rows = list(self._reader().read(csv_file)) assert len(rows) == 1 idx, row = rows[0] assert idx == 1 assert row.account_id == 123 assert row.contract_type == ContractType.DISTRIBUTION def test_missing_file_raises(self): with pytest.raises(FileNotFoundError): list(self._reader().read('/nonexistent.csv')) def test_extra_columns_ignored(self, tmp_path): csv_file = _write_csv(tmp_path, [{**MINIMAL_ROW, 'Extra': 'ignored'}]) rows = list(self._reader().read(csv_file)) assert len(rows) == 1 assert not hasattr(rows[0][1], 'extra') def test_unsupported_extension_raises_when_no_default(self, tmp_path): path = tmp_path / 'data.xlsx' path.write_text('') reader = RowReader(readers={}, default=None) with pytest.raises(UnsupportedFormatError, match='xlsx'): list(reader.read(str(path))) def test_no_extension_uses_default(self, tmp_path): # Write CSV content to a file with no extension path = tmp_path / 'data' with open(path, 'w', newline='') as f: writer = csv.DictWriter(f, fieldnames=MINIMAL_ROW.keys()) writer.writeheader() writer.writerow(MINIMAL_ROW) rows = list(self._reader().read(str(path))) assert len(rows) == 1 assert rows[0][1].account_id == 123 def test_default_must_be_in_readers(self): with pytest.raises(ValueError, match="default '.xlsx' is not a registered"): RowReader(readers={'.csv': CsvFileReader()}, default='.xlsx') class TestContractRowIteratorValidation: def test_valid_rows_yield_contract_row(self, tmp_path): csv_file = _write_csv(tmp_path, [MINIMAL_ROW]) raw = CsvFileReader().read(csv_file) dict_iter = DictRowIterator(raw) contract_iter = ContractRowIterator(dict_iter) rows = list(contract_iter) assert len(rows) == 1 _, row = rows[0] assert isinstance(row, ContractRow) def test_invalid_row_yields_validation_error(self, tmp_path): """Rows that fail Pydantic validation are yielded as RowValidationError.""" csv_file = _write_csv( tmp_path, [ MINIMAL_ROW, # Row with missing required columns still parses (fields are None) # but a truly invalid row (e.g., entirely wrong structure) # would trigger a validation error. Since ContractRow is lenient, # we test via RowReader which validates headers first. ], ) raw = CsvFileReader().read(csv_file) dict_iter = DictRowIterator(raw) contract_iter = ContractRowIterator(dict_iter) results = list(contract_iter) # All rows should be ContractRow since ContractRow accepts None for all fields assert all(isinstance(r[1], ContractRow) for r in results) def test_fieldnames_available_on_contract_row_iterator(self, tmp_path): csv_file = _write_csv(tmp_path, [MINIMAL_ROW]) raw = CsvFileReader().read(csv_file) dict_iter = DictRowIterator(raw) contract_iter = ContractRowIterator(dict_iter) assert 'Account ID' in contract_iter.fieldnames class TestHeaderValidation: def _reader(self): return RowReader(readers={'.csv': CsvFileReader()}) def test_valid_headers(self, tmp_path): csv_file = _write_csv(tmp_path, [MINIMAL_ROW]) list(self._reader().read(csv_file)) # should not raise def test_missing_account_id(self, tmp_path): csv_file = _write_csv( tmp_path, [{'Contract Name': 'X', 'Contract Type': 'Distribution'}], ) with pytest.raises(MissingColumnsError, match='Account ID'): list(self._reader().read(csv_file)) def test_missing_all_required(self, tmp_path): csv_file = _write_csv(tmp_path, [{'Signing Entity': 'acme'}]) with pytest.raises(MissingColumnsError) as exc_info: list(self._reader().read(csv_file)) assert len(exc_info.value.missing) == 3