"""Unit testcases for AdjustmentsJsonImport processor.""" import pathlib from pathlib import Path from unittest.mock import MagicMock, patch import pandas as pd import pytest from adjustments_json_import.constants.adjustments_json_import import ( ERROR_NO_ADJUSTMENT_FILE_FOUND, ERROR_NO_REFERENCE_ADJUSTMENT_TYPES_FOUND, ERROR_NO_STATEMENT_PERIODS_FOUND, FILE_IS_EMPTY, INVALID_FILE_PATH, INVALID_GZIP_FILE, ) from adjustments_json_import.error_handling import NotFoundError, ValidationError from adjustments_json_import.mysql_query_template import render_sql_template from adjustments_json_import.processor import AdjustmentsJsonImportProcessor def setup_mock_pandas(mock_obj, methods): """Mock pandas library methods.""" current = mock_obj for method in methods: current = getattr(current, method).return_value return current @patch('adjustments_json_import.processor.NamedTemporaryFile') @patch('adjustments_json_import.processor.get_s3_connection') @patch('adjustments_json_import.processor.execute_query') def test__get_statement_period_adjustment_file( mock_execute_query, mock_get_s3_connection, mock_name_temporary_file, mock_event ): """Test to get adjustment file.""" mock_statement_period_adjustment_file = [ { 'valid_file_location': 's3://qa-abacus-adjustments/test.json.gz', 'statement_period_id': 282, 'statement_period_adjustment_file_id': 1, } ] mock_temp_file = MagicMock(spec=Path) mock_temp_file.name = '/tmp/test123.json.gz' mock_name_temporary_file.return_value = mock_temp_file mock_execute_query.return_value = mock_statement_period_adjustment_file mock_mysql_conn = MagicMock() mock_get_s3_connection.return_value.download_file.return_value = True processor = AdjustmentsJsonImportProcessor(mock_event, mock_mysql_conn) processor._is_valid_gzipped_file = MagicMock(return_value=True) result = processor._get_statement_period_adjustment_file() mock_get_s3_connection.return_value.download_file.assert_called_once_with( 'qa-abacus-adjustments', 'test.json.gz', '/tmp/test123.json.gz' ) assert result == '/tmp/test123.json.gz' @patch('adjustments_json_import.processor.get_s3_connection') @patch('adjustments_json_import.processor.execute_query') def test__get_statement_period_adjustment_file_error( mock_execute_query, mock_get_s3_connection, mock_event ): """Test raises an error if there is no record for adjustment file.""" mock_statement_period_adjustment_file = None mock_execute_query.return_value = mock_statement_period_adjustment_file mock_mysql_conn = MagicMock() processor = AdjustmentsJsonImportProcessor(mock_event, mock_mysql_conn) processor._is_valid_gzipped_file = MagicMock(return_value=True) with pytest.raises(NotFoundError) as excinfo: processor._get_statement_period_adjustment_file() assert str(excinfo.value) == ERROR_NO_ADJUSTMENT_FILE_FOUND.format(1) mock_get_s3_connection.assert_not_called() @patch('adjustments_json_import.processor.NamedTemporaryFile') @patch('adjustments_json_import.processor.get_s3_connection') @patch('adjustments_json_import.processor.execute_query') def test__get_statement_period_adjustment_file_invalid_path( mock_execute_query, mock_get_s3_connection, mock_name_temporary_file, mock_event ): """Test raise ValidationError for invalid zip file format.""" mock_statement_period_adjustment_file = [ { 'valid_file_location': 's3://qa-abacus-adjustments/test.zip', 'statement_period_id': 282, 'statement_period_adjustment_file_id': 1, } ] mock_temp_file = MagicMock(spec=Path) mock_temp_file.name = '/tmp/test123.zip' mock_name_temporary_file.return_value = mock_temp_file mock_execute_query.return_value = mock_statement_period_adjustment_file mock_mysql_conn = MagicMock() mock_get_s3_connection.return_value.download_file.return_value = True processor = AdjustmentsJsonImportProcessor(mock_event, mock_mysql_conn) processor._is_valid_gzipped_file = MagicMock( side_effect=ValidationError(INVALID_GZIP_FILE) ) with pytest.raises(ValidationError) as excinfo: processor._get_statement_period_adjustment_file() assert str(excinfo.value) == INVALID_GZIP_FILE @patch('adjustments_json_import.processor.get_s3_connection') @patch('adjustments_json_import.processor.execute_query') def test__get_statement_period_adjustment_file_path_empty( mock_execute_query, mock_get_s3_connection, mock_event ): """Test raises an error if file path is empty.""" mock_statement_period_adjustment_file = [ { 'valid_file_location': None, 'statement_period_id': 282, 'statement_period_adjustment_file_id': 1, } ] mock_execute_query.return_value = mock_statement_period_adjustment_file mock_mysql_conn = MagicMock() processor = AdjustmentsJsonImportProcessor(mock_event, mock_mysql_conn) processor._is_valid_gzipped_file = MagicMock(return_value=True) with pytest.raises(ValidationError) as excinfo: processor._get_statement_period_adjustment_file() assert str(excinfo.value) == INVALID_FILE_PATH mock_get_s3_connection.assert_not_called() def test__is_valid_gzipped_file_success(mock_event): """Test returns True if file is valid json gzip file.""" mock_path = pathlib.PurePath('/tmp/test_file.json.gz') mock_mysql_conn = MagicMock() processor = AdjustmentsJsonImportProcessor(mock_event, mock_mysql_conn) result = processor._is_valid_gzipped_file(mock_path) assert result is True def test__is_valid_gzipped_file_raises_error(mock_event): """Test raise ValidationError for invalid zip file format.""" mock_path = MagicMock(spec=Path) mock_path.name = 'test_file.json.gz' mock_mysql_conn = MagicMock() processor = AdjustmentsJsonImportProcessor(mock_event, mock_mysql_conn) with pytest.raises(ValidationError) as excinfo: processor._is_valid_gzipped_file(mock_path) assert str(excinfo.value) == INVALID_GZIP_FILE def test__strip_special_characters_and_spaces(mock_event): """Test to remove specific unicode characters and trim whitespace.""" mock_mysql_conn = MagicMock() input_data = pd.Series( [ ' Hello World ', 'NormalText', 'Hidden\xa0Space', '\u200bZeroWidth', ' \xa0 Mixed \u200b ', ] ) expected_data = pd.Series( ['Hello World', 'NormalText', 'HiddenSpace', 'ZeroWidth', 'Mixed'] ) processor = AdjustmentsJsonImportProcessor(mock_event, mock_mysql_conn) result = processor._strip_special_characters_and_spaces(input_data) pd.testing.assert_series_equal(result, expected_data) def test__strip_with_nan_values(mock_event): """Test to handle NaN values gracefully without crashing.""" mock_mysql_conn = MagicMock() input_data = pd.Series([' Clean ', None, float('nan')]) expected_data = pd.Series(['Clean', None, float('nan')]) processor = AdjustmentsJsonImportProcessor(mock_event, mock_mysql_conn) result = processor._strip_special_characters_and_spaces(input_data) pd.testing.assert_series_equal(result, expected_data) @patch('adjustments_json_import.processor.os.path.exists') @patch('adjustments_json_import.processor.os.remove') def test__remove_temp_file_success(mock_remove, mock_exists, mock_event): """Test call os.remove when the file exists.""" mock_mysql_conn = MagicMock() mock_file_path = Path('/tmp/test.json.gz') mock_exists.return_value = True processor = AdjustmentsJsonImportProcessor(mock_event, mock_mysql_conn) result = processor._remove_temp_file(mock_file_path) mock_exists.assert_called_once_with(mock_file_path) mock_remove.assert_called_once_with(mock_file_path) assert result is True @patch('adjustments_json_import.processor.os.path.exists') @patch('adjustments_json_import.processor.os.remove') def test__remove_temp_file_not_found(mock_remove, mock_exists, mock_event): """Test to NOT call os.remove if the file does not exist.""" mock_mysql_conn = MagicMock() mock_file_path = Path('/tmp/test.json.gz') mock_exists.return_value = False processor = AdjustmentsJsonImportProcessor(mock_event, mock_mysql_conn) result = processor._remove_temp_file(mock_file_path) mock_exists.assert_called_once_with(mock_file_path) mock_remove.assert_not_called() assert result is True @patch('adjustments_json_import.processor.gzip.open') @patch('adjustments_json_import.processor.pd') def test__read_adjustment_json_file( mock_pandas, mock_gzip_open, mock_event, mock_adjustments_json_file_content ): """Test reads the adjustments json file successfully.""" mock_adjustments_json_file = pathlib.PurePath('/tmp/test.json.gz') mocked_file = MagicMock() mocked_file.read.return_value = '{' mock_gzip_open.return_value.__enter__.return_value = mocked_file mock_adjustments_json_data = MagicMock( return_value=enumerate(mock_adjustments_json_file_content) ) setup_mock_pandas( mock_pandas.read_json.return_value, ['astype', 'replace', 'dropna', 'astype', 'where', 'apply'], ).iterrows = mock_adjustments_json_data setup_mock_pandas( mock_pandas.read_json.return_value, ['astype', 'replace', 'dropna', 'astype', 'where', 'apply'], ).empty = False mock_mysql_conn = MagicMock() adjustments_json_import = AdjustmentsJsonImportProcessor( mock_event, mock_mysql_conn ) with patch('adjustments_json_import.processor.open', mocked_file): result = adjustments_json_import._read_adjustment_json_file( mock_adjustments_json_file ) assert result.iterrows == mock_adjustments_json_data @patch('adjustments_json_import.processor.gzip.open') @patch('adjustments_json_import.processor.pd') def test__read_adjustment_json_file_error( mock_pandas, mock_gzip_open, mock_adjustments_json_file_content, mock_event ): """Test raises an error if adjustments json file is empty.""" mock_adjustments_json_file = pathlib.PurePath('/tmp/test.json.gz') mocked_file = MagicMock() mocked_file.read.return_value = '[' mock_gzip_open.return_value.__enter__.return_value = mocked_file mock_adjustments_json_data = MagicMock( return_value=enumerate(mock_adjustments_json_file_content) ) setup_mock_pandas( mock_pandas.read_json.return_value, ['astype', 'replace', 'dropna', 'astype', 'where', 'apply'], ).iterrows = mock_adjustments_json_data setup_mock_pandas( mock_pandas.read_json.return_value, ['astype', 'replace', 'dropna', 'astype', 'where', 'apply'], ).empty = True mock_mysql_conn = MagicMock() adjustmentsJsonImport = AdjustmentsJsonImportProcessor(mock_event, mock_mysql_conn) with ( patch('adjustments_json_import.processor.open', mocked_file), pytest.raises(Exception) as excinfo, ): adjustmentsJsonImport._read_adjustment_json_file(mock_adjustments_json_file) assert str(excinfo.value) == FILE_IS_EMPTY @patch('adjustments_json_import.processor.bulk_insert_query') def test__insert_worksheet_adjustment_entries( mock_bulk_insert_query, mock_event, mock_adjustments_json_file_content ): """Test bulk insert for worksheet_adjustment records.""" mock_mysql_conn = MagicMock() worksheet_adjustments_insert_query = render_sql_template( f'{Path(__file__).parents[2]}/adjustments_json_import/sql_templates/insert_worksheet_adjustments.sql' ) adjustmentsJsonImport = AdjustmentsJsonImportProcessor(mock_event, mock_mysql_conn) adjustmentsJsonImport._insert_worksheet_adjustment_entries( mock_adjustments_json_file_content ) mock_bulk_insert_query.assert_called_once_with( worksheet_adjustments_insert_query, mock_adjustments_json_file_content, mock_mysql_conn, ) def test__get_statement_years(mock_event): """Test to get the statement years.""" data = {'activity_year': [2021, 2022, 2021], 'statement_year': [2022, 2023, 2023]} df = pd.DataFrame(data) mock_mysql_conn = MagicMock() adjustmentsJsonImport = AdjustmentsJsonImportProcessor(mock_event, mock_mysql_conn) result = adjustmentsJsonImport._get_statement_years(df) assert result == ['2021', '2022', '2023'] def test_get_statement_years_empty_cols(mock_event): """Test return an empty list if no valid years are found.""" data = {'activity_year': [None, None], 'statement_year': [None, None]} df = pd.DataFrame(data) mock_mysql_conn = MagicMock() adjustmentsJsonImport = AdjustmentsJsonImportProcessor(mock_event, mock_mysql_conn) result = adjustmentsJsonImport._get_statement_years(df) assert result == [] @patch('adjustments_json_import.processor.execute_query') def test__get_statement_periods( mock_execute_query, mock_statement_periods_list, mock_event ): """Test to get the statement periods.""" mock_execute_query.return_value = mock_statement_periods_list mock_mysql_conn = MagicMock() adjustmentsJsonImport = AdjustmentsJsonImportProcessor(mock_event, mock_mysql_conn) result = adjustmentsJsonImport._get_statement_periods(['2026']) assert result == {2026: {1: 1, 2: 2}} @patch('adjustments_json_import.processor.execute_query') def test__get_statement_periods_error(mock_execute_query, mock_event): """Test raise validation error if there are no statement periods.""" mock_execute_query.return_value = None mock_mysql_conn = MagicMock() adjustmentsJsonImport = AdjustmentsJsonImportProcessor(mock_event, mock_mysql_conn) with pytest.raises(NotFoundError) as excinfo: adjustmentsJsonImport._get_statement_periods(['2026']) assert str(excinfo.value) == ERROR_NO_STATEMENT_PERIODS_FOUND.format(['2026']) @patch('adjustments_json_import.processor.execute_query') def test__get_reference_adjustment_types( mock_execute_query, mock_reference_adjustment_types_list, mock_event ): """Test to get the reference adjustment types.""" mock_execute_query.return_value = mock_reference_adjustment_types_list mock_mysql_conn = MagicMock() adjustmentsJsonImport = AdjustmentsJsonImportProcessor(mock_event, mock_mysql_conn) result = adjustmentsJsonImport._get_reference_adjustment_types() assert result == {'label earnings': 1, 'reclass between labels': 2} @patch('adjustments_json_import.processor.execute_query') def test__get_reference_adjustment_types_error(mock_execute_query, mock_event): """Test raise validation error if there are no reference adjustment types.""" mock_execute_query.return_value = None mock_mysql_conn = MagicMock() adjustmentsJsonImport = AdjustmentsJsonImportProcessor(mock_event, mock_mysql_conn) with pytest.raises(NotFoundError) as excinfo: adjustmentsJsonImport._get_reference_adjustment_types() assert str(excinfo.value) == ERROR_NO_REFERENCE_ADJUSTMENT_TYPES_FOUND @patch('adjustments_json_import.processor.AdjustmentsJsonProcessor') def test__read_adjustments_json_file_and_insert_file_content( mock_adjustment_file_processor, mock_statement_periods, mock_reference_adjustment_types, mock_adjustments_json_file_content, mock_event, ): """Test reads the adjustments json file and insert the records.""" mock_adjustments_json_file = Path('/tmp/test.json.gz') iterrows = MagicMock(return_value=enumerate(mock_adjustments_json_file_content)) AdjustmentsJsonImportProcessor._read_adjustment_json_file = MagicMock( return_value=iterrows ) AdjustmentsJsonImportProcessor._get_statement_years = MagicMock(return_value=[2026]) AdjustmentsJsonImportProcessor._get_statement_periods = MagicMock( return_value=mock_statement_periods ) AdjustmentsJsonImportProcessor._get_reference_adjustment_types = MagicMock( return_value=mock_reference_adjustment_types ) AdjustmentsJsonImportProcessor._insert_worksheet_adjustment_entries = MagicMock() mock_adjustment_file_processor.return_value = MagicMock( process=MagicMock(), _worksheet_adjustment_entries=[mock_adjustments_json_file_content[1]], ) mock_mysql_conn = MagicMock() adjustmentsJsonImport = AdjustmentsJsonImportProcessor(mock_event, mock_mysql_conn) adjustmentsJsonImport._read_adjustments_json_file_and_insert_file_content( mock_adjustments_json_file ) AdjustmentsJsonImportProcessor._insert_worksheet_adjustment_entries.assert_called_once_with( [mock_adjustments_json_file_content[1]] ) def test_process(mock_event): """Test process method.""" mock_adjustments_json_file = Path('/tmp/test.json.gz') AdjustmentsJsonImportProcessor._get_statement_period_adjustment_file = MagicMock( return_value=mock_adjustments_json_file ) AdjustmentsJsonImportProcessor._read_adjustments_json_file_and_insert_file_content = MagicMock( return_value=True ) mock_mysql_conn = MagicMock() adjustmentImport = AdjustmentsJsonImportProcessor(mock_event, mock_mysql_conn) adjustmentImport.process() AdjustmentsJsonImportProcessor._read_adjustments_json_file_and_insert_file_content.assert_called_once() AdjustmentsJsonImportProcessor._get_statement_period_adjustment_file.assert_called_once()