import json import logging from collections.abc import Generator from unittest.mock import MagicMock, patch import pytest import typer from typer.testing import CliRunner from backfill import config from backfill.cli.backfill import CorrelationIdGetter, invoke_sfn, run_backfill from backfill.cli.main import app as backfill_cli from backfill.models import Manifest runner = CliRunner() @pytest.mark.parametrize( "args, expected_message, should_fail", [ pytest.param( [ "backfill", "process", "--bucket-name", "qa-pdp-backfill", "--manifest-file", "manifest.json", ], "Processing 'qa-pdp-backfill/manifest.json' in environment 'test'", False, id="Valid bucket and manifest file", ), pytest.param( [ "backfill", "process", "--bucket-name", "qa-pdp-backfill", "--manifest-file", "path/to/manifest.json", ], "Processing 'qa-pdp-backfill/path/to/manifest.json' in environment 'test'", False, id="Valid bucket and manifest file in a subdirectory", ), pytest.param( [ "backfill", "process", "--bucket-name", "qa-pdp-backfill", "--manifest-file", "content-review/2025-03-01/manifest_1.json", ], "Processing 'qa-pdp-backfill/content-review/2025-03-01/manifest_1.json' in environment 'test'", False, id="Valid bucket and manifest file in a subdirectory with numbers", ), pytest.param( [ "backfill", "process", "--bucket-name", "qa-pdp-backfill", "--manifest-file", "2025/05/07/PLATFORM-4468/MANIFEST.json", ], "Processing 'qa-pdp-backfill/2025/05/07/PLATFORM-4468/MANIFEST.json' in environment 'test'", False, id="Valid bucket and manifest file in a subdirectory with with uppercase letters", ), pytest.param( [ "backfill", "process", "--bucket-name", "invalid-bucket", "--manifest-file", "valid_manifest.json", ], "Invalid value for '--bucket-name':", True, id="Invalid bucket name", ), pytest.param( [ "backfill", "process", "--bucket-name", "qa-pdp-backfill", "--manifest-file", "invalid_manifest.txt", ], "Invalid value for '--manifest-file':", True, id="Invalid manifest file - wrong extension", ), pytest.param( [ "backfill", "process", "--bucket-name", "qa-pdp-backfill", "--manifest-file", "invalid manifest.json", ], "Invalid value for '--manifest-file':", True, id="Invalid manifest file - no spaces", ), ], ) @patch("backfill.cli.backfill.typer.secho") @patch("backfill.cli.backfill.config.ENVIRONMENT", "test") @patch("backfill.cli.backfill.run_backfill") def test_process( mock_run_backfill: MagicMock, mock_secho: MagicMock, args: list[str], expected_message: str, should_fail: bool, ) -> None: """Test the process command with mocked typer.secho.""" result = runner.invoke(backfill_cli, args) if should_fail: mock_secho.assert_not_called() assert result.exit_code != 0 assert expected_message in result.output else: mock_secho.assert_called_once_with(expected_message) assert result.exit_code == 0 mock_run_backfill.assert_called_once() @pytest.fixture() def manifest_data_str() -> str: """Mock manifest data.""" return json.dumps( { "bucket": "dev-mybucket", "jobs": [{"job_type": "attach_and_detach", "keys": ["car", "piano"]}], } ) @patch("backfill.cli.backfill.S3") @patch("backfill.cli.backfill.get_ows_pdp_connector") @patch("backfill.cli.backfill.ManifestProcessor") def test_run_backfill__manifest_missing( mock_manifest_processor: MagicMock, mock_get_ows_pdp_connector: MagicMock, mock_s3_connector: MagicMock, caplog: pytest.LogCaptureFixture, backfill_uuid_getter: CorrelationIdGetter, ) -> None: """Test run_backfill when manifest is missing.""" caplog.set_level(logging.ERROR) mock_s3_instance = mock_s3_connector.return_value mock_s3_instance.does_file_exist.return_value = False with pytest.raises(typer.Exit) as excinfo: run_backfill( bucket_name="test-pdp-backfill", manifest_file_key="path/to/manifest.json", backfill_uuid_getter=backfill_uuid_getter, ) assert excinfo.value.exit_code == 1 with caplog.at_level(logging.ERROR): assert "Did not find the manifest file" in caplog.text mock_get_ows_pdp_connector.assert_not_called() mock_manifest_processor.assert_not_called() @patch("backfill.cli.backfill.S3") @patch("backfill.cli.backfill.get_ows_pdp_connector") @patch("backfill.cli.backfill.ManifestProcessor") def test_run_backfill( mock_manifest_processor: MagicMock, mock_get_ows_pdp_connector: MagicMock, mock_s3_connector: MagicMock, backfill_uuid_getter: CorrelationIdGetter, manifest_data_str: str, monkeypatch: pytest.MonkeyPatch, backfill_uuid: str, ) -> None: """Test run_backfill.""" monkeypatch.setattr(config, "ENVIRONMENT", "test") mock_s3_instance = mock_s3_connector.return_value mp_instance = mock_manifest_processor.return_value mock_pdp_connector = mock_get_ows_pdp_connector.return_value mock_s3_instance.does_file_exist.return_value = True mock_s3_instance.get_file_content.return_value = manifest_data_str run_backfill( bucket_name="test-pdp-backfill", manifest_file_key="path/to/manifest.json", backfill_uuid_getter=backfill_uuid_getter, ) mock_s3_instance.get_file_content.assert_called_with( bucket="test-pdp-backfill", key="path/to/manifest.json", ) mock_get_ows_pdp_connector.assert_called_once_with( "test", correlation_id_getter=backfill_uuid_getter ) mock_manifest_processor.assert_called_once_with( manifest=Manifest.model_validate_json(manifest_data_str), ows_pdp_client=mock_pdp_connector, s3_connector=mock_s3_instance, backfill_uuid=backfill_uuid, ) mp_instance.process.assert_called_once() @patch("backfill.cli.backfill.S3") @patch("backfill.cli.backfill.get_ows_pdp_connector") @patch("backfill.cli.backfill.ManifestProcessor") def test_run_backfill__process_error( mock_manifest_processor: MagicMock, mock_get_ows_pdp_connector: MagicMock, mock_s3_connector: MagicMock, manifest_data_str: str, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch, backfill_uuid_getter: CorrelationIdGetter, backfill_uuid: str, ) -> None: """Test run_backfill when process raises an error.""" monkeypatch.setattr(config, "ENVIRONMENT", "test") mock_s3_instance = mock_s3_connector.return_value mock_pdp_connector = mock_get_ows_pdp_connector.return_value mp_instance = mock_manifest_processor.return_value mock_s3_instance.get_file_content.return_value = manifest_data_str mock_s3_instance.does_file_exist.return_value = True mp_instance.process.side_effect = [RuntimeError("ManifestProcessor error.")] with pytest.raises(typer.Exit) as excinfo: run_backfill( bucket_name="test-pdp-backfill", manifest_file_key="path/to/manifest.json", backfill_uuid_getter=backfill_uuid_getter, ) assert excinfo.value.exit_code == 1 with caplog.at_level(logging.WARNING): assert "Failed to run backfill" in str(caplog.text) mock_get_ows_pdp_connector.assert_called_once_with( "test", correlation_id_getter=backfill_uuid_getter ) mp_instance.process.assert_called_once() mock_manifest_processor.assert_called_once_with( manifest=Manifest.model_validate_json(manifest_data_str), ows_pdp_client=mock_pdp_connector, s3_connector=mock_s3_instance, backfill_uuid=backfill_uuid, ) @pytest.fixture def mock_boto3_client() -> Generator[MagicMock, None, None]: with patch("backfill.cli.backfill.boto3") as mock_boto3: yield mock_boto3.client.return_value def test_invoke_sfn( mock_boto3_client: MagicMock, caplog: pytest.LogCaptureFixture, ) -> None: """Test invoke_sfn.""" state_machine_arn = "arn:some-sfn" with caplog.at_level(logging.INFO): invoke_sfn(state_machine_arn) mock_boto3_client.start_execution.assert_called_once_with( stateMachineArn=state_machine_arn, input="{}", ) assert "Starting execution of state machine" in caplog.messages assert "Started state machine" in caplog.messages def test_invoke_sfn_uses_input( mock_boto3_client: MagicMock, caplog: pytest.LogCaptureFixture, ) -> None: """Test invoke_sfn.""" state_machine_arn = "arn:some-sfn" json_input = '{"dogs": "rule"}' with caplog.at_level(logging.INFO): invoke_sfn(state_machine_arn, json_input) mock_boto3_client.start_execution.assert_called_once_with( stateMachineArn=state_machine_arn, input=json_input, ) assert "Starting execution of state machine" in caplog.messages assert "Started state machine" in caplog.messages