"""Test ManifestProcessor.""" import logging from unittest.mock import MagicMock, call, patch import pytest from backfill.connectors.ows_pdp.ows_pdp import OwsPdpClient from backfill.connectors.s3_connector import S3, FailedDownloadException from backfill.constants import JobType from backfill.job_processors import MissingFileException from backfill.job_processors.attach_and_detach_processor import AttachAndDetachProcessor from backfill.manifest_processor import ManifestProcessor, UnsupportedJobTypeException from backfill.models import Job, Manifest @pytest.fixture() def bucket() -> str: return "dev-backfill-bucket" @pytest.fixture() def manifest(bucket: str) -> Manifest: return Manifest( bucket=bucket, jobs=[ Job( job_type=JobType.JOB_TYPE_ATTACH_AND_DETACH, keys=[ "first.csv", "second.csv", ], ), Job( job_type=JobType.JOB_TYPE_ATTACH_AND_DETACH, keys=[ "a.csv", "b.csv", ], ), ], ) @pytest.fixture() def mock_ows_pdp_client() -> MagicMock: return MagicMock(spec=OwsPdpClient) @pytest.fixture() def mock_s3_connector() -> MagicMock: return MagicMock(spec=S3) @pytest.fixture() def processor( manifest: Manifest, mock_ows_pdp_client: MagicMock, mock_s3_connector: MagicMock, backfill_uuid: str, ) -> ManifestProcessor: """Return ManifestProcessor.""" return ManifestProcessor( manifest, mock_ows_pdp_client, mock_s3_connector, backfill_uuid=backfill_uuid, ) @patch.object(ManifestProcessor, "_process_job") def test_process( mock__process_job: MagicMock, bucket: str, manifest: Manifest, processor: ManifestProcessor, ) -> None: """Test process.""" processor.process() mock__process_job.assert_has_calls( [ call(bucket, manifest.jobs[0]), call(bucket, manifest.jobs[1]), ] ) @patch.object(ManifestProcessor, "_process_file") def test__process_job( mock__process_file: MagicMock, bucket: str, manifest: Manifest, processor: ManifestProcessor, ) -> None: """Test _process_job.""" processor._process_job(bucket, manifest.jobs[0]) mock__process_file.assert_has_calls( [ call(bucket, manifest.jobs[0].keys[0], manifest.jobs[0].job_type), call(bucket, manifest.jobs[0].keys[1], manifest.jobs[0].job_type), ] ) @patch("backfill.manifest_processor.SUPPORTED_JOB_PROCESSORS") @patch("backfill.manifest_processor.generate_local_filepath") @patch("backfill.manifest_processor.os") def test__process_file( mock_os: MagicMock, mock_generate_local_filepath: MagicMock, mock_supported_job_processors: MagicMock, processor: ManifestProcessor, mock_ows_pdp_client: MagicMock, mock_s3_connector: MagicMock, backfill_uuid: str, ) -> None: """Test _process_file, happy path.""" mock_csv_filepath = "temp/local/storage.csv" mock_generate_local_filepath.return_value = mock_csv_filepath mock_s3_connector.download_file.return_value = mock_csv_filepath mock_attach_and_detach_processor = MagicMock(autospec=AttachAndDetachProcessor) mock_supported_job_processors.get.return_value = mock_attach_and_detach_processor processor._process_file( "some_bucket", "some/key", JobType.JOB_TYPE_ATTACH_AND_DETACH ) mock_s3_connector.download_file.assert_called_once_with( "some_bucket", "some/key", mock_csv_filepath ) mock_attach_and_detach_processor.process.assert_called_once_with( mock_csv_filepath, mock_ows_pdp_client, backfill_uuid=backfill_uuid, ) mock_os.remove.assert_called_once_with(mock_csv_filepath) @patch("backfill.manifest_processor.SUPPORTED_JOB_PROCESSORS") @patch("backfill.manifest_processor.generate_local_filepath") @patch("backfill.manifest_processor.os") def test__process_file_exception_from_s3_connector( mock_os: MagicMock, mock_generate_local_filepath: MagicMock, mock_supported_job_processors: MagicMock, processor: ManifestProcessor, mock_s3_connector: MagicMock, caplog: pytest.LogCaptureFixture, ) -> None: """Test _process_file, exception from S3 download_file.""" mock_csv_filepath = "temp/local/storage.csv" mock_generate_local_filepath.return_value = mock_csv_filepath mock_s3_connector.download_file.side_effect = FailedDownloadException("no file") mock_attach_and_detach_processor = MagicMock(autospec=AttachAndDetachProcessor) mock_supported_job_processors.get.return_value = mock_attach_and_detach_processor with caplog.at_level(logging.ERROR): processor._process_file( "some_bucket", "some/key", JobType.JOB_TYPE_ATTACH_AND_DETACH ) mock_s3_connector.download_file.assert_called_once_with( "some_bucket", "some/key", mock_csv_filepath ) mock_attach_and_detach_processor.process.assert_not_called() mock_os.remove.assert_not_called() assert "Failed download from S3" in caplog.text @patch("backfill.manifest_processor.SUPPORTED_JOB_PROCESSORS") @patch("backfill.manifest_processor.generate_local_filepath") @patch("backfill.manifest_processor.os") def test__process_file_exception_from_processor( mock_os: MagicMock, mock_generate_local_filepath: MagicMock, mock_supported_job_processors: MagicMock, processor: ManifestProcessor, mock_ows_pdp_client: MagicMock, mock_s3_connector: MagicMock, caplog: pytest.LogCaptureFixture, backfill_uuid: str, ) -> None: """Test _process_file, exception from AttachAndDetachProcessor.""" mock_csv_filepath = "temp/local/storage.csv" mock_generate_local_filepath.return_value = mock_csv_filepath mock_s3_connector.download_file.return_value = mock_csv_filepath mock_attach_and_detach_processor = MagicMock(autospec=AttachAndDetachProcessor) mock_supported_job_processors.get.return_value = mock_attach_and_detach_processor mock_attach_and_detach_processor.process.side_effect = MissingFileException( "some error" ) with caplog.at_level(logging.ERROR): processor._process_file( "some_bucket", "some/key", JobType.JOB_TYPE_ATTACH_AND_DETACH ) mock_s3_connector.download_file.assert_called_once_with( "some_bucket", "some/key", mock_csv_filepath ) mock_attach_and_detach_processor.process.assert_called_once_with( mock_csv_filepath, mock_ows_pdp_client, backfill_uuid=backfill_uuid, ) mock_os.remove.assert_called_once_with(mock_csv_filepath) assert "some error" in caplog.text @patch("backfill.manifest_processor.generate_local_filepath") @patch("backfill.manifest_processor.os") def test__process_file_unsuppported_processor( mock_os: MagicMock, mock_generate_local_filepath: MagicMock, processor: ManifestProcessor, ) -> None: """Test _process_file, unsupported processor.""" mock_csv_filepath = "temp/local/storage.csv" mock_generate_local_filepath.return_value = mock_csv_filepath with pytest.raises(UnsupportedJobTypeException) as exc: processor._process_file( "some_bucket", "some/key", "fake job type", # type: ignore[arg-type] ) assert exc.value.args == ("fake job type is not supported",) mock_generate_local_filepath.assert_not_called() mock_os.remove.assert_not_called()