"""Test the backfill CLI.""" import json import re import pytest from tests.integration.utils import run_command def test_backfill() -> None: """Test sync one command works. The files from `test/integration/data` have been uploaded to S3. """ stdout, _ = run_command( [ "poetry", "run", "pdpbackfill", "backfill", "process", "--bucket-name=qa-pdp-backfill", "--manifest-file=integration-test/manifest.json", ], expected_returncode=0, ) # "integration-test/01_test_detach.csv" has 3 detach lines attach_matches = re.findall("OPERATION_DETACH", stdout) assert len(attach_matches) == 3 # "integration-test/02_test_attach.csv" has 3 attach lines attach_matches = re.findall("OPERATION_ATTACH", stdout) assert len(attach_matches) == 3 # There are 2 files in the backfill job. # - "integration-test/01_test_detach.csv" # - "integration-test/02_test_attach.csv" # # Both contain 1 unique identity id. Each file should have 1 # `success`ful PDP attach-detach request and 0 `fail`. summary_matches = re.findall(r'"summary": ({"success": 1, "fail": 0})', stdout) assert len(summary_matches) == 2 for match in summary_matches: assert json.loads(match) == {"success": 1, "fail": 0} @pytest.mark.parametrize( "filename", [ pytest.param( "bom_manifest.json", id="manifest is utf-8 with bom encoded, referencing a csv that is utf-8 with bom-encoded", ), pytest.param( "utf8_manifest_ref_bom_csv.json", id="manifest is utf-8 encoded, referencing a csv that is utf-8 with bom-encoded", ), ], ) def test_backfill_bom(filename: str) -> None: """Test backfill command works with utf-8 with bom-encoded files. The files from `test/integration/data` have been uploaded to S3. """ stdout, _ = run_command( [ "poetry", "run", "pdpbackfill", "backfill", "process", "--bucket-name=qa-pdp-backfill", f"--manifest-file=integration-test/{filename}", ], expected_returncode=0, ) # "integration-test/bom_supported.csv" has 3 attach lines # "integration-test/neo4j_export_style.csv" has 3 attach lines attach_matches = re.findall("OPERATION_ATTACH", stdout) assert len(attach_matches) == 6 # There are 2 files in the backfill job. # - "integration-test/bom_supported.csv" # - "integration-test/neo4j_export_style.csv" # # It contains 2 unique identity ids and should have 2 # `success`ful PDP attach-detach request and 0 `fail`. summary_matches = re.findall(r'"summary": ({"success": 1, "fail": 0})', stdout) assert len(summary_matches) == 2 for match in summary_matches: assert json.loads(match) == {"success": 1, "fail": 0} @pytest.mark.parametrize( "additional_args,expected_error_message", [ pytest.param( [], "Missing option '--bucket-name'.", id="Missing both args", ), pytest.param( ["--bucket-name=qa-pdp-backfill"], "Missing option '--manifest-file'.", id="--manifest-file is missing", ), pytest.param( ["--manifest-file=integration-test/manifest.json"], "Missing option '--bucket-name'.", id="--bucket-name is missing", ), ], ) def test_process_arguments( additional_args: list[str], expected_error_message: str, ) -> None: """Test sync all command requires environment argument.""" command_args = [ "poetry", "run", "pdpbackfill", "backfill", "process", ] command_args.extend(additional_args) stdout, _ = run_command( command_args, expected_returncode=2, description=json.dumps(f"Additional arguments: {additional_args}"), ) assert expected_error_message in stdout