"""Process CLI""" import logging import re import uuid from typing import Annotated import boto3 import typer from backfill import config from backfill.connectors.ows_pdp.ows_pdp import get_ows_pdp_connector from backfill.connectors.s3_connector import S3 from backfill.error_handlers import handle_uncaught_errors from backfill.manifest_processor import ManifestProcessor from backfill.models import Manifest logger: logging.Logger = logging.getLogger(__name__) backfill_cli: typer.Typer = typer.Typer( short_help="Commands to perform pdp backfill", no_args_is_help=True ) VALID_MANIFEST_INPUT = re.compile(r"^[a-zA-Z0-9\-/_]+\.json$").match VALID_BUCKET_INPUT = re.compile(r"^(qa|dev|prod|uat)-pdp-backfill$") def manifest_bucket_validator(value: str) -> str: """Validate the bucket name input.""" if not VALID_BUCKET_INPUT.match(value): raise typer.BadParameter( f"The bucket name must be one of 'qa', 'dev', 'prod', or 'uat' followed by '-pdp-backfill'. Found: '{value}'" ) return value def manifest_file_validator(value: str) -> str: """Validate the filepath input.""" if not VALID_MANIFEST_INPUT(value): raise typer.BadParameter( f"The file path must be a snake_case string with json extension. Found: '{value}'" # noqa: E501 ) return value @backfill_cli.command( "process", short_help="Perform a PDP backfill defined by manifest.json." ) @handle_uncaught_errors def process( bucket_name: Annotated[ str, typer.Option( help="Name of the S3 bucket to process.", callback=manifest_bucket_validator, ), ], manifest_file: Annotated[ str, typer.Option( help="Path to manifest json file in S3.", callback=manifest_file_validator ), ], ) -> None: """Handler for the `process` command.""" typer.secho( f"Processing '{bucket_name}/{manifest_file}' in environment '{config.ENVIRONMENT}'" # noqa: E501 ) backfill_uuid_getter = CorrelationIdGetter(correlation_id=str(uuid.uuid4())) run_backfill( bucket_name=bucket_name, manifest_file_key=manifest_file, backfill_uuid_getter=backfill_uuid_getter, ) class CorrelationIdGetter: def __init__(self, correlation_id: str) -> None: self.correlation_id = correlation_id def __call__(self) -> str: """Return a correlation Id""" return self.correlation_id def run_backfill( bucket_name: str, manifest_file_key: str, backfill_uuid_getter: CorrelationIdGetter ) -> None: """Run the backfill command.""" s3_connector = S3() if not s3_connector.does_file_exist( bucket=bucket_name, key=manifest_file_key, ): logger.error( "Did not find the manifest file: '%s/%s'", bucket_name, manifest_file_key, extra={ "resources": { "bucket": bucket_name, "key": manifest_file_key, }, "correlation_id": backfill_uuid_getter(), }, ) raise typer.Exit(1) try: manifest_file_contents = s3_connector.get_file_content( bucket=bucket_name, key=manifest_file_key, ) manifest = Manifest.model_validate_json(manifest_file_contents) ows_pdp_client = get_ows_pdp_connector( config.ENVIRONMENT, correlation_id_getter=backfill_uuid_getter ) mp = ManifestProcessor( manifest=manifest, ows_pdp_client=ows_pdp_client, s3_connector=s3_connector, backfill_uuid=backfill_uuid_getter(), ) mp.process() except Exception as ex: logger.warning( "Failed to run backfill for '%s/%s'. Error: %s", bucket_name, manifest_file_key, str(ex), extra={ "resources": { "bucket": bucket_name, "key": manifest_file_key, }, "correlation_id": backfill_uuid_getter(), }, ) raise typer.Exit(1) from ex else: logger.info( "Completed backfill for '%s/%s'", bucket_name, manifest_file_key, extra={ "resources": { "bucket": bucket_name, "key": manifest_file_key, }, "correlation_id": backfill_uuid_getter(), }, ) @backfill_cli.command("invoke_sfn", short_help="Invoke a sfn.") def invoke_sfn( state_machine_arn: Annotated[ str, typer.Option( help="The Amazon Resource Name (ARN) of the state machine to execute.", ), ], input: Annotated[ str, typer.Option(help="Contains the JSON input data for the execution") ] = "{}", ) -> None: """Invokes a step function, usually to perform a step function..""" client = boto3.client("stepfunctions") logger.info( "Starting execution of state machine", extra={ "state_machine_arn": state_machine_arn, }, ) result = client.start_execution( stateMachineArn=state_machine_arn, input=input, ) logger.info( "Started state machine", extra={ "state_machine_arn": state_machine_arn, "execution_details": result, }, )