"""Asset copy worker entrypoint.""" import json import uuid import boto3 from mypy_boto3_s3.service_resource import S3ServiceResource from mypy_boto3_s3.type_defs import CopySourceTypeDef from owslogger.logger import OwsLoggingAdapter from src.config import ( EXECUTION_NAME, EXPORT_ID, PART_INDEX, S3_INPUT_BUCKET, S3_OUTPUT_BUCKET, app_logger, ) def main(correlation_id: uuid.UUID) -> None: """Run application.""" logger = OwsLoggingAdapter(app_logger, {"correlation_id": correlation_id}) s3_client: S3ServiceResource = boto3.resource("s3") asset_locations = get_asset_locations(s3_client, logger) copy_errors = copy_assets(s3_client, asset_locations, logger) if copy_errors: error_filename = ( f"{EXPORT_ID}/{EXECUTION_NAME}/asset_copy_errors_{PART_INDEX}.json" ) try: s3_client.Object(S3_INPUT_BUCKET, error_filename).put( Body=json.dumps(copy_errors), ContentType="application/json" ) logger.error( f"There were errors while copying assets. " f"See {S3_INPUT_BUCKET}/{error_filename}." ) except Exception: logger.exception( f"There were errors while copying assets for export {EXPORT_ID}, " f"but the error report for part {PART_INDEX} failed to write to S3.", stack_info=True, ) logger.info( f"Asset copy finished for export: {EXPORT_ID} " f"- execution name: {EXECUTION_NAME} " f"- part: {PART_INDEX} with {len(copy_errors)} errors." ) def get_asset_locations( s3_client: S3ServiceResource, logger: OwsLoggingAdapter ) -> list[dict[str, str]]: obj = s3_client.Object( S3_INPUT_BUCKET, f"{EXPORT_ID}/{EXECUTION_NAME}/asset_locations_{PART_INDEX}.json", ) response = obj.get() body = response["Body"].read().decode("utf-8") asset_locations_object = json.loads(body) if ( not asset_locations_object or asset_locations_object.get("asset_locations") is None or asset_locations_object.get("asset_locations") == [] ): logger.error( f"No asset locations found for export: {EXPORT_ID} " f"- execution name: {EXECUTION_NAME} - part: {PART_INDEX}" ) return [] asset_locations: list[dict[str, str]] = asset_locations_object["asset_locations"] return asset_locations def copy_assets( s3_client: S3ServiceResource, asset_locations: list[dict[str, str]], logger: OwsLoggingAdapter, ) -> list[dict[str, str]]: asset_copy_errors: list[dict[str, str]] = [] logger.info( f"Copying assets for export: {EXPORT_ID} " f"- execution name: {EXECUTION_NAME} - part: {PART_INDEX}..." ) for asset_location in asset_locations: source = asset_location["source"].split("/") source_dict: CopySourceTypeDef = { "Bucket": source[0], "Key": "/".join(source[1:]), } destination = f"{EXPORT_ID}/{asset_location['destination']}" try: s3_client.meta.client.copy( CopySource=source_dict, Bucket=S3_OUTPUT_BUCKET, Key=destination, ) except Exception as e: asset_copy_errors.append( { **asset_location, "error_reason": f"{e.__class__.__name__}: {str(e)}", } ) return asset_copy_errors if __name__ == "__main__": main(uuid.uuid4())