"""Lambda trigger ECS tasks module.""" import time import boto3 import sentry_sdk from urllib import parse from botocore.client import Config from botocore.exceptions import ClientError as S3ClientError from boto3.s3.transfer import TransferConfig from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration import config from config import logger sentry_sdk.init( dsn=config.SENTRY_DSN, integrations=[AwsLambdaIntegration()], traces_sample_rate=1.0, ) transfer_config = TransferConfig( max_concurrency=config.MAX_CONCURRENCY, multipart_chunksize=config.MULTIPART_CHUNKSIZE, ) config = Config( max_pool_connections=config.MAX_POOL_CONNECTIONS, retries=dict( max_attempts=config.MAX_ATTEMPTS, ), ) s3_client = boto3.client('s3', config=config) def handler(event, context): """Lambda entry point.""" job_id = event.get('job').get('id') invocation_id = event.get('invocationId') invocation_schema_version = event.get('invocationSchemaVersion') destination_bucket = event.get('job').get('userArguments').get('destination-bucket') destination_prefix = ( event.get('job').get('userArguments').get('destination-prefix', None) ) destination_storage_class = ( event.get('job') .get('userArguments') .get('destination-storage-class', 'STANDARD') ) copy_metadata = ( event.get('job').get('userArguments').get('copy-metadata', 'false').lower() == 'true' ) copy_tags = ( event.get('job').get('userArguments').get('copy-tags', 'false').lower() == 'true' ) task_id = event.get('tasks')[0].get('taskId') source_bucket = event.get('tasks')[0].get('s3Bucket') source_key = parse.unquote_plus(event.get('tasks')[0].get('s3Key')) source_version_id = event.get('tasks')[0].get('s3VersionId') my_args = dict( ACL='bucket-owner-full-control', StorageClass=destination_storage_class, ) logger.info(f'Job ID: {job_id}') # logger.info(f'Invocation ID: {invocation_id}') logger.info(f'Invocation Schema Version: {invocation_schema_version}') logger.info(f'Destination Bucket: {destination_bucket}') logger.info(f'Destination Prefix: {destination_prefix}') logger.info(f'Destination Storage Class: {destination_storage_class}') logger.info(f'Copy Metadata: {copy_metadata}') logger.info(f'Copy Tags: {copy_tags}') # logger.info(f'Task ID: {task_id}') logger.info(f'Source Bucket: {source_bucket}') logger.info(f'Source Key: {source_key}') logger.info(f'Source Version ID: {source_version_id}') results = [] try: result_code = None result_string = None copy_source = dict( Bucket=source_bucket, Key=source_key, ) if source_version_id: copy_source['VersionId'] = source_version_id if copy_metadata: get_metadata = s3_client.head_object( Bucket=source_bucket, Key=source_key, VersionId=source_version_id ) if copy_tags: get_tags = s3_client.get_object_tagging( Bucket=source_bucket, Key=source_key, VersionId=source_version_id ) else: if copy_metadata: get_metadata = s3_client.head_object( Bucket=source_bucket, Key=source_key ) if copy_tags: get_tags = s3_client.get_object_tagging( Bucket=source_bucket, Key=source_key ) if destination_prefix and len(destination_prefix) > 0: destination_key = f'{destination_prefix}/{source_key}' else: destination_key = source_key if copy_metadata: logger.info('Metadata will be copied') cache_control = get_metadata.get('CacheControl') content_disposition = get_metadata.get('ContentDisposition') content_encoding = get_metadata.get('ContentEncoding') content_language = get_metadata.get('ContentLanguage') metadata = get_metadata.get('Metadata') website_redirect_location = get_metadata.get('WebsiteRedirectLocation') expires = get_metadata.get('Expires') if cache_control: my_args['CacheControl'] = cache_control if content_disposition: my_args['ContentDisposition'] = content_disposition if content_encoding: my_args['ContentEncoding'] = content_encoding if content_language: my_args['ContentLanguage'] = content_language if metadata: my_args['Metadata'] = metadata if website_redirect_location: my_args['WebsiteRedirectLocation'] = website_redirect_location if expires: my_args['Expires'] = expires else: logger.info('Metadata will not be copied') if copy_tags: logger.info('Tags will be copied') existing_tag_set = get_tags.get('TagSet') tagging_to_s3 = '&'.join( [ f"{parse.quote_plus(d['Key'])}={parse.quote_plus(d['Value'])}" for d in existing_tag_set ] ) if existing_tag_set: my_args['Tagging'] = tagging_to_s3 else: logger.info('Tags will not be copied') logger.info( f'Copying object from {source_bucket} to {destination_bucket} with key {destination_key}' ) start_time = time.time() result = s3_client.copy( copy_source, destination_bucket, destination_key, Config=transfer_config, ExtraArgs=my_args, ) end_time = time.time() duration = end_time - start_time # Get source object size for transfer speed calculation try: source_object = s3_client.head_object( Bucket=source_bucket, Key=source_key, VersionId=source_version_id if source_version_id else None ) size_bytes = source_object['ContentLength'] speed_mbps = (size_bytes / 1024 / 1024) / duration logger.info(f'Successfully copied object in {duration:.2f} seconds ({speed_mbps:.2f} MB/s)') except Exception as e: logger.info(f'Successfully copied object in {duration:.2f} seconds') logger.debug(f'Error: {e}') logger.info('Successfully copied object') result_code = 'Succeeded' result_string = str(result) except S3ClientError as e: logger.exception(str(e)) sentry_sdk.capture_exception(e) result_code = 'PermanentFailure' result_string = str(e) except Exception as error: logger.exception(str(error)) sentry_sdk.capture_exception(error) result_code = 'PermanentFailure' result_string = str(error) finally: results.append( { 'taskId': task_id, 'resultCode': result_code, 'resultString': result_string, } ) return { 'invocationId': invocation_id, 'invocationSchemaVersion': invocation_schema_version, 'results': results, 'treatMissingKeysAs': 'PermanentFailure', }