"""Lambda function module.""" from owsrequest import request from sentry_sdk import capture_exception from src import config from src.config import logger from constants import general from constants import errors from src import lambda_exceptions from src import s3 from src import status from src import util def skip_if_asset_upload_already_handled(filename): """Skip if asset upload already handled.""" response = request.process( application=config.APPLICATION_NAME, environment=config.ENVIRONMENT, method='GET', service_name=config.OWS_ASSETS_SERVICE_NAME, path=config.GET_ASSET_STATUS_PATH_TEMPLATE.format(filename=filename), headers={'Orchard-User-Id': 'oa:179'} ) if response.status_code == 404: return if response.status_code == 200: asset_status = response.json()['status'] if asset_status in ('upload_complete', 'acknowledge_error'): return raise lambda_exceptions.AssetUploadAlreadyHandled() raise Exception(response.text) def post_asset( upc, track_unique_id, product_id, filename, bucket, original_filename, is_correction): """Call ows-asset post asset handlers. Args: upc (str): UPC. track_unique_id (int): Track unique id. product_id (int): Product id. filename (str): Unique filename with extension. bucket (str): Bucket where file is stored. original_filename (str): Original asset filename. is_correction (int): Flag that identifies which workflow will be used in RBE for processing (1 - error correction,0 - upload or other) Returns: bool: True if success else raise Exception. """ try: data = { 'filename': filename, } # When asset uploads are initiated by POST /upload-token, metadata like product_id and track_unique_id isn't # stored in the asset_upload table; instead, the uploader attaches the metadata to the S3 object. This lambda # retrieves the metadata and sends it to ows-assets to be persisted. # # When asset uploads are initiated by POST /v2/assets/upload, metadata is stored in the asset_upload table # right away, and metadata can't be set on the S3 object by the uploader. # We call ows-assets in this case, not to update any metadata, but to ensure that the metadata has already been # persisted in the asset_upload table. if product_id: data.update({ 'upc': upc, 'track_unique_id': int(track_unique_id), 'product_id': int(product_id), 'original_filename': original_filename, 'is_correction': bool(is_correction) }) response = request.process( application=config.APPLICATION_NAME, environment=config.ENVIRONMENT, method='POST', service_name=config.OWS_ASSETS_SERVICE_NAME, path=config.POST_ASSET_PATH, json=data ) if response.status_code != 200: error_text = errors.POST_ASSET_ERROR_MESSAGE.format( source='acknowledge lambda', status=response.status_code, text=response.text) logger.exception(error_text) lambda_exceptions.notify_and_raise( general.LAMBDA_NAME, general.ACKNOWLEDGE_ERROR_STATUS, errors.POST_ASSET_ERROR_CODE, filename, bucket, { 'code': response.status_code}, data) except Exception as e: logger.exception(str(e)) raise return True def get_metadata(s3_obj, filename, bucket): """Get asset metadata from S3 object. Args: s3_obj (dict): S3 Object information filename (str): Current processed asset filename. bucket (str): Current processed asset source bucket name. Returns: dict: Uploaded asset metadata """ required_keys = ['product_id', 'upc', 'original_filename'] result = {} metadata = s3_obj.get('Metadata') for key in required_keys: result[key] = metadata.get(key) result['track_unique_id'] = metadata.get('track_unique_id', 0) result['is_correction'] = int(metadata.get('is_correction', 0)) result['filename'] = filename result['bucket'] = bucket return result def handler(event, context): """Lambda entry point.""" key = None bucket = None try: bucket, key = util.extract_triggered_key(event) if config.PAUSE_TRANSCODING: logger.critical(f'Transcoding paused: {bucket} {key}') raise lambda_exceptions.StopProcessingException( errors.STOP_V2_PROCESSING_CODE) s3_obj = s3.head_object(bucket, key) if not s3_obj: lambda_exceptions.notify_and_raise( general.LAMBDA_NAME, general.UPLOAD_ERROR_STATUS, errors.S3_FILE_NOT_FOUND_CODE, key, bucket, { 'key': key, 'bucket': bucket}) metadata = get_metadata(s3_obj, key, bucket) # This check is to prevent reprocessing of the same asset upload due to # duplicate s3 event notifications since they are guaranteed to be sent # at least once, and may be sent more than once. # This check lets some duplicate s3 event notifications slip through # since the status check and update are not done atomically within a lock. # TODO: Remove this in favor of the solution outlined here: # https://github.com/theorchard/lambda-assets/pull/370/files#r1529188357 skip_if_asset_upload_already_handled(metadata['filename']) status.send_general_status( general.LAMBDA_NAME, general.UPLOAD_COMPLETE_STATUS, key, input_params=metadata.copy(), bucket=bucket) post_asset(**metadata) status.send_general_status( general.LAMBDA_NAME, general.ACKNOWLEDGE_COMPLETE_STATUS, key, input_params=metadata.copy(), bucket=bucket) return { 'bucket': bucket, 'key': key, 'acknowledge_status': 'ok', } except lambda_exceptions.AssetUploadAlreadyHandled: return { 'acknowledge_status': 'asset_upload_already_handled', } except ( lambda_exceptions.StopProcessingException, lambda_exceptions.LoggedException, lambda_exceptions.UnexpectedEventBody ) as e: capture_exception(e) raise except Exception as e: capture_exception(e) lambda_exceptions.notify_and_raise( general.LAMBDA_NAME, general.ACKNOWLEDGE_ERROR_STATUS, errors.ACKNOWLEDGE_ERROR_CODE, key, bucket, { 'message': str(e)})