""" Update Status Lambda ==================== The lambda is listening to s3 events (all create events), find the file, parse the s3 key and update the particular dynamodb item with the parsed information. Important: this is python 2.7, since Lambda does not support python 3. """ from __future__ import print_function import collections import boto3 dynamodb = boto3.client('dynamodb') TABLE_NAME = 'prod_accounting_statement_export' KEY_FORMAT = '{period_ids}__{trans_types}__AVRO__en_US' Key = collections.namedtuple( 'Key', ['user_id_type', 'user_params', 'period_ids']) def _get_bucket_key(event): """Get bucket and key from the S3 put event. Args: event (dict): event dictionary. Returns: tuple(str, str): bucket and key tuple. """ # Get the object from the event and show its content type bucket = event['Records'][0]['s3']['bucket']['name'] key = event['Records'][0]['s3']['object']['key'].replace( '%3D', '=').replace('%2C', ',') return bucket, key def _parse_s3_object_key(key): """Parse key and structure it in a namedtuple. Args: key (str): object key path's parts. Example: "schematized_files/199_label_all_all_month/ user_id_type=18805L/file.avro". Returns: namedtuple: namedtuple with properties: period_ids, user_id_type and user_params. """ key_parts = key.split('/') vfolder_parts = key_parts[1].split('_') user_id_type = key_parts[2].replace('user_id_type=', '') user_params = KEY_FORMAT.format( period_ids=vfolder_parts[0], trans_types=vfolder_parts[3]) period_ids = vfolder_parts[0] return Key( user_id_type=user_id_type, user_params=user_params, period_ids=period_ids) def lambda_handler(event, context): """Lambda handler for updating status Args: event (dict): S3 put event information. context (dict): statistic information. """ bucket, key = _get_bucket_key(event) file_metadata = _parse_s3_object_key(key) s3_path = 's3://{}/{}'.format(bucket, key) dynamodb = boto3.client('dynamodb') try: # Update dynamodb items dynamodb.put_item( TableName=TABLE_NAME, Item={ 'user_id_type': { 'S': file_metadata.user_id_type }, 'user_params': { 'S': file_metadata.user_params }, 'file_type': { 'S': 'AVRO' }, 'period_ids': { 'S': file_metadata.period_ids }, 's3_path': { 'S': s3_path }, 'status': { 'S': 'GENERATED' } } ) except Exception as e: print( 'Error saving status for key: {} bucket: {}.'.format(key, bucket)) raise e