""" Aggregation Metric Lambda ========================= The lambda is listening to s3 events (all create events), find the file, go through it and ship progressively to DynamoDB the different values. The process is very simple and it does not do data validation and transformation. Important: this is python 2.7, since Lambda does not support python 3. """ from __future__ import print_function import json import urllib import boto3 # Constants TABLE_NAME = 'prod_aggregate_analytics' s3 = boto3.client('s3') dynamodb = boto3.client('dynamodb', 'us-east-1') def lambda_handler(event, context): """Handle the lambda event. Args: event (dict): information about the event that needs to be treated. context (dict): additional contextual information. Returns: str: s3 object's content type. """ # This code is borrowed directly from the lambda generator. record = event['Records'][0]['s3'] bucket = record['bucket']['name'] key = urllib.unquote_plus(record['object']['key']).decode('utf8') try: return process_file(bucket, key) except Exception as e: # TODO(mortali): add sentry to capture events. print(e) raise e def process_file(bucket, key): """Processing the file. The system will go line by line, loads the data from the json, process each line and when it has completed, it will remove the file it has consumed. Args: bucket (str): the name of the bucket. key (str): the key in the bucket. Returns: str: s3 object's content type. """ s3_object = s3.get_object(Bucket=bucket, Key=key) content = s3_object['Body'].read() content = content.decode('utf8') lines = content.split('\n') for line in lines: if not line: continue data = json.loads(line) process_data(data) s3.delete_object(Bucket=bucket, Key=key) return s3_object['ContentType'] def process_data(data): """Process the data. The data in s3 contains the metric, the month, label id, subaccount id and for each day the paid / all values. Processing the data just means sending the same line in a format that dynamodb understands and sending it. Args: data (dict): the data to process. """ # Quick wins - just make the code a bit more readable. String = lambda val: {'S': str(val)} Put = lambda val: {'Value': val, 'Action': 'PUT'} # The table key contains the metric and the month. If one is missing an # exception is thrown and the lambda will stop working. table_key = dict( metric=String(data.pop('metric')), month=String(data.pop('month'))) attributes = dict() for field in ('label_id', 'subaccount_id'): field_value = data.pop(field, None) if field_value: attributes.update({ field: Put(String(field_value)) }) for day, day_values in data.viewitems(): attributes.update({ day: Put({ 'M': { 'all': String(day_values.get('all', '0')), 'paid': String(day_values.get('paid', '0')) } }) }) dynamodb.update_item( TableName=TABLE_NAME, Key=table_key, AttributeUpdates=attributes)