"""Lambda for saving DynamoDB items to SnowFlake.""" from __future__ import print_function import base64 import json from aws_kinesis_agg.deaggregator import iter_deaggregate_records import const # noqa import field_const # noqa from connectors import elasticache # noqa from connectors import snowflakedb # noqa from models import snowflake as snowflake_model # noqa redis_conn = elasticache.get_redis() snowflake_context = snowflakedb.get_snowflake_context() def lambda_handler(event, context): """Consumer Lambda function handler. Args: event (dict): Lambda event information context (dict): Lambda event context Returns: str: result message """ raw_kinesis_records = event['Records'] deaggregated_records = _deaggregate_records(raw_kinesis_records) job_id = _get_job_id(deaggregated_records) print('Job id: {}'.format(job_id)) records_count = len(deaggregated_records) print('Deaggregated {} records'.format(records_count)) snowflake_model.update_snowflake( deaggregated_records, 'INSERT', snowflake_context) update_redis_counter(job_id, records_count) finish_message = 'Successfully processed {} records.'.format(records_count) print(finish_message) return finish_message def update_redis_counter(job_id, records_processed): """Update records counter in Redis. Args: job_id (str): import job id. records_processed (int): number of records processed. """ if not job_id: return try: items_left = redis_conn.hincrby( const.KINESIS_COUNTER, job_id, records_processed) print('{} items processed'.format(items_left)) except Exception as ex: print(ex) def _deaggregate_records(raw_kinesis_records): """Deaggregate raw kinesis records. Kinesis data in Python Lambdas is base64 encoded. Args: raw_kinesis_records (dict): event records Returns: deaggregated_records (list): list of deaggregated records """ deaggregated_records = [] for record in iter_deaggregate_records(raw_kinesis_records): payload = base64.b64decode(record['kinesis']['data']) json_payload = json.loads(payload) deaggregated_records.append(json_payload) return deaggregated_records def _get_job_id(deaggregated_records): """Get job_id from one of deaggregated records. It is expected to be the same for all records in given event. Supposed to be used for records of single event. Args: deaggregated_records (list): event records Returns: job_id (str): job id of any record in event """ return deaggregated_records[0].get(field_const.JOB_ID, '')