"""Lambda function module.""" import base64 from collections import defaultdict import json from aws_kinesis_agg import deaggregator from src.connectors import logging from src.constants import errors from src.constants import key_fields from src.constants import maxwell from src.logic import dms_delivery_spec from src.logic import jobs from src.logic import orders from src.utils import exceptions def process_records(records): """Process maxwell records. Args: records (dict): key (str) table of dict: key (str) operation of list of deaggregated maxwell records (dict) """ process_mapping = { maxwell.TABLE_ENCODING_QUEUE: orders.process_encoding_queue, maxwell.TABLE_DMS_DELIVERY_SPEC: dms_delivery_spec.process_dms_delivery_spec, maxwell.TABLE_ENCODING_QUEUE_DETAIL: jobs.process_details } for table, changes in records.items(): if changes: logging.logger.debug( 'Starting processing %s table changes: %s.', table, changes) process_mapping[table](changes) logging.logger.debug( 'Finished processing %s table changes.', table) def deaggregate_records(records): """Deaggregate all records. Args: records (list): kinesis records (aggregated maxwell records) Returns: dict: key (str) table of dict: key (str) operation of list of deaggregated maxwell records (dict) """ logging.logger.debug('Got %s Kinesis (aggregated) records.', len(records)) result = defaultdict(lambda: defaultdict(list)) count = 0 for raw_record in deaggregator.iter_deaggregate_records(records): kinesis_record = raw_record[maxwell.RECORD_KINESIS] record_bytes = kinesis_record[maxwell.KINESIS_DATA] record_str = base64.b64decode(record_bytes) record = json.loads(record_str.decode()) skip = False for key in [ maxwell.MAXWELL_TABLE, maxwell.MAXWELL_TYPE, maxwell.MAXWELL_DATA ]: if key not in record: logging.logger.error(errors.MISSING_ITEM, key) skip = True break if skip: continue table = record[maxwell.MAXWELL_TABLE] operation = record[maxwell.MAXWELL_TYPE] data = record[maxwell.MAXWELL_DATA] if not ( table in maxwell.PROCESSING_RECORDS and operation in maxwell.PROCESSING_RECORDS[table]): logging.logger.info( 'Unknown combination for table: %s and operation: %s.', table, operation) continue result[table][operation].append(data) logging.logger.debug( 'Table: %s, data: %s', table, {f: data[f] for f in key_fields.TABLE_KEY[table]}) count += 1 logging.logger.debug("Deaggregated %s Maxwell's Daemon records.", count) return result @exceptions.sentry_capture_exception def handler(event, context): """Lambda main function.""" if maxwell.EVENT_RECORDS not in event: logging.logger.error( errors.MISSING_ITEM, maxwell.EVENT_RECORDS) return records = deaggregate_records(event[maxwell.EVENT_RECORDS]) process_records(records)