"""Lambda dd_snowflake_sync function module.""" import base64 from collections import defaultdict import json from aws_kinesis_agg import deaggregator from sentry_sdk import capture_exception from config import logger from src.constants import errors from src.constants import maxwell from src.logic.maxwell_record import log_records def deaggregate_records(records): """Deaggregate all records. Args: records (list): kinesis records (aggregated maxwell records) Returns: dict: key (str) table of dict: key (str) operation of list of deaggregated maxwell records (dict) """ result = defaultdict(list) count = 0 for raw_record in deaggregator.iter_deaggregate_records(records): kinesis_record = raw_record[maxwell.RECORD_KINESIS] record_bytes = kinesis_record[maxwell.KINESIS_DATA] record_str = base64.b64decode(record_bytes).decode('utf-8') record = json.loads(record_str) logger.debug(f'json data from kinesis {record}') skip = False for key in [maxwell.MAXWELL_TABLE, maxwell.MAXWELL_TYPE, maxwell.MAXWELL_DATA]: if key not in record: logger.error(errors.MISSING_ITEM, key) skip = True break if skip: continue table = record[maxwell.MAXWELL_TABLE] operation = record[maxwell.MAXWELL_TYPE] if not (table in maxwell.TABLES_TO_LOG and operation in maxwell.ALLOWED_OPERATIONS): logger.warning(errors.SKIPPING_RECORD, table, operation) continue result[table].append(record) count += 1 logger.debug(f"Deaggregated {count} Maxwell's Daemon records.") return result def handler(event, context): """Lambda entry point.""" try: if maxwell.EVENT_RECORDS not in event: logger.error( errors.MISSING_ITEM, maxwell.EVENT_RECORDS) return logger.debug(event) records = deaggregate_records(event[maxwell.EVENT_RECORDS]) log_records(records) return {'status': 'OK'} except Exception as e: capture_exception(e) logger.exception(str(e)) raise e