"""Client amount snapshot lambda. By taking snapshots of the client amount aggregates in time, we can ensure the ETLs do not have bugs where the amount goes down. This checks and takes snapshots of UPCs in the client_amount table. If any UPCs have a lower sum than the last snapshot, a notification is sent. """ from datetime import datetime from decimal import Decimal import json import urllib import boto3 import pymysql import config import query import mysql _database_connection = None def get_database(): """Get the database connection, or create one if it does not exist. Returns: Connection: mysql connection object. """ global _database_connection if not _database_connection: _database_connection = mysql._connection( cursorclass=pymysql.cursors.DictCursor) return _database_connection def verify_event_source(event): """Verify the lambda event source is from the expected topic and ETL. The SNS topic that this lambda listens to is messaged by many ETLs. This function ensures the lambda continues only if the trigger is from a valid source. Args: event (dict): lambda triggering event. Returns: bool: continue flag. """ if config.IS_DEV: return True try: message = json.loads(event['Records'][0]['Sns']['Message']) except json.JSONDecodeError: return False correlation_id = message.get('correlation_id') source = message.get('source') return correlation_id and source == config.VALID_SOURCE def generate_snapshot_report(good_snapshots, bad_snapshots): """Generate a report of the latest snapshots. Args: good_snapshots (list): UPCs with no amount issues. bad_snapshots (list): UPCs with amount issues. Returns: str: plain text report of all snapshots. """ report_header = f'Snapshot Report: {datetime.now()}' if good_snapshots: good_report = f'Good snapshots found: {len(good_snapshots)}' else: good_report = 'No good snapshots found' if bad_snapshots: bad_list = '\n'.join( f"{s['upc']}, {s['amount_latest']}, {s['amount_previous']}" for s in bad_snapshots) bad_report = \ f'Bad snapshots found (upc, latest, previous):\n{bad_list}' else: bad_report = 'No bad snapshots found' report = '{report_header}\n\n{good_report}\n\n{bad_report}' return report.format( report_header=report_header, good_report=good_report, bad_report=bad_report) def notify_sns(report): """Send snapshot report to SNS topic. Args: report (str): plain text report of snapshots. """ client = boto3.client(service_name='sns', region_name='us-east-1') client.publish(Message=report, TopicArn=config.DATA_INTEGRITY_SNS_ARN) def notify_slack(report): """Send snapshot report to slack channel. Args: report (str): plain text report of snapshots. """ webhook_url = config.SLACK_WEBHOOK_URL if not webhook_url: return urllib.request.urlopen( webhook_url, data=json.dumps({'text': report}).encode('ascii')) def insert_new_snapshots(): """Insert new snapshots into the snapshot table from client_amount.""" timestamp = datetime.now() sql = query.INSERT_NEW_SNAPSHOT.format(timestamp=timestamp) conn = get_database() cursor = conn.cursor() cursor.execute(sql) conn.commit() def get_upcs(): """Get all UPCs in the snapshot table. Returns: list: UPCs found in the snapshot table. """ conn = get_database() cursor = conn.cursor() cursor.execute(query.GET_UPCS) return [row['upc'] for row in cursor.fetchall()] def get_snapshots(upcs): """Get snapshots for given UPCs. Args: upcs (list): UPCs to look up. Returns: list: snapshot data rows. """ snapshots = [] conn = get_database() cursor = conn.cursor() for upc in upcs: cursor.execute(query.GET_RECENT_UPC_AMOUNTS, {'upc': upc}) row = cursor.fetchone() snapshots.append(row) return snapshots def compare_snapshots(): """Lookup and compare all UPC snapshots. Returns: tuple: good and bad snapshots with upc and amounts. """ upcs = get_upcs() snapshots = get_snapshots(upcs) bad_snapshots = [] good_snapshots = [] for snapshot in snapshots: latest = snapshot['amount_latest'] # previous defaults to 0 value if db returns None (a new snapshot) previous = snapshot['amount_previous'] or Decimal(0) if latest >= previous: good_snapshots.append(snapshot) else: bad_snapshots.append(snapshot) return good_snapshots, bad_snapshots def main(event, context): """AWS Lambda entry point. Args: event (dict): triggering event. context (LambdaContext): lambda execution runtime. """ if not verify_event_source(event): return insert_new_snapshots() good_snapshots, bad_snapshots = compare_snapshots() report = generate_snapshot_report(good_snapshots, bad_snapshots) notify_sns(report) notify_slack(report) # Command line entry point to run logic if __name__ == '__main__': main(None, None)