"""Dynamo aggregator experiment.""" import sys import time import uuid import boto3 from boto3.dynamodb.types import TypeSerializer from boto3.dynamodb.types import TypeDeserializer S = TypeSerializer() D = TypeDeserializer() DYNAMO_TABLE = 'dev_lambda_notifications_event_buffer' CLIENT = boto3.client('dynamodb') # event types and their aggregation delay in seconds EVENT_TYPES = { 'trending_track': 15, 'playlist_placement': 60 } def main(): """Parse CLI args and run command.""" action = sys.argv[1] if action == 'add_event': event_id = sys.argv[2] event_type = sys.argv[3] if event_type not in EVENT_TYPES.keys(): raise Exception('invalid event type {event_type}') profile_id = sys.argv[4] profile_type = sys.argv[5] add_event( event_id, event_type, profile_id, profile_type ) elif action == 'process_queue': process_queue() else: raise Exception(f'unrecognized command {action}') def _list_key(event_type, profile_id, profile_type): """Dynmao primary key identifier for buffer.""" return S.serialize(f'{event_type}:{profile_id}:{profile_type}') def add_event(event_id, event_type, profile_id, profile_type): """Add event as if it was received from webhook or other process.""" # append event to profile-event buffer result = CLIENT.update_item( TableName=DYNAMO_TABLE, Key={ 'buffer_id': _list_key(event_type, profile_id, profile_type) }, AttributeUpdates={ 'events': { 'Value': S.serialize( [ { 'event_id': event_id, 'uuid': str(uuid.uuid4()) } ] ), 'Action': 'ADD' } }, ReturnValues='ALL_OLD' ) # if first element in buffer, mark timestamp for future processing if 'Attributes' not in result: delay = EVENT_TYPES[event_type] CLIENT.update_item( TableName=DYNAMO_TABLE, Key={ 'buffer_id': _list_key(event_type, profile_id, profile_type) }, AttributeUpdates={ 'process_after': { 'Value': S.serialize(int(time.time() + delay)), 'Action': 'PUT' }, } ) def process_queue(): """Read ordered sets and process work with timestamps less than now.""" # scan table for buffers ready for processing result = CLIENT.scan( TableName=DYNAMO_TABLE, FilterExpression='process_after < :now', ExpressionAttributeValues={ ':now': S.serialize(int(time.time())) }, ConsistentRead=True ) # delete buffer and read into memory to ensure once-and-only-once process for item in result['Items']: batch = CLIENT.delete_item( TableName=DYNAMO_TABLE, Key={ 'buffer_id': item['buffer_id'] }, ReturnValues='ALL_OLD' ) events = D.deserialize(batch['Attributes']['events']) print(events) print(type(events)) if __name__ == '__main__': main()