"""Redis aggregator experiment.""" import json import sys import time import redis # establish redis connection CACHE = redis.Redis( host='localhost', port=6379, db=0 ) # event types and their aggregation delay in seconds EVENT_TYPES = { 'trending_track': 15, 'playlist_placement': 60 } def main(): """Parse CLI args and run command.""" action = sys.argv[1] if action == 'add_event': event_id = sys.argv[2] event_type = sys.argv[3] if event_type not in EVENT_TYPES.keys(): raise Exception('invalid event type {event_type}') profile_id = sys.argv[4] profile_type = sys.argv[5] add_event( event_id, event_type, profile_id, profile_type ) elif action == 'process_queue': process_queue() else: raise Exception(f'unrecognized command {action}') def _list_key(event_type, profile_id, profile_type): """Redis key identifier for buffer.""" return f'buffer:{event_type}:{profile_id}:{profile_type}' def _queue_key(event_type): """Redis key identifier for work queue.""" return f'queue:{event_type}' def add_event(event_id, event_type, profile_id, profile_type): """Add event as if it was received from webhook or other process.""" list_key = _list_key(event_type, profile_id, profile_type) event = { 'id': event_id, 'created_at': time.time() } # push on to head of buffer result = CACHE.lpush(list_key, json.dumps(event)) print(f'added to buffer {list_key} new len is {result}') # result is new length of buffer after push # need to be careful if process step fails and queue empties if result == 1: delay = EVENT_TYPES[event_type] queue_key = _queue_key(event_type) # add to ordered set that points to buffer with min time to process CACHE.zadd(queue_key, { list_key: time.time() + delay }) print(f'added to queue {queue_key} with delay {delay}') def process_queue(): """Read ordered sets and process work with timestamps less than now.""" for event_type in EVENT_TYPES.keys(): print(f'processing for event type of {event_type}') queue_key = _queue_key(event_type) while True: # get item in ordered set with smallest timestamp result = CACHE.zrange(queue_key, 0, 0, withscores=True) # if set is empty, no work to do if not result: print('no more items to process!') break # if item timestamp in future, no work to do min_process_time = result[0][1] if min_process_time > time.time(): print('no old enough items to process!') break # remove item from queue buffer_key = result[0][0] result = CACHE.zrem(queue_key, buffer_key) if result == 1: print(f'removed {buffer_key} from queue') # pull entire buffer data for item # alternatively, could use POP instead of LRANGE and DEL result = CACHE.lrange(buffer_key, 0, -1) events = [json.dumps(x.decode('utf-8')) for x in result] print(f'batch events to process {events}') # clear buffer (done before processing to prevent dupes) result = CACHE.delete(buffer_key) if result: print(f'cleared buffer at {buffer_key}') print('aggregate and process message here...') else: print('something went wrong clearing buffer {buffer_key}') else: print(f'something went wrong removing {buffer_key}') if __name__ == '__main__': main()