"""Lambda buffer-poller function module.""" import time import uuid import boto3 from boto3.dynamodb.types import TypeDeserializer from boto3.dynamodb.types import TypeSerializer from lambdacommon.common_config import logger from lambdacommon.util import init_sentry_for_lambda from config import AWS_REGION from config import BATCH_SIZE from config import DYNAMO_TABLE from config import SQS_URL init_sentry_for_lambda() dynamo_client = boto3.client('dynamodb', region_name=AWS_REGION) sqs_client = boto3.client('sqs', region_name=AWS_REGION) S = TypeSerializer() D = TypeDeserializer() def handler(event, context): """Lambda entry point.""" try: result = dynamo_client.scan( TableName=DYNAMO_TABLE, FilterExpression='process_after < :now', ExpressionAttributeValues={ ':now': S.serialize(int(time.time())) }, ConsistentRead=True ) for batch in batch_items(result['Items'], BATCH_SIZE): messages = [ { 'Id': str(uuid.uuid4()), 'MessageBody': D.deserialize(x['buffer_id']) } for x in batch ] sqs_client.send_message_batch( QueueUrl=SQS_URL, Entries=messages ) logger.info(f'Messages sent: {messages}') except Exception as e: logger.exception(str(e)) raise e def batch_items(items, batch_size): """Break a list into batches of specified size. Args: items (list): list of things to be batches batch_size (int): size of each chunk Returns: list: contains lists of all items bucketed into batches """ return [ items[x:x + batch_size] # bucket items by offsets for x in range(0, len(items), batch_size) # list offset indexes ]