"""Job to generate replacement dynamo items.""" import boto3 from boto3.dynamodb.conditions import Attr from boto3.dynamodb.conditions import Key from reports import config from reports.report import Report class ReplacementGenerator: """Replacement generator.""" @staticmethod def run(period): """Run. Get a list of s3 paths relative to the prefix. i.e. s3://bucket/prefix/{path} Check if each s3 path has a matching dynamodb entry. If no item is in dynamo, generate a replacement item. """ reports = ReplacementGenerator.get_s3_listings(period) table = ReplacementGenerator.get_dynamo_table() print('Found {num} {interval}ly {account_type}s'.format( # noqa T001 num=len(reports), interval=period.interval, account_type=period.account_type)) for report in reports: if ReplacementGenerator.dynamo_record_is_missing(report, table): # TODO: stash found items print('found missing item', report.ddb_item) # noqa T001 # TODO: batch save replacement items to dynamoDB @staticmethod def get_s3_listings(period): """S3 dir parsing.""" page_iterator = ReplacementGenerator.get_period_reports(period) reports = [] for page in page_iterator: for item in page.get('Contents', []): reports.append( Report(period, config.S3_REPORT_BUCKET, item.get('Key'))) return reports @staticmethod def get_period_reports(period): """Get a configured boto3 s3 paginator.""" paginator = ReplacementGenerator.get_s3_paginator() opts = {'Bucket': config.S3_REPORT_BUCKET, 'Prefix': period.prefix} return paginator.paginate(**opts) @staticmethod def get_s3_paginator(): """Get boto3 s3 paginator.""" return boto3.client('s3').get_paginator('list_objects') @staticmethod def get_dynamo_table(): """Connect to the dynamo table.""" return boto3.resource('dynamodb', region_name=config.AWS_REGION).Table( config.DYNAMO_TABLE) @staticmethod def dynamo_record_is_missing(report, table): """Check if the s3 path has a corresponding dynamo item.""" key_ex = ReplacementGenerator.get_key_condition_expression(report) filter_ex = Attr('file_type').eq(report.file_type) result = table.query( KeyConditionExpression=key_ex, FilterExpression=filter_ex) return len(result.get('Items', [])) == 0 @staticmethod def get_key_condition_expression(report): """Get key condition expression.""" pk_filter = Key('user_id_type').eq(report.user_id_type) sk_filter = Key('user_params').eq(report.user_params) return pk_filter & sk_filter