"""Job to generate replacement dynamo items.""" import boto3 from boto3.dynamodb.conditions import Attr from boto3.dynamodb.conditions import Key from reports import config from reports.logging import app_logger from reports.report import Report class ReplacementGenerator: """Replacement generator.""" @staticmethod def run(period_report_group): """Run. Get a list of s3 paths relative to the prefix. i.e. s3://bucket/prefix/{path} Check if each s3 path has a matching dynamodb entry. If no item is in dynamo, generate a replacement item. """ app_logger.info( 'Running for ' f'{period_report_group.interval}ly ' f'{period_report_group.account_type}s ' f'In period(s) {period_report_group.period_ids}' ) reports = ReplacementGenerator.get_s3_listings(period_report_group) table = ReplacementGenerator.get_dynamo_table() total_record_count = len(reports) app_logger.info( f'Found {total_record_count} ' f'{period_report_group.interval}ly ' f'{period_report_group.account_type}s with sales' ) missing_report_count = 0 app_logger.info('Searching for missing reports...') for index, report in enumerate(reports): if not (index + 1) % config.PRINT_AFTER_MODULO: app_logger.debug(f'Searching... ({index + 1}/{total_record_count})') if ReplacementGenerator.dynamo_record_is_missing(report, table): missing_report_count += 1 if config.REPLACE_MISSING_ITEMS: result = table.put_item(Item=report.ddb_item) app_logger.debug(f'Replaced Item {report.ddb_item} {result}') app_logger.info( f'Found {missing_report_count} missing ' f'{period_report_group.interval}ly ' f'{period_report_group.account_type} reports\n' ) @staticmethod def get_s3_listings(period): """S3 dir parsing.""" page_iterator = ReplacementGenerator.get_period_reports(period) reports = [] for page in page_iterator: for item in page.get('Contents', []): if item.get('Key').endswith('/'): continue reports.append( Report( period, config.S3_REPORT_BUCKET, item.get('Key') ) ) return reports @staticmethod def get_period_reports(period): """Get a configured boto3 s3 paginator.""" paginator = ReplacementGenerator.get_s3_paginator() return paginator.paginate(**{ 'Bucket': config.S3_REPORT_BUCKET, 'Prefix': period.prefix }) @staticmethod def get_s3_paginator(): """Get boto3 s3 paginator.""" return boto3.client('s3').get_paginator('list_objects') @staticmethod def get_dynamo_table(): """Connect to the dynamo table.""" return boto3.resource( 'dynamodb', region_name=config.AWS_REGION ).Table(config.DYNAMO_TABLE) @staticmethod def dynamo_record_is_missing(report, table): """Check if the s3 path has a corresponding dynamo item.""" key_ex = ReplacementGenerator.get_key_condition_expression(report) filter_ex = Attr('file_type').eq(report.file_type) result = table.query( KeyConditionExpression=key_ex, FilterExpression=filter_ex ) return len(result.get('Items', [])) == 0 @staticmethod def get_key_condition_expression(report): """Get key condition expression.""" pk_filter = Key('user_id_type').eq(report.user_id_type) sk_filter = Key('user_params').eq(report.user_params) return pk_filter & sk_filter