"""A script to back fill missing metadata for certain custom exports.""" import argparse import json from pprint import pprint import boto3 from boto3.dynamodb import conditions def get_session(profile_name='default'): """Get AWS session using profile name credentials. Args: profile_name (str): configured AWS profile name Returns: boto3.session.Session: AWS session """ session = boto3.session.Session(profile_name=profile_name) return session def get_dynamodb(session=None): """Get DynamoDB resource. Args: session (boto3.session.Session): AWS session Returns: boto3.resources.factory.dynamodb.ServiceResource: DynamoDB resource """ if session is None: session = get_session() return session.resource('dynamodb') def get_table(session, table_name): """Get DynamoDB table using session. Args: session (boto3.session.Session): AWS session table_name (str): DynamoDB table name Returns: boto3.resources.factory.dynamodb.Table: DynamoDB table """ dynamodb = get_dynamodb(session) table = dynamodb.Table(table_name) return table def update_item(key, data, table): """Update a vendor item processing status in the table. Args: key (dict): DynamoDB item key data (dict): missing metadata table (boto3.resources.factory.dynamodb.Table): DynamoDB table Returns: dict: DynamoDB response dict """ user_params_condition = conditions.Key( 'user_params').eq(key['user_params']) user_id_type_condition = conditions.Key( 'user_id_type').eq(key['user_id_type']) condition = conditions.And(user_params_condition, user_id_type_condition) expression = ( 'SET requested_datetime = :r_datetime, requested_by = :r_by') expr_attributes = { ':r_datetime': data['requested_datetime'], ':r_by': data['requested_by'] } try: resp = table.update_item( Key=key, ConditionExpression=condition, ReturnValues='NONE', UpdateExpression=expression, ExpressionAttributeValues=expr_attributes, ReturnConsumedCapacity='TOTAL', ) except table.meta.client.exceptions.ConditionalCheckFailedException as e: resp = {'ResponseMetadata': {'HTTPStatusCode': 404}, 'error': str(e)} return resp def load_data(file_path): """Read data from json file. Args: file_path (str): file path Returns: dict: loaded json """ with open(file_path, 'r') as data_file: return json.load(data_file) def update_records(missing_records, table): """Update DynamoDB records using DynamoDB table. Args: missing_records (dict): backfill data, DynamoDB keys and values table (boto3.resources.factory.dynamodb.Table): DynamoDB table Returns: list: list of errors (dict) """ errors = [] for data in missing_records: key = data['key'] response = update_item(key, data, table) if response['ResponseMetadata']['HTTPStatusCode'] != 200: errors.append({'data': data, 'response': response}) return errors def main(profile_name, file_path, table_name): """Script entry point. Args: profile_name (str): AWS shared credentials profile name file_path (str): path to json file table_name (str): DynamoDB table name. Should be accessible within profile access rights """ session = get_session(profile_name) print('Reading file.',) data = load_data(file_path) row_count = len(data) print('Got {} row(s)'.format(row_count)) table = get_table(session, table_name) print('Updating records') errors = update_records(data, table) errors_count = len(errors) print('{} of {} record(s) updated. {} errors.'.format( row_count - errors_count, row_count, errors_count)) if not errors: return print('{:=^79}'.format(' Error details ')) for error in errors: pprint(error.get('data')) pprint(error.get('response')) print('-' * 79) if __name__ == '__main__': parser = argparse.ArgumentParser( description=( 'Back-fill missing metadata for custom export DynamoDB records.')) parser.add_argument( '-f', '--file-path', required=True, help='Path to json file with data') parser.add_argument( '-p', '--profile-name', help='Shared AWS credentials profile name.', default='default') parser.add_argument( '-t', '--table-name', help='DynamoDB table name.') args = parser.parse_args() profile_name = args.profile_name file_path = args.file_path table_name = args.table_name main(profile_name, file_path, table_name)