"""DynamoDB connector.""" import decimal import os import boto3 from boto3.dynamodb.conditions import Attr from boto3.dynamodb.conditions import Key from boto3.dynamodb.types import DYNAMODB_CONTEXT DYNAMODB_TABLE = os.getenv('ACCOUNTING_STATEMENT_EXPORT_TABLE') def _get_expressions(status, **kwargs): """Get update_expression, attributes and names forr update_item method. Args: status (str): status to be updated kwargs (dict): optional additional attributes to be updated Returns: tuple: tuple with first item is update_expression, second is expression_attribute_names and third expression_attribute_values Example: ( 'SET #status = :status,#s3_path = :s3_path', {'#s3_path': 's3_path', '#status': 'status'}, {':s3_path': {'S': 's3://test_bucket/test_prefix'}, ':status': {'S': 'generating'}}) """ update_expression = ('SET #status = :status') expression_attribute_names = { '#status': 'status' } expression_attribute_values = { ':status': {'S': status} } if len(kwargs) > 0: update_express_arr = [] for arg, argv in kwargs.items(): if arg not in ['user_id_type', 'user_params', 'status'] and argv: update_express_arr.append('#{} = :{}'.format(arg, arg)) expression_attribute_names['#{}'.format(arg)] = arg type_str = 'S' if type(argv) is list: type_str = 'SS' expression_attribute_values[':{}'.format(arg)] = { type_str: argv } update_expression = '{},{}'.format( update_expression, ', '.join(update_express_arr)) return ( update_expression, expression_attribute_names, expression_attribute_values) def set_status(user_id_type, user_params, status, **kwargs): """Set status of the ingestion job. Args: user_id_type (str): user_id_type partition user_params (str): user_params status (str): status code constant """ key = { 'user_id_type': {'S': user_id_type}, 'user_params': {'S': user_params} } expressions = _get_expressions(status, **kwargs) resource = boto3.resource('dynamodb') table = resource.Table(DYNAMODB_TABLE) table.update_item( Key=key, UpdateExpression=expressions[0], ExpressionAttributeNames=expressions[1], ExpressionAttributeValues=expressions[2] ) def get_items(user_id_type, **filters): """Get dynamodb item for user_id_type and filters. Args: user_id_type (str): user_id_type partition kwargs (dict): optional filters Returns: mixed (False | iterable ResultSets): False if items not found. items if items are found. """ key_expression = Key('user_id_type').eq(user_id_type) filter_expressions = None for attribute, value in filters.items(): if not filter_expressions: filter_expressions = Attr(attribute).eq(value) filter_expressions = filter_expressions & Attr(attribute).eq(value) resource = boto3.resource('dynamodb') table = resource.Table(DYNAMODB_TABLE) results = table.query( KeyConditionExpression=key_expression, FilterExpression=filter_expressions ) if 'Items' in results: return results['Items'] return False def scan_table_by_attribute(**filters): """Scan table for items with specified attributes. Example call: dynamodb.scan_table_by_attribute( user_id_type='18805L',status='pending') @todo(pkuong): update so that if user_id_type is passed in, use query instead of scan. Args: filters (dict): attribute and value pairs. Returns: list: list of dictionaries, each dictionary contains all information in the item. Example: [{'transaction_types': 'DA,DT', 'lines_scanned': '93136', 'generation_total_time': '300.0411696434021', 'download_avro_file_end': '2016-04-29 22:54:09', 'download_avro_file_start': '2016-04-29 22:53:41', 'number_format': 'es_ES', 'number_of_partitions': '1', 'file_type': 'txt', 'user_params': '199__DA,DT__es_ES__txt', 'generation_end': '2016-04-29 22:58:41', 'generation_start': '2016-04-29 22:53:41'}] """ filter_expressions = None for attribute, value in filters.items(): if not filter_expressions: filter_expressions = Attr(attribute).eq(value) filter_expressions = filter_expressions & Attr(attribute).eq(value) resource = boto3.resource('dynamodb') table = resource.Table(DYNAMODB_TABLE) results = table.scan(FilterExpression=filter_expressions) if 'Items' in results: return results['Items'] return False def delete_item_object(item): """Delete a DynamoDB item. Args: item (dict): DynamoDB item """ resource = boto3.resource('dynamodb') table = resource.Table(DYNAMODB_TABLE) table.delete_item( Key={'user_id_type': item.get('user_id_type')} ) def get_status_using_session(key, table_name, session): """Get DynamoDB item status using session. Args: key (dict): DynamoDB key (user_id_type (str), user_params (str)) table_name (str): DynamoDB table name session (boto3.session.Session): AWS session Returns: str: item status """ dynamodb = session.resource('dynamodb') table = dynamodb.Table(table_name) item = table.get_item(Key=key) return item.get('Item', {}).get('status', '') def put_item( partition_key, partition_key_value, sort_key=None, sort_key_value=None, **data): """Insert an item. Args: table (dynamodb.Table): Table to insert into. partition_key (str): Partition key name. partition_key_value (str): Partition key value. sort_key (str?): Sort key name. sort_key_value (str?): Sort key value. data (**Any): Content to add to the item. """ data[partition_key] = partition_key_value if sort_key and sort_key_value: data[sort_key] = sort_key_value with decimal.localcontext(DYNAMODB_CONTEXT) as ctx: ctx.traps[decimal.Inexact] = False ctx.traps[decimal.Rounded] = False for key, value in data.items(): if isinstance(value, float): data[key] = ctx.create_decimal_from_float(value) resource = boto3.resource('dynamodb') table = resource.Table(DYNAMODB_TABLE) table.put_item(Item=data)