"""Helper for DynamoDB requests.""" import concurrent.futures import time import boto3 from boto3.dynamodb.conditions import Attr, Key from ddtrace import tracer from flask import g from analytics import config HASH_KEY = "first_id" RANGE_KEY = "second_id" INDEX_NAME = "second_id-first_id" KEY_FIELD_MAP = { "product": "product_id", "label": "label_id", "subaccount": "subaccount_id", } # instantiate at module level to allow boto3 to reuse connections DYNAMO_CLIENT = boto3.resource("dynamodb", region_name="us-east-1") def create_product_key(product_id): """Create a key in the format label:product. Args: product_id (number): Product identifier Returns: key(str) """ return "product:{}".format(product_id) def create_track_key(isrc): """Create a key in the format isrc:ABC123. Args: isrc (src): ISRC Returns: key (str) """ return "isrc:{}".format(isrc) def parse_key(key): """Extract key values into a dictionary. Args: key (str): Hash or range key Returns: dictionary of key values """ parts = key.split(":") values = {} i = 0 while i < len(parts) - 1: key = KEY_FIELD_MAP.get(parts[i], parts[i]) values[key] = parts[i + 1] i += 2 return values @tracer.wrap(name="dynamo_get_table") def _get_table(): """Get table instance.""" # todo(jpenner): try instantating table at client level for performance return DYNAMO_CLIENT.Table(config.ANALYTICS_METADATA_TABLE) def _query_by_hash_key(hash_key, table, filter_expression): """Query DynamoDB for a set of matching given hash_key.""" if filter_expression: response = table.query( KeyConditionExpression=Key(HASH_KEY).eq(hash_key), FilterExpression=filter_expression, ) else: response = table.query( KeyConditionExpression=Key(HASH_KEY).eq(hash_key), ) return response.get("Items", []) @tracer.wrap(name="dynamo_query_by_hash_key") def query_by_hash_key(hash_key, permissions_filter=None): """Query DynamoDB for a set of matching given hash_key. Args: hash_key (str): Key to match permissions_filter (dict): dict containing resources users can access Returns: An array of matching items """ table = _get_table() return _query_by_hash_key( hash_key, table, _get_filter_expression(permissions_filter) ) def _query_by_range_key(range_key, table, filter_expression=None): """Query DynamoDB for a set of matching given range_key.""" start_time = time.time() if filter_expression: response = table.query( IndexName=INDEX_NAME, KeyConditionExpression=Key(RANGE_KEY).eq(range_key), FilterExpression=filter_expression, ) else: response = table.query( IndexName=INDEX_NAME, KeyConditionExpression=Key(RANGE_KEY).eq(range_key) ) end_time = time.time() duration = end_time - start_time if duration > config.SLOW_DYNAMO_TIME and response: message = ( "Slow dynamodb query with request-id: " "{request_id} took {duration} seconds" ) request_id = response.get("ResponseMetadata", {}).get("RequestId") message = message.format(request_id=request_id, duration=duration) g.ows.log.info(message) return response.get("Items", []) @tracer.wrap(name="dynamo_query_by_range_key") def query_by_range_key(range_key, permissions_filter=None): """Query DynamoDB for a set of matching given range_key. Args: range_key (str): Key to match permissions_filter (dict): dict containing resources users can access Returns: An array of matching items """ table = _get_table() return _query_by_range_key( range_key, table, _get_filter_expression(permissions_filter) ) @tracer.wrap(name="dynamo_query_by_keys") def query_by_keys(keys, key_type="range", permissions_filter=None): """Query DynamoDB for a set of matching given list of keys. Args: keys (list): Keys to match key_type (str): 'range' or 'hash' permissions_filter (dict): dict containing resources users can access Returns: An array of matching items """ if not keys: return None table = _get_table() func = _query_by_range_key if key_type == "range" else _query_by_hash_key with concurrent.futures.ThreadPoolExecutor(max_workers=len(keys)) as executor: futures = { executor.submit( func, key, table, _get_filter_expression(permissions_filter) ): key for key in keys } concurrent.futures.wait(futures) results = list(map(lambda f: {futures[f]: f.result()}, futures)) return results def _get_filter_expression(permissions_filter): """Get a filter expression for dynamo for a permissions filter. Args: permissions_filter (dict): dict containing resources users can access Returns: A filter expression """ if not permissions_filter: return None if permissions_filter["artist_ids"]: return Attr("artist_id").is_in(permissions_filter["artist_ids"]) elif permissions_filter["subaccount_ids"]: return Attr("product.subaccount_id").is_in( permissions_filter["subaccount_ids"] ) | Attr("subaccount_id").is_in(permissions_filter["subaccount_ids"]) elif permissions_filter["label_ids"]: return Attr("label_id").is_in(permissions_filter["label_ids"])