"""Analytics digest utils.""" from datetime import datetime import json from multiprocessing.pool import ThreadPool from snowflake_connector.snowflake_conn import get_session from snowflake_connector.snowflake_conn import SQLLoader from activity_detector import base_config from activity_detector.flows.analytics_digest import config from activity_detector.utils import ows_notifications sql_loader = SQLLoader(config.QUERY_PATH) def generate_digest( account_type, start_date, end_date, account_ids, last_dates_per_store): """Generate the digest and create the notification for each account. Args: account_type (str): the account type start_date (str): the start date end_date (str): the end date account_ids (str): comma-separated list of account ids: 123,456,789. last_dates_per_store ([dict]): the list of last date per store Returns: None """ with get_session() as session: accounts = fetch_accounts(account_type, session, account_ids) exec_list = [] for account in accounts: account_id = account[0] exec_list.append( { 'function_to_call': generate_label_digest, 'params': [ account_id, start_date, end_date, last_dates_per_store, account_type, session ] } ) with ThreadPool(8) as pool: results = pool.map(exec_func, exec_list) for processed_result in results: if not processed_result: return processed_result def exec_func(exec_item): """Make a request as part of a multi-thread series of requests. Args: exec_item (dict): containing a function and parameters. Returns: Result of calling the given function with the given parameters. """ return exec_item.get('function_to_call')(*exec_item.get('params')) def generate_label_digest( account_id, start_date, end_date, last_dates_per_store, account_type, session): """Generate the digest and create the notification for an account. Args: account_type (str): the account type start_date (str): the start date end_date (str): the end date account_ids (str): comma-separated list of account ids: 123,456,789. last_dates_per_store ([dict]): the list of last date per store Returns: None """ with get_session() as session: digest = {} account_name = fetch_account_name(session, account_id) if account_name: digest[account_id] = { 'actor': 'The Orchard Activity Detector', 'verb': 'Generated', 'object': 'Digest', 'start_date': start_date, 'end_date': end_date, 'account_name': account_name, 'top_releases': get_top_releases( account_type, account_id, session), 'top_new_releases': get_top_releases( account_type, account_id, session, is_new_releases=True), 'top_tracks': get_top_tracks( account_type, account_id, session), 'top_physical_shipments': get_top_physical_shipments( account_type, account_id, session, start_date, end_date) } missing_data_per_store = check_missing_data_per_store( last_dates_per_store, end_date) if missing_data_per_store: digest[account_id]['notes'] = missing_data_per_store create_notification(account_type, digest) def fetch_account_name(session, account_id): """Fetch account name based on account_id. Args: session (snowflake.session): the snowflake session account_id (int): the account id Returns: str: the account name """ sql = sql_loader.load_query('fetch_account_name') sql = sql.replace('{env}', base_config.ENVIRONMENT) sql = sql.replace('{account_id}', str(account_id)) result = session.execute(sql) label_name = result.cursor.fetchone() if not label_name: return '' if label_name[0]: return label_name[0] def check_missing_data_per_store(last_dates_per_store, end_date): """Check if missing data based on store. Args: last_dates_per_store ([dict]): the list of last date per store end_date (str): the top ten end date Returns: str: the stores with missing data """ if not last_dates_per_store: return '' end_date = datetime.strptime(end_date, config.DATE_FORMAT) missing_data_store = [] for date_per_store in last_dates_per_store: store_name = date_per_store['store_name'] if 'last_date' not in date_per_store: missing_data_store.append(store_name) else: last_date = datetime.strptime( date_per_store['last_date'], config.DATE_FORMAT) if last_date < end_date: missing_data_store.append(store_name) if not missing_data_store: return '' return ', '.join(missing_data_store) def fetch_accounts(account_type, session, account_ids): """Fetch all labels or subaccounts from the top_releases table. Args: account_type (str): the account type session (snowflake.session): the snowflake session account_ids (str): comma-separated list of account ids: 123,456,789. Returns: list: a list of tuples containing the label or subaccount IDs """ if account_ids: account_ids_lst = account_ids.split(',') if account_ids else [] account_ids_lst = list(map(int, account_ids_lst)) return [(i,) for i in account_ids_lst] else: query = 'fetch_{0}s'.format(account_type) sql = sql_loader.load_query(query) sql = sql.replace('{env}', base_config.ENVIRONMENT) result = session.execute(sql) return result.cursor.fetchall() def get_top_releases(account_type, account_id, session, is_new_releases=False): """Get the sorted list of top releases for a label or a subaccount. Args: account_type (str): the account type account_id (int): the account ID session (snowflake.session): the snowflake session is_new_releases (bool): If new releases Returns: list: the sorted list of top releases """ return sort_top_items_streams( map_top_items_to_list( get_top_releases_dict( account_type, account_id, session, is_new_releases ) ) )[:config.NUMBER_OF_RELEASES] def get_top_tracks(account_type, account_id, session): """Get the sorted list of top tracks for a label or a subaccount. Args: account_type (str): the account type account_id (int): the account ID session (snowflake.session): the snowflake session Returns: list: the sorted list of top tracks """ return sort_top_items_streams( map_top_items_to_list(get_top_tracks_dict(account_type, account_id, session)) )[:config.NUMBER_OF_TRACKS] def get_top_physical_shipments( account_type, account_id, session, start_date, end_date): """Get the sorted list of top physical shipments for a label or a subaccount. Args: account_type (str): the account type account_id (int): the account ID session (snowflake.session): the snowflake session start_date (str): the start date end_date (str): the end date Returns: list: the sorted list of top physical shipments """ return sort_top_items_transactions( map_top_items_to_list( get_top_physical_shipments_dict( account_type, account_id, session, start_date, end_date ) ) ) def get_top_releases_dict(account_type, account_id, session, is_new_releases): """Get a dict containing the top releases for a label or a subaccount. Args: account_type (str): the account type account_id (int): the account ID session (snowflake.session): the snowflake session is_new_releases (bool): If new releases Returns: dict: the dict containing the top releases keyed by release_id """ top_releases = {} for item in fetch_top_items( 'release', account_type, account_id, session, is_new_releases): if is_new_releases: (release_id, release_name, release_format, artist_name, store_id, streams_per_store, total_streams) = item growth_percentage_per_store = None else: (release_id, release_name, release_format, artist_name, store_id, streams_per_store, total_streams, growth_percentage, growth_percentage_per_store) = item if release_id not in top_releases: top_releases[release_id] = { 'release_id': release_id, 'release_name': release_name, 'release_format': release_format, 'artist_name': artist_name, 'total_streams': int(total_streams) } if not is_new_releases: top_releases[release_id]['growth_percentage'] = \ round(growth_percentage) if growth_percentage else None growth_percentage_per_store = \ round(growth_percentage_per_store) if \ growth_percentage_per_store else None update_item_with_store_percentage( top_releases[release_id], store_id, streams_per_store, total_streams, growth_percentage_per_store) return top_releases def get_top_tracks_dict(account_type, account_id, session): """Get a dict containing the top tracks for a label or a subaccount. Args: account_type (str): the account type account_id (int): the account ID session (snowflake.session): the snowflake session Returns: dict: the dict containing the top tracks keyed by isrc """ top_tracks = {} for item in fetch_top_items('track', account_type, account_id, session): (isrc, track_name, artist_names, store_id, streams_per_store, total_streams, growth_percentage, growth_percentage_per_store) = item if isrc not in top_tracks: artist_names = json.loads(artist_names) top_tracks[isrc] = { 'isrc': isrc, 'track_name': track_name, 'artist_names': artist_names, 'total_streams': int(total_streams), 'growth_percentage': round(growth_percentage) if growth_percentage else None } growth_percentage_per_store = round(growth_percentage_per_store) \ if growth_percentage_per_store else None update_item_with_store_percentage( top_tracks[isrc], store_id, streams_per_store, total_streams, growth_percentage_per_store) return top_tracks def get_top_physical_shipments_dict( account_type, account_id, session, start_date, end_date): """Get a dict containing the top physical shipments for a label or a subaccount. Args: account_type (str): the account type account_id (int): the account ID session (snowflake.session): the snowflake session start_date (str): the start date end_date (str): the end date Returns: dict: the dict containing the top physical shipment keyed by raas_no """ top_physical = {} for item in fetch_top_items( 'physical_shipment', account_type, account_id, session, False, start_date, end_date): (raas_no, product_name, product_type, artist_name, vendor_id, total_transactions) = item if raas_no not in top_physical: top_physical[raas_no] = { 'raas_no': raas_no, 'product_name': product_name, 'product_type': product_type, 'artist_name': artist_name, 'total_transactions': int(total_transactions) if total_transactions else 0 } return top_physical def fetch_top_items( item_type, account_type, account_id, session, is_new_releases=False, start_date=None, end_date=None): """Fetch the top items for an account. Args: item_type (str): the item type account_type (str): the account type account_id (int): the account ID session (snowflake.session): the snowflake session is_new_releases (bool): If new releases start_date (str): the start date end_date (str): the end date Returns: list: a list of tuples containing the top releases """ if item_type == 'release' and is_new_releases: item_type = 'new_release' query = 'fetch_{0}_top_{1}s'.format(account_type, item_type) params = {'{0}_id'.format(account_type): account_id} sql = sql_loader.load_query(query) sql = sql.replace('{env}', base_config.ENVIRONMENT) if item_type == 'physical_shipment': params['start_date'] = start_date params['end_date'] = end_date result = session.execute(sql, params) return result.cursor.fetchall() def update_item_with_store_percentage( item, store_id, streams_per_store, total_streams, growth_percentage_per_store): """Update a release or track item with percentage of streams for a store. Args: item (dict): the release or track item store_id (int): the store ID streams_per_store (int): the number of streams for the store total_streams (int): the number of streams accross all stores Returns: None """ if 'store_percentages' not in item: item['store_percentages'] = {} if store_id in config.STORES: item['store_percentages'][config.STORES[store_id]] = \ calculate_store_percentage( streams_per_store, total_streams, growth_percentage_per_store) else: if 'other' not in item['store_percentages']: item['store_percentages']['other'] = 0 item['store_percentages']['other'] += streams_per_store def calculate_store_percentage( streams_per_store, total_streams, growth_percentage_per_store=None): """Calculate the percentage of streams for a store. Args: streams_per_store (int): the number of streams for the store total_streams (int): the number of streams accross all stores Returns: str: the percentage as a string """ percentage = int(round(streams_per_store * 100 / total_streams)) return { 'percentage': '{0}%'.format(percentage), 'streams_per_store': int(streams_per_store), 'growth_percentage_per_store': int(growth_percentage_per_store) if growth_percentage_per_store else None } def map_top_items_to_list(top_items_dict): """Map a dict containing the top items to a list of cleaned up items. Args: top_releases_dict (dict): the dict containing the top items Returns: list: the list of cleaned up top items """ top_items = [] for top_item in top_items_dict.values(): clean_top_item(top_item) top_items.append(top_item) return top_items def clean_top_item(item): """Clean a top release or track item. Args: item (dict): the top release or track item Returns: dict: the the cleaned up top item """ if 'store_percentages' in item: if 'other' in item['store_percentages']: item['store_percentages']['other'] = calculate_store_percentage( item['store_percentages']['other'], item['total_streams']) def sort_top_items_streams(top_items): """Sort the top items by total_streams descending. Args: top_items (list): the list of top items Returns: list: the list of top items sorted by total_streams descending """ return list(reversed(sorted( top_items, key=lambda r: r['total_streams']))) def sort_top_items_transactions(top_items): """Sort the top items by total_transactions descending. Args: top_items (list): the list of top items Returns: list: the list of top items sorted by total_transactions descending """ return list(reversed(sorted( top_items, key=lambda r: r['total_transactions']))) def create_notification(account_type, digest): """Create a notification for an account by calling ows-notifications. Args: account_type (str): the account type digest (dict): the digest dict keyed by account ID Returns: None """ for account_id, account_digest in digest.items(): if not account_digest['top_releases'] and not \ account_digest['top_new_releases'] and not \ account_digest['top_tracks'] and not \ account_digest['top_physical_shipments']: return notification = { 'feed_name': config.FEED_NAME, 'feed_id': '{0}_{1}'.format( base_config.FEED_ACCOUNT_TYPES[account_type], account_id), 'payload': account_digest } ows_notifications.create_notification(notification)