"""fetch_trending_tracks task.""" from datetime import datetime import json import time from garcon import task from snowflake_connector.snowflake_conn import get_session from snowflake_connector.snowflake_conn import SQLLoader from activity_detector import base_config from activity_detector.flows.trending_tracks import config from activity_detector.utils import ows_assets from activity_detector.utils import ows_notifications sql_loader = SQLLoader(config.QUERY_PATH) @task.decorate(timeout=28800) def fetch_trending_tracks( activity, country_groups, top_ten_start_date, top_ten_end_date, all_track_start_date, all_track_end_date, last_dates_per_store): """Fetch the trending tracks for all country groups. Args: activity (ActivityWorker): garcon activity worker country_groups ([dict]): the list of country groups top_ten_start_date (str): the top ten start date top_ten_end_date (str): the top ten end date all_track_start_date (str): the all tracks start date all_track_end_date (str): the all tracks end date last_dates_per_store ([dict]): the list of last date per store Returns: None """ if not country_groups: return with get_session() as session: global_country_group = extract_global_country_group( country_groups.copy()) global_trending_tracks = get_global_trending_tracks( session, global_country_group, top_ten_start_date, top_ten_end_date, all_track_start_date, all_track_end_date) missing_data_per_store = check_missing_data_per_store( last_dates_per_store, top_ten_end_date) for country_group in country_groups: if country_group['country_group_name'] == 'Global': payload = { 'country_group_name': country_group['country_group_name'], 'trending_tracks': global_trending_tracks, 'start_date': top_ten_start_date, 'end_date': top_ten_end_date } if missing_data_per_store: payload['notes'] = missing_data_per_store else: countries = fetch_countries(session, country_group) payload = { 'country_group_name': country_group['country_group_name'], 'global_trending_tracks': global_trending_tracks, 'trending_tracks': get_trending_tracks( session, countries, country_group, top_ten_start_date, top_ten_end_date, all_track_start_date, all_track_end_date), 'start_date': top_ten_start_date, 'end_date': top_ten_end_date } if missing_data_per_store: payload['notes'] = missing_data_per_store if base_config.ENVIRONMENT in [base_config.ENVIRONMENT_PROD, base_config.ENVIRONMENT_QA]: time.sleep(60) create_notification(payload) def check_missing_data_per_store(last_dates_per_store, top_ten_end_date): """Check if missing data based on store. Args: last_dates_per_store ([dict]): the list of last date per store top_ten_end_date (str): the top ten end date Returns: str: the stores with missing data """ if not last_dates_per_store: return '' end_date = datetime.strptime(top_ten_end_date, config.DATE_FORMAT) missing_data_store = [] for date_per_store in last_dates_per_store: store_name = date_per_store['store_name'] if 'last_date' not in date_per_store: missing_data_store.append(store_name) else: last_date = datetime.strptime( date_per_store['last_date'], config.DATE_FORMAT) if last_date < end_date: missing_data_store.append(store_name) if not missing_data_store: return '' return ', '.join(missing_data_store) def extract_global_country_group(country_groups): """Extract the global country group. Args: country_groups ([dict]): the list of country groups Returns: dict: the global country group """ country_groups = [ d for d in country_groups if d.get('country_group_name') == 'Global'] return country_groups[0] def get_global_trending_tracks( session, global_country_group, top_ten_start_date, top_ten_end_date, all_track_start_date, all_track_end_date): """Get trending tracks for the global country group. Args: session: the db session global_country_group (dict): the global country group top_ten_start_date (str): the top ten start date top_ten_end_date (str): the top ten end date all_track_start_date (str): the all tracks start date all_track_end_date (str): the all tracks end date Returns: [dict]: the trending tracks for the global country group """ countries = fetch_countries(session, global_country_group) return get_trending_tracks( session, countries, global_country_group, top_ten_start_date, top_ten_end_date, all_track_start_date, all_track_end_date) def fetch_countries(session, country_group): """Fetch the countries for a country group. Args: session: the db session country_group (dict): the country group Returns: [tuple]: the list of countries """ sql = sql_loader.load_query('fetch_countries') sql = sql.replace('{env}', base_config.ENVIRONMENT) group_id_clause = 'WHERE ttcgc.trending_tracks_country_group_id = {0}' group_id_clause = group_id_clause.format( country_group['country_group_id']) sql = sql.replace('{group_id_clause}', group_id_clause) result = session.execute(sql) return result.cursor.fetchall() def fetch_all_time_streams(session, isrcs, labelids): """Fetch all time streams for isrcs. Args: session: the db session isrcs [string]: the list of isrc Returns: [tuple]: the list of all streams based on isrcs """ sql = sql_loader.load_query('fetch_all_time_streams') sql = sql.replace('{env}', base_config.ENVIRONMENT) isrc_condition = ', '.join(["\'{0}\'".format(isrc) for isrc in isrcs]) labelid_condition = ', '.join( ["\'{0}\'".format(labelid) for labelid in labelids]) where_clause = ( f'WHERE dt.isrc IN ({isrc_condition}) ' f"AND dr.deletions = \'N\' " f'AND fa.labelid IN ({labelid_condition}) ' f'AND fa.feedid NOT IN (43, 44, 49, 50, 51)') sql = sql.replace('{where_clause}', where_clause) result = session.execute(sql) return result.cursor.fetchall() def get_trending_tracks( session, countries, country_group, top_ten_start_date, top_ten_end_date, all_track_start_date, all_track_end_date): """Get the trending tracks for a country group. Args: session: the session db countries ([tuple]): the list of countries country_group (dict): the country group top_ten_start_date (str): the top ten start date top_ten_end_date (str): the top ten end date all_track_start_date (str): the all tracks start date all_track_end_date (str): the all tracks end date Returns: ([dict]): the trending tracks for the country group """ sql = sql_loader.load_query('fetch_trending_tracks') sql = sql.replace('{env}', base_config.ENVIRONMENT) country_group_name = country_group['country_group_name'] sql = update_get_trending_tracks_sql_with_clauses( sql, countries, country_group_name) query_params = { 'weekly_streams_minimum': country_group['country_group_floor'], 'top_ten_start_date': top_ten_start_date, 'top_ten_end_date': top_ten_end_date, 'all_track_start_date': all_track_start_date, 'all_track_end_date': all_track_end_date, } result = session.execute(sql, query_params) trending_tracks = result.cursor.fetchall() if not trending_tracks: return [] trending_tracks_isrcs = [track[2]for track in trending_tracks] trending_tracks_labelids = [track[0] for track in trending_tracks] all_streams_by_isrc = fetch_all_time_streams( session, trending_tracks_isrcs, trending_tracks_labelids) return map_trending_tracks( add_total_streams(trending_tracks, all_streams_by_isrc)) def add_total_streams(trending_tracks, all_streams_by_isrc): """Add all time streams to trending tracks. Args: trending_tracks [tuple]: the trending tracks all_streams_by_isrc [tuple]: the all streams by isrc Returns: [tuple]: the list of trending tracks with all streams """ trending_tracks_with_total_streams = [] for track in trending_tracks: isrc = track[2] for all_stream in all_streams_by_isrc: if all_stream[0] == isrc: track = track + (all_stream[1],) trending_tracks_with_total_streams.append(track) return trending_tracks_with_total_streams def update_get_trending_tracks_sql_with_clauses( sql, countries, country_group_name): """Extrapolate SQL template with filtering condition. Args: sql (str): the sql query countries ([tuple]): the list of countries country_group_name (str): the country group name Returns: sql (str): the formatted sql """ country_clause = '' limit_clause = 'LIMIT 15' if country_group_name != 'Global': country_ids = [str(country[0]) for country in countries] country_ids = ','.join(country_ids) country_clause = ' AND fa.countryid IN ({0})'.format(country_ids) limit_clause = 'LIMIT 30' sql = sql.replace('{limit_clause}', limit_clause) return sql.replace('{country_clause}', country_clause) def map_trending_tracks(trending_tracks): """Map trending tracks. Args: trending_tracks ([dict]): the trending tracks Returns: [dict]: the mapped trending tracks """ mapped_tracks = [] for trending_track in trending_tracks: (label_id, label_name, isrc, track_name, most_recent_week_streams, upc, product_id, release_date, expected_weekly_average, percent_above_avg, spike_score, artist_names, total_streams) = trending_track artist_names = json.loads(artist_names) mapped_tracks.append({ 'label_id': label_id, 'label_name': label_name, 'isrc': isrc, 'track_name': track_name, 'artist_names': artist_names, 'most_recent_week_streams': most_recent_week_streams, 'expected_weekly_average': float(expected_weekly_average), 'percent_above_avg': round(percent_above_avg), 'spike_score': spike_score, 'thumbnail_url': get_release_image_url(product_id, upc), 'streams_all_time': total_streams, 'release_date': release_date.strftime(config.DATE_FORMAT) }) return mapped_tracks def create_notification(payload): """Create a notification by calling ows-notifications. Args: payload (dict): the trending tracks payload for a country group Returns: None """ if not payload['trending_tracks'] and not \ payload['global_trending_tracks']: return feed_name = '{0}_{1}'.format( config.FEED_NAME, config.FEEDS[payload['country_group_name']]) notification = { 'feed_name': feed_name, 'feed_id': config.FEED_ID, 'payload': { 'actor': 'The Orchard Activity Detector', 'verb': 'Detected', 'object': 'Trend', 'start_date': payload['start_date'], 'end_date': payload['end_date'], 'country_group_name': payload['country_group_name'], 'trending_tracks': payload['trending_tracks'] } } if 'global_trending_tracks' in payload: notification['payload']['global_trending_tracks'] = \ payload['global_trending_tracks'] if 'notes' in payload: notification['payload']['notes'] = payload['notes'] ows_notifications.create_notification(notification) def get_release_image_url(product_id, upc, logger=None): """Get release image url by product_id. If ows-assets could not process the request use ows-images as a fall-back. Args: upc (str): release upc. product_id (int): product_id from art_relations.releases. logger: logger instance. Returns: str: image url. """ image_url = None try: image_url = ows_assets.get_release_image_url( product_id, logger=logger) except Exception as e: if logger: logger.warning(e) if image_url: return image_url return None