"""spike_detector workflow.""" from datetime import datetime from datetime import timedelta from itertools import groupby import json from ddtrace import tracer as ddtracer from garcon import task from snowflake_connector.snowflake_conn import get_session from snowflake_connector.snowflake_conn import SQLLoader from activity_detector.flows.spike_detector import config from activity_detector.flows.spike_detector.notifications import SpikedTrack, SpikeNotification from activity_detector.utils import dynamodb from activity_detector.utils import ows_assets sql_loader = SQLLoader(__file__) @ddtracer.wrap(resource='task.spike_detector', name='create_notifications') def send_notifications(logger, target_date, spiked_tracks, level): """Send notifications. Primarily needed for easy testing. Args: logger: logger object. target_date (str): target date. spiked_tracks (list): list of rows from db. level (str): label or subaccount. """ def get_id(track): if level == 'label': return track.label_id else: return track.subaccount_id def is_sorted(rows): return all( get_id(f) <= get_id(s) for f, s in zip(rows, rows[1:])) if not is_sorted(spiked_tracks): raise Exception('Spiked tracks should by sorted by account id') # result is already sorted by id so we can use groupby groups = groupby(spiked_tracks, key=get_id) for acc_id, tracks in groups: logger.info( 'Sending notification to account: %s', acc_id) tracks_sorted = list(sorted( tracks, key=lambda t: t.num_streams, reverse=True)) label_name = tracks_sorted[0].label_name subacc_name = tracks_sorted[0].subaccount_name SpikeNotification( target_date, acc_id, tracks_sorted, label_name, subacc_name, level).send() def get_release_image_url(product_id, upc, logger=None): """Get release image url by product_id. If ows-assets could not process the request use ows-images as a fall-back. Args: upc (str): release upc. product_id (int): product_id from art_relations.releases. logger: logger instance. Returns: str: image url. """ image_url = None try: image_url = ows_assets.get_release_image_url( product_id, logger=logger) except Exception as e: logger.warning(e) if image_url: return image_url return None @task.decorate(timeout=1000) def bootstrap(activity, date, label_ids, subaccount_ids, window_days, run_id): """Bootstrap workflow. Args: activity (ActivityWorker): The activity worker. date (str): target date. label_ids (str): comma-separated list of label ids: 134,4454,234214. subaccount_ids (str): comma-separated list of subaccount ids: 134,4454,234214. window_days (int): Number of days back used to calculate avg and std. run_id (str): Unique id for workflow run. Returns: dict: Context. """ activity.logger.info('Bootstrap flow: {}'.format(date)) if date: target_date = datetime.strptime(date, '%Y-%m-%d') else: target_date = datetime.today() - timedelta( days=config.DEFAULT_SHIFT_DAYS) dlt = timedelta(days=window_days or config.DEFAULT_WINDOW_DAYS) start_date = target_date - dlt # if no label passed trigger detection for all the labels label_ids_lst = label_ids.split(',') if label_ids else [] label_ids_lst = list(map(int, label_ids_lst)) # if no subaccount passed don't trigger detection subaccount_ids_lst = subaccount_ids.split(',') if subaccount_ids else [] subaccount_ids_lst = list(map(int, subaccount_ids_lst)) return dict( target_date=target_date.strftime('%Y-%m-%d'), start_date=start_date.strftime('%Y-%m-%d'), label_ids=label_ids_lst, subaccount_ids=subaccount_ids_lst, run_id=run_id) @task.decorate(timeout=14400) def create_temp_activity_history_table( activity, target_date, start_date, run_id, source_type, label_ids=None, subaccount_ids=None ): """Create temp activity history table. Args: activity (ActivityWorker): The activity worker. label_ids (str): comma-separated list of label ids: 134,4454,234214. subaccount_ids (str): comma-separated list of subaccount ids: 134,4454,234214. target_date (str): target date. start_date (str): start date. run_id (str): Unique id for workflow run. source_type (str): Streams or tiktok. """ filters = construct_id_filter(label_ids, subaccount_ids) activity.logger.info(filters['logger_info']) sql_tmpl = sql_loader.load_query( f'create_temp_activity_history_{source_type}') sql = sql_tmpl.format( labelid_filter=filters['labelid_filter'], subaccountid_filter=filters['subaccountid_filter'], table_name=config.TEMP_TABLE_NAME.format(type=source_type)) with get_session() as session: session.execute( sql, { 'label_ids': label_ids, 'subaccount_ids': subaccount_ids, 'target_date': target_date, 'start_date': start_date, 'store_ids': list(config.STORES), 'min_track_streams': config.MIN_TRACK_STREAMS, 'max_tracks_per_account': config.MAX_TRACKS_PER_ACCOUNT, 'z_score_threshold': config.Z_SCORE_THRESHOLD, 'min_release_date_shift_days': ( config.MIN_RELEASE_DATE_SHIFT_DAYS), 'min_stream_gain': config.MIN_STREAM_GAIN, 'run_id': run_id }) activity.logger.info( f'Temp activity spike history {source_type} table created.') @task.decorate(timeout=14400) def populate_activity_history_table(activity): """Populate activity_spike_history table with data from Temp table. Args: activity (ActivityWorker): The activity worker. Returns: dict: Context dictionary. """ activity.logger.info('Populating activity_spike_history table.') sql = sql_loader.load_query('load_activity_history') with get_session() as session: session.execute(sql) activity.logger.info('activity_spike_history table is populated.') @task.decorate(timeout=14400) def drop_temp_table(activity): """Drop temporary table. Args: activity (ActivityWorker): The activity worker. """ activity.logger.info('Dropping temp_activity_spike_history table.') sql = sql_loader.load_query('drop_temp_table') with get_session() as session: for source_type in config.TYPES: table_name = config.TEMP_TABLE_NAME.format(type=source_type) session.execute(sql.format(table_name=table_name)) activity.logger.info(f'{table_name} has be dropped') @task.decorate(timeout=14400) def detect_spikes(activity, ids, target_date, start_date, level, run_id): """Detect spikes at the label level. Args: activity (ActivityWorker): The activity worker. ids (list): comma-separated list of account ids: 134,4454,234214. target_date (str): target date. start_date (str): start date. level (str): label or subaccount. run_id (str): Unique id for workflow run. Returns: dict: Context dictionary. """ if ddtracer.enabled: with ddtracer.trace(resource='task.label_spike_detector', name='detect_spikes') as dd_span: dd_span.set_tag('level', level) dd_span.set_tag('ids', ids) status = config.STATUS_PROCESSED if ids and ids[0] == -1: return { 'status': status } stores = list(config.STORES.keys()) activity.logger.info( f'Detecting spikes [{level}]: [{start_date} - {target_date}] ({ids}) ' f'stores {stores}') with get_session() as session: sql = sql_loader.load_query('get_spiked_tracks_{}'.format(level)) result = session.execute( sql, { 'max_spikes_per_account': config.MAX_SPIKES_PER_ACCOUNT, 'target_date': target_date, 'playlist_placement_shift_days': ( config.PLAYLIST_PLACEMENTS_SHIFT_DAYS), 'max_playlists_to_show': config.MAX_PLAYLISTS_TO_SHOW, 'playlist_streams_threshold': config.PLAYLIST_STREAMS_THRESHOLD, 'run_id': run_id, 'store_ids': list(stores) } ) def row_to_track(row): image_url = get_release_image_url( row.product_id, row.upc, activity.logger) artist_names = json.loads(row.artist_names) recent_playlist_placements = json.loads( row.recent_playlist_placements) return SpikedTrack( label_id=row.labelid, label_name=row.labelname, subaccount_name=row.subaccountname, track_name=row.trackname, artist_names=artist_names, isrc=row.isrc, release_date=row.releasedate.strftime('%Y-%m-%d'), num_streams=int(row.target_day_streams), num_streams_avg=int(row.avg_isrc_streams), percent_above_avg=int(row.pct_above_avg), image_url=image_url, store_name=config.STORES[row.storeid], subaccount_id=( row.subaccountid if level == 'subaccount' else None), recent_playlist_placements=recent_playlist_placements, country_code=row.country_code) spiked_tracks = [row_to_track(row) for row in result] if spiked_tracks: send_notifications(activity.logger, target_date, spiked_tracks, level) status = config.STATUS_PROCESSED_NOTIF_SENT return { 'status': status } @task.decorate(timeout=300) def check_dynamo_status(activity, date): """Check the activity status in DynamoDB. Args: activity (ActivityWorker): The activity worker. date (str): target date. """ res = dynamodb.get_status(config.ACTIVITY_TYPE_NAME, date) status = res and res['status'] if status and status == config.STATUS_PROCESSED_NOTIF_SENT: return {'should_run': False} return {'should_run': True} @task.decorate(timeout=300) def set_dynamo_status(activity, date, status_label, status_subaccount): """Set activity status in DynamoDB. Args: activity (ActivityWorker): The activity worker. date (str): target date. status_label (str): status of label level detection. status_subaccount (str): status of subaccount level detection. """ activity.logger.info('Setting dynamo status: %s', date) if config.STATUS_PROCESSED_NOTIF_SENT in ( status_label, status_subaccount): status = config.STATUS_PROCESSED_NOTIF_SENT else: status = config.STATUS_PROCESSED dynamodb.set_status( config.ACTIVITY_TYPE_NAME, date, status) def construct_id_filter(label_ids, subaccount_ids): """Build correct id filters. Args: label_ids (str): comma-separated list of label ids: 134,4454,234214. subaccount_ids (str): comma-separated list of subaccount ids: 134,4454,234214. Returns: list: label_id_filter (str) and subaccount_id_filter (str) str: logger text """ if label_ids and subaccount_ids: return { 'labelid_filter': 'fa.labelid IN (:label_ids)', 'subaccountid_filter': 'fa.subaccountid IN (:subaccount_ids)', 'logger_info': 'Creating spikes table for label ids: {l_ids} and ' 'subaccount ids: {s_ids}'.format(l_ids=label_ids, s_ids=subaccount_ids) } if label_ids: return { 'labelid_filter': 'fa.labelid IN (:label_ids)', 'subaccountid_filter': 'False', 'logger_info': 'Creating spikes table for label ids: ' '{}'.format(label_ids) } if subaccount_ids: return { 'labelid_filter': 'False', 'subaccountid_filter': 'fa.subaccountid IN (:subaccount_ids)', 'logger_info': 'Creating spikes table for subaccount ids: ' '{}'.format(subaccount_ids) } return { 'labelid_filter': 'True', 'subaccountid_filter': 'False', 'logger_info': 'Creating spikes table for all accounts' }