"""chartmetric_spike_detector workflow tasks.""" from datetime import datetime from ddtrace import tracer as ddtracer from garcon import task from snowflake_connector.snowflake_conn import get_session from snowflake_connector.snowflake_conn import SQLLoader from activity_detector.flows.chartmetric_spike_detector import config from activity_detector.utils import dynamodb from activity_detector.utils import ows_notifications sql_loader = SQLLoader(__file__) @task.decorate(timeout=1000) def bootstrap(activity, date, run_id): """Startup flow with parameters from exec command.""" if not date: date = datetime.today().strftime('%Y-%m-%d') return dict( target_date=date ) @task.decorate(timeout=300) def check_dynamo_status(activity, date): """Check the activity status in DynamoDB. Args: activity (ActivityWorker): The activity worker date (str): YYYY-MM-DD target date """ res = dynamodb.get_status(config.ACTIVITY_TYPE_NAME, date) status = res and res['status'] if status and status == config.STATUS_PROCESSED_NOTIF_SENT: return {'should_run': False} return {'should_run': True} @task.decorate(timeout=600) def load_instagram_data(activity, date): """Insert instagram rows social spike table. Args: activity (ActivityWorker): The activity worker date (str): YYYY-MM-DD target date """ return _load_data(date, 'instagram') @task.decorate(timeout=600) def load_youtube_data(activity, date): """Insert youtube rows social spike table. Args: activity (ActivityWorker): The activity worker date (str): YYYY-MM-DD target date """ return _load_data(date, 'youtube') def _load_data_format_query(social_network): return sql_loader.load_query('load_data').format( cm_table=config.CM_DATA[social_network]['table'], cm_column=config.CM_DATA[social_network]['column'], social=social_network ) def _load_data(date, social_network): """Insert rows social spike table. Args: date (str): YYYY-MM-DD date to load data for social_network (str): name of social network to load data for """ check_sql = sql_loader.load_query('check_activity_exists') load_sql = _load_data_format_query(social_network) with get_session() as session: # check if data for network already loaded for date row_count = session.execute( check_sql, { 'date': date, 'social': social_network } ).fetchone()['c'] # only insert rows if no data exists for date if row_count == 0: session.execute( load_sql, { 'date': date } ) return dict() @task.decorate(timeout=7200) def detect_spikes(activity, date): """Find spikes in social data and send data to ows-notifications. Args: activity (ActivityWorker): The activity worker date (str): YYYY-MM-DD target date Returns: dict: resulting number of failures when sending to API """ return _detect_spikes(activity, date) @ddtracer.wrap(resource='task.chartmetric_spike_detector', name='detect_social_spikes') def _detect_spikes(activity, date): """Find spikes in social data to send them to ows-notifications. @ddtracer.wrap decorator is incompatible with garcon task runner and causes failures in the chartmetric_spike_detector workflow execution. To enable full tracing without impacting the workflow, all related logic was moved here instead. """ get_activity_sql = sql_loader.load_query('get_activity') # get spikes in social data for day rows = [] with get_session() as session: rows = session.execute( get_activity_sql, { 'date': date, 'youtube_min_score': config.YOUTUBE_MIN_SCORE, 'youtube_min_new_followers': config.YOUTUBE_MIN_NEW_FOLLOWERS, 'instagram_min_score': config.INSTAGRAM_MIN_SCORE, 'instagram_min_new_followers': config.YOUTUBE_MIN_NEW_FOLLOWERS } ).fetchall() activity.logger.info(f'Found {len(rows)} spikes in social data for {date}') # make calls to ows-notifications to add events failures = 0 for row in rows: success = ows_notifications.create_social_spike_notification( str(row['date']), row['social'], row['new_followers_today'], row['cm_artist_id'] ) if not success: failures += 1 return dict( failures=failures ) @task.decorate(timeout=300) def set_dynamo_status(activity, date, failures): """Set status of workflow run according to params. Args: activity (ActivityWorker): The activity worker date (str): YYYY-MM-DD target date failures (int): number of API request failures """ status = config.STATUS_PROCESSED_NOTIF_SENT if not failures \ else config.STATUS_PROCESSED dynamodb.set_status( config.ACTIVITY_TYPE_NAME, date, status)