"""YouTube Red Marketshare Garcon tasks. Tasks to ingest YouTube Red Marketshare data. """ from datetime import datetime from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows.helpers import get_sf_config from feed_ingestion.flows.youtube_red_marketshare import config from feed_ingestion.flows.youtube_red_marketshare import util from feed_ingestion.flows.youtube_red_marketshare.snowflake_executor import \ YouTubeRedMarketshareSF @task.decorate(timeout=1000) def bootstrap(activity, date, reload=None, dw_config=None): """Bootstrap workflow by injecting initial context from config. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). reload (str or None): If 'True' then clear all feed statuses. dw_config (dict): Dictionary of data warehouse config options. Returns: dict: Context for the workflow. """ date_obj = datetime.strptime(date, '%Y-%m-%d') first_day, last_day = util.get_first_last_day(date_obj) report_start_date = first_day.strftime('%Y-%m-%d') report_end_date = last_day.strftime('%Y-%m-%d') feed_name = config.feed_name status = garcon_feed_status.get_overall_status(feed_name, report_start_date) if reload != 'True' and status == garcon_feed_status.STATUS_INGESTED: activity.logger.info( f'Feed {feed_name} already ingested for {report_start_date}' ) return { 'stop': True, 'message': f'{feed_name} is already ingested for {report_start_date}' } return { 'feed_name': config.feed_name, 'secrets_path': config.secrets_path, 'date': report_start_date, 'report_start_date': report_start_date, 'report_end_date': report_end_date, 'monthly_staging_raw_tables': config.youtube_monthly_reports_and_tables, 'staging_raw_table': config.staging_raw_table } @task.decorate(timeout=1000) def check_youtube_monthly_reports_status(activity, feed_name, date, staging_raw_tables): """Check if the youtube_monthly date has been ingested. Args: activity (ActivityWorker): The activity worker. feed_name (str): Feed name of workflow execution for status updates. date (str): Reporting date (YYYY-MM-DD). staging_raw_tables (dict): youtube monthly staging raw tables. Returns: dict: entry indicating all the required staging raw tables are unavailable. """ for report in staging_raw_tables.keys(): report_feed_name = 'youtube_monthly_' + report if garcon_feed_status.get_overall_status(report_feed_name, date) == \ garcon_feed_status.STATUS_INGESTED: activity.logger.info('Data ingested into {} for {' '}'.format(report, date)) else: return { 'stop': True, 'message': 'Data was not ingested into {} for {}'.format( report, date) } @task.decorate(timeout=3600) def load_staging_raw_table(activity, feed_name, start_date, end_date, monthly_staging_raw_tables): """Create temp staging raw table. Args: activity (ActivityWorker): The activity worker. feed_name (str): The name of the feed. start_date (str): stating date (YYYY-MM-DD) of the reporting month. end_date (str): end date (YYYY-MM-DD) of the reporting month. monthly_staging_raw_tables (dict): youtube monthly staging raw tables. """ staging_raw_red_summary_music = \ monthly_staging_raw_tables['red_summary_v1_1'][ 'staging_raw_red_summary_music'] staging_raw_red_label_summary_music = monthly_staging_raw_tables[ 'red_label_summary_v1_1']['staging_raw_red_label_summary_music'] staging_raw_red_label_summary_subscribers = monthly_staging_raw_tables[ 'red_label_summary_v1_1']['staging_raw_red_label_summary_subscribers'] with YouTubeRedMarketshareSF(get_sf_config(config.secrets_path)) as \ sf_executor: sf_executor.clean_staging_raw_table( start_date=start_date, ) sf_executor.load_staging_raw_table( staging_raw_red_summary_music=staging_raw_red_summary_music, staging_raw_red_label_summary_music=staging_raw_red_label_summary_music, # noqa: E501 staging_raw_red_label_summary_subscribers=staging_raw_red_label_summary_subscribers, # noqa: E501 first_day=start_date, last_day=end_date ) activity.logger.info('Loaded staging raw table for YouTube Red ' 'Marketshare')