"""YouTube Video Report Workflow.""" from datetime import date as date_module from datetime import datetime from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.conf.config import merge_configs from feed_ingestion.flows import YoutubeVideo from feed_ingestion.flows.helpers import get_sf_config from feed_ingestion.flows.youtube_video import config STOP_RESPONSE = {'stop': True} @task.decorate(timeout=1000) def bootstrap(activity, date, licensor=None, reload=None, skip_corrupted_rows=None, dw_config=None): """Bootstrap workflow by injecting initial context from config. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). licensor (str): one of config.licensors reload (str): If 'True' delete feed status in dynamodb. skip_corrupted_rows (str): If 'True' use the skip_corrupted_rows param. dw_config (dict): Dictionary of data warehouse config options. Returns: dict: Initial context for the workflow. """ if not licensor: licensor = 'theorchard' assert licensor in config.licensors, f'unsupported licensor "{licensor}"' date = date or date_module.today().strftime('%Y-%m-%d') parsed_date = datetime.strptime(date, '%Y-%m-%d') feed_name = '_'.join([config.feed_name, licensor]) report_status_name = feed_name cms_dict = { 'theorchard': config.cms_dict } if reload == 'True': activity.logger.info( 'Delete status for feed: {} {} '.format(report_status_name, date)) garcon_feed_status.delete_status(report_status_name, date) else: overall_status = garcon_feed_status.get_overall_status( report_status_name, date) if overall_status == garcon_feed_status.STATUS_INGESTED: return STOP_RESPONSE archive_path = config.s3['archive_path'].format( date=parsed_date, licensor=licensor) download_path = config.s3['download_path'] credentials_path = config.credentials_paths.get(licensor) source_file_pattern = config.source_file_pattern[licensor] temp_staging_raw_table = config.snowflake[ 'temp_staging_raw'].format( licensor=licensor, date=datetime.strptime(date, '%Y-%m-%d')) corrupted_rows_error = skip_corrupted_rows or 'False' return { 'feed_name': feed_name, 'secrets_path': config.secrets_path, 'date': date, 'licensor': licensor, 'report_name': config.youtube_report_full_name, 's3_bucket': config.s3_bucket, 's3_download_path': download_path, 's3_archive_path': archive_path, 'credentials_path': credentials_path, 's3_dir_path': 's3://{}/{}'.format(config.s3_bucket, archive_path), 'source_files_path': archive_path, 'source_files_pattern': source_file_pattern, 'skip_corrupted_rows': corrupted_rows_error, 'replace_archive_files': config.replace_archive_files, 'temp_staging_raw_table': temp_staging_raw_table, 'channel_names_table': config.snowflake.get('channel_names_table'), 'cms_dict': cms_dict.get(licensor) } @task.decorate(timeout=1000) def clean_staging_raw_table( activity, feed_name, sfdb_params, licensor): """Update isrc in staging_raw_youtube_video_report table . Args: activity (ActivityWorker): The activity worker. feed_name (str): Name of the feed to get executor class. sfdb_params (dict): Dict with params to optionally override default ones (Snowflake db and schema name). licensor (str): one of config.licensors. """ sf_config = merge_configs(get_sf_config(config.secrets_path), sfdb_params) with YoutubeVideo(sf_config) as sf_executor: sf_executor.clean_staging_raw_table(licensor) activity.logger.info('Cleaned the staging raw table.') @task.decorate(timeout=12600) def load_staging_raw_table( activity, feed_name, sfdb_params, temp_staging_raw_table): """Update isrc in staging_raw_youtube_video_report table . Args: activity (ActivityWorker): The activity worker. feed_name (str): Name of the feed to get executor class. temp_staging_raw_table (str): Namae of a Snowflake table sfdb_params (dict): Dict with params to optionally override default ones (Snowflake db and schema name). """ sf_config = merge_configs(get_sf_config(config.secrets_path), sfdb_params) with YoutubeVideo(sf_config) as sf_executor: sf_executor.load_staging_raw_table(temp_staging_raw_table) activity.logger.info('Loaded the staging raw table.') @task.decorate(timeout=600) def drop_temp_table(activity, temp_staging_raw_table): """Drop temp staging raw tables. Args: activity (ActivityWorker): The Garcon activity worker. temp_staging_raw_table (str): Table name to drop. """ sf_config = get_sf_config(config.secrets_path) with YoutubeVideo(sf_config) as executor: executor.drop_table(temp_staging_raw_table) activity.logger.info('{} was dropped'.format(temp_staging_raw_table)) @task.decorate(timeout=12600) def update_staging_raw_table( activity, date, feed_name, sfdb_params, licensor): """Update isrc in staging_raw_youtube_video_report table . Args: activity (ActivityWorker): The activity worker. date (str): Reporting date (YYYY-MM-DD). feed_name (str): Name of the feed to get executor class. sfdb_params (dict): Dict with params to optionally override default ones (Snowflake db and schema name). licensor (str): one of config.licensors. """ # only for theorchard licensor if licensor != 'theorchard': return sf_config = merge_configs(get_sf_config(config.secrets_path), sfdb_params) with YoutubeVideo(sf_config) as sf_executor: sf_executor.update_staging_raw_table(date, licensor='theorchard') activity.logger.info('Updated isrc in staging_raw_youtube_video_report.') @task.decorate(timeout=12600) def update_mnc_and_owner_columns( activity, date, feed_name, sfdb_params, channel_names_table_name, licensor): """Update MCN and OWNER columns in dim_youtube_channel_names table. Args: activity (ActivityWorker): The activity worker. date (str): Reporting date (YYYY-MM-DD). feed_name (str): Name of the feed to get executor class. sfdb_params (dict): Dict with params to optionally override default ones (Snowflake db and schema name). licensor (str): one of config.licensors. """ # only for theorchard licensor if licensor != 'theorchard': return sf_config = merge_configs(get_sf_config(config.secrets_path), sfdb_params) with YoutubeVideo(sf_config) as sf_executor: sf_executor.update_mnc_and_owner_columns(date, channel_names_table_name, licensor='theorchard') activity.logger.info('Updated MCN and OWNER columns')