"""YouTube Facts Workflow.""" from datetime import date as date_module from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.conf.config import merge_configs from feed_ingestion.flows import registered_executors from feed_ingestion.flows.helpers import get_sf_config from feed_ingestion.flows.youtube_facts import config from feed_ingestion.tasks import check_status STOP_RESPONSE = {'stop': True} @task.decorate(timeout=1000) def bootstrap(activity, date, licensor, reload, report_type): """Bootstrap workflow by injecting initial context from config. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). licensor (str): one of config.licensors reload (str): If 'True' delete feed status in dynamodb. report_type (str): Type of the report to ingest. Returns: dict: Initial context for the workflow. """ if not licensor: licensor = 'theorchard' if licensor not in config.licensors: raise ValueError(f'Unsupported licensor "{licensor}"') if report_type not in config.report_dynamo_status_names.keys(): raise ValueError(f'Unsupported report_type "{report_type}"') date = date or date_module.today().strftime('%Y-%m-%d') feed_name = '_'.join([config.feed_name, licensor]) report_status_name = '_'.join([feed_name, report_type]) if reload == 'True': activity.logger.info( 'Delete status for feed: {} {} '.format(report_status_name, date)) garcon_feed_status.delete_status(report_status_name, date) else: overall_status = garcon_feed_status.get_overall_status( report_status_name, date) if overall_status == garcon_feed_status.STATUS_INGESTED: return STOP_RESPONSE executor_kwargs = { 'licensor': licensor } return { 'feed_name': feed_name, 'date': date, 'licensor': licensor, 'report_type': report_type, 'report_status_name': report_status_name, 'kwargs': executor_kwargs, } @task.decorate(timeout=1000) def check_staging_status(activity, date, report_type, licensor): """Check if all required reports is available. This function checks if all tables for 'date' have been populated based on DynamoDB statuses. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). report_type (str): Type of the report to ingest. licensor (str): one of config.licensors. Returns: dict: {} or STOP_RESPONSE if not all reports are available. """ statuses = _dependant_reports_statuses(date, report_type, licensor) not_ingested = {report: status for report, status in statuses.items() if status != garcon_feed_status.STATUS_INGESTED} if not_ingested: message = f'Some dependant reports are not ready: {not_ingested}' activity.logger.info(message) return {'stop': True, 'message': message} activity.logger.info(f'All dependent reports for {report_type} {date} ' f'are ready') return {'statuses': statuses} def _dependant_reports_statuses(date, report_type, licensor): """Get status of all dependant reports. Args: date (str): Reporting date (YYYY-MM-DD). report_type (str): Type of the report to ingest. licensor (str): one of config.licensors. Returns: dict: report to status map """ if report_type not in config.report_dynamo_status_names: raise ValueError(f'Unknown report type "{report_type}"') if licensor not in config.licensors: raise ValueError(f'Unknown licensor "{report_type}"') reports = config.report_dynamo_status_names[report_type][licensor] return {report: garcon_feed_status.get_overall_status(report, date) for report in reports} @task.decorate(timeout=36000) @check_status() def load_demographics_table(activity, date, feed_name, sfdb_params, kwargs): """Load demographics table. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). feed_name (str): Feed name of workflow execution for status updates. sfdb_params (dict): Dict with params to optionally override default ones (Snowflake db and schema name). kwargs (dict): Custom executor params. """ sf_config = get_sf_config(config.secrets_path) sf_config_custom = merge_configs(sf_config, sfdb_params) ExecutorFA = registered_executors.get(feed_name) with ExecutorFA(sf_config_custom) as sf_executor: activity.logger.info( 'Deleting rows for {date} from demographics for feed ' '{feed_name}'.format(date=date, feed_name=feed_name)) sf_executor.delete_from_demographics_table(date, **kwargs) activity.logger.info( 'Loading demographics for feed {}'.format(feed_name)) sf_executor.load_demographics_data(date, **kwargs) @task.decorate(timeout=36000) @check_status() def load_mapping_table(activity, date, feed_name, report_type, sfdb_params): """Load youtube_video_asset_type_mapping. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). feed_name (str): Feed name of workflow execution for status updates. report_type (str): The type of the loading data (e.g. video, asset). sfdb_params (dict): Dict with params to optionally override default ones (Snowflake db and schema name). """ # update only if report = asset if report_type != 'asset': return STOP_RESPONSE sf_config = get_sf_config(config.secrets_path) sf_config_custom = merge_configs(sf_config, sfdb_params) ExecutorFA = registered_executors.get(feed_name) with ExecutorFA(sf_config_custom) as sf_executor: activity.logger.info( 'Loading mapping_table {}'.format(feed_name)) sf_executor.load_youtube_video_asset_type_mapping(date)