""" YouTube Red Marketshare Workflow. Ingest YouTube Red marketshare data into marketshare table in Snowflake. """ from garcon.param import StaticParam from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows import base from feed_ingestion.flows.base import FlowBase from feed_ingestion.flows.base import FlowConfigMixin from feed_ingestion.flows.base import FlowLoadMarketshareMixinSF from feed_ingestion.flows.youtube_red_marketshare import config from feed_ingestion.flows.youtube_red_marketshare import tasks from feed_ingestion.tasks import overall_status_tasks class Flow(FlowBase, FlowConfigMixin, FlowLoadMarketshareMixinSF): """YouTube Red Marketshare Flow class. This flow downloads YouTube Red marketshare data and loads to a main_market_share. """ def __init__(self): """Initialize an Spotify Marketshare workflow.""" super(Flow, self).__init__(config.feed_name, config.feed_version) def decider(self, schedule): """Orchestrate a YouTube Red Marketshare workflow. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) if bootstrap.result.get('bootstrap.stop'): return check_youtube_monthly_reports_status = schedule( 'check_youtube_monthly_reports_status', self.check_youtube_monthly_reports_status, requires=[bootstrap]) if check_youtube_monthly_reports_status.result.get( 'check_youtube_monthly_reports_status.stop') is True: return reset_dynamo_db_status = schedule( 'reset_dynamo_db_status', self.reset_dynamo_db_status, requires=[check_youtube_monthly_reports_status]) load_staging_raw_table = schedule( 'load_staging_raw_table', self.load_staging_raw_table, requires=[reset_dynamo_db_status]) load_marketshare_table = schedule( 'load_market_share_table', self.load_marketshare_table, requires=[load_staging_raw_table]) schedule( 'set_status_to_ingested', self.set_status_to_ingested, requires=[load_marketshare_table]) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', reload='reload'))) @property def check_youtube_monthly_reports_status(self): """Check if youtube_monthly data has been ingested for that month.""" return self.create( name='check_youtube_monthly_reports_status', tasks=base.SyncRunner( tasks.check_youtube_monthly_reports_status.fill( namespace='check_youtube_monthly_reports_status', feed_name='bootstrap.feed_name', date='bootstrap.report_start_date', staging_raw_tables='bootstrap.monthly_staging_raw_tables' ''))) @property def load_staging_raw_table(self): """Load the staging raw table.""" return self.create( name='load_staging_raw_table', tasks=base.SyncRunner( tasks.load_staging_raw_table.fill( namespace='load_staging_raw_table', feed_name='bootstrap.feed_name', start_date='bootstrap.report_start_date', end_date='bootstrap.report_end_date', monthly_staging_raw_tables='bootstrap' '.monthly_staging_raw_tables'))) @property def set_status_to_ingested(self): """Set overall feed status to INGESTED.""" return self.create( name='set_status_to_ingested', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_ingested', feed_name='bootstrap.feed_name', date='bootstrap.date', status=StaticParam( garcon_feed_status.STATUS_INGESTED))))