"""Youtube Weekly Ingestion Workflow.""" from garcon.param import StaticParam from garcon_contrib.dynamo_feed_status import garcon_feed_status from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.flows import base from feed_ingestion.flows.youtube_weekly import config from feed_ingestion.flows.youtube_weekly import tasks from feed_ingestion.flows.youtube_weekly.stage_loader import YoutubeWeekly from feed_ingestion.tasks import overall_status_tasks from feed_ingestion.tasks.s3_tasks import copy_files class Flow(base.FlowBase, base.FlowConfigMixin, base.FlowLoadFactMixinSF): """Class representing the workflow.""" def __init__(self): """Initialize flow object.""" super(Flow, self).__init__(config.feed_name, config.feed_version) def decider(self, schedule): """Activity decider. Args: schedule (callable): The scheduler method. """ check_feed_status = schedule( 'check_feed_status', self.check_feed_status) if check_feed_status.result.get('check_feed_status.stop') is True: return bootstrap = schedule( 'bootstrap', self.bootstrap, requires=[check_feed_status]) if bootstrap.result.get('bootstrap.stop') is True: return grab_drop_files = schedule( 'grab_drop_files', self.grab_drop_files, requires=[bootstrap]) if grab_drop_files.result.get('grab_drop_files.stop') is True: schedule('set_status_to_not_available', self.set_status_to_not_available, requires=[grab_drop_files]) return # Status updated to DOWNLOADED if files are present set_status_to_downloaded = schedule( 'set_status_to_downloaded', self.set_status_to_downloaded, requires=[grab_drop_files]) load_staging_raw_table = schedule( 'load_staging_raw_table', self.load_staging_raw_table, requires=[set_status_to_downloaded]) set_status_to_populated_raw_table = schedule( 'set_status_to_populated_raw_table', self.set_status_to_populated_raw_table, requires=[load_staging_raw_table]) load_staging_fact_table = schedule( 'load_staging_fact_table', self.load_staging_fact_table, requires=[set_status_to_populated_raw_table]) schedule( 'load_fact_tables', self.load_fact_tables, requires=[load_staging_fact_table]) return True @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='check_feed_status.date', reload='reload'))) @property def grab_drop_files(self): """Archive needed files from the drop location to archive location.""" return self.create( name='grab_drop_files', tasks=base.SyncRunner( copy_files.fill( namespace='grab_drop_files', s3_archive_path='bootstrap.s3_archive_path', s3_download_path='bootstrap.s3_download_path', source_files_dict='bootstrap.source_files_dict', need_all_files=StaticParam(True)))) load_staging_raw_table = YoutubeWeekly.load_activity( feed_name=config.feed_name, secrets_path=config.secrets_path, sql_loader=SQLLoader(__file__), requirements=dict( date='bootstrap.date', s3_dir_path='bootstrap.s3_archive_path', staging_raw_table_name='bootstrap.staging_raw_table', source_files_dict='grab_drop_files.source_files_dict', sfdb_params='sfdb_params' ) ) @property def check_feed_status(self): """Check and reset feed status if it is needed.""" return self.create( name='check_feed_status', tasks=base.SyncRunner( tasks.check_feed_status.fill( namespace='check_feed_status', date='context_date', reload='reload'))) @property def set_status_to_downloaded(self): """Set overall feed status to DOWNLOADED.""" return self.create( name='set_status_to_downloaded', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_downloaded', feed_name='bootstrap.feed_name', date='bootstrap.date', status=StaticParam( garcon_feed_status.STATUS_DOWNLOADED)))) @property def set_status_to_populated_raw_table(self): """Set overall feed status to POPULATED_RAW_TABLE.""" return self.create( name='set_status_to_populated_raw_table', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_populated_raw_table', feed_name='bootstrap.feed_name', date='bootstrap.date', status=StaticParam( garcon_feed_status.STATUS_POPULATED_RAW_TABLE)))) @property def set_status_to_not_available(self): """Set overall feed status to NOT_AVAILABLE.""" return self.create( name='set_status_to_not_available', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_not_available', feed_name='bootstrap.feed_name', date='bootstrap.date', status=StaticParam( garcon_feed_status.STATUS_NOT_AVAILABLE))))