"""Ritmogestion Ingestion Workflow.""" from garcon_contrib.dynamo_feed_status import garcon_feed_status as feed_status from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.common.staging_raw_sf.snowflake_stage_loader import StageLoader # noqa: E501 from feed_ingestion.flows import base from feed_ingestion.flows.ritmogestion import config from feed_ingestion.flows.ritmogestion import tasks class Flow(base.FlowBase, base.FlowConfigMixin): """Class representing the workflow.""" def __init__(self): """Initialize flow object.""" super(Flow, self).__init__(feed_name=config.feed_name, version='2.0') def decider(self, schedule, context): """Activity decider. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) # Stop flow if feed already ingested if bootstrap.result.get('bootstrap.stop'): return fetch_from_http = schedule( 'fetch_from_http', self.fetch_from_http, requires=[bootstrap]) # Stop flow if file not available if fetch_from_http.result.get('fetch_from_http.stop'): return set_overall_status_downloaded = schedule( 'set_overall_status_downloaded', self.set_overall_status, requires=[fetch_from_http], input={'status': feed_status.STATUS_DOWNLOADED}) load_staging_raw_table = schedule( 'load_staging_raw_table', self.load_staging_raw_table, requires=[set_overall_status_downloaded]) schedule( 'set_overall_status_ingested', self.set_overall_status, requires=[load_staging_raw_table], input={'status': feed_status.STATUS_INGESTED}) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', reload='reload', licensor='licensor'))) @property def fetch_from_http(self): """Move file from HTTP URL to S3 archive bucket.""" return self.create( name='fetch_from_http', tasks=base.SyncRunner( tasks.fetch_from_http.fill( namespace='fetch_from_http', feed_name='bootstrap.feed_name', date='bootstrap.date', year='bootstrap.year', week_number='bootstrap.week_number', source_url='bootstrap.source_url', s3_archive_path='bootstrap.s3_archive_path', drop_file_name='bootstrap.drop_file_name', fixed_file_name='bootstrap.fixed_file_name', ))) load_staging_raw_table = StageLoader.load_activity( feed_name=config.feed_name, secrets_path=config.secrets_path, sql_loader=SQLLoader(__file__), requirements=dict( date='bootstrap.date', feed_name='bootstrap.feed_name', source_files_dict='fetch_from_http.source_files_dict', s3_dir_path='bootstrap.s3_full_path_fixed', staging_raw_table_name='bootstrap.staging_raw_table'))