"""GfK Streaming Ingestion Workflow.""" from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.flows import base from feed_ingestion.flows.gfk_streaming import config, tasks from feed_ingestion.flows.gfk_streaming.snowflake_executor import ( GfKStreamingSFExecutor, ) from feed_ingestion.flows.gfk_streaming.stage_loader import GfKStreamingSL sql_loader = SQLLoader(__file__) class Flow(base.FlowBase, base.FlowConfigMixin): """Class representing the workflow.""" def __init__(self): """Initialize flow object.""" super(Flow, self).__init__(feed_name=config.feed_name, version='1.0') def decider(self, schedule): """Activity decider. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) if bootstrap.result.get('bootstrap.stop'): return fetch_from_ftps = schedule( 'fetch_from_ftps', self.fetch_from_ftps, requires=[bootstrap] ) if fetch_from_ftps.result.get('fetch_from_ftps.stop'): return load_staging_raw_table = schedule( 'load_staging_raw_table', self.load_staging_raw_table, requires=[fetch_from_ftps], ) delete_processed_file = schedule( 'delete_processed_file', self.delete_processed_file, requires=[load_staging_raw_table], ) schedule( 'set_overall_status_ingested', self.set_status_to_ingested, requires=[delete_processed_file], ) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', reload='reload', ) ), ) @property def fetch_from_ftps(self): """Download ZIP from FTPS, extract CSV, upload to S3.""" return self.create( name='fetch_from_ftps', tasks=base.SyncRunner( tasks.fetch_from_ftps.fill( namespace='fetch_from_ftps', feed_name='bootstrap.feed_name', date='bootstrap.date', filename='bootstrap.filename', archive_path='bootstrap.archive_path', processed_path='bootstrap.processed_path', ) ), ) @property def delete_processed_file(self): """Delete the extracted CSV from S3.""" return self.create( name='delete_processed_file', tasks=base.SyncRunner( tasks.delete_processed_file.fill( namespace='delete_processed_file', processed_path='bootstrap.processed_path', ) ), ) load_staging_raw_table = GfKStreamingSL.load_activity( secrets_path=config.secrets_path, sql_loader=sql_loader, executor_class=GfKStreamingSFExecutor, requirements=dict( date='bootstrap.date', feed_name='bootstrap.feed_name', staging_raw_table_name='bootstrap.staging_raw_table', s3_dir_path='bootstrap.s3_dir_path', ), )