"""FeatureFMFacebook Ingestion Workflow.""" from garcon.param import StaticParam from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.flows import base from feed_ingestion.flows.feature_fm_facebook import config from feed_ingestion.flows.feature_fm_facebook import tasks from feed_ingestion.flows.feature_fm_facebook.stage_loader \ import FeatureFmFacebookSL class Flow(base.FlowBase, base.FlowConfigMixin, base.FlowLoadFactMixinSF): """Class representing the workflow.""" def __init__(self): """Initialize flow object.""" self.timeout = 21600 # 6 hours (60 *60 * 6) super(Flow, self).__init__(feed_name=config.feed_name, version='1.0') def decider(self, schedule, context): """Activity decider. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) # Stop flow if feed already ingested if bootstrap.result.get('bootstrap.stop'): return # probably it is needed by DS-4409 # when it will be decided what will be with historical data # preprocess_source_files = schedule( # 'preprocess_source_files', # self.preprocess_source_files_activity, # requires=[bootstrap] # ) load_staging_raw_table = schedule( 'load_staging_raw_table', self.load_staging_raw_table, requires=[bootstrap] ) load_staging_fact_table = schedule( 'load_staging_fact_table', self.load_staging_fact_table, requires=[load_staging_raw_table]) load_fact_tables = schedule( 'load_fact_tables', self.load_fact_tables, requires=[load_staging_fact_table]) schedule( 'deduplicate_fact_table', self.deduplicate_fact_table, requires=[load_fact_tables]) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', reload='reload', report_type='report_type'))) @property def preprocess_source_files_activity(self): """Preprocess source files.""" return self.create( name='preprocess_source_files', tasks=base.SyncRunner( tasks.preprocess_source_files.fill( namespace='preprocess_source_files', date='bootstrap.date', file_name_map=( 'bootstrap.preprocess_file_name_map'), fetch_files_dict=( 'bootstrap.source_files_dict'), s3_dir_path='bootstrap.s3_dir_path' ) ) ) load_staging_raw_table = FeatureFmFacebookSL.load_activity( feed_name=config.feed_name, secrets_path=config.secrets_path, sql_loader=SQLLoader(__file__), requirements=dict( date='bootstrap.date', source_files_dict='bootstrap.source_files_dict', s3_dir_path='bootstrap.s3_dir_path', staging_raw_table_name=StaticParam( config.snowflake_table_names['staging_raw']))) @property def deduplicate_fact_table(self): """Deduplicate FACT_AD_SPEND.""" return self.create( name='deduplicate_fact_table', tasks=base.SyncRunner( tasks.deduplicate_fact_table.fill( namespace='deduplicate_fact_table', date='bootstrap.date', ) ) )