"""Class SpotifyArtificialStreamsFlow represents spotify fraud ETL.""" from garcon.param import StaticParam from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows import base from feed_ingestion.flows.spotify_artificial_streams import config from feed_ingestion.flows.spotify_artificial_streams import tasks from feed_ingestion.tasks import load_raw_table_tasks_sf from feed_ingestion.tasks import overall_status_tasks from feed_ingestion.tasks import validate_raw_data_tasks_sf class Flow( base.FlowBase, base.FlowConfigMixin, ): """spotify_artificial_streams workflow class.""" def __init__(self): """Initialize flow object.""" super().__init__( feed_name=config.feed_name, version=config.feed_version, ) self.timeout = config.timeout_between_executions_sec def decider(self, schedule): """Activity decider. Args: schedule (Callable): The scheduler method. """ # 1: Check if the ELT run has already been done check_feed_status_activity = schedule( 'check_feed_status_activity', self.check_feed_status_activity, requires=[], ) # 2: Stop execution if it is not reload and the ELT run has been done if check_feed_status_activity.result.get( 'check_feed_status_activity.stop', ): return # 3: Prepare ELT inputs bootstrap_activity = schedule( 'bootstrap', self.bootstrap_activity, requires=[check_feed_status_activity], ) # 4: Copy blob to archive folder prefix copy_blob_to_archive_activity = schedule( 'copy_blob_to_archive', self.copy_blob_to_archive_activity, requires=[bootstrap_activity], ) # 5: Stop execution on any archive blobs copy error if copy_blob_to_archive_activity.result.get( 'copy_blob_to_archive.stop', ): return # 6: Create temp staging raw table create_temp_staging_raw_table_activity = schedule( 'create_temp_staging_raw_table', self.create_temp_staging_raw_table_activity, requires=[copy_blob_to_archive_activity], ) # 7: Load data to temp staging raw table from S3 bucket load_temp_staging_raw_table_activity = schedule( 'load_temp_staging_raw_table', self.load_temp_staging_raw_table_activity, requires=[create_temp_staging_raw_table_activity], ) # 8: Load data to staging raw table load_staging_raw_table_activity = schedule( 'load_staging_raw_table', self.load_staging_raw_table_activity, requires=[load_temp_staging_raw_table_activity], ) # 9: Set overall status as INGESTED for the flow schedule( 'set_overall_status_ingested', self.set_overall_status, requires=[load_staging_raw_table_activity], input={'status': garcon_feed_status.STATUS_INGESTED}, ) @property def bootstrap_activity(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', schedule_to_start=config.default_schedule_to_start, tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', reload='reload', ), ), ) @property def check_feed_status_activity(self): """Check and reset feed status if it is needed.""" return self.create( name='check_feed_status', schedule_to_start=config.default_schedule_to_start, tasks=base.SyncRunner( tasks.check_feed_status.fill( namespace='check_feed_status', date='context_date', reload='reload', ), ), ) @property def copy_blob_to_archive_activity(self): """Copy blobs to process from drop bucket to archive one.""" return self.create( name='copy_blob_to_archive', schedule_to_start=config.default_schedule_to_start, tasks=base.SyncRunner( tasks.copy_blob_to_archive.fill( namespace='copy_blob_to_archive', feed_name='bootstrap.feed_name', date='bootstrap.date', archive_blob_prefix='bootstrap.archive_blob_prefix', drop_bucket_name='bootstrap.drop_bucket_name', drop_blob_path='bootstrap.drop_blob_path', drop_blob_wildcard='bootstrap.drop_blob_wildcard', reload='reload', ), ), ) @property def create_temp_staging_raw_table_activity(self): """Create temp staging raw table.""" return self.create( name='create_temp_staging_raw_table', schedule_to_start=config.default_schedule_to_start, tasks=base.SyncRunner( load_raw_table_tasks_sf.create_temp_staging_raw_table.fill( namespace='create_temp_staging_raw_table', date='bootstrap.date', sfdb_params='sfdb_params', feed_name='bootstrap.feed_name', secrets_path=StaticParam(config.secrets_path), temp_staging_raw_table='bootstrap.temp_staging_raw_name', ), ), ) @property def load_staging_raw_table_activity(self): """Load permanent staging raw table.""" activity_name = 'load_staging_raw_table' return self.create( name=activity_name, schedule_to_start=config.default_schedule_to_start, tasks=base.SyncRunner( load_raw_table_tasks_sf.load_staging_raw_table.fill( namespace=activity_name, date='bootstrap.date', feed_name='bootstrap.feed_name', secrets_path=StaticParam(config.secrets_path), sfdb_params='sfdb_params', staging_raw_table='bootstrap.staging_raw_table', temp_staging_raw_table='bootstrap.temp_staging_raw_name', kwargs='bootstrap.staging_raw_load_kwargs', ), overall_status_tasks.set_overall_status.fill( date='bootstrap.date', feed_name='bootstrap.feed_name', set_status_once=StaticParam(True), status=StaticParam( garcon_feed_status.STATUS_POPULATED_RAW_TABLE, ), ), ), ) @property def load_temp_staging_raw_table_activity(self): """Load temp staging raw table.""" return self.create( name='load_temp_staging_raw_table', schedule_to_start=config.default_schedule_to_start, tasks=base.SyncRunner( validate_raw_data_tasks_sf.load_temp_staging_raw_table.fill( namespace='load_temp_staging_raw_table', date='bootstrap.date', sfdb_params='sfdb_params', feed_name='bootstrap.feed_name', secrets_path=StaticParam(config.secrets_path), kwargs='bootstrap.temp_staging_raw_load_kwargs', key_dir='bootstrap.archive_blob_prefix', temp_staging_raw_table='bootstrap.temp_staging_raw_name', error_limit='bootstrap.snowflake_error_limit', ), ), )