"""Spotify Marquee Ingestion Workflow.""" from garcon import param from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows import base from feed_ingestion.flows.spotify_marquee import config from feed_ingestion.flows.spotify_marquee import tasks from feed_ingestion.tasks import \ feed_status_tasks, load_raw_table_tasks_sf, overall_status_tasks class Flow(base.FlowLicensor, base.FlowConfigMixin, base.FlowLoadRawMixinSF, base.FlowLoadFactMixinSF): """Class representing the workflow.""" def __init__(self): """Initialize flow object.""" super(Flow, self).__init__(feed_name=config.feed_name, version='1.0') self.timeout = 60 * 60 * 5 # set "Execution Start To Close Timeout" def decider(self, schedule, context): """Activity decider. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) # Stop flow if feed already ingested if bootstrap.result.get('bootstrap.stop') is True: return grab_drop_file = schedule( 'grab_drop_file', self.grab_drop_file, requires=[bootstrap]) # Stop if file does not exist if grab_drop_file.result.get('grab_drop_file.stop') is True: return create_temp_staging_raw_table = schedule( 'create_temp_staging_raw_table', self.create_temp_staging_raw_table, requires=[grab_drop_file]) load_temp_staging_raw_table = schedule( 'load_temp_staging_raw_table', self.load_temp_staging_raw_table, requires=[create_temp_staging_raw_table]) populate_staging_raw = schedule( 'populate_staging_raw', self.populate_staging_raw, requires=[load_temp_staging_raw_table]) drop_temp_stage_table = schedule( 'drop_temp_stage_table', self.drop_temp_stage_table, requires=[populate_staging_raw]) mark_ingested_file = schedule( 'mark_ingested_file', self.mark_ingested_file, requires=[drop_temp_stage_table] ) schedule( 'set_status_to_ingested', self.set_status_to_ingested, requires=[mark_ingested_file], ) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', report_name='report_name', licensor='licensor', reload='reload', ))) @property def grab_drop_file(self): """Grab files and upload to archive bucket on S3 for sme.""" return self.create( name='grab_drop_file', tasks=base.AsyncRunner( tasks.grab_drop_file.fill( namespace='grab_drop_file', feed_name='bootstrap.feed_name', date='bootstrap.date', file_name='bootstrap.file_name', s3_drop_path='bootstrap.s3_drop_path', s3_archive_path='bootstrap.s3_archive_path', ))) @property def create_temp_staging_raw_table(self): """Create temp staging raw table.""" return self.create( name='create_temp_staging_raw_tables', schedule_to_start=48000, tasks=base.SyncRunner( load_raw_table_tasks_sf.create_temp_staging_raw_table.fill( namespace='create_temp_staging_raw_tables', date='bootstrap.date', sfdb_params='sfdb_params', feed_name=param.StaticParam(config.feed_name), secrets_path=param.StaticParam(config.secrets_path), temp_staging_raw_table='bootstrap.temp_table_name', kwargs='bootstrap.sf_kwargs'))) @property def load_temp_staging_raw_table(self): """Load temp staging raw table.""" return self.create( name='load_temp_staging_raw_table', schedule_to_start=48000, tasks=base.SyncRunner( load_raw_table_tasks_sf.load_temp_staging_raw_table.fill( namespace='load_temp_staging_raw_table', aws=param.StaticParam(self.conf_aws), date='bootstrap.date', sfdb_params='sfdb_params', feed_name=param.StaticParam(config.feed_name), secrets_path=param.StaticParam(config.secrets_path), kwargs='bootstrap.sf_kwargs', key_dir='bootstrap.s3_archive_path', temp_staging_raw_table='bootstrap.temp_table_name', error_limit=param.StaticParam(config.snowflake_error_limit) ))) @property def populate_staging_raw(self): """Join temp tables and unload data in the staging_raw table.""" return self.create( name='load_staging_raw', schedule_to_start=48000, tasks=base.SyncRunner( tasks.load_staging_raw_table.fill( namespace='load_staging_raw_table', date='bootstrap.date', temp_staging_raw_table='bootstrap.temp_table_name', filename='bootstrap.file_name', sfdb_params='sfdb_params', staging_raw_table='bootstrap.staging_raw_table', report_name='bootstrap.report_name', feed_name=param.StaticParam(config.feed_name), secrets_path=param.StaticParam(config.secrets_path), licensor='bootstrap.licensor'))) @property def drop_temp_stage_table(self): """Drop the staging_raw temp table.""" return self.create( name='drop_temp_stage_table', schedule_to_start=48000, tasks=base.SyncRunner( tasks.drop_temp_table.fill( namespace='drop_temp_table', temp_table_name='bootstrap.temp_table_name', sfdb_params='sfdb_params', secrets_path=param.StaticParam(config.secrets_path), report_name='bootstrap.report_name'))) @property def mark_ingested_file(self): """Put list of ingested files in DynamoDB.""" return self.create( name='mark_ingested_files', tasks=base.SyncRunner( feed_status_tasks.mark_ingested_files.fill( namespace='mark_ingested_files', feed_name='bootstrap.feed_name', date='bootstrap.date', source_files_dict='bootstrap.source_files_dict'))) @property def set_status_to_ingested(self): """Set overall feed status to INGESTED.""" return self.create( name='set_status_to_ingested', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_ingested', feed_name='bootstrap.feed_name', date='bootstrap.date', set_status_once=param.StaticParam(True), status=param.StaticParam( garcon_feed_status.STATUS_INGESTED ), ), ), )