"""Music Analytics Reports Flow.""" from garcon.param import StaticParam from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows import base from feed_ingestion.flows.music_analytics_reports import config from feed_ingestion.flows.music_analytics_reports import tasks from feed_ingestion.tasks import load_raw_table_tasks_sf from feed_ingestion.tasks import overall_status_tasks class Flow(base.FlowBase, base.FlowConfigMixin, base.FlowLoadRawMixinSF): """Class representing the workflow.""" def __init__(self): """Initialize flow object.""" super(Flow, self).__init__(feed_name=config.feed_name, version='1.0') def decider(self, schedule): """Activity decider. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) if bootstrap.result.get('bootstrap.stop'): return get_report_file = schedule( 'get_report_file', self.get_report_file, requires=[bootstrap]) if get_report_file.result.get('get_report_file.stop'): return load_report_to_s3 = schedule( 'load_report_to_s3', self.load_report_to_s3, requires=[get_report_file]) if load_report_to_s3.result.get('load_report_to_s3.failed'): return set_status_to_downloaded = schedule( 'set_status_to_downloaded', self.set_status_to_downloaded, requires=[load_report_to_s3]) create_temp_staging_raw_table = schedule( 'create_temp_staging_raw_table', self.create_temp_staging_raw_table, requires=[set_status_to_downloaded]) load_temp_staging_raw_table = schedule( 'load_temp_staging_raw_table', self.load_temp_staging_raw_table, requires=[create_temp_staging_raw_table]) requires = [load_temp_staging_raw_table] file = load_report_to_s3.result.get( 'load_report_to_s3.full_s3_path') if file and file.get('errors'): create_temp_errors_table = schedule( 'create_temp_staging_raw_table', self.create_temp_errors_table, requires=requires) load_temp_errors_table = schedule( 'load_errors_table', self.load_temp_errors_table, requires=[create_temp_errors_table]) load_errors_table = schedule( 'load_errors_table', self.load_errors_table, requires=[load_temp_errors_table]) requires = [load_errors_table] load_staging_raw_table = schedule( 'load_staging_raw_table', self.load_staging_raw_table, requires=requires) schedule( 'set_overall_status_ingested', self.set_overall_status_ingested, requires=[load_staging_raw_table]) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', reload='reload', report_type='report_type'))) @property def get_report_file(self): """Get a report file and its S3 archive bucket.""" return self.create( name='get_report_file', schedule_to_start=7200, tasks=base.SyncRunner( tasks.get_report_file.fill( namespace='get_report_file', date='bootstrap.date', report_type='bootstrap.report_type'))) @property def load_report_to_s3(self): """Load Music Analytics Utils file to the S3 archive bucket.""" return self.create( name='load_report_to_s3', schedule_to_start=7200, tasks=base.SyncRunner( tasks.load_report_to_s3.fill( namespace='load_report_to_s3', file='get_report_file.file', date='bootstrap.date', feed_name='feed_name', report_type='bootstrap.report_type'))) @property def create_temp_errors_table(self): """Create temp staging raw table.""" return self.create( name='create_temp_errors_table', tasks=base.SyncRunner( load_raw_table_tasks_sf.create_temp_staging_raw_table.fill( namespace='create_temp_errors_table', date='bootstrap.date', sfdb_params='sfdb_params', feed_name='bootstrap.feed_name', secrets_path=StaticParam(config.secrets_path), temp_staging_raw_table='bootstrap.temp_errors_table', ))) @property def load_temp_staging_raw_table(self): """Load temp staging raw table.""" return self.create( name='load_temp_staging_raw_table', tasks=base.SyncRunner( load_raw_table_tasks_sf.load_temp_staging_raw_table.fill( namespace='load_temp_staging_raw_table', date='bootstrap.date', sfdb_params='sfdb_params', feed_name='bootstrap.feed_name', secrets_path=StaticParam(config.secrets_path), key_dir='load_report_to_s3.full_s3_path', temp_staging_raw_table= # noqa 'bootstrap.temp_staging_raw_table'))) @property def load_temp_errors_table(self): """Load temp errors table.""" return self.create( name='load_temp_errors_table', tasks=base.SyncRunner( load_raw_table_tasks_sf.load_temp_staging_raw_table.fill( namespace='load_temp_errors_table', date='bootstrap.date', sfdb_params='sfdb_params', feed_name='bootstrap.feed_name', secrets_path=StaticParam(config.secrets_path), key_dir='load_report_to_s3.full_s3_path', temp_staging_raw_table='bootstrap.temp_errors_table', ))) @property def load_errors_table(self): """Load temp staging raw table.""" return self.create( name='load_errors_table', tasks=base.SyncRunner( load_raw_table_tasks_sf.load_staging_raw_table.fill( namespace='load_errors_table', date='bootstrap.date', feed_name='bootstrap.feed_name', temp_staging_raw_table='bootstrap.temp_errors_table', staging_raw_table='bootstrap.errors_table', sfdb_params='sfdb_params', clean=StaticParam('True'), secrets_path=StaticParam(config.secrets_path), kwargs='load_report_to_s3.full_s3_path'))) @property def load_staging_raw_table(self): """Load permanent staging raw table.""" activity_name = 'load_staging_raw_table' return self.create( name=activity_name, tasks=base.SyncRunner( load_raw_table_tasks_sf.load_staging_raw_table.fill( namespace=activity_name, date='bootstrap.date', feed_name='bootstrap.feed_name', temp_staging_raw_table='bootstrap.temp_staging_raw_table', staging_raw_table='bootstrap.staging_raw_table', sfdb_params='sfdb_params', clean=StaticParam('True'), secrets_path=StaticParam(config.secrets_path), kwargs='load_report_to_s3.full_s3_path'))) @property def set_status_to_downloaded(self): """Set overall feed status to DOWNLOADED.""" return self.create( name='set_status_to_downloaded', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_downloaded', feed_name='bootstrap.feed_name', date='bootstrap.date', status=StaticParam( garcon_feed_status.STATUS_DOWNLOADED) ))) @property def set_overall_status_ingested(self): """Set overall feed status INGESTED if all reports are completed.""" return self.create( name='set_overall_status_ingested', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( feed_name='bootstrap.feed_name', date='bootstrap.date', set_status_once=StaticParam(True), status=StaticParam(garcon_feed_status.STATUS_INGESTED) )))