"""Apple Id Mapping Sme Ingestion Workflow.""" from garcon.param import StaticParam from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows import base from feed_ingestion.flows.apple_id_mapping_sme import config from feed_ingestion.flows.apple_id_mapping_sme import tasks from feed_ingestion.tasks import load_raw_table_tasks_sf from feed_ingestion.tasks import overall_status_tasks from feed_ingestion.tasks import validate_raw_data_tasks_sf class Flow(base.FlowBase, base.FlowConfigMixin, base.FlowLoadRawMixinSF): """Class representing the workflow.""" def __init__(self): """Initialize flow object.""" super(Flow, self).__init__(feed_name=config.feed_name, version='1.0') def decider(self, schedule): """Activity decider. Args: schedule (callable): The scheduler method. """ check_feed_status = schedule( 'check_feed_status', self.check_feed_status) if check_feed_status.result.get('check_feed_status.stop'): return bootstrap = schedule( 'bootstrap', self.bootstrap, requires=[check_feed_status]) if bootstrap.result.get('bootstrap.stop'): return fetch_from_drop_location = schedule( 'fetch_from_drop_location', self.fetch_from_drop_location, requires=[bootstrap]) # Stop flow if files aren't available if fetch_from_drop_location.result.get( 'fetch_from_drop_location.stop'): return # Status updated to DOWNLOADED if files are present, else NOT_AVAILABLE set_status_to_downloaded = schedule( 'set_status_to_downloaded', self.set_status_to_downloaded, requires=[fetch_from_drop_location]) create_temp_staging_raw_table = schedule( 'create_temp_staging_raw_table', self.create_temp_staging_raw_table, requires=[set_status_to_downloaded]) load_temp_staging_raw_table = schedule( 'load_temp_staging_raw_table', self.load_temp_staging_raw_table, requires=[create_temp_staging_raw_table]) load_staging_raw_table = schedule( 'load_staging_raw_table', self.load_staging_raw_table, requires=[load_temp_staging_raw_table]) set_status_to_populated_raw_table = schedule( 'set_status_to_populated_raw_table', self.set_status_to_populated_raw_table, requires=[load_staging_raw_table]) update_apple_id_mapping = schedule( 'update_apple_id_mapping', self.update_apple_id_mapping, requires=[set_status_to_populated_raw_table]) schedule( 'set_overall_status_ingested', self.set_overall_status_ingested, requires=[update_apple_id_mapping]) @property def check_feed_status(self): """Check and reset feed status if it is needed.""" return self.create( name='check_feed_status', tasks=base.SyncRunner( tasks.check_feed_status.fill( namespace='check_feed_status', date='context_date', reload='reload'))) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', reload='reload', snowflake_error_limit='snowflake_error_limit'))) @property def fetch_from_drop_location(self): """Move file from S3 SME location to S3 archive bucket.""" return self.create( name='fetch_from_drop_location', tasks=base.SyncRunner( tasks.fetch_from_drop_location.fill( namespace='fetch_from_drop_location', date='bootstrap.date', feed_name='bootstrap.feed_name', s3_archive_path='bootstrap.s3_archive_path', reports='bootstrap.reports'))) @property def create_temp_staging_raw_table(self): """Create temp staging raw table.""" return self.create( name='create_temp_staging_raw_table', generators=[self.temp_staging_tables_generator], tasks=base.SyncRunner( load_raw_table_tasks_sf.create_temp_staging_raw_table.fill( namespace='create_temp_staging_raw_table', date='bootstrap.date', sfdb_params='sfdb_params', feed_name='bootstrap.feed_name', secrets_path=StaticParam(config.secrets_path), temp_staging_raw_table='temp_staging_raw_table', kwargs='kwargs'))) @property def load_temp_staging_raw_table(self): """Load temp staging raw table.""" return self.create( name='load_temp_staging_raw_table', generators=[self.temp_staging_tables_generator], tasks=base.SyncRunner( validate_raw_data_tasks_sf.load_temp_staging_raw_table.fill( namespace='load_temp_staging_raw_table', date='bootstrap.date', sfdb_params='sfdb_params', feed_name='bootstrap.feed_name', secrets_path=StaticParam(config.secrets_path), kwargs='kwargs', key_dir='key_dir', temp_staging_raw_table='temp_staging_raw_table', error_limit='bootstrap.snowflake_error_limit'))) @property def load_staging_raw_table(self): """Load permanent staging raw table.""" activity_name = 'load_staging_raw_table' return self.create( name=activity_name, generators=[self.temp_staging_tables_generator], tasks=base.SyncRunner( load_raw_table_tasks_sf.load_staging_raw_table.fill( namespace=activity_name, date='bootstrap.date', feed_name='bootstrap.feed_name', secrets_path=StaticParam(config.secrets_path), sfdb_params='sfdb_params', staging_raw_table='staging_raw_table', temp_staging_raw_table='temp_staging_raw_table', clean=StaticParam('False'), set_complete=StaticParam('False'), kwargs='kwargs'))) @property def update_apple_id_mapping(self): """Update apple_id_mapping with new data.""" return self.create( name='update_apple_id_mapping', tasks=base.SyncRunner( tasks.update_apple_id_mapping.fill( namespace='update_apple_id_mapping', date='bootstrap.date', feed_name='bootstrap.feed_name'))) @property def set_status_to_downloaded(self): """Set overall feed status to DOWNLOADED.""" return self.create( name='set_status_to_downloaded', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_downloaded', feed_name='bootstrap.feed_name', date='bootstrap.date', status=StaticParam( garcon_feed_status.STATUS_DOWNLOADED)))) @property def set_status_to_populated_raw_table(self): """Set overall feed status to POPULATED_RAW_TABLE.""" return self.create( name='set_status_to_downloaded', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_downloaded', feed_name='bootstrap.feed_name', date='bootstrap.date', status=StaticParam( garcon_feed_status.STATUS_POPULATED_RAW_TABLE)))) @property def set_overall_status_ingested(self): """Set overall feed status INGESTED if all reports are completed.""" return self.create( name='set_overall_status_ingested', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( feed_name='bootstrap.feed_name', date='bootstrap.date', set_status_once=StaticParam(True), status=StaticParam(garcon_feed_status.STATUS_INGESTED)))) def temp_staging_tables_generator(self, context): """Generate parameters for temporary staging tables. Used by the create_temp_staging_raw_table, load_temp_staging_raw_table activity and drop_temp_staging_table. Args: context (dict): The current context. Yields: dict: Dictionary of temporary staging table names. """ reports = context['bootstrap.reports'] s3_archive_path = context['bootstrap.s3_archive_path'] for report, description in reports.items(): key_dir = 's3://{bucket}/{s3_archive_path}{filename}'.format( bucket=config.archive_bucket, s3_archive_path=s3_archive_path, filename=description['filename']) yield dict( temp_staging_raw_table=description['temp_table'], key_dir=key_dir, staging_raw_table=config.reports[report], kwargs=dict(report=report))