"""Cable Ingestion ETL Decider.""" from garcon_contrib.dynamo_feed_status import garcon_feed_status from garcon.param import StaticParam from flows import config as base_config from flows import flow from flows import runner from flows.cable_ingestion import config from flows.cable_ingestion import queries from flows.cable_ingestion import tasks class Flow(flow.FlowBase): """Flow class for the Cable Ingestion ETL decider.""" def decider(self, schedule): """Schedule the next appropriate activity for the SWF execution.""" extract_from_source = schedule( 'extract_from_source', self.extract_from_source_activity) valid_files = extract_from_source.result.get( 'validate_source_files.valid_files') if not valid_files: return create_temp_raw_table_activity = schedule( 'create_temp_raw_table', self.create_temp_raw_table_activity, requires=[extract_from_source]) insert_to_temp_raw_table_activity = schedule( 'insert_to_temp_raw_table', self.insert_to_temp_raw_table_activity, requires=[create_temp_raw_table_activity]) insert_select_to_raw_table_activity = schedule( 'insert_select_to_raw_table', self.insert_select_to_raw_table_activity, requires=[insert_to_temp_raw_table_activity]) update_etl_status_activity = schedule( 'update_etl_status', self.update_etl_status_activity, requires=[insert_select_to_raw_table_activity]) schedule( 'update_dashboard_status', self.update_dashboard_status_activity, requires=[update_etl_status_activity]) @property def extract_from_source_activity(self): """Activity to move the data from the raw source into S3. Returns: garcon.activity.Activity instance. """ return self.create( name='extract_from_source', retry=10, tasks=runner.Sync( tasks.download_from_server.fill( correlation_id='correlation_id', date_end='date_end', date_start='date_start', sftp_credentials=StaticParam( config.SFTP_RENTRAK_CREDENTIALS), namespace='download_from_server'), tasks.validate_source_files.fill( correlation_id='correlation_id', directory='download_from_server.directory', namespace='validate_source_files'), tasks.upload_to_s3_archive.fill( bucket=StaticParam(base_config.DATA_BUCKET), correlation_id='correlation_id', destination=StaticParam(config.RAW_ARCHIVE_DESTINATION), namespace='upload_to_s3_archive', valid_files='validate_source_files.valid_files'), tasks.cleanup_local_raw.fill( correlation_id='correlation_id', directory='download_from_server.directory', namespace='cleanup_local_raw', valid_files='validate_source_files.valid_files'))) @property def create_temp_raw_table_activity(self): """Activity to transform the archived raw csv in s3 to the datastore. Returns: garcon.activity.Activity instance. """ return self.create( name='create_temp_raw_table', retry=10, tasks=runner.Sync( tasks.create_temp_raw_table.fill( correlation_id='correlation_id', namespace='create_temp_raw_table', sql=StaticParam(queries.CREATE_TEMP_TABLE), table_name=StaticParam(queries.TEMP_TABLE_NAME)))) @property def insert_to_temp_raw_table_activity(self): """Activity to transform the archived raw csv in s3 to the datastore. Returns: garcon.activity.Activity instance. """ return self.create( name='insert_to_temp_raw_table', retry=10, tasks=runner.Sync( tasks.insert_to_temp_raw_table.fill( correlation_id='correlation_id', delete=StaticParam(queries.DELETE_FROM_TABLE_BY_DATE), insert=StaticParam(queries.INSERT_TO_TEMP_TABLE), namespace='insert_to_temp_raw_table', s3_files='upload_to_s3_archive.s3_files', table_name='create_temp_raw_table.table_name'))) @property def insert_select_to_raw_table_activity(self): """Activity to transform the archived raw csv in s3 to the datastore. Returns: garcon.activity.Activity instance. """ return self.create( name='insert_select_to_raw_table', retry=10, tasks=runner.Sync( tasks.insert_select_to_raw_table.fill( correlation_id='correlation_id', namespace='insert_select_to_raw_table', delete=StaticParam(queries.DELETE_FROM_TABLE_BY_DATE), drop=StaticParam(queries.DROP_TEMP_TABLE), insert=StaticParam(queries.INSERT_SELECT_TO_RAW_TABLE), raw_table_name=StaticParam(queries.RAW_TABLE_NAME), table_date_range=StaticParam( queries.TABLE_DATE_RANGE), temp_table_name='create_temp_raw_table.table_name'))) @property def update_etl_status_activity(self): """Set the etl log status to COMPLETED. Returns: garcon.activity.Activity instance. """ return self.create( name='update_etl_status', retry=10, tasks=runner.Sync( tasks.set_final_status.fill( correlation_id='correlation_id'))) @property def update_dashboard_status_activity(self): """Set the global data dashboard status to INGESTED. Returns: garcon.activity.Activity instance. """ return self.create( name='update_dashboard_status', retry=10, tasks=runner.Sync( tasks.update_dashboard_status.fill( correlation_id='correlation_id', status=StaticParam( garcon_feed_status.STATUS_INGESTED))))