"""Projections ETL Decider.""" from garcon.param import StaticParam from flows import config as base_config from flows import runner from flows.flow import FlowBase from flows.projections import tasks class Flow(FlowBase): """Projections SWF flow class.""" def decider(self, schedule): """Schedule the next appropriate activity for the SWF execution.""" bootstrap = schedule('bootstrap', self.bootstrap) determine_anchor_date = schedule( 'determine_anchor_date', self.determine_anchor_date_activity, requires=[bootstrap]) no_anchor_date = determine_anchor_date.result.get('stop') if no_anchor_date: return create_temp_table = schedule( 'create_temp_table', self.create_temp_table, requires=[determine_anchor_date]) download_to_db = schedule( 'download_to_db', self.download_to_db, requires=[create_temp_table]) insert_new_data = schedule( 'insert_new_data', self.insert_new_data, requires=[download_to_db]) drop_temp_table = schedule( 'drop_temp_table', self.drop_temp_table, requires=[insert_new_data]) archive_dropped_csv = schedule( 'archive_dropped_csv', self.archive_dropped_csv_activity, requires=[drop_temp_table]) set_final_status = schedule( 'set_final_status', self.set_final_status, requires=[archive_dropped_csv]) schedule( 'send_success_notification', self.send_success_notification, requires=[set_final_status]) @property def bootstrap(self): """Prepare parameters for flow execution. Returns: Activity: garcon.activity.Activity instance. """ return self.create( name='bootstrap', retry=10, tasks=runner.Sync( tasks.bootstrap.fill( correlation_id='correlation_id', workflow_run_id='execution.run_id', s3_bucket='s3_bucket', s3_key='s3_key', projection_type='projection_type'))) @property def archive_dropped_csv_activity(self): """Accept and archive the dropped csv file. Returns: Activity: garcon.activity.Activity instance. """ return self.create( name='archive_dropped_csv', retry=10, tasks=runner.Sync( tasks.archive_dropped_csv.fill( archive_path='s3_archive_path', bucket=StaticParam(base_config.DATA_BUCKET), correlation_id='correlation_id', drop_url='drop_url', log_table='log_table'))) @property def determine_anchor_date_activity(self): """Determine the anchor date for a upc. Returns: Activity: garcon.activity.Activity instance. """ return self.create( name='determine_anchor_date', retry=10, tasks=runner.Sync( tasks.determine_anchor_date.fill( correlation_id='correlation_id', upc='upc', log_table='log_table', projection_type='projection_type'))) @property def create_temp_table(self): """Create temp table. Returns: Activity: garcon.activity.Activity instance. """ return self.create( name='create_temp_table', retry=10, tasks=runner.Sync( tasks.create_temp_table.fill( correlation_id='correlation_id', log_table='log_table', revenue_table='revenue_table'))) @property def download_to_db(self): """Download csv data to temp table. Returns: Activity: garcon.activity.Activity instance. """ return self.create( name='download_to_db', retry=10, tasks=runner.Sync( tasks.download_to_db.fill( anchor_date='anchor_date', correlation_id='correlation_id', drop_url='drop_url', temp_table_name='temp_table_name', upc='upc', log_table='log_table', transaction_types='transaction_types'))) @property def insert_new_data(self): """Insert recently ingested data to prod table. Returns: Activity: garcon.activity.Activity instance. """ return self.create( name='insert_new_data', retry=10, tasks=runner.Sync( tasks.insert_new_data.fill( correlation_id='correlation_id', temp_table_name='temp_table_name', log_table='log_table', revenue_table='revenue_table'))) @property def drop_temp_table(self): """Drop temp table. Returns: Activity: garcon.activity.Activity instance. """ return self.create( name='drop_temp_table', retry=10, tasks=runner.Sync( tasks.drop_temp_table.fill( correlation_id='correlation_id', temp_table_name='temp_table_name', log_table='log_table'))) @property def set_final_status(self): """Set status INGESTED to the db log. Returns: Activity: garcon.activity.Activity instance. """ return self.create( name='set_final_status', retry=10, tasks=runner.Sync( tasks.set_final_status.fill( correlation_id='correlation_id', log_table='log_table'))) @property def send_success_notification(self): """Send notification about success ingestion. Returns: Activity: garcon.activity.Activity instance. """ return self.create( name='send_success_notification', retry=10, tasks=runner.Sync( tasks.send_success_notification.fill( correlation_id='correlation_id', upc='upc')))