"""Digital ETL Decider.""" from garcon_contrib.aws import garcon_s3 from garcon.param import StaticParam from flows import config as base_config from flows import runner from flows.digital import queries from flows.digital import status from flows.digital import tasks from flows.flow import DatabaseParam from flows.flow import FlowBase class Flow(FlowBase): """Digital SWF flow class.""" def decider(self, schedule): """Schedule the next appropriate activity for the SWF execution.""" unload_from_datawarehouse = schedule( 'unload_from_datawarehouse', self.unload_from_datawarehouse_activity) load_to_temp_table = schedule( 'load_to_temp_table', self.load_to_temp_table_activity, requires=[unload_from_datawarehouse]) clean_unload_files = schedule( 'cleanup_unload_files', self.cleanup_unload_files_activity, requires=[load_to_temp_table]) move_temp_table_to_digital_revenue = schedule( 'move_temp_table_to_digital_revenue', self.move_temp_table_to_digital_revenue_activity, requires=[clean_unload_files]) send_notification = schedule( 'send_notification', self.send_notification_activity, requires=[move_temp_table_to_digital_revenue]) schedule( 'finalize_etl_status', self.finalize_etl_status_activity, requires=[send_notification]) @property def unload_from_datawarehouse_activity(self): """Describe the unload tasks. Returns: garcon.activity.Activity instance. """ return self.create( name='unload_from_datawarehouse', retry=10, tasks=runner.Sync( tasks.unload_from_datawarehouse.fill( bucket=StaticParam(base_config.DATA_BUCKET), correlation_id='correlation_id', date_end='date_end', date_start='date_start', namespace='unload_from_datawarehouse', upcs=DatabaseParam('upcs')))) @property def load_to_temp_table_activity(self): """Task description for loading from S3 to a temp datastore table. Returns: garcon.activity.Activity instance. """ return self.create( name='load_to_temp_table', retry=10, tasks=runner.Sync( tasks.create_temp_table.fill( correlation_id='correlation_id', create=StaticParam(queries.CREATE_TEMP_TABLE), namespace='create_temp_table', temp_table_name=StaticParam(queries.TEMP_TABLE_NAME)), tasks.insert_to_temp_table.fill( bucket=StaticParam(base_config.DATA_BUCKET), correlation_id='correlation_id', insert=StaticParam(queries.INSERT_TO_TEMP_TABLE), batch_count='unload_from_datawarehouse.batch_count', temp_table_name='create_temp_table.table_name'))) @property def cleanup_unload_files_activity(self): """Task description for cleaning up s3 files from redshift unload. Returns: garcon.activity.Activity instance. """ return self.create( name='cleanup_unload_files', retry=10, tasks=runner.Sync( s3.remove_files_from_path.fill( path='unload_from_datawarehouse.unload_path'), tasks.update_etl_status.fill( correlation_id='correlation_id', status=StaticParam(status.UNLOAD_DATA_CLEANED)))) @property def move_temp_table_to_digital_revenue_activity(self): """Task description for moving temp table rows to live. Returns: garcon.activity.Activity instance. """ return self.create( name='move_temp_table_to_digital_revenue', retry=10, tasks=runner.Sync( tasks.move_temp_table_to_digital_revenue.fill( correlation_id='correlation_id', date_end='date_end', date_start='date_start', temp_table_name='create_temp_table.table_name', upcs=DatabaseParam('upcs')))) @property def send_notification_activity(self): """Task description for notifying any listening apps. Returns: garcon.activity.Activity instance. """ return self.create( name='send_notification', retry=10, tasks=runner.Async( tasks.send_sns.fill( correlation_id='correlation_id', date_end='date_end', date_start='date_start', upcs='upcs'), tasks.queue_build_cache.fill( correlation_id='correlation_id', date_end='date_end', date_start='date_start', upcs='upcs'))) @property def finalize_etl_status_activity(self): """Complete the ETL by setting the status to COMPLETED. Returns: garcon.activity.Activity instance. """ return self.create( name='finalize_etl_status', retry=10, tasks=runner.Sync( tasks.set_dynamo_status.fill( correlation_id='correlation_id', date='date_end'), tasks.update_etl_status.fill( correlation_id='correlation_id', status=StaticParam(status.COMPLETED))))