"""Theatrical ETL Flow.""" from garcon.param import StaticParam from owslogger import logger from flows import log from flows import runner from flows.flow import DatabaseParam from flows.flow import FlowBase from flows.theatrical import config as config from flows.theatrical import tasks class InvalidDataException(Exception): """Class for data errors.""" pass class Flow(FlowBase): """Theatrical ETL Flow.""" def decider(self, schedule): """Schedule the next appropriate activity for the SWF execution.""" bootstrap = schedule('bootstrap', self.bootstrap) if bootstrap.result.get('stop'): return download_to_db = schedule( 'download_to_db', self.download_to_db, requires=[bootstrap]) if download_to_db.result.get('stop'): correlation_id = download_to_db.result.get('correlation_id') error_file = download_to_db.result.get('error_file') error_details = download_to_db.result.get('error_details') error_message = 'Invalid file: {}. details: {}'.format( error_file, error_details) base_log = log.get_logger() task_logger = logger.OwsLoggingAdapter( base_log, {'correlation_id': correlation_id}) task_logger.error(error_message) return clean_dynamo_status = schedule( 'clean_dynamo_status', self.clean_dynamo_status, requires=[download_to_db]) create_theatrical_revenue_temp_table = schedule( 'create_theatrical_revenue_temp_table', self.create_theatrical_revenue_temp_table, requires=[clean_dynamo_status]) insert_new_data = schedule( 'insert_new_data', self.insert_new_data, requires=[create_theatrical_revenue_temp_table]) move_source_files_to_archive = schedule( 'move_source_files_to_archive', self.move_source_files_to_archive, requires=[insert_new_data]) send_notification = schedule( 'send_notification', self.send_notification, requires=[move_source_files_to_archive]) schedule( 'set_status', self.set_status, requires=[send_notification]) @property def bootstrap(self): """Task description for prepare parameters for flow execution. Returns: Activity: garcon.activity.Activity instance. """ return self.create( name='bootstrap', retry=10, tasks=runner.Sync( tasks.bootstrap.fill( correlation_id='correlation_id', date_start='date_start', date_end='date_end', with_archive='with_archive', unload_bucket=StaticParam(config.UNLOAD_BUCKET), unload_prefix=StaticParam(config.UNLOAD_PREFIX), archive_bucket=StaticParam(config.ARCHIVE_BUCKET), archive_prefix=StaticParam(config.ARCHIVE_PREFIX)))) @property def clean_dynamo_status(self): """Clean DynamoDB statuses for files. Returns: Activity: garcon.activity.Activity instance. """ return self.create( name='clean_dynamo_status', retry=10, tasks=runner.Sync( tasks.clean_dynamo_status.fill( correlation_id='correlation_id', source_files='source_files'))) @property def download_to_db(self): """Task description for unload CSV from s3 to DB. Returns: Activity: garcon.activity.Activity instance. """ return self.create( name='download_to_db', retry=10, tasks=runner.Sync( tasks.download_to_db.fill( correlation_id='correlation_id', source_files='source_files'))) @property def create_theatrical_revenue_temp_table(self): """Create temp revenue table. Returns: Activity: garcon.activity.Activity instance. """ return self.create( name='create_theatrical_revenue_temp_table', retry=10, tasks=runner.Sync( tasks.create_theatrical_revenue_temp_table.fill( correlation_id='correlation_id'), tasks.insert_to_theatrical_revenue_temp_table.fill( correlation_id='correlation_id', temp_table_name='temp_table_name', source_files='source_files', upcs=DatabaseParam('upcs')))) @property def insert_new_data(self): """Task description for copying new data to the prod table. Returns: Activity: garcon.activity.Activity instance. """ return self.create( name='insert_new_data', retry=10, tasks=runner.Sync( tasks.insert_new_data.fill( correlation_id='correlation_id', temp_table_name='temp_table_name', source_files='source_files', upcs=DatabaseParam('upcs')))) @property def move_source_files_to_archive(self): """Move source files to the archive directory. Returns: Activity: garcon.activity.Activity instance. """ return self.create( name='move_source_files_to_archive', retry=10, tasks=runner.Sync( tasks.move_source_files_to_archive.fill( correlation_id='correlation_id', source_files='source_files'))) @property def send_notification(self): """Tell the world the ETL has finished. Returns: Activity: garcon.activity.Activity instance. """ return self.create( name='send_notification', retry=10, tasks=runner.Sync( tasks.send_notification.fill( correlation_id='correlation_id', upcs='upcs'))) @property def set_status(self): """Set DynamoDB statuses for files. Returns: Activity: garcon.activity.Activity instance. """ return self.create( name='set_status', retry=10, tasks=runner.Sync( tasks.set_dynamo_status.fill( correlation_id='correlation_id', source_files='source_files'), tasks.set_final_status.fill( correlation_id='correlation_id')))