"""MR-Snowflake Flow.""" from garcon import activity from garcon import runner import raven from flows import config from flows import tasks class MrSnowflake: """Class representing a SWF workflow.""" def __init__(self): """Init flow.""" self.domain = config.SWF_DOMAIN self.name = config.SWF_NAME self.version = config.SWF_VERSION self.timeout = 10800 # 3 hours self.create = activity.create( self.domain, self.name, version=self.version, on_exception=self.on_exception) if config.SENTRY_DSN: self.sentry_client = raven.Client(dsn=config.SENTRY_DSN) else: self.sentry_client = None def on_exception(self, actor, exception): """Capture an exception that has occurred in the application. Args: actor (ActivityWorker, DeciderWorker): the actor that has received the exception. exception (Exception): the exception to capture. """ if self.sentry_client: self.sentry_client.captureException() if isinstance(actor, activity.Activity): actor.logger.error(exception, exc_info=True) else: config.logger.error(exception, exc_info=True) def workflow_id(self): """Generate the workflow id.""" return config.SWF_NAME def decider(self, schedule, context=None): """Flow decider. Args: schedule (callable): The scheduler method. context (dict): Context object. """ bootstrap = schedule( 'bootstrap', self.bootstrap_activity) drop_create_tmp_table = schedule( 'drop_create_tmp_table', self.drop_create_temporary_table_activity, requires=[bootstrap]) import_from_s3 = schedule( 'import_from_s3', self.import_from_s3_activity, requires=[drop_create_tmp_table]) merge_into_mr_table = schedule( 'merge_into_mr_table', self.merge_into_mr_table_activity, requires=[import_from_s3] ) create_flattened_mr_table = schedule( 'create_flattened_mr_table', self.create_flattened_mr_table_activity, requires=[merge_into_mr_table] ) fill_flattened_mr_table = schedule( 'fill_flattened_mr_table', self.fill_flattened_mr_table_activity, requires=[create_flattened_mr_table] ) fill_flattened_mr_locked_table = schedule( 'fill_flattened_mr_locked_table', self.fill_flattened_mr_locked_table_activity, requires=[fill_flattened_mr_table] ) import_audit_from_s3 = schedule( 'import_audit_from_s3', self.import_audit_from_s3_activity, requires=[fill_flattened_mr_locked_table] ) fill_flattened_mr_audit_table = schedule( 'fill_flattened_mr_audit_table', self.fill_flattened_mr_audit_table_activity, requires=[import_audit_from_s3] ) schedule( 'update_dynamo_status', self.update_dynamo_status_activity, requires=[fill_flattened_mr_audit_table] ) @property def bootstrap_activity(self): """Bootstrap the configuration. Get the initial values from the context and create the flow. """ return self.create( name='bootstrap', tasks=runner.Sync(tasks.bootstrap.fill( namespace='bootstrap', date='date'))) # TODO: replace with a real snowflake import task @property def test_snowflake_activity(self): """Test snowflake activity.""" return self.create( name='test_snowflake', retry=2, tasks=runner.Sync(tasks.test_snowflake.fill( date='bootstrap.date'))) @property def drop_create_temporary_table_activity(self): """Drop and create a temporary table for MR events.""" return self.create( name='drop_create_tmp_table', retry=2, tasks=runner.Sync(tasks.drop_create_temp_table.fill())) @property def import_from_s3_activity(self): """Import the date from S3 bucket into a temporary table.""" return self.create( name='import_from_s3', retry=2, tasks=runner.Sync(tasks.import_from_s3.fill( date='bootstrap.date'))) @property def merge_into_mr_table_activity(self): """Merge the log of changes into MR Snowflake table.""" return self.create( name='merge_into_mr_table', retry=2, tasks=runner.Sync(tasks.merge_into_mr_table.fill())) @property def create_flattened_mr_table_activity(self): """Create table with flattened data from MR Snowflake table.""" return self.create( name='create_flattened_mr_table', retry=2, tasks=runner.Sync(tasks.create_flattened_mr_table.fill())) @property def fill_flattened_mr_table_activity(self): """Fill the table with flattened data from MR Snowflake table.""" return self.create( name='fill_flattened_mr_table', retry=2, tasks=runner.Sync(tasks.fill_flattened_mr_table.fill())) @property def fill_flattened_mr_locked_table_activity(self): """Fill the table with flattened data for locked territories.""" return self.create( name='fill_flattened_mr_locked_table', retry=2, tasks=runner.Sync(tasks.fill_flattened_mr_locked_table.fill())) @property def import_audit_from_s3_activity(self): """Import audit data from S3 bucket into a temporary table.""" return self.create( name='import_audit_from_s3', retry=2, tasks=runner.Sync(tasks.import_audit_from_s3.fill( date='bootstrap.date'))) @property def fill_flattened_mr_audit_table_activity(self): """Fill the table with flattened data for MR_AUDIT.""" return self.create( name='fill_flattened_mr_audit_table', retry=2, tasks=runner.Sync(tasks.fill_flattened_mr_audit_table.fill())) @property def update_dynamo_status_activity(self): """Update the final status of Flow in DynamoDB.""" return self.create( name='update_dynamo_status', retry=2, tasks=runner.Sync(tasks.update_dynamo_status.fill( date='bootstrap.date')))