"""Synchronization to DynamoDB workflow.""" from garcon import runner from garcon.param import StaticParam from garcon_contrib.aws import garcon_dynamodb from garcon_contrib.aws import garcon_s3 from dim_refresh_etl.flows.base import BaseFlow from dim_refresh_etl.flows.dynamo_sync import generators from dim_refresh_etl.flows.dynamo_sync import tasks class Flow(BaseFlow): """Dynamo synchronization flow. This flow unloads data from Snowflake and then loads to DynamoDB. The flow is configured by explicitly passed scope type value. """ timeout = 60 * 60 * 7 # 6 hours def __init__(self): """Create a Dynamo Synchronization Workflow.""" super().__init__(name='dynamo_sync', version='1.3') def workflow_id(self, context): """Generate workflow id. Args: context (dict): The initial context for the flow. Returns: str: workflow id with `dynamo_sync_` prefix and corresponding data type from initial context. """ return 'dynamo_sync_{}'.format(context['dim_type']) def decider(self, schedule, context): """Flow decider. Arg: schedule (callable): Call the scheduler. context (dict): initial context workflow was launched with. """ bootstrap = schedule( 'bootstrap', self.bootstrap) unload_data_to_s3 = schedule( 'unload_data_to_s3', self.unload_data_to_s3, requires=[bootstrap]) # increase_table_capacity = schedule( # 'increase_table_capacity', # self.increase_table_capacity, # requires=[unload_data_to_s3]) upload_to_dynamodb = schedule( 'upload_to_dynamodb', self.upload_to_dynamodb, requires=[unload_data_to_s3]) # decrease_table_capacity = schedule( # 'decrease_table_capacity', # self.decrease_table_capacity, # requires=[upload_to_dynamodb]) schedule( 'health_check', self.health_check, requires=[upload_to_dynamodb]) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=runner.Sync( tasks.bootstrap.fill( namespace='bootstrap', sync_scope='dim_type', full_refresh='full_refresh' ) ) ) @property def unload_data_to_s3(self): """Unload data to S3.""" return self.create( name='unload_data_to_s3', generators=[generators.model_types], tasks=runner.Sync( garcon_s3.remove_files_from_path.fill( path='s3_unload_path', return_deleted_files=StaticParam(False) ), tasks.unload_data_to_s3.fill( model_type='model_type', s3_unload_path='s3_unload_path', sync_from_date='bootstrap.sync_from_date', full_refresh='bootstrap.full_refresh', aws=StaticParam(self.conf_aws) ) ) ) @property def increase_table_capacity(self): """Increase write capacity for the target table.""" return self.create( name='increase_table_capacity', tasks=runner.Sync( garcon_dynamodb.set_table_throughput.fill( table_name='bootstrap.analytics_metadata_table', throughput_settings='bootstrap.table_throughput_settings' ), garcon_dynamodb.wait_for_table_task_completion.fill( table_name='bootstrap.analytics_metadata_table' ) ) ) @property def upload_to_dynamodb(self): """Upload data to DynamoDB.""" return self.create( name='upload_to_dynamodb', generators=[generators.model_types], schedule_to_start=48000, tasks=runner.Sync( tasks.upload_to_dynamodb.fill( namespace='upload_to_dynamodb', model_type='model_type', s3_unload_path='s3_unload_path' ) ) ) @property def decrease_table_capacity(self): """Decrease write capacity for the target table.""" return self.create( name='decrease_table_capacity', tasks=runner.Sync( garcon_dynamodb.wait_for_table_task_completion.fill( table_name='bootstrap.analytics_metadata_table' ), garcon_dynamodb.set_table_throughput.fill( table_name='bootstrap.analytics_metadata_table', throughput_settings='bootstrap.' 'table_normal_throughput_settings' ) ) ) @property def health_check(self): """Health check of upload process.""" return self.create( name='health_check', tasks=runner.Sync( tasks.health_check.fill( namespace='health_check', error='upload_to_dynamodb.error' ) ) )