"""Synchronization to DynamoDB workflow tasks.""" from datetime import datetime from datetime import timedelta import sys import traceback import distutils.util from garcon import task from dim_refresh_etl.conf.config import SF_CONFIG from dim_refresh_etl.flows.dynamo_sync import config from dim_refresh_etl.flows.dynamo_sync import consts from dim_refresh_etl.flows.dynamo_sync.dynamo_upload import dynamo_upload from dim_refresh_etl.flows.dynamo_sync.snowflake_executor import \ SnowflakeSyncExecutor from dim_refresh_etl.util import sentry_utils @task.decorate(timeout=600) def bootstrap(activity, sync_scope, full_refresh): """Bootstrap workflow by injecting initial context from config. Args: activity (ActivityWorker): The Garcon activity worker. sync_scope (str): Type of synchronized data. full_refresh (str): Flag for force data reload. Returns: dict: Initial context for the workflow. """ if sync_scope != 'analytics_metadata': raise TypeError('Incorrect sync scope type') full_refresh_flag = _param_to_bool(full_refresh) table_throughput_settings = config.table_throughput_settings table_normal_throughput_settings = \ config.table_normal_throughput_settings if full_refresh_flag: sync_from_date = '1970-01-01 00:00:00' else: sync_from_date = ( datetime.now() - timedelta(days=config.sync_from_days_back) ).strftime('%Y-%m-%d %H:%M:%S') model_type_configs = { model_type: { 's3_unload_path': config.s3_unload_path.format( s3_bucket=config.s3_bucket, model_type=model_type)} for model_type in consts.TABLE_ENTITY_TYPES } analytics_metadata_table = config.analytics_metadata_table return { 'sync_scope': sync_scope, 'model_type_configs': model_type_configs, 'sync_from_date': sync_from_date, 'analytics_metadata_table': analytics_metadata_table, 'table_throughput_settings': table_throughput_settings, 'table_normal_throughput_settings': table_normal_throughput_settings, 'full_refresh': full_refresh_flag } @task.decorate(timeout=3600) def unload_data_to_s3( activity, model_type, s3_unload_path, sync_from_date, full_refresh, aws): """Bootstrap workflow by injecting initial context from config. Args: activity (ActivityWorker): The Garcon activity worker. model_type (str): Type of synchronized model. sync_from_date (str): We're looking for any changes after this date. full_refresh (bool): The flag whether the run is incremental or the full refresh. s3_unload_path (str): S3 path for data uploading. aws (dict): AWS credentials. """ activity.logger.info( 'Starting upload {} to {}'.format(model_type, s3_unload_path)) with SnowflakeSyncExecutor(SF_CONFIG) as snowflake_executor: snowflake_executor.unload_to_s3( model_type, s3_unload_path, sync_from_date, full_refresh, aws) activity.logger.info( '{} model was successfully uploaded to {}'.format( model_type, s3_unload_path)) @task.decorate(timeout=36000) def upload_to_dynamodb(activity, model_type, s3_unload_path): """Upload model data to DynamoDB. All potential errors in uploading process are caught here. Exception details are sent to Sentry and logged. We don't want to interrupt the flow before DynamoDB capacity decreased back to normal. Returns: dict: Error details if any occurred. It is used for further exception invocation. """ try: dynamo_upload.start_upload(model_type, s3_unload_path) except: # noqa sentry_utils.capture_exception() activity.logger.error(traceback.format_exc()) return {'error': { 'message': repr(sys.exc_info()[1]), 'model_type': model_type}} @task.decorate(timeout=600) def health_check(activity, error): """Check if any error occurred during the upload.""" if error: raise Exception( 'An error occurred during the execution of the flow. ' 'Error: "{}". ' 'Please see Sentry or logs for more details.'.format(error)) activity.logger.info( 'The execution of the flow was successfully finished. ' 'No errors were found.') def _param_to_bool(param): """Convert passed param to a boolean value. If the passed param is a str and equals to 'True', 'TRUE', 'true' it will be converted to boolean True. If it's already a boolean returns the same value. If it's None or 'False', 'FALSE', 'false' and etc. will return boolean False. If the param is a random string or any other object the ValueError exception will be raised. Returns: bool: Converted to boolean value. """ return bool(param and distutils.util.strtobool(str(param)))