"""Dimension Refresh Tasks. ========================== Garcon task for bootstrapping a dimension refresh workflow. """ import time from garcon import task from dim_refresh_etl import dimension from dim_refresh_etl.conf.config import SF_CONFIG from dim_refresh_etl.util import environment @task.decorate(timeout=60) def bootstrap_task(activity, dim_type): """Bootstraps the dimension specific args for the workflow. Args: activity (ActivityWorker): the activity worker. dim_type (str): name of dimension being refreshed. Returns: dict: global context variables to be passed down the workflow. """ activity.logger.info('Bootstrapping dimension type %s', dim_type) if environment.name == environment.TEST: test_mode = True else: test_mode = False dim_entity = dimension.create_entity(dim_type, test_mode) current_timestamp = time.strftime('%Y-%m-%d %H:%M:%S') update_sql = dim_entity.hydrate_query_list( query_list=dim_entity.update_sql, current_timestamp=current_timestamp) update_new_rows_sql = dim_entity.hydrate_query_list( query_list=dim_entity.update_new_rows_sql, current_timestamp=current_timestamp) insert_sql = dim_entity.hydrate_query_list( query_list=dim_entity.insert_sql, current_timestamp=current_timestamp) delete_sql = dim_entity.hydrate_query_list( query_list=dim_entity.delete_sql, current_timestamp=current_timestamp) related_sql = dim_entity.hydrate_query_list( query_list=dim_entity.related_sql, current_timestamp=current_timestamp) insert_count_sql = dim_entity.hydrate_query( sql=dim_entity.insert_count_sql, current_timestamp=current_timestamp) update_count_sql = dim_entity.hydrate_query( sql=dim_entity.update_count_sql, current_timestamp=current_timestamp) if dim_entity.export_sql: export_sql = dim_entity.hydrate_query( sql=dim_entity.export_sql) else: export_sql = None empty_staging_table_sql = 'TRUNCATE {db}.{schema}.{table};'.format( db=SF_CONFIG['db'], schema=SF_CONFIG['schema'], table=dim_entity.staging_table) dt = time.strftime('%Y-%m-%d') s3_staging_bucket = dimension.s3_staging_bucket(environment.name) s3_staging_key = dimension.s3_staging_key(dim_entity.dim_table, dt) s3_staging_path = dimension.s3_staging_path( environment.name, dim_entity.dim_table, dt) return { 'dim_refresh.dim_type': dim_type, 'dim_refresh.source_db': dim_entity.source_db, 'dim_refresh.staging_table': dim_entity.staging_table, 'dim_refresh.dim_table': dim_entity.dim_table, 'dim_refresh.export_sql': export_sql, 'dim_refresh.update_sql': update_sql, 'dim_refresh.update_new_rows_sql': update_new_rows_sql, 'dim_refresh.insert_sql': insert_sql, 'dim_refresh.delete_sql': delete_sql, 'dim_refresh.related_sql': related_sql, 'dim_refresh.insert_count_sql': insert_count_sql, 'dim_refresh.update_count_sql': update_count_sql, 'dim_refresh.empty_staging_table_sql': empty_staging_table_sql, 'dim_refresh.staging_bucket': s3_staging_bucket, 'dim_refresh.staging_key': s3_staging_key, 'dim_refresh.staging_path': s3_staging_path, 'dim_refresh.current_timestamp': current_timestamp } @task.decorate(timeout=60) def sync_bootstrap_task(activity, dim_type, sync_dimensions, schema): """Bootstraps the dimension specific args for the workflow. Args: activity (ActivityWorker): the activity worker. dim_type (str): name of dimension being refreshed. Returns: dict: global context variables to be passed down the workflow. """ activity.logger.info('Bootstrapping sync dimension') dt = time.strftime('%Y-%m-%d') if dim_type == 'all': dimensions = sync_dimensions elif dim_type in sync_dimensions: dimensions = [dim_type] else: raise Exception( 'Invalid dim_type passed to sync flow {}'.format(dim_type)) return { 'dt': dt, 'schema': schema, 'dims_to_sync': dimensions }