"""Amazon Unlimited Marketshare Garcon tasks. Tasks to ingest Amazon Unlimited Marketshare data. """ from datetime import datetime from garcon import task from feed_ingestion.flows.amazon_unlimited_marketshare import config from feed_ingestion.flows.amazon_unlimited_marketshare.snowflake_executor \ import AmazonUnlimitedMarketshareSF from feed_ingestion.flows.helpers import get_sf_config from feed_ingestion.tasks import bootstrap as reload STOP_RESPONSE = {'stop': True} @task.decorate(timeout=1000) @reload.reset_dynamodb_status_on_reload(config.feed_name) def bootstrap(activity, date, dw_config=None): """Bootstrap workflow by injecting intial context from config. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). dw_config (dict): Dictionary of data warehouse config options. Returns: dict: Initial context for the workflow. """ date_obj = datetime.strptime(date, '%Y-%m-%d') temp_staging_raw_table = config.snowflake_table_names[ 'temp_staging_raw'].format(date=date_obj) return { 'feed_name': config.feed_name, 'secrets_path': config.secrets_path, 'date': date, 's3_archive_path': config.s3['archive'].format(date=date_obj), 'file_pattern': config.file_pattern.format(date=date_obj), 'temp_staging_raw_table': temp_staging_raw_table, 'staging_raw_table': config.snowflake_table_names['staging_raw'] } @task.decorate(timeout=600) def drop_temp_table(activity, temp_table_name): """Drop temp staging raw table. Args: activity (ActivityWorker): The Garcon activity worker. temp_table_name (str): Table name to drop. """ sf_config = get_sf_config(config.secrets_path) with AmazonUnlimitedMarketshareSF(sf_config) as executor: executor.drop_table(temp_table_name) activity.logger.info('{} was dropped'.format(temp_table_name))