"""Tasks for Ingestion Workflow.""" import datetime from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.conf.config import merge_configs from feed_ingestion.flows.delphi_crm import config from feed_ingestion.flows.delphi_crm.snowflake_executor \ import DelphiCrmSnowflakeExecutor from feed_ingestion.flows.helpers import get_sf_config from feed_ingestion.tasks import check_status @task.decorate(timeout=700) def bootstrap(activity, date, report, reload): """Bootstrap workflow by getting the correct configurations. Args: activity (ActivityWorker): The activity worker. date (str): Reporting date (YYYY-MM-DD). report (str): one of config.reports reload (str or None): If 'True' reload status. Returns: dict: Context. """ assert report in config.reports report_config = config.reports[report] feed_name = f'{config.feed_name}_{report}' date_obj = datetime.datetime.strptime(date, '%Y-%m-%d') date = date_obj.strftime('%Y-%m-%d') if reload == 'True': activity.logger.info('Delete status for feed: {} {} '.format( feed_name, date)) garcon_feed_status.delete_status(feed_name, date) elif garcon_feed_status.get_overall_status( feed_name, date) == garcon_feed_status.STATUS_INGESTED: activity.logger.info('Feed already ingested for {}'.format(date)) return { 'stop': True, 'message': '{feed_name} is already ingested for {date}'.format( feed_name=feed_name, date=date)} activity.logger.info('Bootstrap flow for {} date: {}'.format(report, date)) s3_dir_path = f's3://{config.drop_bucket}/{config.drop_path}' filename = report_config['filename_template'].format( date=date_obj ) file_pattern = rf'.*\/{filename}' return dict( feed_name=feed_name, date=date, report_name=report, secrets_path=config.secrets_path, drop_bucket=config.drop_bucket, drop_path=config.drop_path, source_file_name=filename, s3_dir_path=s3_dir_path, file_pattern=file_pattern, staging_raw_table=report_config['staging_raw_table'], ) @task.decorate(timeout=3600*5) @check_status() def load_staging_raw_table_reports( activity, feed_name, date, report_name, s3_dir_path, staging_raw_table_name, file_name, ): """Save data to staging raw table(s) for given report. Args: activity (ActivityWorker): The Garcon activity worker. feed_name (str): Feed name of workflow execution for status updates. date (str): Reporting date (YYYY-MM-DD). report_name (str): Name of the report to ingest. staging_raw_table_name (str): The staging table name for report. """ temp_table_name = 'temp_{staging_raw_table_name}_{date}'.format( staging_raw_table_name=staging_raw_table_name, date=date.replace('-', '')) sf_config = merge_configs(get_sf_config(config.secrets_path), {}) with DelphiCrmSnowflakeExecutor(sf_config) as executor: executor.create_temp_staging_raw_table( report_name=report_name, temp_staging_raw_table=temp_table_name ) activity.logger.info(f'{temp_table_name} was created') executor.load_temp_staging_raw_table( temp_staging_raw_table=temp_table_name, file_name=[file_name], s3_dir_path=s3_dir_path, ) executor.clean_staging_raw_table( staging_raw_table=staging_raw_table_name, date=date, ) executor.load_staging_raw_table( staging_raw_table=staging_raw_table_name, temp_staging_raw_table=temp_table_name, report_name=report_name, date=date, file_name=file_name, ) executor.drop_table(table=temp_table_name)