"""Kuwo Data Ingestion Workflow.""" from datetime import datetime from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows.kuwo import config STOP_RESPONSE = {'stop': True} @task.decorate(timeout=1000) def bootstrap(activity, date, licensor=None, reload=None, skip_corrupted_rows=None, dw_config=None): """Bootstrap workflow by getting the correct configurations. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). licensor (str): The licensor to ingest. Returns: dict: Context. """ feed_name = '_'.join([config.feed_name, licensor]) date_obj = date or datetime.today().strftime('%Y-%m-%d') activity.logger.info( 'Bootstrapping {feed_name} for {date}'.format( feed_name=feed_name, date=date_obj)) replace_archive_files = True if reload == 'True': activity.logger.info( 'Delete status for feed: {} {} '.format(feed_name, date_obj)) garcon_feed_status.delete_status(feed_name, date_obj) else: overall_status = garcon_feed_status.get_overall_status( feed_name, date) if overall_status == garcon_feed_status.STATUS_INGESTED: return STOP_RESPONSE drop_path = config.s3['drop'][licensor] archive_path = config.s3['archive'][licensor].format(date=date_obj) s3_full_path = 's3://{data_bucket}/'.format( data_bucket=config.data_bucket) + archive_path # Select file name/format version by reporting date. The matching # staging_raw COPY version is resolved separately by SQLLoader via the # queries// sub folder, using the same date keys as config.formats. file_format = config.format_for(date_obj) filename_template = file_format['filename'] licensor_names = file_format['licensors'][licensor] source_file_pattern = file_format['source_file_pattern'][licensor] # array to store files to download expected_files = [] licensor_files = licensor_names.split(',') for licensor_file in licensor_files: filename = filename_template.format( date=datetime.strptime(date_obj, '%Y-%m-%d'), licensor_filename=licensor_file) expected_files.append(filename) staging_raw_table = config.snowflake_table_names['staging_raw'] corrupted_rows_error = skip_corrupted_rows or 'False' return dict( feed_name=feed_name, date=date_obj, licensor=licensor, store_id=config.storeid, s3_bucket=config.data_bucket, drop_path=drop_path, archive_path=archive_path, s3_full_path=s3_full_path, source_file_pattern=source_file_pattern, skip_corrupted_rows=corrupted_rows_error, expected_files=expected_files, secrets_path=config.secrets_path, staging_raw_table=staging_raw_table, replace_archive_files=replace_archive_files )