"""Tasks for the Peloton Monthly Ingestion Workflow.""" import calendar from datetime import datetime from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows.peloton import config from feed_ingestion.tasks import bootstrap as reload from feed_ingestion.tasks import check_status from feed_ingestion.tasks import s3_tasks @task.decorate(timeout=1000) @reload.reset_dynamodb_status_on_reload(config.feed_name) def bootstrap(activity, date, dw_config=None): """Bootstrap the workflow for the reporting month. Args: activity (ActivityWorker): The activity worker. date (str): Reporting date (YYYY-MM-DD). Returns: dict: Initial context for the workflow. """ date_obj = ( datetime.strptime(date, '%Y-%m-%d') if date else datetime.today()) # Normalise to the reporting month: first and last calendar day. first_day = date_obj.replace(day=1) last_day = date_obj.replace( day=calendar.monthrange(date_obj.year, date_obj.month)[1]) activity.logger.info('Bootstrap flow for {:%Y-%m}'.format(first_day)) archive_dir = config.archive_path.format(date=first_day) # One file per country - build the source (SME) and destination (archive) # keys for each. source_files = [] for country in config.countries: file_name = config.drop_filename.format( country=country, first_day=first_day, last_day=last_day) source_files.append(dict( country=country, file_name=file_name, source_key_name='{}{}'.format(config.drop_path, file_name), destination_key_name='{}{}'.format(archive_dir, file_name))) s3_dir_path = 's3://{}/{}'.format(config.data_bucket, archive_dir) return dict( feed_name=config.feed_name, date=first_day.strftime('%Y-%m-%d'), source_files=source_files, s3_dir_path=s3_dir_path, secrets_path=config.secrets_path, staging_raw_table=config.staging_raw_table) @task.decorate(timeout=3600) @check_status() def grab_drop_files(activity, feed_name, date, source_files): """Copy the monthly Peloton files from the SME bucket to the archive. Every country file for the reporting month must be present before the workflow proceeds. If any is still missing the workflow stops (leaving the feed status as SOURCE_NOT_AVAIL) and picks the files up on a later run. Args: activity (ActivityWorker): The activity worker. feed_name (str): The name of the feed. date (str): Reporting date (YYYY-MM-DD). source_files (list): Per-country dicts with ``source_key_name`` and ``destination_key_name``. Returns: dict: source_files_dict for the archived files, or a stop signal if not all files have arrived yet. """ archived = [] missing = [] for entry in source_files: result = s3_tasks.copy_file_from_sme_s3_to_theocrhard( activity, secrets_path=config.sme_secrets_path, source_bucket_name=config.drop_bucket, source_key_name=entry['source_key_name'], destination_bucket_name=config.data_bucket, destination_key_name=entry['destination_key_name'], replace=True) if result.get(entry['file_name']) is True: archived.append(dict( file_name=entry['file_name'], file_size=result.get('file_size', 1), found=True)) else: missing.append(entry['file_name']) # Wait for the full monthly delivery before moving on. if missing: activity.logger.info( 'Waiting for Peloton files, still missing: {}'.format(missing)) garcon_feed_status.set_overall_status( feed_name, date, garcon_feed_status.STATUS_NOT_AVAILABLE) return {'stop': True, 'missing_files': missing} return dict(source_files_dict={'files': archived})