"""Tasks for Ingestion Workflow.""" from datetime import datetime from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows.physical_reporting import config from feed_ingestion.tasks import s3_tasks from feed_ingestion.tasks.overall_status_tasks import ( set_overall_status_enhanced ) from feed_ingestion.util import task_status @task.decorate(timeout=700) def bootstrap(activity, date, report): """Bootstrap workflow by getting the correct configurations. Args: activity (ActivityWorker): The activity worker. date (str): Reporting date (YYYY-MM-DD). report (str): one of config.reports. Returns: dict: Context. """ assert report in config.reports report_config = config.reports[report] date_obj = (datetime.strptime( date, '%Y-%m-%d') if date else datetime.today()) activity.logger.info( 'Bootstrap flow for {} date: {}'.format(report, date)) staging_raw = report_config['staging_raw'] drop_file_name = report_config['drop_file_name'].format(date=date_obj) s3_archive_path = report_config['archive_path'].format(date=date_obj) s3_download_path = report_config['drop_path'] return dict( feed_name='_'.join([config.feed_name, report]), date=date_obj.strftime('%Y-%m-%d'), report_name=report, secrets_path=config.secrets_path, s3_download_path=s3_download_path, s3_archive_path=s3_archive_path, drop_file_name=drop_file_name, staging_raw_table_name=staging_raw, ) @task.decorate(timeout=3600) def grab_drop_files( activity, feed_name, date, s3_archive_path, s3_download_path, drop_file_name): """Copy The Orchard file from the drop location into the archive folder. Args: activity (ActivityWorker): The activity worker. feed_name (str): The name of the feed. date (str): Reporting date (YYYY-MM-DD). s3_archive_path (str): S3 path to archive the file. s3_download_path (str): S3 path to download the file for processing. drop_file_name (str): Name of the file to copy. Returns: dict: Context with source_files_dict or stop flag. """ activity.logger.info( 'Grabbing files from drop location for feed: {}, date: {}'.format( feed_name, date)) is_completed = task_status.is_completed_task( feed_name, date, 'grab_drop_files') # Build source_files_dict for copy_files source_files_dict = { 'files': [ { 'file_name': drop_file_name, # If the task is already completed, # we can assume the file was found 'found': is_completed, } ] } if is_completed: return source_files_dict result = s3_tasks.copy_files( activity=activity, s3_archive_path=s3_archive_path, s3_download_path=s3_download_path, source_files_dict=source_files_dict, need_all_files=True) if result.get('stop'): activity.logger.info( 'File {} not found in {}'.format(drop_file_name, s3_download_path)) set_overall_status_enhanced( feed_name, date, garcon_feed_status.STATUS_NOT_AVAILABLE, activity) return result activity.logger.info( 'Successfully copied file {} from {} to {}'.format( drop_file_name, s3_download_path, s3_archive_path)) task_status.mark_completed_task( feed_name, date, 'grab_drop_files') set_overall_status_enhanced( feed_name, date, garcon_feed_status.STATUS_DOWNLOADED, activity) return result