"""YouTube Video and Claim Summary Workflow.""" from datetime import date as date_module from datetime import datetime from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows.youtube_video_and_claim_summary import config STOP_RESPONSE = {'stop': True} @task.decorate(timeout=1000) def bootstrap(activity, date, reload, report_name): """Bootstrap workflow by injecting initial context from config. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). reload (bool): If True delete feed status in dynamodb. report_name (str): Name of the report to ingest. Returns: dict: Initial context for the workflow. """ if report_name is None: activity.logger.error('Report name is required.') return STOP_RESPONSE date = date or date_module.today().strftime('%Y-%m-%d') report_status_name = '_'.join([config.feed_name, report_name]) if reload is True: activity.logger.info( 'Delete status for feed: {} {} '.format(report_status_name, date)) garcon_feed_status.delete_status(report_status_name, date) else: overall_status = garcon_feed_status.get_overall_status( report_status_name, date) if overall_status == garcon_feed_status.STATUS_INGESTED: return STOP_RESPONSE date_obj = datetime.strptime(date, '%Y-%m-%d') s3_archive_path = 's3://{bucket}/{s3_path}'.format( bucket=config.data_bucket, s3_path=config.s3['archive_path'].format(date=date_obj)) s3_download_path = 's3://{bucket}/{s3_path}'.format( bucket=config.data_bucket, s3_path=config.s3['download_path'].format(date=date_obj)) source_files_dict = { 'files': [{ 'file_name': config.file_template.format( file_type=file_type, date=date_obj, report_name=report_name), } for file_type in config.file_types]} return { 'feed_name': config.feed_name, 'date': date, 'report_name': report_name, 'report_status_name': report_status_name, 's3_archive_path': s3_archive_path, 's3_download_path': s3_download_path, 'source_files_dict': source_files_dict, 'staging_raw_table': 'staging_raw_youtube_{}'.format(report_name)}