"""YouTube Asset Report Workflow.""" from datetime import date as date_module from datetime import datetime import re from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows.youtube_asset import config from feed_ingestion.util.aws import s3 STOP_RESPONSE = {'stop': True} @task.decorate(timeout=1000) def bootstrap( activity, date, licensor=None, reload=None, skip_grab_reports_files=None): """Bootstrap workflow by injecting initial context from config. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). licensor (str): one of config.licensors reload (str): If 'True' delete feed status in dynamodb. skip_grab_reports_files (str): If 'True' get theorchard files from S3 archive location. This is a workaround for API failures when the files are available via CMS, but not available via API. We must download and place the files into the archive location manually. Returns: dict: Initial context for the workflow. """ if not licensor: licensor = 'theorchard' assert licensor in config.licensors, f'unsupported licensor "{licensor}"' date = date or date_module.today().strftime('%Y-%m-%d') parsed_date = datetime.strptime(date, '%Y-%m-%d') feed_name = '_'.join([config.feed_name, licensor]) report_status_name = feed_name cms_dict = { 'theorchard': config.cms_dict } if reload == 'True': activity.logger.info( 'Delete status for feed: {} {} '.format(report_status_name, date)) garcon_feed_status.delete_status(report_status_name, date) else: overall_status = garcon_feed_status.get_overall_status( report_status_name, date) if overall_status == garcon_feed_status.STATUS_INGESTED: return STOP_RESPONSE archive_path = config.s3['archive_path'].format( date=parsed_date, licensor=licensor) credentials_path = config.credentials_paths.get(licensor) return { 'feed_name': feed_name, 'date': date, 'licensor': licensor, 'skip_grab_reports_files': skip_grab_reports_files, 's3_bucket': config.s3_bucket, 's3_archive_path': archive_path, 'credentials_path': credentials_path, 's3_dir_path': 's3://{}/{}'.format( config.s3_bucket, archive_path), 'cms_dict': cms_dict.get(licensor) } def remove_prefix(text, prefix): """Remove prefix from string if string starts with it.""" return text[text.startswith(prefix) and len(prefix):] @task.decorate(timeout=1000) def source_files(activity, licensor, s3_bucket, s3_archive_path): """Get list of files to ingest for given date and licensor. Args: activity (ActivityWorker): The Garcon activity worker. licensor (str): one of config.licensors s3_bucket (str): s3 bucket containing source files s3_archive_path (str): archive path for the date in s3 bucket. Returns: dict: dict containing 'source_files_dict' with list of source files metadata """ s3_path_prefix = s3_archive_path s3_full_path = 's3://{}/{}'.format( s3_bucket, s3_archive_path) filenames = s3.get_list_of_files_and_directories(s3_full_path) source_files_dict = {'files': []} for file_path in filenames: file_name = remove_prefix(file_path, s3_path_prefix) file_pattern = config.source_file_pattern[licensor] if not re.match(file_pattern, file_name): activity.logger.warning( f'Skipping non-matching source file {file_name}') continue file_size = s3.get_key_size( 's3://{}/{}'.format(config.s3_bucket, file_path), config.expected_bucket_owner ) source_files_dict['files'].append({ 'file_name': file_name, 'file_size': file_size}) if not source_files_dict['files']: raise ValueError(f'No source files found in {s3_full_path}') return { 'source_files_dict': source_files_dict, }