"""YouTube Bulk Reports Workflow.""" from datetime import date as date_module from datetime import datetime from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows.youtube_bulk_reports import config from feed_ingestion.flows.youtube_bulk_reports.config import reports_mapping STOP_RESPONSE = {'stop': True} @task.decorate(timeout=1000) def bootstrap(activity, date, reload, report_name, licensor, selected_owner=None): """Bootstrap workflow by injecting initial context from config. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). reload (str): If "True" delete feed status in dynamodb. report_name (str): Name of the report to ingest. licensor (str): one of config.licensors selected_owner (str): Single content owner to work with provided via context. Returns: dict: Initial context for the workflow. """ assert licensor in config.licensors, f'unsupported licensor "{licensor}"' # ignore unsupported selected_owner owner = None if selected_owner: owners_map = config.orchard_content_owners_map.items() owner = { k: v for k, v in owners_map if v == selected_owner.upper()} if not owner: activity.logger.error('Unknown content owner: {}'.format( selected_owner.upper())) return STOP_RESPONSE if report_name is None: activity.logger.error('Report name is required.') return STOP_RESPONSE only_download = None if report_name in config.reports_download_only[licensor]: only_download = 'True' date = date or date_module.today().strftime('%Y-%m-%d') date_obj = datetime.strptime(date, '%Y-%m-%d') feed_name = '_'.join([config.feed_name, licensor]) report_status_name = '_'.join([feed_name, report_name]) if reload == 'True': activity.logger.info( 'Delete status for feed: {} {} '.format(report_status_name, date)) garcon_feed_status.delete_status(report_status_name, date) else: if only_download == 'True': stopping_status = garcon_feed_status.STATUS_DOWNLOADED else: stopping_status = garcon_feed_status.STATUS_INGESTED overall_status = garcon_feed_status.get_overall_status( report_status_name, date) if overall_status == stopping_status: return STOP_RESPONSE youtube_report_id = reports_mapping[report_name] normalized_report_name = report_name.replace('_', '') staging_raw_table = '{}_intermediate'.format( config.v1_reports.get( report_name, normalized_report_name)) archive_path = config.archive_paths[licensor].format( date=date_obj, licensor=licensor, report_name=report_name ) credentials_path = config.credentials_paths.get(licensor) result = { 'feed_name': feed_name, 'date': date, 'licensor': licensor, 'only_download': only_download, 'selected_owner': owner, 'report_name': report_name, 'youtube_report_id': youtube_report_id, 'report_status_name': report_status_name, 's3_bucket': config.s3_bucket, 'archive_path': archive_path, 'credentials_path': credentials_path, 's3_dir_path': 's3://{}/{}'.format(config.s3_bucket, archive_path), 'staging_raw_table': staging_raw_table} return result @task.decorate(timeout=1000) def bootstrap_report_group(activity, feed_name, date, reload, report_list, licensor): """Bootstrap workflow by injecting initial context from config. This activity is used to generate context for loading multiple reports. Args: activity (ActivityWorker): The Garcon activity worker. feed_name (str): Feed name. date (str): Reporting date (YYYY-MM-DD). reload (str): If "True" delete feed status in dynamodb. report_list (list): List of the reports to ingest. licensor (str): one of config.licensors Returns: dict: Initial context for the workflow. """ def clear_statuses(date, report_list): """Clear DynamoDB status for reports from the list. Args: date (str): Reporting date (YYYY-MM-DD). report_list (list): List of the reports to clear. """ for report in report_list: garcon_feed_status.delete_status( report['report_status_name'], date) if not licensor: licensor = 'theorchard' assert licensor in config.licensors, f'unsupported licensor "{licensor}"' date = date or date_module.today().strftime('%Y-%m-%d') date_obj = datetime.strptime(date, '%Y-%m-%d') # feed_name shall be unique across licensors feed_name = '_'.join([feed_name, licensor]) credentials_path = config.credentials_paths.get(licensor) owners_dict = { 'theorchard': config.orchard_content_owners_map } def generate_context(report_list): """Generate partial contexts for the list of reports. Args: report_list (list): List of the reports to ingest. Returns: dict: Initial context for the workflow. """ report_context = [] for report in report_list: report_name = report['report_name'] normalized_report_name = report_name.replace('_', '') archive_path = config.archive_paths[licensor].format( date=date_obj, licensor=licensor, report_name=report_name ) youtube_report_id = reports_mapping[report_name] report_context.append({ 'report_name': report_name, 'archive_path': archive_path, 'youtube_report_id': youtube_report_id, 's3_bucket': config.s3_bucket, 's3_dir_path': 's3://{}/{}'.format( config.s3_bucket, archive_path), 'report_status_name': report['report_status_name'], 'staging_raw_table': '{}_intermediate'.format( config.v1_reports.get( report_name, normalized_report_name))}) return report_context if licensor == 'sme': # SME do not share estimated revenue reports with us yet remove_reports = ['estimated_revenue', 'asset_estimated_revenue'] report_list = [r for r in report_list if r not in remove_reports] report_statuses = [] for r in report_list: report_status_name = '_'.join([config.feed_name, licensor, r]) report_statuses.append({ 'report_name': r, 'report_status_name': report_status_name, 'status': garcon_feed_status.get_overall_status( report_status_name, date) }) missing_reports = [ r for r in report_statuses if r['status'] != garcon_feed_status.STATUS_INGESTED] if not (missing_reports or reload == 'True'): return STOP_RESPONSE context = { 'feed_name': feed_name, 'date': date, 'licensor': licensor, 'credentials_path': credentials_path, 'cms_dict': owners_dict.get(licensor), } if reload == 'True': clear_statuses(date, report_statuses) context['reports'] = generate_context(report_statuses) return context else: context['reports'] = generate_context(missing_reports) return context