import datetime from pprint import pprint from feed_ingestion.bin.backload_monitoring import get_spreadsheet_api, SPREADSHEET_ID all_reports = """ active_claims active_references ad_rates annotations asset asset_annotations asset_basic asset_cards asset_combined asset_conflict asset_demographics asset_device_os asset_end_screens asset_playback_location asset_province asset_sharing_service asset_traffic_source basic cards combined demographics device_os end_screens playback_location playlist_basic playlist_combined playlist_device_os playlist_playback_location playlist_province playlist_traffic_source province sharing_service subtitles traffic_source video_metadata""".split() reports_full_load = """ asset_annotations asset_basic asset_cards asset_combined asset_demographics asset_device_os asset_playback_location asset_province asset_sharing_service asset_traffic_source basic cards combined demographics device_os playback_location province sharing_service subtitles traffic_source""".split() # do not load bulk reports skip_reports = """ video_metadata asset asset_annotations annotations """.split() bulk_reports_all = set(all_reports) - set(skip_reports) bulk_reports_full_ingest = set(reports_full_load) - set(skip_reports) bulk_reports_only_download = set(all_reports) - set(reports_full_load) - set(skip_reports) def create_sheet(spreadsheets, sheet_name): """Update spreadsheet values. The spreadsheet ID configured in global SPREADSHEET_ID Args: spreadsheets (object) google spreadsheets service sheet_name (str) name start_row_num (int) rows (list) of str with values """ body = {"requests": [ { "addSheet": { "properties": { "title": sheet_name, } } } ]} response = spreadsheets.batchUpdate( spreadsheetId=SPREADSHEET_ID, body=body).execute() return response def create_sheets(): api = get_spreadsheet_api() for report in all_reports: create_sheet(api, f'YT-{report}') def gen_config(report_name): return f'youtube_bulk_reports_sme_{report_name}', { 'start_date': datetime.date(year=2020, month=8, day=20), 'end_date': datetime.date(year=2020, month=11, day=15), 'sheet_name': f'YT-{report_name}', 'workflow_type': 'youtube_bulk_reports_feed_ingestion', 'workflow_name_template': f'youtube_bulk_reports_feed_ingestion_{report_name}-sme-{{date}}', 'feed_name': f'youtube_bulk_reports_sme_{report_name}', }, def gen_configs(): configs = {} for report in all_reports: if report in ['asset_annotations', 'annotations', 'asset', 'video_metadata']: continue key, value = gen_config(report) configs[key] = value # pprint(configs, width=75) print(configs) if __name__ == '__main__': gen_configs()