# run all bulk reports for sme athena import json import os import boto3 import time import dotenv import jenkins import loguru import youtube_monitoring logger = loguru.logger ENVIRONMENT = 'prod' # EXEC_JENKINS_JOB = 'swf-youtube-bulk-report-sf-garcon-exec' EXEC_JENKINS_JOB = 'swf-backload-scheduler-exec' SWF_DOMAIN = f'{ENVIRONMENT}_swf_feed_ingestion' # SWF_DOMAIN = 'dev_efedorov' dotenv.load_dotenv() # connect to jenkins. # JENKINGS_PASS - it should be token generated in jenkins server = jenkins.Jenkins( 'https://scheduler.theorchard.io', username='efedorov', password=os.environ['JENKINS_PASS'] ) assert server.job_exists(EXEC_JENKINS_JOB) client = boto3.client('swf', region_name='us-east-1') def schedule_all(): for report_name in youtube_monitoring.bulk_reports_all: raise ValueError('Think before start') is_only_dowload = report_name in youtube_monitoring.bulk_reports_only_download logger.info(f'Report {report_name}. Only download? {is_only_dowload}') if report_name in ['asset_basic', 'ad_rates']: logger.warning(f'Skip it {report_name}') continue context = { "report_name": report_name, } if is_only_dowload: context['only_download'] = 'True' parameters = { "Environment": "prod", "flow_name": "youtube_bulk_reports", "licensor": "sme", "period_start": "2020-08-20", "period_end": "2020-11-18", "reload": "None", "context": json.dumps(context), } logger.info(f'Launching... {parameters}') job_id = server.build_job( EXEC_JENKINS_JOB, parameters ) logger.info(f'JOB_ID: {job_id}') time.sleep(3) to_reload_reports = """ playlist_traffic_source\t2020-11-19 2020-12-12 """.split("\n") def schedule_reload(): for report_str in to_reload_reports: if not report_str: continue logger.info(to_reload_reports) report_name, dates = report_str.split('\t') for date_entry in dates.split(','): date_entry = date_entry.strip() entries = date_entry.split(' ') if len(entries) > 1: period_start, period_end = entries else: period_start, period_end = entries[0], entries[0] is_only_dowload = report_name in youtube_monitoring.bulk_reports_only_download logger.info(f'Report {report_name}. Only download? {is_only_dowload}') context = { "report_name": report_name, } if is_only_dowload: context['only_download'] = 'True' parameters = { "Environment": "prod", "flow_name": "youtube_bulk_reports", "licensor": "sme", "period_start": period_start, "period_end": period_end, "reload": "None", "context": json.dumps(context), } logger.info(f'Launching... {parameters}') job_id = server.build_job(EXEC_JENKINS_JOB, parameters) logger.info(f'JOB_ID: {job_id}') time.sleep(3) if __name__ == '__main__': schedule_reload()