import datetime import logging import time from ssavva.amazon_unlimited.backfilling import swf_utils, config from ssavva.amazon_unlimited.backfilling import jenkins_utils logger = logging.getLogger(__name__) def run_ingestion_task(date): context = '{{"context_date": "{}", "reload": "True"}}'.format(date) jenkins_utils.build_job_and_wait_the_end( config.JENKINS_INGESTION_JOB, { 'CONTEXT': context, 'WORKFLOW': 'amazon_unlimited' } ) def backfill_dates(dates): for date in dates: logger.info('Starting process new date {}'.format(date)) while True: if swf_utils.can_run_flow( config.SWF_INGESTION_DOMAIN, config.SWF_INGESTION_WORKFLOW): logger.info('Can run in SWF {} ingestion'.format(date)) run_ingestion_task(date) print(date, datetime.datetime.now()) logger.info('Run in SWF {} ingestion'.format(date)) break else: logger.info('Waiting for SWF availability...') time.sleep(60) if __name__ == '__main__': dates_for_ingestion = ['2019-03-24'] backfill_dates(dates_for_ingestion)