import json from argparse import ArgumentParser from datetime import datetime, timedelta import boto3 import smart_open from sme_logger.logger import SimpleLoggerFactory from slz_downloader.scripts.appreciationengine_backfill import EnvironmentEnum def run(logger, batch_client, params): start_date = datetime.strptime(params.start_date, '%Y-%m-%d') end_date = datetime.strptime(params.end_date, '%Y-%m-%d') if start_date > end_date: logger.error("Start Date is older than End Date") with smart_open.open(f's3://{params.env}-delphi-configs/batch-run-config.json', 'r') as fin: config = json.load(fin) while start_date != end_date: try: start_date_str = start_date.strftime('%Y-%m-%d') end_date_str = (start_date + timedelta(days=1)).strftime('%Y-%m-%d') response = batch_client.submit_job( jobName= f'appreciationengine_backfill_{params.licensor}_{start_date_str}_{end_date_str}', jobQueue=f'{params.env}-delphi-main', jobDefinition=config['JobDefinition'], containerOverrides={ "command": [ f'--env={params.env}', f'--start-date={start_date_str}', f'--end-date={end_date_str}', f'--logs-path=s3://{params.env}-delphi-configs/logs/{params.licensor}', f'--licensor={params.licensor}', '--debug', ], "environment": [ { "name": "AWS_DEFAULT_REGION", "value": "us-east-1", }, { "name": "PACKAGE", "value": "appreciationengine-backfill", } ], # 'vcpus': '4', # "memory": 8000, }, timeout={ 'attemptDurationSeconds': 7200, }, ) logger.info('batch submission result: %s', response) except Exception as exc: logger.exception('Failed to submit task: %s', exc) finally: start_date += timedelta(days=1) def get_params(): parser = ArgumentParser() parser.add_argument( '-sd', '--start-date', type=str, required=True, help='start report date %YYYY-%MM-%DD (e.g. 2020-03-01)' ) parser.add_argument( '-ed', '--end-date', type=str, required=True, help='end report date %YYYY-%MM-%DD (e.g. 2020-03-01)' ) parser.add_argument( '-e', '--env', type=str, choices=[x.value for x in list(EnvironmentEnum)], required=True, ) parser.add_argument( '--licensor', type=str, required=True, ) parser.add_argument('--debug', action='store_true') parser.add_argument('--humanize', action='store_true') params = parser.parse_args() return params if __name__ == '__main__': batch_client = boto3.client('batch') params = get_params() logger_factory = SimpleLoggerFactory( 'appreciationengine_runner', params.debug, (not params.humanize), trace_id=None, ) logger = logger_factory.get_logger() run(logger, batch_client, params)