import json import boto3 client = boto3.client('batch') download_chunk_size = 16 download_concurrency = 5 upload_concurrency = 15 upload_queue_size = 4 jid = "apple-20191011-sme-amStreams-v1_2-v2-2" uow = { "uow_id": jid, "job_id": jid, "dsp": "apple", "report_type": "amStreams", "subtype": "Detailed", "version": "v1_2", "report_date": "2019-10-11", "licensor": "sme", "extension": "txt", "config_bucket": "dev-delphi-configs", "ecs_run_config_key": "batch-run-config.json", "is_reprocessing": False, "contexts": [ "4632::80026921", "4632::80026923", "4632::80030469", "4632::80031941", "4632::80039627", "4632::85407043", "4632::86007832", "4632::86759190", "4632::86759191", "4632::86759192", "4632::86759193", "4632::86759194", "4632::86759195", "4632::86759255", "4632::86759336", "4632::86759337", "4632::86759338", "4632::86759339", "4632::86759340", "4632::86759341", "4632::86759342", "4632::86759343", "4632::86759344", "4632::86759356", "4632::86759357", "4632::86759358", "4632::86759359", "4632::86759360", "4632::86759361", "4632::86759362", "4632::86759363", "4632::86759364", "4632::86759365", "4632::86759366", "4632::86759367", "4632::86759368", "4632::86759369", "4632::86759370", "4632::86759371", "4632::86759372", "4632::86759373", "4632::86759374", "4632::86759375", "4632::86759376", "4632::86759377", "4632::86759378", "4632::86759379", "4632::86759380", "4632::86759381" ] } jsonified_uow = json.dumps(uow) command = [ '--environment=dev', #f'--download-concurrency={download_concurrency}', #'--chunk-size={}'.format(str(download_chunk_size*1024*1024)), #f'--upload-concurrency={upload_concurrency}', #f'--upload-queue-size={upload_queue_size}', "--content-status-table-name=dev-delphi-slz_content_status", "--bucket-archive=dev-sme-data-archive", "--bucket-decompressed=dev-sme-data-decompressed", "--bucket-corrupted=dev-delphi-sme-data-corrupted", "--input={}".format(jsonified_uow).replace(" ", "", -1), ] print(" \\ ".join(command)) response = client.submit_job( jobName=uow['uow_id'], jobQueue='dev-delphi-main', jobDefinition='dev-delphi-slz-downloader:11', containerOverrides={ "command": command, "environment": [{ "name": "AWS_DEFAULT_REGION", "value": "us-east-1", }], # "memory": 8000, }, ) print(response)