import json import boto3 client = boto3.client('batch') download_chunk_size = 16 download_concurrency = 5 upload_concurrency = 15 upload_queue_size = 4 uow = { "uow_id": "spotify-20190822-sme-streams-v2-manycontexts-{0}mb-{1}dc-{2}uc-{3}uqs-c5n-6".format( download_chunk_size, download_concurrency, upload_concurrency, upload_queue_size, ), "dsp": "spotify", "extension": "json", "config_key": "ecs-run-config.json", "licensor": "sme", "config_bucket": "dev-delphi-configs", "report_type": "streams", "contexts": [ "US", "AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MV", "MW", "MX", "MY", "MZ", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "SK", "TW", "UY", "UZ", "VA", "VE", "YE", "YT", "ZA", "ZM", "ZW", "ZZ" ], "version": "v2", "report_date": "2019-08-22", "job_id": "srjob", } jsonified_uow = json.dumps(uow) command = [ '--environment=dev', f'--download-concurrency={download_concurrency}', '--chunk-size={}'.format(str(download_chunk_size * 1024 * 1024)), f'--upload-concurrency={upload_concurrency}', f'--upload-queue-size={upload_queue_size}', "--content-status-table-name=dev-delphi-slz_content_status", "--bucket-archive=dev-sme-data-archive", "--bucket-decompressed=dev-sme-data-decompressed", "--bucket-corrupted=dev-delphi-sme-data-corrupted", "--input={}".format(jsonified_uow).replace(" ", "", -1), ] print(" \\ ".join(command)) response = client.submit_job( jobName=uow['uow_id'], jobQueue='dev-delphi-c5n', jobDefinition='dev-delphi-slz-download-1:4', containerOverrides={ "command": command, "environment": [{ "name": "AWS_DEFAULT_REGION", "value": "us-east-1", }], "memory": 8000, }, ) print(response)