"""Docstring.""" import csv import json import sys import time import boto3 from botocore.exceptions import ClientError def main(): """Entrypoint.""" filename = sys.argv[1] env = sys.argv[2] after = sys.argv[3] if len(sys.argv) > 3 else None max_running = 15 sfn_arn = f'arn:aws:states:us-east-1:437795906767:stateMachine:{env}-sr-delivery-sfn' # noqa:E501 sfn_client = boto3.client('stepfunctions', region_name='us-east-1') # read in CSV file and validate osrs = [] with open(filename, 'r', encoding='utf-8-sig') as f: reader = csv.DictReader(f) print(reader.fieldnames) if set(reader.fieldnames) != set(['SOUND_RECORDING_ID', 'SOUND_RECORDING_VERSION_ID', 'STEP_FUNCTION_EXECUTION_ID', 'EXECUTION_TYPE']): # noqa:E501 print('Invalid input CSV column format!') exit(1) osrs = [x for x in reader] # sort to allow re-processing from middle process = False osrs = sorted(osrs, key=lambda x: x['SOUND_RECORDING_ID']) num_processed = 0 total_osrs = len(osrs) for osr in osrs: osr_id = osr['SOUND_RECORDING_ID'] version_id = osr['SOUND_RECORDING_VERSION_ID'] exc_name = osr['STEP_FUNCTION_EXECUTION_ID'] is_full_delivery = osr['EXECUTION_TYPE'] == 'FULL_DELIVERY' # skip if processing only after specific OSR if not process and after: num_processed += 1 if version_id == after: process = True continue # throttle based on num running state machines while True: try: sfn_list_response = sfn_client.list_executions( stateMachineArn=sfn_arn, statusFilter='RUNNING', maxResults=max_running ) num_running = len(sfn_list_response['executions']) if num_running >= max_running: print("SLEEP") time.sleep(5) else: break except ClientError as e: if e.response['Error']['Code'] == 'ThrottlingException': print(f'ThrottlingException for osr {version_id}') time.sleep(60) break else: raise e # kick-off new execution payload = { 'sound_recording': { 'id': osr_id, 'version': version_id }, 'upload_asset': is_full_delivery, } sfn_client.start_execution( stateMachineArn=sfn_arn, name='-'.join([version_id, 'manual-delivery', str(int(time.time()))]), input=json.dumps(payload) ) num_processed += 1 percent_processed = (num_processed / total_osrs) * 100 print(f'Processed {osr_id} {num_processed} / {total_osrs} : {percent_processed}%') # noqa:E501 if __name__ == '__main__': main()