"""Docstring.""" import csv import json import sys import time import boto3 import botocore from botocore.exceptions import ClientError from botocore.exceptions import ConnectionClosedError class RetryableException(Exception): """Retryable operation.""" def __init__(self, message): """Init.""" super().__init__(message) lambda_client = boto3.client( 'lambda', config=botocore.config.Config( retries={'max_attempts': 0}, read_timeout=900 # wait for max lambda runtime ), region_name='us-east-1' ) def main(): """Entrypoint.""" filename = sys.argv[1] env = sys.argv[2] after = sys.argv[3] if len(sys.argv) > 3 else None max_retries = 5 backoff = 180 print(filename, env, after) # read CSV file and validate osrs = [] with open(filename, 'r', encoding='utf-8-sig') as f: reader = csv.DictReader(f) if set(reader.fieldnames) != set(['OSR_ID']): print('Invalid input CSV column format!') exit(1) osrs = [x for x in reader] # sort to allow re-processing from middle process = False osrs = sorted(osrs, key=lambda x: x['OSR_ID']) num_processed = 0 total_osrs = len(osrs) for osr in osrs: osr_id = osr['OSR_ID'].replace('"', '') retries = max_retries # skip if processing only after specific osr if not process and after: if osr_id == after: process = True num_processed += 1 continue while True: if retries == 0: print(f'Retried {max_retries} times.. Stopping.') break try: result = invoke_lambda_compilation(osr_id, env) num_processed += 1 percent_processed = (num_processed / total_osrs) * 100 print(f'Processed {num_processed} / {total_osrs} : {percent_processed}%') break except RetryableException as re: print(f'Retryable Exception.. {re}') retries -= 1 time.sleep(backoff) except Exception as e: print(f'Non-Retryable Exception.. {e}') raise e # all done print('Complete!') exit() def invoke_lambda_compilation(osr_id, env): """Invoke lambda for new compiled version.""" print(f'Invoking {env}-lambda-sr-add-version for OSR ID: {osr_id}') osr_payload = { 'label': 'OrchardSoundRecording', 'id': osr_id } try: lambda_response = lambda_client.invoke( FunctionName=f'{env}-lambda-sr-add-version', Payload=json.dumps(osr_payload).encode('utf-8') ) except ConnectionClosedError as connection_e: raise RetryableException(f'Error Connection close: {connection_e}') payload = json.loads(lambda_response['Payload'].read().decode('utf-8')) if 'errorMessage' in payload: print(payload) raise RetryableException(f'Add version invoke failed for OSR ID: {osr_id}') print('Successful compilation...') return payload['results'] if __name__ == '__main__': main()