"""Docstring.""" import csv import json import sys import time import boto3 from botocore.exceptions import ClientError output_fieldnames = [ 'SOUND_RECORDING_ID', 'SOUND_RECORDING_VERSION_ID' ] def main(): """Entrypoint.""" filename = sys.argv[1] env = sys.argv[2] after = sys.argv[3] if len(sys.argv) > 3 else None max_running = 15 print('filename', filename) print('env', env) print('after', after) s3_client = boto3.client('s3', region_name='us-east-1') # read in CSV file and validate osrs = [] with open(filename, 'r') as f: reader = csv.DictReader(f) if set(reader.fieldnames) != set(['OSR_ID']): # noqa:E501 print('Invalid input CSV column format!') exit(1) osrs = [x for x in reader] # sort to allow re-processing from middle process = False osrs = sorted(osrs, key=lambda x: x['OSR_ID']) num_processed = 0 total_osrs = len(osrs) with open('output.csv', 'w', newline='') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=output_fieldnames) writer.writeheader() for osr in osrs: osr_id = osr['OSR_ID'] # skip if processing only after specific asset if not process and after: if osr_id == after: process = True num_processed += 1 continue try: response = s3_client.head_object( Bucket='prod-sr-versions-bucket', Key=f'{osr_id}' ) version_id = response['VersionId'] except ClientError as e: if e.response['Error']['Code'] == '404': print(f'no such key for osr {osr_id}') num_processed += 1 continue else: raise e writer.writerow({ 'SOUND_RECORDING_ID': osr_id, 'SOUND_RECORDING_VERSION_ID': version_id }) num_processed += 1 percent_processed = (num_processed / total_osrs) * 100 print(f'Processed {osr_id} {num_processed} / {total_osrs} : {percent_processed}%') # noqa:E501 if __name__ == '__main__': main()