"""Creates batches of v2 transcoding orders via ows-transcoding. Usage: python create_transcoding_orders.py input_file [last_file_name] - input_file: Path of a text file listing v2 asset filenames to re-enqueue for transcoding. Example of an asset file name: fb73a74f_1027_45f6_9ec8_ad751956a558.wav - last_file_name (optional): Name of the last file in the list that was processed. This is useful for resuming processing of a list that has already been partially processed. @todos: - Make batch size and sleep_seconds CLI args - Use python-owsrequest for the API calls - Maybe add a thread that polls the status of orders and backs off if too many are still pending """ import json import sys import time import requests POST_HEADERS = { 'Content-Type': 'application/json' } TRANSCODING_URL = ( 'https://prod-ows-transcoding.theorchard.io' '/transcoding/preset/prod_mezzanine' ) BATCH_SIZE = 60 SLEEP_SECS = 60 def create_order(asset_file_name): payload = { 'status_topic_alias': 'prod_mezzanine_sns_alias', 'input': { 'bucket': 'prod-orcd-raw-assets', 'key': asset_file_name } } return requests.post( TRANSCODING_URL, data=json.dumps(payload), headers=POST_HEADERS ) def stream_files(input_file_name, last_file=None): skip_files = False if last_file: print('Skipping files through {}'.format(last_file)) skip_files = True with open(input_file_name) as input_fp: for line in input_fp: current_file = line.strip() if skip_files: if current_file == last_file: # This will be the last file that we skip skip_files = False print('Resuming processing after {}'.format(current_file)) else: yield current_file def create_orders(asset_file_names): order_count = 0 for asset_file_name in asset_file_names: response = create_order(asset_file_name) if response.status_code == 200: order_count += 1 order_id = response.json()['transcoding_order_id'] print('Created order {} for {}'.format(order_id, asset_file_name)) else: print('Received status {} for file {}'.format( response.status_code, asset_file_name)) print(response.json()) if order_count % BATCH_SIZE == 0: print('Waiting {} seconds...'.format(SLEEP_SECS)) time.sleep(SLEEP_SECS) return order_count if __name__ == '__main__': input_file_name = sys.argv[1] last_file = None if len(sys.argv) > 2: last_file = sys.argv[2] create_orders(stream_files(input_file_name, last_file))