"""Script that trigger ACRCloud recognizer for multiple files""" import boto3 from botocore.exceptions import ClientError import mysql import config import query import util BUCKET = 'dev-orcd-raw-assets' PREFIX = '2020-07-20//' PAGE_SIZE = 100 LIMIT = 10000 s3_client = boto3.client('s3') lambda_client = boto3.client('lambda', region_name='us-east-1') paginator_config = { 'PageSize': PAGE_SIZE } batch = boto3.client('batch') JOB_QUEUE = 'arn:aws:batch:us-east-1:103233932089:job-queue/dev-audiblemagic-batch-queue' JOB_DEFINITION = 'dev-audiblemagic-identifier-job' if __name__ == '__main__': with mysql.mysql_connection(**config.DB_CONFIG) as conn: with conn.cursor() as cursor: cursor.execute(query.SELECT_SONGS) songs = list(cursor) files = dict() for song in songs: files[song['song_name']] = song['audiblemagic'] filenames = files.keys() paginator = s3_client.get_paginator('list_objects') page_iterator = paginator.paginate( Bucket=BUCKET, Prefix=PREFIX, PaginationConfig=paginator_config ) files_to_recognize = [] for page in page_iterator: items = page['Contents'] # One page of image files. for item in items: key = item['Key'] filename = util.get_audio_name_from_path(key)[0] if filename not in filenames or files[filename] is None: files_to_recognize.append(key) if len(files_to_recognize) == LIMIT: break if len(files_to_recognize) == LIMIT: break for key in files_to_recognize: try: f = util.get_audio_name_from_path(key)[0] response = batch.submit_job( jobName='audiblemagic-{}'.format(f), jobQueue=JOB_QUEUE, jobDefinition=JOB_DEFINITION, parameters={ 'S3bucket': BUCKET, 'S3key': key } ) print(response) except ClientError as e: print(e)