"""Script that trigger ACRCloud recognizer for multiple files""" import json from time import sleep import boto3 from lambdacommon import common_query from lambdacommon import mysql import config import util BUCKET = 'prod-acrcloud-recognizer' PAGE_SIZE = 100 LIMIT = 5000 s3_client = boto3.client('s3') lambda_client = boto3.client('lambda', region_name='us-east-1') paginator_config = { 'PageSize': PAGE_SIZE } # first batch is located at fingerprinting/2020-07-20// # AudibleMagicRights ID test located at fingerprinting/2020-09-18/ def recognize_files(files): for key in files: response = lambda_client.invoke( FunctionName='{}-prod'.format(config.APPLICATION_NAME), InvocationType='Event', LogType='None', Payload=json.dumps({ 'detail': { 'requestParameters': { 'bucketName': BUCKET, 'key': key }, } }), ) print(response) def get_files_from_db(flag): with mysql.mysql_connection(**config.DB_CONFIG) as conn: with conn.cursor() as cursor: cursor.execute(common_query.SELECT_SONGS) songs = list(cursor) files = dict() for song in songs: files[song['song_name']] = song[flag] return files def get_unrecognized_from_s3(path): files = get_files_from_db('acrcloud_rights') print(len(files)) filenames = files.keys() paginator = s3_client.get_paginator('list_objects') page_iterator = paginator.paginate( Bucket=BUCKET, Prefix=path, PaginationConfig=paginator_config ) files_to_recognize = [] for page in page_iterator: items = page['Contents'] # One page of image files. for item in items: key = item['Key'] filename = util.get_audio_name_from_path(key)[0] if filename not in filenames or files[filename] is None: files_to_recognize.append(key) if len(files_to_recognize) == LIMIT: break if len(files_to_recognize) == LIMIT: break return files_to_recognize if __name__ == '__main__': f = get_unrecognized_from_s3('') print(len(f)) recognize_files(f)