"""Script that trigger ACRCloud recognizer for multiple files""" # DYLD_LIBRARY_PATH=/Users/raed/work/orchard/collab/ratoui/fingerprinting-services/audiblemagic/audiblemagic_toolkit/mac/bin from typing import Dict, List import boto3 import config import mysql import query import util from index import process_file BUCKET = 'dev-orcd-raw-assets' PAGE_SIZE = 100 LIMIT = 3000 s3_client = boto3.client('s3') lambda_client = boto3.client('lambda', region_name='us-east-1') paginator_config = { 'PageSize': PAGE_SIZE } batch = boto3.client('batch') # first batch is located at fingerprinting/2020-07-20// # AudibleMagicRights ID test located at fingerprinting/2020-09-18/ def get_files_from_db() -> List[Dict[str, bool]]: with mysql.mysql_connection(**config.DB_CONFIG) as conn: with conn.cursor() as c: c.execute(query.SELECT_SONGS) songs = list(c) files = dict() for song in songs: files[song['song_name']] = song['audiblemagic'] return files def get_unidentified_files_from_s3(prefix: str) -> List[str]: files = get_files_from_db() filenames = files.keys() paginator = s3_client.get_paginator('list_objects') page_iterator = paginator.paginate( Bucket=BUCKET, Prefix=prefix, PaginationConfig=paginator_config ) files_to_recognize = [] for page in page_iterator: items = page['Contents'] # One page of image files. for item in items: k = item['Key'] filename = util.get_audio_name_from_path(k)[0] if filename not in filenames or files[filename] is None: files_to_recognize.append(k) if len(files_to_recognize) == LIMIT: break if len(files_to_recognize) == LIMIT: break return files_to_recognize if __name__ == '__main__': f = get_unidentified_files_from_s3('2020-07-20//') with mysql.mysql_connection(**config.DB_CONFIG) as conn: with conn.cursor() as cursor: for key in f: try: process_file(cursor, BUCKET, key) except Exception as e: print(e)