import argparse import json import requests import time from Queue import Queue from threading import Thread import fastingest_beta import fp fingerprint_queue = Queue() errored_track_ids = [] def load_queue(files): print('Loading queue with fingerprints for querying') with open(files[0]) as f: fingerprints = json.loads(f.read()) for fingerprint in fingerprints: fingerprint_queue.put(fingerprint) def load_api(files): for file in files: with open(file) as f: fingerprints = json.loads(f.read()) for fingerprint in fingerprints: metadata = fingerprint.get('metadata') data = { 'track_id': metadata.get('track_id', None), 'fp_code': fingerprint.get('code', None), 'artist': metadata.get('artist', None), 'release': metadata.get('release', None), 'track': metadata.get('title', None), 'length': metadata.get('duration', None), 'codever': fingerprint.get('codever', None)} r = requests.post('http://localhost:8555/ingest', data=data) def match(q, mode, local): while True: code_data = q.get() start_time = time.time() track_id = code_data.get('metadata').get('track_id') code = code_data.get('code') if mode == 'cut': code = fp.cut_code_string_length(fp.decode_code_string(code)) print('Matching track_id {}'.format(track_id)) try: result = fp.best_match_for_query(code) print(result.match()) print("--- %s seconds ---" % (time.time() - start_time)) except Exception as e: errored_track_ids.append(track_id) print('ERROR!') print(e) q.task_done() def get_cl_args(): help_str = """ Run fingerprint matching with given file_path and thread. Use --clean to erase solr and tyrant and reload with new fingerprints """ parser = argparse.ArgumentParser(help_str) parser.add_argument('--file_path', type=str, required=True, nargs='*', help='file path to fingerprints', dest='file_path') parser.add_argument('--threads', type=int, required=False, help='number of threads to run this with', default=1, dest='threads'), parser.add_argument('--clean', action='store_true', dest='clean') parser.add_argument('--reload', action='store_true', dest='reload') parser.add_argument('--reload_amount', type=int, required=False, default=99999999, dest='reload_amount') parser.add_argument('--match', action='store_true', dest='match') parser.add_argument('--match_mode', required=False, default='default', dest='match_mode') parser.add_argument('--local', action='store_true', dest='local') parser.add_argument('--split', action='store_true', dest='split') parser.add_argument('--cut', action='store_true', dest='cut') return parser.parse_args() if __name__ == '__main__': args = get_cl_args() file_path = args.file_path load_queue(file_path) if args.clean: print('erasing database') fp.erase_database(really_delete=True) if args.reload: print('reloading fingerprints') if args.local: load_api(file_path) else: fastingest_beta.load(file_path, cut=args.cut, split=args.split, reload_amount=args.reload_amount) if args.match: threads = args.threads print('starting workers with {} threads'.format(str(threads))) for i in range(int(threads)): worker = Thread( target=match, args=(fingerprint_queue, args.match_mode, args.local,)) worker.setDaemon(True) worker.start() while not fingerprint_queue.empty(): time.sleep(1) print('done')