import time import sys from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import NullPool import config import queries try: NUMBER_OF_TRACKS_TO_UPDATE_PER_TX = int(sys.argv[1]) except IndexError: print('please provide how many tracks you would like to process at a time') exit(1) try: START_ROW = int(sys.argv[2]) except IndexError: print('provide the start row offset') exit(1) try: END_ROW = int(sys.argv[3]) except IndexError: print('provide the last row to process') exit(1) try: PROCESS_NUMBER = int(sys.argv[4]) except IndexError: print('please provide process number') exit(1) try: SLEEP_TIME_PER_UPDATE = int(sys.argv[5]) except IndexError: print('please provide sleep time in seconds between each update') exit(1) _art_relations_engine = create_engine( config.DB_URL, poolclass=NullPool) _art_relations_session = sessionmaker(bind=_art_relations_engine) session = _art_relations_session() rows = [] tuids_not_processed =[] queries_ran = [] batch_size = NUMBER_OF_TRACKS_TO_UPDATE_PER_TX for offset in range(START_ROW, END_ROW, NUMBER_OF_TRACKS_TO_UPDATE_PER_TX): tracks = [] if offset + NUMBER_OF_TRACKS_TO_UPDATE_PER_TX >= END_ROW: batch_size = (END_ROW - START_ROW) % NUMBER_OF_TRACKS_TO_UPDATE_PER_TX queries_ran.append(queries.log_paginated_track_select_query.format( batch_size, offset)) tracks = session.execute( queries.paginated_track_select_query, {'limit': batch_size, 'offset': offset}) tuids = [] for track in tracks: tuids.append(track['id']) try: print(tuids) session.execute( queries.dmclips_update_query, {'dmclips': 'cloudsearch_backfill', 'tuids': tuids}) session.commit() time.sleep(SLEEP_TIME_PER_UPDATE) except Exception: tuids_not_processed = tuids_not_processed + tuids session.close() with open('process_{}.log'.format(PROCESS_NUMBER), 'w+') as f: f.write('\n'.join(queries_ran)) if tuids_not_processed: with open('error-process_{}.log'.format(PROCESS_NUMBER), 'w+') as f: f.write(','.join(tuids_not_processed))