from datetime import datetime import argparse import logging import os import signal import sys import pymysql sys.path.append(os.path.join(os.path.dirname(__file__), '..')) from artists.connector import mysql from artists.sql import artists log = logging.getLogger('main') log.setLevel(logging.DEBUG) fmt = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') sh = logging.StreamHandler(sys.stdout) sh.setFormatter(fmt) log.addHandler(sh) OFFSET = 0 LIMIT = 10000 START_DATE = '2020-03-19 00:00:00' END_DATE = datetime.now().strftime('%Y-%m-%d %H:%M:%S') def signal_handler(sig, frame): log.info('You pressed Ctrl+C!') sys.exit(0) signal.signal(signal.SIGINT, signal_handler) def update_artists(rows, cursor, ar_db): if not rows: return for artist in rows: try: cursor.execute( artists.UPDATESQL, { 'artist_id': artist['artist_id'] } ) ar_db.commit() log.info('UPDATE SUCCESS FOR artist_id %s' % artist['artist_id']) except (pymysql.MySQLError, pymysql.Warning) as e: log.error( 'UPDATE FAILED FOR artist_id %s error : %s', artist['artist_id'], e) continue except Exception: log.exception('Failed to update artists.') exit(1) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--offset', type=int, default=OFFSET) parser.add_argument('--limit', type=int, default=LIMIT) parser.add_argument('--start_date', type=str, default=START_DATE) parser.add_argument('--end_date', type=str, default=END_DATE) args = parser.parse_args() log.info('START') log.info( 'start_date: %s end_date: %s limit: %s offset: %s delay: %s', args.start_date, args.end_date, args.limit, args.offset, args.delay) artists_count = 0 ar_db = None cursor = None try: ar_db = mysql.get_ar_mysql_connection() cursor = ar_db.cursor() cursor.execute(artists.SELECTSQL.format( limit=args.limit, offset=args.offset), { 'start_date': args.start_date, 'end_date': args.end_date } ) artists_rows = cursor.fetchall() artists_count += len(artists_rows) update_artists(artists_rows, cursor, ar_db) except pymysql.Warning as e: log.error('MySQL Warning : %s', str(e)) pass except Exception: log.exception('Failed to query artists.') exit(1) finally: if cursor: cursor.close() if ar_db: ar_db.close() log.info('All %s artists have been updated.', artists_count) log.info('COMPLETE')