from datetime import datetime import argparse import logging import os import signal import sys import time import pymysql sys.path.append(os.path.join(os.path.dirname(__file__), '..')) from product.connector import mysql from product.sql import releases log = logging.getLogger('main') log.setLevel(logging.DEBUG) fmt = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') sh = logging.StreamHandler(sys.stdout) sh.setFormatter(fmt) log.addHandler(sh) OFFSET = 0 LIMIT = 10000 # Just a date from the past, I do not think that anyone has ever touched any # releases earlier. START_DATE = '1986-02-01 00:00:00' END_DATE = datetime.now().strftime('%Y-%m-%d %H:%M:%S') DELAY_SECONDS = 5 def signal_handler(sig, frame): log.info('You pressed Ctrl+C!') sys.exit(0) signal.signal(signal.SIGINT, signal_handler) def update_releases(rows, cursor, ar_db): if not rows: return for release in rows: try: cursor.execute( releases.UPDATESQL, { 'release_id': release['release_id'] } ) ar_db.commit() log.info('UPDATE SUCCESS FOR release_id %s' % release['release_id']) except (pymysql.MySQLError, pymysql.Warning) as e: log.error( 'UPDATE FAILED FOR release_id %s error : %s', release['release_id'], e) continue except Exception: log.exception('Failed to update releases.') exit(1) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--offset', type=int, default=OFFSET) parser.add_argument('--limit', type=int, default=LIMIT) parser.add_argument('--start_date', type=str, default=START_DATE) parser.add_argument('--end_date', type=str, default=END_DATE) parser.add_argument('--delay', type=int, default=DELAY_SECONDS) parser.add_argument('--release_ids', type=str, default=None) args = parser.parse_args() log.info('START') log.info( 'start_date: %s end_date: %s limit: %s offset: %s delay: %s', args.start_date, args.end_date, args.limit, args.offset, args.delay) release_rows = True release_count = 0 ar_db = None cursor = None while release_rows: try: ar_db = mysql.get_ar_mysql_connection() cursor = ar_db.cursor() if not args.release_ids: cursor.execute(releases.SELECTSQL.format( limit=args.limit, offset=args.offset), { 'start_date': args.start_date, 'end_date': args.end_date } ) else: cursor.execute( releases.SELECT_BY_PRODUCT_ID_SQL.format( release_ids=args.release_ids) ) release_rows = cursor.fetchall() release_count += len(release_rows) update_releases(release_rows, cursor, ar_db) # Stop the loop if updates were done for specific product_ids if args.release_ids: release_rows = False continue log.info('Sleeping for %s sec..', args.delay) time.sleep(args.delay) except pymysql.Warning as e: log.error('MySQL Warning : %s', str(e)) pass except Exception: log.exception('Failed to query releases.') exit(1) finally: if cursor: cursor.close() if ar_db: ar_db.close() log.info('All %s releases have been updated.', release_count) log.info('COMPLETE')