import os import sys import time from multiprocessing.pool import ThreadPool import requests import sqlalchemy from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker PRICING_DB_URL = os.environ.get('PRICING_DB_URL', '') OWS_PRICING_URL = os.environ.get('OWS_PRICING_URL', '') STATUS_BEFORE = os.environ.get('STATUS_BEFORE', 'NeedPhase1Migration') STATUS_AFTER = os.environ.get('STATUS_AFTER', 'Phase1MigrationDone') PRODUCT_BATCH_SIZE = os.environ.get('PRODUCT_BATCH_SIZE', '10') POOL_SIZE = os.environ.get('POOL_SIZE', '4') GET_PRODUCTS_QUERY = """ SELECT pms.product_id FROM product_migration_status pms LEFT OUTER JOIN product_orchard_pricing_tier popt on pms.product_id = popt.product_id WHERE pms.migration_status=:status_before AND popt.product_orchard_pricing_tier_id IS NULL LIMIT :product_batch_size """ db_engine = create_engine(PRICING_DB_URL) db_session = sessionmaker(bind=db_engine) def get_session(): return db_session() def get_products(): session = get_session() try: result = session.execute( sqlalchemy.text(GET_PRODUCTS_QUERY), { 'status_before': STATUS_BEFORE, 'product_batch_size': int(PRODUCT_BATCH_SIZE) } ).fetchall() session.commit() return result except Exception as e: session.rollback() raise e finally: session.close() def get_migration_url(product_id): qs = 'status_before={0}&status_after={1}'.format( STATUS_BEFORE, STATUS_AFTER) return '{0}/legacy/music/product/{1}/execute?{2}'.format( OWS_PRICING_URL, product_id, qs) def get_time(start, end): diff = end - start return '%.2f'%(diff) def migrate_product(row): print('About to migrate product {0}'.format(row['product_id'])) t3 = time.time() pricing_response = requests.get(get_migration_url(row['product_id'])) t4 = time.time() if pricing_response.status_code == 400: print('Product {0} has already been migrated'.format( row['product_id'])) return elif pricing_response.status_code != 200: pricing_response.raise_for_status() print('Migrated product {0} in {1}s'.format( row['product_id'], get_time(t3, t4))) try: print('About to fetch products to migrate') t1 = time.time() product_rows = get_products() t2 = time.time() print('Fetched {0} products to migrate in {1}s'.format( len(product_rows), get_time(t1, t2))) with ThreadPool(int(POOL_SIZE)) as p: p.map(migrate_product, product_rows) t5 = time.time() print('Done migrating in {0}s'.format(get_time(t1, t5))) sys.exit() except Exception as e: sys.exit(e)