import os import sys import time from functools import partial from multiprocessing.pool import ThreadPool import requests import sqlalchemy from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker PRICING_DB_URL = os.environ.get('PRICING_DB_URL', '') OWS_PRICING_URL = os.environ.get('OWS_PRICING_URL', '') POOL_SIZE = os.environ.get('POOL_SIZE', '32') GET_EXISITING_PRODUCT_IDS_QUERY = """ SELECT product_id FROM product_migration_status WHERE migration_status='NeedPhase1Migration' OR migration_status='Phase1MigrationDone' """ CREATE_PRODUCT_MIGRATION_STATUS = """ INSERT INTO product_migration_status (product_id, migration_status) VALUES (:product_id, 'NeedPhase1Migration') """ db_engine = create_engine(PRICING_DB_URL) db_session = sessionmaker(bind=db_engine) def get_session(): return db_session() def get_time(start, end): diff = end - start return '%.2f'%(diff) def get_legacy_product_ids(): url = '{0}/legacy/music-releases'.format(OWS_PRICING_URL) response = requests.get(url) if response.status_code != 200: response.raise_for_status() return response.json()['items'] def get_existing_product_ids(): session = get_session() try: rows = session.execute(sqlalchemy.text(GET_EXISITING_PRODUCT_IDS_QUERY)).fetchall() session.commit() return [row['product_id'] for row in rows] except Exception as e: session.rollback() raise e finally: session.close() def create_product_migration_status(legacy_product_id, existing_product_ids): print('About to create product migration status for product {0}'.format( legacy_product_id)) if legacy_product_id in existing_product_ids: print('Skipping product {0}'.format(legacy_product_id)) return session = get_session() query = CREATE_PRODUCT_MIGRATION_STATUS try: session.execute(sqlalchemy.text(query), {'product_id': legacy_product_id}) session.commit() print('Created product migration status for product {0}'.format( legacy_product_id)) except Exception as e: session.rollback() raise e finally: session.close() try: print('About to fetch legacy product IDs to populate') t1 = time.time() legacy_product_ids = get_legacy_product_ids() t2 = time.time() print('Fetched {0} legacy product IDs to populate in {1}s'.format( len(legacy_product_ids), get_time(t1, t2))) print('About to fetch existing product IDs') t3 = time.time() existing_product_ids = get_existing_product_ids() t4 = time.time() print('Fetched {0} existing product IDs in {1}s'.format( len(existing_product_ids), get_time(t3, t4))) partial_map = partial( create_product_migration_status, existing_product_ids=existing_product_ids) with ThreadPool(int(POOL_SIZE)) as p: p.map( partial_map, legacy_product_ids) t5 = time.time() print('Done populating in {0}s'.format(get_time(t1, t5))) sys.exit() except Exception as e: sys.exit(e)