import os from sqlalchemy.engine import url from sqlalchemy.pool import NullPool from urllib.parse import urlparse DEV_ENVIRONMENT = 'dev' ENVIRONMENT = os.environ.get('Environment', DEV_ENVIRONMENT) DEBUG = os.environ.get('DEBUG', False) parsed = urlparse(os.environ.get('AR_DB_URL')) AR_HOST = parsed.hostname AR_USER = parsed.username AR_PASSWORD = parsed.password AR_DATABASE = (parsed.path).strip('/') AR_PORT = os.environ.get('AR_PORT', 3306) S3_PATH = os.environ.get('S3_PATH', 'mr-registry-backfill') YT_FILES = {} YT_FILES['orch'] = os.environ.get( 'YT_ORCH_FILE', 'asset_full_report_theorchardmusic_L.csv') YT_FILES['ioda'] = os.environ.get( 'YT_IODA_FILE', 'asset_full_report_IODA_L.csv') MEAT_FILE = os.environ.get('MEAT_FILE', 'MEAT-00000-of-00001.csv') MAPPER_NUMBER = os.environ.get('MAPPER_NUMBER', 30) parsed = urlparse(os.environ.get('TARGET_DB_URL')) TARGET_HOST = LOG_HOST = parsed.hostname TARGET_USER = LOG_USER = parsed.username TARGET_PASSWORD = LOG_PASSWORD = parsed.password TARGET_DATABASE = LOG_DATABASE = (parsed.path).strip('/') TARGET_TABLE = os.environ.get('TARGET_TABLE', 'mr_backfill_staging') TARGET_PORT = LOG_PORT = os.environ.get('TARGET_PORT', 3306) LOG_TABLE = os.environ.get('LOG_TABLE', 'mr_backfill_staging_log') TARGET_DB_URL=url.URL( 'mysql+pymysql', TARGET_USER, TARGET_PASSWORD, TARGET_HOST, TARGET_PORT, TARGET_DATABASE) FTP_MEAT_URL = os.environ.get('FTP_MEAT_URL') FTP_MEAT_USER = os.environ.get('FTP_MEAT_USER') FTP_MEAT_PASSWORD = os.environ.get('FTP_MEAT_PASSWORD') FTP_TIMEOUT = os.environ.get('FTP_TIMEOUT', 120) POOLCLASS = NullPool TEST_HOST = os.environ.get('TEST_HOST') TEST_USER = os.environ.get('TEST_USER') TEST_PASSWORD = os.environ.get('TEST_PASSWORD') TEST_DATABASE = os.environ.get('TEST_DATABASE') TEST_TABLE = os.environ.get('TEST_TABLE', 'youtube_asset_map') TEST_PORT = os.environ.get('TEST_PORT', 3306) if os.environ.get('FS_DB_URL'): parsed = urlparse(os.environ.get('FS_DB_URL')) FS_HOST = parsed.hostname FS_USER = parsed.username FS_PASSWORD = parsed.password FS_DATABASE = (parsed.path).strip('/') FS_PORT = os.environ.get('FS_PORT', 5439) FS_DB_URL=url.URL( 'postgresql+psycopg2', FS_USER, FS_PASSWORD, FS_HOST, FS_PORT, FS_DATABASE) ROW_STEP = os.environ.get('ROW_STEP', 100000) if ENVIRONMENT != DEV_ENVIRONMENT: S3_BUCKET = '{}-orcdbucket'.format(ENVIRONMENT) else: S3_BUCKET = 'dev-orchdbucket' TRUNCATE_TARGET=os.environ.get('TRUNCATE_TARGET', 'False') log_fields = ['id', 'filename', 'reason', 'asset_id', 'recent_daily_average', 'asset_type', 'custom_id', 'isrc', 'grid', 'upc', 'artist', 'song', 'album', 'label', 'constituent_asset_ids', 'active_reference_ids', 'inactive_reference_ids', 'is_merged_asset', 'embedded_asset_id']