"""Config file.""" from distutils.util import strtobool import json import os import sys from constants import api as api_consts from constants import database as db_consts from integration_scripts.common_config import ENVIRONMENT, DEV_ENVIRONMENT, \ QA_ENVIRONMENT, PROD_ENVIRONMENT # Work-around urllib3 bug on OS X. See: https://bugs.python.org/issue28342 # https://blog.yimingliu.com/2015/07/22/python-multiprocessing-code-crashes-on-os-x-under-ipython/ # noqa if sys.platform == 'darwin': os.environ['no_proxy'] = '*' # Number of concurrent threads NUM_THREADS = int(os.environ.get('NUM_THREADS', 5)) # noqa # Environment ---------------------------------------------------------------- # VAPI endpoints, SSL, and S3 Warehouses based on target environment. if ENVIRONMENT == DEV_ENVIRONMENT: # noqa BULK_UPLOAD_API_URL = os.environ.get( 'BULK_UPLOAD_API_URL', api_consts.BULK_UPLOAD_DEV_API_URL) USE_SSL = strtobool( str(os.environ.get('USE_SSL', False))) # Or use hard-coded VAPI for QA elif ENVIRONMENT == QA_ENVIRONMENT: # noqa BULK_UPLOAD_API_URL = api_consts.BULK_UPLOAD_QA_API_URL USE_SSL = True SNOWFLAKE_WAREHOUSE = db_consts.QA_WAREHOUSE # Or use hard-coded VAPI for PROD elif ENVIRONMENT == PROD_ENVIRONMENT: # noqa BULK_UPLOAD_API_URL = api_consts.BULK_UPLOAD_PROD_API_URL USE_SSL = True SNOWFLAKE_WAREHOUSE = db_consts.PROD_WAREHOUSE # Snowflake ------------------------------------------------------------------ SNOWFLAKE_FILE_FORMAT = os.environ.get( 'SNOWFLAKE_FILE_FORMAT', 'BULK_UPLOAD_CSV') SNOWFLAKE_FILE_FORMAT_OUTPUT = os.environ.get( 'SNOWFLAKE_FILE_FORMAT_OUTPUT', 'BULK_UPLOAD_CSV_OUTPUT') # The table which contains the pre-formatted rows SNOWFLAKE_SOURCE_TABLE = os.environ.get('SNOWFLAKE_SOURCE_TABLE', '') SNOWFLAKE_SYNC_TARGET_DB = os.environ.get( 'SNOWFLAKE_SYNC_TARGET_DB', 'ORCHARD_APP_REPORTING_V2') SNOWFLAKE_SYNC_TARGET_SCHEMA = os.environ.get( 'SNOWFLAKE_SYNC_TARGET_SCHEMA', 'DEV_INTEGRATIONS_BULK_UPLOAD') # S3 ------------------------------------------------------------------------- # Controls whether S3 Ingestion and Output are used S3_INGEST = strtobool(str(os.environ.get('S3_INGEST', True))) S3_OUTPUT = strtobool(str(os.environ.get('S3_OUTPUT', True))) S3_MOVE_INGESTED = strtobool(str(os.environ.get('S3_MOVE_INGESTED', True))) S3_FOLDER = os.environ.get('S3_FOLDER', 'bulk-upload-feeder') # App Specific --------------------------------------------------------------- # A single release id to process INGESTION_KEY_VALUE = str(os.environ.get('INGESTION_KEY_VALUE', '')) # A single release id to process INGESTION_KEY_FIELD = str(os.environ.get('INGESTION_KEY_FIELD', '')) # Limit labels being uploaded - MUST CONTAIN AT LEAST ONE LABEL IN DATA SET BULK_UPLOAD_LABELS = json.loads(os.environ.get('BULK_UPLOAD_LABEL_LIST', '[]')) # Files and Folders FILE_OUTPUT_PATH = os.environ.get( 'FILE_OUTPUT_PATH', 'tmp/bulk_upload_feeder/{date}') # Automatically reload and resend failed products one (1) additional time REQUEUE_FAILED = strtobool( str(os.environ.get('REQUEUE_FAILED', True))) # if INGESTION_KEY_VALUE exists, this forces that release into this Label_ID FORCE_LABEL_ID = int(os.environ.get('FORCE_LABEL_ID', 0)) # If provided, this resumes the session specified by this id. RESUME_SESSION_ID = os.environ.get('RESUME_SESSION_ID', None) # Timeout to delay processing until backend has a chance to process all queries NEW_ARTIST_SLEEP = int(os.environ.get('NEW_ARTIST_SLEEP', 60)) CHECK_ARTISTS = strtobool(str(os.environ.get('CHECK_ARTISTS', False))) # Allow app to continue when all NEW artists do not appear in art_relations # after the first pass ALLOW_INCOMPLETE_ARTISTS = strtobool( str(os.environ.get('ALLOW_INCOMPLETE_ARTISTS', False))) # ------- App Logging -------------------------------------------------------- # Write requests and responses to files # If enabled, debug info is written to files in the ./tmp/ directory # (grouped by date). This includes the entirety of the console log, # plus each request and response. DEBUG_WRITE_TO_FILE = strtobool(str(os.environ.get( 'DEBUG_WRITE_TO_FILE', False))) # Development: Disables logging of multiprocess responses DEBUG_SKIP_LOG = strtobool( str(os.environ.get('DEBUG_SKIP_LOG', False))) DEBUG_LOG_RESPONSE_CONSOLE = strtobool( str(os.environ.get('DEBUG_LOG_RESPONSE_CONSOLE', False))) # Development: Enables the logging the entire request body to the Snowflake DB # WARNING: this causes large logging delays due to request size. # TODO: Test that this LOG_REQUEST still works with RDS (maybe better?!?) LOG_REQUEST = strtobool(str(os.environ.get('LOG_REQUEST', False))) # Log to file level FILE_LOG_LEVEL = str(os.environ.get('FILE_LOG_LEVEL', 'INFO')) # Determines what heading map to expect for ingested file INPUT_FORMAT = str(os.environ.get('INPUT_FORMAT', 'BULK_UPLOAD')) # ------- DEBUGGING ---------------------------------------------------------- # Development: If True, VAPI requests are mocked for testing. MOCK_REQUESTS = strtobool(str(os.environ.get('MOCK_REQUESTS', False))) # Development: If True, enables replacement of values during preprocessing REPAIR_VALUES = strtobool( str(os.environ.get('REPAIR_VALUES', True))) # Development: If True, all rows have Master Rights Ownership field stripped SEND_BLANK_MR_OWNERSHIP = strtobool( str(os.environ.get('SEND_BLANK_MR_OWNERSHIP', False))) # ------ FEATURE FLAGS ------------------------------------------------------- CREATE_PARTICIPANTS = strtobool( str(os.environ.get('CREATE_PARTICIPANTS', False))) SET_DEFAULT_TRACKS = strtobool( str(os.environ.get('SET_DEFAULT_TRACKS', False))) SET_PROJECT_NAMES = strtobool(str(os.environ.get('SET_PROJECT_NAMES', False))) SET_NOT_FOR_DIST = strtobool(str(os.environ.get('SET_NOT_FOR_DIST', False))) APPROVE_RELEASES = strtobool(str(os.environ.get('APPROVE_RELEASES', False))) # Forces outfiles to be written at end of run regardless of fivetran success FORCE_WRITE_OUTFILES = strtobool( str(os.environ.get('FORCE_WRITE_OUTFILES', False))) STRIP_WHITESPACE = strtobool( str(os.environ.get('STRIP_WHITESPACE', False))) PAD_DIGITAL_UPC = strtobool( str(os.environ.get('PAD_DIGITAL_UPC', False))) RETAIN_LEADING_ZEROES = strtobool( str(os.environ.get('RETAIN_LEADING_ZEROES', False))) REMOVE_NAN = strtobool(str(os.environ.get('REMOVE_NAN', False))) # Blank out the lyrics field STRIP_LYRICS = strtobool( str(os.environ.get('STRIP_LYRICS', False))) FIVETRAN_SYNC_WAIT = int(os.environ.get('FIVETRAN_SYNC_WAIT', 900)) FIVETRAN_SYNC_TRIES = int(os.environ.get('FIVETRAN_SYNC_TRIES', 4))