from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from distutils.util import strtobool import os USE_SSL = True # Environment ---------------------------------------------------------------- PROD_ENVIRONMENT = 'prod' DEV_ENVIRONMENT = 'dev' QA_ENVIRONMENT = 'qa' TEST_ENVIRONMENT = 'test' ENVIRONMENT = os.environ.get('ENVIRONMENT', DEV_ENVIRONMENT).lower() # Environment ---------------------------------------------------------------- # S3 Warehouses based on target environment. if ENVIRONMENT == DEV_ENVIRONMENT: # noqa SNOWFLAKE_WAREHOUSE = 'DEV_OWS_WAREHOUSE' if ENVIRONMENT == QA_ENVIRONMENT: # noqa SNOWFLAKE_WAREHOUSE = 'QA_OWS_WAREHOUSE' elif ENVIRONMENT == PROD_ENVIRONMENT: # noqa SNOWFLAKE_WAREHOUSE = 'PROD_OWS_WAREHOUSE' # Application Vars ----------------------------------------------------------- APPLICATION_NAME = os.environ.get( 'APPLICATION_NAME', 'bulk-insert-targeted-metadata') APP_VERSION = os.environ.get('APP_VERSION', '1.0') SLEEP_TIME = float(os.environ.get('SLEEP_TIME', '0.1')) # Orchard-Specific ----------------------------------------------------------- # OA user id with creds to access Bulk Upload VAPI endpoint and insert rows OA_USER_LOGIN = str(os.environ.get('OA_USER_LOGIN', '')) # Logging -------------------------------------------------------------------- # Logger DSN (WIP) ----------------------------------------------------------- LOGGER_DSN = os.environ.get('LOGGER_DSN', None) LOGGER_NAME = \ os.environ.get('LOGGER_NAME', 'bulk-insert-targeted-metadata-logger') LOGGER_LEVEL = str(os.environ.get('LOGGER_LEVEL', 'DEBUG')) LOG_FREQUENCY = int(os.environ.get('LOG_FREQUENCY', 10)) # Log to Screen level CONSOLE_LOG_LEVEL = str(os.environ.get('CONSOLE_LOG_LEVEL', 'INFO')) # Logging Targets ------------------------------------------------------------ LOG_DIR = os.environ.get('LOG_DIR', str(os.getcwd() + '/logs')) MASTER_LOG = os.environ.get('MASTER_LOG', None) JSON_LOG = os.environ.get('JSON_LOG', None) CSV_LOG = os.environ.get('CSV_LOG', None) OTHER_LOG = os.environ.get('OTHER_LOG', None) # Snowflake ------------------------------------------------------------------ SNOWFLAKE_ACCOUNT = os.environ.get('SNOWFLAKE_ACCOUNT', 'orchard') SNOWFLAKE_USER = os.environ.get('SNOWFLAKE_USER', '') SNOWFLAKE_ROLE = os.environ.get('SNOWFLAKE_ROLE', 'DEV_ENGINEERING') SNOWFLAKE_PASSWORD = os.environ.get('SNOWFLAKE_PASSWORD', '') SNOWFLAKE_WAREHOUSE = os.environ.get( 'SNOWFLAKE_WAREHOUSE', 'DEV_OWS_WAREHOUSE') SNOWFLAKE_DATABASE = os.environ.get('SNOWFLAKE_DATABASE', '') SNOWFLAKE_SCHEMA = os.environ.get('SNOWFLAKE_SCHEMA', '') SNOWFLAKE_KEY_PASSPHRASE = os.environ.get('SNOWFLAKE_KEY_PASSPHRASE', '') SNOWFLAKE_PRIVATE_KEY_PATH = os.environ.get('SNOWFLAKE_PRIVATE_KEY_PATH', '') # ----------- Snowflake PEM-key RSA auth ------------------------------------- if SNOWFLAKE_KEY_PASSPHRASE: try: HOME = os.environ['HOME'] except KeyError: HOME = str( os.environ['HOMEDRIVE'] + os.environ['HOMEPATH'] ).replace('\\', '/') # Snowflake key if not SNOWFLAKE_PRIVATE_KEY_PATH: SNOWFLAKE_PRIVATE_KEY_PATH = '{}/.ssh/snowflake/rsa_key.p8'.format( HOME) with open(SNOWFLAKE_PRIVATE_KEY_PATH, 'rb') as key: p_key = serialization.load_pem_private_key( key.read(), password=SNOWFLAKE_KEY_PASSPHRASE.encode(), backend=default_backend()) pkb = p_key.private_bytes( encoding=serialization.Encoding.DER, format=serialization.PrivateFormat.PKCS8, encryption_algorithm=serialization.NoEncryption()) SNOWFLAKE_CONNECT_ARGS = {'private_key': pkb} else: SNOWFLAKE_CONNECT_ARGS = {} # AWS ------------------------------------------------------------------------ AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID', None) AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY', None) AWS_REGION = os.environ.get('AWS_REGION', 'us-east-1') # S3 ------------------------------------------------------------------------- S3_BUCKET = os.environ.get('S3_BUCKET', 'dev-orchdbucket') # S3_FOLDER_TEMPLATE = os.environ.get('S3_FOLDER_TEMPLATE', '') # Sentry --------------------------------------------------------------------- SENTRY = os.environ.get('SENTRY_DSN', None) SENTRY_TRACE_LEVEL = os.environ.get('SENTRY_TRACE_LEVEL', 1.0) # Enable Debug Logging ------------------------------------------------------- DEBUG_LOG = strtobool(str(os.environ.get('DEBUG_LOG', 'False'))) # GraphQL -------------------------------------------------------------------- GRAPHQL_GATEWAY_URL = os.environ.get('GRAPHQL_GATEWAY_URL') OA_USER = os.environ.get('OA_USER') ORCH_HEADER_IDENTITY_ID = os.environ.get('ORCH_HEADER_IDENTITY_ID', None) ORCH_HEADER_PROFILE_ID = os.environ.get('ORCH_HEADER_PROFILE_ID', None) ORCH_HEADER_PROFILE_TYPE = os.environ.get('ORCH_HEADER_PROFILE_TYPE', None) ORCH_HEADER_ROLE = os.environ.get('ORCH_HEADER_ROLE', None) ORCH_HEADER_PROFILE_UUID = os.environ.get('ORCH_HEADER_PROFILE_UUID', None) # Application-Specific ------------------------------------------------------- VENDOR_ID = os.environ.get('VENDOR_ID', None) SUBACCOUNT_ID = os.environ.get('SUBACCOUNT_ID', 0) # SET_TRACK_PARTICIPANTS = \ # strtobool(os.environ.get('SET_TRACK_PARTICIPANTS', 'False')) # ROLE_MAPPING = os.environ.get('ROLE_MAPPING', 'wmg').lower() PRINT_STACK_TRACES = strtobool(os.environ.get('PRINT_STACK_TRACES', 'False')) ALLOW_NULL_ARTIST_IDS = \ strtobool(os.environ.get('ALLOW_NULL_ARTIST_IDS', 'False')) # Row limits NROWS = int(os.environ.get('NROWS', 0)) # Input File INPUT_FILE_NAME = os.environ.get('INPUT_FILE_NAME', None) PERFORMER_COUNT = int(os.environ.get('PERFORMER_COUNT', 5)) ABORT_ON_FAIL = strtobool(os.environ.get('ABORT_ON_FAIL', 'True')) WRITE_MASTER_OUTPUT = strtobool(os.environ.get('WRITE_MASTER_OUTPUT', 'True')) CACHED_DATA_PATH = os.environ.get('CACHED_DATA_PATH', None) ASYNC_POOL_SIZE = int(os.environ.get('ASYNC_POOL_SIZE', 1)) ASYNC_CHUNK_SIZE = int(os.environ.get('ASYNC_CHUNK_SIZE', 10)) REQUEST_DELAY = float(os.environ.get('REQUEST_DELAY', 1)) TASK_DELAY = float(os.environ.get('TASK_DELAY', 0.1)) OUTPUT_FILE_PATH = os.environ.get('OUTPUT_FILE_PATH', "data/output") INPUT_FILE_STARTING_ROW_OFFSET = int( os.environ.get('INPUT_FILE_STARTING_ROW_OFFSET', 0)) JITTER_FACTOR = float(os.environ.get('JITTER_FACTOR', 1.0)) CREATE_ZIP_FILES = strtobool( os.environ.get('CREATE_ZIP_FILES', 'False')) # If set, only process rows with the specified value for TUID CHOOSE_TUID = int(os.environ.get('CHOOSE_TUID', 0)) # If set, only process rows with the specified value for RELEASE_ID CHOOSE_RELEASE_ID = int(os.environ.get('CHOOSE_RELEASE_ID', 0)) # If set, only process rows with the specified value for UPC CHOOSE_UPC = str(os .environ.get('CHOOSE_UPC', None)) # If set, only process rows with the specified value for VENDOR_ID CHOOSE_VENDOR_ID = int(os.environ.get('CHOOSE_VENDOR_ID', 0)) # If set, only process rows with the specified value for ISRC CHOOSE_ISRC = str(os.environ.get('CHOOSE_ISRC', None)) # If set, only process rows with the specified value for RELEASE NAME CHOOSE_RELEASE_NAME = str(os.environ.get('CHOOSE_RELEASE_NAME', None)) # If set, only process rows with the specified value for TRACK NAME CHOOSE_TRACK_NAME = str(os.environ.get('CHOOSE_TRACK_NAME', None)) # If set, only process rows with the specified value for VOLUME CHOOSE_VOLUME = int(os.environ.get('CHOOSE_VOLUME', 0)) # If set, only process rows with the specified value for TRACK NUMBER CHOOSE_TRACK_NUMBER = int(os.environ.get('CHOOSE_TRACK_NUMBER', 0)) # If set, only process rows with the specified value for LOCAL_LANG CHOOSE_LOCAL_LANG = str(os.environ.get('CHOOSE_LOCAL_LANG', None)) # If set, only process rows with the specified value for TRACK LOCAL NAME CHOOSE_TRACK_LOCAL_NAME = str(os.environ.get('CHOOSE_TRACK_LOCAL_NAME', None)) # If set, only process rows with the specified value for RELEASE LOCAL NAME CHOOSE_RELEASE_LOCAL_NAME = \ str(os.environ.get('CHOOSE_RELEASE_LOCAL_NAME', None)) # If set, will remove any rows with bad data, as opposed to failing the entire # process OMIT_BAD_ROWS = strtobool(os.environ.get('OMIT_BAD_ROWS', 'False')) # If True, when an error occurs, the program will print the entire failed # payload to the log REPORT_FAILED_PAYLOADS = strtobool( os.environ.get('REPORT_FAILED_PAYLOADS', 'True')) # Define the task that the process runner will run MAP_TASK = str(os.environ.get('MAP_TASK', None))