"""App configuration.""" import logging import os from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from secrets_manager.python_ext import PythonSecretsManager # Service information SERVICE_NAME = "ows-data-export" SERVICE_VERSION = "0.1.0" # Health endpoint HEALTH_CHECK = "/hello/" # Production environment PROD_ENVIRONMENT = "prod" QA_ENVIRONMENT = "qa" DEV_ENVIRONMENT = "dev" LOCAL_ENVIRONMENT = "local" TEST_ENVIRONMENT = "test" ENVIRONMENT = os.getenv("Environment") or LOCAL_ENVIRONMENT secrets_manager_client = PythonSecretsManager( environment=ENVIRONMENT, service_name=SERVICE_NAME, force_remote=True) LOGGER_DSN = os.getenv("LOGGER_DSN") or None LOGGER_LEVEL = logging.getLevelName(os.getenv("LOG_LEVEL") or "INFO") LOGGER_NAME = "ows-data-export" # SNOWFLAKE CONFIGURATION SNOWFLAKE_ACCOUNT = os.getenv("SNOWFLAKE_ACCOUNT") SNOWFLAKE_USER = os.getenv("SNOWFLAKE_USER") SNOWFLAKE_ROLE = os.getenv("SNOWFLAKE_ROLE") SNOWFLAKE_WAREHOUSE = os.getenv("SNOWFLAKE_WAREHOUSE") SNOWFLAKE_SCHEMA = os.getenv("SNOWFLAKE_SCHEMA", "DEV") SNOWFLAKE_DATABASE = os.getenv("SNOWFLAKE_DATABASE") SNOWFLAKE_POOL_SIZE = 15 # number of gunicorn threads SNOWFLAKE_POOL_RECYCLE = 4 * 55 * 60 # Snowflake timeout: ~4 hours SNOWFLAKE_POOL_MAX_OVERFLOW = 5 SNOWFLAKE_POOL_PRE_PING = False SNOWFLAKE_POOL_RESET_ON_RETURN = None # for LOCAL by default it takes snowflake pass or key from local environment if ENVIRONMENT in [TEST_ENVIRONMENT, LOCAL_ENVIRONMENT]: SENTRY = os.getenv("SENTRY_DSN") or None SNOWFLAKE_PASSWORD = os.getenv("SNOWFLAKE_PASSWORD") SNOWFLAKE_PRIVATE_KEY_PATH = str(os.getenv("SNOWFLAKE_PRIVATE_KEY_PATH")) SNOWFLAKE_KEY_PASSPHRASE = os.getenv("SNOWFLAKE_KEY_PASSPHRASE") SNOWFLAKE_ACCOUNT = os.getenv("SNOWFLAKE_ACCOUNT") SNOWFLAKE_USER = os.getenv("SNOWFLAKE_USER") SNOWFLAKE_ROLE = os.getenv("SNOWFLAKE_ROLE") SNOWFLAKE_WAREHOUSE = os.getenv("SNOWFLAKE_WAREHOUSE") SNOWFLAKE_SCHEMA = os.getenv("SNOWFLAKE_SCHEMA", "DEV") SNOWFLAKE_DATABASE = os.getenv("SNOWFLAKE_DATABASE") else: creds: dict = secrets_manager_client.get_cred("CREDENTIALS") SENTRY = creds.get("SENTRY_DSN") SNOWFLAKE_ACCOUNT = creds.get("SNOWFLAKE_ACCOUNT") SNOWFLAKE_USER = creds.get("SNOWFLAKE_USER") SNOWFLAKE_PASSWORD = creds.get("SNOWFLAKE_PASSWORD") SNOWFLAKE_PRIVATE_KEY = creds.get("SNOWFLAKE_PRIVATE_KEY") SNOWFLAKE_KEY_PASSPHRASE = creds.get("SNOWFLAKE_KEY_PASSPHRASE") SNOWFLAKE_ROLE = creds.get("SNOWFLAKE_ROLE") SNOWFLAKE_WAREHOUSE = creds.get("SNOWFLAKE_WAREHOUSE") SNOWFLAKE_SCHEMA = creds.get("SNOWFLAKE_SCHEMA") SNOWFLAKE_DATABASE = creds.get("SNOWFLAKE_DATABASE") if SNOWFLAKE_KEY_PASSPHRASE: if ENVIRONMENT not in [TEST_ENVIRONMENT, LOCAL_ENVIRONMENT]: SNOWFLAKE_PRIVATE_KEY = ("-----BEGIN ENCRYPTED PRIVATE KEY-----\n" f"{SNOWFLAKE_PRIVATE_KEY}\n" "-----END ENCRYPTED PRIVATE KEY-----") p_key = serialization.load_pem_private_key( SNOWFLAKE_PRIVATE_KEY.encode(), password=SNOWFLAKE_KEY_PASSPHRASE.encode(), backend=default_backend(), ) else: with open(SNOWFLAKE_PRIVATE_KEY_PATH, "rb") as key: p_key = serialization.load_pem_private_key( key.read(), password=SNOWFLAKE_KEY_PASSPHRASE.encode(), backend=default_backend(), ) pkb = p_key.private_bytes( encoding=serialization.Encoding.DER, format=serialization.PrivateFormat.PKCS8, encryption_algorithm=serialization.NoEncryption(), ) SNOWFLAKE_CONNECT_ARGS = {"private_key": pkb} else: # pragma: no cover SNOWFLAKE_CONNECT_ARGS = {} # AWS CONFIGURATION if ENVIRONMENT in [TEST_ENVIRONMENT, LOCAL_ENVIRONMENT]: AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID') AWS_SECRET = os.getenv('AWS_SECRET') AWS_TOKEN = os.getenv('AWS_TOKEN') # KAFKA CONSUMER CONFIGURATION TOPICS = os.getenv("KAFKA_TOPICS", "event.snowflake.query.status") AUTO_OFFSET_RESET = "earliest" ENABLE_AUTO_COMMIT = True GROUP_ID = "ows-data-export" SECURITY_PROTOCOL = "SSL" BOOTSTRAP_SERVERS = [ "b-1.dev-managed-kafka-cdc.rk4es0.c11.kafka.us-east-1.amazonaws.com:9094", "b-2.dev-managed-kafka-cdc.rk4es0.c11.kafka.us-east-1.amazonaws.com:9094", "b-2.dev-managed-kafka-cdc.rk4es0.c11.kafka.us-east-1.amazonaws.com:9094" ]