"""Application configuration.""" import dataclasses import logging import os from dbdeploy.dtos import EnvType from dbdeploy.dtos import KafkaCluster from dbdeploy.util.aws.helpers import get_msk_bootstrap_brokers from dbdeploy.util.aws.helpers import get_secret from dbdeploy.util.common import get_project_root from dbdeploy.util.logger import get_logger # Service information SERVICE_NAME = 'kafka-db-deploy' SERVICE_VERSION = '1.0.0' PROJECT_ROOT = get_project_root() LOG_LEVEL = logging.getLevelName(os.getenv('LOG_LEVEL', 'INFO')) AWS_REGION = os.getenv('AWS_REGION', 'us-east-1') # Environment settings ENVIRONMENT = os.getenv('Environment', 'dev') LOGGER = get_logger(service_name=SERVICE_NAME, log_level=LOG_LEVEL) # Snowflake client credentials and configuration if EnvType(ENVIRONMENT) is EnvType.TEST: ORCHARD_SNOWFLAKE_CREDENTIALS = {} DELPHI_SNOWFLAKE_CREDENTIALS = {} DELPHI_DEV_SNOWFLAKE_CREDENTIALS = {} MUSIC_GRAPH_CREDENTIALS = {} NR_GRAPH_CREDENTIALS = {} else: ORCHARD_SNOWFLAKE_CREDENTIALS = get_secret( aws_region=AWS_REGION, environment=ENVIRONMENT, service_name=SERVICE_NAME, secret_key='SNOWFLAKE_CREDENTIALS') DELPHI_SNOWFLAKE_CREDENTIALS = get_secret( aws_region=AWS_REGION, environment=ENVIRONMENT, service_name=SERVICE_NAME, secret_key='DELPHI_CREDENTIALS') DELPHI_DEV_SNOWFLAKE_CREDENTIALS = get_secret( aws_region=AWS_REGION, environment=ENVIRONMENT, service_name=SERVICE_NAME, secret_key='DELPHI_DEV_CREDENTIALS') MUSIC_GRAPH_CREDENTIALS = get_secret( aws_region=AWS_REGION, environment=ENVIRONMENT, service_name=SERVICE_NAME, secret_key='MUSIC_GRAPH_CREDENTIALS') NR_GRAPH_CREDENTIALS = get_secret( aws_region=AWS_REGION, environment=ENVIRONMENT, service_name=SERVICE_NAME, secret_key='NR_GRAPH_CREDENTIALS') class SnowflakeAccount: ORCHARD = 'orchard' DELPHI = 'delphi' DELPHI_DEV = 'delphi-dev' @dataclasses.dataclass class SnowflakeAccountConfig: SNOWFLAKE_USER: str | None SNOWFLAKE_ACCOUNT: str | None SNOWFLAKE_PRIVATE_KEY_STRING: str | None SNOWFLAKE_KEY_PASSPHRASE: str | None SNOWFLAKE_ACCOUNTS = { SnowflakeAccount.ORCHARD: SnowflakeAccountConfig( SNOWFLAKE_USER=ORCHARD_SNOWFLAKE_CREDENTIALS.get('SNOWFLAKE_USER'), SNOWFLAKE_ACCOUNT=ORCHARD_SNOWFLAKE_CREDENTIALS.get('SNOWFLAKE_ACCOUNT'), SNOWFLAKE_PRIVATE_KEY_STRING=ORCHARD_SNOWFLAKE_CREDENTIALS.get('SNOWFLAKE_PRIVATE_KEY'), SNOWFLAKE_KEY_PASSPHRASE=ORCHARD_SNOWFLAKE_CREDENTIALS.get('SNOWFLAKE_KEY_PASSPHRASE')), SnowflakeAccount.DELPHI: SnowflakeAccountConfig( SNOWFLAKE_USER=DELPHI_SNOWFLAKE_CREDENTIALS.get('SNOWFLAKE_USER'), SNOWFLAKE_ACCOUNT=DELPHI_SNOWFLAKE_CREDENTIALS.get('SNOWFLAKE_ACCOUNT'), SNOWFLAKE_PRIVATE_KEY_STRING=DELPHI_SNOWFLAKE_CREDENTIALS.get('SNOWFLAKE_PRIVATE_KEY'), SNOWFLAKE_KEY_PASSPHRASE=DELPHI_SNOWFLAKE_CREDENTIALS.get('SNOWFLAKE_KEY_PASSPHRASE')), SnowflakeAccount.DELPHI_DEV: SnowflakeAccountConfig( SNOWFLAKE_USER=DELPHI_DEV_SNOWFLAKE_CREDENTIALS.get('SNOWFLAKE_USER'), SNOWFLAKE_ACCOUNT=DELPHI_DEV_SNOWFLAKE_CREDENTIALS.get('SNOWFLAKE_ACCOUNT'), SNOWFLAKE_PRIVATE_KEY_STRING=DELPHI_DEV_SNOWFLAKE_CREDENTIALS.get('SNOWFLAKE_PRIVATE_KEY'), SNOWFLAKE_KEY_PASSPHRASE=DELPHI_DEV_SNOWFLAKE_CREDENTIALS.get('SNOWFLAKE_KEY_PASSPHRASE')) } SNOWFLAKE_ROLE = os.getenv('SNOWFLAKE_ROLE') SNOWFLAKE_WAREHOUSE = os.getenv('SNOWFLAKE_WAREHOUSE') SNOWFLAKE_DATABASE = os.getenv('SNOWFLAKE_DATABASE') SNOWFLAKE_SCHEMA = os.getenv('SNOWFLAKE_SCHEMA') SNOWFLAKE_BATCH_SIZE = int(os.getenv('SNOWFLAKE_BATCH_SIZE', 5000)) # Kafka and Kafka-Connect configuration CONNECT_CLUSTER_NAME = f'{ENVIRONMENT}-kafka-connect-db-deploy' if EnvType(ENVIRONMENT) in [EnvType.PROD, EnvType.QA]: CONNECT_CLUSTER_URL = f'https://{CONNECT_CLUSTER_NAME}.theorchard.io' # noqa: E501 NEO4J_MUSIC_GRAPH_URI = f'neo4j+s://{ENVIRONMENT}-neo4j-cluster.theorchard.io:7687' KAFKA_CLUSTER_NAME = f'{ENVIRONMENT}-managed-kafka-db-deploy' else: CONNECT_CLUSTER_URL = 'https://dev-kafka-connect-db-deploy.dev.theorchard.io' # noqa: E501 NEO4J_MUSIC_GRAPH_URI = 'neo4j+s://dev-neo4j-cluster.dev.theorchard.io:7687' KAFKA_CLUSTER_NAME = 'dev-managed-kafka-cdc-destination' KAFKA_TOPIC_PREFIX = os.getenv('KAFKA_TOPIC_PREFIX', 'db.deploy.') KAFKA_DLQ_TOPIC_PREFIX = os.getenv('KAFKA_DLQ_TOPIC_PREFIX', 'dlq.db.deploy.') KAFKA_CONNECTION_RETRIES = 5 TOPIC_DELETE_RETRIES = 5 RETRY_BACKOFF = 60 # seconds OTHER_ROLES_FOR_CLUSTER_ACCESS = os.getenv( 'OTHER_ROLES_FOR_CLUSTER_ACCESS', '').split(',') if EnvType(ENVIRONMENT) is not EnvType.TEST: KAFKA_BOOTSTRAP_BROKERS = get_msk_bootstrap_brokers(KAFKA_CLUSTER_NAME) else: KAFKA_BOOTSTRAP_BROKERS = '' KAFKA_CLUSTER = KafkaCluster( name=KAFKA_CLUSTER_NAME, bootstrap_brokers=KAFKA_BOOTSTRAP_BROKERS, security_protocol=os.getenv('KAFKA_SECURITY_PROTOCOL', 'SSL')) CONNECTOR_BATCH_SIZE = int(os.getenv('CONNECTOR_BATCH_SIZE', 5)) CONNECTOR_TASKS_MAX = 1 CONNECTOR_MAX_POLL_INTERVAL = 3 * 24 * 60 * 60 * 1000 # 3 days in milliseconds MYSQL_DB_NAME = os.getenv('MYSQL_DB_NAME', None) MYSQL_SECRET_KEY = f'MYSQL_CREDENTIALS_{MYSQL_DB_NAME}'.upper() # Fargate task configuration (deprecated) FARGATE_TASK_HEALTH_CHECK_BACKOFF_TIMEOUT = 60 # seconds FARGATE_TASK_HEALTH_CHECK_GRACE_PERIOD = 60 * 4 # seconds # Deploy job execution lock configuration DYNAMODB_TABLE = f'{ENVIRONMENT}_kafka_db_deploy_status' LOCK_ACQUISITION_TIMEOUT = 60 * 15 # secods LOCK_ACQUISITION_BACKOFF = 60 # secods FORCE_LOCK_RELEASE = str(os.getenv('FORCE_NEO4J_KAFKA_LOCK_RELEASE')).lower() == "true" # noqa: E501 CONTINUE_ON_DLQ = str(os.getenv('CONTINUE_ON_DLQ', 'false')).lower() == "true" # XML Schemas configuration FLOW_NAME = os.getenv('FLOW_NAME', '') XML_SCHEMA_PATH = os.path.join(PROJECT_ROOT, 'schemas') XML_SCHEMA_FILE = { 'neo4j': os.path.join(XML_SCHEMA_PATH, 'Neo4jSchema.xsd'), 'kafka': os.path.join(XML_SCHEMA_PATH, 'KafkaSchema.xsd'), 'snowflake-neo4j': os.path.join(XML_SCHEMA_PATH, 'SnowflakeNeo4jSchema.xsd'), # noqa: E501 'snowflake-kafka': os.path.join(XML_SCHEMA_PATH, 'SnowflakeKafkaSchema.xsd'), # noqa: E501 'snowflake-mysql': os.path.join(XML_SCHEMA_PATH, 'SnowflakeMysqlSchema.xsd'), # noqa: E501 } class Neo4jServerNames: MUSIC_GRAPH = 'music-graph' NR_GRAPH = 'nr-graph' @dataclasses.dataclass class Neo4jServerConfig: uri: str secret_name: str NEO4J_SERVERS = { Neo4jServerNames.MUSIC_GRAPH: Neo4jServerConfig( uri=os.getenv('NEO4J_MUSIC_GRAPH_URI', '') or MUSIC_GRAPH_CREDENTIALS.get('neo4j_uri', ''), secret_name=os.getenv('NEO4J_MUSIC_GRAPH_SECRET_NAME') or 'MUSIC_GRAPH_CREDENTIALS'), Neo4jServerNames.NR_GRAPH: Neo4jServerConfig( uri=os.getenv('NEO4J_NR_GRAPH_URI', '') or NR_GRAPH_CREDENTIALS.get('neo4j_uri', ''), secret_name=os.getenv('NEO4J_NR_GRAPH_SECRET_NAME') or 'NR_GRAPH_CREDENTIALS'), }