"""Application configuration.""" import logging import os import boto3 from dotenv import load_dotenv from secrets_manager.lambda_ext import LambdaSecretsManager # Load env file if it exists load_dotenv(verbose=True) # Service information SERVICE_NAME = os.environ.get('SERVICE_NAME', 'neo4j-cluster-restore') SERVICE_VERSION = '4.0.0' # Logger Name LOGGER_NAME = os.environ.get('LOGGER_NAME', 'neo4j-cluster-restore') LOGGER_LEVEL = logging.INFO LOGGER_DSN = os.environ.get( 'LOGGER_DSN', 'https://qa-fluentd-applications.theorchard.io:8888/application' ) ENVIRONMENT = os.environ.get('Environment', 'dev') print(f'The environment is {ENVIRONMENT}') ENVIRONMENTS = ['dev', 'qa', 'prod'] if ENVIRONMENT not in ENVIRONMENTS: raise Exception('The environment used is not properly named.') YES_I_REALLY_MEAN_PROD = os.environ.get('YES_I_REALLY_MEAN_PROD', None) if ENVIRONMENT == 'prod' and not YES_I_REALLY_MEAN_PROD: raise Exception('Will not run on prod unless YES_I_REALLY_MEAN_PROD is defined') # AWS settings AWS_REGION = os.environ.get('AWS_DEFAULT_REGION', 'us-east-1') sm_client = LambdaSecretsManager( environment=ENVIRONMENT, service_name=SERVICE_NAME, aws_region=AWS_REGION, force_remote=True, ) # Neo4j node restore settings NEO4J_CONNECTION_USER = sm_client.get_cred('NEO4J_CONNECTION_USER') NEO4J_CONNECTION_PASSWORD = sm_client.get_cred('NEO4J_CONNECTION_PASSWORD') NEO4J_DATABASE_NAME = os.environ.get('NEO4J_DATABASE_NAME', 'graph.db') NEO4J_DROP_DATABASE_TIMEOUT = int(os.environ.get('NEO4J_DROP_DATABASE_TIMEOUT', 900)) NEO4J_HEALTH_CHECK_TIMEOUT_SECONDS = int( os.environ.get('NEO4J_HEALTH_CHECK_TIMEOUT_SECONDS', 300) ) NEO4J_HTTP_PORT = os.environ.get('NEO4J_HTTP_PORT', '7474') NEO4J_CHECK_MAX_RETRIES = int(os.environ.get('NEO4J_CHECK_MAX_RETRIES', 5)) NEO4J_MINIMUM_CLUSTER_MEMBERS = int(os.environ.get('NEO4J_MINIMUM_CLUSTER_MEMBERS', 3)) NEO4J_RESTORE_BACKUP_DATE = os.environ.get('NEO4J_RESTORE_BACKUP_DATE', None) NEO4J_RESTORE_DATABASE_NAME = os.environ.get('NEO4J_RESTORE_DATABASE_NAME', 'graph.db') NEO4J_RESTORE_DIRECTORY = os.environ.get( 'NEO4J_RESTORE_DIRECTORY', '/var/lib/neo4j/data/databases' ) NEO4J_RESTORE_KMS_KEY_ID = sm_client.get_cred('NEO4J_RESTORE_KMS_KEY_ID') NEO4J_RESTORE_STAGING_DIRECTORY = os.environ.get( 'NEO4J_RESTORE_STAGING_DIRECTORY', '/var/lib/neo4j/backup' ) NEO4J_RESTORE_S3_CROSS_ACCOUNT_ROLE_ARN = os.environ.get( 'NEO4J_RESTORE_S3_CROSS_ACCOUNT_ROLE_ARN', '' ) NEO4J_RESTORE_S3_CROSS_ACCOUNT_ROLE_EXTERNAL_ID = os.environ.get( 'NEO4J_RESTORE_S3_CROSS_ACCOUNT_ROLE_EXTERNAL_ID', 'neo4j-cluster-restore' ) NEO4J_RESTORE_S3_BUCKET = sm_client.get_cred('NEO4J_RESTORE_S3_BUCKET') NEO4J_RESTORE_S3_PREFIX = sm_client.get_cred('NEO4J_RESTORE_S3_PREFIX') NEO4J_SYSTEM_USER = os.environ.get('NEO4J_USER', 'neo4j') # Manager settings NEO4J_CLUSTER_DNS_NAME = os.environ.get( 'NEO4J_CLUSTER_DNS_NAME', f'{ENVIRONMENT}-neo4j-cluster.theorchard.io' ) NEO4J_RESTORE_SSH_PRIVATE_KEY = sm_client.get_cred('NEO4J_RESTORE_SSH_PRIVATE_KEY') NEO4J_RESTORE_SSH_USER = sm_client.get_cred('NEO4J_RESTORE_SSH_USER') NEO4J_RESTORE_REPO_LOCATION = os.environ.get( 'NEO4J_RESTORE_REPO_LOCATION', '/var/lib/neo4j/python-neo4j-utils' ) NEO4J_RESTORE_PYTHON_LOCATION = os.environ.get( 'NEO4J_RESTORE_PYTHON_LOCATION', f'{NEO4J_RESTORE_REPO_LOCATION}/restore/venv/bin/python', ) NEO4J_RESTORE_DOWNLOAD_TIMEOUT = int( os.environ.get('NEO4J_RESTORE_DOWNLOAD_TIMEOUT', 7200) ) NEO4J_RESTORE_SCRIPT_TIMEOUT = int(os.environ.get('NEO4J_RESTORE_SCRIPT_TIMEOUT', 7200)) # Sentry project SENTRY_DSN = sm_client.get_cred('SENTRY_DSN') # This client is used specifically for the Datadog API key datadog_sm_client = boto3.client('secretsmanager', region_name=AWS_REGION) datadog_api_key = f'{ENVIRONMENT}/datadog/DD_API_KEY' datadog_app_key = f'{ENVIRONMENT}/datadog/DD_APP_KEY' # Datadog settings DATADOG_API_KEY = datadog_sm_client.get_secret_value(SecretId=datadog_api_key)[ 'SecretString' ] DATADOG_APP_KEY = datadog_sm_client.get_secret_value(SecretId=datadog_app_key)[ 'SecretString' ] DATADOG_RESTORE_ATTEMPT_METRIC = 'restore_attempt' DATADOG_RESTORE_FAILURE_METRIC = 'restore_failure' DATADOG_RESTORE_SUCCESS_METRIC = 'restore_success' # How long to wait for a database to become online when selected (seconds) NEO4J_DATABASE_ONLINE_TIMEOUT_SECONDS = int( os.environ.get('NEO4J_DATABASE_ONLINE_TIMEOUT_SECONDS', 8 * 60 * 60) ) # Dry run mode for manager actions (only log intended actions, do not execute) NEO4J_DRY_RUN = os.environ.get('NEO4J_DRY_RUN', 'false').lower() == 'true' # Creates a dict of config that can be passed to subprocess via env vars/ # Filters out None values, so they would not be interpreted as 'None' strings. NEO4J_CONFIG = { key: str(value) for key, value in vars().items() if (key.startswith(('NEO4J_', 'LOGGER_', 'ENVIRONMENT')) and value) } # Kafka Connect Neo4j Source connector (CDC) KC_NEO4J_SRC_CONNECTOR_URL = os.environ.get( 'KC_NEO4J_SRC_CONNECTOR_URL', 'https://qa-kc-neo-v5-src.theorchard.io/connectors/neo4j_cdc_source', )