"""Application configuration.""" import logging import os import boto3 from dotenv import load_dotenv from secrets_manager.lambda_ext import LambdaSecretsManager # Load env file if it exists load_dotenv(verbose=True) # Service information SERVICE_NAME = 'neo4j-cluster-backup-fargate' SERVICE_VERSION = '4.0.0' # Logger Name LOGGER_NAME = 'neo4j-cluster-backup' LOGGER_LEVEL = logging.INFO LOGGER_DSN = os.environ.get('LOGGER_DSN') or None ENVIRONMENT = os.environ.get('Environment', 'prod') ENVIRONMENTS = ['dev', 'qa', 'prod'] if ENVIRONMENT not in ENVIRONMENTS: raise Exception('The environment used is not properly named.') REFRESH_ENVIRONMENT = os.environ.get('REFRESH_ENVIRONMENT', 'qa') REFRESH_ENVIRONMENTS = ['dev', 'qa'] if REFRESH_ENVIRONMENT not in REFRESH_ENVIRONMENTS: raise Exception('The refresh environment used is not properly named.') # This is for local development DRY_RUN = False # AWS settings AWS_REGION = os.environ.get('AWS_DEFAULT_REGION', 'us-east-1') secrets_manager_client = LambdaSecretsManager( environment=ENVIRONMENT, service_name=SERVICE_NAME, aws_region=AWS_REGION, force_remote=True, ) REFRESH_SERVICE_NAME = 'neo4j-cluster-restore' refresh_secrets_manager_client = LambdaSecretsManager( environment=REFRESH_ENVIRONMENT, service_name=REFRESH_SERVICE_NAME, aws_region=AWS_REGION, force_remote=True, ) # Neo4j connection settings NEO4J_HOST = os.environ.get('NEO4J_HOST', 'prod-neo4j-cluster.theorchard.io') NEO4J_REFRESH_HOST = os.environ.get('NEO4J_REFRESH_HOST', 'qa-neo4j-5-cluster.theorchard.io') # Neo4j backup settings NEO4J_BACKUP_UPLOAD_CONCURRENCY = os.environ.get('NEO4J_BACKUP_UPLOAD_CONCURRENCY', '10') # NEO4J_BACKUP_DATABASE = os.environ.get('NEO4J_BACKUP_DATABASE', 'graph.db') NEO4J_BACKUP_DIRECTORY = os.environ.get('NEO4J_BACKUP_DIRECTORY', '/var/lib/neo4j/') # defaults to false so backups will be incremental NEO4J_BACKUP_DIRECTORY_CLEAN = os.environ.get('NEO4J_BACKUP_DIRECTORY_CLEAN', 'true').lower() == 'true' NEO4J_BACKUP_FROM_FOLLOWER = os.environ.get('NEO4J_BACKUP_FROM_FOLLOWER', 'true').lower() == 'true' NEO4J_BACKUP_HEAP_SIZE = os.environ.get('NEO4J_BACKUP_HEAP_SIZE', '12G') NEO4J_BACKUP_COMPRESS_BACKUP = os.environ.get('NEO4J_BACKUP_COMPRESS_BACKUP', 'false').lower() == 'true' NEO4J_URL = f'neo4j+s://{NEO4J_HOST}' NEO4j_USERNAME = secrets_manager_client.get_cred('NEO4j_USERNAME') NEO4j_PASSWORD = secrets_manager_client.get_cred('NEO4j_PASSWORD') NEO4J_REFRESH_URL = f'neo4j+s://{NEO4J_REFRESH_HOST}' NEO4J_REFRESH_CONNECTION_USER = refresh_secrets_manager_client.get_cred('NEO4J_CONNECTION_USER') NEO4J_REFRESH_CONNECTION_PASSWORD = refresh_secrets_manager_client.get_cred('NEO4J_CONNECTION_PASSWORD') NEO4J_SERVER_MAJOR_VERSION = int(os.environ.get('NEO4J_SERVER_MAJOR_VERSION', '4')) if NEO4J_BACKUP_FROM_FOLLOWER and (not NEO4J_URL or not NEO4j_USERNAME or not NEO4j_PASSWORD): raise Exception('Need NEO4J_URL, NEO4j_USERNAME and NEO4j_PASSWORD to backup from follower') if NEO4J_BACKUP_DIRECTORY == '/' or NEO4J_BACKUP_DIRECTORY == '' or NEO4J_BACKUP_DIRECTORY[-1] != '/': error = f'NEO4J_BACKUP_DIRECTORY: ({NEO4J_BACKUP_DIRECTORY}) can\'t be empty, be "/" or not end with "/"' raise Exception(error) NEO4J_BACKUP_INCLUDE_METADATA = os.environ.get('NEO4J_BACKUP_INCLUDE_METADATA', 'all') NEO4J_BACKUP_KMS_KEY_ID = os.environ.get('NEO4J_BACKUP_KMS_KEY_ID', '839dc350-52d3-4bd0-b37f-da708ba77aca') NEO4J_BACKUP_PAGECACHE = os.environ.get('NEO4J_BACKUP_PAGECACHE', '12G') NEO4J_BACKUP_PORT = os.environ.get('NEO4J_BACKUP_PORT', 6362) NEO4J_BACKUP_S3_BUCKET = os.environ.get('NEO4J_BACKUP_S3_BUCKET', 'prod-orcdbucket') NEO4J_BACKUP_S3_PREFIX = os.environ.get('NEO4J_BACKUP_S3_PREFIX', f'backup/neo4j/{NEO4J_HOST}/prod-5') # Set default timeout to 4hrs NEO4J_BACKUP_TIMEOUT = int(os.environ.get('NEO4J_BACKUP_TIMEOUT', 14400)) # Timeout (seconds) for the `neo4j-admin backup inspect` metadata step NEO4J_BACKUP_INSPECT_TIMEOUT = int(os.environ.get('NEO4J_BACKUP_INSPECT_TIMEOUT', 600)) NEO4J_BACKUP_UPLOAD_TIMEOUT = int(os.environ.get('NEO4J_BACKUP_UPLOAD_TIMEOUT', 3600)) NEO4J_BACKUP_UPLOAD_TO_S3 = os.environ.get('NEO4J_BACKUP_UPLOAD_TO_S3', 'true').lower() == 'true' # Whether to refresh QA backup pointer after upload NEO4J_BACKUP_REFRESH_QA = os.environ.get('NEO4J_BACKUP_REFRESH_QA', 'false').lower() == 'true' # This client is used specifically for the Datadog API key datadog_sm_client = boto3.client('secretsmanager', region_name=AWS_REGION) datadog_api_key = f'{ENVIRONMENT}/datadog/DD_API_KEY' datadog_app_key = f'{ENVIRONMENT}/datadog/DD_APP_KEY' # Datadog settings DATADOG_API_KEY = datadog_sm_client.get_secret_value(SecretId=datadog_api_key)['SecretString'] DATADOG_APP_KEY = datadog_sm_client.get_secret_value(SecretId=datadog_app_key)['SecretString'] DATADOG_BACKUP_ATTEMPT_METRIC = 'backup_attempt' DATADOG_BACKUP_FAILURE_METRIC = 'backup_failure' DATADOG_BACKUP_SUCCESS_METRIC = 'backup_success' DATADOG_BACKUP_UPLOAD_FAILURE_METRIC = 'backup_upload_failure'