"""Application configuration.""" import requests import time from lambdacommon import util from neo4j import GraphDatabase from owslogger import logger import sentry_sdk import config # Global logger log = logger.setup( config.ENVIRONMENT, config.LOGGER_NAME, config.LOGGER_LEVEL, config.SERVICE_NAME, config.SERVICE_VERSION, dsn=config.LOGGER_DSN, ) # Sentry handler sentry_sdk.init(config.SENTRY_DSN) def main(): """Execute main entrypoint.""" call_datadog_with_metric(config.DATADOG_RESTORE_ATTEMPT_METRIC) log.info(f'Running restore manager script for {config.NEO4J_CLUSTER_DNS_NAME}') uri = f'neo4j+ssc://{config.NEO4J_CLUSTER_DNS_NAME}' driver = GraphDatabase.driver( uri, auth=(config.NEO4J_CONNECTION_USER, config.NEO4J_CONNECTION_PASSWORD), ) with driver.session(database='system') as session: # 1) Get graph.* databases not targeted by any alias alias_targets = _get_alias_targets(session) graph_records = session.run('SHOW DATABASES').data() candidate_names = [ r.get('name') for r in graph_records if isinstance(r.get('name'), str) and r.get('name').startswith('graph') and r.get('name') not in alias_targets ] distinct_names = sorted(set(candidate_names)) log.info(f'Graph database candidates (no alias targets): {distinct_names}') if not distinct_names: # TODO: kick off a create database job here log.info('No eligible graph.* databases found to promote. Exiting.') raise RuntimeError('No eligible graph.* databases found to promote.') # 2) Pick latest lexicographically (names share fixed timestamp format) selected = distinct_names[-1] log.info(f'Selected database candidate: {selected}') # 3) Wait until selected database is online _wait_until_online( session, selected, timeout_seconds=config.NEO4J_DATABASE_ONLINE_TIMEOUT_SECONDS, ) # 4) Point alias graph.db to selected database _replace_graph_db_alias(session, selected) # 5) Ensure alias target is also the default database ensure_alias_target_is_also_default_database(session) # 6) Restart Neo4j Kafka Connector (CDC) _restart_cdc_source_connector() # TODO: kick off a create database job here log.info(f'Completed restore manager script for {config.NEO4J_CLUSTER_DNS_NAME}') call_datadog_with_metric(config.DATADOG_RESTORE_SUCCESS_METRIC) def call_datadog_with_metric(metric): """ Wrap the DataDog metric call. Args: metric (str): custom metric name to send to DataDog. """ with util.datadog_connection( api_key=config.DATADOG_API_KEY, app_key=config.DATADOG_APP_KEY ) as datadog: now = time.time() datadog.Metric.send( [ { 'metric': f'{config.SERVICE_NAME}.{metric}', 'type': 'count', 'interval': 60, 'points': (now, 1), 'tags': [ f'environment:{config.ENVIRONMENT}', f'service_name:{config.SERVICE_NAME}', ], } ] ) def _get_alias_targets(session) -> set[str]: """Return a set of databases that are targets of any alias.""" targets = set() records = session.run('SHOW ALIASES FOR DATABASE').data() for record in records: # Neo4j 5 may use keys 'database' or 'target' target = record.get('database') if isinstance(target, str) and target: targets.add(target) return targets def _wait_until_online(session, database_name: str, timeout_seconds: int): """Poll currentStatus until 'online' or timeout, checking once per minute.""" deadline = time.time() + int(timeout_seconds) while time.time() < deadline: data = session.run( 'SHOW DATABASES YIELD name, currentStatus WHERE name = $n RETURN currentStatus', n=database_name, ).data() status = data[0]['currentStatus'] if data else None log.info(f"Database '{database_name}' status: {status}") if status == 'online': return time.sleep(60) raise TimeoutError( f"Timed out after {timeout_seconds}s waiting for database '{database_name}' to become online" ) def _replace_graph_db_alias(session, database_name: str): """Point alias 'graph.db' to the given database (create or replace).""" escaped = database_name.replace('`', '``') cypher = f'CREATE OR REPLACE ALIAS `graph.db` FOR DATABASE `{escaped}`' if config.NEO4J_DRY_RUN: log.info( f"DRY RUN: Would update alias 'graph.db' to target '{database_name}' with query: {cypher}" ) else: log.info(f"Updating alias 'graph.db' to target '{database_name}'") session.run(cypher) _configure_graph_db_after_alias(session) def _configure_graph_db_after_alias(session): """Run required configuration and grants after alias creation.""" commands = [ 'ALTER DATABASE `graph.db` SET OPTION txLogEnrichment "FULL"', 'GRANT ACCESS ON DATABASE `graph.db` TO architect', 'GRANT EXECUTE PROCEDURE db.cdc.query ON DBMS TO architect', 'GRANT EXECUTE BOOSTED PROCEDURE db.cdc.query ON DBMS TO architect', ] for command in commands: if config.NEO4J_DRY_RUN: log.info(f'DRY RUN: Would run post-alias command: {command}') else: log.info(f'Running post-alias command: {command}') session.run(command) def _restart_cdc_source_connector(): """Restart the Kafka Connect Neo4j Source connector via its REST API.""" url = config.KC_NEO4J_SRC_CONNECTOR_URL if config.NEO4J_DRY_RUN: log.info( f'DRY RUN: Would restart Kafka Connect Neo4j Source connector at {url}' ) return log.info(f'Restarting Kafka Connect Neo4j Source connector at {url}') response = requests.post(f'{url}/tasks/0/restart') if not response.ok: raise RuntimeError( f'Failed to restart Kafka Connect Neo4j Source connector at {url}: {response.status_code} {response.text}' ) log.info('Successfully restarted Kafka Connect Neo4j Source connector') def ensure_alias_target_is_also_default_database(session): alias_target = _get_alias_targets(session).pop() log.info(f'alias_target: {alias_target}') current_databases = session.run('SHOW DATABASES').data() default_databases = [r.get('name') for r in current_databases if r.get('default')] if not default_databases: log.info('No default database found') set_database_as_default(session) return default_database = default_databases.pop() if default_database == alias_target: log.info('default_database is already the alias target') return elif ( default_database < alias_target ): # only stops the default database if it is older than the alias target stop_database(session, default_database) set_database_as_default(session, alias_target) def set_database_as_default(session, database_name: str = 'graph.db'): if config.NEO4J_DRY_RUN: log.info(f'DRY RUN: Would set default database to {database_name}') else: log.info(f'Setting default database to {database_name}') session.run(f'CALL dbms.setDefaultDatabase("{database_name}")') def stop_database(session, database_name: str): if config.NEO4J_DRY_RUN: log.info(f'DRY RUN: Would stop database {database_name}') else: log.info(f'Stopping database {database_name}') session.run(f'STOP DATABASE `{database_name}`') if __name__ == '__main__': main()