#!/usr/bin/env python3 """Application configuration.""" from json.decoder import JSONDecodeError from random import randint import sys import time import backoff from owslogger import logger import requests from requests.adapters import HTTPAdapter from requests.exceptions import RequestException import sentry_sdk import config # Global logger log = logger.setup( config.ENVIRONMENT, config.LOGGER_NAME, config.LOGGER_LEVEL, config.SERVICE_NAME, config.SERVICE_VERSION, dsn=config.LOGGER_DSN, ) # Sentry handler sentry_sdk.init(config.SENTRY_DSN) def main(): """Execute main entrypoint.""" log.info('Starting status checks') # Run health checks for ( attempt, val, ) in enumerate(range(0, config.NEO4J_CHECK_MAX_RETRIES)): timeout = time.time() + config.NEO4J_HEALTH_CHECK_TIMEOUT_SECONDS while time.time() < timeout: if check_neo4j_cluster_health( database=config.NEO4J_RESTORE_DATABASE_NAME, user=config.NEO4J_CONNECTION_USER, password=config.NEO4J_CONNECTION_PASSWORD, port=config.NEO4J_HTTP_PORT, min_members=config.NEO4J_MINIMUM_CLUSTER_MEMBERS, max_retries=config.NEO4J_CHECK_MAX_RETRIES, ): log.info('Health checks passed.') return True else: # Add some randomness so that nodes do not restart at the exact # same cadence that caused the prior cluster start to fail time.sleep(randint(1, 10)) # If timeout is reached and function has not returned, fail. log.error('Max retry attempts reached and cluster is not healthy.') sys.exit(1) def check_neo4j_cluster_health( database, user, password, port, min_members, max_retries ): """ Check cluster health according to a series of metrics. Returns: bool: cluster availability """ status = get_neo4j_status( database=database, user=user, password=password, port=port, max_retries=max_retries, ) if status and 'healthy' in status: log.info(f'Node status: {status}') if 'healthy' in status and status['healthy'] is not True: log.info('Node is not healthy') return False log.info('Node is healthy') if ( 'votingMembers' in status and len(status['votingMembers']) < min_members ): log.info('Minimum voting nodes not reached') return False log.info('Minimum voting nodes reached') # Check to make sure node is in voting members if ( 'memberId' and 'votingMembers' in status and status['memberId'] not in status['votingMembers'] ): log.info('Node is not present in voting members of cluster') return False log.info('Node is present in voting members of cluster') # Check to make sure leader is present and part of voting members if ( 'leader' in status and status['leader'] not in status['votingMembers'] ): log.info('Leader is not present and in voting members of cluster') return False log.info('Leader is present and in voting members of cluster') # Check to make sure node is participating in raft group if ( 'participatingInRaftGroup' in status and status['participatingInRaftGroup'] is not True ): log.info('Node is not participating in raft group') return False log.info('Node is participating in raft group') return True else: return False @backoff.on_exception( backoff.expo, requests.exceptions.RequestException, max_time=config.NEO4J_HEALTH_CHECK_TIMEOUT_SECONDS, ) def get_neo4j_status(database, user, password, port, max_retries): """ Run various checks to check for service health. https://neo4j.com/docs/operations-manual/current/monitoring/causal-cluster/http-endpoints/ Returns: dict: dictionary of status fields """ status_endpoint_url = f'http://127.0.0.1:{port}' try: # Check node availability session = requests.Session() session.mount('http://', HTTPAdapter(max_retries=max_retries)) availability = session.get( f'{status_endpoint_url}/db/{database}/cluster/available', auth=(user, password), ) availability.status_code == 200 and log.info('Node is available') status_response = session.get( f'{status_endpoint_url}/db/{database}/cluster/status', auth=(user, password), ) return status_response.json() except (RequestException, JSONDecodeError) as error: """ If the service is running but Neo4j has not fully started, the HTTP interface will not be available. This is expected behavior. """ log.error( f'Exception making request to {status_endpoint_url}: {error}. ' 'Retrying...' ) return {} if __name__ == '__main__': main()