import logging import os import subprocess import sys import time # Global logger logging.basicConfig( level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", filename=os.environ.get("LOGGER_FILE_NAME", "/var/log/neo4j/watchdog.log"), ) def run_subprocess(command, timeout=60, check_return_code=False): """ Run subprocess call. Args: command (list): The formatted command to run. timeout (int): Subprocess timeout check_return_code (bool): Whether or not to throw exceptions on non-zero return code Returns: subprocess.CompletedProcess.stdout: resulting stdout """ try: subprocess_result = subprocess.run( command, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout, check=check_return_code, ) logging.info("Ran command: {}".format(" ".join(command))) subprocess_result.returncode != 0 and logging.error( "Non-zero exit code from command: {}: {}, {}".format(" ".join( command), subprocess_result.stderr, subprocess_result.stderr)) return subprocess_result.stdout except subprocess.SubprocessError as error: logging.error("Exception running command {}: {}".format( " ".join(command), error)) def check_neo4j_service(): """Check status of Neo4j service.""" check_neo4j_service_format = [ "systemctl", "is-active", "neo4j", ] result = run_subprocess(check_neo4j_service_format, 60, False) return result def check_java_process(): """Check for Java process.""" check_java_process_format = [ "pgrep", "java", ] result = run_subprocess(check_java_process_format, 60, False) return result def check_neo4j_listening_ports(): """Checks that Neo4j is listening on Bolt ports""" check_neo4j_listening_ports_format = [ "ss", "-Hl", "( sport = :7687 )", ] result = run_subprocess(check_neo4j_listening_ports_format, 60, True) return result def restart_neo4j_service(): """Start Neo4j service.""" restart_neo4j_service_format = [ "systemctl", "restart", "neo4j", ] run_subprocess(restart_neo4j_service_format, 300, True) def main(): """Do sundry Neo4j checks and restart service if needed""" systemd_result = check_neo4j_service() if systemd_result and systemd_result.startswith("active"): logging.info("Neo4j service is running") else: logging.info("Neo4j is not active. Systemd status: {}\n. Systemd " "service will be restarted".format(systemd_result)) restart_neo4j_service() time.sleep(5) running_java_process = check_java_process() if not running_java_process: logging.info("Neo4j java process is not running. {}".format( systemd_result)) restart_neo4j_service() time.sleep(5) else: logging.info("Neo4j java process is running") neo4j_polling_timeout = int(os.environ.get('NEO4J_POLLING_TIMEOUT', 3600)) timeout = time.time() + neo4j_polling_timeout logging.info('Checking for listening ports. Timeout: {} seconds'.format( neo4j_polling_timeout)) while time.time() < timeout: port_listening = check_neo4j_listening_ports() if port_listening: logging.info('Neo4j is running and listening on bolt port 7687') sys.exit(0) else: logging.info('Neo4j is not listening on bolt port 7687. Waiting ' 'up to {} seconds.'.format(neo4j_polling_timeout)) time.sleep(10) logging.error('Max timeout reached and node is not healthy.') if __name__ == "__main__": main()