# Monitor for Neo4j service
resource "datadog_monitor" "neo4j_status_monitor" {
  count = var.neo4j_monitors_enabled ? 1 : 0
  name               = "${var.environment}-${var.service_name}-neo4j-service-monitor"
  type               = "service check"
  message            = "Neo4j is not running on {{host.name}} in the ${var.environment}-${var.service_name} cluster. Notify: ${var.notification_endpoints}"
  escalation_message = "Escalation to ${var.escalation_notification_endpoints}"

  query = "\"${local.neo4j_dd_service_check_name}\".over(\"neo4j_cluster:${var.environment}-${var.service_name}\").by(\"host\").last(4).count_by_status()"

  monitor_thresholds {
    ok       = 1
    warning  = 2
    critical = 3
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = true

  tags = local.tags
}

# Monitor for Neo4j cluster memory
resource "datadog_monitor" "neo4j_memory_monitor" {
  count = var.neo4j_monitors_enabled ? 1 : 0
  name               = "${var.environment}-${var.service_name}-memory-monitor"
  type               = "query alert"
  message            = "Memory utililization is high on {{host.name}} in the ${var.environment}-${var.service_name} cluster. Notify: ${var.notification_endpoints}"
  escalation_message = "Escalation to ${var.escalation_notification_endpoints}"

  query = "avg(last_5m):100 * ( 1 - sum:system.mem.pct_usable{neo4j_cluster:${var.environment}-${var.service_name}} by {host} ) > ${var.memory_critical_number}"

  monitor_thresholds {
    warning           = var.memory_warning_number
    warning_recovery  = var.memory_warning_recovery_number
    critical          = var.memory_critical_number
    critical_recovery = var.memory_critical_recovery_number
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = true

  tags = local.tags
}

# Monitor for Neo4j cluster low memory utilization, also a bad thing
resource "datadog_monitor" "neo4j_low_memory_monitor" {
  count = var.neo4j_monitors_enabled ? 1 : 0
  name               = "${var.environment}-${var.service_name}-low-memory-monitor"
  type               = "query alert"
  message            = "Memory utililization is too low on {{host.name}} in the ${var.environment}-${var.service_name} cluster. Notify: ${var.notification_endpoints}"
  escalation_message = "Escalation to ${var.escalation_notification_endpoints}"

  query = "avg(last_5m):100 * ( 1 - sum:system.mem.pct_usable{neo4j_cluster:${var.environment}-${var.service_name}} by {host} ) < ${var.low_memory_critical_number}"

  monitor_thresholds {
    warning           = var.low_memory_warning_number
    warning_recovery  = var.low_memory_warning_recovery_number
    critical          = var.low_memory_critical_number
    critical_recovery = var.low_memory_critical_recovery_number
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = true

  tags = local.tags
}

# Monitor for Neo4j cluster CPU
resource "datadog_monitor" "neo4j_cpu_monitor" {
  count = var.neo4j_monitors_enabled ? 1 : 0
  name               = "${var.environment}-${var.service_name}-cpu-monitor"
  type               = "query alert"
  message            = "CPU utililization is high on {{host.name}} in the ${var.environment}-${var.service_name} cluster. Notify: ${var.notification_endpoints}"
  escalation_message = "Escalation to ${var.escalation_notification_endpoints}"

  query = "avg(last_5m):sum:system.cpu.system{neo4j_cluster:${var.environment}-${var.service_name}} by {host} + sum:system.cpu.user{neo4j_cluster:${var.environment}-${var.service_name}} by {host} > ${var.cpu_critical_number}"

  monitor_thresholds {
    warning           = var.cpu_warning_number
    warning_recovery  = var.cpu_warning_recovery_number
    critical          = var.cpu_critical_number
    critical_recovery = var.cpu_critical_recovery_number
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = true

  tags = local.tags
}

# Monitor for Neo4j cluster disk space
resource "datadog_monitor" "neo4j_disk_space_monitor" {
  count = var.neo4j_monitors_enabled ? 1 : 0
  name               = "${var.environment}-${var.service_name}-disk-space-monitor"
  type               = "query alert"
  message            = "Disk space is low on {{device.name}} on {{host.name}} in the ${var.environment}-${var.service_name} cluster. Notify: ${var.notification_endpoints}"
  escalation_message = "Escalation to ${var.escalation_notification_endpoints}"

  query = "avg(${var.disk_space_evaluation_window}):max:system.disk.in_use{neo4j_cluster:${var.environment}-${var.service_name},!device:devtmpfs,!device:tmpfs, !device:rootfs} by {device,host} * 100 > ${var.disk_space_critical_number}"

  monitor_thresholds {
    warning           = var.disk_space_warning_number
    warning_recovery  = var.disk_space_warning_recovery_number
    critical          = var.disk_space_critical_number
    critical_recovery = var.disk_space_critical_recovery_number
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = true

  tags = local.tags
}

# Monitor for Neo4j Java OOM log events
resource "datadog_monitor" "neo4j_java_oom_monitor" {
  count = var.neo4j_monitors_enabled ? 1 : 0
  name               = "${var.environment}-${var.service_name}-java-oom-monitor"
  type               = "log alert"
  message            = "Java OOM event on {{host.name}} in the ${var.environment}-${var.service_name} cluster. Notify: ${var.notification_endpoints}"
  escalation_message = "Escalation to ${var.escalation_notification_endpoints}"

  query = "logs(\"service:neo4j neo4j_cluster:${var.environment}-${var.service_name} java.lang.OutOfMemoryError\").index(\"main\").rollup(\"count\").by(\"host\").last(\"5m\") > 0"
  monitor_thresholds {
    critical = 0
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = true

  tags = local.tags
}

# Monitor queries failed with memory allocation threshold reached
resource "datadog_monitor" "neo4j_cypher_transaction_max_size_exceeded" {
  count = var.neo4j_monitors_enabled ? 1 : 0
  name               = "${var.environment}-${var.service_name} cyphers exceeded dbms.memory.transaction.max_size"
  type               = "log alert"
  message            = join(" ", [
    "{{#is_alert}}\n{{value}} queries on ${var.environment}-${var.service_name} failed because they exceeded dbms.memory.transaction.max_size threshold. More info:\n- executingUser: {{log.attributes.executingUser}}\n{{/is_alert}}\n",
    "{{#is_warning}}\n{{value}} queries on ${var.environment}-${var.service_name} failed because they exceeded dbms.memory.transaction.max_size threshold. More info:\n- executingUser: {{log.attributes.executingUser}}\n{{/is_warning}}\n",
    "{{#is_recovery}}\nNo more queries on ${var.environment}-${var.service_name} that exceeded dbms.memory.transaction.max_size threshold.\n{{/is_recovery}}\n\n",
    "Notify: ${var.notification_endpoints}"
  ])
  escalation_message = "Escalation to ${var.escalation_notification_endpoints}"

  query = "logs(\"environment:${var.environment} sourcecategory:query service:neo4j @event:fail @failureReason:*dbms.memory.transaction.max_size?threshold?reached*\").index(\"*\").rollup(\"count\").last(\"5m\") > ${var.number_of_failed_queries_critical}"
  monitor_thresholds {
    critical = var.number_of_failed_queries_critical
    warning = var.number_of_failed_queries_warning
  }

  include_tags        = true
  enable_logs_sample  = false
  on_missing_data     = "default"
  notify_audit        = false
  renotify_interval   = 60
  require_full_window = true
  groupby_simple_monitor = false

  tags = local.tags
}