data "datadog_role" "datadog_admin_role" {
  filter = "Datadog Admin Role"
}

# Datadog monitors and dashboards
module "datadog_neo4j_dashboards_and_monitors" {
  source                            = "git@github.com:theorchard/terraform-datadog.git//modules/neo4j?ref=6.16.1"
  environment                       = var.environment
  service_name                      = var.service_name_v5
  neo4j_major_version_number        = 5
  cpu_warning_recovery_number       = 55
  cpu_warning_number                = 60
  cpu_critical_recovery_number      = 65
  cpu_critical_number               = 70
  memory_critical_number            = 97
  memory_warning_number             = 95
  memory_warning_recovery_number    = 94
  notification_endpoints            = "@slack-neo4j-db"
  escalation_notification_endpoints = "@slack-neo4j-db"
  dependent_applications            = var.dependent_applications
  neo4j_monitors_enabled            = true
}

resource "datadog_monitor" "session_error_monitor" {
  name               = "${var.environment}-${var.service_name_v5}-session-error-monitor"
  type               = "query alert"
  message            = "Neo4j sessions are experiencing a high error rate. Notify: ${var.neo4j_escalation_endpoints}"
  escalation_message = "Escalation to ${var.neo4j_escalation_endpoints}"
  restricted_roles   = [data.datadog_role.datadog_admin_role.id]
  query              = "sum(last_15m):sum:trace.neo4j.session.run.errors{env:${var.environment},service:neo4j}.as_count() > ${var.neo4j_session_errors_critical_number}"

  monitor_thresholds {
    critical = var.neo4j_session_errors_critical_number
  }

  include_tags      = true
  notify_no_data    = false
  notify_audit      = true
  renotify_interval = 30
  priority          = 1

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name_v5}",
  ]
}

resource "datadog_monitor" "bookmark_lag_errors_monitor" {
  name               = "[${upper(var.environment)} neo4j] Increased transaction replication bookmark lag errors"
  type               = "log alert"
  message            = "[${upper(var.environment)} neo4j] High number of `TransientError: Database not up to the requested version.` errors. Cluster load may be too high, and queries might keep failing. Notify: @slack-monitoring @slack-data-alerts @slack-kafka-data-highway-alerts"
  escalation_message = "Number of transaction replication lag errors is too high. Investigation is required. Notify: @slack-neo4j-db"

  query = "logs(\"service:neo4j filename:query.log environment:${var.environment} *:\"not up to the requested\"\").index(\"*\").rollup(\"count\").last(\"15m\") > 250"

  monitor_thresholds {
    critical          = 250
    warning           = 100
    warning_recovery  = 0
    critical_recovery = 100
  }

  enable_logs_sample  = true
  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 15
  renotify_statuses   = ["alert"]
  timeout_h           = 1
  require_full_window = false

  priority = 2

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name_v5}",
    "application_family:data-platform",
  ]
}

resource "datadog_monitor" "asg_event_monitor" {
  name    = "Neo4j ${title(var.environment)} Auto Scaling Group Event Monitor"
  type    = "event-v2 alert"
  message = "{{#is_alert}}Auto Scaling Group Event has occurred for Neo4j Prod V5 cluster.\n[This documentation](https://www.notion.so/Neo4j-V5-Cluster-Management-20097177520f80c987b5ff881f018dd7) serves as a general guideline on what to check for. \nNotify: ${var.neo4j_escalation_endpoints}{{/is_alert}}"

  query = "events(\"source:amazon_auto_scaling autoscaling_group:${var.environment}*neo4j* -autoscaling_group:*neo4j-backup*\").rollup(\"count\").last(\"15m\") >= 1"

  on_missing_data = "resolve"

  monitor_thresholds {
    critical = 1
  }

  include_tags = true

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:devops",
  ]
}

resource "datadog_monitor" "dns_record_updated_monitor_v5" {
  name    = "Neo4j ${title(var.environment)} V5 DNS record has been updated"
  type    = "log alert"
  message = "{{#is_warning}}DNS record for Neo4j ${title(var.environment)} V5 cluster has been updated with new node information\nNotify: ${var.dns_lambda_notification_endpoints}{{/is_warning}}"

  query = "logs(\"service:networking-${var.dns_updater_lambda_name} env:${var.environment} \\\"DNS record updated for Neo4j ${var.environment} v5\\\"\").index(\"*\").rollup(\"count\").last(\"15m\") >= 10"

  monitor_thresholds {
    critical = 10
    warning  = 1
  }
  include_tags = true
  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:devops",
  ]
}