resource "datadog_monitor" "high_error_rate_monitor" {
  for_each = var.services

  name    = "${var.environment}-${var.service_name} service ${each.key} has a high error rate"
  query   = "sum(last_10m):(sum:trace.${each.value}.request.errors{env:${var.environment},service:${each.key}}.as_count() / sum:trace.${each.value}.request.hits{env:${var.environment},service:${each.key}}.as_count()) > 0.05"
  type    = "query alert"
  message = <<EOF
High Error Rate Alert: The error rate for service `${each.key}` has exceeded acceptable limits. Immediate attention required.

${var.slack_channel}"
EOF

  monitor_thresholds {
    critical = 0.05
    warning  = 0.01
  }

  require_full_window = false
  notify_no_data      = false
  renotify_interval   = 0
  new_host_delay      = 300
  restricted_roles    = [data.datadog_role.datadog_admin_role.id]

  notify_audit = false
  timeout_h    = 0
  include_tags = true

  priority = 4

  tags = concat(["service:${each.key}"], local.common_tags)
}

resource "datadog_monitor" "high_average_latency_monitor" {
  for_each = var.services

  name    = "${var.environment}-${var.service_name} service ${each.key} has a high average latency"
  query   = "avg(last_10m):( sum:trace.${each.value}.request.duration{env:${var.environment},service:${each.key}}.rollup(sum).fill(zero) / sum:trace.${each.value}.request.hits{env:${var.environment},service:${each.key}} ) > 0.5"
  type    = "query alert"
  message = <<EOF
Latency Alert: The average latency for service `${each.key}` is higher than expected. Investigate potential performance bottlenecks.

${var.slack_channel}"
EOF

  monitor_thresholds {
    critical = 0.5
    warning  = 0.3
  }

  require_full_window = false
  notify_no_data      = false
  renotify_interval   = 0
  new_host_delay      = 300
  restricted_roles    = [data.datadog_role.datadog_admin_role.id]

  notify_audit = false
  timeout_h    = 0
  include_tags = true

  priority = 4

  tags = concat(["service:${each.key}"], local.common_tags)
}

resource "datadog_monitor" "high_p90_latency_monitor" {
  for_each = var.services

  name    = "${var.environment}-${var.service_name} service ${each.key} has a high p90 latency"
  query   = "percentile(last_10m):p90:trace.${each.value}.request{env:${var.environment},service:${each.key}} > 1"
  type    = "query alert"
  message = <<EOF
Latency Alert: The p90 latency for service `${each.key}` is higher than expected. Investigate potential performance bottlenecks.

${var.slack_channel}"
EOF

  monitor_thresholds {
    critical = 1
  }

  require_full_window = false
  notify_no_data      = false
  renotify_interval   = 0
  new_host_delay      = 300
  restricted_roles    = [data.datadog_role.datadog_admin_role.id]

  notify_audit = false
  timeout_h    = 0
  include_tags = true

  priority = 4

  tags = concat(["service:${each.key}"], local.common_tags)
}