locals {
  common_tags = [
    "service:${var.service_name}",
    "env:${var.environment}",
    "terraform:true",
    "application_family:${var.application_family}",
  ]
}

resource "datadog_monitor" "service_latency_p50" {
  name = "${var.environment}-${var.service_name} High p50 Latency"
  type = "metric alert"

  message = <<-EOM
    Service ${var.service_name} in ${var.environment} is experiencing high p50 (median) latency.

    {{#is_alert}}CRITICAL: p50 latency is {{value}} ms, above {{threshold}} ms.{{/is_alert}}
    {{#is_warning}}WARNING: p50 latency is {{value}} ms, above {{threshold}} ms.{{/is_warning}}

    ${var.notification_endpoints}
  EOM

  tags = local.common_tags

  query = "avg(last_5m):p50:trace.flask.request{env:${var.environment},service:${var.service_name}} > 120"

  monitor_thresholds {
    critical = 120
    warning  = 100
  }

  notify_audit      = false
  timeout_h         = 0
  evaluation_delay  = 300
  renotify_interval = 0
  notify_no_data    = false
}

resource "datadog_monitor" "service_latency_p90" {
  name = "${var.environment}-${var.service_name} High p90 Latency"
  type = "metric alert"

  message = <<-EOM
    Service ${var.service_name} in ${var.environment} is experiencing high p90 latency.

    {{#is_alert}}CRITICAL: p90 latency is {{value}} ms, above {{threshold}} ms.{{/is_alert}}
    {{#is_warning}}WARNING: p90 latency is {{value}} ms, above {{threshold}} ms.{{/is_warning}}

    ${var.notification_endpoints}
  EOM

  tags = local.common_tags

  query = "avg(last_5m):p90:trace.flask.request{env:${var.environment},service:${var.service_name}} > 600"

  monitor_thresholds {
    critical = 600
    warning  = 400
  }

  notify_audit      = false
  timeout_h         = 0
  evaluation_delay  = 300
  renotify_interval = 0
  notify_no_data    = false
}

resource "datadog_monitor" "service_latency_p99" {
  name = "${var.environment}-${var.service_name} High p99 Latency"
  type = "metric alert"

  message = <<-EOM
    Service ${var.service_name} in ${var.environment} is experiencing high p99 latency.

    {{#is_alert}}CRITICAL: p99 latency is {{value}} ms, above {{threshold}} ms.{{/is_alert}}
    {{#is_warning}}WARNING: p99 latency is {{value}} ms, above {{threshold}} ms.{{/is_warning}}

    ${var.notification_endpoints}
  EOM

  tags = local.common_tags

  query = "avg(last_5m):p99:trace.flask.request{env:${var.environment},service:${var.service_name}} > 1000"

  monitor_thresholds {
    critical = 1000
    warning  = 800
  }

  notify_audit      = false
  timeout_h         = 0
  evaluation_delay  = 300
  renotify_interval = 0
  notify_no_data    = false
}

resource "datadog_monitor" "service_throughput" {
  name = "${var.environment}-${var.service_name} Low Throughput"
  type = "query alert"

  # Anomaly detection instead of a static floor: normal off-peak RPS regularly
  # dips to 0.02-0.5 req/s, which made a flat "< 1 rps" threshold flap on
  # ordinary bursty/low traffic instead of catching a real drop.
  query = "avg(last_4h):anomalies(avg:trace.flask.request.hits{env:${var.environment},service:${var.service_name}}.as_rate(), 'basic', 2, direction='below', alert_window='last_15m', interval=60, count_default_zero='true') >= 1"

  monitor_thresholds {
    critical = 1
    warning  = 0.5
  }

  message = <<-EOM
    Service ${var.service_name} in ${var.environment} is experiencing throughput (RPS) that is anomalously low for this time of day.

    {{#is_alert}}CRITICAL: Throughput is anomalously low.{{/is_alert}}
    {{#is_warning}}WARNING: Throughput is trending anomalously low.{{/is_warning}}

    ${var.notification_endpoints}
  EOM

  tags = local.common_tags

  evaluation_delay  = 300
  no_data_timeframe = 15
  notify_no_data    = true

  notify_audit      = false
  renotify_interval = 0
  timeout_h         = 0
}

resource "datadog_downtime_schedule" "service_throughput" {
  message          = "Mute low-throughput."
  display_timezone = "America/New_York"

  monitor_identifier {
    monitor_id = datadog_monitor.service_throughput.id
  }

  scope = "env:${var.environment} service:${var.service_name}"

  recurring_schedule {
    timezone = "America/New_York"

    # Weeknights 10:00 PM - 11:59 PM ET
    recurrence {
      duration = "2h"
      rrule    = "FREQ=WEEKLY;INTERVAL=1;BYDAY=MO,TU,WE,TH,FR"
      start    = "2025-09-29T22:00:00"
    }

    # Weekdays 00:00 AM - 04:00 AM ET
    recurrence {
      duration = "4h"
      rrule    = "FREQ=WEEKLY;INTERVAL=1;BYDAY=MO,TU,WE,TH,FR"
      start    = "2025-09-29T00:00:00"
    }

    # Weekends
    recurrence {
      duration = "1d"
      rrule    = "FREQ=WEEKLY;INTERVAL=1;BYDAY=SA,SU"
      start    = "2025-10-04T00:00:00"
    }
  }
}