resource "datadog_monitor" "ameba_msk_max_offset_Lag_is_100" {
  name = "AMEBA MSK | Max Offset Lag is > 100"
  type = "query alert"
  query = <<EOT
avg(last_1h):max:aws.kafka.max_offset_lag{cluster_name:${var.msk_cluster_name},consumer_group:* ,!consumer_group:amazon* , !consumer_group:eom.oms-asset-ingest.*test.*} by {consumer_group,topic} > 100
EOT
  message = <<EOT
{{#is_alert}}
### Max Offset Lag is > 100
PRIORITY=HIGH

@slack-The_Orchard-ameba-alerts

{{/is_alert}}

{{#is_recovery}}
### Max Offset Lag recovered (below 50)

@slack-The_Orchard-ameba-alerts

{{/is_recovery}}
EOT
  draft_status = "published"
  escalation_message = <<EOT
EOT
  tags = ["team:ameba-team", "application_family:aoma-core"]
  evaluation_delay = 900
  new_group_delay = 60
  notify_by = ["*"]
  on_missing_data = "show_and_notify_no_data"
  renotify_interval = 30
  renotify_occurrences = 5
  renotify_statuses = ["alert", "warn"]
  require_full_window = false
  monitor_thresholds {
    critical = 100
    critical_recovery = 49
    warning_recovery = 19
  }
}
resource "datadog_monitor" "ameba_msk_monitor_mlib_max_offset_lag_max_offset_lag_is_100" {
  name = "AMEBA | MSK Monitor MLIB Max Offset Lag  | Max Offset Lag is > 100"
  type = "query alert"
  query = <<EOT
avg(last_1h):max:aws.kafka.sum_offset_lag{cluster_name:${var.msk_cluster_name} AND (consumer_group:mlib* OR consumer_group:delivery* )} by {topic,consumer_group} > 100
EOT
  message = <<EOT
{{#is_alert}}

### Max Offset Lag is > 100

PRIORITY=HIGH

@slack-The_Orchard-ameba-alerts

{{/is_alert}}

{{#is_recovery}}
### Max Offset Lag recovered (below 50)

@slack-The_Orchard-ameba-alerts

{{/is_recovery}}
EOT
  draft_status = "published"
  escalation_message = <<EOT
EOT
  tags = ["team:ameba-team", "application_family:aoma-core"]
  evaluation_delay = 900
  include_tags = false
  new_group_delay = 60
  notify_by = ["*"]
  on_missing_data = "default"
  renotify_interval = 30
  renotify_occurrences = 5
  renotify_statuses = ["alert", "warn"]
  require_full_window = false
  monitor_thresholds {
    critical = 100
    critical_recovery = 49
    warning_recovery = 19
  }
}

resource "datadog_monitor" "ameba_msk_free_storage_monitor_percent" {
  name = "AMEBA | MSK Empty Space"
  type = "query alert"
  query = <<EOT
avg(last_15m):max:aws.kafka.kafka_data_logs_disk_used{cluster_name:${var.msk_cluster_name}} by {broker_id} > ${var.msk_disk_used_alert_critical}
EOT
  message = <<EOT
{{#is_alert}}

#### MSK Empty Space is < ${100 - var.msk_disk_used_alert_critical}%

**Cluster:** ${var.msk_cluster_display_name}
**Broker:** {{broker_id.name}}

PRIORITY=HIGH

**Action Required:** Investigate disk usage patterns. Consider:
- Reviewing Kafka log retention policies
- Scaling broker storage if sustained high usage

@oncall-ameba-team
@slack-The_Orchard-ameba-alerts

{{/is_alert}}

{{#is_recovery}}
### MSK disk space recovered

@oncall-ameba-team
@slack-The_Orchard-ameba-alerts

{{/is_recovery}}
EOT
  draft_status = "published"
  escalation_message = <<EOT
EOT
  tags = ["team:ameba-team", "application_family:aoma-core"]
  evaluation_delay = 900
  include_tags = false
  new_group_delay = 60
  notify_by = ["*"]
  on_missing_data = "default"
  renotify_interval = 30
  renotify_occurrences = 5
  renotify_statuses = ["alert", "warn"]
  require_full_window = false
  monitor_thresholds {
    critical = var.msk_disk_used_alert_critical
    critical_recovery = var.msk_disk_used_alert_critical - 5
  }
}