resource "datadog_monitor" "global_ds_dev_pod_crashloopbackoff" {
  name    = "Global-DS - DEV - EKS Pod CrashLoopBackOff on global-ds"
  type    = "query alert"
  message = <<EOT
{{#is_alert}}
### Pod **{{pod_name.name}}** is CrashLoopBackOff on namespace **{{kube_namespace.name}}**
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### Pod **{{pod_name.name}}** recovered from CrashLoopBackOff on namespace **{{kube_namespace.name}}**
PRIORITY=LOW
{{/is_recovery}}

@ameba.developers@sonymusic.com
@slack-The_Orchard-ameba-alerts
EOT
  query = "max(last_1h):default_zero(max:kubernetes_state.container.status_report.count.waiting{reason:crashloopbackoff,kube_cluster_name:${var.gds_eks_cluster_name},kube_namespace:${var.gds_dev_namespace}} by {kube_cluster_name,kube_namespace,pod_name}) >= 1"

  monitor_thresholds {
    critical          = 1
    critical_recovery = 0
  }

  draft_status        = "published"
  evaluation_delay    = 300
  notify_audit        = true
  require_full_window = false
  include_tags        = true
  renotify_interval   = 30
  renotify_occurrences = 5
  renotify_statuses   = ["alert", "warn"]
  on_missing_data     = "default"
  notify_by           = ["*"]

  tags = ["integration:kubernetes", "team:ameba-team", "application_family:global-ds"]
}

resource "datadog_monitor" "global_ds_dev_msk_free_space_low" {
  name = "Global-DS - DEV - MSK free space is low"
  type = "query alert"
  message = <<EOT
{{#is_alert}}
### MSK disk usage on broker {{broker_id.name}} is {{value}}% > {{threshold}}% (free space < 15%)
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### MSK disk usage on broker {{broker_id.name}} is {{value}}% < {{threshold}}% (free space recovered)
PRIORITY=LOW
{{/is_recovery}}



@slack-The_Orchard-ameba-alerts
EOT
  query = "avg(last_1h):avg:aws.kafka.kafka_data_logs_disk_used{cluster_name:${var.gds_msk_dev_cluster_name}} by {broker_id} > 85"

  monitor_thresholds {
    critical          = 85
    critical_recovery = 80
  }

  draft_status         = "published"
  notify_audit         = false
  on_missing_data      = "default"
  include_tags         = true
  evaluation_delay     = 900
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "warn"]
  notify_by            = ["*"]

  tags = ["team:ameba-team", "application_family:global-ds"]
}

resource "datadog_monitor" "global_ds_uat_msk_free_space_low" {
  name = "Global-DS - UAT - MSK free space is low"
  type = "query alert"
  message = <<EOT
{{#is_alert}}
### MSK disk usage on broker {{broker_id.name}} is {{value}}% > {{threshold}}% (free space < 15%)
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### MSK disk usage on broker {{broker_id.name}} is {{value}}% < {{threshold}}% (free space recovered)
PRIORITY=LOW
{{/is_recovery}}



@slack-The_Orchard-ameba-alerts
EOT
  query = "avg(last_1h):avg:aws.kafka.kafka_data_logs_disk_used{cluster_name:${var.gds_msk_uat_cluster_name}} by {broker_id} > 85"

  monitor_thresholds {
    critical          = 85
    critical_recovery = 80
  }

  draft_status         = "published"
  notify_audit         = false
  on_missing_data      = "default"
  include_tags         = true
  evaluation_delay     = 900
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "warn"]
  notify_by            = ["*"]

  tags = ["team:ameba-team", "application_family:global-ds"]
}

# UAT shares the global-ds-dev EKS cluster with DEV (separated by namespace, not by cluster),
# unlike MSK which has dedicated dev/uat clusters. UAT monitors below intentionally filter on
# kube_cluster_name:global-ds-dev.
resource "datadog_monitor" "global_ds_dev_eks_node_not_ready" {
  name = "Global-DS - DEV/UAT - EKS node NotReady"
  type = "query alert"
  message = <<EOT
{{#is_alert}}
### EKS node **{{kube_node.name}}** is NotReady on cluster global-ds-dev
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### EKS node **{{kube_node.name}}** is Ready again on cluster global-ds-dev
PRIORITY=LOW
{{/is_recovery}}

@slack-The_Orchard-ameba-alerts
EOT
  query = "max(last_5m):sum:kubernetes_state.node.by_condition{condition:ready,status:false,kube_cluster_name:${var.gds_eks_cluster_name}} by {kube_node} >= 1"

  monitor_thresholds {
    critical          = 1
    critical_recovery = 0
  }

  draft_status         = "published"
  evaluation_delay     = 300
  notify_audit         = true
  require_full_window  = false
  include_tags         = true
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "warn"]
  on_missing_data      = "default"
  notify_by            = ["*"]

  tags = ["integration:kubernetes", "team:ameba-team", "application_family:global-ds"]
}

resource "datadog_monitor" "global_ds_dev_pending_pods_spike" {
  name = "Global-DS - DEV - Pending pods spike"
  type = "query alert"
  message = <<EOT
{{#is_alert}}
### Pending pods in namespace **{{kube_namespace.name}}** are {{value}} > {{threshold}}
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### Pending pods in namespace **{{kube_namespace.name}}** recovered to {{value}} < {{threshold}}
PRIORITY=LOW
{{/is_recovery}}

@slack-The_Orchard-ameba-alerts
EOT
  query = "avg(last_10m):sum:kubernetes_state.pod.status_phase{phase:pending,kube_cluster_name:${var.gds_eks_cluster_name},kube_namespace:${var.gds_dev_namespace}} by {kube_namespace} > 5"

  monitor_thresholds {
    critical          = 5
    critical_recovery = 2
  }

  draft_status         = "published"
  evaluation_delay     = 300
  notify_audit         = true
  require_full_window  = false
  include_tags         = true
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "warn"]
  on_missing_data      = "default"
  notify_by            = ["*"]

  tags = ["integration:kubernetes", "team:ameba-team", "application_family:global-ds"]
}

resource "datadog_monitor" "global_ds_dev_container_restarts_spike" {
  name = "Global-DS - DEV - Container restarts spike on global-ds"
  type = "query alert"
  message = <<EOT
{{#is_alert}}
### Container restarts for pod **{{pod_name.name}}** in namespace **{{kube_namespace.name}}** are {{value}} > {{threshold}}
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### Container restarts for pod **{{pod_name.name}}** in namespace **{{kube_namespace.name}}** recovered to {{value}} < {{threshold}}
PRIORITY=LOW
{{/is_recovery}}

@slack-The_Orchard-ameba-alerts
EOT
  query = "max(last_15m):sum:kubernetes.containers.restarts{kube_cluster_name:${var.gds_eks_cluster_name},kube_namespace:${var.gds_dev_namespace}} by {kube_namespace,pod_name} > 5"

  monitor_thresholds {
    critical          = 5
    critical_recovery = 3
  }

  draft_status         = "published"
  evaluation_delay     = 300
  notify_audit         = true
  require_full_window  = false
  include_tags         = true
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "warn"]
  on_missing_data      = "default"
  notify_by            = ["*"]


  tags = ["integration:kubernetes", "team:ameba-team", "application_family:global-ds"]
}

resource "datadog_monitor" "global_ds_uat_pod_crashloopbackoff" {
  name    = "Global-DS - UAT - EKS Pod CrashLoopBackOff on global-ds-uat"
  type    = "query alert"
  message = <<EOT
{{#is_alert}}
### Pod **{{pod_name.name}}** is CrashLoopBackOff on namespace **{{kube_namespace.name}}**
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### Pod **{{pod_name.name}}** recovered from CrashLoopBackOff on namespace **{{kube_namespace.name}}**
PRIORITY=LOW
{{/is_recovery}}

@slack-The_Orchard-ameba-alerts
EOT
  query = "max(last_1h):default_zero(max:kubernetes_state.container.status_report.count.waiting{reason:crashloopbackoff,kube_cluster_name:${var.gds_eks_cluster_name},kube_namespace:${var.gds_uat_namespace}} by {kube_cluster_name,kube_namespace,pod_name}) >= 1"

  monitor_thresholds {
    critical          = 1
    critical_recovery = 0
  }

  draft_status        = "published"
  evaluation_delay    = 300
  notify_audit        = true
  require_full_window = false
  include_tags        = true
  renotify_interval   = 30
  renotify_occurrences = 5
  renotify_statuses   = ["alert", "warn"]
  on_missing_data     = "default"
  notify_by           = ["*"]

  tags = ["integration:kubernetes", "team:ameba-team", "application_family:global-ds"]
}

resource "datadog_monitor" "global_ds_uat_pending_pods_spike" {
  name = "Global-DS - UAT - Pending pods spike on global-ds-uat"
  type = "query alert"
  message = <<EOT
{{#is_alert}}
### Pending pods in namespace **{{kube_namespace.name}}** are {{value}} > {{threshold}}
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### Pending pods in namespace **{{kube_namespace.name}}** recovered to {{value}} < {{threshold}}
PRIORITY=LOW
{{/is_recovery}}

@slack-The_Orchard-ameba-alerts
EOT
  query = "avg(last_10m):sum:kubernetes_state.pod.status_phase{phase:pending,kube_cluster_name:${var.gds_eks_cluster_name},kube_namespace:${var.gds_uat_namespace}} by {kube_namespace} > 5"

  monitor_thresholds {
    critical          = 5
    critical_recovery = 2
  }

  draft_status         = "published"
  evaluation_delay     = 300
  notify_audit         = true
  require_full_window  = false
  include_tags         = true
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "warn"]
  on_missing_data      = "default"
  notify_by            = ["*"]

  tags = ["integration:kubernetes", "team:ameba-team", "application_family:global-ds"]
}

resource "datadog_monitor" "global_ds_uat_container_restarts_spike" {
  name = "Global-DS - UAT - Container restarts spike on global-ds-uat"
  type = "query alert"
  message = <<EOT
{{#is_alert}}
### Container restarts for pod **{{pod_name.name}}** in namespace **{{kube_namespace.name}}** are {{value}} > {{threshold}}
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### Container restarts for pod **{{pod_name.name}}** in namespace **{{kube_namespace.name}}** recovered to {{value}} < {{threshold}}
PRIORITY=LOW
{{/is_recovery}}

@slack-The_Orchard-ameba-alerts
EOT
  query = "max(last_15m):sum:kubernetes.containers.restarts{kube_cluster_name:${var.gds_eks_cluster_name},kube_namespace:${var.gds_uat_namespace}} by {kube_namespace,pod_name} > 5"

  monitor_thresholds {
    critical          = 5
    critical_recovery = 3
  }

  draft_status         = "published"
  evaluation_delay     = 300
  notify_audit         = true
  require_full_window  = false
  include_tags         = true
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "warn"]
  on_missing_data      = "default"
  notify_by            = ["*"]

  tags = ["integration:kubernetes", "team:ameba-team", "application_family:global-ds"]
}

resource "datadog_monitor" "global_ds_dev_msk_consumer_lag_high" {
  name = "Global-DS - DEV - MSK consumer lag is high"
  type = "query alert"
  message = <<EOT
{{#is_alert}}
### MSK max offset lag is {{value}} > {{threshold}} for **{{consumer_group.name}}** on topic **{{topic.name}}**
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### MSK max offset lag recovered for **{{consumer_group.name}}** on topic **{{topic.name}}**
PRIORITY=LOW
{{/is_recovery}}


@slack-The_Orchard-ameba-alerts
EOT
  query = "avg(last_1h):max:aws.kafka.max_offset_lag{cluster_name:${var.gds_msk_dev_cluster_name},consumer_group:*} by {consumer_group,topic} > 100"

  monitor_thresholds {
    critical          = 100
    critical_recovery = 49
  }

  draft_status         = "published"
  evaluation_delay     = 900
  notify_audit         = false
  require_full_window  = false
  include_tags         = true
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "warn"]
  on_missing_data      = "default"
  notify_by            = ["*"]


  tags = ["team:ameba-team", "application_family:global-ds"]
}

resource "datadog_monitor" "global_ds_uat_msk_consumer_lag_high" {
  name = "Global-DS - UAT - MSK consumer lag is high"
  type = "query alert"
  message = <<EOT
{{#is_alert}}
### MSK max offset lag is {{value}} > {{threshold}} for **{{consumer_group.name}}** on topic **{{topic.name}}**
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### MSK max offset lag recovered for **{{consumer_group.name}}** on topic **{{topic.name}}**
PRIORITY=LOW
{{/is_recovery}}


@slack-The_Orchard-ameba-alerts
EOT
  query = "avg(last_1h):max:aws.kafka.max_offset_lag{cluster_name:${var.gds_msk_uat_cluster_name},consumer_group:*} by {consumer_group,topic} > 100"

  monitor_thresholds {
    critical          = 100
    critical_recovery = 49
  }

  draft_status         = "published"
  evaluation_delay     = 900
  notify_audit         = false
  require_full_window  = false
  include_tags         = true
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "warn"]
  on_missing_data      = "default"
  notify_by            = ["*"]


  tags = ["team:ameba-team", "application_family:global-ds"]
}

resource "datadog_monitor" "global_ds_dev_http_check_gateway_url" {
  name = "Global-DS - DEV - HTTP Check Failed on global-ds-gw URL"
  type = "service check"
  query = <<EOT
"http.can_connect".over("instance:gds_dev_url", "url:https://global-ds-dev-gw.smcdp-aws.net/actuator/health").by("instance", "url").last(3).count_by_status()
EOT
  message = <<EOT
Endpoint source: Datadog service check instance `gds_dev_url` (URL reported as {{url.name}})
Purpose: External availability check for the Global-DS dev gateway.
Expected behavior: URL is reachable from the Datadog cluster check.

{{#is_alert}}
### HTTP check is failing for **{{url.name}}**
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### HTTP check recovered for **{{url.name}}**
PRIORITY=LOW
{{/is_recovery}}

{{#is_no_data}}
### No data from HTTP check for **{{url.name}}**
PRIORITY=LOW
{{/is_no_data}}

@slack-The_Orchard-ameba-alerts
EOT

  draft_status         = "published"
  notify_audit         = true
  include_tags         = true
  require_full_window  = false
  notify_no_data       = true
  no_data_timeframe    = 120
  new_group_delay      = 300
  notify_by            = ["*"]
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "no data"]

  monitor_thresholds {
    critical = 2
    warning  = 1
    ok       = 1
  }

  tags = ["team:ameba-team", "application_family:global-ds"]
}

resource "datadog_monitor" "global_ds_uat_http_check_gateway_url" {
  name = "Global-DS - UAT - HTTP Check Failed on global-ds-gw URL"
  type = "service check"
  query = <<EOT
"http.can_connect".over("instance:gds_uat_url", "url:https://global-ds-uat-gw.smcdp-aws.net/actuator/health").by("instance", "url").last(3).count_by_status()
EOT
  message = <<EOT
Endpoint source: Datadog service check instance `gds_uat_url` (URL reported as {{url.name}})
Purpose: External availability check for the Global-DS uat gateway.
Expected behavior: URL is reachable from the Datadog cluster check.

{{#is_alert}}
### HTTP check is failing for **{{url.name}}**
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### HTTP check recovered for **{{url.name}}**
PRIORITY=LOW
{{/is_recovery}}

{{#is_no_data}}
### No data from HTTP check for **{{url.name}}**
PRIORITY=LOW
{{/is_no_data}}

@slack-The_Orchard-ameba-alerts
EOT

  draft_status         = "published"
  notify_audit         = true
  include_tags         = true
  require_full_window  = false
  notify_no_data       = true
  no_data_timeframe    = 120
  new_group_delay      = 300
  notify_by            = ["*"]
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "no data"]

  monitor_thresholds {
    critical = 2
    warning  = 1
    ok       = 1
  }

  tags = ["team:ameba-team", "application_family:global-ds"]
}