resource "datadog_monitor" "ameba_dev_eks_node_not_ready" {
  name = "AMEBA-DEV - EKS node NotReady"
  type = "query alert"
  query = <<EOT
max(last_5m):sum:kubernetes_state.node.by_condition{condition:ready,status:false,kube_cluster_name:ameba-dev} by {kube_node} >= 1
EOT
  message = <<EOT
{{#is_alert}}
### EKS node **{{kube_node.name}}** is NotReady on cluster **ameba-dev**
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### EKS node **{{kube_node.name}}** is Ready again on cluster **ameba-dev**
PRIORITY=LOW
{{/is_recovery}}

@slack-The_Orchard-ameba-alerts
EOT
  draft_status         = "published"
  evaluation_delay     = 300
  notify_audit         = true
  require_full_window  = false
  include_tags         = true
  on_missing_data      = "default"
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "warn"]
  notify_by            = ["*"]

  monitor_thresholds {
    critical          = 1
    critical_recovery = 0
  }

  tags = ["team:ameba-team", "application_family:aoma-core"]
}

resource "datadog_monitor" "ameba_dev_pending_pods_spike" {
  name = "AMEBA-DEV - Pending pods spike"
  type = "query alert"
  query = <<EOT
avg(last_10m):sum:kubernetes_state.pod.status_phase{phase:pending,kube_namespace:ameba-dev,kube_cluster_name:ameba-dev} by {kube_namespace} > 5
EOT
  message = <<EOT
{{#is_alert}}
### Pending pods in **{{kube_namespace.name}}** on **ameba-dev** are {{value}} > {{threshold}}
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### Pending pods in **{{kube_namespace.name}}** on **{{kube_cluster_name.name}}** recovered to {{value}} < {{threshold}}
PRIORITY=LOW
{{/is_recovery}}

@slack-The_Orchard-ameba-alerts
EOT
  draft_status         = "published"
  evaluation_delay     = 300
  notify_audit         = true
  require_full_window  = false
  include_tags         = true
  on_missing_data      = "default"
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "warn"]
  notify_by            = ["*"]

  monitor_thresholds {
    critical          = 5
    critical_recovery = 2
  }

  tags = ["team:ameba-team", "application_family:aoma-core"]
}

resource "datadog_monitor" "ameba_dev_container_restarts_spike" {
  name = "AMEBA-DEV - Container restarts spike"
  type = "query alert"
  query = <<EOT
max(last_15m):sum:kubernetes.containers.restarts{kube_namespace:ameba-dev,kube_cluster_name:ameba-dev} by {kube_namespace,pod_name} > 5
EOT
  message = <<EOT
{{#is_alert}}
### Container restarts for pod **{{pod_name.name}}** in **{{kube_namespace.name}}** on **{{kube_cluster_name.name}}** are {{value}} > {{threshold}}
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### Container restarts for pod **{{pod_name.name}}** in **{{kube_namespace.name}}** on **{{kube_cluster_name.name}}** recovered to {{value}} < {{threshold}}
PRIORITY=LOW
{{/is_recovery}}

@slack-The_Orchard-ameba-alerts
EOT
  draft_status         = "published"
  evaluation_delay     = 300
  notify_audit         = true
  require_full_window  = false
  include_tags         = true
  on_missing_data      = "default"
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "warn"]
  notify_by            = ["*"]

  monitor_thresholds {
    critical          = 5
    critical_recovery = 3
  }

  tags = ["team:ameba-team", "application_family:aoma-core"]
}

resource "datadog_monitor" "ameba_stage_pending_pods_spike" {
  name = "AMEBA-STAGE - Pending pods spike"
  type = "query alert"
  query = <<EOT
avg(last_10m):sum:kubernetes_state.pod.status_phase{phase:pending,kube_namespace:ameba-stage,kube_cluster_name:ameba-dev} by {kube_namespace} > 5
EOT
  message = <<EOT
{{#is_alert}}
### Pending pods in **{{kube_namespace.name}}** on **ameba-dev** are {{value}} > {{threshold}}
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### Pending pods in **{{kube_namespace.name}}** on **{{kube_cluster_name.name}}** recovered to {{value}} < {{threshold}}
PRIORITY=LOW
{{/is_recovery}}

@slack-The_Orchard-ameba-alerts
EOT
  draft_status         = "published"
  evaluation_delay     = 300
  notify_audit         = true
  require_full_window  = false
  include_tags         = true
  on_missing_data      = "default"
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "warn"]
  notify_by            = ["*"]

  monitor_thresholds {
    critical          = 5
    critical_recovery = 2
  }

  tags = ["team:ameba-team", "application_family:aoma-core"]
}

resource "datadog_monitor" "ameba_stage_container_restarts_spike" {
  name = "AMEBA-STAGE - Container restarts spike"
  type = "query alert"
  query = <<EOT
max(last_15m):sum:kubernetes.containers.restarts{kube_namespace:ameba-stage,kube_cluster_name:ameba-dev} by {kube_namespace,pod_name} > 5
EOT
  message = <<EOT
{{#is_alert}}
### Container restarts for pod **{{pod_name.name}}** in **{{kube_namespace.name}}** on **{{kube_cluster_name.name}}** are {{value}} > {{threshold}}
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### Container restarts for pod **{{pod_name.name}}** in **{{kube_namespace.name}}** on **{{kube_cluster_name.name}}** recovered to {{value}} < {{threshold}}
PRIORITY=LOW
{{/is_recovery}}

@slack-The_Orchard-ameba-alerts
EOT
  draft_status         = "published"
  evaluation_delay     = 300
  notify_audit         = true
  require_full_window  = false
  include_tags         = true
  on_missing_data      = "default"
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "warn"]
  notify_by            = ["*"]

  monitor_thresholds {
    critical          = 5
    critical_recovery = 3
  }

  tags = ["team:ameba-team", "application_family:aoma-core"]
}

resource "datadog_monitor" "ameba_dev_msk_consumer_lag_high" {
  name = "AMEBA-DEV - MSK consumer lag is high"
  type = "query alert"
  query = <<EOT
avg(last_1h):max:aws.kafka.max_offset_lag{cluster_name:aomad2-ameba-msk-01,consumer_group:*} by {consumer_group,topic} > 100
EOT
  message = <<EOT
{{#is_alert}}
### MSK max offset lag is {{value}} > {{threshold}} for **{{consumer_group.name}}** on **{{topic.name}}** (cluster: {{cluster_name.name}})
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### MSK max offset lag recovered for **{{consumer_group.name}}** on **{{topic.name}}** (cluster: {{cluster_name.name}})
PRIORITY=LOW
{{/is_recovery}}

@ameba.developers@sonymusic.com
@slack-The_Orchard-ameba-alerts
EOT
  draft_status         = "published"
  evaluation_delay     = 900
  notify_audit         = false
  require_full_window  = false
  include_tags         = true
  on_missing_data      = "default"
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "warn"]
  notify_by            = ["*"]

  monitor_thresholds {
    critical          = 100
    critical_recovery = 49
  }

  tags = ["team:ameba-team", "application_family:aoma-core"]
}

resource "datadog_monitor" "ameba_stage_msk_consumer_lag_high" {
  name = "AMEBA-STAGE - MSK consumer lag is high"
  type = "query alert"
  query = <<EOT
avg(last_1h):max:aws.kafka.max_offset_lag{cluster_name:aomas-ameba-msk-01,consumer_group:*} by {consumer_group,topic} > 100
EOT
  message = <<EOT
{{#is_alert}}
### MSK max offset lag is {{value}} > {{threshold}} for **{{consumer_group.name}}** on **{{topic.name}}** (cluster: {{cluster_name.name}})
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### MSK max offset lag recovered for **{{consumer_group.name}}** on **{{topic.name}}** (cluster: {{cluster_name.name}})
PRIORITY=LOW
{{/is_recovery}}

@slack-The_Orchard-ameba-alerts
EOT
  draft_status         = "published"
  evaluation_delay     = 900
  notify_audit         = false
  require_full_window  = false
  include_tags         = true
  on_missing_data      = "default"
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "warn"]
  notify_by            = ["*"]

  monitor_thresholds {
    critical          = 100
    critical_recovery = 49
  }

  tags = ["team:ameba-team", "application_family:aoma-core"]
}

resource "datadog_monitor" "ameba_dev_http_check_gateway_url" {
  name = "AMEBA-DEV - HTTP Check Failed on ameba-gw URL"
  type = "service check"
  query = <<EOT
"http.can_connect".over("instance:ameba_gw_dev", "url:https://ameba-dev-gw.smcdp-aws.net/actuator/health").by("instance", "url").last(3).count_by_status()
EOT
  message = <<EOT
Endpoint source: Datadog service check instance `ameba_gw_dev` (URL reported as {{url.name}})
Purpose: External availability check for the AMEBA dev gateway.
Expected behavior: URL is reachable from the Datadog cluster check.

{{#is_alert}}
### HTTP check is failing for **{{url.name}}**
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### HTTP check recovered for **{{url.name}}**
PRIORITY=LOW
{{/is_recovery}}

{{#is_no_data}}
### No data from HTTP check for **{{url.name}}**
PRIORITY=LOW
{{/is_no_data}}

@slack-The_Orchard-ameba-alerts
EOT

  draft_status         = "published"
  notify_audit         = true
  include_tags         = true
  require_full_window  = false
  notify_no_data       = true
  no_data_timeframe    = 120
  new_group_delay      = 300
  notify_by            = ["*"]
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "no data"]

  monitor_thresholds {
    critical = 2
    warning  = 1
    ok       = 1
  }

  tags = ["team:ameba-team", "application_family:aoma-core"]
}

resource "datadog_monitor" "ameba_stage_http_check_gateway_url" {
  name = "AMEBA-STAGE - HTTP Check Failed on ameba-gw URL"
  type = "service check"
  query = <<EOT
"http.can_connect".over("instance:ameba_gw_stage", "url:https://ameba-stage-gw.smcdp-aws.net/actuator/health").by("instance", "url").last(3).count_by_status()
EOT
  message = <<EOT
Endpoint source: Datadog service check instance `ameba_gw_stage` (URL reported as {{url.name}})
Purpose: External availability check for the AMEBA stage gateway.
Expected behavior: URL is reachable from the Datadog cluster check.

{{#is_alert}}
### HTTP check is failing for **{{url.name}}**
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
### HTTP check recovered for **{{url.name}}**
PRIORITY=LOW
{{/is_recovery}}

{{#is_no_data}}
### No data from HTTP check for **{{url.name}}**
PRIORITY=LOW
{{/is_no_data}}

@slack-The_Orchard-ameba-alerts
EOT

  draft_status         = "published"
  notify_audit         = true
  include_tags         = true
  require_full_window  = false
  notify_no_data       = true
  no_data_timeframe    = 120
  new_group_delay      = 300
  notify_by            = ["*"]
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "no data"]

  monitor_thresholds {
    critical = 2
    warning  = 1
    ok       = 1
  }

  tags = ["team:ameba-team", "application_family:aoma-core"]
}