resource "datadog_monitor" "ameba_prod_eks_disk_pressure_on_worker_node_on_cluster_name" {
  name = "AMEBA-Prod - EKS - Disk pressure on worker node"
  type = "query alert"
  query = <<EOT
max(last_5m):avg:system.disk.free{cluster_name:ameba-prod, !device_name:shm} by {kube_node,device_name} < 1100000000
EOT
  message = <<EOT
{{#is_alert}}
###Disk free space on AMEBA-Prod worker node is {{value}} < {{threshold}}
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
###Disk free space on AMEBA-Prod worker node is {{value}} > {{threshold}}
PRIORITY=HIGH
{{/is_recovery}}

@oncall-ameba-team
@ameba.developers@sonymusic.com
@slack-The_Orchard-ameba-alerts
EOT
  draft_status = "published"
  escalation_message = <<EOT

EOT
  tags = ["team:ameba-team", "application_family:aoma-core"]
  evaluation_delay = 300
  new_group_delay = 0
  notify_by = ["*"]
  on_missing_data = "show_and_notify_no_data"
  renotify_interval = 30
  renotify_occurrences = 5
  renotify_statuses = ["alert", "warn"]
  require_full_window = false
  monitor_thresholds {
    critical = 1100000000
    critical_recovery = 1200000000
  }
}
resource "datadog_monitor" "ameba_prod_eks_internal_workernode_status_pods_not_schedulable" {
  name = "AMEBA-Prod - EKS - Internal Workernode status : Pods Not Schedulable"
  type = "query alert"
  query = <<EOT
avg(last_5m):sum:kubernetes_state.node.status{cluster_name:ameba-prod, name:ameba-prod-eks-int-worker-node-node, status:schedulable} < 2
EOT
  message = <<EOT
{{#is_alert}}
###Internal Worker node status is NOT Schedulable, Please check the node status/Evicted Pods on the cluster
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
###Internal Worker node status is Schedulable now.
PRIORITY=HIGH
{{/is_recovery}}

@ameba.developers@sonymusic.com
@slack-The_Orchard-ameba-alerts
EOT
  tags = ["team:ameba-team", "application_family:aoma-core"]
  draft_status = "published"
  evaluation_delay = 300
  on_missing_data = "show_and_notify_no_data"
  renotify_interval = 30
  renotify_occurrences = 5
  renotify_statuses = ["alert", "warn"]
  require_full_window = false
  monitor_thresholds {
    critical = 2
    critical_recovery = 3
  }
}

resource "datadog_monitor" "ameba_prod_eks_pod_status_failed_on_namespace_ameba_prod" {
  name = "AMEBA-Prod - EKS - Pod Status Failed on namespace: ameba-prod"
  type = "query alert"
  query = <<EOT
avg(last_5m):sum:kubernetes.pods.running{kube_namespace:ameba-prod, cluster_name:ameba-prod} < 14
EOT
  message = <<EOT
{{#is_alert}}
###Number of pods running on Cluster ameba-prod, Namespace: ameba-prod is < {{threshold}} (count)
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
###Number of pods running on Cluster ameba-prod, Namespace: ameba-prod is updated to desired state: {{threshold}} (count)
PRIORITY=HIGH
{{/is_recovery}}


@ameba.developers@sonymusic.com
@slack-The_Orchard-ameba-alerts
EOT
  tags = ["team:ameba-team", "application_family:aoma-core"]
  draft_status = "published"
  on_missing_data = "show_and_notify_no_data"
  require_full_window = false
  monitor_thresholds {
    critical = 14
    critical_recovery = 15
  }
}
resource "datadog_monitor" "ameba_prod_eks_pod_status_failed_on_namespace_gw_prod" {
  name = "AMEBA-Prod - EKS - Pod Status Failed on namespace: gw-prod"
  type = "query alert"
  query = <<EOT
avg(last_5m):sum:kubernetes.pods.running{kube_namespace:gw-prod, cluster_name:ameba-prod} < 4
EOT
  message = <<EOT
{{#is_alert}}
###Number of pods running on Cluster ameba-prod, Namespace: gw-prod is < {{threshold}} (count)
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
###Number of pods running on Cluster ameba-prod, Namespace: gw-prod is updated to desired state: {{threshold}} (count)
PRIORITY=HIGH
{{/is_recovery}}

@ameba.developers@sonymusic.com
@slack-The_Orchard-ameba-alerts
EOT
  tags = ["team:ameba-team", "application_family:aoma-core"]
  draft_status = "published"
  on_missing_data = "show_and_notify_no_data"
  require_full_window = false
  monitor_thresholds {
    critical = 4
    critical_recovery = 5
  }
}
resource "datadog_monitor" "ameba_prod_eks_pod_status_failed_on_cluster_namespace_aoma3" {
  name = "AMEBA-Prod - EKS - Pod Status Failed on namespace: aoma3"
  type = "query alert"
  query = <<EOT
avg(last_5m):sum:kubernetes.pods.running{kube_namespace IN (aoma3) AND cluster_name:ameba-prod} by {kube_namespace,cluster_name} < 12
EOT
  message = <<EOT
{{#is_alert}}
###Number of pods running on Cluster {{cluster_name.name}} , Namespace: {{kube_namespace.name}}  is < {{threshold}} (count)
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
###Number of pods running on Cluster {{cluster_name.name}} , Namespace: {{kube_namespace.name}}  is updated to desired state: {{threshold}} (count)
PRIORITY=HIGH
{{/is_recovery}}

@ameba.developers@sonymusic.com
@slack-The_Orchard-ameba-alerts
EOT
  tags = ["team:ameba-team", "application_family:aoma-core"]
  draft_status = "published"
  new_group_delay = 60
  on_missing_data = "show_and_notify_no_data"
  require_full_window = false
  monitor_thresholds {
    critical = 12
    critical_recovery = 13
  }
}
resource "datadog_monitor" "ameba_prod_eks_pod_status_failed_on_cluster_namespace_digital_archive" {
  name = "AMEBA-Prod - EKS - Pod Status Failed on namespace: digital-archive"
  type = "query alert"
  query = <<EOT
avg(last_5m):sum:kubernetes.pods.running{kube_namespace:digital-archive, cluster_name:ameba-prod} by {kube_namespace,cluster_name} < 3
EOT
  message = <<EOT
{{#is_alert}}
###Number of pods running on Cluster {{cluster_name.name}} , Namespace: {{kube_namespace.name}}  is < {{threshold}} (count)
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
###Number of pods running on Cluster {{cluster_name.name}} , Namespace: {{kube_namespace.name}}  is updated to desired state: {{threshold}} (count)
PRIORITY=HIGH
{{/is_recovery}}

@ameba.developers@sonymusic.com
@slack-The_Orchard-ameba-alerts
EOT
  tags = ["team:ameba-team", "application_family:aoma-core"]
  draft_status = "published"
  new_group_delay = 60
  on_missing_data = "show_and_notify_no_data"
  require_full_window = false
  monitor_thresholds {
    critical = 3
    critical_recovery = 4
  }
}
resource "datadog_monitor" "ameba_prod_eks_worker_node_disk_space_usage_on_hostname_device" {
  name = "AMEBA-Prod - EKS Worker-node - Disk space usage is high"
  type = "query alert"
  query = <<EOT
avg(last_15m):avg:system.disk.in_use{kube_cluster_name:ameba-prod} by {host,device} * 100 >= ${var.eks_worker_disk_usage_critical}
EOT
  message = <<EOT
{{#is_alert}}
### Disk space usage on {{host.name}}: **{{device.name}}** is {{value}}% > {{threshold}}%
PRIORITY=HIGH

**Action Required:**
- Check which filesystem is filling on the worker node.
- Clean up logs/cache or expand disk if usage remains high.
{{/is_alert}}

{{#is_recovery}}
### Disk space usage on {{host.name}}: **{{device.name}}** is {{value}}% < {{threshold}}%
PRIORITY=HIGH
{{/is_recovery}}

@oncall-ameba-team
@ameba.developers@sonymusic.com
@slack-The_Orchard-ameba-alerts
EOT
  tags = ["team:ameba-team", "application_family:aoma-core"]
  draft_status = "published"
  evaluation_delay = 300
  new_group_delay = 60
  notify_by = ["*"]
  on_missing_data = "default"
  renotify_interval = 30
  renotify_occurrences = 5
  renotify_statuses = ["alert", "warn"]
  require_full_window = false
  monitor_thresholds {
    critical = var.eks_worker_disk_usage_critical
    critical_recovery = 85
  }
}

resource "datadog_monitor" "ameba_prod_eks_worker_node_down_less_than_7" {
  name = "AMEBA-Prod - EKS - worker node down less than 7"
  type = "query alert"
  query = <<EOT
max(last_1m):sum:kubernetes_state.node.status{status:schedulable,cluster_name:ameba-prod} < 7
EOT
  message = <<EOT
{{#is_alert}}
###Number of EKS worker nodes available on {{cluster_name.name}} is {{value}} < {{threshold}}
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
###Number of EKS worker nodes available on {{cluster_name.name}} is {{value}} > {{threshold}}
PRIORITY=HIGH
{{/is_recovery}}



@ameba.developers@sonymusic.com
@slack-The_Orchard-ameba-alerts
EOT
  draft_status = "published"
  escalation_message = <<EOT
EOT
  tags = ["team:ameba-team", "application_family:aoma-core"]
  evaluation_delay = 300
  notify_audit = true
  on_missing_data = "show_and_notify_no_data"
  renotify_interval = 30
  renotify_occurrences = 5
  renotify_statuses = ["alert", "warn"]
  require_full_window = false
  monitor_thresholds {
    critical = 7
    critical_recovery = 8
  }
}

resource "datadog_monitor" "ameba_prod_eks_pod_not_running_or_completed_more_than_5m" {
  name = "AMEBA-Prod - EKS - Pod has non-running status for more than 5 minutes on namespace: ameba-prod"
  type = "query alert"
  query = <<EOT
max(last_5m):sum:kubernetes_state.pod.status_phase{cluster_name:ameba-prod,kube_namespace:ameba-prod,!phase:running,!phase:succeeded} by {kube_namespace,pod_name,phase} >= 1
EOT
  message = <<EOT
{{#is_alert}}
### Pod **{{pod_name.name}}** in namespace **{{kube_namespace.name}}** is in status **{{phase.name}}** for more than 5 minutes.
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
### Pod **{{pod_name.name}}** in namespace **{{kube_namespace.name}}** returned to Running/Completed state.
PRIORITY=HIGH
{{/is_recovery}}

@oncall-ameba-team
@ameba.developers@sonymusic.com
@slack-The_Orchard-ameba-alerts
EOT
  tags = ["team:ameba-team", "application_family:aoma-core"]
  draft_status = "published"
  evaluation_delay = 300
  on_missing_data = "default"
  renotify_interval = 30
  renotify_occurrences = 5
  renotify_statuses = ["alert", "warn"]
  require_full_window = false
  notify_by = ["*"]
  monitor_thresholds {
    critical          = 1
    critical_recovery = 0
  }
}

resource "datadog_monitor" "ameba_prod_eks_pod_not_running_or_completed_more_than_5m_namespace_gw_prod" {
  name = "AMEBA-Prod - EKS - Pod has non-running status for more than 5 minutes on namespace: gw-prod"
  type = "query alert"
  query = <<EOT
max(last_5m):sum:kubernetes_state.pod.status_phase{cluster_name:ameba-prod,kube_namespace:gw-prod,!phase:running,!phase:succeeded} by {kube_namespace,pod_name,phase} >= 1
EOT
  message = <<EOT
{{#is_alert}}
### Pod **{{pod_name.name}}** in namespace **{{kube_namespace.name}}** is in status **{{phase.name}}** for more than 5 minutes.
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
### Pod **{{pod_name.name}}** in namespace **{{kube_namespace.name}}** returned to Running/Completed state.
PRIORITY=HIGH
{{/is_recovery}}

@oncall-ameba-team
@ameba.developers@sonymusic.com
@slack-The_Orchard-ameba-alerts
EOT
  tags = ["team:ameba-team", "application_family:aoma-core"]
  draft_status = "published"
  evaluation_delay = 300
  on_missing_data = "default"
  renotify_interval = 30
  renotify_occurrences = 5
  renotify_statuses = ["alert", "warn"]
  require_full_window = false
  notify_by = ["*"]
  monitor_thresholds {
    critical          = 1
    critical_recovery = 0
  }
}

resource "datadog_monitor" "ameba_clock_sync_with_ntp" {
  name = "AMEBA-Prod - Clock Sync with NTP"
  type = "query alert"
  query = <<EOT
avg(last_5m):avg:ntp.offset{cluster_name:ameba-prod} > 3
EOT
  message = <<EOT
<!--StartFragment-->

{{#is_alert}}

### Clock NOT in sync with NTP - Host ** {{host.name}} **. Check NTP service status and time on the host.

{{/is_alert}}

{{#is_recovery}}

### Clock Sync with NTP - Host ** {{host.name}} **

{{/is_recovery}}
 @oncall-ameba-team
 @ameba.developers@sonymusic.com
 @slack-The_Orchard-ameba-alerts

<!--EndFragment-->
EOT
  tags = ["team:ameba-team", "application_family:aoma-core"]
  draft_status = "published"
  include_tags = false
  on_missing_data = "default"
  require_full_window = false
  monitor_thresholds {
    critical = 3
    critical_recovery = 2
  }
}

resource "datadog_monitor" "ameba_prod_http_check_gateway_url" {
  name = "AMEBA-Prod - HTTP Check Failed on ameba-gw URL"
  type = "service check"
  query = <<EOT
"http.can_connect".over("instance:ameba_gw_prod", "url:https://ameba-gw.smcdp-aws.net/actuator/health").by("instance", "url").last(3).count_by_status()
EOT
  message = <<EOT
Endpoint source: Datadog service check instance `ameba_gw_prod` (URL reported as {{url.name}})
Purpose: External availability check for the AMEBA production gateway.
Expected behavior: URL is reachable from the Datadog cluster check.

{{#is_alert}}
### HTTP check is failing for **{{url.name}}**
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
### HTTP check recovered for **{{url.name}}**
PRIORITY=HIGH
{{/is_recovery}}

{{#is_no_data}}
### No data from HTTP check for **{{url.name}}**
PRIORITY=HIGH
{{/is_no_data}}

@oncall-ameba-team
@ameba.developers@sonymusic.com
@slack-The_Orchard-ameba-alerts
EOT

  draft_status         = "published"
  notify_audit         = true
  include_tags         = true
  require_full_window  = false
  notify_no_data       = true
  no_data_timeframe    = 120
  new_group_delay      = 300
  notify_by            = ["*"]
  renotify_interval    = 30
  renotify_occurrences = 5
  renotify_statuses    = ["alert", "no data"]

  monitor_thresholds {
    critical = 2
    warning  = 1
    ok       = 1
  }

  tags = ["team:ameba-team", "application_family:aoma-core"]
}