resource "datadog_monitor" "amp_p_eks_cluster_memory_pressure_on_worker_nodes_on_cluster_n" {
  name = "AMP-P | EKS Cluster | Memory pressure on worker nodes on {{cluster_name.name}}"
  type = "query alert"
  query = <<EOT
min(last_1m):sum:kubernetes_state.node.by_condition{condition:memorypressure,cluster_name:amp-prod-as-eks} by {host,cluster_name} > 1
EOT
  message = <<EOT
{{#is_alert}}
Memory pressure on worker nodes on {{cluster_name.name}} is {{value}} < {{threshold}}
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
Memory pressure on worker nodes on {{cluster_name.name}} is {{value}} > {{threshold}}
PRIORITY=HIGH
{{/is_recovery}}

@pagerduty-AMP
@amp-microservices@sonymusic.com

EOT
  tags = ["team:amp-team","managed_by:amp-terraform"]
  notify_audit = false
  require_full_window = false
  include_tags = true
  new_group_delay = 60
  on_missing_data = "show_no_data"
  group_retention_duration = "2h"
  monitor_thresholds {
    critical = 1
  }
}
resource "datadog_monitor" "amp_p_eks_cluster_cpu_usage_on_worker_nodes_hostname_is_value" {
  name = "AMP-P | EKS Cluster | CPU usage on worker nodes {{host.name}} is {{value}}%"
  type = "query alert"
  query = <<EOT
avg(last_10m):avg:system.cpu.system{cluster_name:amp-prod-as-eks ,!host:i-08db9d6d794fb125d} by {host} + avg:system.cpu.system{cluster_name:amp-prod-as-eks ,!host:i-08db9d6d794fb125d} by {host} > 80
EOT
  message = <<EOT
{{#is_alert}}
### CPU usage is HIGH on **{{host.name}}**  {{value}}% > {{threshold}}% (threshold)
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
### CPU usage is NORMAL on {{host.name}}  {{value}}%  <= {{threshold}}% (threshold)
PRIORITY=HIGH
{{/is_recovery}}

@pagerduty-AMP
@amp-microservices@sonymusic.com

EOT
  tags = ["team:amp-team","managed_by:amp-terraform"]
  draft_status = "published"
  new_group_delay = 60
  on_missing_data = "default"
  require_full_window = false
  monitor_thresholds {
    critical = 80
  }
}
resource "datadog_monitor" "amp_p_eks_cluster_desired_number_of_replicas_on_kube_cluster_nam" {
  name = "AMP-P | EKS Cluster | Desired number of replicas on {{kube_cluster_name.name}}"
  type = "query alert"
  query = <<EOT
min(last_30m):sum:kubernetes_state.deployment.replicas_desired{kube_cluster_name:amp-prod-as-eks} by {kube_cluster_name} - sum:kubernetes_state.deployment.replicas_available{kube_cluster_name:amp-prod-as-eks} by {kube_cluster_name} > 1
EOT
  message = <<EOT
{{#is_alert}}
### ALERT -  Desired number of replicas not met on {{kube_cluster_name.name}}  is {{value}} < {{threshold}}
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
### NORMAL - Desired number of replicas met on {{kube_cluster_name.name}}  is {{value}} > {{threshold}}
{{/is_recovery}}

@pagerduty-AMP
@amp-microservices@sonymusic.com

EOT
  tags = ["team:amp-team","managed_by:amp-terraform"]
  draft_status = "published"
  new_group_delay = 60
  on_missing_data = "show_and_notify_no_data"
  require_full_window = false
  monitor_thresholds {
    critical = 1
  }
}
resource "datadog_monitor" "amp_p_eks_cluster_disk_pressure_on_worker_node_on_cluster_namena" {
  name = "AMP-P | EKS Cluster | Disk pressure on worker node on {{cluster_name.name}}"
  type = "query alert"
  query = <<EOT
min(last_1m):max:kubernetes_state.node.by_condition{condition:diskpressure,cluster_name:amp-prod-as-eks} by {host,kube_cluster_name} < 1
EOT
  message = <<EOT
{{#is_alert}}
###Number of EKS worker nodes available on {{cluster_name.name}} is {{value}} < {{threshold}}
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
###Number of EKS worker nodes available on {{cluster_name.name}} is {{value}} > {{threshold}}
PRIORITY=HIGH
{{/is_recovery}}

@pagerduty-AMP
@amp-microservices@sonymusic.com

EOT
  tags = ["team:amp-team","managed_by:amp-terraform"]
  notify_audit = false
  include_tags = true
  require_full_window = false
  new_group_delay = 60
  on_missing_data = "show_no_data"
  group_retention_duration = "2h"
  monitor_thresholds {
    critical = 1
  }
}
resource "datadog_monitor" "amp_p_eks_cluster_pod_status_on_cluster_namename" {
  name = "AMP-P | EKS Cluster | Pod Status on {{cluster_name.name}}"
  type = "query alert"
  query = <<EOT
min(last_5m):(avg:kubernetes_state.pod.count{cluster_name:amp-prod-as-eks} by {cluster_name} - avg:kubernetes.pods.running{cluster_name:amp-prod-as-eks} by {cluster_name}) * 100 >= 1
EOT
  message = <<EOT
{{#is_alert}}
### {{value}} Pods NOT RUNNING on {{cluster_name.name}}. Please check.

PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
### All pods are running on {{cluster_name.name}}
PRIORITY=HIGH
{{/is_recovery}}

@pagerduty-AMP
@amp-microservices@sonymusic.com

EOT
  tags = ["team:amp-team","managed_by:amp-terraform"]
  draft_status = "published"
  new_group_delay = 60
  on_missing_data = "show_and_notify_no_data"
  require_full_window = false
  monitor_thresholds {
    critical = 1
  }
}
resource "datadog_monitor" "amp_p_eks_cluster_worker_node_down_on_cluster_namename" {
  name = "AMP-P | EKS Cluster | Worker node down on {{cluster_name.name}}"
  type = "query alert"
  query = <<EOT
max(last_1m):sum:kubernetes_state.node.status{status:schedulable,cluster_name:amp-prod-as-eks} < 2
EOT
  message = <<EOT
{{#is_alert}}
###Number of EKS worker nodes available on {{cluster_name.name}} is {{value}} < {{threshold}}
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
###Number of EKS worker nodes available on {{cluster_name.name}} is {{value}} > {{threshold}}
PRIORITY=HIGH
{{/is_recovery}}

@pagerduty-AMP
@amp-microservices@sonymusic.com

EOT
  tags = ["team:amp-team","managed_by:amp-terraform"]
  draft_status = "published"
  include_tags = false
  on_missing_data = "show_and_notify_no_data"
  require_full_window = false
  monitor_thresholds {
    critical = 2
  }
}
resource "datadog_monitor" "amp_p_eks_cluster_memory_usage_on_kube_deploymentname_is_HIGH" {
  name = "AMP-P | EKS-Cluster | Memory Usage on {{kube_deployment.name}} is HIGH"
  type = "query alert"
  query = <<EOT
avg(last_10m):avg:kubernetes.memory.usage_pct{cluster_name:amp-prod-as-eks} by {kube_deployment,cluster_name} > 0.95
EOT
  message = <<EOT
{{#is_alert}}
Memory Usage on {{kube_deployment.name}} is {{value}} %  > {{threshold}} %
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
Memory Usage on {{kube_deployment.name}} is {{value}} %  < {{threshold}} %
PRIORITY=HIGH
{{/is_recovery}}

@pagerduty-AMP
@amp-microservices@sonymusic.com

EOT
  tags = ["team:amp-team","managed_by:amp-terraform"]
  draft_status = "published"
  new_group_delay = 60
  on_missing_data = "show_and_notify_no_data"
  require_full_window = false
  monitor_thresholds {
    critical = 0.95
  }
}