resource "datadog_monitor" "arvatok8s_prod_amino_pod_status_aoma-prod" {
  name = "ArvatoK8S - Prod- AMINO Pod Status - aoma-prod"
  type = "query alert"
  query = <<EOT
avg(last_5m):sum:kubernetes_state.deployment.replicas_desired{kube_cluster_name:arvato-prod,kube_namespace:aoma-prod , kube_deployment:amino-*} by {kube_deployment} - sum:kubernetes.pods.running{cluster_name:arvato-prod ,kube_namespace:aoma-prod, kube_deployment:amino-*} by {kube_deployment} >= 1
EOT
  message = <<EOT
{{#is_alert}}
### {{value}} AMINO Pods NOT RUNNING on PROD/{{cluster_name.name}}. Please check.

PRIORITY=HIGH
{{/is_alert}}

@team-amp-developers

{{#is_recovery}}
### All AMINO pods are running on PROD/{{cluster_name.name}}.
PRIORITY=HIGH
{{/is_recovery}}
EOT
  tags = ["env:Arvatok8sProd", "team:amp-developers", "application_family:aoma-core"]
  draft_status = "published"
  new_group_delay = 0
  notification_preset_name = "hide_query"
  on_missing_data = "default"
  require_full_window = false
  monitor_thresholds {
    critical = 1
  }
}
resource "datadog_monitor" "arvatok8s_prod_cpu_usage_on_worker_nodes_hostname_is_value" {
  name = "ArvatoK8S - PROD - CPU usage on worker nodes {{host.name}} is {{value}}%"
  type = "query alert"
  query = <<EOT
avg(last_15m):avg:system.cpu.system{cluster_name:arvatok8s-prod} by {host} > 90
EOT
  message = <<EOT
{{#is_alert}}
### CPU usage is HIGH on **{{host.name}}**  {{value}}% > {{threshold}}% (threshold)
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
### CPU usage is NORMAL on {{host.name}}  {{value}}%  <= {{threshold}}% (threshold)
PRIORITY=HIGH
{{/is_recovery}}

@slack-The_Orchard-ameba-alerts
@ameba.developers@sonymusic.com
EOT
  tags = ["team:ameba-team", "env:Arvatok8sProd", "application_family:aoma-core"]
  draft_status = "published"
  new_group_delay = 60
  notification_preset_name = "hide_all"
  on_missing_data = "show_and_notify_no_data"
  require_full_window = false
  monitor_thresholds {
    critical = 90
  }
}
resource "datadog_monitor" "arvatok8s_prod_daemonset_kube_app_name_not_running_on_all_no" {
  name = "ArvatoK8S - PROD - DaemonSet {{kube_app_name}} not running on all nodes {{cluster_name.name}}"
  type = "query alert"
  query = <<EOT
avg(last_5m):sum:kubernetes_state.daemonset.daemons_unavailable{kube_cluster_name:arvatok8s-prod} by {kube_app_name} >= 1
EOT
  message = <<EOT
<!--StartFragment-->

{{#is_alert}}

### DaemonSet {{kube_app_name}} not running on all nodes {{cluster_name.name}} -   Please check.

PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}

### DaemonSet {{kube_app_name}} running again on all nodes {{cluster_name.name}} -  Recovered

PRIORITY=HIGH
{{/is_recovery}}

@slack-The_Orchard-ameba-alerts
@ameba.developers@sonymusic.com

<!--EndFragment-->
EOT
  draft_status = "published"
  escalation_message = <<EOT
<!--StartFragment-->
### DaemonSet {{kube_app_name}} not running on all nodes {{cluster_name.name}} -   REPEAT ALARM

PRIORITY=HIGH

@slack-The_Orchard-ameba-alerts
@ameba.developers@sonymusic.com

<!--EndFragment-->
EOT
  tags = ["team:ameba-team", "application_family:aoma-core"]
  new_group_delay = 60
  on_missing_data = "show_and_notify_no_data"
  renotify_interval = 60
  renotify_statuses = ["alert", "no data"]
  require_full_window = false
  monitor_thresholds {
    critical = 1
  }
}
resource "datadog_monitor" "arvatok8s_prod_deployment_unavailable_on_namespace_kube_name" {
  name = "ArvatoK8s - PROD - Deployment unavailable on namespace {{kube_namespace.name}}, deployment {{deployment.name}}"
  type = "query alert"
  query = <<EOT
avg(last_15m):avg:kubernetes_state.deployment.replicas_unavailable{kube_cluster_name:arvatok8s-prod, kube_namespace:aoma-prod} by {kube_namespace,kube_deployment} >= 1
EOT
  message = <<EOT
{{#is_alert}}
### Deployment unavailable on namespace {{kube_namespace.name}}, deployment {{deployment.name}}   {{value}} > {{threshold}} (threshold)
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
### Deployment unavailable on namespace {{kube_namespace.name}}, deployment {{deployment.name}} >  {{threshold}} (threshold)
PRIORITY=HIGH
{{/is_recovery}}

@slack-The_Orchard-ameba-alerts
@ameba.developers@sonymusic.com
EOT
  tags = ["team:ameba-team", "env:Arvatok8sProd", "application_family:aoma-core"]
  draft_status = "published"
  new_group_delay = 60
  notification_preset_name = "hide_all"
  on_missing_data = "show_and_notify_no_data"
  require_full_window = false
  monitor_thresholds {
    critical = 1
  }
}
resource "datadog_monitor" "arvatok8s_prod_ephemeral_storage_usage_is_very_high_on_pod_n" {
  name = "ArvatoK8s - PROD - Ephemeral Storage Usage is very high on {{pod_name.name}}, namespace {{kube_namespace.name}}"
  type = "query alert"
  query = <<EOT
avg(last_5m):(avg:kubernetes.ephemeral_storage.usage{kube_cluster_name:arvato-prod, kube_namespace:aoma-prod} by {pod_name} / avg:kubernetes.ephemeral_storage.limits{kube_cluster_name:arvato-prod, kube_namespace:aoma-prod} by {pod_name}) * 100 > 95
EOT
  message = <<EOT
{{#is_alert}}

### Ephemeral storage usage is very high on {{pod_name.name}}, namespace {{kube_namespace.name}} {{value}} > {{threshold}} (threshold)

PRIORITY=HIGH {{/is_alert}}

{{#is_recovery}}

### Ephemeral storage is very high on {{pod_name.name}}, namespace {{kube_namespace.name}} {{value}} > {{threshold}} (threshold)

PRIORITY=HIGH {{/is_recovery}}

@oncall-ameba-team
@slack-The_Orchard-ameba-alerts
@ameba.developers@sonymusic.com
EOT
  tags = ["team:ameba-team", "env:Arvatok8sProd", "application_family:aoma-core"]
  draft_status = "published"
  new_group_delay = 60
  notification_preset_name = "hide_all"
  on_missing_data = "resolve"
  require_full_window = false
  monitor_thresholds {
    critical = 95
  }
}
resource "datadog_monitor" "arvatok8s_prod_memory_usage_is_very_high_on_pod_name_namespa" {
  name = "ArvatoK8s - PROD - Memory usage is very high on {{pod_name}}, namespace {{kube_namespace.name}}"
  type = "query alert"
  query = <<EOT
avg(last_5m):((avg:kubernetes.memory.usage{kube_cluster_name:arvatok8s-prod, kube_namespace:aoma-prod, !kube_container_name:amp* , !kube_container_name:amino-service , ! kube_container_name:aoma-ui-proxy} by {pod_name} - avg:kubernetes.memory.cache{kube_cluster_name:arvatok8s-prod, kube_namespace:aoma-prod , !kube_container_name:amp* , !kube_container_name:amino-service , !kube_container_name:aoma-ui-proxy} by {pod_name}) / avg:kubernetes.memory.limits{kube_cluster_name:arvatok8s-prod, kube_namespace:aoma-prod , !kube_container_name:amp* , !kube_container_name:amino-service , !kube_container_name:aoma-ui-proxy} by {pod_name}) * 100 > 98
EOT
  message = <<EOT
{{#is_alert}}
### Memory usage is very high on {{pod_name}}, namespace {{kube_namespace.name}} {{value}} > {{threshold}} (threshold)
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
### Memory usage is very high on {{pod_name}}, namespace {{kube_namespace.name}} {{value}} > {{threshold}} (threshold)
PRIORITY=HIGH
{{/is_recovery}}

@slack-The_Orchard-ameba-alerts
@ameba.developers@sonymusic.com
EOT
  tags = ["team:ameba-team", "env:Arvatok8sProd", "application_family:aoma-core"]
  draft_status = "published"
  new_group_delay = 60
  notification_preset_name = "hide_all"
  on_missing_data = "default"
  require_full_window = false
  monitor_thresholds {
    critical = 98
    warning = 95
  }
}
resource "datadog_monitor" "arvatok8s_prod_node_encountered_freediskspacefailed_event" {
  name = "ArvatoK8s - PROD - Node encountered FreeDiskSpaceFailed event"
  type = "event-v2 alert"
  query = <<EOT
events("FreeDiskSpaceFailed host:*avarto-prod").rollup("count").last("5m") > 1
EOT
  message = <<EOT
{{#is_alert}}
  {{event.text}}
{{/is_alert}}

@oncall-ameba-team
@slack-The_Orchard-ameba-alerts
@ameba.developers@sonymusic.com
EOT
  tags = ["team:ameba-team", "env:Arvatok8sProd", "application_family:aoma-core"]
  draft_status = "published"
  include_tags = false
  notification_preset_name = "hide_all"
  on_missing_data = "default"
  require_full_window = false
  monitor_thresholds {
    critical = 1
  }
}
resource "datadog_monitor" "arvatok8s_prod_persistentvolume_usage_is_high" {
  name = "ArvatoK8s - PROD - PersistentVolume usage is high"
  type = "query alert"
  query = <<EOT
avg(last_5m):(sum:kubernetes.kubelet.volume.stats.used_bytes{persistentvolumeclaim:pvc-amp-*} * 100) / sum:kubernetes_state.persistentvolume.capacity{*} > 90
EOT
  message = <<EOT
{{#is_alert}}

### Persistent Volume usage is HIGH on ArvatoK8s node **{{host.name}}** {{value}}% < {{threshold}}% (threshold)

PRIORITY=HIGH {{/is_alert}}

@slack-The_Orchard-ameba-alerts
@ameba.developers@sonymusic.com

{{#is_recovery}}

### Persistent Volume usage is HIGH on ArvatoK8s node **{{host.name}}** {{value}}% < {{threshold}}% (threshold)

PRIORITY=HIGH {{/is_recovery}}
EOT
  tags = ["team:ameba-team", "env:Arvatok8sProd", "application_family:aoma-core"]
  draft_status = "published"
  include_tags = false
  notification_preset_name = "hide_all"
  on_missing_data = "default"
  require_full_window = false
  monitor_thresholds {
    critical = 90
    warning = 80
  }
}
resource "datadog_monitor" "arvatok8s_prod_kubernetes_pod_pod_namename_is_crashloopbacko" {
  name = "ArvatoK8S - PROD - [kubernetes] Pod {{pod_name.name}} is CrashloopBackOff on namespace {{kube_namespace.name}}"
  type = "query alert"
  query = <<EOT
max(last_15m):default_zero(max:kubernetes_state.container.status_report.count.waiting{reason:crashloopbackoff, kube_cluster_name:arvato-prod, kube_namespace:aoma-prod, ! deployment:amp-search} by {kube_namespace,pod_name}) >= 1
EOT
  message = <<EOT
Pod {{pod_name.name}} on namespace {{kube_namespace.name}} is in CrashloopBackOff on {{kube_namespace.name}}
 This alert could generate several alerts for a bad deployment. Adjust the thresholds of the query to suit your infrastructure.


@slack-The_Orchard-ameba-alerts
@ameba.developers@sonymusic.com
EOT
  tags = ["team:ameba-team", "env:Arvatok8sProd", "application_family:aoma-core"]
  draft_status = "published"
  escalation_message = <<EOT
EOT
  new_group_delay = 60
  notification_preset_name = "hide_all"
  notify_audit = true
  on_missing_data = "default"
  require_full_window = false
  monitor_thresholds {
    critical = 1
  }
}
resource "datadog_monitor" "arvatok8s_prod_pod_status_on_namespace_kube_namespacename" {
  name = "ArvatoK8S - PROD - Pod Status on namespace - {{kube_namespace.name}}"
  type = "query alert"
  query = <<EOT
avg(last_15m):max:kubernetes_state.pod.count{cluster_name:arvatok8s-prod , kube_namespace:aoma-prod} by {cluster_name,kube_namespace} - max:kubernetes.pods.running{cluster_name:arvatok8s-prod, kube_namespace:aoma-prod} by {cluster_name,kube_namespace} >= 1
EOT
  message = <<EOT
{{#is_alert}}
### {{value}} Pods NOT RUNNING on {{cluster_name.name}}. Please check.

PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
### All pods are running on {{cluster_name.name}}
PRIORITY=HIGH
{{/is_recovery}}

@slack-The_Orchard-ameba-alerts
@ameba.developers@sonymusic.com
EOT
  tags = ["team:ameba-team", "env:Arvatok8sProd", "application_family:aoma-core"]
  draft_status = "published"
  new_group_delay = 60
  notification_preset_name = "hide_all"
  on_missing_data = "default"
  require_full_window = false
  monitor_thresholds {
    critical = 1
  }
}
resource "datadog_monitor" "arvatok8s_prod_pvc_usage_is_high_on_hostname_at_value" {
  name = "ArvatoK8s - PROD - PVC usage is high on {{host.name}} at {{value}}%"
  type = "query alert"
  query = <<EOT
avg(last_5m):(max:kubernetes.kubelet.volume.stats.used_bytes{persistentvolumeclaim:pvc-amp-elasticsearch-*} * 100) / max:kubernetes.kubelet.volume.stats.capacity_bytes{persistentvolumeclaim:pvc-amp-elasticsearch-*} > 90
EOT
  message = <<EOT
{{#is_alert}}

### PVC usage is high on {{host.name}}  {{value}}% < {{threshold}}% (threshold)

PRIORITY=HIGH {{/is_alert}}

@slack-The_Orchard-ameba-alerts
@ameba.developers@sonymusic.com

{{#is_recovery}}

### PVC usage is high on {{host.name}}  {{value}}% < {{threshold}}% (threshold)

PRIORITY=HIGH {{/is_recovery}}
EOT
  tags = ["team:ameba-team", "env:Arvatok8sProd", "application_family:aoma-core"]
  draft_status = "published"
  include_tags = false
  notification_preset_name = "hide_all"
  on_missing_data = "default"
  require_full_window = false
  monitor_thresholds {
    critical = 90
    warning = 80
  }
}
resource "datadog_monitor" "arvatok8s_prod_worker_node_usable_memory_on_node_hostname_is" {
  name = "ArvatoK8s - PROD - worker node Usable memory on node {{host.name}} is {{value}}%"
  type = "query alert"
  query = <<EOT
avg(last_15m):avg:system.mem.pct_usable{cluster_name:arvatok8s-prod} by {host} < 0.1
EOT
  message = <<EOT
{{#is_alert}}
### Usable memory is LOW on ArvatoK8s node **{{host.name}}**  {{value}}% < {{threshold}}% (threshold)
PRIORITY=HIGH
{{/is_alert}}

@slack-The_Orchard-ameba-alerts
@ameba.developers@sonymusic.com

{{#is_recovery}}
### Usable memory is LOW on ArvatoK8s node {{host.name}}  {{value}}%  < {{threshold}}% (threshold)
PRIORITY=HIGH
{{/is_recovery}}
EOT
  tags = ["team:ameba-team", "env:Arvatok8sProd", "application_family:aoma-core"]
  draft_status = "published"
  new_group_delay = 60
  notification_preset_name = "hide_all"
  on_missing_data = "show_and_notify_no_data"
  require_full_window = false
  monitor_thresholds {
    critical = 0.1
  }
}
resource "datadog_monitor" "arvato_prod_disk_pressure_on_worker_node_on_kube_nodename" {
  name = "Arvato-Prod - Disk pressure on worker node on {{kube_node.name}}"
  type = "query alert"
  query = <<EOT
max(last_5m):avg:system.disk.free{cluster_name:arvato-prod , !device_name:tmpfs, ! device_name:shm} by {kube_node,device_name} < 10000000000
EOT
  message = <<EOT
{{#is_alert}}
###Number of EKS worker nodes available on {{cluster_name.name}} is {{value}} < {{threshold}}
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
###Number of EKS worker nodes available on {{cluster_name.name}} is {{value}} > {{threshold}}
PRIORITY=HIGH
{{/is_recovery}}

@slack-The_Orchard-ameba-alerts
@ameba.developers@sonymusic.com
EOT
  tags = ["team:ameba-team", "application_family:aoma-core"]
  draft_status = "published"
  escalation_message = <<EOT
EOT
  new_group_delay = 0
  on_missing_data = "show_and_notify_no_data"
  monitor_thresholds {
    critical = 10000000000
    warning = 15000009999
  }
}