resource "datadog_monitor" "arvatok8s_dev_amino_pod_status_aoma-dev" {
  name = "ArvatoK8S - Dev - AMINO Pod Status - aoma-dev"
  type = "query alert"
  query = <<EOT
avg(last_5m):sum:kubernetes_state.deployment.replicas_desired{kube_cluster_name:arvato-devstage,kube_namespace:aoma-dev, kube_deployment:amino-*} by {kube_deployment} - sum:kubernetes.pods.running{cluster_name:arvato-devstage ,kube_namespace:aoma-dev, kube_deployment:amino-*} by {kube_deployment} >= 1
EOT
  message = <<EOT
{{#is_alert}}
### {{value}} AMINO Pods NOT RUNNING on DEV/{{cluster_name.name}}. Please check.

PRIORITY=HIGH
{{/is_alert}}

@team-amp-developers

{{#is_recovery}}
### All AMINO pods are running on DEV/{{cluster_name.name}}.
PRIORITY=HIGH
{{/is_recovery}}
EOT
  tags = ["team:amp-developers", "env:Arvatok8sDev", "application_family:aoma-core"]
  draft_status = "published"
  new_group_delay = 0
  notification_preset_name = "hide_query"
  on_missing_data = "default"
  require_full_window = false
  monitor_thresholds {
    critical = 1
  }
}
resource "datadog_monitor" "arvatok8s_dev_deployment_unavailable_on_namespace_kube_names" {
  name = "ArvatoK8s - Dev - Deployment unavailable on namespace {{kube_namespace.name}}, deployment {{kube_deployment.name}} }}"
  type = "query alert"
  query = <<EOT
avg(last_15m):avg:kubernetes_state.deployment.replicas_unavailable{kube_cluster_name:arvato-devstage , kube_namespace:aoma-dev} by {kube_namespace,kube_deployment} >= 1
EOT
  message = <<EOT
{{#is_alert}}
### Deployment unavailable on namespace {{kube_namespace.name}}, deployment {{kube_deployment.name}}   {{value}} > {{threshold}} (threshold)
PRIORITY=HIGH
{{/is_alert}}

@slack-The_Orchard-ameba-alerts
@ameba.developers@sonymusic.com

{{#is_recovery}}
### Deployment unavailable on namespace {{kube_namespace.name}}, deployment {{kube_deployment.name}} >  {{threshold}} (threshold)
PRIORITY=HIGH
{{/is_recovery}}
EOT
  tags = ["team:ameba-team", "env:Arvatok8sDev", "application_family:aoma-core"]
  draft_status = "published"
  include_tags = false
  new_group_delay = 960
  notification_preset_name = "hide_all"
  on_missing_data = "show_and_notify_no_data"
  require_full_window = false
  monitor_thresholds {
    critical = 1
  }
}
resource "datadog_monitor" "arvatok8s_dev_desired_number_of_pods_on_namespace_for_deploy" {
  name = "ArvatoK8s - Dev - Desired number of Pods on {{namespace}} for {{deployment.name}} < {{threshold}} (threshold)"
  type = "query alert"
  query = <<EOT
sum(last_1h):avg:kubernetes_state.deployment.replicas_available{kube_namespace:aoma-dev, kube_cluster_name:arvato-devstage, ! kube_deployment:amp-search} by {kube_namespace,kube_deployment} < 1
EOT
  message = <<EOT
{{#is_alert}}
### Desired number of Pods on {{deployment.name}}  < {{threshold}} (threshold) for the past hour.
PRIORITY=HIGH
{{/is_alert}}

 @slack-The_Orchard-ameba-alerts
@ameba.developers@sonymusic.com

{{#is_recovery}}
### Desired number of Pods on {{deployment.name}}  > {{threshold}} (threshold)
PRIORITY=HIGH
{{/is_recovery}}
EOT
  tags = ["team:ameba-team", "env:Arvatok8sDev", "application_family:aoma-core"]
  draft_status = "published"
  evaluation_delay = 900
  include_tags = false
  new_group_delay = 60
  notification_preset_name = "hide_all"
  on_missing_data = "show_and_notify_no_data"
  require_full_window = false
  monitor_thresholds {
    critical = 1
  }
}
resource "datadog_monitor" "arvatok8s_dev_memory_usage_is_very_high_on_pod_name_namespac" {
  name = "ArvatoK8s - DEV - Memory usage is very high on {{pod_name}}, namespace {{kube_namespace.name}}"
  type = "query alert"
  query = <<EOT
avg(last_5m):((avg:kubernetes.memory.usage{kube_namespace:aoma-dev , !kube_container_name:amp* , !kube_container_name:amino-service , ! kube_container_name:aoma-ui-proxy} by {pod_name} - avg:kubernetes.memory.cache{kube_namespace:aoma-dev , !kube_container_name:amp* , !kube_container_name:amino-service , !kube_container_name:aoma-ui-proxy} by {pod_name}) / avg:kubernetes.memory.limits{kube_namespace:aoma-dev , !kube_container_name:amp* , !kube_container_name:amino-service , !kube_container_name:aoma-ui-proxy} by {pod_name}) * 100 > 98
EOT
  message = <<EOT
{{#is_alert}}
### Memory usage is very high on {{pod_name}}, namespace {{kube_namespace.name}} {{value}} > {{threshold}} (threshold)
PRIORITY=HIGH
{{/is_alert}}

{{#is_recovery}}
### Memory usage is very high on {{pod_name}}, namespace {{kube_namespace.name}} {{value}} > {{threshold}} (threshold)
PRIORITY=HIGH
{{/is_recovery}}

@slack-The_Orchard-ameba-alerts
@ameba.developers@sonymusic.com
EOT
  tags = ["team:ameba-team", "env:Arvatok8sDev", "application_family:aoma-core"]
  draft_status = "published"
  escalation_message = <<EOT
EOT
  new_group_delay = 60
  notification_preset_name = "hide_all"
  notify_by = ["*"]
  on_missing_data = "default"
  renotify_interval = 30
  renotify_statuses = ["alert", "no data"]
  require_full_window = false
  monitor_thresholds {
    critical = 98
    warning = 95
  }
}
resource "datadog_monitor" "arvatok8s_dev_monitor_Kubernetes_Deployments_replica_pods_de" {
  name = "ArvatoK8s - Dev - Monitor Kubernetes Deployments Replica Pods (Desired Pods - Available Pods)"
  type = "query alert"
  query = <<EOT
avg(last_30m):avg:kubernetes_state.deployment.replicas_desired{kube_cluster_name:arvato-devstage , kube_namespace:aoma-dev} by {kube_namespace,kube_deployment} - avg:kubernetes_state.deployment.replicas_available{kube_cluster_name:arvato-devstage , kube_namespace:aoma-dev} by {kube_namespace,kube_deployment} > 1
EOT
  message = <<EOT
### More than one Deployments Replica's pods are down in Deployment {{kube_namespace.name}}
@slack-The_Orchard-ameba-alerts
@ameba.developers@sonymusic.com
EOT
  tags = ["team:ameba-team", "env:Arvatok8sDev", "application_family:aoma-core"]
  draft_status = "published"
  new_group_delay = 60
  notification_preset_name = "hide_all"
  on_missing_data = "show_and_notify_no_data"
  require_full_window = false
  monitor_thresholds {
    critical = 1
  }
}
resource "datadog_monitor" "arvatok8s_dev_kubernetes_pod_pod_namename_is_crashloopbackof" {
  name = "ArvatoK8s - Dev - [kubernetes] Pod {{pod_name.name}} is CrashloopBackOff on namespace {{kube_namespace.name}}"
  type = "query alert"
  query = <<EOT
max(last_10m):default_zero(max:kubernetes_state.container.status_report.count.waiting{reason:crashloopbackoff, kube_cluster_name:zplq4ngvmw, kube_namespace:aoma-dev} by {kube_cluster_name,kube_namespace,pod_name}) >= 1
EOT
  message = <<EOT
Pod {{pod_name.name}} on namespace {{kube_namespace.name}} is in CrashloopBackOff on {{kube_namespace.name}}
 This alert could generate several alerts for a bad deployment. Adjust the thresholds of the query to suit your infrastructure.


@slack-The_Orchard-ameba-alerts
@ameba.developers@sonymusic.com
EOT
  tags = ["team:ameba-team", "integration:kubernetes", "env:Arvatok8sDev", "application_family:aoma-core"]
  draft_status = "published"
  escalation_message = <<EOT
EOT
  new_group_delay = 60
  notification_preset_name = "hide_all"
  notify_audit = true
  on_missing_data = "default"
  require_full_window = false
  monitor_thresholds {
    critical = 1
  }
}
resource "datadog_monitor" "arvatok8s_dev_pod_status_on_namespace_aoma-dev" {
  name = "ArvatoK8S - Dev - Pod Status on namespace - aoma-dev"
  type = "query alert"
  query = <<EOT
avg(last_15m):avg:kubernetes_state.pod.count{cluster_name:arvato-devstage ,kube_namespace:aoma-dev} by {cluster_name} - avg:kubernetes.pods.running{cluster_name:arvato-devstage , kube_namespace:aoma-dev} by {cluster_name} > 1
EOT
  message = <<EOT
{{#is_alert}}
### {{value}} Pods NOT RUNNING on {{cluster_name.name}}. Please check.

PRIORITY=HIGH
{{/is_alert}}

@slack-The_Orchard-ameba-alerts
@ameba.developers@sonymusic.com
@team-amp-developers

{{#is_recovery}}
### All pods are running on {{cluster_name.name}}
PRIORITY=HIGH
{{/is_recovery}}
EOT
  tags = ["team:ameba-team", "env:Arvatok8sDev", "application_family:aoma-core"]
  draft_status = "published"
  new_group_delay = 60
  notification_preset_name = "hide_all"
  on_missing_data = "default"
  require_full_window = false
  monitor_thresholds {
    critical = 1
  }
}