resource "datadog_monitor" "amp_elasticsearch_domain_status_is_not_green" {
  name = "AMP-P | ElasticSearch Domain Status is not Green"
  type = "query alert"
  query = <<EOT
avg(last_15m):sum:aws.es.cluster_statusgreen{domainname:amp-asset-srv-applogs-prod} by {domainname} < 1
EOT
  message = <<EOT
{{#is_alert}}
AMP ElasticSearch Cluster status is not Green PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
AMP ElasticSearch Cluster status is Green PRIORITY=LOW
{{/is_recovery}}

@pagerduty-AMP
@amp-microservices@sonymusic.com

EOT
  draft_status = "published"
  escalation_message = <<EOT
EOT
  tags = ["team:amp-team", "managed_by:amp-terraform"]
  new_group_delay = 0
  on_missing_data = "show_no_data"
  monitor_thresholds {
    critical = 1
  }
}
resource "datadog_monitor" "amp_elasticsearch_domain_amp_prod_asset_srv_applogs_cpu_usage_on" {
  name = "AMP-P | ElasticSearch domain (amp-asset-srv-applogs-prod) CPU usage on node {{nodeid.name}} is {{value}}%"
  type = "query alert"
  query = <<EOT
avg(last_10m):sum:aws.es.cpuutilization{name:amp-asset-srv-applogs-prod} by {nodeid} >= 90
EOT
  message = <<EOT

{{#is_alert}}
CPU usage on node {{nodeid.name}} {{value}} >= {{threshold}}
 PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
CPU usage on node {{nodeid.name}} {{value}} <= {{threshold}}
PRIORITY=LOW
{{/is_recovery}}

@pagerduty-AMP
@amp-microservices@sonymusic.com

EOT
  draft_status = "published"
  escalation_message = <<EOT
EOT
  tags = ["team:amp-team","managed_by:amp-terraform"]
  new_group_delay = 0
  on_missing_data = "show_no_data"
  monitor_thresholds {
    critical = 90
  }
}
resource "datadog_monitor" "amp_elasticsearch_domain_amp_prod_asset_srv_applogs_jvm_memory_o" {
  name = "AMP-P | ElasticSearch domain (amp-asset-srv-applogs-prod) JVM memory on node {{nodeid.name}} is {{value}}%"
  type = "query alert"
  query = <<EOT
avg(last_10m):sum:aws.es.jvmmemory_pressure{domainname:amp-asset-srv-applogs-prod} by {nodeid} > 80
EOT
  message = <<EOT

{{#is_alert}}
JVM Memory usage on node {{nodeid.name}} {{value}} > {{threshold}}
 PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
JVM Memory usage on node {{nodeid.name}} {{value}} <= {{threshold}}
 PRIORITY=LOW
{{/is_recovery}}

@pagerduty-AMP
@amp-microservices@sonymusic.com

EOT
  draft_status = "published"
  escalation_message = <<EOT
EOT
  tags = ["team:amp-team","managed_by:amp-terraform"]
  new_group_delay = 0
  on_missing_data = "show_no_data"
  monitor_thresholds {
    critical = 80
  }
}
resource "datadog_monitor" "amp_elasticsearch_domain_amp_prod_asset_srv_applogs_free_storage" {
  name = "AMP-P | ElasticSearch domain (amp-asset-srv-applogs-prod) free storage space on node {{nodeid.name}} is {{value}}"
  type = "query alert"
  query = <<EOT
avg(last_10m):sum:aws.es.free_storage_space{domainname:amp-asset-srv-applogs-prod} by {nodeid} <= 51200
EOT
  message = <<EOT

{{#is_alert}}
Free storage space on node {{nodeid.name}} is <= {{threshold}}
PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}}
Free storage space on node {{nodeid.name}} is >= {{threshold}}
PRIORITY=LOW
{{/is_recovery}}

@pagerduty-AMP
@amp-microservices@sonymusic.com

EOT
  draft_status = "published"
  escalation_message = <<EOT
EOT
  tags = ["team:amp-team","managed_by:amp-terraform"]
  new_group_delay = 0
  on_missing_data = "show_no_data"
  monitor_thresholds {
    critical = 51200
  }
}
resource "datadog_monitor" "amp_elasticsearch_domain_unhealthy_nodes_in_domainnamename" {
  name = "AMP-P | ElasticSearch domain unhealthy nodes in {{domainname.name}}"
  type = "query alert"
  query = <<EOT
avg(last_10m):sum:aws.es.nodes{domainname:amp-asset-srv-applogs-prod} by {domainname} < 1
EOT
  message = <<EOT

{{#is_alert}}
Healthy nodes in {{domainname.name}} is < {{threshold}}
 PRIORITY=LOW
{{/is_alert}}

{{#is_recovery}} Healthy nodes in {{domainname.name}} is >= {{threshold}}
PRIORITY=LOW
{{/is_recovery}}

@pagerduty-AMP
@amp-microservices@sonymusic.com

EOT
  draft_status = "published"
  escalation_message = <<EOT
EOT
  tags = ["team:amp-team","managed_by:amp-terraform"]
  new_group_delay = 0
  on_missing_data = "show_no_data"
  monitor_thresholds {
    critical = 1
  }
}