resource "datadog_monitor" "es_cluster_status_monitor" {
  name    = "${var.environment}-${var.service_name} status has changed to red for the cluster: {{domainname.name}}"
  type    = "metric alert"
  message = <<EOF
Cluster status has changed to RED for domain: `{{domainname.name}}`

Your Elasticsearch cluster can enter red status for the following reasons:

- Multiple data node failures;
- Using a corrupt or red shard for an index;
- High JVM memory pressure or CPU utilization;
- Low disk space or disk skew;

Check out [this article](https://aws.amazon.com/premiumsupport/knowledge-center/elasticsearch-red-yellow-status/) for troubleshooting instructions.

${var.slack_channel}
EOF

  query = "max(last_1h):max:aws.es.cluster_statusred{alias:${var.account_alias}} by {domainname} > 0"

  monitor_thresholds {
    critical = 0
  }

  evaluation_delay = 1800 // minimum recommended value for AWS metrics 900

  notify_no_data    = false
  renotify_interval = 300
  restricted_roles  = [data.datadog_role.datadog_admin_role.id]

  notify_audit = false
  timeout_h    = 0
  include_tags = true


  tags = local.common_tags
}

resource "datadog_monitor" "es_free_storage_space_monitor" {
  name    = "${var.environment}-${var.service_name} domain {{domainname.name}} free storage space at {{value}}. {{#is_alert}}Threshold exceeded: Set at {{threshold}}%.{{/is_alert}}{{#is_warning}}Warning threshold: Set at {{warn_threshold}}%.{{/is_warning}}"
  type    = "metric alert"
  message = <<EOF
{{#is_warning}}Warning: Free storage space for {{domainname.name}} is approaching limits. Consider optimizing or scaling resources.{{/is_warning}}
{{#is_alert}}Critical Alter: Free storage space for {{domainname.name}} is too low. Consider optimizing or scaling resources.{{/is_alert}}
{{#is_recovery}}Recovery: Free storage space for {{domainname.name}} is back to normal. No further actions are required.{{/is_recovery}}

{{^is_recovery}}
Troubleshooting:

- Add more space;
- Remove unused indexes if any;
{{/is_recovery}}

${var.slack_channel}
EOF

  query = "avg(last_1h):avg:aws.es.free_storage_space{alias:${var.account_alias}} by {domainname} < 2560"

  monitor_thresholds {
    critical = 2560
    warning  = 3072
  }

  evaluation_delay = 1800 // minimum recommended value for AWS metrics 900

  notify_no_data    = false
  renotify_interval = 300
  restricted_roles  = [data.datadog_role.datadog_admin_role.id]

  notify_audit = false
  timeout_h    = 0
  include_tags = true

  tags = local.common_tags
}

resource "datadog_monitor" "es_cpu_utilization_monitor" {
  name    = "${var.environment}-${var.service_name} domain {{domainname.name}} CPU utilization at {{value}}%. {{#is_alert}}Threshold exceeded: Set at {{threshold}}%.{{/is_alert}}{{#is_warning}}Warning threshold: Set at {{warn_threshold}}%.{{/is_warning}}"
  type    = "metric alert"
  message = <<EOF
{{#is_warning}}Warning: CPU Utilization for {{domainname.name}} is approaching limits. Consider optimizing or scaling resources.{{/is_warning}}
{{#is_alert}}Critical Alter: CPU Utilization for {{domainname.name}} is too high. Consider optimizing or scaling resources.{{/is_alert}}
{{#is_recovery}}Recovery: CPU Utilization for {{domainname.name}} is back to normal. No further actions are required.{{/is_recovery}}

{{^is_recovery}}
Troubleshooting:

- Upgrade instance tier;
- Add more instances;
- Reduce data ingestion if possible;
{{/is_recovery}}

${var.slack_channel}
EOF

  query = "max(last_1h):max:aws.es.cpuutilization{alias:${var.account_alias}} by {domainname} > 90"

  monitor_thresholds {
    critical = 90
    warning  = 80
  }

  evaluation_delay = 1800 // minimum recommended value for AWS metrics 900

  notify_no_data    = false
  renotify_interval = 300
  restricted_roles  = [data.datadog_role.datadog_admin_role.id]

  notify_audit = false
  timeout_h    = 0
  include_tags = true

  tags = local.common_tags
}

resource "datadog_monitor" "es_jvm_memory_pressure_monitor" {
  name    = "${var.environment}-${var.service_name} domain {{domainname.name}} JVM memory pressure at {{value}}%. {{#is_alert}}Threshold exceeded: Set at {{threshold}}%.{{/is_alert}}{{#is_warning}}Warning threshold: Set at {{warn_threshold}}%.{{/is_warning}}"
  type    = "metric alert"
  message = <<EOF
{{#is_alert}}Critical Alter: JVM memory pressure for {{domainname.name}} is too high. Consider optimizing or scaling resources.{{/is_alert}}
{{#is_warning}}Warning: JVM memory pressure for {{domainname.name}} is approaching limits. Consider optimizing or scaling resources.{{/is_warning}}
{{#is_recovery}}Recovery: JVM memory pressure for {{domainname.name}} is back to normal. No further actions are required.{{/is_recovery}}

{{^is_recovery}}
Troubleshooting:

- Upgrade instance tier;
- Reduce data ingestion if possible;
{{/is_recovery}}

${var.slack_channel}
EOF

  query = "max(last_1h):max:aws.es.jvmmemory_pressure{alias:${var.account_alias}} by {domainname} > 90"

  monitor_thresholds {
    critical = 90
    warning  = 80
  }

  evaluation_delay = 1800 // minimum recommended value for AWS metrics 900

  notify_no_data    = false
  renotify_interval = 300
  restricted_roles  = [data.datadog_role.datadog_admin_role.id]

  notify_audit = false
  timeout_h    = 0
  include_tags = true

  tags = local.common_tags
}