locals {
  delphi_gcp_tags = concat(
    local.common_tags,
    ["cloud:gcp"]
  )
}

resource "datadog_monitor" "bigtable_storage_usage_monitor" {
  name    = "${var.environment}-${var.service_name} bigtable cluster storage space usage at {{value}} on {{cluster.name}}. {{#is_alert}}Threshold exceeded: Set at {{threshold}}.{{/is_alert}}{{#is_warning}}Warning threshold: Set at {{warn_threshold}}.{{/is_warning}}"
  type    = "metric alert"
  message = <<EOT
{{^is_recovery}}
Storage usage at `{{value}}` which exceeds `{{threshold}}` on `{{cluster.name}}`

Please consider one of the following actions:
- Add nodes to cluster
{{/is_recovery}}

{{#is_recovery}}
Storage usage on `{{cluster.name}}` is below the threshold.
{{/is_recovery}}

${var.slack_channels["infra"]} ${var.slack_channels["api"]}
EOT

  query = "avg(last_1h):avg:gcp.bigtable.cluster.storage_utilization{project_id:${var.project_id}} by {cluster} > 0.9"

  monitor_thresholds {
    warning  = 0.8
    critical = 0.9
  }

  evaluation_delay = 300 // minimum recommended value for GCP metrics 300

  notify_no_data    = false
  renotify_interval = 300
  restricted_roles  = [data.datadog_role.datadog_admin_role.id]

  notify_audit = false
  timeout_h    = 0
  include_tags = true

  tags = local.delphi_gcp_tags
}

resource "datadog_monitor" "bigtable_read_latency_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
Read latency was above 45 second for the last 5 minutes

{{^is_recovery}}
Please consider one of the following actions:
Add nodes to cluster
Reduce batch job write load
Stop any user initiated queries
{{/is_recovery}}

${var.slack_channels["infra"]} ${var.slack_channels["api"]}
EOT

  name            = "${var.environment}-${var.service_name} read latency is high"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "events(\"tags:\"service:bigtable\" source:google_cloud_bigtable status:error\").rollup(\"count\").last(\"1m\") >= 1"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_gcp_tags
  timeout_h            = 0
  type                 = "event-v2 alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "1"
  }
}