locals {
  cpu_usage_documentation_content = <<-EOT
    Bigtable CPU usage is above the threshold.

    Please consider one of the following actions:
      - Add nodes to cluster;
      - Reduce batch job write load;

EOT

  latency_documentation_content = <<-EOT
    Bigtable read latency is above the threshold.

    Please consider one of the following actions:
      - Reduce batch job write load;
      - Check currently running user initiated queries;

EOT
}

resource "google_monitoring_notification_channel" "pubsub_to_datadog_channel" {
  display_name = "${local.name_prefix}-datadog"
  type         = "pubsub"
  labels = {
    "topic" = google_pubsub_topic.stackdriver_notifications_topic.id
  }
}

data "google_monitoring_notification_channel" "slack_channel" {
  display_name = "Sonymusic-PDE"
}

/* resource "google_monitoring_alert_policy" "cluster_cpu_usage" {
  combiner     = "OR"
  display_name = "${local.name_prefix}-bigtable-cluster-cpu-usage"

  conditions {
    display_name = "Cloud Bigtable Cluster - CPU load"
    condition_threshold {
      comparison      = "COMPARISON_GT"
      duration        = "600s"
      filter          = "resource.type = \"bigtable_cluster\" AND (resource.labels.cluster = \"delphi-ca-ssd-${local.region_zones[0]}\" AND resource.labels.instance = \"${google_bigtable_instance.ca_instance_ssd.name}\" AND resource.labels.project_id = \"${local.project_id}\") AND metric.type = \"bigtable.googleapis.com/cluster/cpu_load\""
      threshold_value = 0.7

      aggregations {
        alignment_period   = "300s"
        per_series_aligner = "ALIGN_MAX"
      }
    }
  }

  notification_channels = [
    data.google_monitoring_notification_channel.slack_channel.name,
  ]

  documentation {
    content   = local.cpu_usage_documentation_content
    mime_type = "text/markdown"
  }
} */

/* resource "google_monitoring_alert_policy" "cluster_hottest_node_cpu_usage" {
  combiner     = "OR"
  display_name = "${local.name_prefix}-bigtable-cluster-hottest-node-cpu-usage"

  conditions {
    display_name = "Cloud Bigtable Cluster - CPU load (hottest node)"
    condition_threshold {
      comparison      = "COMPARISON_GT"
      duration        = "600s"
      filter          = "resource.type = \"bigtable_cluster\" AND (resource.labels.cluster = \"delphi-ca-ssd-${local.region_zones[0]}\" AND resource.labels.instance = \"${google_bigtable_instance.ca_instance_ssd.name}\" AND resource.labels.project_id = \"${local.project_id}\") AND metric.type = \"bigtable.googleapis.com/cluster/cpu_load_hottest_node\""
      threshold_value = 0.9

      aggregations {
        alignment_period   = "300s"
        per_series_aligner = "ALIGN_MAX"
      }
    }
  }

  notification_channels = [
    data.google_monitoring_notification_channel.slack_channel.name,
  ]

  documentation {
    content   = local.cpu_usage_documentation_content
    mime_type = "text/markdown"
  }
} */

// alert if 99th percentile latency was above 5000ms for the past 5 minutes
resource "google_monitoring_alert_policy" "cluster_read_latency" {
  combiner     = "OR"
  display_name = "${local.name_prefix}-bigtable-cluster-read-latency"

  alert_strategy {
    auto_close = "3600s"
  }

  conditions {
    display_name = "Cloud Bigtable Cluster - read latency"
    condition_threshold {
      comparison      = "COMPARISON_GT"
      duration        = "0s"
      filter          = "resource.type = \"bigtable_table\" AND (resource.labels.cluster = \"delphi-ca-ssd-us-east4-a\" AND resource.labels.instance = \"delphi-consumer-analytics-ssd\" AND resource.labels.project_id = \"delphi-prod\") AND metric.type = \"bigtable.googleapis.com/server/latencies\""
      threshold_value = 45000

      aggregations {
        alignment_period     = "300s"
        per_series_aligner   = "ALIGN_PERCENTILE_50"
        cross_series_reducer = "REDUCE_MAX"
      }

      trigger {
        count   = 0
        percent = 100
      }
    }
  }

  notification_channels = [
    google_monitoring_notification_channel.pubsub_to_datadog_channel.name,
    data.google_monitoring_notification_channel.slack_channel.name,
  ]

  documentation {
    content   = local.latency_documentation_content
    mime_type = "text/markdown"
  }
}