locals {
  delphi_api_tags = concat(
    local.common_tags,
    ["application_sub_family:delphi-api"]
  )
}

resource "datadog_monitor" "api_anomaly_detection_monitor" {
  evaluation_delay = 0

  include_tags = true

  message         = "Anomalous activity has been detected. ${var.slack_channels["api"]}"
  name            = "${var.environment}-${var.service_name} Anomaly detected on the delphi-api service"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "events(\"tags:(story_category:apm AND env:${var.environment} AND service:\"prod-delphi-api\") source:watchdog\").rollup(\"count\").by(\"datacenter,service,resource_name\").last(\"30m\") > 0"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_api_tags
  timeout_h            = 0
  type                 = "event-v2 alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "0"
  }
}

resource "datadog_monitor" "get_streams_high_average_latency_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
`get_/v3/streams` average latency is too high.
This may indicate high latency from Bigtable.

${var.slack_channels["api"]}
EOT

  name            = "${var.environment}-${var.service_name} Resource get_/v3/streams has a high average latency on env:${var.environment}"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "avg(last_10m):( sum:trace.servlet.request.duration{resource_name:get_/v3/streams,env:${var.environment},service:delphi-api}.rollup(sum).fill(zero) / sum:trace.servlet.request.hits{resource_name:get_/v3/streams,env:${var.environment},service:delphi-api}.rollup(sum).fill(zero) ) > 2"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_api_tags
  timeout_h            = 0
  type                 = "query alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "2"
    warning  = "1"
  }
}

resource "datadog_monitor" "get_streams_high_error_rate_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
`get_/v3/streams` error rate is too high.

${var.slack_channels["api"]}
EOT

  name            = "${var.environment}-${var.service_name} Resource get_/v3/streams has a high error rate on env:${var.environment}"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "sum(last_10m):( sum:trace.servlet.request.errors{resource_name:get_/v3/streams,env:${var.environment},service:delphi-api*}.as_count() / sum:trace.servlet.request.hits{resource_name:get_/v3/streams,env:${var.environment},service:delphi-api}.as_count() ) > 0.02"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_api_tags
  timeout_h            = 0
  type                 = "query alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "0.02"
    warning  = "0.01"
  }
}

resource "datadog_monitor" "get_streams_hhigh_p90_latency_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
`get_/v3/streams` 90th percentile latency is too high.
This may indicate high latency from Bigtable.

${var.slack_channels["api"]}
EOT

  name            = "${var.environment}-${var.service_name} Resource get_/v3/streams has a high p90 latency on env:${var.environment}"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "avg(last_10m):p90:trace.servlet.request{env:${var.environment},resource_name:get_/v3/streams,service:delphi-api} > 5"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_api_tags
  timeout_h            = 0
  type                 = "metric alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "5"
    warning  = "3"
  }
}

resource "datadog_monitor" "get_tracks_high_error_rate_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
`get_/v3/tracks` error rate is too high.

${var.slack_channels["api"]}
EOT

  name            = "${var.environment}-${var.service_name} Resource get_/v3/tracks has a high error rate on env:${var.environment}"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "sum(last_10m):( sum:trace.servlet.request.errors{resource_name:get_/v3/tracks,env:${var.environment},service:delphi-api}.as_count() / sum:trace.servlet.request.hits{resource_name:get_/v3/tracks,env:${var.environment},service:delphi-api*}.as_count() ) > 0.02"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_api_tags
  timeout_h            = 0
  type                 = "query alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "0.02"
    warning  = "0.01"
  }
}

resource "datadog_monitor" "delphi_api_high_average_latency_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
`delphi-api` average latency is above alert threshold.

${var.slack_channels["api"]}
EOT

  name            = "${var.environment}-${var.service_name} Service delphi-api has a high average latency on env:${var.environment}"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "avg(last_10m):( sum:trace.servlet.request.duration{env:${var.environment},service:delphi-api*,!resource_name:get_/health}.rollup(sum).fill(zero) / sum:trace.servlet.request.hits{env:${var.environment},service:delphi-api,!resource_name:get_/health}.rollup(sum).fill(zero) ) > 2"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_api_tags
  timeout_h            = 0
  type                 = "query alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "2"
    warning  = "1"
  }
}

resource "datadog_monitor" "delphi_api_error_rate_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
`delphi-api` error rate is above alert threshold.

{{#is_alert}}${var.slack_channels["outage"]}{{/is_alert}}

${var.slack_channels["api"]}
EOT

  name            = "${var.environment}-${var.service_name} Service delphi-api has a high error rate on env:${var.environment}"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "sum(last_15m):( sum:trace.servlet.request.errors{env:${var.environment},service:delphi-api*,!resource_name:get_/health}.as_count() / sum:trace.servlet.request.hits{env:${var.environment},service:delphi-api*,!resource_name:get_/health}.as_count() ) > 0.05"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_api_tags
  timeout_h            = 0
  type                 = "query alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "0.05"
    warning  = "0.03"
  }
}

resource "datadog_monitor" "delphi_api_error_rate_on_health_endpoint_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
`delphi-api` error rate is too high.

{{#is_alert}}${var.slack_channels["outage"]}{{/is_alert}}

${var.slack_channels["api"]}
EOT

  name            = "${var.environment}-${var.service_name} Service delphi-api has a high error rate on /health endpoint on env:${var.environment}"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "sum(last_10m):( sum:trace.servlet.request.errors{env:${var.environment},resource_name:get_/health,service:delphi-api*}.as_count() / sum:trace.servlet.request.hits{env:${var.environment},resource_name:get_/health,service:delphi-api}.as_count() ) > 0.1"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_api_tags
  timeout_h            = 0
  type                 = "query alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "0.1"
    warning  = "0.05"
  }
}

resource "datadog_monitor" "delphi_api_p90_latency_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
`delphi-api` 90th percentile latency is above alert threshold.


{{#is_alert}}${var.slack_channels["outage"]}{{/is_alert}}

${var.slack_channels["api"]}
EOT

  name            = "${var.environment}-${var.service_name} Service delphi-api has a high p90 latency on env:${var.environment}"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "avg(last_30m):p90:trace.servlet.request{env:${var.environment},service:delphi-api} > 20"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_api_tags
  timeout_h            = 0
  type                 = "query alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "20"
    warning  = "18"
  }
}

resource "datadog_monitor" "delphi_api_abnormal_change_in_apdex_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
`delphi-api` Apdex deviated too much from its usual value.

${var.slack_channels["api"]}
EOT

  name            = "${var.environment}-${var.service_name} Service delphi-api has an abnormal change in Apdex on env:${var.environment}"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "avg(last_10m):avg:trace.servlet.request.apdex.by.service{env:${var.environment},service:delphi-api} < 0.9"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_api_tags
  timeout_h            = 0
  type                 = "metric alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "0.9"
    warning  = "0.95"
  }
}

resource "datadog_monitor" "delphi_api_nginx_error_rate_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
Error rate is above alert threshold.

{{#is_alert}}${var.slack_channels["outage"]}{{/is_alert}}

${var.slack_channels["api"]}
EOT

  name            = "${var.environment}-${var.service_name} Service delphi-api-nginx has a high error rate on env:${var.environment}"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "sum(last_10m):( sum:trace.nginx.handle.errors{env:${var.environment},service:delphi-api-nginx}.as_count() / sum:trace.nginx.handle.hits{env:prod,service:delphi-api-nginx}.as_count() ) > 0.05"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_api_tags
  timeout_h            = 0
  type                 = "query alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "0.05"
    warning  = "0.03"
  }
}

resource "datadog_monitor" "postgresql_error_rate_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
`postgresql` error rate is too high.

${var.slack_channels["api"]}
EOT

  name            = "${var.environment}-${var.service_name} Service postgresql has a high error rate on env:${var.environment}"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 2
  query                = "sum(last_10m):(sum:trace.postgresql.query.errors{env:${var.environment},service:postgresql}.as_count() / sum:trace.postgresql.query.hits{env:prod,service:postgresql}.as_count()) > 0.2"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_api_tags
  timeout_h            = 0
  type                 = "query alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "0.2"
    warning  = "0.15"
  }
}

resource "datadog_monitor" "service_postgresql_has_a_high_p90_latency_on_env_prod" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
`postgresql` 90th percentile latency is too high.

{{#is_alert}}${var.slack_channels["outage"]}{{/is_alert}}

${var.slack_channels["api"]}
EOT

  name            = "${var.environment}-${var.service_name} Service postgresql has a high p90 latency on env:${var.environment}"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 3
  query                = "avg(last_10m):p90:trace.postgresql.query{env:${var.environment},service:postgresql} > 20"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_api_tags
  timeout_h            = 0
  type                 = "query alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "20"
    warning  = "18"
  }
}

resource "datadog_monitor" "postgresql_abnormal_change_in_throughput_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
`postgresql` throughput deviated too much from its usual value.

${var.slack_channels["api"]}
EOT

  name            = "${var.environment}-${var.service_name} Service postgresql has an abnormal change in throughput on env:${var.environment}"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 5
  query                = "avg(last_4h):anomalies(sum:trace.postgresql.query.hits{env:${var.environment},service:postgresql}, 'agile', 3, direction='both', alert_window='last_15m', interval=60, count_default_zero='true', seasonality='weekly') >= 1"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_api_tags
  timeout_h            = 0
  type                 = "query alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_threshold_windows {
    recovery_window = "last_15m"
    trigger_window  = "last_15m"
  }

  monitor_thresholds {
    critical          = "1"
    critical_recovery = "0"
  }
}

resource "datadog_monitor" "delphi_slzapi_average_latency_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
`prod-delphi-slzapi` average latency is too high.

${var.slack_channels["slz"]}
EOT

  name            = "${var.environment}-${var.service_name} Service prod-delphi-slzapi has a high average latency on env:none"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "avg(last_10m):( sum:trace.flask.request.duration{service:prod-delphi-slzapi}.rollup(sum).fill(zero) / sum:trace.flask.request.hits{service:prod-delphi-slzapi}.rollup(sum).fill(zero) ) > 0.5"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_api_tags
  timeout_h            = 0
  type                 = "query alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "0.5"
    warning  = "0.3"
  }
}

resource "datadog_monitor" "delphi_slzapi_error_rate_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
`prod-delphi-slzapi` error rate is too high.


${var.slack_channels["slz"]}
EOT

  name            = "${var.environment}-${var.service_name} Service prod-delphi-slzapi has a high error rate on env:none"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "sum(last_10m):( sum:trace.flask.request.errors{service:prod-delphi-slzapi}.as_count() / sum:trace.flask.request.hits{service:prod-delphi-slzapi}.as_count() ) > 0.05"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_api_tags
  timeout_h            = 0
  type                 = "query alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "0.05"
    warning  = "0.01"
  }
}

resource "datadog_monitor" "delphi_slzapi_p90_latency_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
`prod-delphi-slzapi` 90th percentile latency is too high.

${var.slack_channels["slz"]}
  EOT

  name            = "${var.environment}-${var.service_name} Service prod-delphi-slzapi has a high p90 latency on env:none"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "avg(last_10m):p90:trace.flask.request{service:prod-delphi-slzapi} > 1"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_api_tags
  timeout_h            = 0
  type                 = "metric alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "1"
    warning  = "0.8"
  }
}

resource "datadog_monitor" "delphi_slzapi_abnormal_change_in_throughput_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
`prod-delphi-slzapi` throughput deviated too much from its usual value.

${var.slack_channels["slz"]}
EOT

  name            = "${var.environment}-${var.service_name} Service prod-delphi-slzapi has an abnormal change in throughput on env:none"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "avg(last_4h):anomalies(sum:trace.flask.request.hits{service:prod-delphi-slzapi}, 'basic', 3, direction='both', alert_window='last_15m', interval=60, count_default_zero='true') >= 1"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_api_tags
  timeout_h            = 0
  type                 = "query alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_threshold_windows {
    recovery_window = "last_15m"
    trigger_window  = "last_15m"
  }

  monitor_thresholds {
    critical          = "1"
    critical_recovery = "0"
  }
}