# these are not in datadog-collector but in snowflake web integration > custom metrics.
# this is just to keep all the monitors in same place as old custom metric ones.

resource "datadog_monitor" "snowflake_custom_query_lsr_isrc_case_mismatch" {
  name    = "LabelSoundRecordings with ISRC mismatch has increased"
  type    = "query alert"
  message = "There are #{{value}} LabelSoundRecordings that have ISRCs that are not in uppercase.\nNotify: @slack-kafka-data-highway-alerts"
  query   = "avg(last_5m):avg:snowflake.snowflake.custom.ar_to_neo_sync.lsr_isrc_case_mismatch{*} > 10"

  monitor_thresholds {
    ok       = 0
    warning  = 1
    critical = 10
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  timeout_h           = 2
  require_full_window = false

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:kafka-data-highway"
  ]
}

resource "datadog_monitor" "snowflake_custom_query_track_missing_lsr" {
  name    = "Tracks missing LabelSoundRecordings has increased"
  type    = "query alert"
  message = "There are #{{value}} Tracks that are missing LabelSoundRecordings.\nNotify: @slack-kafka-data-highway-alerts"
  query   = "avg(last_5m):avg:snowflake.snowflake.custom.ar_to_neo_sync.track_missing_lsr{*} > 14290"

  monitor_thresholds {
    warning  = 14280 # current query count is 14279
    critical = 14290
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  timeout_h           = 2
  require_full_window = false

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:kafka-data-highway"
  ]
}

resource "datadog_monitor" "snowflake_custom_query_track_dupe_lsr" {
  name    = "Tracks with multiple LabelSoundRecordings has increased"
  type    = "query alert"
  message = "There are #{{value}} Tracks have multiple LabelSoundRecordings.\nNotify: @slack-kafka-data-highway-alerts"
  query   = "avg(last_5m):avg:snowflake.snowflake.custom.ar_to_neo_sync.track_dupe_lsr{*} > 2085"

  monitor_thresholds {
    warning  = 2075 # current query count is 2074
    critical = 2085
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  timeout_h           = 2
  require_full_window = false

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:kafka-data-highway"
  ]
}

resource "datadog_monitor" "snowflake_custom_query_gsr_isrc_case_mismatch" {
  name    = "GlobalSoundRecordings with ISRC mismatch has increased"
  type    = "query alert"
  message = "There are #{{value}} GlobalSoundRecordings that have ISRCs that are not in uppercase.\nNotify: @slack-kafka-data-highway-alerts"
  query   = "avg(last_5m):avg:snowflake.snowflake.custom.ar_to_neo_sync.gsr_isrc_case_mismatch{*} > 37"

  monitor_thresholds {
    warning  = 27 # current query count is 26
    critical = 37
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  timeout_h           = 2
  require_full_window = false

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:kafka-data-highway"
  ]
}

resource "datadog_monitor" "snowflake_custom_query_product_on_two_projects" {
  name    = "Products with two or more projects has increased"
  type    = "query alert"
  message = "There are #{{value}} Products that have two or more projects linked to it.\nNotify: @slack-kafka-data-highway-alerts"
  query   = "avg(last_5m):avg:snowflake.snowflake.custom.ar_to_neo_sync.product_on_two_projects{*} > 10"

  monitor_thresholds {
    warning  = 5 # current query count is 0
    critical = 10
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  timeout_h           = 2
  require_full_window = false

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:kafka-data-highway"
  ]
}

resource "datadog_monitor" "snowflake_custom_query_product_on_no_projects" {
  name    = "Products with no projects has increased"
  type    = "query alert"
  message = "There are #{{value}} Products that have no projects linked to it.\nNotify: @slack-kafka-data-highway-alerts"
  query   = "avg(last_5m):avg:snowflake.snowflake.custom.ar_to_neo_sync.product_on_no_projects{*} > 16"

  monitor_thresholds {
    warning  = 8 # current query count fluctuates between 0-4
    critical = 16
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  timeout_h           = 2
  require_full_window = false

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:kafka-data-highway"
  ]
}


# Anomaly Monitor for Art Relations to Neo4j Sync metrics from dashboard.

resource "datadog_monitor" "anomaly_monitor" {
  for_each = local.all_dashboard_metrics

  name               = "Anomaly Monitor for ${each.key} AR to Neo4j Sync"
  type               = "query alert"
  message            = "There has been an {{value}} increase of ${each.key} between AR and Neo4j. \nIt can be because of Kinesis lag or backfills or some issue processing the messages by lambda. \nNotify: ${each.value.notification_endpoints}"
  escalation_message = "Escalation to ${var.escalation_notification_endpoints}"

  # 1 deviations from the predicted data using the agile algorithm. direction to monitor only increase in numbers.
  # Use a rollup interval of 600 seconds but alert window is 1h since we only get metrics data every hour.
  query = "avg(last_2d):anomalies(${each.value.metrics}, 'agile', 1, direction='above', interval=600, alert_window='last_1h', count_default_zero='true', seasonality='daily') >= ${each.value.critical}"

  monitor_thresholds {
    warning  = each.value.warning
    critical = each.value.critical
  }

  on_missing_data     = "show_no_data"
  include_tags        = true
  notify_audit        = false
  require_full_window = false
  renotify_interval   = 120

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:kafka-data-highway"
  ]
}

resource "datadog_monitor" "snowflake_custom_query_label_participant_with_different_vendors" {
  name    = "LabelParticipant with different vendors (direct vendor and subaccount vendor)"
  type    = "query alert"
  message = "There are #{{value}} LabelParticipants with different vendors (direct and subaccount vendor, previously 1226).\nNotify: @slack-kafka-data-highway-alerts"
  query   = "avg(last_5m):avg:snowflake.custom.ar_to_neo_sync.label_participant_with_different_vendors{*} > 1226"

  monitor_thresholds {
    critical = 1226
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  timeout_h           = 2
  require_full_window = false

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:kafka-data-highway"
  ]
}