/*Metrics and Alarms:

Apple:
1 ScraperAppleRequests
period: 15 mins
min: 15k
max: 60k

2 ScraperAppleRateLimitErrors
period: 15 mins
max: 600

3 ScraperApplePlaylistProcessed
period: 6 hour
min: 1000

4 ScraperApplePlaylistProcessed, TransformServiceInputApplePlaylists
period: 6 hours
relation = TransformServiceInputApplePlaylists / ScraperApplePlaylistProcessed
threshold: [0.85, 1.5]

Spotify:
5 ScraperSpotifyPlaylistProcessed
period: 6 hour
min: 12000
max: 15000

6 ScraperSpotifyPlaylistProcessed, TransformServiceInputSpotifyPlaylists
period: 6 hours
relation = TransformServiceInputSpotifyPlaylists / ScraperSpotifyPlaylistProcessed
threshold= [0.85, 1.5]

All:
7 GetRecords.IteratorAgeMilliseconds
period = 15 min
max: 600k (this is 10 mins)
consecutive data points count: 3
Need it for all 8 metrics, both for Apple and Spotify
*/

resource "datadog_monitor" "ScraperAppleRequests_high" {
  name = "DAPD monitor: Scraper Apple requests too high"
  type = "query alert"

  message = <<EOT
{{#is_alert}}
${local.alert_message_env_title} amount of ScraperAppleRequests is > 80000 or  for the last 20 minutes.
{{/is_alert}}

{{#is_recovery}}
${local.alert_message_env_title} amount of ScraperAppleRequest is back to normal values.
{{/is_recovery}}
${local.alert_channels}

EOT

  # The values should be between 20000 and 80000
  # Math explained https://docs.datadoghq.com/monitors/guide/monitor-for-value-within-a-range/
  # query = "sum(last_20m):abs(50000 - sum:dapd.ScraperAppleRequests.sum{*} by {alias}) - 30000 > 0"
  # Disabled due to strange behaviour with sum function
  query = "sum(last_20m):sum:dapd.ScraperAppleRequests.sum{*} by {alias} > 80000"


  evaluation_delay    = 600 // minimum recommended value for GCP metrics 300
  require_full_window = true
  notify_no_data      = true
  no_data_timeframe   = 30
  renotify_interval   = 1440
  include_tags        = true
  restricted_roles    = [data.datadog_role.datadog_admin_role.id]

  tags = local.common_tags
}

resource "datadog_monitor" "ScraperAppleRequests_low" {
  name = "DAPD monitor: Scraper Apple requests too low"
  type = "query alert"

  message = <<EOT
{{#is_alert}}
${local.alert_message_env_title} amount of ScraperAppleRequests is < 20000 for the last 20 minutes.
{{/is_alert}}

{{#is_recovery}}
${local.alert_message_env_title} amount of ScraperAppleRequest is back to normal values.
{{/is_recovery}}
${local.alert_channels}
EOT

  query = "sum(last_20m):sum:dapd.ScraperAppleRequests.sum{*} by {alias} < 20000"


  evaluation_delay    = 600 // minimum recommended value for GCP metrics 300
  require_full_window = true
  notify_no_data      = true
  no_data_timeframe   = 30
  renotify_interval   = 1440
  include_tags        = true
  restricted_roles    = [data.datadog_role.datadog_admin_role.id]

  tags = local.common_tags
}

resource "datadog_monitor" "ScraperAppleRateLimitErrors" {
  name = "DAPD monitor: ScraperAppleRateLimitErrors too high"
  type = "query alert"

  message = <<EOT
{{#is_alert}}
${local.alert_message_env_title} amount of ScraperAppleRateLimitErrors is > 800 for the last 20 minutes.
{{/is_alert}}

{{#is_recovery}}
${local.alert_message_env_title} amount of ScraperAppleRateLimitErrors is back to normal values.
{{/is_recovery}}
${local.alert_channels}
EOT

  query = "sum(last_20m):sum:dapd.ScraperAppleRateLimitErrors.sum{*} by {alias} > 800"


  evaluation_delay    = 600 // minimum recommended value for GCP metrics 300
  require_full_window = true
  notify_no_data      = true
  no_data_timeframe   = 30
  renotify_interval   = 1440
  include_tags        = true
  restricted_roles    = [data.datadog_role.datadog_admin_role.id]

  tags = local.common_tags
}

resource "datadog_monitor" "ScraperApplePlaylistProcessed" {
  name = "DAPD monitor: ScraperApplePlaylistProcessed too low"
  type = "query alert"

  message = <<EOT
{{#is_alert}}
${local.alert_message_env_title} amount of ScraperApplePlaylistProcessed is < 1000 for the last 6 hours.
{{/is_alert}}

{{#is_recovery}}
${local.alert_message_env_title} amount of ScraperApplePlaylistProcessed is back to normal values.
{{/is_recovery}}
${local.alert_channels}
EOT

  query = "sum(last_6h):sum:dapd.ScraperApplePlaylistProcessed.sum{*} by {alias} < 1000"


  evaluation_delay    = 600 // minimum recommended value for GCP metrics 300
  require_full_window = true
  notify_no_data      = true
  no_data_timeframe   = 30
  renotify_interval   = 1440
  include_tags        = true
  restricted_roles    = [data.datadog_role.datadog_admin_role.id]

  tags = local.common_tags
}

/*
4 ScraperApplePlaylistProcessed, TransformServiceInputApplePlaylists
period: 6 hours
relation = TransformServiceInputApplePlaylists / ScraperApplePlaylistProcessed
threshold: [0.85, 1.5]
*/

resource "datadog_monitor" "ApplePlaylist_TransformService_to_Scraper_ratio_high" {
  name = "DAPD monitor: TransformServiceInputApplePlaylists / ScraperApplePlaylistProcessed ratio too high"
  type = "query alert"

  message = <<EOT
{{#is_alert}}
${local.alert_message_env_title} relation TransformServiceInputApplePlaylists / ScraperApplePlaylistProcessed is >1.5  for the last 6 hours.
{{/is_alert}}

{{#is_recovery}}
${local.alert_message_env_title} relation TransformServiceInputApplePlaylists / ScraperApplePlaylistProcessed is back to normal values.
{{/is_recovery}}
${local.alert_channels}
EOT

  query = "avg(last_10m):sum:dapd.TransformServiceInputApplePlaylists.sum{*} by {alias}.rollup(sum, 21600)/sum:dapd.ScraperApplePlaylistProcessed.sum{*} by {alias}.rollup(sum, 21600) > 1.5"


  evaluation_delay    = 21600 // minimum recommended value for GCP metrics 300
  require_full_window = true
  notify_no_data      = true
  no_data_timeframe   = 30
  renotify_interval   = 1440
  include_tags        = true
  restricted_roles    = [data.datadog_role.datadog_admin_role.id]

  tags = local.common_tags
}

resource "datadog_monitor" "ApplePlaylist_TransformService_to_Scraper_ratio_low" {
  name = "DAPD monitor: TransformServiceInputApplePlaylists / ScraperApplePlaylistProcessed ratio too low"
  type = "query alert"

  message = <<EOT
{{#is_alert}}
${local.alert_message_env_title} relation TransformServiceInputApplePlaylists / ScraperApplePlaylistProcessed is < 0.85  for the last 6 hours.
{{/is_alert}}

{{#is_recovery}}
${local.alert_message_env_title} relation TransformServiceInputApplePlaylists / ScraperApplePlaylistProcessed is back to normal values.
{{/is_recovery}}
${local.alert_channels}
EOT

  query = "max(last_10h):sum:dapd.TransformServiceInputApplePlaylists.sum{*} by {alias}.rollup(sum, 21600)/sum:dapd.ScraperApplePlaylistProcessed.sum{*} by {alias}.rollup(sum, 21600) < 0.85"


  evaluation_delay    = 21600 // minimum recommended value for GCP metrics 300
  require_full_window = true
  notify_no_data      = true
  no_data_timeframe   = 30
  renotify_interval   = 1440
  include_tags        = true
  restricted_roles    = [data.datadog_role.datadog_admin_role.id]

  tags = local.common_tags
}