locals {
  delphi_dapd_tags = concat(
    local.common_tags,
    ["application_sub_family:delphi-dapd"]
  )
  delphi_apps_etl_tags = concat(
    local.common_tags,
    ["application_sub_family:delphi-apps-etl"]
  )
  delphi_slz_tags = concat(
    local.common_tags,
    ["application_sub_family:delphi-slz"]
  )
}

resource "datadog_monitor" "content_status_failure_count_monitor" {
  enable_logs_sample     = true
  evaluation_delay       = 0
  groupby_simple_monitor = false

  include_tags    = true
  message         = "Environment: {{environment}} Host: {{host.name}} has logged failure_count >= {{threshold}} .  ${var.slack_channels["etl"]}"
  name            = "${var.environment}-${var.service_name} ContentStatus failure_count at {{value}}. {{#is_alert}}Threshold exceeded: Set at {{threshold}}.{{/is_alert}}"
  new_group_delay = 0

  no_data_timeframe    = 2
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "logs(\"service:cloudwatch\").index(\"main\").rollup(\"min\", \"@failure_count\").last(\"10m\") >= 5"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.common_tags
  timeout_h            = 0
  type                 = "log alert"

  monitor_thresholds {
    critical = "5"
  }
}

resource "datadog_monitor" "dapd_apple_dtf_fails_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
{{#is_alert}} Apple DAPD DTF unit was failed! {{/is_alert}}

{{^is_alert}}  Apple DAPD DTF unit came alive! {{/is_alert}}

${var.slack_channels["dtf"]}
EOT

  name                 = "${var.environment}-${var.service_name} Apple DAPD DTF fails"
  new_group_delay      = 60
  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "sum(last_1h):sum:dtf_importer_task.uow_status{status:import_completed AND version:v2 AND env:${var.environment} AND project IN (dapd_apple)} by {env}.as_count() <= 0"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_dapd_tags
  timeout_h            = 0
  type                 = "query alert"

  monitor_thresholds {
    critical = "0"
  }
}

resource "datadog_monitor" "dapd_api_errors_monitor" {
  evaluation_delay = 0

  include_tags = false

  message         = "DAPD API return some errors. ${var.slack_channels["dapd"]}"
  name            = "${var.environment}-${var.service_name} DAPD API Errors"
  new_group_delay = 60

  no_data_timeframe        = 0
  notification_preset_name = "hide_query"
  notify_audit             = false
  notify_by                = []
  notify_no_data           = false
  priority                 = 1
  query                    = "sum(last_10m):sum:dapd_api_scraper.api_errors{env:${var.environment} AND status_code IN (401,429,500,501,502,503,504)} by {dsp,entity,status_code,env}.as_count() > 200"
  renotify_interval        = 0
  renotify_occurrences     = 0
  require_full_window      = false
  tags                     = local.delphi_dapd_tags
  timeout_h                = 0
  type                     = "query alert"

  monitor_thresholds {
    critical = "200"
  }
}

resource "datadog_monitor" "dapd_apple_music_playlists_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
dapd-api-scraper apple_music playlists process below threshold

${var.slack_channels["dapd"]}
EOT

  name            = "${var.environment}-${var.service_name} DAPD Apple music playlists"
  new_group_delay = 60

  no_data_timeframe    = 5
  notify_audit         = false
  notify_by            = []
  notify_no_data       = true
  priority             = 2
  query                = "avg(last_1d):moving_rollup(avg:dapd_api_scraper.records_processed{entity:playlist AND dsp:apple_music AND env:${var.environment}} by {env,dsp,entity}.as_rate(), 86400, 'avg') < 18"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_dapd_tags
  timeout_h            = 0
  type                 = "query alert"

  monitor_thresholds {
    critical = "18"
  }
}

resource "datadog_monitor" "dapd_playlists_scraping_anomaly_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
dapd-api-scraper records processed anomaly detected

${var.slack_channels["dapd"]}
EOT

  name            = "${var.environment}-${var.service_name} DAPD Playlists scraping anomaly"
  new_group_delay = 60

  no_data_timeframe    = 15
  notify_audit         = false
  notify_by            = []
  notify_no_data       = true
  priority             = 0
  query                = "avg(last_12h):anomalies(moving_rollup(avg:dapd_api_scraper.records_processed{entity:playlist AND env:${var.environment}} by {env,dsp,entity}.as_rate(), 3600, 'avg'), 'robust', 4, direction='below', interval=120, alert_window='last_15m', timezone='utc', seasonality='hourly', count_default_zero='true') >= 0.75"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_dapd_tags
  timeout_h            = 0
  type                 = "query alert"

  monitor_threshold_windows {
    recovery_window = "last_15m"
    trigger_window  = "last_15m"
  }

  monitor_thresholds {
    critical          = "0.75"
    critical_recovery = "0"
  }
}

resource "datadog_monitor" "dapd_spotify_playlists_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
dapd-api-scraper spotify playlists process below threshold

${var.slack_channels["dapd"]}
EOT

  name            = "${var.environment}-${var.service_name} DAPD Spotify playlists"
  new_group_delay = 60

  no_data_timeframe    = 5
  notify_audit         = false
  notify_by            = []
  notify_no_data       = true
  priority             = 2
  query                = "avg(last_1d):moving_rollup(avg:dapd_api_scraper.records_processed{entity:playlist AND dsp:spotify AND env:${var.environment}} by {env,dsp,entity}.as_rate(), 86400, 'avg') < 28"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_dapd_tags
  timeout_h            = 0
  type                 = "query alert"

  monitor_thresholds {
    critical = "28"
  }
}

resource "datadog_monitor" "apps_etl_execution_time_monitor" {
  evaluation_delay = 0

  include_tags = false

  message = <<-EOT
- **environment** = `{{environment.name}}`
- **jobName** = `{{[mdc.delphi.etl.apps.appname].name}}`
- **traceId** = `{{[mdc.delphi.etl.apps.traceid].name}}`
- **executionTime** = `{{value}}s`

${var.slack_channels["etl"]}
EOT

  name            = "${var.environment}-${var.service_name} Apps ETL Execution Time"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "max(last_1m):min:etl.apps.execution_time{environment:${var.environment},!mdc.delphi.etl.apps.appname:apolloplaylistisrcdatedayaggregation} by {service,mdc.delphi.etl.apps.traceid,mdc.delphi.etl.apps.appname} > 3000"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = concat(local.delphi_dapd_tags, ["service:databricks"])
  timeout_h            = 24
  type                 = "query alert"

  monitor_thresholds {
    critical          = "3000"
    critical_recovery = "899"
    warning           = "900"
    warning_recovery  = "899"
  }
}

resource "datadog_monitor" "apollo_etl_execution_time_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
- **environment** = `{{environment.name}}`
- **jobName** = `{{[mdc.delphi.etl.apps.appname].name}}`
- **traceId** = `{{[mdc.delphi.etl.apps.traceid].name}}`
- **executionTime** = `{{value}}s`

${var.slack_channels["etl"]}
EOT

  name            = "${var.environment}-${var.service_name} Apollo ETL Execution Time"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "avg(last_5m):min:etl.apps.execution_time{environment:${var.environment},mdc.delphi.etl.apps.appname:apolloplaylistisrcdatedayaggregation} by {service,mdc.delphi.etl.apps.traceid,mdc.delphi.etl.apps.appname} > 35000"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = concat(local.delphi_dapd_tags, ["service:databricks"])
  timeout_h            = 24
  type                 = "query alert"

  monitor_thresholds {
    critical          = "35000"
    critical_recovery = "25000"
    warning           = "30000"
    warning_recovery  = "25000"
  }
}

resource "datadog_monitor" "high_failure_count_metric_monitor" {
  evaluation_delay = 0

  include_tags = false

  message = <<-EOT
- **environment** = `{{environment.name}}`
- **failure_count** = `{{value}}`
- **content_name** = `{{content_name.name}}`
- **uow_id** = `{{[job.uow_id].name}}`
- **failure_description** = `{{error.name}}`

${var.slack_channels["slz"]}
EOT

  name            = "${var.environment}-${var.service_name} High failure_count Metric"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "max(last_15m):min:slz.failure_count{environment:production} by {environment,job.uow_id,content_name,error} >= 5"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_slz_tags
  timeout_h            = 24
  type                 = "metric alert"

  monitor_thresholds {
    critical = "5"
    warning  = "1"
  }
}

resource "datadog_monitor" "delphi_apps_etl_status_message_error_monitor" {
  enable_logs_sample     = true
  evaluation_delay       = 0
  groupby_simple_monitor = false

  include_tags    = true
  message         = "- **environment** = `{{environment.name}}` ${var.slack_channels["etl"]}"
  name            = "${var.environment}-${var.service_name} Delphi Apps ETL- Status Message: Error"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "logs(\"service:delphi-apps-etl status:error @mdc.delphi.etl.env:prod\").index(\"main\").rollup(\"count\").by(\"status\").last(\"5m\") >= 5"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = concat(local.delphi_apps_etl_tags, ["service:databricks"])
  timeout_h            = 0
  type                 = "log alert"

  monitor_thresholds {
    critical = "5"
    warning  = "1"
  }
}


resource "datadog_monitor" "apps_etl_failed_jobs_monitor" {
  enable_logs_sample     = true
  evaluation_delay       = 0
  groupby_simple_monitor = false

  include_tags    = true
  message         = var.slack_channels["etl"]
  name            = "${var.environment}-${var.service_name} Prod Apps ETL Job Failure"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "logs(\"service:delphi-apps-etl @level:ERROR @mdc.delphi.etl.apps.env:${var.environment} Job failed\").index(\"main\").rollup(\"count\").last(\"5m\") >= 1"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = concat(local.delphi_apps_etl_tags, ["service:databricks"])
  timeout_h            = 0
  type                 = "log alert"

  monitor_thresholds {
    critical = "1"
  }
}

resource "datadog_monitor" "slz_file_size_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
Context {{[job.uow_id].name}}:{{[job.context].name}} file size difference is {{value}}.

UoW - `{{[job.uow_id].name}}`
Context - `{{[job.context].name}}`
${var.slack_channels["slz"]} @victorops
EOT

  name            = "${var.environment}-${var.service_name} SLZ file size validation"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "max(last_5m):avg:slz.validation.difference{environment:production} by {job.uow_id,job.context} > 30"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_slz_tags
  timeout_h            = 24
  type                 = "query alert"

  monitor_thresholds {
    critical          = "30"
    critical_recovery = "19"
    warning           = "20"
    warning_recovery  = "9"
  }
}

resource "datadog_monitor" "slz_uow_status_monitor" {
  evaluation_delay = 0

  include_tags = true

  message         = "SLZ-UoW Staying active 'N' days ${var.slack_channels["slz"]}"
  name            = "${var.environment}-${var.service_name} SLZ-UoW Staying active 'N' days"
  new_group_delay = 0

  no_data_timeframe    = 1440
  notify_audit         = false
  notify_by            = []
  notify_no_data       = true
  priority             = 0
  query                = "sum(last_1d):count:slz.uow_item.completeness_status.min_complete{environment:production} by {dsp}.as_count() + count:slz.uow_item.completeness_status.complete{environment:production} by {dsp}.as_count() < 0"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = true
  tags                 = local.delphi_slz_tags
  timeout_h            = 24
  type                 = "query alert"

  monitor_thresholds {
    critical          = "0"
    critical_recovery = "1"
  }
}

resource "datadog_monitor" "spotify_dapd_dtf_unit_status_monitor" {
  evaluation_delay = 0

  include_tags = true

  message = <<-EOT
{{#is_alert}} Spotify DAPD DTF unit was failed! {{/is_alert}}

{{^is_alert}}  Spotify DAPD DTF unit came alive! {{/is_alert}}

${var.slack_channels["dtf"]}
EOT

  name            = "${var.environment}-${var.service_name} Spotify DAPD DTF fails"
  new_group_delay = 60

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "sum(last_1h):sum:dtf_importer_task.uow_status{status:import_completed AND version:v2 AND env:${var.environment} AND project IN (dapd_spotify)} by {env}.as_count() <= 0"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_dapd_tags
  timeout_h            = 0
  type                 = "query alert"

  monitor_thresholds {
    critical = "0"
  }
}

resource "datadog_monitor" "unit_of_work_status_monitor" {
  evaluation_delay = 0

  include_tags = true

  message         = "UOW {{[uow.unit_of_work_code].name}} is in ACTIVE state more than configured time. ${var.slack_channels["slz"]}"
  name            = "${var.environment}-${var.service_name} UOW monitoring: Uow stuck in ACTIVE status"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "sum(last_1m):avg:slz.uow_monitoring{environment:production} by {uow.unit_of_work_code,uow.unit_of_work_id} >= 1"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.delphi_slz_tags
  timeout_h            = 0
  type                 = "query alert"

  monitor_thresholds {
    critical = "1"
  }
}