resource "datadog_monitor" "healthy_tasks_monitor" {
  count = var.healthy_tasks_monitor_enabled ? 1 : 0
  name  = "${var.environment}-${var.service_name} healthy tasks monitor"
  type  = "metric alert"
  message = templatefile("${path.module}/templates/notifications.tftpl", {
    notification_endpoints         = coalesce(var.healthy_tasks_notification_overrides, local.default_notification_settings).endpoints
    alert_notification_endpoints   = coalesce(var.healthy_tasks_notification_overrides, local.default_notification_settings).alert_endpoints
    no_data_notification_endpoints = coalesce(var.healthy_tasks_notification_overrides, local.default_notification_settings).no_data_endpoints
  })
  escalation_message = "Escalation to ${coalesce(var.healthy_tasks_notification_overrides, local.default_notification_settings).escalation_endpoints}"
  # (desired tasks count - running task count) should be 0,
  # our threshold is based on how many times the metric reports a non-zero value in the evaluation window
  # this helps us avoid false positives due to transient issues or SPOT instances
  query = "sum(${var.healthy_tasks_evaluation_window}):count_nonzero(max:aws.ecs.service.desired{${local.healthy_tasks_filter[var.environment_type]}} - max:aws.ecs.service.running{${local.healthy_tasks_filter[var.environment_type]}}) >= ${var.healthy_tasks_critical_number}"

  monitor_thresholds {
    ok                = var.healthy_tasks_ok_number
    critical          = var.healthy_tasks_critical_number
  }

  evaluation_delay    = 900 # Recommended for AWS metrics
  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = false
  no_data_timeframe   = 0

  tags = local.combined_resource_tags
}

resource "datadog_monitor" "service_cpu_monitor" {
  count = var.service_cpu_monitor_enabled ? 1 : 0
  name  = "${var.environment}-${var.service_name} cpu monitor"
  type  = "metric alert"
  message = templatefile("${path.module}/templates/notifications.tftpl", {
    notification_endpoints         = coalesce(var.service_cpu_notification_overrides, local.default_notification_settings).endpoints
    alert_notification_endpoints   = coalesce(var.service_cpu_notification_overrides, local.default_notification_settings).alert_endpoints
    no_data_notification_endpoints = coalesce(var.service_cpu_notification_overrides, local.default_notification_settings).no_data_endpoints
  })
  escalation_message = "Escalation to ${coalesce(var.service_cpu_notification_overrides, local.default_notification_settings).escalation_endpoints}"

  query = "${var.service_cpu_monitor_evaluation_function}(${var.service_cpu_time_window}):avg:${local.service_cpu_monitor_metric[var.environment_type]}{${local.service_cpu_monitor_filter[var.environment_type]}} > ${var.service_cpu_critical_number}"

  monitor_thresholds {
    ok                = var.service_cpu_ok_number
    warning           = var.service_cpu_warning_number
    warning_recovery  = var.service_cpu_warning_recovery_number
    critical          = var.service_cpu_critical_number
    critical_recovery = var.service_cpu_critical_recovery_number
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = false
  no_data_timeframe   = 0

  tags = local.combined_resource_tags
}

resource "datadog_monitor" "kafka_connector_errors_monitor" {
  count   = var.error_monitor_enabled ? 1 : 0
  name    = "${var.environment}-${var.service_name} error monitor"
  type    = "log alert"
  message = "[${var.environment}] ${var.service_name} Kafka Connector error. Notify: ${var.notification_endpoints}"

  query = "logs(\"source:${var.environment}-${var.service_name} (\\\"ERROR\\\" OR *EXCEPTION*) ${var.error_monitor_extra_query}\").index(\"*\").rollup(\"count\").last(\"${var.error_monitor_query_interval}\") >= ${var.error_monitor_critical_number_errors}"

  monitor_thresholds {
    critical = var.error_monitor_critical_number_errors
  }

  enable_logs_sample  = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = false

  tags = local.combined_resource_tags
}


resource "datadog_monitor" "kafka_connector_dlq_monitor" {
  count   = var.dlq_topic_name != "" ? 1 : 0
  name    = "${var.environment}-${var.service_name} DLQ monitor"
  type    = "metric alert"
  message = "[${var.environment}] ${var.service_name} Connector sent a message into DLQ topic. Notify: ${var.notification_endpoints}"

  query = "max(${var.dlq_query_interval}):avg:aws.kafka.messages_in_per_sec{cluster_name:${var.monitor_kafka_topics_cluster_name},topic:${lower(var.dlq_topic_name)}} > ${var.dlq_number_errors}"

  monitor_thresholds {
    critical = var.dlq_number_errors
  }

  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 1
  require_full_window = false
  renotify_statuses   = ["alert"]
  tags                = local.combined_resource_tags
}

resource "datadog_monitor" "debezium_seconds_since_last_event_monitor" {
  count   = local.debezium_seconds_since_last_event_monitor_enabled ? 1 : 0
  name    = "${var.environment}-${var.service_name} SecondsSinceLastEvent monitor"
  type    = "query alert"
  message            = join(" ", [
    "{{#is_alert}}\n[${var.environment}] ${var.service_name} Connector has not processed new messages since {{value}} seconds.\n{{/is_alert}}\n",
    "{{#is_warning}}\n[${var.environment}] ${var.service_name} Connector has not processed new messages since {{value}} seconds.\n{{/is_warning}}\n",
    "{{#is_recovery}}\n[${var.environment}] ${var.service_name} Connector has started processing new messages and the delay has gone down to {{value}} seconds.\n{{/is_recovery}}\n\n",
    "Notify: ${var.notification_endpoints}"
  ])

  query = "avg(last_5m):max:jmx.debezium.mysql.milli_seconds_since_last_event{task_name:${var.environment}-${var.service_name}} / 1000 > ${var.seconds_since_last_event_critical}"

  monitor_thresholds {
    critical = var.seconds_since_last_event_critical
    warning = var.seconds_since_last_event_warning
  }

  notify_no_data      = true
  no_data_timeframe   = var.seconds_since_last_event_no_data_timeframe
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 1
  require_full_window = false
  renotify_statuses   = ["alert"]
  tags                = local.combined_resource_tags
}

resource "datadog_monitor" "debezium_seconds_behind_source_monitor" {
  count   = local.debezium_seconds_behind_source_monitor_enabled ? 1 : 0
  name    = "${var.environment}-${var.service_name} SecondsBehindSource monitor"
  type    = "query alert"
  message            = join(" ", [
    "{{#is_alert}}\n[${var.environment}] ${var.service_name} Connector is {{value}} seconds behind source.\n{{/is_alert}}\n",
    "{{#is_warning}}\n[${var.environment}] ${var.service_name} Connector is {{value}} seconds behind source.\n{{/is_warning}}\n",
    "{{#is_recovery}}\n[${var.environment}] ${var.service_name} Connector is recovering and is now {{value}} seconds behind source.\n{{/is_recovery}}\n\n",
    "Notify: ${var.notification_endpoints}"
  ])

  query = "avg(last_5m):max:jmx.debezium.mysql.milli_seconds_behind_source{task_name:${var.environment}-${var.service_name}} / 1000 > ${var.seconds_behind_source_critical}"

  monitor_thresholds {
    critical = var.seconds_behind_source_critical
    warning = var.seconds_behind_source_warning
  }

  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 1
  require_full_window = false
  renotify_statuses   = ["alert"]
  tags                = local.combined_resource_tags
}

resource "datadog_downtime_schedule" "healthy_tasks_monitor_downtime" {
  count = var.healthy_tasks_monitor_silenced ? 1 : 0
  scope = "*"
  monitor_identifier {
    monitor_id = datadog_monitor.healthy_tasks_monitor[count.index].id
  }

  recurring_schedule {
    recurrence {
      duration = "24h"
      rrule    = "FREQ=DAILY;INTERVAL=1"
    }
  }
}

resource "datadog_downtime_schedule" "service_cpu_monitor_downtime" {
  count = var.service_cpu_monitor_silenced ? 1 : 0
  scope = "*"
  monitor_identifier {
    monitor_id = datadog_monitor.service_cpu_monitor[count.index].id
  }

  recurring_schedule {
    recurrence {
      duration = "24h"
      rrule    = "FREQ=DAILY;INTERVAL=1"
    }
  }
}