# Datadog configs for creating custom monitors
data "aws_secretsmanager_secret" "datadog_api" {
  name = "${var.environment}/datadog/DD_API_KEY"
}

data "aws_secretsmanager_secret_version" "datadog_api" {
  secret_id = data.aws_secretsmanager_secret.datadog_api.id
}

data "aws_secretsmanager_secret" "datadog_app_key" {
  name = "${var.environment}/datadog/DD_APP_KEY"
}

data "aws_secretsmanager_secret_version" "datadog_app_key" {
  secret_id = data.aws_secretsmanager_secret.datadog_app_key.id
}

data "datadog_role" "datadog_admin_role" {
  filter = "Datadog Admin Role"
}

terraform {
  required_providers {
    datadog = {
      source = "DataDog/datadog"
    }
  }
}

resource "datadog_monitor" "ecs_task_failure_monitor" {
  name    = "🚨 ECS Task Failure: dbt-accounting"
  type    = "query alert"
  message = "ECS Task Failure detected for dbt-accounting. ${var.notification_alerts_endpoint}"

  query = "avg(last_5m):sum:aws.ecs.task.exitCode{clustername:${var.environment}-${var.service_name},taskdefinitionfamily:${var.environment}-${var.service_name}} by {taskdefinitionfamily} > 0"

  monitor_thresholds {
    critical = 0
  }

  include_tags       = true
  notify_audit       = false
  on_missing_data    = "default"
  new_group_delay    = 60
  renotify_interval  = 0
  escalation_message = "ECS Task Failure for dbt-accounting. Immediate attention required! ${var.notification_escalation_endpoint}"

  tags = [
    "env:${var.environment}",
    "service:${var.service_name}",
    "team:${var.team_name}",
  ]
}

data "aws_lambda_function" "datadog_lambda_function" {
  function_name = "DatadogLambdaFunction"
}

resource "aws_cloudwatch_log_subscription_filter" "dbt_accounting_sm_datadog_log_filter" {
  name            = "${var.environment}-${var.service_name}-sm-log-subscription-filter"
  log_group_name  = aws_cloudwatch_log_group.dbt_accounting_sm_logs.name
  filter_pattern  = ""
  destination_arn = data.aws_lambda_function.datadog_lambda_function.arn
  distribution    = "ByLogStream"
}

resource "datadog_monitor" "step_function_failure_monitor" {
  name    = "🚨 Show to Customer Accounting DBT Refresh Failed"
  type    = "query alert"
  message = "Step Function execution either failed, timed out, or aborted for ${var.environment}-${var.service_name}-sm. ${var.notification_alerts_endpoint}"

  # Alert on any failed, timed-out, or aborted executions of the dbt-accounting state machine in the last 5 minutes.
  query = "sum(last_5m):sum:aws.states.executions_failed{statemachinename:${var.environment}-${var.service_name}-sm}.as_count() + sum:aws.states.executions_timed_out{statemachinename:${var.environment}-${var.service_name}-sm}.as_count() + sum:aws.states.executions_aborted{statemachinename:${var.environment}-${var.service_name}-sm}.as_count() > 1"

  monitor_thresholds {
    critical = 1
    warning  = 0
  }

  include_tags       = true
  notify_audit       = false
  on_missing_data    = "default"
  renotify_interval  = 0
  escalation_message = "Show to Customer Accounting DBT Refresh Failed. Immediate attention required! ${var.notification_escalation_endpoint}"

  tags = [
    "env:${var.environment}",
    "service:${var.service_name}",
    "team:${var.team_name}",
  ]
}