# Datadog configs for creating custom monitors
data "aws_secretsmanager_secret" "datadog_api" {
  name = "${var.environment}/datadog/DD_API_KEY"
}

data "aws_secretsmanager_secret_version" "datadog_api" {
  secret_id = data.aws_secretsmanager_secret.datadog_api.id
}

data "aws_secretsmanager_secret" "datadog_app_key" {
  name = "${var.environment}/datadog/DD_APP_KEY"
}

data "aws_secretsmanager_secret_version" "datadog_app_key" {
  secret_id = data.aws_secretsmanager_secret.datadog_app_key.id
}

# Datadog provider
provider "datadog" {
  api_key = data.aws_secretsmanager_secret_version.datadog_api.secret_string
  app_key = data.aws_secretsmanager_secret_version.datadog_app_key.secret_string
}

# Datadog Monitors
resource "datadog_monitor" "delivery_order_failed" {
  count   = var.datadog_monitor_enabled ? 1 : 0
  name    = "[${var.environment}] Ownership Delivery order failure"
  type    = "log alert"
  message = <<EOF
An ownership delivery order has failed, please investigate.

Step function run: https://us-east-1.console.aws.amazon.com/states/home?region=us-east-1#/v2/executions/details/{{log.attributes.context.Execution.Id}}

Order ID: {{log.attributes.context.id}}

Notify: @slack-collections-prod-alarms

EOF

  query = "logs(\"@environment:${var.environment} service:lambda-nr-ownership-delivery-order-finalizer \\\"Error running in order-finalizer\\\"\").rollup(\"count\").last(\"10m\") >= 1"
  monitor_thresholds {
    critical = 1
  }

  enable_logs_sample  = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  include_tags        = true
  require_full_window = false
  priority            = 5

  tags = [
    "env:${var.environment}",
    "service:lambda-nr-ownership-delivery-order-finalizer",
    "team:${var.team_name}"
  ]
}

# this is technically an alert for the graphql service, but it pertains to the ownership delivery process, so it is included here
resource "datadog_monitor" "eligibility_query_slow" {
  count   = 1
  name    = "[${var.environment}] Ownership Delivery slow eligibility query"
  type    = "query alert"
  message = <<EOF
    The ownership delivery eligibility query took a long time to run. If the query took longer than 60 seconds, the request will have been cut off by the router. Please check that the order has not failed. The order-populator/step fns will retry, so the order may still be running successfully.
    If this alert is triggering regularly, we should investigate the performance of the eligibility query. This may become a problem with our current architecture due to the 60 second hard limit.
    This alert should not be attached to pagerduty as it's not a production outage.
    Notify: ${var.notification_endpoints}
    EOF

  query = "max(last_30m):max:trace.graphql.execute{resource_name:${var.eligibility_query_resource_name},env:${var.environment}} > 45"

  monitor_thresholds {
    warning  = 30
    critical = 45
  }

  on_missing_data     = "resolve"
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  include_tags        = true
  require_full_window = false
  priority            = 5

  tags = [
    "env:${var.environment}",
    "service:graphql-neighbouring-rights",
    "team:${var.team_name}"
  ]
}