# Datadog configs for creating custom monitors
data "aws_secretsmanager_secret" "datadog_api" {
  name = "${var.environment}/datadog/DD_API_KEY"
}

data "aws_secretsmanager_secret_version" "datadog_api" {
  secret_id = data.aws_secretsmanager_secret.datadog_api.id
}

data "aws_secretsmanager_secret" "datadog_app_key" {
  name = "${var.environment}/datadog/DD_APP_KEY"
}

data "aws_secretsmanager_secret_version" "datadog_app_key" {
  secret_id = data.aws_secretsmanager_secret.datadog_app_key.id
}

data "datadog_role" "datadog_admin_role" {
  filter = "Datadog Admin Role"
}

# Terraform 0.13+ uses the Terraform Registry:
terraform {
  backend "s3" {
    bucket  = "orcd-terraform-state"
    key     = "prod/ows-moneyhub/datadog/terraform.tfstate"
    region  = "us-east-1"
    encrypt = "true"
  }

  required_providers {
    datadog = {
      source = "DataDog/datadog"
    }
  }
}

# Datadog provider
provider "datadog" {
  api_key = data.aws_secretsmanager_secret_version.datadog_api.secret_string
  app_key = data.aws_secretsmanager_secret_version.datadog_app_key.secret_string
}

# Datadog Monitors
resource "datadog_monitor" "_5xx_resource_monitor" {
  name               = "🚨Resource Monitor: 5xx Status Codes Count Too High on ${var.service_name} 🚨"
  type               = "trace-analytics alert"
  message            = "${var.five_hundred_resource_monitor_message} ${var.notification_alerts_endpoint}"

  query = "trace-analytics(\"env:${var.environment} service:${var.service_name} @http.status_code:5**\").rollup(\"count\").by(\"resource_name\").last(\"5m\") >= 5"

  monitor_thresholds {
    warning  = 2
    critical = 5
  }

  include_tags        = true
  on_missing_data     = "default"
  new_group_delay     = 60
  renotify_interval   = 30
  renotify_statuses   = ["alert"]
  escalation_message  = "${var.five_hundred_resource_monitor_escalation_message} ${var.notification_main_endpoint}"
  priority            = 2

  tags = concat([
    "application_family:${var.application_family}",
    "env:${var.environment}",
    "service:${var.service_name}",
    "team:${var.team_handle}"],
    [for app in var.dependent_applications : "dependent_application:${app}"]
  )
}

resource "datadog_monitor" "gateway_timeout_resource_monitor" {
  name               = "🚨Resource Monitor: 504 Status Codes Count Too High on ${var.service_name} 🚨"
  type               = "trace-analytics alert"
  message            = "${var.gateway_timeout_resource_monitor_message} ${var.notification_alerts_endpoint} ${var.escalation_endpoints}"

  query = "trace-analytics(\"env:${var.environment} service:${var.service_name} @http.status_code:504\").rollup(\"count\").by(\"resource_name\").last(\"5m\") >= 7"

  monitor_thresholds {
    warning  = 2
    critical = 7
  }

  include_tags        = true
  on_missing_data     = "default"
  new_group_delay     = 60
  renotify_interval   = 30
  renotify_statuses   = ["alert"]
  escalation_message  = "${var.gateway_timeout_resource_monitor_escalation_message} ${var.notification_main_endpoint}"
  priority            = 2

  tags = concat([
    "application_family:${var.application_family}",
    "env:${var.environment}",
    "team:${var.team_handle}",
    "service:${var.service_name}"
  ],
    [for app in var.dependent_applications : "dependent_application:${app}"]
  )
}

resource "datadog_monitor" "_5xx_resource_monitor_statement_periods" {
  name               = "🚨Resource Monitor: GET Statement Periods 5xx Status Codes Count Too High on ${var.service_name} 🚨"
  type               = "trace-analytics alert"
  message            = "${var.five_hundred_resource_monitor_statement_periods_message} ${var.notification_alerts_endpoint} ${var.escalation_endpoints}"

  query = "trace-analytics(\"env:${var.environment} service:${var.service_name} resource_name:(\\\"GET /account/{account_id}/statement-periods\\\") @http.status_code:5**\").rollup(\"count\").by(\"resource_name\").last(\"5m\") >= 7"

  monitor_thresholds {
    warning  = 2
    critical = 7
  }

  include_tags        = true
  on_missing_data     = "default"
  new_group_delay     = 60
  renotify_interval   = 30
  renotify_statuses   = ["alert"]
  escalation_message  = "${var.five_hundred_resource_monitor_statement_periods_escalation_message} ${var.escalation_endpoints}"
  priority            = 2

  tags = concat([
    "application_family:${var.application_family}",
    "env:${var.environment}",
    "service:${var.service_name}",
    "team:${var.team_handle}"],
    [for app in var.dependent_applications : "dependent_application:${app}"]
  )
}

resource "datadog_monitor" "_5xx_resource_monitor_reference_signing_entity" {
  name               = "🚨Resource Monitor: GET Reference Signing Entity 5xx Status Codes Count Too High on ${var.service_name} 🚨"
  type               = "trace-analytics alert"
  message            = "${var.five_hundred_resource_monitor_reference_signing_entity_message} ${var.notification_alerts_endpoint} ${var.escalation_endpoints}"

  query = "trace-analytics(\"env:${var.environment} service:${var.service_name} resource_name:(\\\"GET /reference-signing-entity/contract/{contract_id}\\\") @http.status_code:5**\").rollup(\"count\").by(\"resource_name\").last(\"5m\") >= 7"

  monitor_thresholds {
    warning  = 2
    critical = 7
  }

  include_tags        = true
  on_missing_data     = "default"
  new_group_delay     = 60
  renotify_interval   = 30
  renotify_statuses   = ["alert"]
  escalation_message  = "${var.five_hundred_resource_monitor_reference_signing_entity_escalation_message} ${var.escalation_endpoints}"
  priority            = 2

  tags = concat([
    "application_family:${var.application_family}",
    "env:${var.environment}",
    "service:${var.service_name}",
    "team:${var.team_handle}"
  ],
    [for app in var.dependent_applications : "dependent_application:${app}"]
  )
}


resource "datadog_monitor" "custom_reports_monitor" {
  name               = "🚨Custom Reports Monitor: Customers Are Unable to Get/Download Custom Reports on ${var.service_name}"
  type               = "trace-analytics alert"
  message            = "${var.custom_reports_monitor_message} ${var.notification_alerts_endpoint}"

  query = "trace-analytics(\"env:${var.environment} service:${var.service_name} resource_name:(\\\"GET /reports/custom/account/{account_id}\\\" OR \\\"GET /reports/custom/{report_custom_id}/download\\\") @http.status_code:5**\").rollup(\"count\").by(\"resource_name\").last(\"5m\") > 5"

  monitor_thresholds {
    warning  = 2
    critical = 5
  }

  include_tags        = true
  on_missing_data     = "default"
  new_group_delay     = 60
  renotify_interval   = 30
  renotify_statuses   = ["alert"]
  escalation_message  = "${var.custom_reports_monitor_escalation_message} ${var.notification_main_endpoint}"
  priority            = 1

  tags = concat([
    "application_family:${var.application_family}",
    "env:${var.environment}",
    "service:${var.service_name}",
    "team:${var.team_handle}"
  ],
    [for app in var.dependent_applications : "dependent_application:${app}"]
  )
}


resource "datadog_monitor" "statement_attachments_monitor" {
  name               = "🚨Statement Attachments Monitor: ${var.service_name} is Unable to retrieve/create Statement Attachments"
  type               = "query alert"
  message            = "${var.statement_attachments_monitor_message} ${var.notification_alerts_endpoint}"

  query = "sum(last_5m):sum:trace.fastapi.request.errors{env:${var.environment} AND service:${var.service_name} AND (resource_name:get_/statement-attachment/account/_account_id_/statement-period/_statement_period_id_ OR resource_name:post_/statement-attachment/account/_account_id_/statement-period/_statement_period_id_/* OR resource_name:get_/statement-attachment/_statement_attachment_id_/download* )} by {resource_name}.as_count() >= 5"

  monitor_thresholds {
    warning  = 2
    critical = 5
  }

  include_tags        = true
  notify_no_data      = false
  new_group_delay     = 60
  renotify_interval   = 30
  require_full_window = false
  renotify_statuses   = ["alert"]
  escalation_message  = "${var.statement_attachments_monitor_escalation_message} ${var.notification_main_endpoint}"
  priority            = 1

  tags = concat([
    "application_family:${var.application_family}",
    "env:${var.environment}",
    "service:${var.service_name}",
    "team:${var.team_handle}"],
    [for app in var.dependent_applications : "dependent_application:${app}"]
  )
}


resource "datadog_monitor" "latency_p75_monitor" {
  name               = "🐌 Latency Monitor: ${var.service_name} has a high p75 latency on ${var.environment}"
  type               = "query alert"
  message            = "${var.latency_p75_monitor_message} ${var.notification_alerts_endpoint}"

  query = "avg(last_10m):p75:trace.fastapi.request{env:${var.environment},service:${var.service_name}} > 5"

  monitor_thresholds {
    warning  = 3
    critical = 5
  }

  no_data_timeframe   = 10
  renotify_interval   = 0
  priority            = 3

  tags = [
    "application_family: ${var.application_family}",
    "env:${var.environment}",
    "service:${var.service_name}",
    "team:${var.team_handle}",
  ]
}


resource "datadog_monitor" "latency_p95_monitor" {
  name               = "🐌 Latency Monitor: ${var.service_name} has a high p95 latency on ${var.environment}"
  type               = "query alert"
  message            = "${var.latency_p95_monitor_message} ${var.notification_alerts_endpoint}"

  query = "avg(last_10m):p95:trace.fastapi.request{env:${var.environment},service:${var.service_name}} > 5"

  monitor_thresholds {
    warning  = 3
    critical = 5
  }

  no_data_timeframe   = 10
  renotify_interval   = 0
  priority            = 3

  tags = concat([
    "application_family: ${var.application_family}",
    "env:${var.environment}",
    "service:${var.service_name}",
    "team:${var.team_handle}"
  ],
    [for app in var.dependent_applications : "dependent_application:${app}"]
  )
}


resource "datadog_monitor" "security_monitor" {
  name               = "🚓 Security Monitor: High Number of Unauthorised Requests on ${var.service_name}"
  type               = "trace-analytics alert"
  message            = "${var.security_monitor_message} ${var.notification_alerts_endpoint}"

  query = "trace-analytics(\"env:${var.environment} service:${var.service_name} @http.status_code:(401 OR 403)\").rollup(\"count\").by(\"service\").last(\"5m\") > 15"

  monitor_thresholds {
    warning  = 5
    critical = 15
  }

  include_tags        = true
  on_missing_data     = "default"
  new_group_delay     = 60
  renotify_interval   = 30
  renotify_statuses   = ["alert"]
  escalation_message  = "${var.security_monitor_escalation_message} ${var.notification_main_endpoint}"
  priority            = 1

  tags = [
    "application_family: ${var.application_family}",
    "env:${var.environment}",
    "service:${var.service_name}",
    "team:${var.team_handle}",
  ]
}


# Datadog from dashboards & monitors created from terraform-datadog
module "ows_moneyhub_fargate_service_dashboard" {
  source = "git@github.com:theorchard/terraform-datadog.git//modules/service?ref=6.13.7"

  environment          = var.environment
  environment_type     = "fargate"
  service_name         = var.service_name
  application_family   = var.application_family
  teams                = ["${var.team_handle}"]

  notification_endpoints            = var.notification_alerts_endpoint
  escalation_notification_endpoints = var.notification_alerts_endpoint
  service_4xx_monitor_enabled       = false
  service_5xx_monitor_enabled       = false
  dependent_applications            = var.dependent_applications
  service_cpu_time_window           = "last_10m"
}

module "datadog_redis_elasticache_monitors_and_dashboard" {
  source       = "git@github.com:theorchard/terraform-datadog.git//modules/elasticache?ref=6.13.7"

  environment        = var.environment
  service_name       = var.service_name
  application_family = var.application_family
  teams              = ["${var.team_handle}"]

  notification_endpoints            = var.notification_alerts_endpoint
  escalation_notification_endpoints = var.notification_main_endpoint
}

module "datadog_team" {
  source       = "git@github.com:theorchard/terraform-datadog.git//modules/team?ref=6.13.7"

  team_handle  = var.application_family
  team_name    = var.team_name
  team_members = var.team_members
}