# Datadog configs for creating custom monitors
data "aws_secretsmanager_secret" "datadog_api" {
  name = "${var.environment}/datadog/DD_API_KEY"
}

data "aws_secretsmanager_secret_version" "datadog_api" {
  secret_id = data.aws_secretsmanager_secret.datadog_api.id
}

data "aws_secretsmanager_secret" "datadog_app_key" {
  name = "${var.environment}/datadog/DD_APP_KEY"
}

data "aws_secretsmanager_secret_version" "datadog_app_key" {
  secret_id = data.aws_secretsmanager_secret.datadog_app_key.id
}

# Datadog provider
provider "datadog" {
  api_key = data.aws_secretsmanager_secret_version.datadog_api.secret_string
  app_key = data.aws_secretsmanager_secret_version.datadog_app_key.secret_string
}

# Define datadog monitors

module "datadog_close_batch_lambda" {
  source = "git@github.com:theorchard/terraform-datadog.git//modules/lambda?ref=6.13.4"

  environment                       = var.environment
  service_name                      = "lambda-${var.lambda_close_batch_name}"
  lambda_invocation_monitor_enabled = true
  lambda_error_monitor_enabled      = true
  notification_endpoints            = var.notification_endpoints
  escalation_notification_endpoints = var.escalation_notification_endpoints
}

module "datadog_find_batch_to_close_lambda" {
  source = "git@github.com:theorchard/terraform-datadog.git//modules/lambda?ref=6.13.4"

  environment                       = var.environment
  service_name                      = "lambda-${var.lambda_find_batch_to_close_name}"
  lambda_invocation_monitor_enabled = true
  lambda_error_monitor_enabled      = true
  notification_endpoints            = var.notification_endpoints
  escalation_notification_endpoints = var.escalation_notification_endpoints
}

module "datadog_ready_deliveries_lambda" {
  source = "git@github.com:theorchard/terraform-datadog.git//modules/lambda?ref=6.13.4"

  environment                           = var.environment
  service_name                          = "${var.service_name}-${var.lambda_ready_deliveries_name}"
  lambda_invocation_monitor_enabled     = false
  lambda_error_monitor_enabled          = true
  lambda_error_warning_number           = 11
  lambda_error_critical_number          = 15
  lambda_error_critical_recovery_number = 12
  lambda_error_warning_recovery_number  = 8
  lambda_error_ok_number                = 6
  notification_endpoints                = var.notification_endpoints
  escalation_notification_endpoints     = var.escalation_notification_endpoints
}

module "datadog_ready_deliveries_manager_lambda" {
  source = "git@github.com:theorchard/terraform-datadog.git//modules/lambda?ref=6.13.4"

  environment                           = var.environment
  service_name                          = "${var.service_name}-${var.lambda_ready_deliveries_manager_name}"
  lambda_invocation_monitor_enabled     = false
  lambda_error_monitor_enabled          = true
  lambda_error_warning_number           = 2
  lambda_error_critical_number          = 3
  lambda_error_critical_recovery_number = 0
  lambda_error_warning_recovery_number  = 0
  lambda_error_ok_number                = 0
  notification_endpoints                = var.notification_endpoints
  escalation_notification_endpoints     = var.escalation_notification_endpoints
}

# resource "datadog_monitor" "ready-deliveries-monitor-exclude-folder-creation-errors" {
#   name               = "${var.environment} ready deliveries error monitor"
#   type               = "log alert"
#   message            = "Error rate is elevated. Here is the error log:\n{{log.message}} Notify: ${var.notification_endpoints}"
#   escalation_message = "Error rate is still elevated. Escalation to ${var.escalation_notification_endpoints}"

#   query = "logs(\"service:lambda-vector-ready-deliveries status:error -\\\"Remote folder creation error\\\" -\\\"Connection timed out for DMS\\\" @environment:prod\").index(\"*\").rollup(\"count\").last(\"5m\") >= ${var.critical_number_ready_deliveries_errors}"

#   monitor_thresholds {
#     ok       = var.ok_number_ready_deliveries_errors
#     warning  = var.warning_number_ready_deliveries_errors
#     critical = var.critical_number_ready_deliveries_errors
#   }

#   enable_logs_sample  = true
#   include_tags        = true
#   notify_no_data      = true
#   notify_audit        = false
#   renotify_interval   = 240
#   timeout_h           = 0
#   require_full_window = true

#   tags = [
#     "environment:${var.environment}",
#     "service_name:lambda-vector-ready-deliveries",
#     "application_family:${var.application_family}",
#   ]
# }

resource "datadog_monitor" "remote_folder_creation_error_monitor" {
  name    = "${var.environment} remote folder creation error monitor"
  type    = "log alert"
  message = "{{#is_alert}}There was an error creating a remote folder. The error log is below:\n{{log.message}}\n Notify: ${var.ops_alert_notification_endpoint}{{/is_alert}}"

  query = "logs(\"service:lambda-vector-ready-deliveries status:warning \\\"Remote folder creation error\\\" @environment:prod\").index(\"*\").rollup(\"count\").by(\"@dms_id\").last(\"5m\") > ${var.critical_number_remote_folder_creation}"

  monitor_thresholds {
    ok       = var.ok_number_remote_folder_creation
    critical = var.critical_number_remote_folder_creation
  }

  enable_logs_sample  = true
  include_tags        = true
  on_missing_data     = "show_and_notify_no_data"
  notify_audit        = false
  renotify_interval   = 240
  timeout_h           = 0
  require_full_window = true

  tags = [
    "environment:${var.environment}",
    "service_name:lambda-vector-ready-deliveries",
    "application_family:${var.application_family}",
  ]
}

resource "datadog_monitor" "encoding_order_lag_sync_monitor" {
  name    = "${var.environment} encoding order lag sync monitor"
  type    = "metric alert"
  message = "Encoding orders have failed to sync from art_relations.encoding_order to direct_delivery.encoding_queue. Notify: ${var.notification_endpoints} {{#is_alert}}${var.paging_notification_endpoints}{{/is_alert}} {{#is_recovery}}${var.paging_notification_endpoints}{{/is_recovery}}"

  query = "sum(last_30m):max:snowflake.custom.query_monitor.encoding_order_sync_lag{*}.as_count() >= 20"

  monitor_thresholds {
    ok       = 0
    warning  = 1
    critical = 20
  }

  require_full_window = false

  renotify_interval   = 60
  renotify_statuses   = ["alert", "warn"]

  tags = [
    "environment:${var.environment}",
    "service_name:lambda-vector",
    "application_family:${var.application_family}",
  ]
}

resource "datadog_monitor" "dms_connection_timeout_error_monitor" {
  name    = "${var.environment} dms connection timeout error monitor"
  type    = "log alert"
  message = "{{#is_alert}}There was an error creating a remote folder. The error log is below:\n{{log.message}}\n Notify: ${var.ops_alert_notification_endpoint}{{/is_alert}}"

  query = "logs(\"service:lambda-vector-ready-deliveries status:error \\\"Connection timed out for DMS\\\" @environment:prod\").index(\"*\").rollup(\"count\").by(\"@dms_id\").last(\"5m\") > ${var.critical_number_remote_folder_creation}"

  monitor_thresholds {
    ok       = var.ok_number_dms_connection_timeout
    critical = var.critical_number_dms_connection_timeout
  }

  enable_logs_sample  = true
  include_tags        = true
  notify_no_data      = true
  notify_audit        = false
  renotify_interval   = 240
  timeout_h           = 0
  require_full_window = true

  tags = [
    "environment:${var.environment}",
    "service_name:lambda-vector-ready-deliveries",
    "application_family:${var.application_family}",
  ]
}

resource "datadog_monitor" "lambda-vector-renew-outage" {
  name               = "${var.environment}-lambda-vector-renew outage monitor"
  type               = "metric alert"
  message            = "Lambda vector renew is not running. Notify: ${var.notification_endpoints}"
  escalation_message = "Lambda vector renew is not running. Escalation to ${var.renew_outage_notification_endpoint}"

  query = "sum(last_20m):sum:renew_manager.run{environment:${var.environment}}.as_count() <= ${var.renew_outage_threshold_critical}"

  monitor_thresholds {
    ok       = var.renew_outage_threshold_ok
    critical = var.renew_outage_threshold_critical
  }

  enable_logs_sample   = true
  include_tags         = true
  notify_no_data       = true
  notify_audit         = false
  renotify_interval    = 20
  renotify_occurrences = 1
  renotify_statuses = [
    "alert",
    "no data"
  ]
  timeout_h           = 0
  require_full_window = true

  tags = [
    "environment:${var.environment}",
    "service_name:lambda-vector-renew",
    "application_family:${var.application_family}",
  ]
}

resource "datadog_monitor" "lambda-vector-ready-deliveries-outage" {
  name               = "${var.environment}-lambda-vector-ready-deliveries outage monitor"
  type               = "metric alert"
  message            = "Lambda vector ready-deliveries is not running. Notify: ${var.notification_endpoints}"
  escalation_message = "Lambda vector ready-deliveries is not running. Escalation to ${var.ready_deliveries_outage_notification_endpoint}"

  query = "sum(last_20m):sum:ready_deliveries_manager.run{environment:${var.environment}}.as_count() <= ${var.ready_deliveries_outage_threshold_critical}"

  monitor_thresholds {
    ok       = var.ready_deliveries_outage_threshold_ok
    critical = var.ready_deliveries_outage_threshold_critical
  }

  enable_logs_sample   = true
  include_tags         = true
  notify_no_data       = true
  notify_audit         = false
  renotify_interval    = 20
  renotify_occurrences = 1
  renotify_statuses = [
    "alert",
    "no data"
  ]
  timeout_h           = 0
  require_full_window = true

  tags = [
    "environment:${var.environment}",
    "service_name:lambda-vector-ready-deliveries",
    "application_family:${var.application_family}",
  ]
}

resource "datadog_monitor" "lambda-vector-throttler-outage" {
  name               = "${var.environment}-lambda-vector-throttler outage monitor"
  type               = "metric alert"
  message            = "Lambda vector throttler is not running. Notify: ${var.notification_endpoints}"
  escalation_message = "Lambda vector throttler is not running. Escalation to ${var.throttler_outage_notification_endpoint}"

  query = "sum(last_20m):sum:throttler_manager.run{environment:${var.environment}}.as_count() <= ${var.throttler_outage_threshold_critical}"

  monitor_thresholds {
    ok       = var.throttler_outage_threshold_ok
    critical = var.throttler_outage_threshold_critical
  }

  enable_logs_sample   = true
  include_tags         = true
  notify_no_data       = true
  notify_audit         = false
  renotify_interval    = 20
  renotify_occurrences = 1
  renotify_statuses = [
    "alert",
    "no data"
  ]
  timeout_h           = 0
  require_full_window = true

  tags = [
    "environment:${var.environment}",
    "service_name:lambda-vector-throttler",
    "application_family:${var.application_family}",
  ]
}

resource "datadog_monitor" "lambda-vector-ar-to-ddb-sync-outage" {
  name               = "${var.environment}-lambda-vector-ar-to-ddb-sync outage monitor"
  type               = "metric alert"
  message            = "Lambda vector ar to ddb sync is not running. Notify: ${var.notification_endpoints}"
  escalation_message = "Lambda vector ar to ddb sync is not running. Escalation to ${var.escalation_notification_endpoints}"

  query = "sum(last_20m):sum:aws.lambda.invocations{service:lambda-vector-ar-to-ddb-sync, environment:${var.environment}}.as_count() <= ${var.ar_to_ddb_sync_outage_threshold_critical}"

  monitor_thresholds {
    ok       = var.ar_to_ddb_sync_outage_threshold_ok
    critical = var.ar_to_ddb_sync_outage_threshold_critical
  }

  enable_logs_sample  = true
  include_tags        = true
  notify_no_data      = true
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = true
  evaluation_delay    = 900
  no_data_timeframe   = 30
  renotify_statuses = [
    "alert",
    "no data"
  ]

  tags = [
    "environment:${var.environment}",
    "service_name:lambda-vector-ar-to-ddb-sync",
    "application_family:${var.application_family}",
  ]
}

module "datadog_ar_to_ddb_sync_lambda" {
  source = "git@github.com:theorchard/terraform-datadog.git//modules/lambda?ref=6.13.4"

  environment                       = var.environment
  service_name                      = "${var.service_name}-${var.lambda_ar_to_ddb_sync_name}"
  lambda_invocation_monitor_enabled = false
  lambda_error_monitor_enabled      = true
  notification_endpoints            = var.notification_endpoints
  escalation_notification_endpoints = var.escalation_notification_endpoints
}

module "datadog_lambda_throttler" {
  source = "git@github.com:theorchard/terraform-datadog.git//modules/lambda?ref=6.13.4"

  environment                       = var.environment
  service_name                      = "${var.service_name}-throttler"
  lambda_invocation_monitor_enabled = false
  lambda_error_monitor_enabled      = true
  notification_endpoints            = var.notification_endpoints
  escalation_notification_endpoints = var.escalation_notification_endpoints
}

module "datadog_lambda_auto_metadata_update" {
  source = "git@github.com:theorchard/terraform-datadog.git//modules/lambda?ref=6.13.4"

  environment                       = var.environment
  service_name                      = "${var.service_name}-auto-metadata-update"
  lambda_invocation_monitor_enabled = false
  lambda_error_monitor_enabled      = true
  notification_endpoints            = var.notification_endpoints
  escalation_notification_endpoints = var.escalation_notification_endpoints
}

module "datadog_lambda_job_priority_rule_match" {
  source = "git@github.com:theorchard/terraform-datadog.git//modules/lambda?ref=6.13.4"

  environment                       = var.environment
  service_name                      = "${var.service_name}-job-priority-rule-match"
  lambda_invocation_monitor_enabled = false
  lambda_error_monitor_enabled      = true
  notification_endpoints            = var.notification_endpoints
  escalation_notification_endpoints = var.paging_notification_endpoints
}

module "datadog_lambda_ddb_streams_handler" {
  source = "git@github.com:theorchard/terraform-datadog.git//modules/lambda?ref=6.13.4"

  environment                       = var.environment
  service_name                      = "${var.service_name}-ddb-streams-handler"
  lambda_invocation_monitor_enabled = false
  lambda_error_monitor_enabled      = true
  notification_endpoints            = var.notification_endpoints
  escalation_notification_endpoints = var.escalation_notification_endpoints
}

module "datadog_lambda_save_to_dd_db" {
  source = "git@github.com:theorchard/terraform-datadog.git//modules/lambda?ref=6.13.4"

  environment                       = var.environment
  service_name                      = "${var.service_name}-save-to-dd-db"
  lambda_invocation_monitor_enabled = false
  lambda_error_monitor_enabled      = true
  notification_endpoints            = var.notification_endpoints
  escalation_notification_endpoints = var.escalation_notification_endpoints
}

module "datadog_lambda_queue_tagger" {
  source = "git@github.com:theorchard/terraform-datadog.git//modules/lambda?ref=6.13.4"

  environment                       = var.environment
  service_name                      = "${var.service_name}-${var.lambda_queue_tagger_name}"
  lambda_invocation_monitor_enabled = false
  lambda_error_monitor_enabled      = true
  notification_endpoints            = var.notification_endpoints
  escalation_notification_endpoints = var.escalation_notification_endpoints
}

resource "datadog_monitor" "batch_to_close_dlq_monitor" {
  name    = "${var.environment}-batch-to-close-deadletter-monitor"
  type    = "metric alert"
  message = "[${var.environment}] batch_to_close-deadletter SQS has messages are not being processed. Notify: ${var.notification_endpoints}"
  query   = "sum(last_1h):sum:aws.sqs.number_of_messages_sent{queuename:${var.environment}-batch_to_close-deadletter}.as_count() > 10"

  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = false

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:${var.application_family}",
    "terraformed: true",
  ]
}