provider "aws" {
  region = var.aws_region
}

terraform {
  backend "s3" {
    bucket  = "orcd-terraform-state"
    key     = "prod/vector/datadog/terraform.tfstate"
    region  = "us-east-1"
    encrypt = "true"
  }
}

data "aws_caller_identity" "current" {}

data "datadog_role" "datadog_admin_role" {
  filter = "Datadog Admin Role"
}

resource "datadog_dashboard" "generate_encoding_orders_fargate_service_timeseries_dashboard" {
  title            = "${var.environment}-generate-encoding-orders timeseries dashboard"
  description      = "Terraformed timeseries dashboard for ${var.environment}-generate-encoding-orders workers"
  restricted_roles = [data.datadog_role.datadog_admin_role.id]
  layout_type      = "ordered"
  reflow_type      = "auto"

  widget {
    timeseries_definition {
      title         = "SQS - ${var.environment}-automated_encoding_order_upcs - number of visible messages"
      show_legend   = true
      legend_layout = "auto"

      request {
        q = "max:aws.sqs.approximate_number_of_messages_visible{queuename:${var.environment}-automated_encoding_order_upcs}"
      }
    }
  }

  widget {
    timeseries_definition {
      title         = "SQS - ${var.environment}-automated_encoding_order_upcs_fc - number of visible messages"
      show_legend   = true
      legend_layout = "auto"

      request {
        q = "max:aws.sqs.approximate_number_of_messages_visible{queuename:${var.environment}-automated_encoding_order_upcs_fc}"
      }
    }
  }

  widget {
    timeseries_definition {
      title         = "SQS - ${var.environment}-error_correction_upcs - number of visible messages"
      show_legend   = true
      legend_layout = "auto"

      request {
        q = "max:aws.sqs.approximate_number_of_messages_visible{queuename:${var.environment}-error_correction_upcs}"
      }
    }
  }

  widget {
    timeseries_definition {
      title         = "SQS - ${var.environment}-carveout_changed_upcs - number of visible messages"
      show_legend   = true
      legend_layout = "auto"

      request {
        q = "max:aws.sqs.approximate_number_of_messages_visible{queuename:${var.environment}-carveout_changed_upcs}"
      }
    }
  }

  widget {
    timeseries_definition {
      title         = "SQS - ${var.environment}-carveout_changed_upcs_fc - number of visible messages"
      show_legend   = true
      legend_layout = "auto"

      request {
        q = "max:aws.sqs.approximate_number_of_messages_visible{queuename:${var.environment}-carveout_changed_upcs_fc}"
      }
    }
  }

  widget {
    timeseries_definition {
      title         = "ECS - ${var.environment}-auto-encoding-order-upc - number of tasks"
      show_legend   = true
      legend_layout = "auto"

      request {
        q = "sum:aws.ecs.service.running{servicename:${var.environment}-auto-encoding-order-upc}"
      }

      request {
        q = "sum:aws.ecs.service.desired{servicename:${var.environment}-auto-encoding-order-upc}"
      }
    }
  }

  widget {
    timeseries_definition {
      title         = "ECS - ${var.environment}-auto-encoding-order-upc-fc - number of tasks"
      show_legend   = true
      legend_layout = "auto"

      request {
        q = "sum:aws.ecs.service.running{servicename:${var.environment}-auto-encoding-order-upc-fc}"
      }

      request {
        q = "sum:aws.ecs.service.desired{servicename:${var.environment}-auto-encoding-order-upc-fc}"
      }
    }
  }

  widget {
    timeseries_definition {
      title         = "ECS - ${var.environment}-error-correction-upc - number of tasks"
      show_legend   = true
      legend_layout = "auto"

      request {
        q = "sum:aws.ecs.service.running{servicename:${var.environment}-error-correction-upc}"
      }

      request {
        q = "sum:aws.ecs.service.desired{servicename:${var.environment}-error-correction-upc}"
      }
    }
  }

  widget {
    timeseries_definition {
      title         = "ECS - ${var.environment}-carveout-changed-upc - number of tasks"
      show_legend   = true
      legend_layout = "auto"

      request {
        q = "sum:aws.ecs.service.running{servicename:${var.environment}-carveout-changed-upc}"
      }

      request {
        q = "sum:aws.ecs.service.desired{servicename:${var.environment}-carveout-changed-upc}"
      }
    }
  }

  widget {
    timeseries_definition {
      title          = "ECS - ${var.environment}-carveout-changed-upc-fc - number of tasks"
      show_legend    = true
      legend_layout  = "auto"
      legend_columns = []

      request {
        q = "sum:aws.ecs.service.running{servicename:${var.environment}-carveout-changed-upc-fc}"
      }

      request {
        on_right_yaxis = false
        q              = "sum:aws.ecs.service.desired{servicename:${var.environment}-carveout-changed-upc-fc}"
      }
    }
  }
}

resource "datadog_monitor" "populate_encoding_order_monitor" {
  name    = "${var.environment}-vector-populate-encoding-order-sqs-monitor"
  type    = "metric alert"
  message = "SQS messages are not being sent to encoding order SQS queues by the populate encoding order script. Notify: ${var.notification_endpoints}"
  query   = "sum(last_12h):sum:aws.sqs.number_of_messages_sent{vector_worker_type:generate_encoding_orders}.as_count() <= 0"

  notify_no_data      = true
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = false
  no_data_timeframe   = 60
  restricted_roles    = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "0"
  }

  tags = [
    "environment:${var.environment}",
    "application_family:${var.application_family}",
    "terraformed:true",
    "vector_worker_type:populate_encoding_orders",
  ]
}

resource "datadog_monitor" "generate_encoding_order_monitor" {
  name    = "${var.environment}-vector-generate-encoding-order-sqs-monitor"
  type    = "metric alert"
  message = "SQS encoding order queues are not being consumed by the generate encoding order scripts. Notify: ${var.notification_endpoints}"
  query   = "sum(last_12h):sum:aws.sqs.number_of_messages_received{vector_worker_type:generate_encoding_orders}.as_count() <= 0"

  notify_no_data      = true
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = false
  no_data_timeframe   = 60
  restricted_roles    = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "0"
  }

  tags = [
    "environment:${var.environment}",
    "application_family:${var.application_family}",
    "terraformed:true",
    "vector_worker_type:generate_encoding_orders",
  ]
}

resource "datadog_monitor" "asset_errors_monitor" {
  name    = "${var.environment}-asset-errors-monitor"
  type    = "query alert"
  message = "The number of assets in error status is high. Notify: ${var.notification_endpoints}"
  query   = "avg(last_5m):sum:mysql.direct_delivery.assets.errors{host:prod-mysql-reportsdd01,!error_type:nosuchkey} > ${var.critical_number_asset_errors}"

  notify_no_data      = true
  notify_audit        = false
  renotify_interval   = 120
  timeout_h           = 0
  require_full_window = false
  no_data_timeframe   = 60
  restricted_roles    = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    warning  = var.warning_number_asset_errors
    critical = var.critical_number_asset_errors
  }

  tags = [
    "environment:${var.environment}",
    "application_family:${var.application_family}",
    "terraformed:true",
  ]
}

# Monitor for direct-delivery-update trace metric error rate
resource "datadog_monitor" "direct_delivery_update_trace_no_hits" {
  name    = "${var.environment}-direct-delivery-update-trace-no-hits"
  type    = "metric alert"
  message = <<-EOT
    {{#is_alert}}
    Direct Delivery Update did not execute as expected.
    {{/is_alert}}

    {{#is_recovery}}
    Direct Delivery Update execution frequency has recovered.
    {{/is_recovery}}

    ${var.notification_endpoints}
  EOT
  query   = "sum(last_30m):sum:trace.direct_delivery_update.php.hits{*}.rollup(sum, 1800) < 1"

  monitor_thresholds {
    critical = 1
  }

  notify_no_data      = true
  no_data_timeframe   = 40
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = false
  restricted_roles    = [data.datadog_role.datadog_admin_role.id]

  tags = [
    "service:direct-delivery-update",
    "environment:${var.environment}",
    "application_family:${var.application_family}",
    "terraformed:true",
  ]
}

# Direct Delivery Update Sync Lag Monitor
resource "datadog_monitor" "delivery_history_sync_lag_monitor" {
  name    = "${var.environment} delivery history sync lag monitor"
  type    = "metric alert"
  message = "Delivery history is failing to sync from direct_delivery.encoding_queue_detail to art_relations.delivery_history. Notify: ${var.notification_endpoints} {{#is_alert}}${var.notification_endpoints}{{/is_alert}} {{#is_recovery}}${var.notification_endpoints}{{/is_recovery}}"

  query = "avg(last_20m):max:snowflake.custom.query_monitor.direct_delivery_update_sync_lag{*} >= 30"

  monitor_thresholds {
    warning  = 15
    critical = 30
  }

  require_full_window = false

  renotify_interval = 60
  renotify_statuses = ["alert", "warn"]

  tags = [
    "environment:${var.environment}",
    "service_name:direct-delivery-update",
    "application_family:${var.application_family}",
  ]
}

# Gras Delivery SQS Queue Monitor
resource "datadog_monitor" "gras_delivery_not_running_monitor" {
  name    = "${var.environment} gras delivery not running monitor"
  type    = "metric alert"
  message = "GRAS delivery SQS queue has a message older than 3 hours. gras-delivery script may not be running. Notify: ${var.notification_endpoints} {{#is_alert}}${var.notification_endpoints}{{/is_alert}} {{#is_recovery}}${var.notification_endpoints}{{/is_recovery}}"

  query = "avg(last_90m):max:aws.sqs.approximate_age_of_oldest_message{queuename:${var.environment}-gras_delivery-queue} > 10800"

  monitor_thresholds {
    warning  = 7200  # 2 hours — 1 missed run
    critical = 10800 # 3 hours — 2 missed runs
  }

  require_full_window = false
  renotify_interval   = 60
  renotify_statuses   = ["alert", "warn"]

  tags = [
    "environment:${var.environment}",
    "service_name:gras-delivery",
    "application_family:${var.application_family}",
  ]
}

# CDAM-2185: per-store delivery-failure monitors.
locals {
  delivery_failure_metric_prefix = "snowflake.custom.query_monitor.vector_delivery"

  delivery_failure_monitor_stores = {
    "1"    = { name = "iTunes/Apple", min_failures = 5 }
    "187"  = { name = "Amazon Digital Services Inc.", min_failures = 5 }
    "286"  = { name = "Spotify", min_failures = 5 }
    "348"  = { name = "Deezer", min_failures = 5 }
    "399"  = { name = "TIDAL", min_failures = 5 }
    "446"  = { name = "VEVO", min_failures = 3 }
    "453"  = { name = "YouTube CID", min_failures = 5 }
    "569"  = { name = "YouTube ATMV", min_failures = 5 }
    "708"  = { name = "Pandora", min_failures = 5 }
    "1173" = { name = "Meta", min_failures = 5 }
    "1294" = { name = "TikTok", min_failures = 5 }
  }
}

resource "datadog_monitor" "delivery_failure_1h" {
  for_each = local.delivery_failure_monitor_stores

  name = "${var.environment} vector delivery failure (1h) - ${each.value.name} (dms_id:${each.key})"
  type = "query alert"

  # Fires only when a single snapshot has >= N new failures AND zero successes.
  # clamp_max(successes, 1) is a 0/1 "any successes" flag; (1 - flag) zeroes the
  # failure count the instant any success appears, so the volume and rate
  # conditions are bound to the same point rather than maxed independently.
  query = "max(last_1h):(max:${local.delivery_failure_metric_prefix}.new_failures_1h{dms_id:${each.key}} * (1 - clamp_max(max:${local.delivery_failure_metric_prefix}.successes_1h{dms_id:${each.key}}, 1))) >= ${each.value.min_failures}"

  message = <<-EOT
    {{#is_alert}}
    All new deliveries to ${each.value.name} (dms_id:${each.key}, encoders 17/18) are failing: 100% failure rate over the last 1h with at least ${each.value.min_failures} new failures.
    {{/is_alert}}

    {{#is_recovery}}
    Delivery to ${each.value.name} (dms_id:${each.key}) has recovered.
    {{/is_recovery}}

    ${var.notification_endpoints} ${var.ops_support_notification_endpoint}
  EOT

  monitor_thresholds {
    critical = each.value.min_failures
  }

  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 0
  timeout_h           = 0
  require_full_window = false
  restricted_roles    = [data.datadog_role.datadog_admin_role.id]

  tags = [
    "service:direct-delivery",
    "environment:${var.environment}",
    "application_family:${var.application_family}",
    "terraformed:true",
    "dms_id:${each.key}"
  ]
}

resource "datadog_monitor" "sound_recordings_takedown_success_monitor" {
  name    = "${var.environment}-sound-recordings-takedown-success-monitor"
  type    = "metric alert"
  message = "Sound Recordings Takedown has not published a success metric in the last 25h; the daily job may not be running. Notify: ${var.notification_endpoints}"
  query   = "sum(last_25h):sum:sound_recordings_takedown.success{*} < 1"

  monitor_thresholds {
    critical = 1
  }

  notify_no_data      = true
  renotify_interval   = 60 * 24
  require_full_window = false
  no_data_timeframe   = 0
  restricted_roles    = [data.datadog_role.datadog_admin_role.id]

  tags = [
    "environment:${var.environment}",
    "service_name:sound-recordings-takedown",
    "application_family:${var.application_family}",
    "terraformed:true",
  ]
}

resource "datadog_monitor" "sound_recordings_takedown_error_monitor" {
  name    = "${var.environment}-sound-recordings-takedown-error-monitor"
  type    = "metric alert"
  message = "Sound Recordings Takedown reported errors in the last 25h. Notify: ${var.notification_endpoints}"
  query   = "sum(last_25h):sum:sound_recordings_takedown.error{*} >= 1"

  monitor_thresholds {
    critical = 1
  }

  notify_no_data      = false
  renotify_interval   = 60
  require_full_window = false
  restricted_roles    = [data.datadog_role.datadog_admin_role.id]

  tags = [
    "environment:${var.environment}",
    "service_name:sound-recordings-takedown",
    "application_family:${var.application_family}",
    "terraformed:true",
  ]
}

resource "datadog_monitor" "delivery_failure_12h" {
  for_each = local.delivery_failure_monitor_stores

  name = "${var.environment} vector delivery failure (12h) - ${each.value.name} (dms_id:${each.key})"
  type = "query alert"

  # Fires only when a single snapshot has >= N new failures AND zero successes.
  # clamp_max(successes, 1) is a 0/1 "any successes" flag; (1 - flag) zeroes the
  # failure count the instant any success appears, so the volume and rate
  # conditions are bound to the same point rather than maxed independently.
  query = "max(last_1h):(max:${local.delivery_failure_metric_prefix}.new_failures_12h{dms_id:${each.key}} * (1 - clamp_max(max:${local.delivery_failure_metric_prefix}.successes_12h{dms_id:${each.key}}, 1))) >= ${each.value.min_failures}"

  message = <<-EOT
    {{#is_alert}}
    Every new delivery to ${each.value.name} (dms_id:${each.key}, encoders 17/18) over the last 12h has failed (>= ${each.value.min_failures}).
    {{/is_alert}}

    {{#is_recovery}}
    Deliveries to ${each.value.name} (dms_id:${each.key}) have recovered.
    {{/is_recovery}}

    ${var.notification_endpoints} ${var.ops_support_notification_endpoint}
  EOT

  monitor_thresholds {
    critical = each.value.min_failures
  }

  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 0
  timeout_h           = 0
  require_full_window = false
  restricted_roles    = [data.datadog_role.datadog_admin_role.id]

  tags = [
    "service:direct-delivery",
    "environment:${var.environment}",
    "application_family:${var.application_family}",
    "terraformed:true",
    "dms_id:${each.key}"
  ]
}