# Slack-only visibility monitors for VECTOR prod delivery workers that die/fail in ways a blind
# restart won't fix — they surface for a human to look at the failing release/config instead.
# Notification only; these deliberately omit the paging handle @oncall-content-delivery-asset-management
# (Datadog On-Call), so they do NOT page.
#
# ECS stopped-task events carry a `containers` array (worker + datadog-agent + fluentbit sidecars) that
# Datadog matches ELEMENT-WISE — it cannot tie an exit code to a specific container. On every normal
# teardown the non-essential sidecars are SIGKILLed to exit 137, so a bare `@containers.exitCode:>0` can't
# tell a benign sidecar 137 from a real worker crash, and a real worker 137 (OOM/SIGKILL) is likewise
# indistinguishable. The companion log pipeline (logs_pipeline.tf) breaks this by extracting the WORKER
# container's own exitCode/reason/name into scalar @worker.* attributes; the crash monitor keys on those
# (see its query comment for the primary + gated-fallback logic) and allowlists
# @stopCode:EssentialContainerExited — every self-inflicted worker death (OOM, timeout, PHP fatal) stops
# the task via that code, while operator/infra/scale stops (UserInitiated, SpotInterruption,
# TerminationNotice, ServiceSchedulerInitiated, TaskFailedToStart) are excluded. Events reach Datadog via
# the terraform-fargate stopped_task event rule (forwards events only — no duplicate monitor).

locals {
  # Same channel the other vector delivery monitors post to (see variables.tf healthy_task_monitors[].notification).
  cdam_alerts_slack = "@slack-content-delivery-and-asset-mgmt-alerts"

  # Base scope shared by the crash monitor + its pipeline-health canary (below) AND the worker-exit pipeline
  # (logs_pipeline.tf). Derived from one value on purpose: the pipeline must decorate every event the monitor
  # can match, or the monitor's gated fallback silently takes over. If you narrow one, narrow both.
  vector_stopped_task_scope = "source:eventbridge aws_service:ecs event:stopped_task environment:prod service_name:vector-delivery-*"
}

resource "datadog_monitor" "vector_delivery_unexpected_task_stop" {
  name = "[PROD] VECTOR delivery — worker task killed unexpectedly (OOM / crash)"
  type = "log alert"
  # These EventBridge events carry the service as the `service_name` tag (= vector-delivery-*), not the
  # reserved `service` attribute (env-prefixed) — so the query and the message's {{log.tags.service_name}}
  # both key off the tag. Monitor 3's app logs use `service`; each matches its source.
  #
  # PRIMARY: @worker.* are the WORKER container's own fields (isolated from the sidecars by logs_pipeline.tf).
  # `@worker.exit_code:>0` catches every worker crash INCLUDING exit 137 — the case a `@containers.*` array
  # query cannot separate from the benign sidecar 137. (`:>0` is correct whether the value is numeric or
  # string.) This clause is structurally free of sidecar false positives.
  #
  # GATED FALLBACK: the `@containers.*` clauses apply ONLY when @worker.* is absent (`-@worker.name:*`), i.e.
  # only if the pipeline hasn't decorated the event — then the monitor degrades to array-only detection
  # instead of going blind. It is gated, NOT OR'd flat, on purpose: an ungated fallback is always-on and
  # would re-admit the very sidecar FPs the pipeline removes (a mid-task sidecar OOM setting a container
  # reason; or a sidecar that starts exiting 143/139 instead of 137). `[1 TO 136] OR >137` = "non-zero but
  # not 137" (positive range, not `-exitCode:137`, so array-wide negation can't drop a real crash when a
  # sidecar also exits 137). When the pipeline is healthy this fallback is inert; the canary monitor below
  # alerts if @worker.* ever stops being populated.
  #
  # @stopCode:EssentialContainerExited allowlist: every real worker death carries it (verified: all 6 crash
  # events + the 06-24 OOMs). It excludes operator/infra stops (UserInitiated/SpotInterruption/
  # TerminationNotice) that would otherwise false-fire "OOM/crash" on a worker SIGKILLed to 137 during a
  # manual/spot stop, and scale/deploy stops (ServiceSchedulerInitiated/TaskFailedToStart). A worker that
  # crashes *during* an externally-initiated stop is attributed to that stopCode and suppressed (rare; the
  # defect recurs on the next task). Task-never-starts (TaskFailedToStart) is covered by healthy_task_monitors.
  query = <<EOT
logs("${local.vector_stopped_task_scope} @stopCode:EssentialContainerExited (@worker.exit_code:>0 OR @worker.reason:*OutOfMemory* OR (-@worker.name:* AND (@containers.exitCode:[1 TO 136] OR @containers.exitCode:>137 OR @containers.reason:*OutOfMemory*)))").index("*").rollup("count").last("15m") >= 1
EOT
  message = <<EOT
{{#is_alert}}
A VECTOR **prod delivery** worker task stopped unexpectedly (OOM-killed or non-zero exit): {{log.tags.service_name}}.
This is usually a package/config-level condition that will **recur on retry** — investigate the failing release rather than blindly restarting. Stuck jobs surface on the "Content VECTOR Stuck Jobs" Sigma board.
{{/is_alert}}
${local.cdam_alerts_slack}
EOT
  tags                = ["team:cdam", "service:vector-delivery", "managed_by:terraform"]
  draft_status        = "published"
  include_tags        = true
  on_missing_data     = "default"
  require_full_window = false
  monitor_thresholds {
    critical = 1
  }
}

# Canary: detects silent failure of the worker-exit pipeline (logs_pipeline.tf). When healthy, every
# EssentialContainerExited teardown (~60/day) is decorated with @worker.name. If the pipeline stops
# (disabled, reordered behind a mutating pipeline, its select filter no longer matching the worker container
# name, or the logs_write_pipelines permission lost), events arrive WITHOUT @worker.name and the crash
# monitor above silently drops to its array-only fallback — which can miss a worker OOM at exit 137. This
# makes that observable. Expect it to fire for up to its 4h window right after the FIRST apply (events from
# before the pipeline existed carry no @worker.name) and then self-resolve.
resource "datadog_monitor" "vector_delivery_worker_exit_pipeline_health" {
  name = "[PROD] VECTOR delivery — worker-exit pipeline not decorating events"
  type = "log alert"
  query = <<EOT
logs("${local.vector_stopped_task_scope} @stopCode:EssentialContainerExited -@worker.name:*").index("*").rollup("count").last("4h") >= 3
EOT
  message = <<EOT
{{#is_alert}}
The worker-exit log pipeline (logs_pipeline.tf) has stopped populating @worker.name on VECTOR delivery stopped-task events, so the "worker task killed unexpectedly" monitor is now on its degraded array-only fallback (which can miss a worker OOM at exit 137). Check the pipeline is enabled and ordered, that its `select` filter still matches the worker container `name` (vector-delivery-*), and that the Datadog key retains logs_write_pipelines.
{{/is_alert}}
${local.cdam_alerts_slack}
EOT
  tags                = ["team:cdam", "service:vector-delivery", "managed_by:terraform"]
  draft_status        = "published"
  include_tags        = true
  on_missing_data     = "default"
  require_full_window = false
  monitor_thresholds {
    critical = 3
  }
}

resource "datadog_monitor" "vector_delivery_out_of_storage" {
  name = "[PROD] VECTOR delivery — out of storage (No space left on device)"
  type = "log alert"
  # Worker application logs (not EventBridge) key the service as `service` (= vector-delivery-*) —
  # see monitor 1 above for why this differs from its `service_name`.
  #
  # Out-of-storage surfaces as a local-write errno at up to three points; matching all three keeps this both
  # low-FP and low-FN for the delivery worker:
  #   - "No space left on device" — kernel ENOSPC from TempStorage mkdir/touch (createDir/createEfsDir).
  #     This is where OOS actually trips on this fleet (every occurrence in 120d). retryableCreate retries
  #     3x over ~2s and a full disk doesn't free in 2s, so the warning always coincides with the terminal
  #     failure — the "recovered-transient false positive" is mechanically implausible and unobserved.
  #   - "cURL error 23" (CURLE_WRITE_ERROR) / "Unable to write to stream" — how OOS would surface if the disk
  #     fills while streaming a large S3 getObject(SaveAs)/GCS downloadToFile to the sink, after temp dirs are
  #     created. 0 occurrences in 120d (OOS trips at mkdir first), so they add no FP and close the download-
  #     stage blind spot. Both are LOCAL-write failures here, not network (network cURL codes are 6/7/28/56).
  # Residual FN (out of scope): OOS reported only via an opaque subprocess message (ffmpeg/iTMSTransporter)
  # runs on the encoder (vector-encoder), not this delivery worker.
  query = <<EOT
logs("environment:prod service:vector-delivery-* (\"No space left on device\" OR \"cURL error 23\" OR \"Unable to write to stream\")").index("*").rollup("count").last("15m") >= 1
EOT
  message = <<EOT
{{#is_alert}}
A VECTOR **prod delivery** worker ran out of ephemeral disk (No space left on device): {{log.service}}.
The staged package + baseline exceeded the local disk. This is a sizing/config condition (compare the package size to USE_EFS_FOR_PACKAGES_LARGER_THAN_GIB), not a transient failure — retrying alone will recur.
{{/is_alert}}
${local.cdam_alerts_slack}
EOT
  tags                = ["team:cdam", "service:vector-delivery", "managed_by:terraform"]
  draft_status        = "published"
  include_tags        = true
  on_missing_data     = "default"
  require_full_window = false
  monitor_thresholds {
    critical = 1
  }
}