##################################################################
## Monitoring and Alerting for Adjustment File Apply Workflow  ##
##################################################################

resource "aws_sqs_queue" "adjustment_file_apply_workflow_dlq" {
  name                       = "${var.environment}-adjustment-file-apply-workflow-dlq"
  message_retention_seconds  = 1209600 # 14 days
  visibility_timeout_seconds = 300
  sqs_managed_sse_enabled    = true

  tags = {
    application_family = var.application_family
    environment        = var.environment
    service_name       = var.service_name
    terraformed        = true
  }
}

resource "aws_sqs_queue_policy" "adjustment_file_apply_workflow_dlq_policy" {
  queue_url = aws_sqs_queue.adjustment_file_apply_workflow_dlq.id

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        Principal = {
          Service = "events.amazonaws.com"
        }
        Action   = "sqs:SendMessage"
        Resource = aws_sqs_queue.adjustment_file_apply_workflow_dlq.arn
        # Both rules dead-letter to this queue: the trigger target (delivery
        # failures starting the SFN) and the failures rule (SFN execution failures).
        Condition = {
          ArnEquals = {
            "aws:SourceArn" = [
              aws_cloudwatch_event_rule.adjustment_file_apply_trigger.arn,
              aws_cloudwatch_event_rule.adjustment_file_apply_workflow_failures.arn
            ]
          }
        }
      }
    ]
  })
}

data "aws_iam_policy_document" "adjustment_file_apply_workflow_dlq_iam_policy" {
  statement {
    effect = "Allow"
    actions = [
      "sqs:DeleteMessage",
      "sqs:GetQueue*",
      "sqs:PurgeQueue",
      "sqs:ReceiveMessage",
      "sqs:SendMessage",
    ]
    resources = [
      aws_sqs_queue.adjustment_file_apply_workflow_dlq.arn
    ]
  }
}

resource "aws_iam_policy" "adjustment_file_apply_workflow_dlq_policy" {
  name        = "${var.environment}-adjustment-file-apply-workflow-dlq-policy"
  description = "Grants accounting team access to monitor and manage the adjustment file apply workflow DLQ"
  policy      = data.aws_iam_policy_document.adjustment_file_apply_workflow_dlq_iam_policy.json

  tags = {
    application_family = var.application_family
    environment        = var.environment
    service_name       = var.service_name
    terraformed        = true
  }
}

resource "aws_iam_role_policy_attachment" "adjustment_file_apply_workflow_dlq_access" {
  policy_arn = aws_iam_policy.adjustment_file_apply_workflow_dlq_policy.arn
  role       = data.aws_iam_role.accounting_role.name
}

resource "aws_cloudwatch_event_rule" "adjustment_file_apply_workflow_failures" {
  name        = "${var.environment}-adjustment-file-apply-workflow-failures"
  description = "Capture failed Step Functions executions for adjustment file apply workflow"

  event_pattern = jsonencode({
    source      = ["aws.states"]
    detail-type = ["Step Functions Execution Status Change"]
    detail = {
      status          = ["FAILED", "TIMED_OUT", "ABORTED"]
      stateMachineArn = [aws_sfn_state_machine.adjustment_file_apply_workflow.arn]
    }
  })

  tags = {
    application_family = var.application_family
    environment        = var.environment
    service_name       = var.service_name
    terraformed        = true
  }
}

resource "aws_cloudwatch_event_target" "adjustment_file_apply_workflow_dlq_target" {
  rule      = aws_cloudwatch_event_rule.adjustment_file_apply_workflow_failures.name
  target_id = "SendToSQSDLQ"
  arn       = aws_sqs_queue.adjustment_file_apply_workflow_dlq.arn
}

resource "datadog_monitor" "adjustment_file_apply_workflow_execution_failures" {
  name    = "${var.environment}-adjustment-file-apply-workflow-execution-failures"
  type    = "query alert"
  message = "Step Functions execution failures detected for adjustment file apply workflow. Notify: ${var.notification_endpoints}"

  query = "sum(last_5m):sum:aws.states.execution_failed{statemachinename:${var.environment}-adjustment-file-apply-workflow}.as_count() >= 3"

  monitor_thresholds {
    warning  = 1
    critical = 3
  }

  evaluation_delay     = 30
  include_tags         = true
  notify_no_data       = false
  notify_audit         = false
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  timeout_h            = 0

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:${var.application_family}",
    "team:abacus-${var.environment}",
  ]
}

resource "datadog_monitor" "adjustment_file_apply_workflow_dlq_monitor" {
  name    = "${var.environment}-adjustment-file-apply-workflow-dlq-alert"
  type    = "query alert"
  message = "Messages detected in adjustment file apply workflow DLQ. Notify: ${var.notification_endpoints}"

  query = "avg(last_5m):avg:aws.sqs.approximate_number_of_messages_visible{queuename:${var.environment}-adjustment-file-apply-workflow-dlq} >= 3"

  monitor_thresholds {
    warning  = 1
    critical = 3
  }

  evaluation_delay     = 30
  include_tags         = true
  notify_no_data       = false
  notify_audit         = false
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  timeout_h            = 0

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:${var.application_family}",
    "team:abacus-${var.environment}",
  ]
}

resource "datadog_monitor" "adjustment_file_apply_workflow_duration" {
  name    = "${var.environment}-adjustment-file-apply-workflow-duration-alert"
  type    = "query alert"
  message = "Adjustment file apply workflow execution duration exceeded threshold. Notify: ${var.notification_endpoints}"

  query = "avg(last_10m):avg:aws.states.execution_time{statemachinename:${var.environment}-adjustment-file-apply-workflow} >= 900000"

  monitor_thresholds {
    warning  = 600000
    critical = 900000
  }

  evaluation_delay     = 30
  include_tags         = true
  notify_no_data       = false
  notify_audit         = false
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  timeout_h            = 0

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:${var.application_family}",
    "team:abacus-${var.environment}",
  ]
}