######################################################
## Monitoring and Alerting for File Upload Workflow ##
######################################################

#######################
## Dead Letter Queue ##
#######################

# SQS Dead Letter Queue for failed Step Functions executions
resource "aws_sqs_queue" "abacus_file_upload_workflow_dlq" {
  name                       = "${var.environment}-abacus-file-upload-workflow-dlq"
  message_retention_seconds  = 1209600 # 14 days
  visibility_timeout_seconds = 300
  sqs_managed_sse_enabled    = true # Enable encryption at rest

  tags = {
    application_family = var.application_family
    environment        = var.environment
    service_name       = var.service_name
    terraformed        = true
  }
}

# SQS Queue Policy to allow EventBridge to send messages
resource "aws_sqs_queue_policy" "abacus_file_upload_workflow_dlq_policy" {
  queue_url = aws_sqs_queue.abacus_file_upload_workflow_dlq.id

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        Principal = {
          Service = "events.amazonaws.com"
        }
        Action   = "sqs:SendMessage"
        Resource = aws_sqs_queue.abacus_file_upload_workflow_dlq.arn
        Condition = {
          ArnEquals = {
            "aws:SourceArn" = aws_cloudwatch_event_rule.abacus_file_upload_workflow_failures.arn
          }
        }
      }
    ]
  })
}

# IAM policy to allow users to purge/delete messages from the dlq
data "aws_iam_group" "accounting" {
  group_name = "accounting"
}

data "aws_iam_policy_document" "abacus_file_upload_workflow_dlq_iam_policy" {
  statement {
    effect = "Allow"

    actions = [
      "sqs:DeleteMessage",
      "sqs:GetQueue*",
      "sqs:PurgeQueue",
      "sqs:ReceiveMessage",
      "sqs:SendMessage",
    ]

    resources = [
      aws_sqs_queue.abacus_file_upload_workflow_dlq.arn
    ]
  }
}

resource "aws_iam_policy" "abacus_file_upload_workflow_dlq_policy" {
  name   = "SQS-${var.environment}-abacus-file-upload-workflow-dlq-policy"
  description = "Grants accounting team access to monitor and manage the file upload workflow DLQ"
  policy = data.aws_iam_policy_document.abacus_file_upload_workflow_dlq_iam_policy.json

  tags = {
    application_family = var.application_family
    environment        = var.environment
    service_name       = var.service_name
    terraformed        = true
  }
}

resource "aws_iam_group_policy_attachment" "abacus_file_upload_workflow_dlq_access" {
  group      = data.aws_iam_group.accounting.group_name
  policy_arn = aws_iam_policy.abacus_file_upload_workflow_dlq_policy.arn
}


##############################
## EventBridge Failure Rule ##
##############################

# EventBridge rule to capture failed Step Functions executions
resource "aws_cloudwatch_event_rule" "abacus_file_upload_workflow_failures" {
  name        = "${var.environment}-abacus-file-upload-workflow-failures"
  description = "Capture failed Step Functions executions for file upload workflow"

  event_pattern = jsonencode({
    source      = ["aws.states"]
    detail-type = ["Step Functions Execution Status Change"]
    detail = {
      status          = ["FAILED", "TIMED_OUT", "ABORTED"]
      stateMachineArn = [aws_sfn_state_machine.abacus_file_upload_workflow.arn]
    }
  })

  tags = {
    application_family = var.application_family
    environment        = var.environment
    service_name       = var.service_name
    terraformed        = true
  }
}

# EventBridge target to send failed executions to DLQ
resource "aws_cloudwatch_event_target" "abacus_file_upload_workflow_dlq_target" {
  rule      = aws_cloudwatch_event_rule.abacus_file_upload_workflow_failures.name
  target_id = "SendToSQSDLQ"
  arn       = aws_sqs_queue.abacus_file_upload_workflow_dlq.arn
}

######################
## Datadog Monitors ##
######################

# Monitor Step Functions execution failures directly
resource "datadog_monitor" "abacus_file_upload_workflow_execution_failures" {
  name    = "${var.environment}-abacus-file-upload-workflow-execution-failures"
  type    = "query alert"
  message = "Step Functions execution failures detected for file upload workflow. Notify: ${var.notification_endpoints}"

  query = "sum(last_5m):sum:aws.states.execution_failed{statemachinename:${var.environment}-abacus-file-upload-workflow}.as_count() >= 3"

  monitor_thresholds {
    warning  = 1
    critical = 3
  }

  evaluation_delay     = 30
  include_tags         = true
  notify_no_data       = false
  notify_audit         = false
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  timeout_h            = 0

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:${var.application_family}",
    "team:abacus-${var.environment}",
  ]
}

# Monitor Dead Letter Queue for failed workflow invocations and executions
resource "datadog_monitor" "abacus_file_upload_workflow_dlq_monitor" {
  name    = "${var.environment}-abacus-file-upload-workflow-dlq-alert"
  type    = "query alert"
  message = "Messages detected in file upload workflow DLQ. Notify: ${var.notification_endpoints}"

  query = "avg(last_5m):avg:aws.sqs.approximate_number_of_messages_visible{queuename:${var.environment}-abacus-file-upload-workflow-dlq} >= 3"

  monitor_thresholds {
    warning  = 1
    critical = 3
  }

  evaluation_delay     = 30
  include_tags         = true
  notify_no_data       = false
  notify_audit         = false
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  timeout_h            = 0

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:${var.application_family}",
    "team:abacus-${var.environment}",
  ]
}