data "aws_caller_identity" "current" {
}

# Datadog ASG Timeboard
resource "datadog_dashboard" "autoscaling_group_timeboard" {
  title        = "${var.environment}-${var.service_name} dashboard"
  description  = "Default terraformed dashboard for ${var.environment}-${var.service_name}"
  layout_type  = "ordered"

  template_variable {
    defaults = [var.environment]
    name     = var.service_name
    prefix   = "environment"
  }

  widget {
    timeseries_definition {
      title       = "Application 2xx Requests"
      show_legend = true

      request {
        q = "sum:aws.elb.httpcode_backend_2xx{loadbalancername:${var.load_balancer_service_name}, environment:${var.environment}}.as_count()"
      }
    }
  }

  widget {
    timeseries_definition {
      title       = "Application 3xx Requests"
      show_legend = true

      request {
        q = "sum:aws.elb.httpcode_backend_3xx{loadbalancername:${var.load_balancer_service_name}, environment:${var.environment}}.as_count()"
      }
    }
  }

  widget {
    timeseries_definition {
      title       = "Application 4xx Requests"
      show_legend = true

      request {
        q = "sum:aws.elb.httpcode_backend_4xx{loadbalancername:${var.load_balancer_service_name}, environment:${var.environment}}.as_count()"
      }
    }
  }

  widget {
    timeseries_definition {
      title       = "Application 5xx Requests"
      show_legend = true

      request {
        q = "sum:aws.elb.httpcode_backend_5xx{loadbalancername:${var.load_balancer_service_name}, environment:${var.environment}}.as_count()"
      }
    }
  }

  widget {
    timeseries_definition {
      title       = "Total Application Requests"
      show_legend = true

      request {
        q = "sum:aws.elb.request_count{loadbalancername:${var.load_balancer_service_name}, environment:${var.environment}}.as_count()"
      }
    }
  }

  widget {
    timeseries_definition {
      title       = "95th Percentile Request Time (from LB to target)"
      show_legend = true

      request {
        q = "avg:aws.elb.latency.p95{loadbalancername:${var.load_balancer_service_name}, environment:${var.environment}}"
      }
    }
  }

  widget {
    timeseries_definition {
      title       = "99th Percentile Request Time (from LB to target)"
      show_legend = true

      request {
        q = "avg:aws.elb.latency.p99{loadbalancername:${var.load_balancer_service_name}, environment:${var.environment}}"
      }
    }
  }

  widget {
    timeseries_definition {
      title       = "Average Request Time (from LB to target)"
      show_legend = true

      request {
        q = "avg:aws.elb.latency{loadbalancername:${var.load_balancer_service_name}, environment:${var.environment}}"
      }
    }
  }

  widget {
    timeseries_definition {
      title       = "Healthy Host Count"
      show_legend = true

      request {
        q = "sum:aws.elb.healthy_host_count_deduped{loadbalancername:${var.load_balancer_service_name}, environment:${var.environment}}"
      }
    }
  }

  # When running instances in an autoscaling group, it is expected that they will run the datadog agent themselves.
  widget {
    timeseries_definition {
      title       = "Load Average (1 min)"
      show_legend = true

      request {
        q = "avg:system.load.1{service_name:${var.service_name}, environment:${var.environment}} by {host}"
      }
    }
  }

  widget {
    timeseries_definition {
      title       = "Load Average (5 min)"
      show_legend = true

      request {
        q = "avg:system.load.5{service_name:${var.service_name}, environment:${var.environment}} by {host}"
      }
    }
  }

  widget {
    timeseries_definition {
      title       = "CPU Utilization"
      show_legend = true

      request {
        q = "max:system.cpu.system{service_name:${var.service_name}, environment:${var.environment}} by {host}"
      }

      request {
        q = "max:system.cpu.user{service_name:${var.service_name}, environment:${var.environment}} by {host}"
      }
    }
  }

  widget {
    timeseries_definition {
      title       = "Disk Utilization"
      show_legend = true

      request {
        q = "max:system.disk.used{service_name:${var.service_name}, environment:${var.environment}} by {host}"
      }

      request {
        q = "max:system.disk.total{service_name:${var.service_name}, environment:${var.environment}} by {host}"
      }
    }
  }

  widget {
    timeseries_definition {
      title       = "Network In"
      show_legend = true

      request {
        q = "max:system.net.bytes_rcvd{service_name:${var.service_name}, environment:${var.environment}} by {host}"
      }
    }
  }

  widget {
    timeseries_definition {
      title       = "Network Out"
      show_legend = true

      request {
        q = "max:system.net.bytes_sent{service_name:${var.service_name}, environment:${var.environment}} by {host}"
      }
    }
  }

  tags = local.dashboard_tags
}

resource "datadog_monitor" "healthy_instance_monitor" {
  count              = var.healthy_tasks_monitor_enabled ? 1 : 0
  name               = "${var.environment}-${var.service_name} healthy instance monitor"
  type               = "metric alert"
  message            = "Monitor triggered. Notify: ${var.notification_endpoints}"
  escalation_message = "Escalation to ${var.escalation_notification_endpoints}"
  query              = "sum(last_15m):sum:aws.elb.healthy_host_count_deduped{loadbalancername:${var.load_balancer_service_name}, environment:${var.environment}} < ${var.healthy_tasks_critical_number}"

  monitor_thresholds {
    ok                = var.healthy_tasks_ok_number
    warning           = var.healthy_tasks_warning_number
    warning_recovery  = var.healthy_tasks_warning_recovery_number
    critical          = var.healthy_tasks_critical_number
    critical_recovery = var.healthy_tasks_critical_recovery_number
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = false

  tags = local.tags
}

resource "datadog_monitor" "service_cpu_monitor" {
  count              = var.service_cpu_monitor_enabled ? 1 : 0
  name               = "${var.environment}-${var.service_name} cpu monitor"
  type               = "metric alert"
  message            = "Monitor triggered. Notify: ${var.notification_endpoints}"
  escalation_message = "Escalation to ${var.escalation_notification_endpoints}"
  query              = "max(last_5m):avg:system.cpu.user{service_name:${var.service_name}, environment:${var.environment}} + avg:system.cpu.system{service_name:${var.service_name}, environment:${var.environment}} > ${var.service_cpu_critical_number}"

  monitor_thresholds {
    ok                = var.service_cpu_ok_number
    warning           = var.service_cpu_warning_number
    warning_recovery  = var.service_cpu_warning_recovery_number
    critical          = var.service_cpu_critical_number
    critical_recovery = var.service_cpu_critical_recovery_number
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = false

  tags = local.tags
}

resource "datadog_monitor" "service_4xx_monitor" {
  count              = var.service_4xx_monitor_enabled ? 1 : 0
  name               = "${var.environment}-${var.service_name} 4xx requests monitor"
  type               = "metric alert"
  message            = "Monitor triggered. Notify: ${var.notification_endpoints}"
  escalation_message = "Escalation to ${var.escalation_notification_endpoints}"
  query              = "max(last_5m):sum:aws.elb.httpcode_backend_4xx{loadbalancername:${var.load_balancer_service_name}, environment:${var.environment}} > ${var.critical_number_4xx}"

  monitor_thresholds {
    ok                = var.ok_number_4xx
    warning           = var.warning_number_4xx
    warning_recovery  = var.warning_recovery_number_4xx
    critical          = var.critical_number_4xx
    critical_recovery = var.critical_recovery_number_4xx
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = false

  tags = local.tags
}

resource "datadog_monitor" "service_5xx_monitor" {
  count              = var.service_5xx_monitor_enabled ? 1 : 0
  name               = "${var.environment}-${var.service_name} 5xx requests monitor"
  type               = "metric alert"
  message            = "Monitor triggered. Notify: ${var.notification_endpoints}"
  escalation_message = "Escalation to ${var.escalation_notification_endpoints}"
  query              = "max(last_5m):sum:aws.elb.httpcode_backend_5xx{loadbalancername:${var.load_balancer_service_name}, environment:${var.environment}} > ${var.critical_number_5xx}"

  monitor_thresholds {
    ok                = var.ok_number_5xx
    warning           = var.warning_number_5xx
    warning_recovery  = var.warning_recovery_number_5xx
    critical          = var.critical_number_5xx
    critical_recovery = var.critical_recovery_number_5xx
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = false

  tags = local.tags
}