data "aws_caller_identity" "current" {
}

# Datadog HAProxy timeboard
resource "datadog_dashboard" "datadog_haproxy" {
  title        = "${var.environment}-${var.cluster_name} HAProxy dashboard"
  description  = "Default terraformed dashboard for HAProxy"
  layout_type  = "ordered"

  template_variable {
    defaults = [var.environment]
    name     = var.cluster_name
    prefix   = "environment"
  }

  # Percentage of frontend requests which are http/2xx
  widget {
    query_value_definition {
      title     = "Front 2xx %"
      autoscale = true
      live_span = "4h"

      request {
        q = "(avg:haproxy.frontend.response.2xx{haproxy_cluster:${var.cluster_name}}/(avg:haproxy.frontend.response.3xx{haproxy_cluster:${var.cluster_name}}+avg:haproxy.frontend.response.4xx{haproxy_cluster:${var.cluster_name}}+avg:haproxy.frontend.response.2xx{haproxy_cluster:${var.cluster_name}}+avg:haproxy.frontend.response.5xx{haproxy_cluster:${var.cluster_name}}))*100"

        conditional_formats {
          palette    = "white_on_red"
          comparator = "<"
          value      = "80"
        }
      }
    }
  }

  # Percentage of backend requests which are http/2xx
  widget {
    query_value_definition {
      title     = "Back 2xx %"
      autoscale = true
      live_span = "4h"

      request {
        q = "(avg:haproxy.backend.response.2xx{haproxy_cluster:${var.cluster_name}}/(avg:haproxy.backend.response.3xx{haproxy_cluster:${var.cluster_name}}+avg:haproxy.backend.response.4xx{haproxy_cluster:${var.cluster_name}}+avg:haproxy.backend.response.2xx{haproxy_cluster:${var.cluster_name}}+avg:haproxy.backend.response.5xx{haproxy_cluster:${var.cluster_name}}))*100"

        conditional_formats {
          palette    = "white_on_red"
          comparator = "<"
          value      = "80"
        }
      }
    }
  }

  # Slowest 5 backends
  widget {
    toplist_definition {
      title = "Slowest 5 Backends"

      request {
        q = "top(avg:haproxy.backend.response.time{haproxy_cluster:${var.cluster_name}} by {backend}, 5, 'last', 'desc')"

        conditional_formats {
          palette    = "white_on_red"
          comparator = ">"
          value      = "2000"
        }

        conditional_formats {
          palette    = "white_on_yellow"
          comparator = ">"
          value      = "1000"
        }

        conditional_formats {
          palette    = "white_on_green"
          comparator = "<="
          value      = "1000"
        }
      }
    }
  }

  # Average Backend Response Time by Host (ms)
  widget {
    timeseries_definition {
      title       = "Average Backend Response Time by Host (ms)"
      show_legend = true

      request {
        q            = "avg:haproxy.backend.response.time{haproxy_cluster:${var.cluster_name}} by {host}"
        display_type = "line"
      }

      marker {
        display_type = "error dashed"
        value        = "y > 500"
      }

      marker {
        display_type = "ok dashed"
        value        = "y < 350"
      }
    }
  }

  # Frontend 2xx http status codes
  widget {
    timeseries_definition {
      title       = "Frontend 2xx"
      show_legend = true

      request {
        q            = "sum:haproxy.frontend.response.2xx{haproxy_cluster:${var.cluster_name}} by {host}"
        display_type = "line"
      }
    }
  }

  # Frontend 3xx http status codes
  widget {
    timeseries_definition {
      title       = "Frontend 3xx"
      show_legend = true

      request {
        q            = "sum:haproxy.frontend.response.3xx{haproxy_cluster:${var.cluster_name}} by {host}"
        display_type = "line"
      }
    }
  }

  # Frontend 4xx http status codes
  widget {
    timeseries_definition {
      title       = "Frontend 4xx"
      show_legend = true

      request {
        q            = "sum:haproxy.frontend.response.4xx{haproxy_cluster:${var.cluster_name}} by {host}"
        display_type = "line"
      }
    }
  }

  # Frontend 5xx http status codes
  widget {
    timeseries_definition {
      title       = "Frontend 5xx"
      show_legend = true

      request {
        q            = "sum:haproxy.frontend.response.5xx{haproxy_cluster:${var.cluster_name}} by {host}"
        display_type = "line"
      }
    }
  }

  # Frontend Sessions
  widget {
    timeseries_definition {
      title       = "Frontend Sessions"
      show_legend = true

      request {
        q            = "sum:haproxy.frontend.session.current{haproxy_cluster:${var.cluster_name}}"
        display_type = "line"

        style {
          palette = "purple"
        }
      }

      request {
        q            = "sum:haproxy.frontend.session.limit{haproxy_cluster:${var.cluster_name}}"
        display_type = "line"

        style {
          palette = "warm"
        }
      }
    }
  }

  # Frontend Session rate graph
  widget {
    timeseries_definition {
      title       = "Frontend Sessions/second"
      show_legend = true

      request {
        q            = "sum:haproxy.frontend.session.rate{haproxy_cluster:${var.cluster_name}}"
        display_type = "line"
      }
    }
  }

  # Frontend Network Traffic Graph
  widget {
    timeseries_definition {
      title       = "Frontend Network Traffic"
      show_legend = true

      request {
        q            = "sum:haproxy.frontend.bytes.in_rate{haproxy_cluster:${var.cluster_name}}"
        display_type = "line"

        style {
          palette = "dog_classic"
        }
      }

      request {
        q = "sum:haproxy.frontend.bytes.out_rate{haproxy_cluster:${var.cluster_name}}"

        style {
          palette = "dog_classic"
        }
      }
    }
  }

  # Frontend request rate graph
  widget {
    timeseries_definition {
      title       = "Frontend Requests/second"
      show_legend = true

      request {
        q            = "sum:haproxy.frontend.requests.rate{haproxy_cluster:${var.cluster_name}}"
        display_type = "line"
      }
    }
  }

  # Backend 2xx http status codes
  widget {
    timeseries_definition {
      title       = "Backend 2xx"
      show_legend = true

      request {
        q            = "sum:haproxy.backend.response.2xx{haproxy_cluster:${var.cluster_name}} by {host}"
        display_type = "line"
      }
    }
  }

  # Backend 3xx http status codes
  widget {
    timeseries_definition {
      title       = "Backend 3xx"
      show_legend = true

      request {
        q            = "sum:haproxy.backend.response.3xx{haproxy_cluster:${var.cluster_name}} by {host}"
        display_type = "line"
      }
    }
  }

  # Backend 4xx http status codes
  widget {
    timeseries_definition {
      title       = "Backend 4xx"
      show_legend = true

      request {
        q            = "sum:haproxy.backend.response.4xx{haproxy_cluster:${var.cluster_name}} by {host}"
        display_type = "line"
      }
    }
  }

  # Backend 5xx http status codes
  widget {
    timeseries_definition {
      title       = "Backend 5xx"
      show_legend = true

      request {
        q            = "sum:haproxy.backend.response.5xx{haproxy_cluster:${var.cluster_name}} by {host}"
        display_type = "line"
      }
    }
  }

  # Backend Sessions
  widget {
    timeseries_definition {
      title       = "Backend Sessions"
      show_legend = true

      request {
        q            = "sum:haproxy.backend.session.current{haproxy_cluster:${var.cluster_name}}"
        display_type = "line"

        style {
          palette = "purple"
        }
      }

      request {
        q            = "sum:haproxy.backend.session.limit{haproxy_cluster:${var.cluster_name}}"
        display_type = "line"

        style {
          palette = "warm"
        }
      }
    }
  }

  # Backend Session rate graph
  widget {
    timeseries_definition {
      title       = "Backend Sessions/second"
      show_legend = true

      request {
        q            = "sum:haproxy.backend.session.rate{haproxy_cluster:${var.cluster_name}}"
        display_type = "line"
      }
    }
  }

  # Backend Network Traffic Graph
  widget {
    timeseries_definition {
      title       = "Backend Network Traffic"
      show_legend = true

      request {
        q            = "sum:haproxy.backend.bytes.in_rate{haproxy_cluster:${var.cluster_name}}"
        display_type = "line"

        style {
          palette = "dog_classic"
        }
      }

      request {
        q            = "sum:haproxy.backend.bytes.out_rate{haproxy_cluster:${var.cluster_name}}"
        display_type = "line"

        style {
          palette = "dog_classic"
        }
      }
    }
  }

  # Backend Response Time by backend
  widget {
    timeseries_definition {
      title       = "Backend Response Times"
      show_legend = true

      request {
        q            = "avg:haproxy.backend.response.time{haproxy_cluster:${var.cluster_name}} by {backend}"
        display_type = "line"
      }
    }
  }

  #System Metrics Area

  # System Load graph
  widget {
    timeseries_definition {
      title       = "System Load Averages"
      show_legend = true

      request {
        q            = "avg:system.load.1{haproxy_cluster:${var.cluster_name}} by {host}, avg:system.load.5{haproxy_cluster:${var.cluster_name}} by {host}, avg:system.load.15{haproxy_cluster:${var.cluster_name}} by {host}"
        display_type = "line"

        style {
          palette = "cool"
        }
      }
    }
  }
  # System CPU graph
  widget {
    timeseries_definition {
      title       = "System CPU"
      show_legend = true

      request {
        q            = "avg:system.cpu.user{haproxy_cluster:${var.cluster_name}} by {host}"
        display_type = "line"
      }

      request {
        q            = "avg:system.cpu.guest{haproxy_cluster:${var.cluster_name}} by {host}"
        display_type = "line"
      }

      request {
        q            = "avg:system.cpu.iowait{haproxy_cluster:${var.cluster_name}} by {host}"
        display_type = "line"
      }

      request {
        q            = "avg:system.cpu.stolen{haproxy_cluster:${var.cluster_name}} by {host}"
        display_type = "line"
      }

      request {
        q            = "avg:system.cpu.system{haproxy_cluster:${var.cluster_name}} by {host}"
        display_type = "line"
      }

      request {
        q            = "avg:system.cpu.interrupt{haproxy_cluster:${var.cluster_name}} by {host}"
        display_type = "line"
      }
    }
  }

  # System Memory used graph
  widget {
    timeseries_definition {
      title       = "System Memory % Used"
      show_legend = true

      request {
        q            = "(max:system.mem.used{haproxy_cluster:${var.cluster_name}} by {host}/avg:system.mem.total{haproxy_cluster:${var.cluster_name}} by {host})*100"
        display_type = "line"

        style {
          palette = "cool"
        }
      }
    }
  }

  # Disk Info Graph
  widget {
    timeseries_definition {
      title       = "Disk Info"
      show_legend = true

      request {
        q            = "avg:system.disk.total{haproxy_cluster:${var.cluster_name}}"
        display_type = "line"

        style {
          palette = "dog_classic"
        }
      }

      request {
        q            = "avg:system.disk.used{haproxy_cluster:${var.cluster_name}}"
        display_type = "line"

        style {
          palette = "warm"
        }
      }
    }
  }

  tags = local.dashboard_tags
}

resource "datadog_monitor" "healthy_haproxy_master_tasks_monitor" {
  count              = var.healthy_haproxy_master_tasks_monitor_enabled ? 1 : 0
  name               = "${var.environment}-${var.cluster_name} healthy haproxy-master tasks monitor"
  type               = "process alert"
  message            = "{{#is_alert}}The {{haproxy_cluster.name}} is missing haproxy-master processes{{/is_alert}}\n{{^is_alert}}The {{haproxy_cluster.name}} has recovered haproxy-master processes{{/is_alert}} ${var.healthy_haproxy_master_tasks_notifications}"
  escalation_message = "Escalation to ${var.healthy_haproxy_master_tasks_notifications}"

  query = "processes('haproxy-systemd-wrapper').over('haproxy_cluster:${var.cluster_name}').rollup('count').last('5m') < ${var.healthy_haproxy_master_tasks_critical_number}"

  monitor_thresholds {
    ok                = var.healthy_haproxy_master_tasks_ok_number
    warning           = var.healthy_haproxy_master_tasks_warning_number
    warning_recovery  = var.healthy_haproxy_master_tasks_warning_recovery_number
    critical          = var.healthy_haproxy_master_tasks_critical_number
    critical_recovery = var.healthy_haproxy_master_tasks_critical_recovery_number
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = false

  tags = local.tags
}

resource "datadog_monitor" "healthy_keepalived_tasks_monitor" {
  count              = var.healthy_keepalived_tasks_monitor_enabled ? 1 : 0
  name               = "${var.environment}-${var.cluster_name} healthy keepalived tasks monitor"
  type               = "process alert"
  message            = "{{#is_alert}}The {{haproxy_cluster.name}} is missing keepalived processes{{/is_alert}}\n{{^is_alert}}The {{haproxy_cluster.name}} has recovered keepalived processes{{/is_alert}} ${var.healthy_keepalived_tasks_notifications}"
  escalation_message = "Escalation to ${var.healthy_keepalived_tasks_notifications}"

  query = "processes('keepalived').over('haproxy_cluster:${var.cluster_name}').rollup('count').last('5m') < ${var.healthy_keepalived_tasks_critical_number}"

  monitor_thresholds {
    ok                = var.healthy_keepalived_tasks_ok_number
    warning           = var.healthy_keepalived_tasks_warning_number
    warning_recovery  = var.healthy_keepalived_tasks_warning_recovery_number
    critical          = var.healthy_keepalived_tasks_critical_number
    critical_recovery = var.healthy_keepalived_tasks_critical_recovery_number
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = false

  tags = local.tags
}