resource "datadog_monitor" "os_node_disk_space_monitor" {
  message             = "[${var.environment}]-${var.service_name} nodes low disk space. It has {{value}} free space left. Notify: ${var.infra_notification_endpoints}"
  name                = "${var.environment}-${var.service_name}-nodes-low-disk-space"
  type                = "query alert"
  query               = "avg(last_5m):avg:aws.es.free_storage_space{domain:${var.environment}-${var.service_name}} < ${var.os_disk_datadog_monitor_critical}"
  notify_no_data      = false
  renotify_interval   = 60
  evaluation_delay    = "900"
  timeout_h           = "0"
  include_tags        = true
  require_full_window = false
  notify_audit        = false
  monitor_thresholds {
    critical = var.os_disk_datadog_monitor_critical
    warning  = var.os_disk_datadog_monitor_warning
  }

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:${var.application_family}",
    "terraformed:true",
  ]
}

# this monitor is for global stream level lag. It will alert if any of the consumers is lagging.
resource "datadog_monitor" "kinesis_processing_lag_monitor" {
  message             = "[${var.environment}]-${var.service_name} Kinesis stream processing lag has increased to {{value}}. Notify: ${var.infra_notification_endpoints}"
  name                = "${var.environment}-${var.service_name}-kinesis-processing-lag"
  type                = "query alert"
  query               = "avg(last_1h):avg:aws.kinesis.get_records_iterator_age.maximum{streamname:${var.environment}-${var.kinesis_stream_name}} > ${var.kinesis_lag_datadog_monitor_critical}"
  notify_no_data      = false
  renotify_interval   = 60
  evaluation_delay    = "900"
  timeout_h           = "0"
  include_tags        = true
  require_full_window = false
  notify_audit        = false
  monitor_thresholds {
    critical = var.kinesis_lag_datadog_monitor_critical
    warning  = var.kinesis_lag_datadog_monitor_warning
  }

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:${var.application_family}",
    "terraformed:true",
  ]
}

# this monitor is for this lambda level kinesis stream lag.
resource "datadog_monitor" "iterator_age_processing_lag_monitor" {
  message             = "[${var.environment}]-${var.lambda_name} Lambda processing lag has increased to {{value}}. Notify: ${var.infra_notification_endpoints}"
  name                = "${var.environment}-${var.lambda_name}-lag"
  type                = "query alert"
  query               = "avg(last_1h):avg:aws.lambda.iterator_age.maximum{service:${var.lambda_name}, env:${var.environment}} > ${var.lambda_lag_datadog_monitor_critical}"
  notify_no_data      = false
  renotify_interval   = 60
  evaluation_delay    = "900"
  timeout_h           = "2"
  include_tags        = true
  require_full_window = false
  notify_audit        = false
  monitor_thresholds {
    critical = var.lambda_lag_datadog_monitor_critical
    warning  = var.lambda_lag_datadog_monitor_warning
  }

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:${var.application_family}",
    "terraformed:true",
  ]
}

# Built in datadog timeboard dashboard for Lambda function errors.
module "datadog_lambda_dd_search" {
  source = "git@github.com:theorchard/terraform-datadog.git//modules/lambda?ref=6.13.7"

  environment                       = var.environment
  service_name                      = var.lambda_name
  lambda_invocation_monitor_enabled = false
  lambda_error_monitor_enabled      = true

  lambda_throttle_monitor_enabled          = true
  lambda_throttle_critical_number          = 40
  lambda_throttle_warning_number           = 30
  lambda_throttle_warning_recovery_number  = 12
  lambda_throttle_critical_recovery_number = 22

  lambda_anomaly_monitor_enabled                  = true
  lambda_max_memory_used_enabled                  = true
  lambda_max_memory_usage_warning_number          = 70 # percentage
  lambda_max_memory_usage_warning_recovery_number = 60 # percentage

  notification_endpoints = var.lambda_notification_endpoints
}

# Custom Datadog dashboard for DD Search (same as prod).
resource "datadog_dashboard" "dd_search_dashboard" {
  title       = "${var.environment}-${var.lambda_name} data sync dashboard"
  description = "${var.environment} DD search v2 data sync status dashboard"
  layout_type = "ordered"
  reflow_type = "fixed"

  widget {
    group_definition {
      background_color = "blue"
      layout_type      = "ordered"
      show_title       = true
      title            = "MySQL Aurora ${var.environment}-direct-delivery"

      widget {
        timeseries_definition {
          title          = "Cluster replication lag"
          title_size     = "16"
          title_align    = "left"
          show_legend    = true
          legend_layout  = "auto"
          legend_columns = ["avg", "min", "max", "value", "sum"]
          request {
            display_type   = "line"
            on_right_yaxis = false
            formula {
              formula_expression = "query1"
            }
            query {
              metric_query {
                name        = "query1"
                data_source = "metrics"
                query       = "avg:aws.rds.aurora_replica_lag_maximum{environment:${var.environment} AND service_name:direct-delivery-cluster}"
              }
            }
          }
        }
        widget_layout {
          height          = 2
          is_column_break = false
          width           = 4
          x               = 0
          y               = 0
        }
      }

      widget {
        timeseries_definition {
          title          = "Running queries"
          title_size     = "16"
          title_align    = "left"
          show_legend    = true
          legend_layout  = "auto"
          legend_columns = ["avg", "min", "max", "value", "sum"]
          request {
            display_type   = "line"
            on_right_yaxis = false
            formula {
              formula_expression = "query1"
            }
            query {
              metric_query {
                name        = "query1"
                data_source = "metrics"
                query       = "max:aws.rds.queries{environment:${var.environment} AND service_name:direct-delivery-cluster}"
              }
            }
          }
        }
        widget_layout {
          height          = 2
          is_column_break = false
          width           = 4
          x               = 4
          y               = 0
        }
      }

      widget {
        timeseries_definition {
          title          = "Read latency"
          title_size     = "16"
          title_align    = "left"
          show_legend    = true
          legend_layout  = "auto"
          legend_columns = ["avg", "min", "max", "value", "sum"]
          request {
            display_type   = "line"
            on_right_yaxis = false
            formula {
              formula_expression = "query1"
            }
            query {
              metric_query {
                name        = "query1"
                data_source = "metrics"
                query       = "max:aws.rds.read_latency{environment:${var.environment} AND service_name:direct-delivery-cluster}"
              }
            }
          }
        }
        widget_layout {
          height          = 2
          is_column_break = false
          width           = 4
          x               = 8
          y               = 0
        }
      }

    } # end of group definition
  }

  widget {
    group_definition {
      background_color = "green"
      layout_type      = "ordered"
      show_title       = true
      title            = "Kinesis ${var.environment}-${var.kinesis_stream_name}"

      widget {
        timeseries_definition {
          title          = "Processing lag"
          title_size     = "16"
          title_align    = "left"
          show_legend    = true
          legend_layout  = "auto"
          legend_columns = ["avg", "min", "max", "value", "sum"]
          request {
            formula {
              formula_expression = "query1"
            }
            query {
              metric_query {
                name        = "query1"
                data_source = "metrics"
                query       = "avg:aws.kinesis.get_records_iterator_age.maximum{name:${var.environment}-${var.kinesis_stream_name}}"
              }
            }
          }
        }
        widget_layout {
          height          = 2
          is_column_break = false
          width           = 4
          x               = 0
          y               = 0
        }
      }

      widget {
        timeseries_definition {
          title          = "Kinesis incoming and outgoing records"
          title_size     = "16"
          title_align    = "left"
          show_legend    = true
          legend_layout  = "auto"
          legend_columns = ["avg", "min", "max", "value", "sum"]
          request {
            formula {
              formula_expression = "query1"
              alias = "incoming_records"
            }
            formula {
              formula_expression = "query2"
              alias = "outgoing_records"
            }
            query {
              metric_query {
                name        = "query1"
                data_source = "metrics"
                query       = "sum:aws.kinesis.incoming_records{streamname:${var.environment}-${var.kinesis_stream_name}} by {streamname}"
              }
            }
            query {
              metric_query {
                name        = "query2"
                data_source = "metrics"
                query       = "sum:aws.kinesis.outgoing_records{streamname:${var.environment}-${var.kinesis_stream_name}} by {streamname}"
              }
            }
          }
        }
        widget_layout {
          height          = 2
          is_column_break = false
          width           = 4
          x               = 4
          y               = 0
        }
      }

      widget {
        timeseries_definition {
          title          = "Maxwell kinesis-producer for ${var.environment}-${var.kinesis_stream_name}"
          title_size     = "16"
          title_align    = "left"
          show_legend    = true
          legend_layout  = "auto"
          legend_columns = ["avg", "min", "max", "value", "sum"]
          request {
            formula {
              formula_expression = "query1"
            }
            query {
              metric_query {
                name        = "query1"
                data_source = "metrics"
                query       = "avg:kinesisproducerlibrary.KinesisRecordsDataPut{streamname:${var.environment}-${var.kinesis_stream_name}} by {streamname}"
              }
            }
          }
        }
        widget_layout {
          height          = 2
          is_column_break = false
          width           = 4
          x               = 8
          y               = 0
        }
      }

    } # end of group definition
  }

  widget {
    group_definition {
      background_color = "purple"
      layout_type      = "ordered"
      show_title       = true
      title            = "Lambda ${var.environment}-${var.lambda_name}"

      widget {
        timeseries_definition {
          title          = "Lambda invocations"
          title_size     = "16"
          title_align    = "left"
          show_legend    = true
          legend_layout  = "auto"
          legend_columns = ["avg", "min", "max", "value", "sum"]
          request {
            formula {
              formula_expression = "query1"
            }
            query {
              metric_query {
                name        = "query1"
                data_source = "metrics"
                query       = "sum:aws.lambda.invocations{functionname:${var.environment}-${var.lambda_name}}.as_count()"
              }
            }
          }
        }
        widget_layout {
          height          = 2
          is_column_break = false
          width           = 4
          x               = 0
          y               = 0
        }
      }

      widget {
        timeseries_definition {
          title          = "Lambda errors"
          title_size     = "16"
          title_align    = "left"
          show_legend    = true
          legend_layout  = "auto"
          legend_columns = ["avg", "min", "max", "value", "sum"]
          request {
            formula {
              formula_expression = "query1"
            }
            query {
              metric_query {
                name        = "query1"
                data_source = "metrics"
                query       = "sum:aws.lambda.errors{functionname:${var.environment}-${var.lambda_name}}.as_count()"
              }
            }
          }
        }
        widget_layout {
          height          = 2
          is_column_break = false
          width           = 4
          x               = 4
          y               = 0
        }
      }

      widget {
        timeseries_definition {
          title          = "Lambda memory used"
          title_size     = "16"
          title_align    = "left"
          show_legend    = true
          legend_layout  = "auto"
          legend_columns = ["avg", "min", "max", "value", "sum"]
          request {
            formula {
              formula_expression = "query1"
            }
            query {
              metric_query {
                name        = "query1"
                data_source = "metrics"
                query       = "max:aws.lambda.enhanced.max_memory_used{functionname:${var.environment}-${var.lambda_name}}.as_count()"
              }
            }
          }
        }
        widget_layout {
          height          = 2
          is_column_break = false
          width           = 4
          x               = 8
          y               = 0
        }
      }

      widget {
        timeseries_definition {
          title          = "Lambda duration"
          title_size     = "16"
          title_align    = "left"
          show_legend    = true
          legend_layout  = "auto"
          legend_columns = ["avg", "min", "max", "value", "sum"]
          request {
            formula {
              formula_expression = "query1"
            }
            query {
              metric_query {
                name        = "query1"
                data_source = "metrics"
                query       = "max:aws.lambda.duration{functionname:${var.environment}-${var.lambda_name}}"
              }
            }
          }
        }
        widget_layout {
          height          = 2
          is_column_break = false
          width           = 4
          x               = 0
          y               = 0
        }
      }

      widget {
        timeseries_definition {
          title          = "Lambda iterator_age"
          title_size     = "16"
          title_align    = "left"
          show_legend    = true
          legend_layout  = "auto"
          legend_columns = ["avg", "min", "max", "value", "sum"]
          request {
            formula {
              formula_expression = "query1"
            }
            query {
              metric_query {
                name        = "query1"
                data_source = "metrics"
                query       = "avg:aws.lambda.iterator_age.maximum{functionname:${var.environment}-${var.lambda_name}}.as_count()"
              }
            }
          }
        }
        widget_layout {
          height          = 2
          is_column_break = false
          width           = 4
          x               = 4
          y               = 2
        }
      }

      widget {
        timeseries_definition {
          title          = "Lambda throttles"
          title_size     = "16"
          title_align    = "left"
          show_legend    = true
          legend_layout  = "auto"
          legend_columns = ["avg", "min", "max", "value", "sum"]
          request {
            formula {
              formula_expression = "query1"
            }
            query {
              metric_query {
                name        = "query1"
                data_source = "metrics"
                query       = "sum:aws.lambda.throttles{functionname:${var.environment}-${var.lambda_name}}.as_count()"
              }
            }
          }
        }
        widget_layout {
          height          = 2
          is_column_break = false
          width           = 4
          x               = 8
          y               = 2
        }
      }

    } # end of group definition
  }

  widget {
    group_definition {
      background_color = "blue"
      layout_type      = "ordered"
      show_title       = true
      title            = "Opensearch instance ${var.environment}-${var.service_name}"

      widget {
        timeseries_definition {
          title          = "Free space"
          title_size     = "16"
          title_align    = "left"
          show_legend    = true
          legend_layout  = "auto"
          legend_columns = ["avg", "min", "max", "value", "sum"]
          request {
            formula {
              formula_expression = "query1"
            }
            query {
              metric_query {
                name        = "query1"
                data_source = "metrics"
                query       = "avg:aws.es.free_storage_space{domain:${var.environment}-${var.service_name}}"
              }
            }
          }
        }
        widget_layout {
          height          = 2
          is_column_break = false
          width           = 4
          x               = 0
          y               = 0
        }
      }

      widget {
        timeseries_definition {
          title          = "CPU utilization"
          title_size     = "16"
          title_align    = "left"
          show_legend    = true
          legend_layout  = "auto"
          legend_columns = ["avg", "min", "max", "value", "sum"]
          request {
            formula {
              formula_expression = "query1"
            }
            query {
              metric_query {
                name        = "query1"
                data_source = "metrics"
                query       = "max:aws.es.cpuutilization.maximum{domainname:${var.environment}-${var.service_name}}"
              }
            }
          }
        }
        widget_layout {
          height          = 2
          is_column_break = false
          width           = 4
          x               = 4
          y               = 0
        }
      }

      widget {
        timeseries_definition {
          title          = "Query failures"
          title_size     = "16"
          title_align    = "left"
          show_legend    = true
          legend_layout  = "auto"
          legend_columns = ["avg", "min", "max", "value", "sum"]
          request {
            formula {
              formula_expression = "query1"
            }
            query {
              metric_query {
                name        = "query1"
                data_source = "metrics"
                query       = "sum:aws.es.query_failure{domainname:${var.environment}-${var.service_name}}"
              }
            }
          }
        }
        widget_layout {
          height          = 2
          is_column_break = false
          width           = 4
          x               = 8
          y               = 0
        }
      }

      widget {
        timeseries_definition {
          title          = "Write IOPS"
          title_size     = "16"
          title_align    = "left"
          show_legend    = true
          legend_layout  = "auto"
          legend_columns = ["avg", "min", "max", "value", "sum"]
          request {
            formula {
              formula_expression = "query1"
            }
            query {
              metric_query {
                name        = "query1"
                data_source = "metrics"
                query       = "sum:aws.es.write_iops{domainname:${var.environment}-${var.service_name}}"
              }
            }
          }
        }
        widget_layout {
          height          = 2
          is_column_break = false
          width           = 4
          x               = 0
          y               = 0
        }
      }

      widget {
        timeseries_definition {
          title          = "Read IOPS"
          title_size     = "16"
          title_align    = "left"
          show_legend    = true
          legend_layout  = "auto"
          legend_columns = ["avg", "min", "max", "value", "sum"]
          request {
            formula {
              formula_expression = "query1"
            }
            query {
              metric_query {
                name        = "query1"
                data_source = "metrics"
                query       = "sum:aws.es.read_iops{domainname:${var.environment}-${var.service_name}}"
              }
            }
          }
        }
        widget_layout {
          height          = 2
          is_column_break = false
          width           = 4
          x               = 4
          y               = 2
        }
      }

      widget {
        timeseries_definition {
          title          = "5xx errors"
          title_size     = "16"
          title_align    = "left"
          show_legend    = true
          legend_layout  = "auto"
          legend_columns = ["avg", "min", "max", "value", "sum"]
          request {
            formula {
              formula_expression = "query1"
            }
            query {
              metric_query {
                name        = "query1"
                data_source = "metrics"
                query       = "sum:aws.es.5xx{domainname:${var.environment}-${var.service_name}}"
              }
            }
            formula {
              formula_expression = "query2"
            }
            query {
              metric_query {
                name        = "query2"
                data_source = "metrics"
                query       = "sum:aws.es.4xx{domainname:${var.environment}-${var.service_name}}"
              }
            }
          }
        }
        widget_layout {
          height          = 2
          is_column_break = false
          width           = 4
          x               = 8
          y               = 2
        }
      }


    } # end of group definition
  }
}