resource "datadog_dashboard" "databricks_spark_dashboard" {
  layout_type = "ordered"
  title       = "Databricks Spark"
  url         = "/dashboard/v4x-gkq-ip4/databricks-spark"
  widget {
    timeseries_definition {
      show_legend = false
      title       = "Avg Execution Time"
      request {
        display_type   = "line"
        on_right_yaxis = false
        q              = "(avg:spark.stage.executor_run_time{*}.as_count()/avg:spark.stage.count{*}.as_count())/60000"
        style {
          line_type  = "solid"
          line_width = "normal"
          palette    = "dog_classic"
        }
      }
      yaxis {
        include_zero = true
        max          = "auto"
        min          = "auto"
        scale        = "linear"
      }
    }
  }
}

resource "datadog_dashboard" "spark_overview_cloned_dashboard" {
  description = <<-EOT
      This dashboard tracks the current status of your Spark jobs, tasks, and stages. It also allows you to track expensive operations like shuffles, which point to possible areas for optimization. Further reading on Spark monitoring:

      - [Datadog's blog post announcing support for Hadoop and Spark monitoring](https://www.datadoghq.com/blog/monitoring-spark/)

      - [How Datadog uses and monitors Spark for highly reliable data pipelines](https://www.datadoghq.com/blog/engineering/highly-reliable-data-pipelines/)

      - [Docs for Datadog's Spark integration](https://docs.datadoghq.com/integrations/spark/)

      Clone this template dashboard to make changes and add your own graph widgets.
        EOT
  layout_type = "free"
  title       = "Spark - Overview (cloned)"
  url         = "/dashboard/wdb-343-skn/spark---overview-cloned"
  widget {
    image_definition {
      has_background = false
      has_border     = false
      sizing         = "fit"
      url            = "/static/images/saas_logos/bot/spark.png"
    }
    widget_layout {
      height          = 11
      is_column_break = false
      width           = 21
      x               = 2
      y               = 2
    }
  }
  widget {
    timeseries_definition {
      legend_size = "0"
      live_span   = "4h"
      show_legend = false
      title       = "Jobs"
      title_align = "left"
      title_size  = "16"
      request {
        display_type   = "bars"
        on_right_yaxis = false
        q              = "sum:spark.job.count{status:succeeded}"
      }
      request {
        display_type   = "bars"
        on_right_yaxis = false
        q              = "sum:spark.job.count{status:running}"
      }
      request {
        display_type   = "bars"
        on_right_yaxis = false
        q              = "avg:spark.job.count{status:failed}"
      }
    }
    widget_layout {
      height          = 15
      is_column_break = false
      width           = 47
      x               = 101
      y               = 15
    }
  }
  widget {
    query_value_definition {
      autoscale   = false
      custom_unit = " "
      live_span   = "1h"
      precision   = 0
      title       = "Tasks Running"
      title_align = "left"
      title_size  = "16"
      request {
        aggregator = "last"
        q          = "sum:spark.job.num_tasks{status:running}"
      }
    }
    widget_layout {
      height          = 11
      is_column_break = false
      width           = 12
      x               = 116
      y               = 2
    }
  }
  widget {
    query_value_definition {
      autoscale   = false
      live_span   = "1h"
      precision   = 0
      title       = "Jobs Running"
      title_align = "left"
      title_size  = "16"
      request {
        aggregator = "last"
        q          = "sum:spark.job.count{status:running}"
      }
    }
    widget_layout {
      height          = 11
      is_column_break = false
      width           = 12
      x               = 88
      y               = 2
    }
  }
  widget {
    query_value_definition {
      autoscale   = false
      live_span   = "1h"
      precision   = 0
      title       = "Executors"
      title_align = "left"
      title_size  = "16"
      request {
        aggregator = "last"
        q          = "sum:spark.executor.count{*}"
      }
    }
    widget_layout {
      height          = 11
      is_column_break = false
      width           = 12
      x               = 74
      y               = 2
    }
  }
  widget {
    query_value_definition {
      autoscale   = false
      live_span   = "1h"
      precision   = 0
      title       = "Stages Running"
      title_align = "left"
      title_size  = "16"
      request {
        aggregator = "last"
        q          = "sum:spark.stage.count{status:active}"
      }
    }
    widget_layout {
      height          = 11
      is_column_break = false
      width           = 12
      x               = 102
      y               = 2
    }
  }
  widget {
    timeseries_definition {
      legend_size = "0"
      live_span   = "4h"
      show_legend = false
      title       = "Job Tasks"
      title_align = "left"
      title_size  = "16"
      request {
        display_type   = "line"
        on_right_yaxis = false
        q              = "sum:spark.job.num_tasks{status:running}"
      }
      request {
        display_type   = "line"
        on_right_yaxis = false
        q              = "sum:spark.job.num_completed_tasks{*}"
      }
      request {
        display_type   = "line"
        on_right_yaxis = false
        q              = "sum:spark.job.num_skipped_tasks{*}"
      }
      request {
        display_type   = "line"
        on_right_yaxis = false
        q              = "sum:spark.job.num_failed_tasks{*}"
      }
    }
    widget_layout {
      height          = 15
      is_column_break = false
      width           = 47
      x               = 101
      y               = 32
    }
  }
  widget {
    timeseries_definition {
      legend_size = "0"
      live_span   = "4h"
      show_legend = false
      title       = "Job Stages"
      title_align = "left"
      title_size  = "16"
      request {
        display_type   = "line"
        on_right_yaxis = false
        q              = "sum:spark.job.num_active_stages{*}"
      }
      request {
        display_type   = "line"
        on_right_yaxis = false
        q              = "sum:spark.job.num_completed_stages{*}"
      }
      request {
        display_type   = "line"
        on_right_yaxis = false
        q              = "sum:spark.job.num_skipped_stages{*}"
      }
      request {
        display_type   = "line"
        on_right_yaxis = false
        q              = "sum:spark.job.num_failed_stages{*}"
      }
    }
    widget_layout {
      height          = 15
      is_column_break = false
      width           = 47
      x               = 52
      y               = 32
    }
  }
  widget {
    timeseries_definition {
      legend_size = "0"
      live_span   = "4h"
      show_legend = false
      title       = "Stage Input/Output Bytes"
      title_align = "left"
      title_size  = "16"
      request {
        display_type   = "line"
        on_right_yaxis = false
        q              = "sum:spark.stage.input_bytes{*}"
      }
      request {
        display_type   = "line"
        on_right_yaxis = false
        q              = "sum:spark.stage.output_bytes{*}"
      }
    }
    widget_layout {
      height          = 15
      is_column_break = false
      width           = 47
      x               = 101
      y               = 49
    }
  }
  widget {
    timeseries_definition {
      legend_size = "0"
      live_span   = "4h"
      show_legend = false
      title       = "Stage Input/Output Records"
      title_align = "left"
      title_size  = "16"
      request {
        display_type   = "line"
        on_right_yaxis = false
        q              = "sum:spark.stage.input_records{*}"
      }
      request {
        display_type   = "line"
        on_right_yaxis = false
        q              = "sum:spark.stage.output_records{*}"
      }
    }
    widget_layout {
      height          = 15
      is_column_break = false
      width           = 47
      x               = 52
      y               = 49
    }
  }
  widget {
    timeseries_definition {
      legend_size = "0"
      live_span   = "4h"
      show_legend = false
      title       = "Shuffle Read/Write Bytes"
      title_align = "left"
      title_size  = "16"
      request {
        display_type   = "line"
        on_right_yaxis = false
        q              = "sum:spark.stage.shuffle_read_bytes{*}"
      }
      request {
        display_type   = "line"
        on_right_yaxis = false
        q              = "sum:spark.stage.shuffle_write_bytes{*}"
      }
      request {
        display_type   = "line"
        on_right_yaxis = false
        q              = "sum:spark.stage.shuffle_write_bytes{*}"
      }
    }
    widget_layout {
      height          = 15
      is_column_break = false
      width           = 47
      x               = 101
      y               = 66
    }
  }
  widget {
    timeseries_definition {
      legend_size = "0"
      live_span   = "4h"
      show_legend = false
      title       = "Shuffle Read/Write Records"
      title_align = "left"
      title_size  = "16"
      request {
        display_type   = "line"
        on_right_yaxis = false
        q              = "avg:spark.stage.shuffle_read_records{*}"
      }
      request {
        display_type   = "line"
        on_right_yaxis = false
        q              = "avg:spark.stage.shuffle_write_records{*}"
      }
    }
    widget_layout {
      height          = 15
      is_column_break = false
      width           = 47
      x               = 52
      y               = 66
    }
  }
  widget {
    timeseries_definition {
      legend_size = "0"
      live_span   = "4h"
      show_legend = false
      title       = "RDD Memory/Disk Usage"
      title_align = "left"
      title_size  = "16"
      request {
        display_type   = "line"
        on_right_yaxis = false
        q              = "sum:spark.rdd.memory_used{*}"
      }
      request {
        display_type   = "line"
        on_right_yaxis = false
        q              = "sum:spark.rdd.disk_used{*}"
      }
    }
    widget_layout {
      height          = 15
      is_column_break = false
      width           = 47
      x               = 52
      y               = 15
    }
  }
  widget {
    event_stream_definition {
      event_size  = "s"
      live_span   = "1d"
      query       = "spark has status"
      title       = "Spark Application Events"
      title_align = "left"
      title_size  = "16"
    }
    widget_layout {
      height          = 66
      is_column_break = false
      width           = 48
      x               = 2
      y               = 15
    }
  }
  widget {
    event_timeline_definition {
      live_span   = "1d"
      query       = "spark has status"
      title       = "Spark Application Event Counts"
      title_align = "left"
      title_size  = "16"
    }
    widget_layout {
      height          = 9
      is_column_break = false
      width           = 47
      x               = 25
      y               = 2
    }
  }
}