resource "datadog_monitor" "snowflake_custom_task_monitor_failed_tasks_count_monitor" {
  name = "Failed Snowflake Task: {{[custom.task-monitor.failed-task-details.name].name}}"
  type = "query alert"
  # If there is no notification_channel tag, default to notification_endpoints,
  # otherwise notify the corresponding team's slack channel
  message = "Monitor triggered for failed Snowflake Task: **{{[custom.task-monitor.failed-task-details.name].name}} **\n {{#is_exact_match '[custom.task-monitor.failed-task-details.notification_channel].name' 'no_channel'}} ${var.notification_endpoints} {{/is_exact_match}} \n {{^is_exact_match '[custom.task-monitor.failed-task-details.notification_channel].name' 'no_channel'}} @slack-{{[custom.task-monitor.failed-task-details.notification_channel].name}} {{/is_exact_match}}"

  query = "max(last_1h):sum:snowflake.custom.task_monitor.failed_task_details.count{*} by {custom.task-monitor.failed-task-details.name,custom.task-monitor.failed-task-details.notification_channel} >= 1"

  monitor_thresholds {
    ok       = 0
    critical = 1
  }

  include_tags        = false
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 1
  require_full_window = false
  no_data_timeframe   = 0
  notify_by = [
    "custom.task-monitor.failed-task-details.name"
  ]

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "team:${var.kdh_team_tag}",
  ]
}

resource "datadog_monitor" "ar_to_neo_sync_kinesis_iterator_age_monitor" {
  name    = "Art Relations to Neo4j Sync kinesis.iterator_age monitor (${var.environment}-${var.service_name})"
  type    = "query alert"
  message = "Monitor triggered for {{value}}. Notify: ${var.notification_endpoints}"

  query = "avg(last_5m):avg:aws.kinesis.iterator_age_milliseconds{name:prod-neo4j-stream} >= 600000"

  monitor_thresholds {
    ok       = 0
    warning  = 360000 # 6 mins
    critical = 600000 # 10 mins
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 0
  require_full_window = false
  no_data_timeframe   = 0

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "team:${var.kdh_team_tag}",
  ]
}

resource "datadog_monitor" "snowflake_custom_query_monitor_query_queued_time_ratio" {
  name = "Snowflake - PROD_OWS_WAREHOUSE - Query Queuing (last hour)"
  type = "query alert"

  message = "---\n\n# ❄️ Snowflake - PROD_OWS_WAREHOUSE - Query Queuing (last hour) - Monitor triggered ❄️\n\n---\n\nIn the last hour, a max of {{value}}% of time was spent queuing. \nThis could soon impact the performance of Insights. \nIn the first instance, check the warehouse dashboard [here](https://app.snowflake.com/sme/orchard/#/compute/warehouses/PROD_OWS_WAREHOUSE?wh=PROD_OWS_WAREHOUSE)\n\n---\n\nNotify: @slack-data-alerts @fumar@sonymusic-pde.com @cbeesley@sonymusic-pde.com @data-squad@sonymusic-pde.com\n"

  query = "max(last_1h):max:snowflake.custom.query_monitor.warehouse_load.QUEUED_RATIO{custom.query-monitor.warehouse_load.warehouse_name:prod_ows_warehouse} * 100 > 60"

  monitor_thresholds {
    ok       = 0
    warning  = 50
    critical = 60
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 1
  require_full_window = false
  no_data_timeframe   = 0

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:data-platform"
  ]
}

resource "datadog_monitor" "snowflake_custom_query_monitor_query_queued_time_ratio_prod_etl_wh" {
  name = "Snowflake - PROD_ETL_WAREHOUSE - Query Queuing (last hour)"
  type = "query alert"

  message = "---\n\n# ❄️ Snowflake - PROD_ETL_WAREHOUSE - Query Queuing (last hour) - Monitor triggered ❄️\n\n---\n\nIn the last hour, a max of {{value}}% of time was spent queuing. \nThis could soon impact data getting to Insights. \nIn the first instance, check the warehouse dashboard [here](https://app.snowflake.com/sme/orchard/#/compute/warehouses/PROD_ETL_WAREHOUSE?wh=PROD_ETL_WAREHOUSE)\n\n---\n\nNotify: @slack-data-alerts @fumar@sonymusic-pde.com @cbeesley@sonymusic-pde.com @data-squad@sonymusic-pde.com\n"

  query = "max(last_1h):max:snowflake.custom.query_monitor.warehouse_load.QUEUED_RATIO{custom.query-monitor.warehouse_load.warehouse_name:prod_etl_warehouse} * 100 > 60"

  monitor_thresholds {
    ok       = 0
    warning  = 50
    critical = 60
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 1
  require_full_window = false
  no_data_timeframe   = 0

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:data-platform"
  ]
}

resource "datadog_monitor" "snowflake_custom_query_monitor_tnx_blocked_time_ratio_ows_wh" {
  name = "Snowflake - PROD_OWS_WAREHOUSE - Transaction Blocking (last hour)"
  type = "query alert"

  message = "---\n\n# ❄️ SNOWFLAKE ALERT ❄️ \n\n##  TRANSACTIONS BLOCKED ratio is **HIGH** on warehouse **PROD_OWS_WAREHOUSE** \n\n---\n\nThis could mean that Insights performance will be degraded. \nIn the first instance, check the warehouse dashboard [here](https://app.snowflake.com/sme/orchard/#/compute/warehouses/PROD_OWS_WAREHOUSE?wh=PROD_OWS_WAREHOUSE) and identify what is causing the locking.\n\n @slack-data-alerts @cbeesley@sonymusic-pde.com"

  query = "max(last_1h):max:snowflake.custom.query_monitor.warehouse_load.BLOCKED_RATIO{custom.query-monitor.warehouse_load.warehouse_name:prod_ows_warehouse} * 100 > 30"

  monitor_thresholds {
    ok       = 0
    warning  = 25
    critical = 30
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 1
  require_full_window = false
  no_data_timeframe   = 0

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:data-platform"
  ]
}


resource "datadog_monitor" "snowflake_custom_query_monitor_tnx_blocked_time_ratio_etl_wh" {
  name = "Snowflake - PROD_ETL_WAREHOUSE - Transaction Blocking (last hour)"
  type = "query alert"

  message = "---\n\n# ❄️ SNOWFLAKE ALERT ❄️ \n\n##  TRANSACTIONS BLOCKED ratio is **HIGH** on warehouse **PROD_ETL_WAREHOUSE** \n\n---\n\nThis could mean that data to Insights will soon be delayed. \nIn the first instance, check the warehouse dashboard [here](https://app.snowflake.com/sme/orchard/#/compute/warehouses/PROD_ETL_WAREHOUSE?wh=PROD_ETL_WAREHOUSE) and identify what is causing the locking.\n\n @slack-data-alerts @cbeesley@sonymusic-pde.com"

  query = "avg(last_2h):avg:snowflake.custom.query_monitor.warehouse_load.BLOCKED_RATIO{custom.query-monitor.warehouse_load.warehouse_name:prod_etl_warehouse} * 100 > 40"

  monitor_thresholds {
    ok       = 0
    warning  = 30
    critical = 40
  }

  include_tags        = true
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 1
  require_full_window = false
  no_data_timeframe   = 0

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "application_family:data-platform"
  ]
}

resource "datadog_monitor" "snowflake_custom_task_suspended_due_to_errors_monitor" {
  name = "Suspended Snowflake Task: {{[custom.task-monitor.suspended-task-details.name].name}}"
  type = "query alert"
  # If there is no notification_channel tag, default to notification_endpoints,
  # otherwise notify the corresponding team's slack channel
  message = "---\n\n# ❄️ SNOWFLAKE ALERT ❄️ \n\n Monitor triggered for Suspended Snowflake Task due to errors in: **{{[custom.task-monitor.suspended-task-details.name].name}} **\n {{#is_exact_match '[custom.task-monitor.suspended-task-details.notification_channel].name' 'no_channel'}} ${var.notification_endpoints} {{/is_exact_match}} \n {{^is_exact_match '[custom.task-monitor.suspended-task-details.notification_channel].name' 'no_channel'}} @slack-{{[custom.task-monitor.suspended-task-details.notification_channel].name}} {{/is_exact_match}}"

  query = "max(last_1h):sum:snowflake.custom.task_monitor.suspended_task_details.count{*} by {custom.task-monitor.suspended-task-details.name,custom.task-monitor.suspended-task-details.notification_channel} >= 1"

  monitor_thresholds {
    ok       = 0
    critical = 1
  }

  include_tags        = false
  notify_no_data      = false
  notify_audit        = false
  renotify_interval   = 60
  timeout_h           = 1
  require_full_window = false
  no_data_timeframe   = 0
  notify_by = [
    "custom.task-monitor.suspended-task-details.name"
  ]

  tags = [
    "environment:${var.environment}",
    "service_name:${var.service_name}",
    "team:${var.kdh_team_tag}",
  ]
}