resource "datadog_monitor" "lambda_increased_cost_monitor" {
  evaluation_delay = 0

  include_tags = true

  message         = "Estimated cost of invocations have increased more than 20% ${var.slack_channel}"
  name            = "${var.environment}-${var.service_name} increased Cost on {{functionname.name}} in {{region.name}} for {{aws_account.name}}"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "pct_change(avg(last_5m),last_5m):avg:aws.lambda.enhanced.estimated_cost{aws_account:${local.account_id}} > 20"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.common_tags
  timeout_h            = 0
  type                 = "metric alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "20"
  }
}

resource "datadog_monitor" "lambda_execution_errors_monitor" {
  evaluation_delay = 900

  include_tags = true

  message = <<-EOF
{{#is_warning}}Percent of execution errors > 85% for lambda: `{{functionname.name}} `{{/is_warning}}
{{#is_alert}}Percent of execution errors > 95% for lambda: `{{functionname.name}}`{{/is_alert}}

Troubleshooting:

 Check logs

${var.slack_channel}
EOF

  name            = "${var.environment}-${var.service_name} lambda execution errors for {{functionname.name}}"
  new_group_delay = 60

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "sum(last_1h):max:aws.lambda.errors{aws_account:${local.account_id}} by {functionname}.as_count() / max:aws.lambda.invocations{aws_account:${local.account_id}} by {functionname}.as_count() >= 1"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.common_tags
  timeout_h            = 0
  type                 = "query alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "1"
    warning  = "0.95"
  }
}

resource "datadog_monitor" "lambda_cold_start_rate_monitor" {
  evaluation_delay = 0

  include_tags = true

  message         = "More than 20% of the function’s invocations were cold starts in the selected time range. Datadog’s [enhanced metrics](https://docs.datadoghq.com/serverless/enhanced_lambda_metrics) and [distributed tracing](https://docs.datadoghq.com/serverless/distributed_tracing) can help you understand the impact of cold starts on your applications today. {{#is_alert}} Resolution: Cold starts occur when your serverless applications receive sudden increases in traffic, and can occur when the function was previously inactive or when it was receiving a relatively constant number of requests. Users may perceive cold starts as slow response times or lag. To get ahead of cold starts, consider enabling [provisioned concurrency](https://www.datadoghq.com/blog/monitor-aws-lambda-provisioned-concurrency/) on your impacted Lambda functions. Note that this could affect your AWS bill. {{/is_alert}} ${var.slack_channel}"
  name            = "${var.environment}-${var.service_name} high Cold Start Rate on {{functionname.name}} in {{region.name}} for {{aws_account.name}}"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "avg(last_15m):sum:aws.lambda.enhanced.invocations{cold_start:true,aws_account:${local.account_id}} by {aws_account,functionname,region}.as_count() / sum:aws.lambda.enhanced.invocations{aws_account:${local.account_id}} by {aws_account,functionname,region}.as_count() >= 0.2"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.common_tags
  timeout_h            = 0
  type                 = "metric alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "0.2"
  }
}

resource "datadog_monitor" "lambda_error_rate_monitor" {
  evaluation_delay = 0

  include_tags = true

  message         = "More than 10% of the function’s invocations were errors in the selected time range. {{#is_alert}} Resolution: Examine the function’s logs, check for recent code or configuration changes with [Deployment Tracking](https://docs.datadoghq.com/serverless/deployment_tracking), or look for failures across microservices with [distributed tracing](https://docs.datadoghq.com/serverless/distributed_tracing).{{/is_alert}} ${var.slack_channel}"
  name            = "${var.environment}-${var.service_name} high Error Rate on {{functionname.name}} in {{region.name}} for {{aws_account.name}}"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "avg(last_15m):sum:aws.lambda.errors{aws_account:${local.account_id}} by {functionname,region,aws_account}.as_count() / sum:aws.lambda.invocations{aws_account:${local.account_id}} by {functionname,region,aws_account}.as_count() >= 0.1"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.common_tags
  timeout_h            = 0
  type                 = "metric alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "0.1"
  }
}

resource "datadog_monitor" "lambda_iterator_rate_monitor" {
  evaluation_delay = 0

  include_tags = true

  message         = "The function’s iterator was older than 24 hours. Iterator age measures the age of the last record for each batch of records processed from a stream. When this value increases, it means your function cannot process data fast enough. {{#is_alert}} Resolution: Enable [distributed tracing](https://docs.datadoghq.com/serverless/distributed_tracing) to isolate why your function has so much data being streamed to it. You can also consider increasing the shard count and batch size of the stream your function reads from. {{/is_alert}}"
  name            = "${var.environment}-${var.service_name} high Iterator Rate on {{functionname.name}} in {{region.name}} for {{aws_account.name}}"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "avg(last_15m):min:aws.lambda.iterator_age.maximum{aws_account:${local.account_id}} by {aws_account,region,functionname} >= 86400"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.common_tags
  timeout_h            = 0
  type                 = "metric alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "86400"
  }
}

resource "datadog_monitor" "lambda_throttles_monitor" {
  evaluation_delay = 0

  include_tags = true

  message         = "More than 10% of invocations in the selected time range were throttled. Throttling occurs when your serverless Lambda applications receive high levels of traffic without adequate [concurrency](https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html). {{#is_alert}} Resolution: Check your [Lambda concurrency metrics](https://docs.datadoghq.com/integrations/amazon_lambda/#metrics) and confirm if `aws.lambda.concurrent_executions.maximum` is approaching your AWS account concurrency level. If so, consider configuring reserved concurrency, or request a service quota increase from AWS. Note that this may affect your AWS bill. {{/is_alert}} ${var.slack_channel}"
  name            = "${var.environment}-${var.service_name} high Throttles on {{functionname.name}} in {{region.name}} for {{aws_account.name}}"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "avg(last_15m):sum:aws.lambda.throttles {aws_account:${local.account_id}} by {aws_account,region,functionname}.as_count() / ( sum:aws.lambda.throttles {aws_account:${local.account_id}} by {aws_account,region,functionname}.as_count() + sum:aws.lambda.invocations{aws_account:${local.account_id}} by {aws_account,region,functionname}.as_count()) >= 0.2"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.common_tags
  timeout_h            = 0
  type                 = "metric alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "0.2"
  }
}

resource "datadog_monitor" "lambda_out_of_memory_monitor" {
  evaluation_delay = 0

  include_tags = true

  message         = "At least one invocation in the selected time range ran out of memory. Resolution: Lambda functions that use more than their allotted amount of memory can be killed by the Lambda runtime. To users, this may look like failed requests to your application. [Distributed tracing](https://docs.datadoghq.com/serverless/distributed_tracing) can help you pinpoint parts of your application using excessive amounts of memory. Consider increasing the amount of memory your Lambda function is allowed to use. ${var.slack_channel}"
  name            = "${var.environment}-${var.service_name} Out of Memory on {{functionname.name}} in {{region.name}} for {{aws_account.name}}"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "avg(last_15m):sum:aws.lambda.enhanced.out_of_memory{aws_account:${local.account_id}} by {aws_account,functionname,region} > 0"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.common_tags
  timeout_h            = 0
  type                 = "metric alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "0"
  }
}

resource "datadog_monitor" "lambda_timeout_monitor" {
  evaluation_delay = 0

  include_tags = true

  message         = "At least one invocation in the selected time range timed out. This occurs when your function runs for longer than the configured timeout or the global Lambda timeout. Resolution: [Distributed tracing](https://docs.datadoghq.com/serverless/distributed_tracing) can help you pinpoint slow requests to APIs and other microservices. You can also consider increasing the timeout of your function. Note that this could affect your AWS bill."
  name            = "${var.environment}-${var.service_name} Timeout on {{functionname.name}} in {{region.name}} for {{aws_account.name}}"
  new_group_delay = 0

  no_data_timeframe    = 0
  notify_audit         = false
  notify_by            = []
  notify_no_data       = false
  priority             = 0
  query                = "avg(last_15m):sum:aws.lambda.duration.maximum{aws_account:${local.account_id}} by {aws_account,functionname,region}.as_count() / (sum:aws.lambda.timeout{aws_account:${local.account_id}} by {aws_account,functionname,region}.as_count() * 1000) >= 1"
  renotify_interval    = 0
  renotify_occurrences = 0
  require_full_window  = false
  tags                 = local.common_tags
  timeout_h            = 0
  type                 = "metric alert"
  restricted_roles     = [data.datadog_role.datadog_admin_role.id]

  monitor_thresholds {
    critical = "1"
  }
}