Skip to content

Commit 029ec52

Browse files
obelehobeleh
committed
switching to out of the box latency SLO
1 parent a5f6e8f commit 029ec52

File tree

4 files changed

+39
-30
lines changed

4 files changed

+39
-30
lines changed

‎README.md

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -161,9 +161,8 @@ burn_rate(\"${local.latency_slo_id}\").over(\"${var.latency_slo_burn_rate_evalua
161161
| latency_slo_filter_override | "" | No | |
162162
| latency_slo_warning | None | No | |
163163
| latency_slo_critical | 99.9 | No | |
164+
| latency_slo_latency_threshold | 1 | No | SLO latency threshold in seconds for APM traces |
164165
| latency_slo_alerting_enabled | True | No | |
165-
| latency_slo_status_ok_filter | ,status:ok | No | Filter string to select the non-errors for the latency SLO, Dont forget to include the comma or (AND or OR) keywords |
166-
| latency_slo_ms_bucket | 250 | No | We defined several latency buckets with custom metrics based on the APM traces that come in. Our buckets are 100, 250, 500, 1000, 2500, 5000, 10000 |
167166
| latency_slo_timeframe | 30d | No | |
168167
| latency_slo_burn_rate_priority | 3 | No | Number from 1 (high) to 5 (low). |
169168
| latency_slo_burn_rate_warning | None | No | |
@@ -176,6 +175,8 @@ burn_rate(\"${local.latency_slo_id}\").over(\"${var.latency_slo_burn_rate_evalua
176175
| latency_slo_burn_rate_notification_channel_override | "" | No | |
177176
| latency_slo_burn_rate_enabled | True | No | |
178177
| latency_slo_burn_rate_alerting_enabled | True | No | |
178+
| latency_slo_custom_numerator | "" | No | |
179+
| latency_slo_custom_denominator | "" | No | |
179180

180181

181182
## Apdex
@@ -242,18 +243,18 @@ Query:
242243
avg(last_10m):avg:trace.${var.trace_span_name}{tag:xxx} > 0.5
243244
```
244245

245-
| variable | default | required | description |
246-
|---------------------------------------|----------|----------|----------------------------------|
247-
| latency_enabled | True | No | |
248-
| latency_warning | 0.3 | No | |
249-
| latency_critical | 0.5 | No | |
250-
| latency_evaluation_period | last_10m | No | |
251-
| latency_note | "" | No | |
252-
| latency_docs | "" | No | |
253-
| latency_filter_override | "" | No | |
254-
| latency_alerting_enabled | True | No | |
255-
| latency_priority | 3 | No | Number from 1 (high) to 5 (low). |
256-
| latency_notification_channel_override | "" | No | |
246+
| variable | default | required | description |
247+
|---------------------------------------|----------|----------|---------------------------------------------|
248+
| latency_enabled | True | No | |
249+
| latency_warning | 0.3 | No | |
250+
| latency_critical | 0.5 | No | Latency threshold in seconds for APM traces |
251+
| latency_evaluation_period | last_10m | No | |
252+
| latency_note | "" | No | |
253+
| latency_docs | "" | No | |
254+
| latency_filter_override | "" | No | |
255+
| latency_alerting_enabled | True | No | |
256+
| latency_priority | 3 | No | Number from 1 (high) to 5 (low). |
257+
| latency_notification_channel_override | "" | No | |
257258

258259

259260
## Module Variables

‎latency-slo-variables.tf

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -29,23 +29,17 @@ variable "latency_slo_critical" {
2929
default = 99.9
3030
}
3131

32+
variable "latency_slo_latency_threshold" {
33+
description = "SLO latency threshold in seconds for APM traces"
34+
type = number
35+
default = 1
36+
}
37+
3238
variable "latency_slo_alerting_enabled" {
3339
type = bool
3440
default = true
3541
}
3642

37-
variable "latency_slo_status_ok_filter" {
38-
type = string
39-
description = "Filter string to select the non-errors for the latency SLO, Dont forget to include the comma or (AND or OR) keywords"
40-
default = ",status:ok"
41-
}
42-
43-
variable "latency_slo_ms_bucket" {
44-
type = number
45-
default = 250
46-
description = "We defined several latency buckets with custom metrics based on the APM traces that come in. Our buckets are 100, 250, 500, 1000, 2500, 5000, 10000"
47-
}
48-
4943
variable "latency_slo_timeframe" {
5044
validation {
5145
condition = contains(["7d", "30d", "90d"], var.latency_slo_timeframe)
@@ -111,3 +105,13 @@ variable "latency_slo_burn_rate_alerting_enabled" {
111105
type = bool
112106
default = true
113107
}
108+
109+
variable "latency_slo_custom_numerator" {
110+
type = string
111+
default = ""
112+
}
113+
114+
variable "latency_slo_custom_denominator" {
115+
type = string
116+
default = ""
117+
}

‎latency-slo.tf

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@ locals {
99
), "")
1010
latency_slo_burn_rate_enabled = var.latency_slo_enabled && var.latency_slo_burn_rate_enabled
1111
latency_slo_id = local.latency_slo_burn_rate_enabled ? datadog_service_level_objective.latency_slo[0].id : ""
12+
13+
latency_slo_numerator = coalesce(var.latency_slo_custom_numerator, "count(v: v<${var.latency_slo_latency_threshold}):trace.${var.trace_span_name}{${local.latency_slo_filter}}")
14+
latency_slo_denominator = coalesce(var.latency_slo_custom_denominator, "count:trace.${var.trace_span_name}{${local.latency_slo_filter}}")
1215
}
1316

1417

@@ -25,8 +28,8 @@ resource "datadog_service_level_objective" "latency_slo" {
2528
}
2629

2730
query {
28-
numerator = "sum:custom_trace.lt.${var.latency_slo_ms_bucket}ms.count{${local.latency_slo_filter}${var.latency_slo_status_ok_filter}}.as_count()"
29-
denominator = "sum:custom_trace.hits{${local.latency_slo_filter}${var.latency_slo_status_ok_filter}}.as_count()"
31+
numerator = local.latency_slo_numerator
32+
denominator = local.latency_slo_denominator
3033
}
3134

3235
tags = local.normalized_tags

‎latency-variables.tf

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,9 @@ variable "latency_warning" {
99
}
1010

1111
variable "latency_critical" {
12-
type = number
13-
default = 0.5
12+
description = "Latency threshold in seconds for APM traces"
13+
type = number
14+
default = 0.5
1415
}
1516

1617
variable "latency_evaluation_period" {

0 commit comments

Comments
 (0)