Skip to content

Commit e148afa

Browse files
haadhaad
authored
Merge pull request #183 from breed808/http_adaptive_quantiles
feat: add per-site latency metrics
2 parents 1a251e9 + b766e09 commit e148afa

File tree

3 files changed

+113
-40
lines changed

3 files changed

+113
-40
lines changed

‎README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,9 @@ Note: `ZONE_<name>` configuration is not supported as flag.
111111
# HELP cloudflare_zone_requests_content_type Number of request for zone per content type
112112
# HELP cloudflare_zone_requests_country Number of request for zone per country
113113
# HELP cloudflare_zone_requests_origin_status_country_host Count of not cached requests for zone per origin HTTP status per country per host
114+
# HELP cloudflare_zone_requests_origin_status_country_host_p50_ms P50 latency of not cached requests for zone per origin HTTP status per country per host, in ms
115+
# HELP cloudflare_zone_requests_origin_status_country_host_p95_ms P95 latency of not cached requests for zone per origin HTTP status per country per host, in ms
116+
# HELP cloudflare_zone_requests_origin_status_country_host_p99_ms P99 latency of not cached requests for zone per origin HTTP status per country per host, in ms
114117
# HELP cloudflare_zone_requests_ssl_encrypted Number of encrypted requests for zone
115118
# HELP cloudflare_zone_requests_status Number of request for zone per HTTP status
116119
# HELP cloudflare_zone_requests_status_country_host Count of requests for zone per edge HTTP status per country per host

‎cloudflare.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,11 @@ type zoneResp struct {
217217
ClientCountryName string `json:"clientCountryName"`
218218
ClientRequestHTTPHost string `json:"clientRequestHTTPHost"`
219219
} `json:"dimensions"`
220+
Quantile struct {
221+
OriginResponseDurationMsP50 uint `json:"originResponseDurationMsP50"`
222+
OriginResponseDurationMsP95 uint `json:"originResponseDurationMsP95"`
223+
OriginResponseDurationMsP99 uint `json:"originResponseDurationMsP99"`
224+
} `json:"quantiles"`
220225
} `json:"httpRequestsAdaptiveGroups"`
221226

222227
HTTPRequestsEdgeCountryHost []struct {
@@ -541,6 +546,11 @@ query ($zoneIDs: [String!], $mintime: Time!, $maxtime: Time!, $limit: Int!) {
541546
clientCountryName
542547
clientRequestHTTPHost
543548
}
549+
quantiles {
550+
originResponseDurationMsP50
551+
originResponseDurationMsP95
552+
originResponseDurationMsP99
553+
}
544554
}
545555
httpRequestsEdgeCountryHost: httpRequestsAdaptiveGroups(limit: $limit, filter: { datetime_geq: $mintime, datetime_lt: $maxtime, requestSource_in: ["eyeball"] }) {
546556
count

‎prometheus.go‎

Lines changed: 100 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -20,46 +20,49 @@ func (mn MetricName) String() string {
2020
}
2121

2222
const (
23-
zoneRequestTotalMetricName MetricName = "cloudflare_zone_requests_total"
24-
zoneRequestCachedMetricName MetricName = "cloudflare_zone_requests_cached"
25-
zoneRequestSSLEncryptedMetricName MetricName = "cloudflare_zone_requests_ssl_encrypted"
26-
zoneRequestContentTypeMetricName MetricName = "cloudflare_zone_requests_content_type"
27-
zoneRequestCountryMetricName MetricName = "cloudflare_zone_requests_country"
28-
zoneRequestHTTPStatusMetricName MetricName = "cloudflare_zone_requests_status"
29-
zoneRequestBrowserMapMetricName MetricName = "cloudflare_zone_requests_browser_map_page_views_count"
30-
zoneRequestOriginStatusCountryHostMetricName MetricName = "cloudflare_zone_requests_origin_status_country_host"
31-
zoneRequestStatusCountryHostMetricName MetricName = "cloudflare_zone_requests_status_country_host"
32-
zoneBandwidthTotalMetricName MetricName = "cloudflare_zone_bandwidth_total"
33-
zoneBandwidthCachedMetricName MetricName = "cloudflare_zone_bandwidth_cached"
34-
zoneBandwidthSSLEncryptedMetricName MetricName = "cloudflare_zone_bandwidth_ssl_encrypted"
35-
zoneBandwidthContentTypeMetricName MetricName = "cloudflare_zone_bandwidth_content_type"
36-
zoneBandwidthCountryMetricName MetricName = "cloudflare_zone_bandwidth_country"
37-
zoneThreatsTotalMetricName MetricName = "cloudflare_zone_threats_total"
38-
zoneThreatsCountryMetricName MetricName = "cloudflare_zone_threats_country"
39-
zoneThreatsTypeMetricName MetricName = "cloudflare_zone_threats_type"
40-
zonePageviewsTotalMetricName MetricName = "cloudflare_zone_pageviews_total"
41-
zoneUniquesTotalMetricName MetricName = "cloudflare_zone_uniques_total"
42-
zoneColocationVisitsMetricName MetricName = "cloudflare_zone_colocation_visits"
43-
zoneColocationEdgeResponseBytesMetricName MetricName = "cloudflare_zone_colocation_edge_response_bytes"
44-
zoneColocationRequestsTotalMetricName MetricName = "cloudflare_zone_colocation_requests_total"
45-
zoneFirewallEventsCountMetricName MetricName = "cloudflare_zone_firewall_events_count"
46-
zoneHealthCheckEventsOriginCountMetricName MetricName = "cloudflare_zone_health_check_events_origin_count"
47-
workerRequestsMetricName MetricName = "cloudflare_worker_requests_count"
48-
workerErrorsMetricName MetricName = "cloudflare_worker_errors_count"
49-
workerCPUTimeMetricName MetricName = "cloudflare_worker_cpu_time"
50-
workerDurationMetricName MetricName = "cloudflare_worker_duration"
51-
poolHealthStatusMetricName MetricName = "cloudflare_zone_pool_health_status"
52-
poolRequestsTotalMetricName MetricName = "cloudflare_zone_pool_requests_total"
53-
poolOriginHealthStatusMetricName MetricName = "cloudflare_pool_origin_health_status"
54-
logpushFailedJobsAccountMetricName MetricName = "cloudflare_logpush_failed_jobs_account_count"
55-
logpushFailedJobsZoneMetricName MetricName = "cloudflare_logpush_failed_jobs_zone_count"
56-
r2StorageTotalMetricName MetricName = "cloudflare_r2_storage_total_bytes"
57-
r2StorageMetricName MetricName = "cloudflare_r2_storage_bytes"
58-
r2OperationMetricName MetricName = "cloudflare_r2_operation_count"
59-
tunnelInfoMetricName MetricName = "cloudflare_tunnel_info"
60-
tunnelHealthStatusMetricName MetricName = "cloudflare_tunnel_health_status"
61-
tunnelConnectorInfoMetricName MetricName = "cloudflare_tunnel_connector_info"
62-
tunnelConnectorActiveConnectionsMetricName MetricName = "cloudflare_tunnel_connector_active_connections"
23+
zoneRequestTotalMetricName MetricName = "cloudflare_zone_requests_total"
24+
zoneRequestCachedMetricName MetricName = "cloudflare_zone_requests_cached"
25+
zoneRequestSSLEncryptedMetricName MetricName = "cloudflare_zone_requests_ssl_encrypted"
26+
zoneRequestContentTypeMetricName MetricName = "cloudflare_zone_requests_content_type"
27+
zoneRequestCountryMetricName MetricName = "cloudflare_zone_requests_country"
28+
zoneRequestHTTPStatusMetricName MetricName = "cloudflare_zone_requests_status"
29+
zoneRequestBrowserMapMetricName MetricName = "cloudflare_zone_requests_browser_map_page_views_count"
30+
zoneRequestOriginStatusCountryHostMetricName MetricName = "cloudflare_zone_requests_origin_status_country_host"
31+
zoneRequestOriginStatusCountryHostP50MetricName MetricName = "cloudflare_zone_requests_origin_status_country_host_p50_ms"
32+
zoneRequestOriginStatusCountryHostP95MetricName MetricName = "cloudflare_zone_requests_origin_status_country_host_p95_ms"
33+
zoneRequestOriginStatusCountryHostP99MetricName MetricName = "cloudflare_zone_requests_origin_status_country_host_p99_ms"
34+
zoneRequestStatusCountryHostMetricName MetricName = "cloudflare_zone_requests_status_country_host"
35+
zoneBandwidthTotalMetricName MetricName = "cloudflare_zone_bandwidth_total"
36+
zoneBandwidthCachedMetricName MetricName = "cloudflare_zone_bandwidth_cached"
37+
zoneBandwidthSSLEncryptedMetricName MetricName = "cloudflare_zone_bandwidth_ssl_encrypted"
38+
zoneBandwidthContentTypeMetricName MetricName = "cloudflare_zone_bandwidth_content_type"
39+
zoneBandwidthCountryMetricName MetricName = "cloudflare_zone_bandwidth_country"
40+
zoneThreatsTotalMetricName MetricName = "cloudflare_zone_threats_total"
41+
zoneThreatsCountryMetricName MetricName = "cloudflare_zone_threats_country"
42+
zoneThreatsTypeMetricName MetricName = "cloudflare_zone_threats_type"
43+
zonePageviewsTotalMetricName MetricName = "cloudflare_zone_pageviews_total"
44+
zoneUniquesTotalMetricName MetricName = "cloudflare_zone_uniques_total"
45+
zoneColocationVisitsMetricName MetricName = "cloudflare_zone_colocation_visits"
46+
zoneColocationEdgeResponseBytesMetricName MetricName = "cloudflare_zone_colocation_edge_response_bytes"
47+
zoneColocationRequestsTotalMetricName MetricName = "cloudflare_zone_colocation_requests_total"
48+
zoneFirewallEventsCountMetricName MetricName = "cloudflare_zone_firewall_events_count"
49+
zoneHealthCheckEventsOriginCountMetricName MetricName = "cloudflare_zone_health_check_events_origin_count"
50+
workerRequestsMetricName MetricName = "cloudflare_worker_requests_count"
51+
workerErrorsMetricName MetricName = "cloudflare_worker_errors_count"
52+
workerCPUTimeMetricName MetricName = "cloudflare_worker_cpu_time"
53+
workerDurationMetricName MetricName = "cloudflare_worker_duration"
54+
poolHealthStatusMetricName MetricName = "cloudflare_zone_pool_health_status"
55+
poolRequestsTotalMetricName MetricName = "cloudflare_zone_pool_requests_total"
56+
poolOriginHealthStatusMetricName MetricName = "cloudflare_pool_origin_health_status"
57+
logpushFailedJobsAccountMetricName MetricName = "cloudflare_logpush_failed_jobs_account_count"
58+
logpushFailedJobsZoneMetricName MetricName = "cloudflare_logpush_failed_jobs_zone_count"
59+
r2StorageTotalMetricName MetricName = "cloudflare_r2_storage_total_bytes"
60+
r2StorageMetricName MetricName = "cloudflare_r2_storage_bytes"
61+
r2OperationMetricName MetricName = "cloudflare_r2_operation_count"
62+
tunnelInfoMetricName MetricName = "cloudflare_tunnel_info"
63+
tunnelHealthStatusMetricName MetricName = "cloudflare_tunnel_health_status"
64+
tunnelConnectorInfoMetricName MetricName = "cloudflare_tunnel_connector_info"
65+
tunnelConnectorActiveConnectionsMetricName MetricName = "cloudflare_tunnel_connector_active_connections"
6366
)
6467

6568
type MetricsSet map[MetricName]struct{}
@@ -123,6 +126,24 @@ var (
123126
}, []string{"zone", "account", "status", "country", "host"},
124127
)
125128

129+
zoneRequestOriginStatusCountryHostP50 = prometheus.NewGaugeVec(prometheus.GaugeOpts{
130+
Name: zoneRequestOriginStatusCountryHostP50MetricName.String(),
131+
Help: "P50 latency of not cached requests for zone per origin HTTP status per country per host, in ms",
132+
}, []string{"zone", "account", "status", "country", "host"},
133+
)
134+
135+
zoneRequestOriginStatusCountryHostP95 = prometheus.NewGaugeVec(prometheus.GaugeOpts{
136+
Name: zoneRequestOriginStatusCountryHostP95MetricName.String(),
137+
Help: "P95 latency of not cached requests for zone per origin HTTP status per country per host, in ms",
138+
}, []string{"zone", "account", "status", "country", "host"},
139+
)
140+
141+
zoneRequestOriginStatusCountryHostP99 = prometheus.NewGaugeVec(prometheus.GaugeOpts{
142+
Name: zoneRequestOriginStatusCountryHostP99MetricName.String(),
143+
Help: "P99 latency of not cached requests for zone per origin HTTP status per country per host, in ms",
144+
}, []string{"zone", "account", "status", "country", "host"},
145+
)
146+
126147
zoneRequestStatusCountryHost = prometheus.NewCounterVec(prometheus.CounterOpts{
127148
Name: zoneRequestStatusCountryHostMetricName.String(),
128149
Help: "Count of requests for zone per edge HTTP status per country per host",
@@ -397,6 +418,15 @@ func mustRegisterMetrics(deniedMetrics MetricsSet) {
397418
if !deniedMetrics.Has(zoneRequestOriginStatusCountryHostMetricName) {
398419
prometheus.MustRegister(zoneRequestOriginStatusCountryHost)
399420
}
421+
if !deniedMetrics.Has(zoneRequestOriginStatusCountryHostP50MetricName) {
422+
prometheus.MustRegister(zoneRequestOriginStatusCountryHostP50)
423+
}
424+
if !deniedMetrics.Has(zoneRequestOriginStatusCountryHostP95MetricName) {
425+
prometheus.MustRegister(zoneRequestOriginStatusCountryHostP95)
426+
}
427+
if !deniedMetrics.Has(zoneRequestOriginStatusCountryHostP99MetricName) {
428+
prometheus.MustRegister(zoneRequestOriginStatusCountryHostP99)
429+
}
400430
if !deniedMetrics.Has(zoneRequestStatusCountryHostMetricName) {
401431
prometheus.MustRegister(zoneRequestStatusCountryHost)
402432
}
@@ -818,6 +848,9 @@ func addHTTPAdaptiveGroups(z *zoneResp, name string, account string) {
818848
// Clear stale series for this zone/account
819849
label := prometheus.Labels{"zone": name, "account": account}
820850
zoneRequestOriginStatusCountryHost.DeletePartialMatch(label)
851+
zoneRequestOriginStatusCountryHostP50.DeletePartialMatch(label)
852+
zoneRequestOriginStatusCountryHostP95.DeletePartialMatch(label)
853+
zoneRequestOriginStatusCountryHostP99.DeletePartialMatch(label)
821854
zoneRequestStatusCountryHost.DeletePartialMatch(label)
822855

823856
for _, g := range z.HTTPRequestsAdaptiveGroups {
@@ -829,6 +862,33 @@ func addHTTPAdaptiveGroups(z *zoneResp, name string, account string) {
829862
"country": g.Dimensions.ClientCountryName,
830863
"host": g.Dimensions.ClientRequestHTTPHost,
831864
}).Add(float64(g.Count))
865+
866+
zoneRequestOriginStatusCountryHostP50.With(
867+
prometheus.Labels{
868+
"zone": name,
869+
"account": account,
870+
"status": strconv.Itoa(int(g.Dimensions.OriginResponseStatus)),
871+
"country": g.Dimensions.ClientCountryName,
872+
"host": g.Dimensions.ClientRequestHTTPHost,
873+
}).Set(float64(g.Quantile.OriginResponseDurationMsP50))
874+
875+
zoneRequestOriginStatusCountryHostP95.With(
876+
prometheus.Labels{
877+
"zone": name,
878+
"account": account,
879+
"status": strconv.Itoa(int(g.Dimensions.OriginResponseStatus)),
880+
"country": g.Dimensions.ClientCountryName,
881+
"host": g.Dimensions.ClientRequestHTTPHost,
882+
}).Set(float64(g.Quantile.OriginResponseDurationMsP95))
883+
884+
zoneRequestOriginStatusCountryHostP99.With(
885+
prometheus.Labels{
886+
"zone": name,
887+
"account": account,
888+
"status": strconv.Itoa(int(g.Dimensions.OriginResponseStatus)),
889+
"country": g.Dimensions.ClientCountryName,
890+
"host": g.Dimensions.ClientRequestHTTPHost,
891+
}).Set(float64(g.Quantile.OriginResponseDurationMsP99))
832892
}
833893

834894
for _, g := range z.HTTPRequestsEdgeCountryHost {

0 commit comments

Comments
 (0)