Changes to fix #116

Many quality of life Improvements, including:
* Many snake_case labels in API Priority & Fairness panels
* verb and resource filters on more request panels
* Updated obsolete timeseries
* Units and legends updates
* Panels resize and organized

Author: afcollins

assets/api-performance-overview/panels.libsonnet

@@ -31,6 +31,7 @@ local g = import 'github.com/grafana/grafonnet/gen/grafonnet-latest/main.libsonn
       + options.tooltip.withSort('desc')
       + timeSeries.queryOptions.withTimeFrom(null)
       + timeSeries.queryOptions.withTimeShift(null)
+      + options.legend.withSortBy('Max')
       + options.legend.withSortDesc(true),
 
     legendRightPlacement(title, unit, targets, gridPos):

assets/api-performance-overview/queries.libsonnet

@@ -5,7 +5,7 @@ local prometheus = g.query.prometheus;
 {
   request_duration_99th_quantile: {
     query():
-      prometheus.withExpr('histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{apiserver=~"$apiserver",instance=~"$instance",resource=~"$resource",subresource!="log",verb!~"WATCH|WATCHLIST|PROXY"}[$interval])) by(verb,le))')
+      prometheus.withExpr('histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{apiserver=~"$apiserver",instance=~"$instance",resource=~"$resource",subresource!="log",verb=~"$verb",verb!~"WATCH|WATCHLIST|PROXY"}[$interval])) by(verb,le))')
       + prometheus.withFormat('time_series')
       + prometheus.withIntervalFactor(2)
       + prometheus.withLegendFormat('{{verb}}')
@@ -23,7 +23,7 @@ local prometheus = g.query.prometheus;
 
   requestDuarationByResource: {
     query():
-      prometheus.withExpr('histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{apiserver=~"$apiserver",instance=~"$instance",resource=~"$resource",subresource!="log",verb!~"WATCH|WATCHLIST|PROXY"}[$interval])) by(resource,le))')
+      prometheus.withExpr('histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{apiserver=~"$apiserver",instance=~"$instance",resource=~"$resource",subresource!="log",verb=~"$verb",verb!~"WATCH|WATCHLIST|PROXY"}[$interval])) by(resource,le))')
       + prometheus.withFormat('time_series')
       + prometheus.withIntervalFactor(2)
       + prometheus.withLegendFormat('{{resource}}')
@@ -75,7 +75,7 @@ local prometheus = g.query.prometheus;
 
   requestRateDropped: {
     query():
-      prometheus.withExpr('sum(rate(apiserver_dropped_requests_total{instance=~"$instance"}[$interval])) by (requestKind)')
+      prometheus.withExpr('sum(rate(apiserver_request_terminations_total{instance=~"$instance"}[$interval])) by (verb)')
       + prometheus.withFormat('time_series')
       + prometheus.withIntervalFactor(2)
       + prometheus.withLegendFormat('')
@@ -102,7 +102,7 @@ local prometheus = g.query.prometheus;
 
   requestsLongRunning: {
     query():
-      prometheus.withExpr('sum(apiserver_longrunning_gauge{instance=~"$instance",resource=~"$resource",verb=~"$verb"}) by(instance)')
+      prometheus.withExpr('sum(apiserver_longrunning_requests{instance=~"$instance",resource=~"$resource",verb=~"$verb"}) by(instance)')
       + prometheus.withFormat('time_series')
       + prometheus.withIntervalFactor(2)
       + prometheus.withLegendFormat('{{instance}}')
@@ -111,19 +111,10 @@ local prometheus = g.query.prometheus;
 
   requestInFlight: {
     query():
-      prometheus.withExpr('sum(apiserver_current_inflight_requests{instance=~"$instance"}) by (instance,requestKind)')
+      prometheus.withExpr('sum(apiserver_current_inflight_requests{instance=~"$instance"}) by (instance,request_kind)')
       + prometheus.withFormat('time_series')
       + prometheus.withIntervalFactor(2)
-      + prometheus.withLegendFormat('{{requestKind}}-{{instance}}')
-      + prometheus.withDatasource('$Datasource'),
-  },
-
-  requestRejectPandF: {
-    query():
-      prometheus.withExpr('sum(rate(apiserver_flowcontrol_rejected_requests_total{instance=~"$instance",flowSchema=~"$flowSchema",priorityLevel=~"$priorityLevel"}[$interval])) by (reason)')
-      + prometheus.withFormat('time_series')
-      + prometheus.withIntervalFactor(2)
-      + prometheus.withLegendFormat('')
+      + prometheus.withLegendFormat('{{request_kind}}-{{instance}}')
       + prometheus.withDatasource('$Datasource'),
   },
 
@@ -138,55 +129,55 @@ local prometheus = g.query.prometheus;
 
   requestQueueLengthPandF: {
     query():
-      prometheus.withExpr('histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_queue_length_after_enqueue_bucket{instance=~"$instance",flowSchema=~"$flowSchema",priorityLevel=~"$priorityLevel"}[$interval])) by(flowSchema, priorityLevel, le))')
+      prometheus.withExpr('histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_queue_length_after_enqueue_bucket{instance=~"$instance",flow_schema=~"$flow_schema",priority_level=~"$priority_level"}[$interval])) by(flow_schema, priority_level, le))')
       + prometheus.withFormat('time_series')
       + prometheus.withIntervalFactor(2)
-      + prometheus.withLegendFormat('{{flowSchema}}:{{priorityLevel}}')
+      + prometheus.withLegendFormat('{{flow_schema}}:{{priority_level}}')
       + prometheus.withDatasource('$Datasource'),
   },
 
   requestWaitDuration99QuatilePandF: {
     query():
-      prometheus.withExpr('histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_wait_duration_seconds_bucket{instance=~"$instance"}[5m])) by(flow_schema, priority_level, le))')
+      prometheus.withExpr('histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_wait_duration_seconds_bucket{instance=~"$instance",flow_schema=~"$flow_schema",priority_level=~"$priority_level"}[5m])) by(flow_schema, priority_level, le))')
       + prometheus.withFormat('time_series')
       + prometheus.withIntervalFactor(2)
-      + prometheus.withLegendFormat('')
+      + prometheus.withLegendFormat('{{flow_schema}}:{{priority_level}}')
       + prometheus.withDatasource('$Datasource'),
   },
 
   requestDispatchRatePandF: {
     query():
-      prometheus.withExpr('sum(rate(apiserver_flowcontrol_dispatched_requests_total{instance=~"$instance",flowSchema=~"$flowSchema",priorityLevel=~"$priorityLevel"}[$interval])) by(flowSchema,priorityLevel)')
+      prometheus.withExpr('sum(rate(apiserver_flowcontrol_dispatched_requests_total{instance=~"$instance",flow_schema=~"$flow_schema",priority_level=~"$priority_level"}[$interval])) by(flow_schema,priority_level)')
       + prometheus.withFormat('time_series')
       + prometheus.withIntervalFactor(2)
-      + prometheus.withLegendFormat('{{flowSchema}}:{{priorityLevel}}')
+      + prometheus.withLegendFormat('{{flow_schema}}:{{priority_level}}')
       + prometheus.withDatasource('$Datasource'),
   },
 
   requestExecutionDurationPandF: {
     query():
-      prometheus.withExpr('histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_execution_seconds_bucket{instance=~"$instance",flowSchema=~"$flowSchema",priorityLevel=~"$priorityLevel"}[$interval])) by(flowSchema, priorityLevel, le))')
+      prometheus.withExpr('histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_execution_seconds_bucket{instance=~"$instance",flow_schema=~"$flow_schema",priority_level=~"$priority_level"}[$interval])) by(flow_schema, priority_level, le))')
       + prometheus.withFormat('time_series')
       + prometheus.withIntervalFactor(2)
-      + prometheus.withLegendFormat('{{flowSchema}}:{{priorityLevel}}')
+      + prometheus.withLegendFormat('{{flow_schema}}:{{priority_level}}')
       + prometheus.withDatasource('$Datasource'),
   },
 
   pendingInQueuePandF: {
     query():
-      prometheus.withExpr('sum(apiserver_flowcontrol_current_inqueue_requests{instance=~"$instance",flowSchema=~"$flowSchema",priorityLevel=~"$priorityLevel"}) by (flowSchema,priorityLevel)')
+      prometheus.withExpr('sum(apiserver_flowcontrol_current_inqueue_requests{instance=~"$instance",flow_schema=~"$flow_schema",priority_level=~"$priority_level"}) by (flow_schema,priority_level)')
       + prometheus.withFormat('time_series')
       + prometheus.withIntervalFactor(2)
-      + prometheus.withLegendFormat('{{flowSchema}}:{{priorityLevel}}')
+      + prometheus.withLegendFormat('{{flow_schema}}:{{priority_level}}')
       + prometheus.withDatasource('$Datasource'),
   },
 
   concurrencyLimitByKubeapiserverPandF: {
     query():
-      prometheus.withExpr('sum(apiserver_flowcontrol_request_concurrency_limit{instance=~".*:6443",priorityLevel=~"$priorityLevel"}) by (instance,priorityLevel)')
+      prometheus.withExpr('sum(apiserver_flowcontrol_request_concurrency_in_use{instance=~".*:6443",priority_level=~"$priority_level"}) by (instance,flow_schema)')
       + prometheus.withFormat('time_series')
       + prometheus.withIntervalFactor(2)
-      + prometheus.withLegendFormat('')
+      + prometheus.withLegendFormat('{{instance}}:{{flow_schema}}')
       + prometheus.withDatasource('$Datasource'),
   },
 }

assets/api-performance-overview/variables.libsonnet

@@ -12,7 +12,7 @@ local var = g.dashboard.variable;
     + var.query.withDatasourceFromVariable(self.Datasource)
     + var.query.selectionOptions.withMulti(false)
     + var.query.selectionOptions.withIncludeAll(true)
-    + var.query.generalOptions.withLabel('apisever')
+    + var.query.generalOptions.withLabel('apiserver')
     + var.query.withRefresh(2),
 
   instance:
@@ -47,16 +47,16 @@ local var = g.dashboard.variable;
     + var.query.generalOptions.withLabel('verb')
     + var.query.withRefresh(2),
 
-  flowSchema:
-    var.query.new('flowSchema', 'label_values(flowSchema)')
+  flow_schema:
+    var.query.new('flow_schema', 'label_values(flow_schema)')
     + var.query.withDatasourceFromVariable(self.Datasource)
     + var.query.selectionOptions.withMulti(false)
     + var.query.selectionOptions.withIncludeAll(true)
     + var.query.generalOptions.withLabel('flow-schema')
     + var.query.withRefresh(2),
 
-  priorityLevel:
-    var.query.new('priorityLevel', 'label_values(priorityLevel)')
+  priority_level:
+    var.query.new('priority_level', 'label_values(priority_level)')
     + var.query.withDatasourceFromVariable(self.Datasource)
     + var.query.selectionOptions.withMulti(false)
     + var.query.selectionOptions.withIncludeAll(true)

assets/hypershift-perf-dashboard/queries.libsonnet

@@ -708,7 +708,7 @@ local prometheus = g.query.prometheus;
 
   requests_dropped_rate: {
     query():
-      prometheus.withExpr('sum(rate(apiserver_dropped_requests_total{namespace=~"$namespace"}[2m])) by (requestKind)')
+      prometheus.withExpr('sum(rate(apiserver_dropped_requests_total{namespace=~"$namespace"}[2m])) by (request_kind)')
       + prometheus.withFormat('time_series')
       + prometheus.withIntervalFactor(2)
       + prometheus.withDatasource('P1BA917A37525EDF3'),
@@ -742,9 +742,9 @@ local prometheus = g.query.prometheus;
 
   request_in_flight: {
     query():
-      prometheus.withExpr('sum(apiserver_current_inflight_requests{namespace=~"$namespace"}) by (instance,requestKind)')
+      prometheus.withExpr('sum(apiserver_current_inflight_requests{namespace=~"$namespace"}) by (instance,request_kind)')
       + prometheus.withFormat('time_series')
-      + prometheus.withLegendFormat('{{requestKind}}-{{instance}}')
+      + prometheus.withLegendFormat('{{request_kind}}-{{instance}}')
       + prometheus.withIntervalFactor(2)
       + prometheus.withDatasource('P1BA917A37525EDF3'),
   },
@@ -768,54 +768,54 @@ local prometheus = g.query.prometheus;
 
   pf_request_queue_length: {
     query():
-      prometheus.withExpr('histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_queue_length_after_enqueue_bucket{namespace=~"$namespace"}[2m])) by(flowSchema, priorityLevel, le))')
+      prometheus.withExpr('histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_queue_length_after_enqueue_bucket{namespace=~"$namespace"}[2m])) by(flow_schema, priority_level, le))')
       + prometheus.withFormat('time_series')
-      + prometheus.withLegendFormat('{{flowSchema}}:{{priorityLevel}}')
+      + prometheus.withLegendFormat('{{flow_schema}}:{{priority_level}}')
       + prometheus.withIntervalFactor(2)
       + prometheus.withDatasource('P1BA917A37525EDF3'),
   },
 
   pf_request_wait_duration_99th_quartile: {
     query():
-      prometheus.withExpr('histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_wait_duration_seconds_bucket{namespace=~"$namespace"}[2m])) by(flowSchema, priorityLevel, le))')
+      prometheus.withExpr('histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_wait_duration_seconds_bucket{namespace=~"$namespace"}[2m])) by(flow_schema, priority_level, le))')
       + prometheus.withFormat('time_series')
-      + prometheus.withLegendFormat('{{flowSchema}}:{{priorityLevel}}')
+      + prometheus.withLegendFormat('{{flow_schema}}:{{priority_level}}')
       + prometheus.withIntervalFactor(2)
       + prometheus.withDatasource('P1BA917A37525EDF3'),
   },
 
   pf_request_execution_duration: {
     query():
-      prometheus.withExpr('histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_execution_seconds_bucket{namespace=~"$namespace"}[2m])) by(flowSchema, priorityLevel, le))')
+      prometheus.withExpr('histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_execution_seconds_bucket{namespace=~"$namespace"}[2m])) by(flow_schema, priority_level, le))')
       + prometheus.withFormat('time_series')
-      + prometheus.withLegendFormat('{{flowSchema}}:{{priorityLevel}}')
+      + prometheus.withLegendFormat('{{flow_schema}}:{{priority_level}}')
       + prometheus.withIntervalFactor(2)
       + prometheus.withDatasource('P1BA917A37525EDF3'),
   },
 
   pf_request_dispatch_rate: {
     query():
-      prometheus.withExpr('sum(rate(apiserver_flowcontrol_dispatched_requests_total{namespace=~"$namespace"}[2m])) by(flowSchema,priorityLevel)')
+      prometheus.withExpr('sum(rate(apiserver_flowcontrol_dispatched_requests_total{namespace=~"$namespace"}[2m])) by(flow_schema,priority_level)')
       + prometheus.withFormat('time_series')
-      + prometheus.withLegendFormat('{{flowSchema}}:{{priorityLevel}}')
+      + prometheus.withLegendFormat('{{flow_schema}}:{{priority_level}}')
       + prometheus.withIntervalFactor(2)
       + prometheus.withDatasource('P1BA917A37525EDF3'),
   },
 
   pf_concurrency_limit: {
     query():
-      prometheus.withExpr('sum(apiserver_flowcontrol_request_concurrency_limit{namespace=~"$namespace"}) by (priorityLevel)')
+      prometheus.withExpr('sum(apiserver_flowcontrol_request_concurrency_limit{namespace=~"$namespace"}) by (priority_level)')
       + prometheus.withFormat('time_series')
-      + prometheus.withLegendFormat('{{priorityLevel}}')
+      + prometheus.withLegendFormat('{{priority_level}}')
       + prometheus.withIntervalFactor(2)
       + prometheus.withDatasource('P1BA917A37525EDF3'),
   },
 
   pf_pending_in_queue: {
     query():
-      prometheus.withExpr('sum(apiserver_flowcontrol_current_inqueue_requests{namespace=~"$namespace"}) by (flowSchema,priorityLevel)')
+      prometheus.withExpr('sum(apiserver_flowcontrol_current_inqueue_requests{namespace=~"$namespace"}) by (flow_schema,priority_level)')
       + prometheus.withFormat('time_series')
-      + prometheus.withLegendFormat('{{flowSchema}}:{{priorityLevel}}')
+      + prometheus.withLegendFormat('{{flow_schema}}:{{priority_level}}')
       + prometheus.withIntervalFactor(2)
       + prometheus.withDatasource('P1BA917A37525EDF3'),
   },

templates/General/api-performance-overview.jsonnet

@@ -23,28 +23,27 @@ g.dashboard.new('API Performance Dashboard')
   variables.resource,
   variables.code,
   variables.verb,
-  variables.flowSchema,
-  variables.priorityLevel,
+  variables.flow_schema,
+  variables.priority_level,
   variables.interval,
 ])
 + g.dashboard.withPanels([
-  panels.timeSeries.legendRightPlacement('request duration - 99th quantile', 'short', queries.request_duration_99th_quantile.query(), { x: 0, y: 0, w: 12, h: 8 }),
+  panels.timeSeries.legendRightPlacement('request duration - 99th quantile', 's', queries.request_duration_99th_quantile.query(), { x: 0, y: 0, w: 12, h: 8 }),
   panels.timeSeries.legendRightPlacement('request rate - by instance', 'short', queries.requestRateByInstance.query(), { x: 12, y: 0, w: 12, h: 8 }),
-  panels.timeSeries.legendRightPlacement('request duration - 99th quantile - by resource', 'short', queries.requestDuarationByResource.query(), { x: 0, y: 8, w: 12, h: 8 }),
+  panels.timeSeries.legendRightPlacement('request duration - 99th quantile - by resource', 's', queries.requestDuarationByResource.query(), { x: 0, y: 8, w: 12, h: 8 }),
   panels.timeSeries.legendRightPlacement('request rate - by resource', 'short', queries.requestRateByResource.query(), { x: 12, y: 8, w: 12, h: 8 }),
-  panels.timeSeries.legendBottomPlacement('request duration - read vs write', 'short', queries.requestDurationReadWrite.query(), { x: 0, y: 16, w: 12, h: 8 }),
+  panels.timeSeries.legendBottomPlacement('request duration - read vs write', 's', queries.requestDurationReadWrite.query(), { x: 0, y: 16, w: 12, h: 8 }),
   panels.timeSeries.legendBottomPlacement('request rate - read vs write', 'short', queries.requestRateReadWrite.query(), { x: 12, y: 16, w: 12, h: 8 }),
   panels.timeSeries.legendBottomPlacement('requests dropped rate', 'short', queries.requestRateDropped.query(), { x: 0, y: 24, w: 12, h: 8 }),
   panels.timeSeries.legendBottomPlacement('requests terminated rate', 'short', queries.requestRateTerminated.query(), { x: 12, y: 24, w: 12, h: 8 }),
   panels.timeSeries.legendRightPlacement('requests status rate', 'short', queries.requestRateStatus.query(), { x: 0, y: 32, w: 12, h: 8 }),
   panels.timeSeries.legendRightPlacement('long running requests', 'short', queries.requestsLongRunning.query(), { x: 12, y: 32, w: 12, h: 8 }),
   panels.timeSeries.legendRightPlacement('request in flight', 'short', queries.requestInFlight.query(), { x: 0, y: 40, w: 12, h: 8 }),
-  panels.timeSeries.legendRightPlacement('p&f - requests rejected', 'short', queries.requestRejectPandF.query(), { x: 12, y: 40, w: 12, h: 8 }),
-  panels.timeSeries.legendRightPlacement('response size - 99th quantile', 'short', queries.responseSize99Quatile.query(), { x: 0, y: 48, w: 12, h: 8 }),
-  panels.timeSeries.legendRightPlacement('p&f - request queue length', 'short', queries.requestQueueLengthPandF.query(), { x: 12, y: 48, w: 12, h: 8 }),
-  panels.timeSeries.withRequestWaitDurationAggregations('p&f - request wait duration - 99th quantile', 'short', queries.requestWaitDuration99QuatilePandF.query(), { x: 0, y: 56, w: 24, h: 8 }),
+  panels.timeSeries.legendRightPlacement('response size - 99th quantile', 'bytes', queries.responseSize99Quatile.query(), { x: 12, y: 40, w: 12, h: 8 }),
+  panels.timeSeries.legendRightPlacement('p&f - request queue length', 'short', queries.requestQueueLengthPandF.query(), { x: 0, y: 48, w: 12, h: 8 }),
+  panels.timeSeries.withRequestWaitDurationAggregations('p&f - request wait duration - 99th quantile', 's', queries.requestWaitDuration99QuatilePandF.query(), { x: 12, y: 48, w: 12, h: 8 }),
   panels.timeSeries.legendRightPlacement('p&f - request dispatch rate', 'short', queries.requestDispatchRatePandF.query(), { x: 0, y: 64, w: 12, h: 8 }),
-  panels.timeSeries.legendRightPlacement('p&f - request execution duration', 'short', queries.requestExecutionDurationPandF.query(), { x: 12, y: 64, w: 12, h: 8 }),
+  panels.timeSeries.legendRightPlacement('p&f - request execution duration', 's', queries.requestExecutionDurationPandF.query(), { x: 12, y: 64, w: 12, h: 8 }),
   panels.timeSeries.legendRightPlacement('p&f - pending in queue', 'short', queries.pendingInQueuePandF.query(), { x: 0, y: 72, w: 12, h: 8 }),
   panels.timeSeries.legendRightPlacement('p&f - concurrency limit by kube-apiserver', 'short', queries.concurrencyLimitByKubeapiserverPandF.query(), { x: 12, y: 72, w: 12, h: 8 }),
 ])