Update Jetty thread pool metrics (#18883)

Add new Jetty thread pool metrics to capture request-serving thread statistics: jetty/threadPool/utilized, jetty/threadPool/ready and jetty/threadPool/utilizationRate.

Unlike some of the existing metrics, these new metrics are intended to serve as an indicator for threads that would serve requests from the thread pool.

Also update the documentation for the new metrics and also fixes up descriptions for some existing Jetty metrics.

Author: abhishekrb19

docs/operations/metrics.md

@@ -149,10 +149,13 @@ to represent the task ID are deprecated and will be removed in a future release.
 |Metric|Description|Normal value|
 |------|-----------|------------|
 |`jetty/numOpenConnections`|Number of open jetty connections.|Not much higher than number of jetty threads.|
-|`jetty/threadPool/total`|Number of total workable threads allocated.|The number should equal to `threadPoolNumIdleThreads` + `threadPoolNumBusyThreads`.|
-|`jetty/threadPool/idle`|Number of idle threads.|Less than or equal to `threadPoolNumTotalThreads`. Non zero number means there is less work to do than configured capacity.|
-|`jetty/threadPool/busy`|Number of busy threads that has work to do from the worker queue.|Less than or equal to `threadPoolNumTotalThreads`.|
-|`jetty/threadPool/isLowOnThreads`|A rough indicator of whether number of total workable threads allocated is enough to handle the works in the work queue.|0|
+|`jetty/threadPool/total`|Number of total threads allocated. This includes internal threads and threads that are ready to serve requests.|Equals the total number of threads in the Jetty thread pool. i.e., `jetty/threadPool/ready` + `jetty/threadPool/utilized` + any reserved threads for internal Jetty usage.|
+|`jetty/threadPool/ready`|Number of threads that are ready to serve requests.|Equals the total number of usable threads in the Jetty thread pool to serve requests. i.e., `jetty/threadPool/idle` + any reserved threads for internal Jetty usage.|
+|`jetty/threadPool/utilized`|Number of threads currently in use by the thread pool to serve requests.|Value < `jetty/threadPool/ready`.|
+|`jetty/threadPool/utilizationRate`|Fraction of thread pool capacity currently in use to serve requests.|0.0 ≤ Value ≤ 1. A value of 0.0 means that the thread pool is not utilized, while a value of 1.0 means that the thread pool is fully utilized.|
+|`jetty/threadPool/idle`|Number of idle threads.|Value ≤ (`jetty/threadPool/ready` - any reserved threads for internal Jetty usage). A non-zero value means there is less work to do than configured capacity.|
+|`jetty/threadPool/busy`|Number of busy threads that have work to do from the worker queue.|Value ≤ (`jetty/threadPool/utilized` + any reserved threads for internal Jetty usage).|
+|`jetty/threadPool/isLowOnThreads`|A rough indicator of whether the number of total workable threads allocated is enough to handle the work in the work queue.|0|
 |`jetty/threadPool/min`|Number of minimum threads allocatable.|`druid.server.http.numThreads` plus a small fixed number of threads allocated for Jetty acceptors and selectors.|
 |`jetty/threadPool/max`|Number of maximum threads allocatable.|`druid.server.http.numThreads` plus a small fixed number of threads allocated for Jetty acceptors and selectors.|
 |`jetty/threadPool/queueSize`|Size of the worker queue.|Not much higher than `druid.server.http.queueSize`.|

extensions-contrib/opentsdb-emitter/src/main/resources/defaultMetrics.json

@@ -29,6 +29,9 @@
   ],
   "jetty/numOpenConnections": [],
   "jetty/threadPool/total": [],
+  "jetty/threadPool/ready": [],
+  "jetty/threadPool/utilized": [],
+  "jetty/threadPool/utilizationRate": [],
   "jetty/threadPool/idle": [],
   "jetty/threadPool/busy": [],
   "jetty/threadPool/isLowOnThreads": [],

extensions-contrib/prometheus-emitter/src/main/resources/defaultMetrics.json

@@ -37,6 +37,9 @@
 
   "jetty/numOpenConnections" : { "dimensions" : [], "type" : "gauge", "help": "Number of open jetty connections."},
   "jetty/threadPool/total" : { "dimensions" : [], "type" : "gauge", "help": "Number of total workable threads allocated."},
+  "jetty/threadPool/ready" : { "dimensions" : [], "type" : "gauge", "help": "Number of threads that are ready to serve requests."},
+  "jetty/threadPool/utilized" : { "dimensions" : [], "type" : "gauge", "help": "Number of threads currently in use to serve requests."},
+  "jetty/threadPool/utilizationRate" : { "dimensions" : [], "type" : "gauge", "help": "Fraction of thread pool capacity currently in use to serve requests."},
   "jetty/threadPool/idle" : { "dimensions" : [], "type" : "gauge", "help": "Number of idle threads."},
   "jetty/threadPool/busy" : { "dimensions" : [], "type" : "gauge", "help": "Number of busy threads that has work to do from the worker queue."},
   "jetty/threadPool/isLowOnThreads" : { "dimensions" : [], "type" : "gauge", "help": "A rough indicator of whether number of total workable threads allocated is enough to handle the works in the work queue."},

extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json

@@ -159,6 +159,9 @@
 
   "jetty/numOpenConnections": { "dimensions" : [], "type" : "gauge" },
   "jetty/threadPool/total": { "dimensions" : [], "type" : "gauge" },
+  "jetty/threadPool/ready": { "dimensions" : [], "type" : "gauge" },
+  "jetty/threadPool/utilized": { "dimensions" : [], "type" : "gauge" },
+  "jetty/threadPool/utilizationRate": { "dimensions" : [], "type" : "gauge" },
   "jetty/threadPool/idle": { "dimensions" : [], "type" : "gauge" },
   "jetty/threadPool/busy": { "dimensions" : [], "type" : "gauge" },
   "jetty/threadPool/isLowOnThreads": { "dimensions" : [], "type" : "gauge" },

server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerModule.java

@@ -528,6 +528,11 @@ public boolean doMonitor(ServiceEmitter emitter)
       if (jettyServerThreadPool != null) {
         emitter.emit(builder.setMetric("jetty/threadPool/total", jettyServerThreadPool.getThreads()));
         emitter.emit(builder.setMetric("jetty/threadPool/idle", jettyServerThreadPool.getIdleThreads()));
+
+        emitter.emit(builder.setMetric("jetty/threadPool/ready", jettyServerThreadPool.getReadyThreads()));
+        emitter.emit(builder.setMetric("jetty/threadPool/utilized", jettyServerThreadPool.getUtilizedThreads()));
+        emitter.emit(builder.setMetric("jetty/threadPool/utilizationRate", jettyServerThreadPool.getUtilizationRate()));
+
         emitter.emit(builder.setMetric("jetty/threadPool/isLowOnThreads", jettyServerThreadPool.isLowOnThreads() ? 1 : 0));
         emitter.emit(builder.setMetric("jetty/threadPool/min", jettyServerThreadPool.getMinThreads()));
         emitter.emit(builder.setMetric("jetty/threadPool/max", jettyServerThreadPool.getMaxThreads()));

server/src/test/java/org/apache/druid/server/initialization/jetty/JettyServerModuleTest.java

@@ -53,6 +53,9 @@ public void testJettyServerModule()
     Mockito.when(jettyServerThreadPool.getMaxThreads()).thenReturn(100);
     Mockito.when(jettyServerThreadPool.getQueueSize()).thenReturn(50);
     Mockito.when(jettyServerThreadPool.getBusyThreads()).thenReturn(60);
+    Mockito.when(jettyServerThreadPool.getReadyThreads()).thenReturn(20);
+    Mockito.when(jettyServerThreadPool.getUtilizedThreads()).thenReturn(5);
+    Mockito.when(jettyServerThreadPool.getUtilizationRate()).thenReturn(0.45);
 
     JettyServerModule.JettyMonitor jettyMonitor = new JettyServerModule.JettyMonitor();
 
@@ -67,6 +70,9 @@ public void testJettyServerModule()
     serviceEmitter.verifyValue("jetty/threadPool/max", 100);
     serviceEmitter.verifyValue("jetty/threadPool/queueSize", 50);
     serviceEmitter.verifyValue("jetty/threadPool/busy", 60);
+    serviceEmitter.verifyValue("jetty/threadPool/ready", 20);
+    serviceEmitter.verifyValue("jetty/threadPool/utilized", 5);
+    serviceEmitter.verifyValue("jetty/threadPool/utilizationRate", 0.45);
   }
 
   @Test