FEATURE: Add new metrics busy_threads from Puma (#333)

benoittgt · web-flow · commit 8e3c307fb681 · 2025-05-09T14:58:52.000+08:00
New Puma version has a [nice addition](puma/puma#3517) of metric on busy threads. > busy_threads: running - how many threads are waiting to receive work + how many requests are waiting for a thread to pick them up. this is a "wholistic" stat reflecting the overall current state of work to be done and the capacity to do it. [source](https://github.com/puma/puma/blob/master/docs/stats.md#single-mode-and-individual-workers-in-cluster-mode)
diff --git a/CHANGELOG b/CHANGELOG
@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+- FEATURE: Added puma_busy_threads metric that provides a holistic view of server workload by calculating (active threads - idle threads) + queued requests
+
 ## [2.2.0] - 2024-12-05
 
 ### Added
diff --git a/README.md b/README.md
@@ -609,15 +609,16 @@ end
 
 #### Metrics collected by Puma Instrumentation
 
-| Type  | Name                        | Description                                                 |
-| ---   | ---                         | ---                                                         |
-| Gauge | `puma_workers`              | Number of puma workers                                      |
-| Gauge | `puma_booted_workers`       | Number of puma workers booted                               |
-| Gauge | `puma_old_workers`          | Number of old puma workers                                  |
-| Gauge | `puma_running_threads`      | Number of puma threads currently running                    |
-| Gauge | `puma_request_backlog`      | Number of requests waiting to be processed by a puma thread |
-| Gauge | `puma_thread_pool_capacity` | Number of puma threads available at current scale           |
-| Gauge | `puma_max_threads`          | Number of puma threads at available at max scale            |
+| Type  | Name                        | Description                                                                                                         |
+| ---   | ---                         | ---                                                                                                                 |
+| Gauge | `puma_workers`              | Number of puma workers                                                                                              |
+| Gauge | `puma_booted_workers`       | Number of puma workers booted                                                                                       |
+| Gauge | `puma_old_workers`          | Number of old puma workers                                                                                          |
+| Gauge | `puma_running_threads`      | How many threads are spawned. A spawned thread may be busy processing a request or waiting for a new request        |
+| Gauge | `puma_request_backlog`      | Number of requests waiting to be processed by a puma thread                                                         |
+| Gauge | `puma_thread_pool_capacity` | Number of puma threads available at current scale                                                                   |
+| Gauge | `puma_max_threads`          | Number of puma threads at available at max scale                                                                    |
+| Gauge | `puma_busy_threads`         | Running - how many threads are waiting to receive work + how many requests are waiting for a thread to pick them up |
 
 All metrics may have a `phase` label and all custom labels provided with the `labels` option.
 
diff --git a/lib/prometheus_exporter/instrumentation/puma.rb b/lib/prometheus_exporter/instrumentation/puma.rb
@@ -61,11 +61,13 @@ def collect_worker_status(metric, status)
       metric[:running_threads] ||= 0
       metric[:thread_pool_capacity] ||= 0
       metric[:max_threads] ||= 0
+      metric[:busy_threads] ||= 0
 
       metric[:request_backlog] += status["backlog"]
       metric[:running_threads] += status["running"]
       metric[:thread_pool_capacity] += status["pool_capacity"]
       metric[:max_threads] += status["max_threads"]
+      metric[:busy_threads] += status["busy_threads"]
     end
   end
 end
diff --git a/lib/prometheus_exporter/server/puma_collector.rb b/lib/prometheus_exporter/server/puma_collector.rb
@@ -13,6 +13,10 @@ class PumaCollector < TypeCollector
       max_threads: "Number of puma threads at available at max scale.",
     }
 
+    if defined?(::Puma::Const) && Gem::Version.new(::Puma::Const::VERSION) >= Gem::Version.new('6.6.0')
+      PUMA_GAUGES[:busy_threads] = "Wholistic stat reflecting the overall current state of work to be done and the capacity to do it"
+    end
+
     def initialize
       @puma_metrics = MetricsContainer.new(ttl: MAX_PUMA_METRIC_AGE)
       @puma_metrics.filter = ->(new_metric, old_metric) do
diff --git a/test/server/collector_test.rb b/test/server/collector_test.rb
@@ -1038,7 +1038,7 @@ def test_it_can_collect_puma_metrics
     mock_puma = Minitest::Mock.new
     mock_puma.expect(
       :stats,
-      '{ "workers": 1, "phase": 0, "booted_workers": 1, "old_workers": 0, "worker_status": [{ "pid": 87819, "index": 0, "phase": 0, "booted": true, "last_checkin": "2018-10-16T11:50:31Z", "last_status": { "backlog":0, "running":8, "pool_capacity":32, "max_threads": 32 } }] }',
+      '{ "workers": 1, "phase": 0, "booted_workers": 1, "old_workers": 0, "worker_status": [{ "pid": 87819, "index": 0, "phase": 0, "booted": true, "last_checkin": "2018-10-16T11:50:31Z", "last_status": { "backlog":0, "running":8, "pool_capacity":32, "max_threads": 32, "busy_threads": 4 } }] }',
     )
 
     instrument = PrometheusExporter::Instrumentation::Puma.new
@@ -1061,6 +1061,10 @@ def test_it_can_collect_puma_metrics
       result.include?('puma_thread_pool_capacity{phase="0",service="service1"} 32'),
       "has pool capacity",
     )
+    assert(
+      result.include?('puma_busy_threads{phase="0",service="service1"} 4'),
+      "has total busy threads",
+    )
     mock_puma.verify
   end
 
@@ -1071,7 +1075,7 @@ def test_it_can_collect_puma_metrics_with_metric_labels
     mock_puma = Minitest::Mock.new
     mock_puma.expect(
       :stats,
-      '{ "workers": 1, "phase": 0, "booted_workers": 1, "old_workers": 0, "worker_status": [{ "pid": 87819, "index": 0, "phase": 0, "booted": true, "last_checkin": "2018-10-16T11:50:31Z", "last_status": { "backlog":0, "running":8, "pool_capacity":32, "max_threads": 32 } }] }',
+      '{ "workers": 1, "phase": 0, "booted_workers": 1, "old_workers": 0, "worker_status": [{ "pid": 87819, "index": 0, "phase": 0, "booted": true, "last_checkin": "2018-10-16T11:50:31Z", "last_status": { "backlog":0, "running":8, "pool_capacity":32, "max_threads": 32, "busy_threads": 4 } }] }',
     )
 
     instrument = PrometheusExporter::Instrumentation::Puma.new({ foo: "bar" })
@@ -1094,6 +1098,10 @@ def test_it_can_collect_puma_metrics_with_metric_labels
       result.include?('puma_thread_pool_capacity{phase="0",service="service1",foo="bar"} 32'),
       "has pool capacity",
     )
+    assert(
+      result.include?('puma_busy_threads{phase="0",service="service1",foo="bar"} 4'),
+      "has total busy threads",
+    )
     mock_puma.verify
   end
 
diff --git a/test/server/puma_collector_test.rb b/test/server/puma_collector_test.rb
@@ -25,6 +25,7 @@ def test_collecting_metrics_for_different_hosts_without_custom_labels
       "running_threads" => 4,
       "thread_pool_capacity" => 10,
       "max_threads" => 10,
+      "busy_threads" => 2,
     )
 
     collector.collect(
@@ -39,6 +40,7 @@ def test_collecting_metrics_for_different_hosts_without_custom_labels
       "running_threads" => 9,
       "thread_pool_capacity" => 10,
       "max_threads" => 10,
+      "busy_threads" => 3,
     )
 
     # overwriting previous metrics from first host
@@ -54,10 +56,11 @@ def test_collecting_metrics_for_different_hosts_without_custom_labels
       "running_threads" => 8,
       "thread_pool_capacity" => 10,
       "max_threads" => 10,
+      "busy_threads" => 4,
     )
 
     metrics = collector.metrics
-    assert_equal 7, metrics.size
+    assert_equal 8, metrics.size
     assert_equal "puma_workers{phase=\"0\"} 3", metrics.first.metric_text
   end
 
@@ -74,6 +77,7 @@ def test_collecting_metrics_for_different_hosts_with_custom_labels
       "running_threads" => 4,
       "thread_pool_capacity" => 10,
       "max_threads" => 10,
+      "busy_threads" => 2,
       "custom_labels" => {
         "hostname" => "test1.example.com",
       },
@@ -91,6 +95,7 @@ def test_collecting_metrics_for_different_hosts_with_custom_labels
       "running_threads" => 9,
       "thread_pool_capacity" => 10,
       "max_threads" => 10,
+      "busy_threads" => 3,
       "custom_labels" => {
         "hostname" => "test2.example.com",
       },
@@ -109,13 +114,14 @@ def test_collecting_metrics_for_different_hosts_with_custom_labels
       "running_threads" => 8,
       "thread_pool_capacity" => 10,
       "max_threads" => 10,
+      "busy_threads" => 4,
       "custom_labels" => {
         "hostname" => "test1.example.com",
       },
     )
 
     metrics = collector.metrics
-    assert_equal 7, metrics.size
+    assert_equal 8, metrics.size
     assert_equal "puma_workers{phase=\"0\",hostname=\"test2.example.com\"} 4\n" \
                    "puma_workers{phase=\"0\",hostname=\"test1.example.com\"} 3",
                  metrics.first.metric_text
diff --git a/test/test_helper.rb b/test/test_helper.rb
@@ -9,10 +9,10 @@
 
 module TestingMod
   class FakeConnection
-    def call_pipelined(_, _, _)
+    def call_pipelined(...)
     end
 
-    def call(_, _)
+    def call(...)
     end
 
     def connected?
@@ -64,6 +64,18 @@ def call_pipelined(command, _config)
 RedisClient::Middlewares.prepend(TestingMod)
 RedisClient.register(RedisValidationMiddleware)
 
+unless defined?(::Puma)
+  module Puma
+    module Const
+      VERSION = "6.6.0"
+    end
+
+    def self.stats
+      '{}'
+    end
+  end
+end
+
 class TestHelper
   def self.wait_for(time, &blk)
     (time / 0.001).to_i.times do
