Merge pull request #11 from lessonnine/TNT-2741/measure-socket-backlog

Pull info on socket backlog

Author: Leszek Zalewski

CHANGELOG.md

@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.1.0 Alpha]
+
+### Added
+
+Socket telemetry, and to be more precise new metric: `sockets.backlog`. If enabled it will
+pull information from Puma sockets about the state of their backlogs (requests waiting to
+be acknowledged by Puma). It will be exposed under `sockets-backlog` metric.
+
+You can enable and test it via `config.sockets_telemetry!` option.
+
 ## [1.0.0] - 2021-09-08
 ### Added
 - Release to Github Packages

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    puma-plugin-telemetry (0.3.1)
+    puma-plugin-telemetry (1.1.0.alpha)
       puma (>= 5.0)
 
 GEM
@@ -14,7 +14,7 @@ GEM
     parallel (1.20.1)
     parser (3.0.2.0)
       ast (~> 2.4.1)
-    puma (5.4.0)
+    puma (5.5.2)
       nio4r (~> 2.0)
     rack (2.2.3)
     rainbow (3.0.0)

lib/puma/plugin/telemetry.rb

@@ -30,8 +30,8 @@ def configure
           yield(config)
         end
 
-        def build
-          puma_telemetry
+        def build(launcher = nil)
+          socket_telemetry(puma_telemetry, launcher)
         end
 
         private
@@ -47,6 +47,16 @@ def puma_telemetry
             .new(stats)
             .metrics(config.puma_telemetry)
         end
+
+        def socket_telemetry(telemetry, launcher)
+          return telemetry if launcher.nil?
+          return telemetry unless config.socket_telemetry?
+
+          telemetry.merge! SocketData.new(launcher.binder.ios)
+                                     .metrics
+
+          telemetry
+        end
       end
 
       # Contents of actual Puma Plugin
@@ -71,7 +81,7 @@ def run!
           loop do
             @launcher.events.debug "plugin=telemetry msg=\"publish\""
 
-            call(Puma::Plugin::Telemetry.build)
+            call(Puma::Plugin::Telemetry.build(@launcher))
           rescue Errno::EPIPE
             # Occurs when trying to output to STDOUT while puma is shutting down
           rescue StandardError => e

lib/puma/plugin/telemetry/config.rb

@@ -58,18 +58,31 @@ class Config
         # - default: DEFAULT_PUMA_TELEMETRY
         attr_accessor :puma_telemetry
 
+        # Whenever to publish socket telemetry.
+        # - default: false
+        attr_accessor :socket_telemetry
+
         def initialize
           @enabled = false
           @initial_delay = 5
           @frequency = 5
           @targets = []
           @puma_telemetry = DEFAULT_PUMA_TELEMETRY
+          @socket_telemetry = false
         end
 
         def enabled?
           !!@enabled
         end
 
+        def socket_telemetry!
+          @socket_telemetry = true
+        end
+
+        def socket_telemetry?
+          @socket_telemetry
+        end
+
         def add_target(name_or_target, **args)
           return @targets.push(name_or_target) unless name_or_target.is_a?(Symbol)
 

lib/puma/plugin/telemetry/data.rb

@@ -95,6 +95,71 @@ def sum_stat(stat)
           end
         end
       end
+
+      # Pulls TCP INFO data from socket
+      class SocketData
+        UNACKED_REGEXP = /\ unacked=(?<unacked>\d+)\ /.freeze
+
+        def initialize(ios)
+          @sockets = ios.select { |io| io.respond_to?(:getsockopt) }
+        end
+
+        # Number of unacknowledged connections in the sockets, which
+        # we know as socket backlog.
+        #
+        # The Socket::Option returned by `getsockopt` doesn't provide
+        # any kind of accessors for data inside. It decodes it on demand
+        # for `inspect` as strings in C implementation. It looks like
+        #
+        #     #<Socket::Option: INET TCP INFO state=LISTEN
+        #                                     ca_state=Open
+        #                                     retransmits=0
+        #                                     probes=0
+        #                                     backoff=0
+        #                                     options=0
+        #                                     rto=0.000000s
+        #                                     ato=0.000000s
+        #                                     snd_mss=0
+        #                                     rcv_mss=0
+        #                                     unacked=0
+        #                                     sacked=5
+        #                                     lost=0
+        #                                     retrans=0
+        #                                     fackets=0
+        #                                     last_data_sent=0.000s
+        #                                     last_ack_sent=0.000s
+        #                                     last_data_recv=0.000s
+        #                                     last_ack_recv=0.000s
+        #                                     pmtu=0
+        #                                     rcv_ssthresh=0
+        #                                     rtt=0.000000s
+        #                                     rttvar=0.000000s
+        #                                     snd_ssthresh=0
+        #                                     snd_cwnd=10
+        #                                     advmss=0
+        #                                     reordering=3
+        #                                     rcv_rtt=0.000000s
+        #                                     rcv_space=0
+        #                                     total_retrans=0
+        #                                     (128 bytes too long)>
+        #
+        # That's why we have to pull the `unacked`  field by parsing
+        # `inspect` output, instead of using something like `opt.unacked`
+        def unacked
+          @sockets.sum do |socket|
+            tcp_info = socket.getsockopt(Socket::SOL_TCP, Socket::TCP_INFO).inspect
+            tcp_match = tcp_info.match(UNACKED_REGEXP)
+
+            tcp_match[:unacked].to_i
+          end
+        end
+
+        def metrics
+          {
+            "sockets.backlog" => unacked
+          }
+        end
+      end
     end
   end
 end

lib/puma/plugin/telemetry/version.rb

@@ -3,7 +3,7 @@
 module Puma
   class Plugin
     module Telemetry
-      VERSION = "1.0.0"
+      VERSION = "1.1.0.alpha"
     end
   end
 end