util/sysutil: add functions for getting TCP round-trip time and variance

Previously, we relied on timing heartbeats in the RPC layer to estimate network
latency (rpc.connection.avg_round_trip_latency). However, because these
measurements are computed within cockroach, they can be confounded by CPU-heavy
workloads.

Luckily, Linux maintains the smoothed round-trip time (SRTT) and RTT variance
for each TCP socket it opens. As kernel-computed metrics, these are less
sensitive to CPU overload.

This patch adds a sysutil function for obtaining SRTT and RTT variance given a
a net.TCPConn pointer.

Part of: #149959
Release note: None

Author: xuchef

pkg/util/sysutil/BUILD.bazel

@@ -16,6 +16,9 @@ go_library(
         "sysutil.go",
         "sysutil_unix.go",
         "sysutil_windows.go",
+        "tcpinfo.go",
+        "tcpinfo_linux.go",
+        "tcpinfo_stub.go",
     ],
     importpath = "github.com/cockroachdb/cockroach/pkg/util/sysutil",
     visibility = ["//visibility:public"],
@@ -94,6 +97,7 @@ go_test(
         "large_file_test.go",
         "sysutil_test.go",
         "sysutil_unix_test.go",
+        "tcpinfo_linux_test.go",
     ],
     embed = [":sysutil"],
     deps = [
@@ -105,6 +109,7 @@ go_test(
             "@org_golang_x_sys//unix",
         ],
         "@io_bazel_rules_go//go/platform:android": [
+            "//pkg/testutils/skip",
             "@com_github_stretchr_testify//assert",
             "@org_golang_x_sys//unix",
         ],
@@ -133,6 +138,7 @@ go_test(
             "@org_golang_x_sys//unix",
         ],
         "@io_bazel_rules_go//go/platform:linux": [
+            "//pkg/testutils/skip",
             "@com_github_stretchr_testify//assert",
             "@org_golang_x_sys//unix",
         ],

pkg/util/sysutil/tcpinfo.go

@@ -0,0 +1,14 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+
+package sysutil
+
+import "time"
+
+// RTTInfo holds the round-trip time information for a TCP connection.
+type RTTInfo struct {
+	RTT    time.Duration
+	RTTVar time.Duration
+}

pkg/util/sysutil/tcpinfo_linux.go

@@ -0,0 +1,52 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+
+//go:build linux
+
+package sysutil
+
+import (
+	"net"
+	"syscall"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+func getTCPInfo(conn syscall.RawConn) (*unix.TCPInfo, error) {
+	var (
+		tcpInfo    *unix.TCPInfo
+		syscallErr error
+	)
+	err := conn.Control(func(fd uintptr) {
+		tcpInfo, syscallErr = unix.GetsockoptTCPInfo(int(fd), unix.IPPROTO_TCP, unix.TCP_INFO)
+	})
+	if syscallErr != nil {
+		return nil, syscallErr
+	}
+	if err != nil {
+		return nil, err
+	}
+	return tcpInfo, nil
+}
+
+// GetRTTInfo retrieves round-trip time information from a TCP connection.
+func GetRTTInfo(conn *net.TCPConn) (value *RTTInfo, ok bool) {
+	if conn == nil {
+		return nil, false
+	}
+	rawConn, err := conn.SyscallConn()
+	if err != nil {
+		return nil, false
+	}
+	tcpInfo, err := getTCPInfo(rawConn)
+	if err != nil {
+		return nil, false
+	}
+	return &RTTInfo{
+		RTT:    time.Duration(tcpInfo.Rtt) * time.Microsecond,
+		RTTVar: time.Duration(tcpInfo.Rttvar) * time.Microsecond,
+	}, true
+}

pkg/util/sysutil/tcpinfo_linux_test.go

@@ -0,0 +1,110 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+
+//go:build linux
+
+package sysutil
+
+import (
+	"context"
+	"io"
+	"net"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
+	"github.com/stretchr/testify/require"
+)
+
+// GetRTTInfo doesn't panic when it's called with a nil connection.
+func TestGetTCPInfoNilConn(t *testing.T) {
+	var conn *net.TCPConn
+	info, ok := GetRTTInfo(conn)
+	require.Nil(t, info)
+	require.False(t, ok)
+}
+
+func TestGetTCPInfoLinux(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		skip.IgnoreLint(t, "skipping test; RTT inspection is only supported on Linux")
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Start a TCP echo server.
+	listener, _, err := createEchoServer(ctx, "127.0.0.1:0")
+	require.NoError(t, err)
+	defer func() {
+		require.NoError(t, listener.Close())
+	}()
+
+	// Connect a client.
+	conn, err := net.Dial("tcp", listener.Addr().String())
+	require.NoError(t, err)
+	defer func() {
+		require.NoError(t, conn.Close())
+	}()
+
+	// Give the connection a moment to be established.
+	time.Sleep(10 * time.Millisecond)
+
+	pingPayload := []byte("ping")
+	pongBuffer := make([]byte, len(pingPayload))
+
+	// Write a "ping".
+	_, err = conn.Write(pingPayload)
+	require.NoError(t, err)
+
+	// Read the "pong" echo.
+	_, err = io.ReadFull(conn, pongBuffer)
+	require.NoError(t, err)
+
+	// RTT info is only available on TCP connections. GetRTTInfo takes a
+	// *net.TCPConn, forcing the caller to perform a type assertion.
+	tcpConn, ok := conn.(*net.TCPConn)
+	require.True(t, ok)
+
+	info, ok := GetRTTInfo(tcpConn)
+
+	require.True(t, ok)
+	require.NotNil(t, info)
+}
+
+// createEchoServer creates a TCP echo server that echoes back all received data.
+// It returns the listener and a channel that will receive any server errors.
+func createEchoServer(ctx context.Context, addr string) (net.Listener, <-chan error, error) {
+	listener, err := net.Listen("tcp", addr)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	serverErrChan := make(chan error, 1)
+	go func() {
+		conn, err := listener.Accept()
+		if err != nil {
+			// If the context is canceled, the listener may be closed,
+			// resulting in an error. This is expected during cleanup.
+			select {
+			case <-ctx.Done():
+				serverErrChan <- nil
+			default:
+				serverErrChan <- err
+			}
+			return
+		}
+		defer conn.Close()
+		// Echo all data received back to the client. This will run until
+		// the client closes the connection, resulting in an io.EOF.
+		if _, err := io.Copy(conn, conn); err != nil && err != io.EOF {
+			serverErrChan <- err
+		} else {
+			serverErrChan <- nil
+		}
+	}()
+
+	return listener, serverErrChan, nil
+}

pkg/util/sysutil/tcpinfo_stub.go

@@ -0,0 +1,15 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+
+//go:build !linux
+
+package sysutil
+
+import "syscall"
+
+// GetRTTInfo is a stub implementation for non-Linux platforms returning (nil, false).
+func GetRTTInfo(conn syscall.RawConn) (value *RTTInfo, ok bool) {
+	return nil, false
+}