server: add information filtering to hot ranges endpoint

This change introduces two enhancements to the hot ranges page. The
first is the omission of table descriptors if specified, the second
allows callers to specify per-node limits on the number of ranges
requested.

Specifying `StatsOnly` on the hot ranges call will cause the call to
skip collecting table descriptors to include in the response, which
means the call will not be required to read from the keyspace.

The `PerNodeLimit` specifies a local limit for a hot ranges call, so
that we only include a number of replicas for each node local call made,
(different than the global limit enforced today).

Fixes: #142595
Epic: CRDB-43150

Release note (general change): Allows api callers to specify statistics
only and a per-node limit for the hot ranges response.

Author: angles-n-daemons

docs/generated/http/full.md

@@ -3599,6 +3599,8 @@ of ranges currently considered “hot” by the node(s).
 | page_token | [string](#cockroach.server.serverpb.HotRangesRequest-string) |  |  | [reserved](#support-status) |
 | tenant_id | [string](#cockroach.server.serverpb.HotRangesRequest-string) |  |  | [reserved](#support-status) |
 | nodes | [string](#cockroach.server.serverpb.HotRangesRequest-string) | repeated |  | [reserved](#support-status) |
+| per_node_limit | [int32](#cockroach.server.serverpb.HotRangesRequest-int32) |  | per_node_limit indicates the maximum number of hot ranges to return for each node. If left empty, the default is 128. | [reserved](#support-status) |
+| stats_only | [bool](#cockroach.server.serverpb.HotRangesRequest-bool) |  | stats_only indicates whether to return only the stats for the hot ranges, without pulling descriptor information. | [reserved](#support-status) |
 
 
 

pkg/server/serverpb/status.proto

@@ -1387,6 +1387,18 @@ message HotRangesRequest {
     (gogoproto.customname) = "Nodes",
     (gogoproto.nullable) = true
   ];
+  // per_node_limit indicates the maximum number of hot ranges
+  // to return for each node. If left empty, the default is 128.
+  int32 per_node_limit = 6 [
+    (gogoproto.customname) = "PerNodeLimit",
+    (gogoproto.nullable) = true
+  ];
+  // stats_only indicates whether to return only the stats
+  // for the hot ranges, without pulling descriptor information.
+  bool stats_only = 7 [
+    (gogoproto.customname) = "StatsOnly",
+    (gogoproto.nullable) = true
+  ];
 }
 
 // HotRangesResponseV2 is a response payload returned by `HotRangesV2` service.

pkg/server/status.go

@@ -7,6 +7,7 @@ package server
 
 import (
 	"bytes"
+	"cmp"
 	"context"
 	"crypto/ecdsa"
 	"crypto/rsa"
@@ -19,6 +20,7 @@ import (
 	"os/exec"
 	"reflect"
 	"regexp"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -2863,7 +2865,7 @@ func (t *statusServer) HotRangesV2(
 	}
 
 	ti, _ := t.sqlServer.tenantConnect.TenantInfo()
-	if ti.TenantID.IsSet() {
+	if ti.TenantID.IsSet() && !req.StatsOnly {
 		err = t.addDescriptorsToHotRanges(ctx, resp)
 		if err != nil {
 			return nil, err
@@ -2923,13 +2925,13 @@ func (s *systemStatusServer) HotRangesV2(
 				return nil, errors.New("cannot call 'local' mixed with other nodes")
 			}
 
-			resp, err := s.localHotRanges(ctx, tenantID, requestedNodeID)
+			resp, err := s.localHotRanges(tenantID, requestedNodeID, int(req.PerNodeLimit))
 			if err != nil {
 				return nil, err
 			}
 
-			// If operating as the system tenant, add descriptor data to the reposnse.
-			if !tenantID.IsSet() {
+			// If explicitly set as the system tenant, or unset, add descriptor data to the reposnse.
+			if !tenantID.IsSet() && !req.StatsOnly {
 				err = s.addDescriptorsToHotRanges(ctx, resp)
 				if err != nil {
 					return nil, err
@@ -2943,7 +2945,12 @@ func (s *systemStatusServer) HotRangesV2(
 		requestedNodes = append(requestedNodes, requestedNodeID)
 	}
 
-	remoteRequest := serverpb.HotRangesRequest{Nodes: []string{"local"}, TenantID: req.TenantID}
+	remoteRequest := serverpb.HotRangesRequest{
+		Nodes:        []string{"local"},
+		TenantID:     req.TenantID,
+		PerNodeLimit: req.PerNodeLimit,
+		StatsOnly:    req.StatsOnly,
+	}
 	nodeFn := func(ctx context.Context, status serverpb.StatusClient, nodeID roachpb.NodeID) ([]*serverpb.HotRangesResponseV2_HotRange, error) {
 		nodeResp, err := status.HotRangesV2(ctx, &remoteRequest)
 		if err != nil {
@@ -2990,7 +2997,7 @@ func (s *systemStatusServer) HotRangesV2(
 // Returns a HotRangesResponseV2 containing detailed information about each hot range,
 // or an error if the operation fails.
 func (s *systemStatusServer) localHotRanges(
-	ctx context.Context, tenantID roachpb.TenantID, requestedNodeID roachpb.NodeID,
+	tenantID roachpb.TenantID, requestedNodeID roachpb.NodeID, localLimit int,
 ) (*serverpb.HotRangesResponseV2, error) {
 	// Initialize response object
 	var resp serverpb.HotRangesResponseV2
@@ -3048,6 +3055,15 @@ func (s *systemStatusServer) localHotRanges(
 		return nil, err
 	}
 
+	// sort the slices by cpu
+	slices.SortFunc(resp.Ranges, func(a, b *serverpb.HotRangesResponseV2_HotRange) int {
+		return cmp.Compare(a.CPUTimePerSecond, b.CPUTimePerSecond)
+	})
+	// truncate the response if localLimit is set
+	if localLimit != 0 && localLimit < len(resp.Ranges) {
+		resp.Ranges = resp.Ranges[:localLimit]
+	}
+
 	return &resp, nil
 }
 

pkg/server/status_test.go

@@ -8,6 +8,7 @@ package server
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"os"
 	"slices"
 	"sync"
@@ -820,3 +821,101 @@ func TestHotRangesByNode(t *testing.T) {
 		require.Error(t, err, "cannot call 'local' mixed with other nodes")
 	})
 }
+
+func TestHotRangesStatsOnly(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	sc := log.ScopeWithoutShowLogs(t)
+	defer sc.Close(t)
+
+	ctx := context.Background()
+
+	s := serverutils.StartServerOnly(t, base.TestServerArgs{
+		DefaultTestTenant: base.TestControlsTenantsExplicitly,
+		StoreSpecs: []base.StoreSpec{
+			base.DefaultTestStoreSpec,
+			base.DefaultTestStoreSpec,
+			base.DefaultTestStoreSpec,
+		},
+		Knobs: base.TestingKnobs{
+			Store: &kvserver.StoreTestingKnobs{
+				ReplicaPlannerKnobs: plan.ReplicaPlannerTestingKnobs{
+					DisableReplicaRebalancing: true,
+				},
+			},
+		},
+	})
+	defer s.Stopper().Stop(ctx)
+
+	for _, test := range []struct {
+		statsOnly      bool
+		hasDescriptors bool
+	}{
+		{true, false},
+		{false, true},
+	} {
+		t.Run(fmt.Sprintf("statsOnly=%t hasDescriptors %t", test.statsOnly, test.hasDescriptors), func(t *testing.T) {
+			testutils.SucceedsSoon(t, func() error {
+				ss := s.StatusServer().(*systemStatusServer)
+				resp, err := ss.HotRangesV2(ctx, &serverpb.HotRangesRequest{NodeID: "local", StatsOnly: test.statsOnly})
+				if err != nil {
+					return err
+				}
+
+				if len(resp.Ranges) == 0 {
+					return errors.New("waiting for hot ranges to be collected")
+				}
+
+				hasDescriptors := false
+				for _, r := range resp.Ranges {
+					allDescriptors := append(r.Databases, append(r.Tables, r.Indexes...)...)
+					if len(allDescriptors) > 0 {
+						hasDescriptors = true
+					}
+				}
+
+				require.Equal(t, test.hasDescriptors, hasDescriptors)
+				return nil
+			})
+		})
+	}
+}
+
+func TestHotRangesNodeLimit(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	sc := log.ScopeWithoutShowLogs(t)
+	defer sc.Close(t)
+
+	ctx := context.Background()
+
+	s := serverutils.StartServerOnly(t, base.TestServerArgs{
+		DefaultTestTenant: base.TestControlsTenantsExplicitly,
+		StoreSpecs: []base.StoreSpec{
+			base.DefaultTestStoreSpec,
+			base.DefaultTestStoreSpec,
+			base.DefaultTestStoreSpec,
+		},
+		Knobs: base.TestingKnobs{
+			Store: &kvserver.StoreTestingKnobs{
+				ReplicaPlannerKnobs: plan.ReplicaPlannerTestingKnobs{
+					DisableReplicaRebalancing: true,
+				},
+			},
+		},
+	})
+	defer s.Stopper().Stop(ctx)
+
+	testutils.SucceedsSoon(t, func() error {
+		ss := s.StatusServer().(*systemStatusServer)
+		resp, err := ss.HotRangesV2(ctx, &serverpb.HotRangesRequest{NodeID: "local", PerNodeLimit: 5})
+		if err != nil {
+			return err
+		}
+
+		if len(resp.Ranges) == 0 {
+			return errors.New("waiting for hot ranges to be collected")
+		}
+
+		require.Equal(t, 5, len(resp.Ranges))
+		return nil
+	})
+}