storage,sever: add debug separated value retrieval profile endpoint

Add a new debug endpoint that returns a text-formatted representation of all
the stack traces that performed a retrieval of a separated value while the
profile was being collected. This provides better observability into what code
paths are suffering more expensive value retrievals.

Epic: none
Informs: #146575
Release note: none

Author: jbowens

pkg/server/debug/server.go

@@ -6,13 +6,15 @@
 package debug
 
 import (
+	"context"
 	"expvar"
 	"fmt"
 	"io"
 	"net/http"
 	"net/http/pprof"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/base/serverident"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver"
@@ -277,6 +279,29 @@ func (ds *Server) RegisterEngines(engines []storage.Engine) error {
 					fmt.Fprintf(w, "error analyzing LSM at %s: %v", dir, err)
 				}
 			})
+		ds.mux.HandleFunc(fmt.Sprintf("/debug/storage/%d/profiles/separated-value-retrievals", storeID),
+			func(w http.ResponseWriter, req *http.Request) {
+				dur := 30 * time.Second
+				if secsStr := req.Header.Get("seconds"); secsStr != "" {
+					secs, err := strconv.ParseInt(secsStr, 10, 64)
+					if err != nil {
+						http.Error(w, "error parsing seconds", http.StatusBadRequest)
+						return
+					}
+					dur = time.Duration(secs) * time.Second
+				}
+				ctx := req.Context()
+				ctx, cancel := context.WithTimeout(ctx, dur)
+				defer cancel()
+				profile, err := eng.ProfileSeparatedValueRetrievals(ctx)
+				if err != nil {
+					http.Error(w, "error profiling separated value retrievals", http.StatusInternalServerError)
+					return
+				}
+				w.Header().Set("Content-Type", "text/plain")
+				w.WriteHeader(http.StatusOK)
+				fmt.Fprint(w, profile.String())
+			})
 	}
 	return nil
 }

pkg/storage/engine.go

@@ -26,6 +26,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/uuid"
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble"
+	"github.com/cockroachdb/pebble/metrics"
 	"github.com/cockroachdb/pebble/rangekey"
 	"github.com/cockroachdb/pebble/vfs"
 	"github.com/cockroachdb/redact"
@@ -926,6 +927,9 @@ type Engine interface {
 	Capacity() (roachpb.StoreCapacity, error)
 	// Properties returns the low-level properties for the engine's underlying storage.
 	Properties() roachpb.StoreProperties
+	// ProfileSeparatedValueRetrievals collects a profile of the engine's
+	// separated value retrievals. It stops when the context is done.
+	ProfileSeparatedValueRetrievals(ctx context.Context) (*metrics.ValueRetrievalProfile, error)
 	// Compact forces compaction over the entire database.
 	Compact(ctx context.Context) error
 	// Env returns the filesystem environment used by the Engine.

pkg/storage/pebble.go

@@ -2136,6 +2136,19 @@ func (p *Pebble) GetEnvStats() (*fs.EnvStats, error) {
 	return stats, nil
 }
 
+// ProfileSeparatedValueRetrievals collects a profile of the engine's
+// separated value retrievals. It stops when the context is done.
+func (p *Pebble) ProfileSeparatedValueRetrievals(
+	ctx context.Context,
+) (*metrics.ValueRetrievalProfile, error) {
+	stop, err := p.db.RecordSeparatedValueRetrievals()
+	if err != nil {
+		return nil, err
+	}
+	<-ctx.Done()
+	return stop(), nil
+}
+
 // GetAuxiliaryDir implements the Engine interface.
 func (p *Pebble) GetAuxiliaryDir() string {
 	return p.auxDir