Merge #147756

147756: kvserver: add useful execution trace regions r=tbg a=tbg

This PR adds regions for the range on which a BatchRequest executes, as well as regions indicating the method of each request while it's being evaluated.

Some samples from running this script:

```bash
#!/bin/bash
set -euo pipefail

c=local-tpcc
wh=10
(roachprod create -n 4 $c || true) &> /dev/null
roachprod stop $c
roachprod put $c cockroach
roachprod start $c:1-3
roachprod run $c:4 -- "if [ ! -f loaded ]; then ./cockroach workload init tpcc --warehouses $wh {pgurl:1-3} && touch loaded; fi"
roachprod run $c:4 -- ./cockroach workload run tpcc --warehouses $wh --concurrency 10 {pgurl:1-3}
```

and then

```
curl -o regiontrace.bin "$(roachprod adminui local-tpcc:1)debug/pprof/trace?seconds=10"
gotraceui regiontrace.bin
```

<img width="735" alt="image" src="https://github.com/user-attachments/assets/938b871b-c5c5-4960-9cd2-420ade908a2b" />

<img width="992" alt="image" src="https://github.com/user-attachments/assets/012ed3c6-75d6-4fde-b00e-f0bf731f172e" />

Related to #146873.

Epic: none
Release note: none


Co-authored-by: Tobias Grieger <[email protected]>

Author: craig[bot]

pkg/kv/kvserver/replica_evaluate.go

@@ -9,6 +9,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"runtime/trace"
 
 	"github.com/cockroachdb/cockroach/pkg/keys"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvnemesis/kvnemesisutil"
@@ -335,9 +336,17 @@ func evaluateBatch(
 		// Note that `reply` is populated even when an error is returned: it
 		// may carry a response transaction and in the case of WriteTooOldError
 		// (which is sometimes deferred) it is fully populated.
+		var reg *trace.Region
+		if trace.IsEnabled() {
+			regName := args.Method().String() // NB: this is cheap, no allocs
+			reg = trace.StartRegion(ctx, regName)
+		}
 		curResult, err := evaluateCommand(
 			ctx, readWriter, rec, ms, ss, baHeader, args, reply, g, st, ui, evalPath, omitInRangefeeds,
 		)
+		if reg != nil {
+			reg.End()
+		}
 
 		if filter := rec.EvalKnobs().TestingPostEvalFilter; filter != nil {
 			filterArgs := kvserverbase.FilterArgs{

pkg/kv/kvserver/replica_send.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"reflect"
 	"runtime/pprof"
+	"runtime/trace"
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/kv/kvpb"
@@ -128,6 +129,9 @@ func (r *Replica) SendWithWriteBytes(
 		ctx = pprof.WithLabels(ctx, pprof.Labels("range_str", r.rangeStr.ID()))
 		pprof.SetGoroutineLabels(ctx)
 	}
+	if trace.IsEnabled() {
+		defer trace.StartRegion(ctx, r.rangeStr.String() /* cheap */).End()
+	}
 	// Add the range log tag.
 	ctx = r.AnnotateCtx(ctx)