Merge pull request #151397 from arulajmani/backport25.3-151367

release-25.3: kv: include tracing for split QueryIntent requests

Author: arulajmani

pkg/kv/kvclient/kvcoord/dist_sender.go

@@ -1548,6 +1548,7 @@ func (ds *DistSender) divideAndSendParallelCommit(
 	} else {
 		ctx, hdl, err := ds.stopper.GetHandle(ctx, stop.TaskOpts{
 			TaskName: taskName,
+			SpanOpt:  stop.ChildSpan,
 		})
 		if err != nil {
 			return nil, kvpb.NewError(err)

pkg/kv/kvclient/kvcoord/txn_test.go

@@ -9,6 +9,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"regexp"
 	"sort"
 	"sync/atomic"
 	"testing"
@@ -26,19 +27,23 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverbase"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/tscache"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
+	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/storage"
 	"github.com/cockroachdb/cockroach/pkg/storage/enginepb"
 	"github.com/cockroachdb/cockroach/pkg/storage/mvccencoding"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/kvclientutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/localtestcluster"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
+	"github.com/cockroachdb/cockroach/pkg/testutils/testcluster"
 	"github.com/cockroachdb/cockroach/pkg/util/admission/admissionpb"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/randutil"
 	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
+	"github.com/cockroachdb/cockroach/pkg/util/tracing"
+	"github.com/cockroachdb/cockroach/pkg/util/tracing/tracingpb"
 	"github.com/cockroachdb/cockroach/pkg/util/uuid"
 	"github.com/cockroachdb/errors"
 	"github.com/stretchr/testify/require"
@@ -2469,3 +2474,50 @@ func TestTxnBufferedWritesOmitAbortSpanChecks(t *testing.T) {
 	require.Error(t, err)
 	require.Regexp(t, "TransactionRetryWithProtoRefreshError: .*WriteTooOldError", err)
 }
+
+// TestTxnTracesSplitQueryIntents verifies that the split out QueryIntent
+// requests are captured in the verbose tracing of a transaction.
+func TestTxnTracesSplitQueryIntents(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	st := cluster.MakeTestingClusterSettings()
+	tc := testcluster.StartTestCluster(t, 1, base.TestClusterArgs{
+		ServerArgs: base.TestServerArgs{
+			Settings: st,
+		},
+	})
+	ctx := context.Background()
+	defer tc.Stopper().Stop(ctx)
+	// NB: Disable buffered writes to ensure writes are pipelined, and therefore
+	// there are QueryIntent requests to split.
+	kvcoord.BufferedWritesEnabled.Override(ctx, &st.SV, false)
+
+	db := tc.Server(0).DB()
+
+	tracer := tracing.NewTracer()
+	traceCtx, sp := tracer.StartSpanCtx(context.Background(), "test-txn", tracing.WithRecording(tracingpb.RecordingVerbose))
+
+	if err := db.Txn(traceCtx, func(ctx context.Context, txn *kv.Txn) error {
+		if err := txn.CPut(ctx, "a", "val", nil); err != nil {
+			return err
+		}
+		if err := txn.Put(ctx, "c", "d"); err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	collectedSpans := sp.FinishAndGetRecording(tracingpb.RecordingVerbose)
+	dump := collectedSpans.String()
+	// dump:
+	//    0.275ms      0.171ms    sending split out pre-commit QueryIntent batch
+	found, err := regexp.MatchString(
+		// The (?s) makes "." match \n. This makes the test resilient to other log
+		// lines being interspersed.
+		`.*sending split out pre-commit QueryIntent batch`,
+		dump)
+	require.NoError(t, err)
+	require.True(t, found, "didn't match: %s", dump)
+}

pkg/sql/opt/exec/execbuilder/testdata/show_trace

@@ -46,6 +46,7 @@ SELECT $$
     FROM [SHOW KV TRACE FOR SESSION]
    WHERE message NOT LIKE '%Z/%'
          AND operation NOT LIKE 'kv.DistSender: sending partial batch%'
+         AND operation NOT LIKE 'kv.DistSender: sending pre-commit query intents%'
          AND message NOT SIMILAR TO '%(PushTxn|ResolveIntent|SystemConfigSpan)%'
          AND tag NOT LIKE '%intExec=%'
          AND tag NOT LIKE '%scExec%'

pkg/sql/opt/exec/execbuilder/testdata/upsert

@@ -1076,6 +1076,7 @@ SET tracing = on,kv,results; UPSERT INTO t.kv(k, v) VALUES (2,3); SET tracing =
 query TT
 SELECT operation, message FROM [SHOW KV TRACE FOR SESSION]
  WHERE operation != 'dist sender send' AND operation != 'kv.DistSender: sending partial batch'
+  AND operation NOT LIKE 'kv.DistSender: sending pre-commit query intents%'
 ----
 colbatchscan  Scan /Table/120/1/2/0 lock Exclusive (Block, Unreplicated)
 count         CPut /Table/120/1/2/0 -> /TUPLE/2:2:Int/3