kv: include tracing for split QueryIntent requests

When performing a parallel commit, we split out QueryIntent requests
for previously pipelined writes that need to be verified as part of
the parallel commit into their own batches. These are then run in
parallel with the EndTxn batch. Previously, we were missing tracing
for these, as we weren't using the correct SpanOpt. This is now fixed.

Epic: none

Release note: None

Author: arulajmani

pkg/kv/kvclient/kvcoord/dist_sender.go

@@ -1538,6 +1538,7 @@ func (ds *DistSender) divideAndSendParallelCommit(
 	} else {
 		ctx, hdl, err := ds.stopper.GetHandle(ctx, stop.TaskOpts{
 			TaskName: taskName,
+			SpanOpt:  stop.ChildSpan,
 		})
 		if err != nil {
 			return nil, kvpb.NewError(err)

pkg/kv/kvclient/kvcoord/txn_test.go

@@ -9,6 +9,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"regexp"
 	"sort"
 	"sync/atomic"
 	"testing"
@@ -26,19 +27,23 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverbase"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/tscache"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
+	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/storage"
 	"github.com/cockroachdb/cockroach/pkg/storage/enginepb"
 	"github.com/cockroachdb/cockroach/pkg/storage/mvccencoding"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/kvclientutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/localtestcluster"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
+	"github.com/cockroachdb/cockroach/pkg/testutils/testcluster"
 	"github.com/cockroachdb/cockroach/pkg/util/admission/admissionpb"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/randutil"
 	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
+	"github.com/cockroachdb/cockroach/pkg/util/tracing"
+	"github.com/cockroachdb/cockroach/pkg/util/tracing/tracingpb"
 	"github.com/cockroachdb/cockroach/pkg/util/uuid"
 	"github.com/cockroachdb/errors"
 	"github.com/stretchr/testify/require"
@@ -2460,3 +2465,50 @@ func TestTxnBufferedWritesOmitAbortSpanChecks(t *testing.T) {
 	require.Error(t, err)
 	require.Regexp(t, "TransactionRetryWithProtoRefreshError: .*WriteTooOldError", err)
 }
+
+// TestTxnTracesSplitQueryIntents verifies that the split out QueryIntent
+// requests are captured in the verbose tracing of a transaction.
+func TestTxnTracesSplitQueryIntents(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	st := cluster.MakeTestingClusterSettings()
+	tc := testcluster.StartTestCluster(t, 1, base.TestClusterArgs{
+		ServerArgs: base.TestServerArgs{
+			Settings: st,
+		},
+	})
+	ctx := context.Background()
+	defer tc.Stopper().Stop(ctx)
+	// NB: Disable buffered writes to ensure writes are pipelined, and therefore
+	// there are QueryIntent requests to split.
+	kvcoord.BufferedWritesEnabled.Override(ctx, &st.SV, false)
+
+	db := tc.Server(0).DB()
+
+	tracer := tracing.NewTracer()
+	traceCtx, sp := tracer.StartSpanCtx(context.Background(), "test-txn", tracing.WithRecording(tracingpb.RecordingVerbose))
+
+	if err := db.Txn(traceCtx, func(ctx context.Context, txn *kv.Txn) error {
+		if err := txn.CPut(ctx, "a", "val", nil); err != nil {
+			return err
+		}
+		if err := txn.Put(ctx, "c", "d"); err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	collectedSpans := sp.FinishAndGetRecording(tracingpb.RecordingVerbose)
+	dump := collectedSpans.String()
+	// dump:
+	//    0.275ms      0.171ms    sending split out pre-commit QueryIntent batch
+	found, err := regexp.MatchString(
+		// The (?s) makes "." match \n. This makes the test resilient to other log
+		// lines being interspersed.
+		`.*sending split out pre-commit QueryIntent batch`,
+		dump)
+	require.NoError(t, err)
+	require.True(t, found, "didn't match: %s", dump)
+}

pkg/sql/opt/exec/execbuilder/testdata/show_trace

@@ -46,6 +46,7 @@ SELECT $$
     FROM [SHOW KV TRACE FOR SESSION]
    WHERE message NOT LIKE '%Z/%'
          AND operation NOT LIKE 'kv.DistSender: sending partial batch%'
+         AND operation NOT LIKE 'kv.DistSender: sending pre-commit query intents%'
          AND message NOT SIMILAR TO '%(PushTxn|ResolveIntent|SystemConfigSpan)%'
          AND tag NOT LIKE '%intExec=%'
          AND tag NOT LIKE '%scExec%'

pkg/sql/opt/exec/execbuilder/testdata/upsert

@@ -1076,6 +1076,7 @@ SET tracing = on,kv,results; UPSERT INTO t.kv(k, v) VALUES (2,3); SET tracing =
 query TT
 SELECT operation, message FROM [SHOW KV TRACE FOR SESSION]
  WHERE operation != 'dist sender send' AND operation != 'kv.DistSender: sending partial batch'
+  AND operation NOT LIKE 'kv.DistSender: sending pre-commit query intents%'
 ----
 colbatchscan  Scan /Table/120/1/2/0 lock Exclusive (Block, Unreplicated)
 count         CPut /Table/120/1/2/0 -> /TUPLE/2:2:Int/3