kvprober: log traced planning query

This commit adds a cluster setting
kv.prober.log_tracing_on_plan_failure.enabled. If enabled, along with
kv.prober.tracing.enabled, the read/write probers will log the traced
planning query (targetting Meta2) if there is an error in the planning
phase.

Epic: None

Release note: None

Author: iskettaneh

pkg/kv/kvprober/kvprober.go

@@ -368,6 +368,7 @@ func (p *Prober) readProbeImpl(ctx context.Context, ops proberOpsI, txns proberT
 	var probeCtx = ctx
 
 	isTracingEnabled := tracingEnabled.Get(&p.settings.SV)
+	logTracingOnPlanFailureEnabled := isTracingEnabled && logTracingOnPlanFailureEnabled.Get(&p.settings.SV)
 	if isTracingEnabled {
 		probeCtx, finishAndGetRecording = tracing.ContextWithRecordingSpan(ctx, p.tracer, "read probe")
 	}
@@ -380,7 +381,11 @@ func (p *Prober) readProbeImpl(ctx context.Context, ops proberOpsI, txns proberT
 		if errorIsExpectedDuringNormalOperation(err) {
 			log.Health.Warningf(ctx, "making a plan failed with expected error: %v", err)
 		} else {
-			log.Health.Errorf(ctx, "can't make a plan: %v", err)
+			if logTracingOnPlanFailureEnabled {
+				log.Health.Errorf(ctx, "can't make a plan: %v, recorded trace: %s", err, finishAndGetRecording())
+			} else {
+				log.Health.Errorf(ctx, "can't make a plan: %v", err)
+			}
 			p.metrics.ProbePlanFailures.Inc(1)
 		}
 		return
@@ -453,6 +458,7 @@ func (p *Prober) writeProbeImpl(ctx context.Context, ops proberOpsI, txns prober
 	var probeCtx = ctx
 
 	isTracingEnabled := tracingEnabled.Get(&p.settings.SV)
+	logTracingOnPlanFailureEnabled := isTracingEnabled && logTracingOnPlanFailureEnabled.Get(&p.settings.SV)
 	if isTracingEnabled {
 		probeCtx, finishAndGetRecording = tracing.ContextWithRecordingSpan(ctx, p.tracer, "write probe")
 	}
@@ -467,7 +473,11 @@ func (p *Prober) writeProbeImpl(ctx context.Context, ops proberOpsI, txns prober
 		if errorIsExpectedDuringNormalOperation(err) {
 			log.Health.Warningf(ctx, "making a plan failed with expected error: %v", err)
 		} else {
-			log.Health.Errorf(ctx, "can't make a plan: %v", err)
+			if logTracingOnPlanFailureEnabled {
+				log.Health.Errorf(ctx, "can't make a plan: %v, recorded trace: %s", err, finishAndGetRecording())
+			} else {
+				log.Health.Errorf(ctx, "can't make a plan: %v", err)
+			}
 			p.metrics.ProbePlanFailures.Inc(1)
 		}
 		return

pkg/kv/kvprober/settings.go

@@ -136,3 +136,10 @@ var tracingEnabled = settings.RegisterBoolSetting(
 	"whether the KV prober should collect traces, in order to log info about"+
 		"leaseholders of ranges probed",
 	false)
+
+var logTracingOnPlanFailureEnabled = settings.RegisterBoolSetting(
+	settings.ApplicationLevel,
+	"kv.prober.log_tracing_on_plan_failure.enabled",
+	"whether the KV prober should print log traces when planning fails; "+
+		"requires kv.prober.tracing.enabled to be set to true",
+	false)