Merge #144307

144307: logictest: raise TxnLivenessThreshold for multitenant configs r=rafiss a=rafiss

Logic tests are flaky due to overload when running in multitenant mode. This patch increases the threshold for transaction heartbeat timeouts, which will make it less likely for foreground operations to be aborted by background jobs like the span config reconciler or the job registry loop to reclaim jobs from dead sessions.

This change was first added in a8ccd6b, but then was later reverted in 84079c9 since we fixed auto-retry behavior. I'm adding it back now since we are still seeing these flakes for statements/transactions that cannot be auto-retried.

fixes #143660
fixes #143946
Release note: None

Co-authored-by: Rafi Shamim <[email protected]>

Author: craig[bot]

pkg/sql/logictest/BUILD.bazel

@@ -62,6 +62,7 @@ go_library(
         "//pkg/kv/kvclient/rangefeed",
         "//pkg/kv/kvserver",
         "//pkg/kv/kvserver/kvserverbase",
+        "//pkg/kv/kvserver/txnwait",
         "//pkg/multitenant/tenantcapabilities",
         "//pkg/multitenant/tenantcapabilitiespb",
         "//pkg/security/username",

pkg/sql/logictest/logic.go

@@ -41,6 +41,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverbase"
+	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait"
 	"github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilities"
 	"github.com/cockroachdb/cockroach/pkg/multitenant/tenantcapabilitiespb"
 	"github.com/cockroachdb/cockroach/pkg/security/username"
@@ -4438,6 +4439,18 @@ func RunLogicTest(
 	if *printErrorSummary {
 		defer lt.printErrorSummary()
 	}
+	if config.UseSecondaryTenant == logictestbase.Always {
+		// Under multitenant configs running in EngFlow, we have seen that logic
+		// tests can be flaky due to an overload condition where schema change
+		// transactions do not heartbeat quickly enough. This prevents background
+		// jobs such as the spanconfig reconciler or the job registry "remove claims
+		// from dead sessions" loop from aborting the queries under test.
+		// See https://github.com/cockroachdb/cockroach/pull/140400#issuecomment-2634346278
+		// and https://github.com/cockroachdb/cockroach/issues/140494#issuecomment-2640208187
+		// for a detailed analysis of this issue.
+		cleanup := txnwait.TestingOverrideTxnLivenessThreshold(10 * time.Second)
+		defer cleanup()
+	}
 	// Each test needs a copy because of Parallel
 	serverArgsCopy := serverArgs
 	serverArgsCopy.ForceProductionValues = serverArgs.ForceProductionValues || nonMetamorphicBatchSizes