catalog: add metamorphic testing of leased catalog views

Previously, when we enabled leased descriptors for catalog views we lost
coverage for the existing default behavior fetching descriptors using
KV. This patch, makes the default metamorhpic, so that 90% of the time
leased descriptors are used and 10% of the time we avoid using them.

Fixes: #159747

Release note: None

Author: fqazi

pkg/BUILD.bazel

@@ -527,7 +527,6 @@ ALL_TESTS = [
     "//pkg/sql/logictest/tests/fakedist-vec-off:fakedist-vec-off_test",
     "//pkg/sql/logictest/tests/fakedist:fakedist_test",
     "//pkg/sql/logictest/tests/local-dist-merge-backfill-declarative-schema-changer:local-dist-merge-backfill-declarative-schema-changer_test",
-    "//pkg/sql/logictest/tests/local-leased-descriptors:local-leased-descriptors_test",
     "//pkg/sql/logictest/tests/local-legacy-schema-changer:local-legacy-schema-changer_test",
     "//pkg/sql/logictest/tests/local-mixed-25.4:local-mixed-25_4_test",
     "//pkg/sql/logictest/tests/local-prepared:local-prepared_test",
@@ -2113,7 +2112,6 @@ GO_TARGETS = [
     "//pkg/sql/logictest/tests/fakedist-vec-off:fakedist-vec-off_test",
     "//pkg/sql/logictest/tests/fakedist:fakedist_test",
     "//pkg/sql/logictest/tests/local-dist-merge-backfill-declarative-schema-changer:local-dist-merge-backfill-declarative-schema-changer_test",
-    "//pkg/sql/logictest/tests/local-leased-descriptors:local-leased-descriptors_test",
     "//pkg/sql/logictest/tests/local-legacy-schema-changer:local-legacy-schema-changer_test",
     "//pkg/sql/logictest/tests/local-mixed-25.4:local-mixed-25_4_test",
     "//pkg/sql/logictest/tests/local-prepared:local-prepared_test",

pkg/sql/catalog/descs/collection.go

@@ -459,7 +459,7 @@ var allowLeasedDescriptorsInCatalogViews = settings.RegisterBoolSetting(
 	settings.ApplicationLevel,
 	"sql.catalog.allow_leased_descriptors.enabled",
 	"if true, catalog views (crdb_internal, information_schema, pg_catalog) can use leased descriptors for improved performance",
-	true,
+	lease.UseLeasedDescriptorsForCatalogDefault,
 	settings.WithPublic,
 )
 

pkg/sql/catalog/lease/BUILD.bazel

@@ -54,6 +54,7 @@ go_library(
         "//pkg/util/hlc",
         "//pkg/util/log",
         "//pkg/util/log/logcrash",
+        "//pkg/util/metamorphic",
         "//pkg/util/metric",
         "//pkg/util/mon",
         "//pkg/util/quotapool",

pkg/sql/catalog/lease/lease.go

@@ -44,6 +44,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/log/logcrash"
+	"github.com/cockroachdb/cockroach/pkg/util/metamorphic"
 	"github.com/cockroachdb/cockroach/pkg/util/metric"
 	"github.com/cockroachdb/cockroach/pkg/util/mon"
 	"github.com/cockroachdb/cockroach/pkg/util/quotapool"
@@ -97,6 +98,21 @@ var LeaseMonitorRangeFeedResetTime = settings.RegisterDurationSetting(
 	time.Minute*25,
 )
 
+// diableLeasedDescriptorsByDefaultThreshold any value above this is considered
+// disabled, making it 10% odds of disabled.
+const diableLeasedDescriptorsByDefaultThreshold = 90
+
+// disableLeasedDescriptorThresholdDefault, by default, leased descriptors
+// are enabled.
+const disableLeasedDescriptorThresholdDefault = 0
+
+// UseLeasedDescriptorsForCatalogDefault determines if leased descrptor are used for catalog
+// views.
+var UseLeasedDescriptorsForCatalogDefault = metamorphic.ConstantWithTestRange("disable-catalog-leased-descriptors-threshold",
+	disableLeasedDescriptorThresholdDefault,
+	0,
+	100) < diableLeasedDescriptorsByDefaultThreshold
+
 var WaitForInitialVersion = settings.RegisterBoolSetting(settings.ApplicationLevel,
 	"sql.catalog.descriptor_wait_for_initial_version.enabled",
 	"enables waiting for the initial version of a descriptor",
@@ -106,7 +122,7 @@ var LockedLeaseTimestamp = settings.RegisterBoolSetting(settings.ApplicationLeve
 	"sql.catalog.descriptor_lease.use_locked_timestamps.enabled",
 	"guarantees transactional version consistency for descriptors used by the lease manager,"+
 		"descriptors used can be intentionally older to support this",
-	true)
+	UseLeasedDescriptorsForCatalogDefault)
 
 var RetainOldVersionsForLocked = settings.RegisterBoolSetting(settings.ApplicationLevel,
 	"sql.catalog.descriptor_lease.lock_old_versions.enabled",

pkg/sql/logictest/logic.go

@@ -1843,23 +1843,6 @@ func (t *logicTest) newCluster(
 				t.Fatal(err)
 			}
 		}
-		if cfg.EnableLeasedDescriptorSupport {
-			if _, err := conn.Exec(
-				"SET CLUSTER SETTING sql.catalog.allow_leased_descriptors.enabled = true",
-			); err != nil {
-				t.Fatal(err)
-			}
-			if _, err := conn.Exec(
-				"SET CLUSTER SETTING sql.catalog.descriptor_lease.use_locked_timestamps.enabled = true",
-			); err != nil {
-				t.Fatal(err)
-			}
-			if _, err := conn.Exec(
-				"SET CLUSTER SETTING sql.catalog.allow_leased_descriptors.prefetch.enabled = true"); err != nil {
-				t.Fatal(err)
-			}
-		}
-
 		if cfg.UseDistributedMergeIndexBackfill {
 			mode := "declarative"
 			if cfg.DisableDeclarativeSchemaChanger {

pkg/sql/logictest/logictestbase/logictestbase.go

@@ -95,9 +95,6 @@ type TestClusterConfig struct {
 	DisableSchemaLockedByDefault bool
 	// PrepareQueries executes queries and statements with Prepare and Execute.
 	PrepareQueries bool
-	// EnableLeasedDescriptorSupport enables leased descriptors for pg_catalog /
-	// crdb_internal and locked leasing behavior.
-	EnableLeasedDescriptorSupport bool
 	// UseDistributedMergeIndexBackfill enables the use of distributed
 	// merge when index backfills are preformed.
 	UseDistributedMergeIndexBackfill bool
@@ -511,19 +508,6 @@ var LogicTestConfigs = []TestClusterConfig{
 		DisableUpgrade:              true,
 		DeclarativeCorpusCollection: true,
 	},
-	{
-		Name:                "local-leased-descriptors",
-		NumNodes:            1,
-		OverrideDistSQLMode: "off",
-		// local is the configuration where we run all tests which have bad
-		// interactions with the default test tenant.
-		//
-		// TODO(#156124): We should review this choice. Why can't we use "Random"
-		// here? If there are specific tests that are incompatible, we can
-		// flag them to run only in a separate config.
-		UseSecondaryTenant:            Never,
-		EnableLeasedDescriptorSupport: true,
-	},
 	{
 		// This config runs a cluster with 3 nodes, with a separate process per
 		// node. The nodes initially start on v25.4.

pkg/sql/logictest/testdata/logic_test/comment_on

@@ -1,4 +1,4 @@
-# LogicTest: default-configs !local-prepared local-leased-descriptors 3node-tenant
+# LogicTest: default-configs !local-prepared 3node-tenant
 
 statement ok
 CREATE DATABASE db

pkg/sql/logictest/testdata/logic_test/crdb_internal

@@ -1,6 +1,6 @@
 # 3node-tenant is blocked from running this file due to heavy reliance on
 # unavailable node IDs in this test.
-# LogicTest: !3node-tenant-default-configs !local-prepared local-leased-descriptors default-configs
+# LogicTest: !3node-tenant-default-configs !local-prepared default-configs
 
 query error database "crdb_internal" does not exist
 ALTER DATABASE crdb_internal RENAME TO not_crdb_internal

pkg/sql/logictest/testdata/logic_test/event_log

@@ -480,6 +480,8 @@ AND info NOT LIKE '%sql.defaults%'
 AND info NOT LIKE '%sql.distsql%'
 AND info NOT LIKE '%sql.testing%'
 AND info NOT LIKE '%sql.stats%'
+AND info NOT LIKE '%sql.catalog.allow_leased_descriptor%'
+AND info NOT LIKE '%sql.catalog.descriptor_lease.use_locked_timestamps.enabled%'
 ORDER BY "timestamp", info
 ----
 1  {"ApplicationName": "$ internal-optInToDiagnosticsStatReporting", "DefaultValue": "true", "EventType": "set_cluster_setting", "SettingName": "diagnostics.reporting.enabled", "Statement": "SET CLUSTER SETTING \"diagnostics.reporting.enabled\" = true", "Tag": "SET CLUSTER SETTING", "User": "node", "Value": "true"}
@@ -500,6 +502,8 @@ AND info NOT LIKE '%sql.defaults%'
 AND info NOT LIKE '%sql.distsql%'
 AND info NOT LIKE '%sql.testing%'
 AND info NOT LIKE '%sql.stats%'
+AND info NOT LIKE '%sql.catalog.allow_leased_descriptor%'
+AND info NOT LIKE '%sql.catalog.descriptor_lease.use_locked_timestamps.enabled%'
 ORDER BY "timestamp", info
 ----
 1  {"ApplicationName": "$ internal-optInToDiagnosticsStatReporting", "DefaultValue": "true", "EventType": "set_cluster_setting", "SettingName": "diagnostics.reporting.enabled", "Statement": "SET CLUSTER SETTING \"diagnostics.reporting.enabled\" = true", "Tag": "SET CLUSTER SETTING", "User": "node", "Value": "true"}

pkg/sql/logictest/testdata/logic_test/fk

@@ -1,4 +1,4 @@
-# LogicTest: !local-prepared default-configs 3node-tenant local-leased-descriptors
+# LogicTest: !local-prepared default-configs 3node-tenant
 statement ok
 SET CLUSTER SETTING sql.cross_db_fks.enabled = TRUE