changefeedccl: remove two usages of disableDeclarativeSchemaChangesForTest

This change removes two usages of `disableDeclarativeSchemaChangesForTest`
from tests and replaces them with `maybeDisableDeclarativeSchemaChangesForTest`
which will use the legacy schema changer 10% of the time.

Release note: None
Informs: cockroachdb#106906
Epic: None

Author: jayshrivastava

pkg/ccl/changefeedccl/alter_changefeed_test.go

@@ -1103,11 +1103,19 @@ func TestAlterChangefeedAddTargetsDuringSchemaChangeError(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
-	rnd, _ := randutil.NewPseudoRand()
+	rnd, seed := randutil.NewPseudoRand()
+	t.Logf("random seed: %d", seed)
 
 	testFn := func(t *testing.T, s TestServerWithSystem, f cdctest.TestFeedFactory) {
 		sqlDB := sqlutils.MakeSQLRunner(s.DB)
-		disableDeclarativeSchemaChangesForTest(t, sqlDB)
+		usingLegacySchemaChanger := maybeDisableDeclarativeSchemaChangesForTest(t, sqlDB, rnd)
+		// NB: For the `ALTER TABLE foo ADD COLUMN ... DEFAULT` schema change,
+		// the expected boundary is different depending on if we are using the
+		// legacy schema changer or not.
+		expectedBoundaryType := jobspb.ResolvedSpan_RESTART
+		if usingLegacySchemaChanger {
+			expectedBoundaryType = jobspb.ResolvedSpan_BACKFILL
+		}
 
 		knobs := s.TestingKnobs.
 			DistSQL.(*execinfra.TestingKnobs).
@@ -1182,10 +1190,9 @@ func TestAlterChangefeedAddTargetsDuringSchemaChangeError(t *testing.T) {
 				return true, nil
 			}
 
-			// A backfill begins when the backfill resolved event arrives, which has a
-			// timestamp such that all backfill spans have a timestamp of
-			// timestamp.Next()
-			if r.BoundaryType == jobspb.ResolvedSpan_BACKFILL {
+			// A backfill begins when the associated resolved event arrives, which has a
+			// timestamp such that all backfill spans have a timestamp of timestamp.Next().
+			if r.BoundaryType == expectedBoundaryType {
 				backfillTimestamp = r.Timestamp
 				return false, nil
 			}

pkg/ccl/changefeedccl/changefeed_test.go

@@ -1866,7 +1866,8 @@ func TestChangefeedSchemaChangeBackfillCheckpoint(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
-	rnd, _ := randutil.NewPseudoRand()
+	rnd, seed := randutil.NewPseudoRand()
+	t.Logf("random seed: %d", seed)
 
 	// This test asserts that a second checkpoint made after resumption does its
 	// best to not lose information from the first checkpoint, therefore the
@@ -1876,7 +1877,14 @@ func TestChangefeedSchemaChangeBackfillCheckpoint(t *testing.T) {
 
 	testFn := func(t *testing.T, s TestServerWithSystem, f cdctest.TestFeedFactory) {
 		sqlDB := sqlutils.MakeSQLRunner(s.DB)
-		disableDeclarativeSchemaChangesForTest(t, sqlDB)
+		usingLegacySchemaChanger := maybeDisableDeclarativeSchemaChangesForTest(t, sqlDB, rnd)
+		// NB: For the `ALTER TABLE foo ADD COLUMN ... DEFAULT` schema change,
+		// the expected boundary is different depending on if we are using the
+		// legacy schema changer or not.
+		expectedBoundaryType := jobspb.ResolvedSpan_RESTART
+		if usingLegacySchemaChanger {
+			expectedBoundaryType = jobspb.ResolvedSpan_BACKFILL
+		}
 
 		knobs := s.TestingKnobs.
 			DistSQL.(*execinfra.TestingKnobs).
@@ -1957,10 +1965,10 @@ func TestChangefeedSchemaChangeBackfillCheckpoint(t *testing.T) {
 				return true, nil
 			}
 
-			// A backfill begins when the backfill resolved event arrives, which has a
+			// A backfill begins when the associated resolved event arrives, which has a
 			// timestamp such that all backfill spans have a timestamp of
-			// timestamp.Next()
-			if r.BoundaryType == jobspb.ResolvedSpan_BACKFILL {
+			// timestamp.Next().
+			if r.BoundaryType == expectedBoundaryType {
 				backfillTimestamp = r.Timestamp
 				return false, nil
 			}

pkg/ccl/changefeedccl/helpers_test.go

@@ -58,6 +58,21 @@ import (
 
 var testSinkFlushFrequency = 100 * time.Millisecond
 
+// disableDeclarativeSchemaChangesForTest will disable the declarative schema
+// changer with a probability of 10% using the provided SQL DB connection. This
+// returns true if the declarative schema changer is disabled.
+func maybeDisableDeclarativeSchemaChangesForTest(
+	t testing.TB, sqlDB *sqlutils.SQLRunner, rnd *rand.Rand,
+) bool {
+	disable := rnd.Float32() < 0.1
+	if disable {
+		t.Log("using legacy schema changer")
+		sqlDB.Exec(t, "SET use_declarative_schema_changer='off'")
+		sqlDB.Exec(t, "SET CLUSTER SETTING  sql.defaults.use_declarative_schema_changer='off'")
+	}
+	return disable
+}
+
 // disableDeclarativeSchemaChangesForTest tests that are disabled due to differences
 // in changefeed behaviour and are tracked by issue #80545.
 func disableDeclarativeSchemaChangesForTest(t testing.TB, sqlDB *sqlutils.SQLRunner) {