upgrades: swallow setting override error in diagnostics migration

Previously if a tenant ran the optInToDiagnosticsStatReporting migration, which
sets diagnostics.reporting.enabled, after the system tenant already overrode
this setting, the migration would enter a fail loop. With this patch, the
migration instead noops.

Epic: none

Release note: none

Author: msbutler

pkg/ccl/logictestccl/testdata/logic_test/tenant_settings

@@ -109,10 +109,10 @@ SHOW CLUSTER SETTING sql.notices.enabled
 true
 
 # Verify that we disallow setting a ApplicationLevel setting that is overridden.
-statement error cluster setting 'sql.notices.enabled' is currently overridden by the operator
+statement error cluster setting 'sql.notices.enabled' cannot be set: cluster setting is overridden by system virtual cluster
 SET CLUSTER SETTING sql.notices.enabled = false
 
-statement error cluster setting 'sql.notices.enabled' is currently overridden by the operator
+statement error cluster setting 'sql.notices.enabled' cannot be set: cluster setting is overridden by system virtual cluster
 RESET CLUSTER SETTING sql.notices.enabled
 
 user host-cluster-root

pkg/crosscluster/physical/standby_read_ts_poller_job_test.go

@@ -43,14 +43,19 @@ func TestStandbyReadTSPollerJob(t *testing.T) {
 	jobutils.WaitForJobToRun(c.T, c.DestSysSQL, jobspb.JobID(ingestionJobID))
 	t.Logf("test setup took %s", timeutil.Since(beginTS))
 
+	readerTenantName := fmt.Sprintf("%s-readonly", args.DestTenantName)
+
+	// Ensures the reader tenant can spin up even if the system tenant overrode
+	// the diagnostics setting, set during a permanent migration.
+	c.DestSysSQL.Exec(t, fmt.Sprintf("ALTER TENANT '%s' SET CLUSTER SETTING diagnostics.reporting.enabled = true", readerTenantName))
+
 	srcTime := c.SrcCluster.Server(0).Clock().Now()
 	c.WaitUntilReplicatedTime(srcTime, jobspb.JobID(ingestionJobID))
 
 	stats := replicationtestutils.TestingGetStreamIngestionStatsFromReplicationJob(t, ctx, c.DestSysSQL, ingestionJobID)
 	readerTenantID := stats.IngestionDetails.ReadTenantID
 	require.NotNil(t, readerTenantID)
 
-	readerTenantName := fmt.Sprintf("%s-readonly", args.DestTenantName)
 	c.ConnectToReaderTenant(ctx, readerTenantID, readerTenantName)
 
 	defaultDBQuery := `

pkg/settings/cluster/cluster_settings.go

@@ -62,6 +62,8 @@ type OverridesInformer interface {
 	IsOverridden(settingKey settings.InternalKey) bool
 }
 
+var SettingOverrideErr = errors.New("cluster setting is overridden by system virtual cluster")
+
 // TelemetryOptOut controls whether to opt out of telemetry (including Sentry) or not.
 var TelemetryOptOut = envutil.EnvOrDefaultBool("COCKROACH_SKIP_ENABLING_DIAGNOSTIC_REPORTING", false)
 

pkg/sql/set_cluster_setting.go

@@ -225,7 +225,7 @@ func (p *planner) SetClusterSetting(
 	}
 
 	if st.OverridesInformer != nil && st.OverridesInformer.IsOverridden(setting.InternalKey()) {
-		return nil, errors.Errorf("cluster setting '%s' is currently overridden by the operator", name)
+		return nil, errors.Wrapf(cluster.SettingOverrideErr, "cluster setting '%s' cannot be set", name)
 	}
 
 	value, err := p.getAndValidateTypedClusterSetting(ctx, name, n.Value, setting)

pkg/upgrade/upgrades/permanent_upgrades.go

@@ -168,6 +168,9 @@ func optInToDiagnosticsStatReporting(
 	_, err := deps.InternalExecutor.Exec(
 		ctx, "optInToDiagnosticsStatReporting", nil, /* txn */
 		`SET CLUSTER SETTING diagnostics.reporting.enabled = true`)
+	if errors.Is(err, cluster.SettingOverrideErr) {
+		return nil
+	}
 	return err
 }