sql: add setting for constraint validation for secondary tenants

This patch adds a cluster setting, ``sql.zone_configs.max_replicas_per_region,
that helps prevent virtual clusters from setting undesired zone configs. This
setting defines the max number of replicas that can be configured for a
single region (0 for unlimited).

Epic: none
Informs: #142856

Release note: None

Author: annrpom

pkg/ccl/logictestccl/testdata/logic_test/multi_region_secondary_tenants_abstractions_allowed

@@ -0,0 +1,62 @@
+# LogicTest: multiregion-9node-3region-3azs-tenant
+# tenant-cluster-setting-override-opt: sql.virtual_cluster.feature_access.multiregion.enabled=true
+
+statement ok
+CREATE DATABASE db PRIMARY REGION "us-east-1"
+
+statement ok
+ALTER DATABASE db ADD REGION "ca-central-1"
+
+statement ok
+SET override_multi_region_zone_config = true
+
+statement ok
+ALTER DATABASE db CONFIGURE ZONE USING constraints = '{+region=us-east-1: 4}', voter_constraints = '{+region=us-east-1: 3}'
+
+subtest constrained_replicas
+
+user root
+
+statement ok
+SET CLUSTER SETTING sql.zone_configs.max_replicas_per_region = 10
+
+user host-cluster-root
+
+# Ensure that the override from the system tenant takes precedence over anything
+# set from within the application tenant.
+statement ok
+ALTER TENANT ALL SET CLUSTER SETTING sql.zone_configs.max_replicas_per_region = 4
+
+user root
+
+# Our setting should not affect non-constraint related zone config updates.
+statement ok
+ALTER DATABASE db CONFIGURE ZONE USING gc.ttlseconds = 1
+
+statement error pgcode 23514 constraint for "us-east-1" exceeds the configured maximum of 4 replicas
+ALTER DATABASE db CONFIGURE ZONE USING constraints = '{+region=us-east-1: 5}'
+
+statement ok
+ALTER DATABASE db CONFIGURE ZONE USING constraints = '{+region=us-east-1: 1, +region=ca-central-1: 1}', voter_constraints = '{+region=us-east-1: 3}'
+
+statement error pgcode 23514 voter constraint for "us-east-1" exceeds the configured maximum of 4 replicas
+ALTER DATABASE db CONFIGURE ZONE USING voter_constraints = '{+region=us-east-1: 5}'
+
+user host-cluster-root
+
+statement ok
+ALTER TENANT ALL SET CLUSTER SETTING sql.zone_configs.max_replicas_per_region = 3
+
+user root
+
+statement ok
+ALTER DATABASE db CONFIGURE ZONE USING gc.ttlseconds = 1
+
+statement ok
+ALTER DATABASE db CONFIGURE ZONE USING constraints = '{+region=us-east-1: 3, +region=ca-central-1: 1}', voter_constraints = '{+region=us-east-1: 2}'
+
+# Ensure that prohibited constraints don't trigger a validation error.
+statement ok
+ALTER DATABASE test CONFIGURE ZONE USING num_replicas = 4, constraints = '{-region=us-east-1: 4}'
+
+subtest end

pkg/ccl/logictestccl/tests/multiregion-9node-3region-3azs-tenant/BUILD.bazel

@@ -9,7 +9,7 @@ go_test(
         "//pkg/ccl/logictestccl:testdata",  # keep
     ],
     exec_properties = {"test.Pool": "large"},
-    shard_count = 17,
+    shard_count = 18,
     tags = ["cpu:4"],
     deps = [
         "//pkg/base",

pkg/ccl/logictestccl/tests/multiregion-9node-3region-3azs-tenant/generated_test.go

@@ -15,13 +15,15 @@ go_library(
         "//pkg/clusterversion",
         "//pkg/keys",
         "//pkg/roachpb",
+        "//pkg/settings",
         "//pkg/sql/pgwire/pgcode",
         "//pkg/sql/pgwire/pgerror",
         "//pkg/sql/sem/tree",
         "//pkg/util/debugutil",
         "//pkg/util/envutil",
         "//pkg/util/log",
         "@com_github_cockroachdb_errors//:errors",
+        "@com_github_cockroachdb_redact//:redact",
         "@com_github_gogo_protobuf//proto",
         "@in_gopkg_yaml_v2//:yaml_v2",
     ],

pkg/config/zonepb/BUILD.bazel

@@ -17,12 +17,14 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/clusterversion"
 	"github.com/cockroachdb/cockroach/pkg/keys"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
+	"github.com/cockroachdb/cockroach/pkg/settings"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode"
 	"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
 	"github.com/cockroachdb/cockroach/pkg/util/envutil"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/errors"
+	"github.com/cockroachdb/redact"
 	"github.com/gogo/protobuf/proto"
 )
 
@@ -91,6 +93,16 @@ var MultiRegionZoneConfigFields = []tree.Name{
 	"lease_preferences",
 }
 
+var MaxReplicasPerRegion = settings.RegisterIntSetting(
+	settings.ApplicationLevel,
+	"sql.zone_configs.max_replicas_per_region",
+	"the maximum number of replicas that can be "+
+		"configured per region (0 for unlimited); this is only enforced on "+
+		"new zone config modifications",
+	0,
+	settings.WithVisibility(settings.Reserved),
+	settings.NonNegativeInt)
+
 // MultiRegionZoneConfigFieldsSet contain the items in
 // MultiRegionZoneConfigFields but in a set form for fast lookup.
 var MultiRegionZoneConfigFieldsSet = func() map[tree.Name]struct{} {
@@ -1474,3 +1486,78 @@ func (v ReplaceMinMaxValVisitor) VisitPre(expr tree.Expr) (recurse bool, newExpr
 
 // VisitPost satisfies the Visitor interface.
 func (ReplaceMinMaxValVisitor) VisitPost(expr tree.Expr) tree.Expr { return expr }
+
+// ValidateNewUniqueConstraintsForSecondaryTenants validates that none of our
+// zonepb.ConstraintsConjunction violate the given max replicas per region
+// set by sql.zone_configs.max_replicas_per_region.
+func ValidateNewUniqueConstraintsForSecondaryTenants(
+	sv *settings.Values, currentZone, newZone *ZoneConfig,
+) error {
+	maxReplicas := MaxReplicasPerRegion.Get(sv)
+	if maxReplicas == 0 {
+		return nil
+	}
+
+	getRequiredConstraintMap := func(
+		constraintsConj []ConstraintsConjunction,
+	) map[Constraint]int32 {
+		constraintsMap := make(map[Constraint]int32)
+		for _, constraints := range constraintsConj {
+			for _, constraint := range constraints.Constraints {
+				if constraint.Type == Constraint_REQUIRED {
+					constraintsMap[constraint] = constraints.NumReplicas
+				}
+			}
+		}
+		return constraintsMap
+	}
+
+	validateConstraints := func(
+		constraintType redact.SafeString,
+		currentConstraints map[Constraint]int32,
+		newConstraints ConstraintsConjunction,
+	) error {
+		for _, constraint := range newConstraints.Constraints {
+			if constraint.Type != Constraint_REQUIRED {
+				continue
+			}
+			newReplicasConstrained := newConstraints.NumReplicas
+			// If the user has not explicitly changed what replicas are
+			// constrained to a region, bypass this validation.
+			if currentReplicasConstrained, ok := currentConstraints[constraint]; ok {
+				if currentReplicasConstrained == newReplicasConstrained {
+					continue
+				}
+			}
+			// If the amount of replicas we have constrained for this region
+			// surpasses the limit we have set, error out early.
+			if int64(newReplicasConstrained) > maxReplicas {
+				return pgerror.Newf(
+					pgcode.CheckViolation,
+					"%sconstraint for %q exceeds the configured "+
+						"maximum of %d replicas",
+					constraintType,
+					constraint.Value,
+					maxReplicas,
+				)
+			}
+		}
+		return nil
+	}
+
+	for _, newConstraints := range newZone.Constraints {
+		err := validateConstraints("",
+			getRequiredConstraintMap(currentZone.Constraints), newConstraints)
+		if err != nil {
+			return err
+		}
+	}
+	for _, newVoterConstraints := range newZone.VoterConstraints {
+		err := validateConstraints("voter ",
+			getRequiredConstraintMap(currentZone.VoterConstraints), newVoterConstraints)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/config/zonepb/zone.go

@@ -601,6 +601,9 @@ func validateZoneLocalitiesForSecondaryTenants(
 	settings *cluster.Settings,
 ) error {
 	toValidate := accumulateNewUniqueConstraints(currentZone, newZone)
+	if err := zonepb.ValidateNewUniqueConstraintsForSecondaryTenants(&settings.SV, currentZone, newZone); err != nil {
+		return err
+	}
 
 	// rs and zs will be lazily populated with regions and zones, respectively.
 	// These should not be accessed directly - use getRegionsAndZones helper

pkg/sql/schemachanger/scbuild/internal/scbuildstmt/zone_config_helpers.go

@@ -941,6 +941,9 @@ func validateZoneLocalitiesForSecondaryTenants(
 	settings *cluster.Settings,
 ) error {
 	toValidate := accumulateNewUniqueConstraints(currentZone, newZone)
+	if err := zonepb.ValidateNewUniqueConstraintsForSecondaryTenants(&settings.SV, currentZone, newZone); err != nil {
+		return err
+	}
 
 	// rs and zs will be lazily populated with regions and zones, respectively.
 	// These should not be accessed directly - use getRegionsAndZones helper