Skip to content

Commit 1bb0d0b

Browse files
spilchenspilchen
committed
sql: fix super region placement in 3-region database with secondary region
When a super region was added to a 3-region database configured for region survivability, we applied zone constraints to ensure all 5 replicas were confined to the 3 defined regions. This was intended to prevent the super region from including any region outside the database. However, the logic mishandled secondary regions. Both the primary and secondary regions receive 2 voter replicas. The zone config logic then incorrectly added 2 more replicas to an arbitrary non-primary region, regardless of whether it was already the secondary. This resulted in an invalid configuration: 2 replicas in the primary, 2 in the secondary, and 2 in another region. This totals 6 replicas, which exceeds the allowed maximum of 5. The fix ensures that the secondary region is correctly accounted for when setting constraints. Fixes #150361 Release note (bug fix): Fixed invalid zone configurations when adding a super region to a 3-region database with a secondary region and region survivability. Previously, this could result in assigning more than the allowed number of replicas.
1 parent b1bb226 commit 1bb0d0b

File tree

3 files changed

+167
-21
lines changed

3 files changed

+167
-21
lines changed

‎pkg/ccl/logictestccl/testdata/logic_test/secondary_region‎

Lines changed: 117 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,24 @@ DATABASE db ALTER DATABASE db CONFIGURE ZONE USING
202202
voter_constraints = '{+region=ap-southeast-2: 2, +region=ca-central-1: 2}',
203203
lease_preferences = '[[+region=ap-southeast-2], [+region=ca-central-1]]'
204204

205+
# All of the table zone configs are the result of an implicit super region. To
206+
# help make sense of those, dump out the current primary and secondary regions
207+
# in the database.
208+
query TT
209+
SELECT
210+
region,
211+
CASE
212+
WHEN "primary" THEN 'primary'
213+
WHEN "secondary" THEN 'secondary'
214+
ELSE 'other'
215+
END AS role
216+
FROM [SHOW REGIONS FROM DATABASE db]
217+
ORDER BY region;
218+
----
219+
ap-southeast-2 primary
220+
ca-central-1 secondary
221+
us-east-1 other
222+
205223
# Verify that the zone configuration on the table is expected.
206224
query TT
207225
SHOW ZONE CONFIGURATION FOR TABLE db.rbt_in_primary
@@ -212,7 +230,7 @@ TABLE db.public.rbt_in_primary ALTER TABLE db.public.rbt_in_primary CONFIGURE Z
212230
gc.ttlseconds = 14400,
213231
num_replicas = 5,
214232
num_voters = 5,
215-
constraints = '{+region=ap-southeast-2: 1, +region=ca-central-1: 2, +region=us-east-1: 1}',
233+
constraints = '{+region=ap-southeast-2: 1, +region=ca-central-1: 1, +region=us-east-1: 1}',
216234
voter_constraints = '{+region=ap-southeast-2: 2, +region=ca-central-1: 2}',
217235
lease_preferences = '[[+region=ap-southeast-2], [+region=ca-central-1]]'
218236

@@ -225,7 +243,7 @@ TABLE db.public.rbt_in_us_east ALTER TABLE db.public.rbt_in_us_east CONFIGURE Z
225243
gc.ttlseconds = 14400,
226244
num_replicas = 5,
227245
num_voters = 5,
228-
constraints = '{+region=ap-southeast-2: 2, +region=ca-central-1: 1, +region=us-east-1: 1}',
246+
constraints = '{+region=ap-southeast-2: 1, +region=ca-central-1: 1, +region=us-east-1: 1}',
229247
voter_constraints = '{+region=ca-central-1: 2, +region=us-east-1: 2}',
230248
lease_preferences = '[[+region=us-east-1], [+region=ca-central-1]]'
231249

@@ -252,7 +270,7 @@ PARTITION "us-east-1" OF TABLE db.public.rbr ALTER PARTITION "us-east-1" OF TAB
252270
gc.ttlseconds = 14400,
253271
num_replicas = 5,
254272
num_voters = 5,
255-
constraints = '{+region=ap-southeast-2: 2, +region=ca-central-1: 1, +region=us-east-1: 1}',
273+
constraints = '{+region=ap-southeast-2: 1, +region=ca-central-1: 1, +region=us-east-1: 1}',
256274
voter_constraints = '{+region=ca-central-1: 2, +region=us-east-1: 2}',
257275
lease_preferences = '[[+region=us-east-1], [+region=ca-central-1]]'
258276

@@ -265,7 +283,7 @@ PARTITION "ap-southeast-2" OF TABLE db.public.rbr ALTER PARTITION "ap-southeast
265283
gc.ttlseconds = 14400,
266284
num_replicas = 5,
267285
num_voters = 5,
268-
constraints = '{+region=ap-southeast-2: 1, +region=ca-central-1: 2, +region=us-east-1: 1}',
286+
constraints = '{+region=ap-southeast-2: 1, +region=ca-central-1: 1, +region=us-east-1: 1}',
269287
voter_constraints = '{+region=ap-southeast-2: 2, +region=ca-central-1: 2}',
270288
lease_preferences = '[[+region=ap-southeast-2], [+region=ca-central-1]]'
271289

@@ -312,7 +330,7 @@ TABLE db.public.rbt_in_us_east ALTER TABLE db.public.rbt_in_us_east CONFIGURE Z
312330
gc.ttlseconds = 14400,
313331
num_replicas = 5,
314332
num_voters = 5,
315-
constraints = '{+region=ap-southeast-2: 2, +region=ca-central-1: 1, +region=us-east-1: 1}',
333+
constraints = '{+region=ap-southeast-2: 1, +region=ca-central-1: 1, +region=us-east-1: 1}',
316334
voter_constraints = '{+region=ca-central-1: 2, +region=us-east-1: 2}',
317335
lease_preferences = '[[+region=us-east-1], [+region=ca-central-1]]'
318336

@@ -339,7 +357,7 @@ PARTITION "us-east-1" OF TABLE db.public.rbr ALTER PARTITION "us-east-1" OF TAB
339357
gc.ttlseconds = 14400,
340358
num_replicas = 5,
341359
num_voters = 5,
342-
constraints = '{+region=ap-southeast-2: 2, +region=ca-central-1: 1, +region=us-east-1: 1}',
360+
constraints = '{+region=ap-southeast-2: 1, +region=ca-central-1: 1, +region=us-east-1: 1}',
343361
voter_constraints = '{+region=ca-central-1: 2, +region=us-east-1: 2}',
344362
lease_preferences = '[[+region=us-east-1], [+region=ca-central-1]]'
345363

@@ -352,7 +370,7 @@ PARTITION "ap-southeast-2" OF TABLE db.public.rbr ALTER PARTITION "ap-southeast
352370
gc.ttlseconds = 14400,
353371
num_replicas = 5,
354372
num_voters = 5,
355-
constraints = '{+region=ap-southeast-2: 1, +region=ca-central-1: 2, +region=us-east-1: 1}',
373+
constraints = '{+region=ap-southeast-2: 1, +region=ca-central-1: 1, +region=us-east-1: 1}',
356374
voter_constraints = '{+region=ap-southeast-2: 2, +region=ca-central-1: 2}',
357375
lease_preferences = '[[+region=ap-southeast-2], [+region=ca-central-1]]'
358376

@@ -427,6 +445,98 @@ PARTITION "ca-central-1" OF TABLE db.public.rbr ALTER PARTITION "ca-central-1"
427445
voter_constraints = '{+region=ca-central-1: 2}',
428446
lease_preferences = '[[+region=ca-central-1]]'
429447

448+
# Ensure that the implicit super region is updated now that the secondary region
449+
# was dropped. We will dump the regions and role so that we can more easily
450+
# understand the table zone configs that follow.
451+
query TT
452+
SELECT
453+
region,
454+
CASE
455+
WHEN "primary" THEN 'primary'
456+
WHEN "secondary" THEN 'secondary'
457+
ELSE 'other'
458+
END AS role
459+
FROM [SHOW REGIONS FROM DATABASE db]
460+
ORDER BY region;
461+
----
462+
ap-southeast-2 primary
463+
ca-central-1 other
464+
us-east-1 other
465+
466+
query TT
467+
SHOW ZONE CONFIGURATION FOR TABLE db.rbt_in_us_east
468+
----
469+
TABLE db.public.rbt_in_us_east ALTER TABLE db.public.rbt_in_us_east CONFIGURE ZONE USING
470+
range_min_bytes = 134217728,
471+
range_max_bytes = 536870912,
472+
gc.ttlseconds = 14400,
473+
num_replicas = 5,
474+
num_voters = 5,
475+
constraints = '{+region=ap-southeast-2: 2, +region=ca-central-1: 1, +region=us-east-1: 1}',
476+
voter_constraints = '{+region=us-east-1: 2}',
477+
lease_preferences = '[[+region=us-east-1]]'
478+
479+
query TT
480+
SHOW ZONE CONFIGURATION FOR TABLE db.rbt_in_ca_central
481+
----
482+
TABLE db.public.rbt_in_ca_central ALTER TABLE db.public.rbt_in_ca_central CONFIGURE ZONE USING
483+
range_min_bytes = 134217728,
484+
range_max_bytes = 536870912,
485+
gc.ttlseconds = 14400,
486+
num_replicas = 5,
487+
num_voters = 5,
488+
constraints = '{+region=ap-southeast-2: 2, +region=ca-central-1: 1, +region=us-east-1: 1}',
489+
voter_constraints = '{+region=ca-central-1: 2}',
490+
lease_preferences = '[[+region=ca-central-1]]'
491+
492+
# Add a different secondary to confirm zone config adjusts based on the secondary.
493+
statement ok
494+
ALTER DATABASE db SET SECONDARY REGION "us-east-1"
495+
496+
# Dump the regions and role so that we can understand table zone configs easier.
497+
query TT
498+
SELECT
499+
region,
500+
CASE
501+
WHEN "primary" THEN 'primary'
502+
WHEN "secondary" THEN 'secondary'
503+
ELSE 'other'
504+
END AS role
505+
FROM [SHOW REGIONS FROM DATABASE db]
506+
ORDER BY region;
507+
----
508+
ap-southeast-2 primary
509+
ca-central-1 other
510+
us-east-1 secondary
511+
512+
query TT
513+
SHOW ZONE CONFIGURATION FOR TABLE db.rbt_in_us_east
514+
----
515+
TABLE db.public.rbt_in_us_east ALTER TABLE db.public.rbt_in_us_east CONFIGURE ZONE USING
516+
range_min_bytes = 134217728,
517+
range_max_bytes = 536870912,
518+
gc.ttlseconds = 14400,
519+
num_replicas = 5,
520+
num_voters = 5,
521+
constraints = '{+region=ap-southeast-2: 2, +region=ca-central-1: 1, +region=us-east-1: 1}',
522+
voter_constraints = '{+region=us-east-1: 2}',
523+
lease_preferences = '[[+region=us-east-1]]'
524+
525+
query TT
526+
SHOW ZONE CONFIGURATION FOR TABLE db.rbt_in_ca_central
527+
----
528+
TABLE db.public.rbt_in_ca_central ALTER TABLE db.public.rbt_in_ca_central CONFIGURE ZONE USING
529+
range_min_bytes = 134217728,
530+
range_max_bytes = 536870912,
531+
gc.ttlseconds = 14400,
532+
num_replicas = 5,
533+
num_voters = 5,
534+
constraints = '{+region=ap-southeast-2: 1, +region=ca-central-1: 1, +region=us-east-1: 1}',
535+
voter_constraints = '{+region=ca-central-1: 2, +region=us-east-1: 2}',
536+
lease_preferences = '[[+region=ca-central-1], [+region=us-east-1]]'
537+
538+
statement ok
539+
ALTER DATABASE db DROP SECONDARY REGION
430540

431541
# Verify super region interactions
432542
statement ok

‎pkg/ccl/logictestccl/testdata/logic_test/super_regions‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -559,6 +559,26 @@ ALTER DATABASE mr3 SURVIVE REGION FAILURE;
559559
statement error pq: super region r1 only has 2 region\(s\): at least 3 regions are required for surviving a region failure
560560
ALTER DATABASE mr3 ADD SUPER REGION "r1" VALUES "us-west-1", "us-central-1";
561561

562+
# All of the table zone configs are easier to understand once we know what the
563+
# primary/secondary regions are. Dump them to make it easier to understand the
564+
# resulting table zone configs.
565+
query TT
566+
SELECT
567+
region,
568+
CASE
569+
WHEN "primary" THEN 'primary'
570+
WHEN "secondary" THEN 'secondary'
571+
ELSE 'other'
572+
END AS role
573+
FROM [SHOW REGIONS FROM DATABASE mr3]
574+
ORDER BY region;
575+
----
576+
ap-southeast-2 other
577+
ca-central-1 other
578+
us-central-1 other
579+
us-east-1 primary
580+
us-west-1 other
581+
562582
# In the case where we have 3 regions within the super region under
563583
# survive region failure mode, the first non-primary region constraint should
564584
# have 2 replicas, this is so all replicas are accounted for.

‎pkg/sql/regions/region_util.go‎

Lines changed: 30 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -333,24 +333,40 @@ func AddConstraintsForSuperRegion(
333333
}
334334
return nil
335335
case descpb.SurvivalGoal_REGION_FAILURE:
336-
// There is a special case where we have 3 regions under survival goal
337-
// region failure where we have to constrain an extra replica to any
338-
// region within the super region to guarantee that all replicas are
339-
// accounted for. In our case, we assign it to the first non-primary region
340-
// in sorted order.
341-
// This happens because we have 5 replicas and 3 regions. 2 voters are
342-
// constrained to the primary region, the other 2 regions each are given a
343-
// replica, the last non-voting replica is not guaranteed to be constrained
344-
// anywhere.
345-
// If we have more than 3 regions, all replicas are accounted for and
346-
// constrained within the super region.
336+
// Under survival goal REGION_FAILURE with 5 replicas, we must ensure all replicas
337+
// are placed within the super region. If any replica is unconstrained, it could be
338+
// allocated outside the super region in larger clusters.
339+
//
340+
// We only apply extra replica constraints when:
341+
// - There are exactly 3 regions, and
342+
// - Either there is no secondary region,
343+
// OR the affinity region is the secondary region.
344+
//
345+
// In those cases, we explicitly assign a second replica to a non-primary,
346+
// non-secondary region.
347+
//
348+
// The existing placement logic works without modification in the following cases:
349+
// - The database has more than 3 regions
350+
// - The database has both a primary and a secondary region, and the super region's
351+
// affinity is not set to the secondary region
352+
//
353+
// In these scenarios, the default logic ensures that all 5 replicas are distributed
354+
// correctly without over-constraining.
355+
//
347356
// See: https://github.com/cockroachdb/cockroach/issues/63617 for more.
348-
extraReplicaToConstrain := len(regions) == 3
357+
secondaryRegion := regionConfig.SecondaryRegion()
358+
359+
shouldDoubleUp := len(regions) == 3 &&
360+
(!regionConfig.HasSecondaryRegion() || affinityRegion == secondaryRegion)
361+
362+
doubleUpAssigned := false
363+
349364
for _, region := range regions {
350365
n := int32(1)
351-
if region != affinityRegion && extraReplicaToConstrain {
366+
if shouldDoubleUp && !doubleUpAssigned &&
367+
region != affinityRegion && region != secondaryRegion {
352368
n = 2
353-
extraReplicaToConstrain = false
369+
doubleUpAssigned = true
354370
}
355371
zc.Constraints = append(zc.Constraints, zonepb.ConstraintsConjunction{
356372
NumReplicas: n,

0 commit comments

Comments
 (0)