Skip to content

Commit 0599a96

Browse files
jeffswensonjeffswenson
committed
backup: split up the multiregion datadriven test
This splits up the multiregion datadriven test so that each test has at most 2 clusters in it. We've been seeing some stuck server shutdowns and this should make them easier to troubleshoot. Release note: none Informs: #145079
1 parent 2a1771e commit 0599a96

File tree

3 files changed

+150
-146
lines changed

3 files changed

+150
-146
lines changed

‎pkg/backup/testdata/backup-restore/multiregion-mismatched-locality‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# disabled to run within tenant because multiregion primitives are not supported within tenant
2+
3+
skip-under-duress
4+
----
5+
6+
new-cluster name=s1 allow-implicit-access disable-tenant localities=us-east-1,us-west-1,eu-central-1
7+
----
8+
9+
set-cluster-setting setting=sql.multiregion.system_database_multiregion.enabled value=true
10+
----
11+
12+
exec-sql
13+
ALTER DATABASE system SET PRIMARY REGION "us-east-1";
14+
CREATE DATABASE d PRIMARY REGION "us-east-1" REGIONS "us-west-1", "eu-central-1" SURVIVE REGION FAILURE;
15+
CREATE TABLE d.t (x INT);
16+
INSERT INTO d.t VALUES (1), (2), (3);
17+
----
18+
19+
query-sql
20+
SELECT region FROM [SHOW REGIONS FROM DATABASE d] ORDER BY 1;
21+
----
22+
eu-central-1
23+
us-east-1
24+
us-west-1
25+
26+
exec-sql
27+
BACKUP DATABASE d INTO 'nodelocal://1/database_backup/';
28+
----
29+
30+
exec-sql
31+
BACKUP INTO 'nodelocal://1/full_cluster_backup/';
32+
----
33+
34+
# A new cluster with different localities settings.
35+
new-cluster name=s3 share-io-dir=s1 allow-implicit-access disable-tenant localities=eu-central-1,eu-north-1
36+
----
37+
38+
exec-sql
39+
RESTORE DATABASE d FROM LATEST IN 'nodelocal://1/database_backup/';
40+
----
41+
pq: detected a mismatch in regions between the restore cluster and the backup cluster, missing regions detected: us-east-1, us-west-1.
42+
HINT: there are two ways you can resolve this issue: 1) update the cluster to which you're restoring to ensure that the regions present on the nodes' --locality flags match those present in the backup image, or 2) restore with the "skip_localities_check" option
43+
44+
exec-sql
45+
RESTORE FROM LATEST IN 'nodelocal://1/full_cluster_backup/';
46+
----
47+
pq: detected a mismatch in regions between the restore cluster and the backup cluster, missing regions detected: us-east-1, us-west-1.
48+
HINT: there are two ways you can resolve this issue: 1) update the cluster to which you're restoring to ensure that the regions present on the nodes' --locality flags match those present in the backup image, or 2) restore with the "skip_localities_check" option
49+
50+
exec-sql
51+
RESTORE FROM LATEST IN 'nodelocal://1/full_cluster_backup/' WITH skip_localities_check;
52+
----
53+
54+
exec-sql
55+
INSERT INTO d.t VALUES (4);
56+
----
57+
58+
exec-sql
59+
SET enable_multiregion_placement_policy='true';
60+
ALTER DATABASE d SURVIVE ZONE FAILURE;
61+
ALTER DATABASE d PLACEMENT RESTRICTED;
62+
ALTER DATABASE d SET PRIMARY REGION 'eu-central-1';
63+
ALTER DATABASE d DROP REGION 'us-east-1';
64+
ALTER DATABASE d DROP REGION 'us-west-1';
65+
ALTER DATABASE d ADD REGION 'eu-north-1';
66+
ALTER DATABASE d SET SECONDARY REGION 'eu-north-1';
67+
----
68+
69+
exec-sql
70+
RESTORE DATABASE d FROM LATEST IN 'nodelocal://1/database_backup/' WITH skip_localities_check, new_db_name='d_new';
71+
----
72+
73+
exec-sql
74+
INSERT INTO d_new.t VALUES (4);
75+
----
76+
77+
exec-sql
78+
SET enable_multiregion_placement_policy='true';
79+
ALTER DATABASE d_new SURVIVE ZONE FAILURE;
80+
ALTER DATABASE d PLACEMENT RESTRICTED;
81+
ALTER DATABASE d_new SET PRIMARY REGION 'eu-central-1';
82+
ALTER DATABASE d_new DROP REGION 'us-east-1';
83+
ALTER DATABASE d_new DROP REGION 'us-west-1';
84+
ALTER DATABASE d_new ADD REGION 'eu-north-1';
85+
ALTER DATABASE d_new SET SECONDARY REGION 'eu-north-1';
86+
----

‎pkg/backup/testdata/backup-restore/multiregion-same-locality‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
# disabled to run within tenant because multiregion primitives are not supported within tenant
2+
3+
skip-under-duress
4+
----
5+
6+
new-cluster name=s1 allow-implicit-access disable-tenant localities=us-east-1,us-west-1,eu-central-1
7+
----
8+
9+
set-cluster-setting setting=sql.multiregion.system_database_multiregion.enabled value=true
10+
----
11+
12+
exec-sql
13+
ALTER DATABASE system SET PRIMARY REGION "us-east-1";
14+
CREATE DATABASE d PRIMARY REGION "us-east-1" REGIONS "us-west-1", "eu-central-1" SURVIVE REGION FAILURE;
15+
CREATE TABLE d.t (x INT);
16+
INSERT INTO d.t VALUES (1), (2), (3);
17+
----
18+
19+
query-sql
20+
SELECT region FROM [SHOW REGIONS FROM DATABASE d] ORDER BY 1;
21+
----
22+
eu-central-1
23+
us-east-1
24+
us-west-1
25+
26+
exec-sql
27+
BACKUP DATABASE d INTO 'nodelocal://1/database_backup/';
28+
----
29+
30+
exec-sql
31+
BACKUP INTO 'nodelocal://1/full_cluster_backup/';
32+
----
33+
34+
# A new cluster with the same locality settings.
35+
new-cluster name=s2 share-io-dir=s1 allow-implicit-access disable-tenant localities=us-east-1,us-west-1,eu-central-1
36+
----
37+
38+
exec-sql
39+
RESTORE FROM LATEST IN 'nodelocal://1/full_cluster_backup/';
40+
----
41+
42+
exec-sql
43+
DROP DATABASE d;
44+
----
45+
46+
exec-sql
47+
RESTORE DATABASE d FROM LATEST IN 'nodelocal://1/database_backup/';
48+
----
49+
50+
query-sql
51+
SHOW DATABASES;
52+
----
53+
d root us-east-1 {eu-central-1,us-east-1,us-west-1} region
54+
data root <nil> <nil> {} <nil>
55+
defaultdb root <nil> <nil> {} <nil>
56+
postgres root <nil> <nil> {} <nil>
57+
system node <nil> <nil> {} <nil>

‎pkg/backup/testdata/backup-restore/multiregion‎ renamed to ‎pkg/backup/testdata/backup-restore/multiregion-single-to-multi‎

Lines changed: 7 additions & 146 deletions
Original file line numberDiff line numberDiff line change
@@ -3,151 +3,7 @@
33
skip-under-duress
44
----
55

6-
new-cluster name=s1 allow-implicit-access disable-tenant localities=us-east-1,us-west-1,eu-central-1
7-
----
8-
9-
set-cluster-setting setting=sql.multiregion.system_database_multiregion.enabled value=true
10-
----
11-
12-
exec-sql
13-
ALTER DATABASE system SET PRIMARY REGION "us-east-1";
14-
CREATE DATABASE d PRIMARY REGION "us-east-1" REGIONS "us-west-1", "eu-central-1" SURVIVE REGION FAILURE SECONDARY REGION "us-west-1";
15-
CREATE TABLE d.t (x INT);
16-
INSERT INTO d.t VALUES (1), (2), (3);
17-
----
18-
19-
query-sql
20-
SELECT region FROM [SHOW REGIONS FROM DATABASE d] ORDER BY 1;
21-
----
22-
eu-central-1
23-
us-east-1
24-
us-west-1
25-
26-
query-sql
27-
SHOW ZONE CONFIGURATION FROM DATABASE d;
28-
----
29-
DATABASE d ALTER DATABASE d CONFIGURE ZONE USING
30-
range_min_bytes = 134217728,
31-
range_max_bytes = 536870912,
32-
gc.ttlseconds = 14400,
33-
num_replicas = 5,
34-
num_voters = 5,
35-
constraints = '{+region=eu-central-1: 1, +region=us-east-1: 1, +region=us-west-1: 1}',
36-
voter_constraints = '{+region=us-east-1: 2, +region=us-west-1: 2}',
37-
lease_preferences = '[[+region=us-east-1], [+region=us-west-1]]'
38-
39-
exec-sql
40-
BACKUP DATABASE d INTO 'nodelocal://1/database_backup/';
41-
----
42-
43-
exec-sql
44-
BACKUP INTO 'nodelocal://1/full_cluster_backup/';
45-
----
46-
47-
# A new cluster with the same locality settings.
48-
new-cluster name=s2 share-io-dir=s1 allow-implicit-access disable-tenant localities=us-east-1,us-west-1,eu-central-1
49-
----
50-
51-
exec-sql
52-
RESTORE FROM LATEST IN 'nodelocal://1/full_cluster_backup/';
53-
----
54-
55-
exec-sql
56-
DROP DATABASE d;
57-
----
58-
59-
exec-sql
60-
RESTORE DATABASE d FROM LATEST IN 'nodelocal://1/database_backup/';
61-
----
62-
63-
query-sql
64-
SHOW DATABASES;
65-
----
66-
d root us-east-1 us-west-1 {eu-central-1,us-east-1,us-west-1} region
67-
data root <nil> <nil> {} <nil>
68-
defaultdb root <nil> <nil> {} <nil>
69-
postgres root <nil> <nil> {} <nil>
70-
system node <nil> <nil> {} <nil>
71-
72-
query-sql
73-
SHOW ZONE CONFIGURATION FROM DATABASE d;
74-
----
75-
DATABASE d ALTER DATABASE d CONFIGURE ZONE USING
76-
range_min_bytes = 134217728,
77-
range_max_bytes = 536870912,
78-
gc.ttlseconds = 14400,
79-
num_replicas = 5,
80-
num_voters = 5,
81-
constraints = '{+region=eu-central-1: 1, +region=us-east-1: 1, +region=us-west-1: 1}',
82-
voter_constraints = '{+region=us-east-1: 2, +region=us-west-1: 2}',
83-
lease_preferences = '[[+region=us-east-1], [+region=us-west-1]]'
84-
85-
# A new cluster with different localities settings.
86-
new-cluster name=s3 share-io-dir=s1 allow-implicit-access disable-tenant localities=eu-central-1,eu-north-1
87-
----
88-
89-
exec-sql
90-
RESTORE DATABASE d FROM LATEST IN 'nodelocal://1/database_backup/';
91-
----
92-
pq: detected a mismatch in regions between the restore cluster and the backup cluster, missing regions detected: us-east-1, us-west-1.
93-
HINT: there are two ways you can resolve this issue: 1) update the cluster to which you're restoring to ensure that the regions present on the nodes' --locality flags match those present in the backup image, or 2) restore with the "skip_localities_check" option
94-
95-
exec-sql
96-
RESTORE FROM LATEST IN 'nodelocal://1/full_cluster_backup/';
97-
----
98-
pq: detected a mismatch in regions between the restore cluster and the backup cluster, missing regions detected: us-east-1, us-west-1.
99-
HINT: there are two ways you can resolve this issue: 1) update the cluster to which you're restoring to ensure that the regions present on the nodes' --locality flags match those present in the backup image, or 2) restore with the "skip_localities_check" option
100-
101-
exec-sql
102-
RESTORE FROM LATEST IN 'nodelocal://1/full_cluster_backup/' WITH skip_localities_check;
103-
----
104-
105-
exec-sql
106-
INSERT INTO d.t VALUES (4);
107-
----
108-
109-
exec-sql
110-
SET enable_multiregion_placement_policy='true';
111-
ALTER DATABASE d SURVIVE ZONE FAILURE;
112-
ALTER DATABASE d PLACEMENT RESTRICTED;
113-
ALTER DATABASE d SET PRIMARY REGION 'eu-central-1';
114-
ALTER DATABASE d DROP SECONDARY REGION;
115-
ALTER DATABASE d DROP REGION 'us-east-1';
116-
ALTER DATABASE d DROP REGION 'us-west-1';
117-
ALTER DATABASE d ADD REGION 'eu-north-1';
118-
ALTER DATABASE d SET SECONDARY REGION 'eu-north-1';
119-
----
120-
121-
exec-sql
122-
RESTORE DATABASE d FROM LATEST IN 'nodelocal://1/database_backup/' WITH skip_localities_check, new_db_name='d_new';
123-
----
124-
125-
exec-sql
126-
INSERT INTO d_new.t VALUES (4);
127-
----
128-
129-
exec-sql
130-
SET enable_multiregion_placement_policy='true';
131-
ALTER DATABASE d_new SURVIVE ZONE FAILURE;
132-
ALTER DATABASE d PLACEMENT RESTRICTED;
133-
ALTER DATABASE d_new SET PRIMARY REGION 'eu-central-1';
134-
ALTER DATABASE d_new DROP SECONDARY REGION;
135-
ALTER DATABASE d_new DROP REGION 'us-east-1';
136-
ALTER DATABASE d_new DROP REGION 'us-west-1';
137-
ALTER DATABASE d_new ADD REGION 'eu-north-1';
138-
ALTER DATABASE d_new SET SECONDARY REGION 'eu-north-1';
139-
----
140-
141-
exec-sql
142-
DROP DATABASE d_new;
143-
----
144-
145-
exec-sql
146-
DROP DATABASE d;
147-
----
148-
149-
exec-sql
150-
DROP DATABASE data;
6+
new-cluster name=s1 allow-implicit-access disable-tenant localities=eu-central-1
1517
----
1528

1539
# Create a database with no regions to check default primary regions.
@@ -186,7 +42,7 @@ exec-sql
18642
RESTORE DATABASE no_region_db FROM LATEST IN 'nodelocal://1/no_region_database_backup/';
18743
----
18844
pq: region "non-existent-region" does not exist
189-
HINT: valid regions: eu-central-1, eu-north-1
45+
HINT: valid regions: eu-central-1
19046
--
19147
set the default PRIMARY REGION to a region that exists (see SHOW REGIONS FROM CLUSTER) then using SET CLUSTER SETTING sql.defaults.primary_region = 'region'
19248

@@ -202,6 +58,7 @@ HINT: to change the default primary region, use SET CLUSTER SETTING sql.defaults
20258
query-sql
20359
SHOW DATABASES;
20460
----
61+
data root <nil> <nil> {} <nil>
20562
defaultdb root <nil> <nil> {} <nil>
20663
no_region_db root eu-central-1 {eu-central-1} zone
20764
postgres root <nil> <nil> {} <nil>
@@ -238,6 +95,8 @@ NOTICE: setting the PRIMARY REGION as eu-north-1 on database defaultdb
23895
HINT: to change the default primary region, use SET CLUSTER SETTING sql.defaults.primary_region = 'region' or use RESET CLUSTER SETTING sql.defaults.primary_region to disable this behavior
23996
NOTICE: setting the PRIMARY REGION as eu-north-1 on database postgres
24097
HINT: to change the default primary region, use SET CLUSTER SETTING sql.defaults.primary_region = 'region' or use RESET CLUSTER SETTING sql.defaults.primary_region to disable this behavior
98+
NOTICE: setting the PRIMARY REGION as eu-north-1 on database data
99+
HINT: to change the default primary region, use SET CLUSTER SETTING sql.defaults.primary_region = 'region' or use RESET CLUSTER SETTING sql.defaults.primary_region to disable this behavior
241100
NOTICE: setting the PRIMARY REGION as eu-north-1 on database no_region_db
242101
HINT: to change the default primary region, use SET CLUSTER SETTING sql.defaults.primary_region = 'region' or use RESET CLUSTER SETTING sql.defaults.primary_region to disable this behavior
243102
NOTICE: setting the PRIMARY REGION as eu-north-1 on database no_region_db_2
@@ -246,6 +105,7 @@ HINT: to change the default primary region, use SET CLUSTER SETTING sql.defaults
246105
query-sql
247106
SHOW DATABASES;
248107
----
108+
data root eu-north-1 {eu-north-1} zone
249109
defaultdb root eu-north-1 {eu-north-1} zone
250110
no_region_db root eu-north-1 {eu-north-1} zone
251111
no_region_db_2 root eu-north-1 {eu-north-1} zone
@@ -266,6 +126,7 @@ RESTORE DATABASE eu_central_db FROM LATEST IN 'nodelocal://1/eu_central_database
266126
query-sql
267127
SHOW DATABASES;
268128
----
129+
data root eu-north-1 {eu-north-1} zone
269130
defaultdb root eu-north-1 {eu-north-1} zone
270131
eu_central_db root eu-central-1 {eu-central-1} zone
271132
no_region_db root eu-north-1 {eu-north-1} zone

0 commit comments

Comments
 (0)