Skip to content

Commit 7bb1ee9

Browse files
ewoutpewoutp
committed
Added upgrade tests
1 parent 7954f88 commit 7bb1ee9

File tree

4 files changed

+258
-1
lines changed

4 files changed

+258
-1
lines changed

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ ifdef TRAVIS
5858
echo Using IP=$(IP)
5959
endif
6060

61-
TEST_TIMEOUT := 20m
61+
TEST_TIMEOUT := 25m
6262

6363
BINNAME := arangodb$(GOEXE)
6464
BIN := $(BINDIR)/$(GOOS)/$(GOARCH)/$(BINNAME)

‎service/upgrade_manager.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1355,6 +1355,14 @@ func (m *upgradeManager) isSuperVisionMaintenanceSupported(ctx context.Context)
13551355

13561356
// disableSupervision blocks supervision of the agency and waits for the agency to acknowledge.
13571357
func (m *upgradeManager) disableSupervision(ctx context.Context) error {
1358+
supported, err := m.isSuperVisionMaintenanceSupported(ctx)
1359+
if err != nil {
1360+
return maskAny(err)
1361+
}
1362+
if !supported {
1363+
m.log.Info().Msg("Supervision maintenance is not supported on this version")
1364+
return nil
1365+
}
13581366
api, err := m.createAgencyAPI()
13591367
if err != nil {
13601368
return maskAny(err)
@@ -1401,6 +1409,15 @@ func getMaintenanceMode(value interface{}) (string, bool) {
14011409

14021410
// enableSupervision enabled supervision of the agency.
14031411
func (m *upgradeManager) enableSupervision(ctx context.Context) error {
1412+
supported, err := m.isSuperVisionMaintenanceSupported(ctx)
1413+
if err != nil {
1414+
return maskAny(err)
1415+
}
1416+
if !supported {
1417+
m.log.Info().Msg("Supervision maintenance is not supported on this version")
1418+
return nil
1419+
}
1420+
14041421
api, err := m.createAgencyAPI()
14051422
if err != nil {
14061423
return maskAny(err)

‎test/docker_cluster_upgrade_test.go‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
//
20+
// Author Ewout Prangsma
21+
//
22+
23+
package test
24+
25+
import (
26+
"fmt"
27+
"os"
28+
"strings"
29+
"testing"
30+
"time"
31+
)
32+
33+
// TestDockerClusterUpgrade runs 3 arangodb starters in docker with default settings.
34+
// Once running it starts a database upgrade.
35+
func TestDockerClusterUpgrade(t *testing.T) {
36+
needTestMode(t, testModeDocker)
37+
needStarterMode(t, starterModeCluster)
38+
if os.Getenv("IP") == "" {
39+
t.Fatal("IP envvar must be set to IP address of this machine")
40+
}
41+
/*
42+
docker volume create arangodb1
43+
docker run -i --name=adb1 --rm -p 8528:8528 \
44+
-v arangodb1:/data \
45+
-v /var/run/docker.sock:/var/run/docker.sock \
46+
arangodb/arangodb-starter \
47+
--docker.container=adb1 \
48+
--starter.address=$IP
49+
*/
50+
volID1 := createDockerID("vol-starter-test-cluster-upgrade1-")
51+
createDockerVolume(t, volID1)
52+
defer removeDockerVolume(t, volID1)
53+
54+
volID2 := createDockerID("vol-starter-test-cluster-upgrade2-")
55+
createDockerVolume(t, volID2)
56+
defer removeDockerVolume(t, volID2)
57+
58+
volID3 := createDockerID("vol-starter-test-cluster-upgrade3-")
59+
createDockerVolume(t, volID3)
60+
defer removeDockerVolume(t, volID3)
61+
62+
// Cleanup of left over tests
63+
removeDockerContainersByLabel(t, "starter-test=true")
64+
removeStarterCreatedDockerContainers(t)
65+
66+
start := time.Now()
67+
68+
cID1 := createDockerID("starter-test-cluster-upgrade1-")
69+
dockerRun1 := Spawn(t, strings.Join([]string{
70+
"docker run -i",
71+
"--label starter-test=true",
72+
"--name=" + cID1,
73+
"--rm",
74+
fmt.Sprintf("-p %d:%d", basePort, basePort),
75+
fmt.Sprintf("-v %s:/data", volID1),
76+
"-v /var/run/docker.sock:/var/run/docker.sock",
77+
"arangodb/arangodb-starter",
78+
"--docker.container=" + cID1,
79+
"--starter.address=$IP",
80+
createEnvironmentStarterOptions(),
81+
}, " "))
82+
defer dockerRun1.Close()
83+
defer removeDockerContainer(t, cID1)
84+
85+
cID2 := createDockerID("starter-test-cluster-upgrade2-")
86+
dockerRun2 := Spawn(t, strings.Join([]string{
87+
"docker run -i",
88+
"--label starter-test=true",
89+
"--name=" + cID2,
90+
"--rm",
91+
fmt.Sprintf("-p %d:%d", basePort+(1*portIncrement), basePort),
92+
fmt.Sprintf("-v %s:/data", volID2),
93+
"-v /var/run/docker.sock:/var/run/docker.sock",
94+
"arangodb/arangodb-starter",
95+
"--docker.container=" + cID2,
96+
"--starter.address=$IP",
97+
createEnvironmentStarterOptions(),
98+
fmt.Sprintf("--starter.join=$IP:%d", basePort),
99+
}, " "))
100+
defer dockerRun2.Close()
101+
defer removeDockerContainer(t, cID2)
102+
103+
cID3 := createDockerID("starter-test-cluster-upgrade3-")
104+
dockerRun3 := Spawn(t, strings.Join([]string{
105+
"docker run -i",
106+
"--label starter-test=true",
107+
"--name=" + cID3,
108+
"--rm",
109+
fmt.Sprintf("-p %d:%d", basePort+(2*portIncrement), basePort),
110+
fmt.Sprintf("-v %s:/data", volID3),
111+
"-v /var/run/docker.sock:/var/run/docker.sock",
112+
"arangodb/arangodb-starter",
113+
"--docker.container=" + cID3,
114+
"--starter.address=$IP",
115+
createEnvironmentStarterOptions(),
116+
fmt.Sprintf("--starter.join=$IP:%d", basePort),
117+
}, " "))
118+
defer dockerRun3.Close()
119+
defer removeDockerContainer(t, cID3)
120+
121+
if ok := WaitUntilStarterReady(t, whatCluster, 3, dockerRun1, dockerRun2, dockerRun3); ok {
122+
t.Logf("Cluster start took %s", time.Since(start))
123+
testCluster(t, insecureStarterEndpoint(0*portIncrement), false)
124+
testCluster(t, insecureStarterEndpoint(1*portIncrement), false)
125+
testCluster(t, insecureStarterEndpoint(2*portIncrement), false)
126+
}
127+
128+
testUpgradeProcess(t, insecureStarterEndpoint(0*portIncrement))
129+
130+
if isVerbose {
131+
t.Log("Waiting for termination")
132+
}
133+
ShutdownStarter(t, insecureStarterEndpoint(0*portIncrement))
134+
ShutdownStarter(t, insecureStarterEndpoint(1*portIncrement))
135+
ShutdownStarter(t, insecureStarterEndpoint(2*portIncrement))
136+
}

‎test/process_cluster_upgrade_test.go‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
//
20+
// Author Ewout Prangsma
21+
//
22+
23+
package test
24+
25+
import (
26+
"context"
27+
"os"
28+
"testing"
29+
"time"
30+
)
31+
32+
// TestProcessClusterUpgrade starts a master starter, followed by 2 slave starters.
33+
// Once running it starts a database upgrade.
34+
func TestProcessClusterUpgrade(t *testing.T) {
35+
removeArangodProcesses(t)
36+
needTestMode(t, testModeProcess)
37+
needStarterMode(t, starterModeCluster)
38+
dataDirMaster := SetUniqueDataDir(t)
39+
defer os.RemoveAll(dataDirMaster)
40+
41+
start := time.Now()
42+
43+
master := Spawn(t, "${STARTER} "+createEnvironmentStarterOptions())
44+
defer master.Close()
45+
46+
dataDirSlave1 := SetUniqueDataDir(t)
47+
defer os.RemoveAll(dataDirSlave1)
48+
slave1 := Spawn(t, "${STARTER} --starter.join 127.0.0.1 "+createEnvironmentStarterOptions())
49+
defer slave1.Close()
50+
51+
dataDirSlave2 := SetUniqueDataDir(t)
52+
defer os.RemoveAll(dataDirSlave2)
53+
slave2 := Spawn(t, "${STARTER} --starter.join 127.0.0.1 "+createEnvironmentStarterOptions())
54+
defer slave2.Close()
55+
56+
if ok := WaitUntilStarterReady(t, whatCluster, 3, master, slave1, slave2); ok {
57+
t.Logf("Cluster start took %s", time.Since(start))
58+
testCluster(t, insecureStarterEndpoint(0*portIncrement), false)
59+
testCluster(t, insecureStarterEndpoint(1*portIncrement), false)
60+
testCluster(t, insecureStarterEndpoint(2*portIncrement), false)
61+
}
62+
63+
testUpgradeProcess(t, insecureStarterEndpoint(0*portIncrement))
64+
65+
if isVerbose {
66+
t.Log("Waiting for termination")
67+
}
68+
SendIntrAndWait(t, master, slave1, slave2)
69+
}
70+
71+
func testUpgradeProcess(t *testing.T, endpoint string) {
72+
t.Log("Starting database upgrade")
73+
c := NewStarterClient(t, endpoint)
74+
ctx := context.Background()
75+
if err := c.StartDatabaseUpgrade(ctx); err != nil {
76+
t.Fatalf("StartDatabaseUpgrade failed: %v", err)
77+
}
78+
// Wait until upgrade complete
79+
recentErrors := 0
80+
deadline := time.Now().Add(time.Minute * 5)
81+
for {
82+
status, err := c.UpgradeStatus(ctx)
83+
if err != nil {
84+
recentErrors++
85+
if recentErrors > 20 {
86+
t.Fatalf("UpgradeStatus failed: %s", err)
87+
} else {
88+
t.Logf("UpgradeStatus failed: %s", err)
89+
}
90+
} else {
91+
recentErrors = 0
92+
if status.Failed {
93+
t.Fatalf("Upgrade failed: %s", status.Reason)
94+
}
95+
if status.Ready {
96+
break
97+
}
98+
}
99+
if time.Now().After(deadline) {
100+
t.Fatal("Upgrade failed to finished in time")
101+
}
102+
time.Sleep(time.Second)
103+
}
104+
}

0 commit comments

Comments
 (0)