Skip to content

Commit 52273b1

Browse files
MLE-12270: added restart util fn and updated e2e tests with post restart checks (#252) (#253)
* added restart util fn and updated existing e2e tests with after restart checks
1 parent f3a47e7 commit 52273b1

17 files changed

+261
-112
lines changed

makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ e2e-test: prepare
114114
sudo sysctl -w vm.nr_hugepages=0
115115

116116
@echo "=====Running e2e tests"
117-
$(if $(saveOutput),gotestsum --junitfile test/test_results/e2e-tests.xml ./test/e2e/$(testSelection) -count=1 -timeout 70m, go test -v -count=1 -timeout 70m ./test/e2e/...)
117+
$(if $(saveOutput),gotestsum --junitfile test/test_results/e2e-tests.xml ./test/e2e/$(testSelection) -count=1 -timeout 180m, go test -v -count=1 -timeout 180m ./test/e2e/...)
118118

119119
@echo "=====Setting hugepages value to 1280 for hugepages-e2e test"
120120
sudo sysctl -w vm.nr_hugepages=1280

test/e2e/admin_secrets_test.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package e2e
22

33
import (
4+
"crypto/tls"
45
"os"
56
"path/filepath"
67
"strings"
@@ -10,6 +11,7 @@ import (
1011
"github.com/gruntwork-io/terratest/modules/helm"
1112
"github.com/gruntwork-io/terratest/modules/k8s"
1213
"github.com/gruntwork-io/terratest/modules/random"
14+
"github.com/marklogic/marklogic-kubernetes/test/testUtil"
1315
)
1416

1517
func TestMlAdminSecrets(t *testing.T) {
@@ -36,7 +38,7 @@ func TestMlAdminSecrets(t *testing.T) {
3638
options := &helm.Options{
3739
KubectlOptions: kubectlOptions,
3840
SetValues: map[string]string{
39-
"persistence.enabled": "false",
41+
"persistence.enabled": "true",
4042
"replicaCount": "1",
4143
"image.repository": imageRepo,
4244
"image.tag": imageTag,
@@ -71,4 +73,8 @@ func TestMlAdminSecrets(t *testing.T) {
7173
if !strings.Contains(podLogs, "MARKLOGIC_WALLET_PASSWORD_FILE is set, using file as secret for wallet-password.") {
7274
t.Errorf("wallet password not set as secret")
7375
}
76+
77+
tlsConfig := tls.Config{}
78+
// restart pods in the cluster and verify its ready and MarkLogic server is healthy
79+
testUtil.RestartPodAndVerify(t, false, []string{podName}, namespaceName, kubectlOptions, &tlsConfig)
7480
}

test/e2e/backup_restore_test.go

Lines changed: 3 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import (
1010
"testing"
1111
"time"
1212

13-
http_helper "github.com/gruntwork-io/terratest/modules/http-helper"
1413
"github.com/gruntwork-io/terratest/modules/k8s"
1514
"github.com/gruntwork-io/terratest/modules/random"
1615
"github.com/imroc/req/v3"
@@ -187,23 +186,6 @@ func TestMlDbBackupRestore(t *testing.T) {
187186
// wait until the pod is in Ready status
188187
k8s.WaitUntilPodAvailable(t, kubectlOptions, podName, 10, 15*time.Second)
189188

190-
tunnel7997 := k8s.NewTunnel(kubectlOptions, k8s.ResourceTypePod, podName, 7997, 7997)
191-
defer tunnel7997.Close()
192-
tunnel7997.ForwardPort(t)
193-
endpoint7997 := fmt.Sprintf("http://%s", tunnel7997.Endpoint())
194-
195-
// verify if 7997 health check endpoint returns 200
196-
http_helper.HttpGetWithRetryWithCustomValidation(
197-
t,
198-
endpoint7997,
199-
&tlsConfig,
200-
10,
201-
15*time.Second,
202-
func(statusCode int, body string) bool {
203-
return statusCode == 200
204-
},
205-
)
206-
207189
//create backup directories and setup permissions
208190
k8s.RunKubectl(t, kubectlOptions, "exec", podName, "--", "/bin/bash", "-c", "cd /tmp && mkdir backup && chmod 777 backup && mkdir backup/incrBackup && chmod 777 backup/incrBackup")
209191

@@ -344,4 +326,7 @@ func TestMlDbBackupRestore(t *testing.T) {
344326
if !strings.Contains(string(result), "<b>two</b>") && !strings.Contains(string(result), "<a>one</a>") {
345327
t.Errorf("Both docs are restored")
346328
}
329+
330+
// restart pods in the cluster and verify its ready and MarkLogic server is healthy
331+
testUtil.RestartPodAndVerify(t, false, []string{podName}, namespaceName, kubectlOptions, &tlsConfig)
347332
}

test/e2e/clustering_test.go

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package e2e
22

33
import (
4+
"crypto/tls"
45
"io"
56
"os"
67
"path/filepath"
@@ -12,6 +13,7 @@ import (
1213
"github.com/gruntwork-io/terratest/modules/k8s"
1314
"github.com/gruntwork-io/terratest/modules/random"
1415
"github.com/imroc/req/v3"
16+
"github.com/marklogic/marklogic-kubernetes/test/testUtil"
1517
"github.com/tidwall/gjson"
1618
)
1719

@@ -33,7 +35,7 @@ func TestClusterJoin(t *testing.T) {
3335
}
3436

3537
if !tagPres {
36-
imageTag = "latest"
38+
imageTag = "latest-11"
3739
t.Logf("No imageTag variable present, setting to default value: " + imageTag)
3840
}
3941

@@ -42,7 +44,7 @@ func TestClusterJoin(t *testing.T) {
4244
options := &helm.Options{
4345
KubectlOptions: kubectlOptions,
4446
SetValues: map[string]string{
45-
"persistence.enabled": "false",
47+
"persistence.enabled": "true",
4648
"replicaCount": "2",
4749
"image.repository": imageRepo,
4850
"image.tag": imageTag,
@@ -62,6 +64,7 @@ func TestClusterJoin(t *testing.T) {
6264
releaseName := "test-join"
6365
helm.Install(t, options, helmChartPath, releaseName)
6466

67+
podZeroName := releaseName + "-0"
6568
podName := releaseName + "-1"
6669

6770
// wait until the pod is in Ready status
@@ -113,4 +116,11 @@ func TestClusterJoin(t *testing.T) {
113116
if numOfHosts != 2 {
114117
t.Errorf("Wrong number of hosts")
115118
}
119+
120+
tlsConfig := tls.Config{}
121+
// restart 1 pod at a time in the cluster and verify its ready and MarkLogic server is healthy
122+
testUtil.RestartPodAndVerify(t, false, []string{podZeroName, podName}, namespaceName, kubectlOptions, &tlsConfig)
123+
124+
// restart all pods in the cluster and verify its ready and MarkLogic server is healthy
125+
testUtil.RestartPodAndVerify(t, true, []string{podZeroName, podName}, namespaceName, kubectlOptions, &tlsConfig)
116126
}

test/e2e/env_param_test.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package e2e
22

33
import (
4+
"crypto/tls"
45
"fmt"
56
"io"
67
"net/http"
@@ -13,6 +14,7 @@ import (
1314
"github.com/gruntwork-io/terratest/modules/helm"
1415
"github.com/gruntwork-io/terratest/modules/k8s"
1516
"github.com/gruntwork-io/terratest/modules/random"
17+
"github.com/marklogic/marklogic-kubernetes/test/testUtil"
1618
"github.com/stretchr/testify/assert"
1719
digestAuth "github.com/xinsnake/go-http-digest-auth-client"
1820
)
@@ -47,7 +49,7 @@ func TestEnableConvertersAndLicense(t *testing.T) {
4749
options := &helm.Options{
4850
KubectlOptions: kubectlOptions,
4951
SetValues: map[string]string{
50-
"persistence.enabled": "false",
52+
"persistence.enabled": "true",
5153
"replicaCount": "1",
5254
"image.repository": imageRepo,
5355
"image.tag": imageTag,
@@ -103,4 +105,8 @@ func TestEnableConvertersAndLicense(t *testing.T) {
103105
assert.Contains(t, logs, "LICENSE_KEY and LICENSEE are defined")
104106
// Verify that converters are getting installed
105107
assert.Contains(t, logs, "INSTALL_CONVERTERS is true, installing converters.")
108+
109+
tlsConfig := tls.Config{}
110+
// restart pods in the cluster and verify its ready and MarkLogic server is healthy
111+
testUtil.RestartPodAndVerify(t, false, []string{podName}, namespaceName, kubectlOptions, &tlsConfig)
106112
}

test/e2e/install_test.go

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ func TestHelmInstall(t *testing.T) {
2222
var resp *http.Response
2323
var body []byte
2424
var err error
25-
var podName string
25+
var podZeroName string
2626
imageRepo, repoPres := os.LookupEnv("dockerRepository")
2727
imageTag, tagPres := os.LookupEnv("dockerVersion")
2828

@@ -37,7 +37,7 @@ func TestHelmInstall(t *testing.T) {
3737
}
3838

3939
options := map[string]string{
40-
"persistence.enabled": "false",
40+
"persistence.enabled": "true",
4141
"replicaCount": "2",
4242
"image.repository": imageRepo,
4343
"image.tag": imageTag,
@@ -55,14 +55,15 @@ func TestHelmInstall(t *testing.T) {
5555
defer t.Logf("====Deleting namespace: " + namespaceName)
5656
defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
5757

58-
podName = testUtil.HelmInstall(t, options, releaseName, kubectlOptions)
58+
podZeroName = testUtil.HelmInstall(t, options, releaseName, kubectlOptions)
59+
podOneName := releaseName + "-1"
5960
tlsConfig := tls.Config{}
6061

6162
// wait until the pod is in Ready status
62-
k8s.WaitUntilPodAvailable(t, kubectlOptions, podName, 15, 15*time.Second)
63+
k8s.WaitUntilPodAvailable(t, kubectlOptions, podZeroName, 15, 15*time.Second)
6364

6465
// verify MarkLogic is ready
65-
_, err = testUtil.MLReadyCheck(t, kubectlOptions, podName, &tlsConfig)
66+
_, err = testUtil.MLReadyCheck(t, kubectlOptions, podZeroName, &tlsConfig)
6667
if err != nil {
6768
t.Fatal("MarkLogic failed to start")
6869
}
@@ -79,7 +80,7 @@ func TestHelmInstall(t *testing.T) {
7980
// the random generated username should have length of 11"
8081
assert.Equal(t, 11, len(username))
8182

82-
tunnel8002 := k8s.NewTunnel(kubectlOptions, k8s.ResourceTypePod, podName, 8002, 8002)
83+
tunnel8002 := k8s.NewTunnel(kubectlOptions, k8s.ResourceTypePod, podZeroName, 8002, 8002)
8384
defer tunnel8002.Close()
8485
tunnel8002.ForwardPort(t)
8586
endpointManage := fmt.Sprintf("http://%s/manage/v2", tunnel8002.Endpoint())
@@ -128,4 +129,9 @@ func TestHelmInstall(t *testing.T) {
128129
t.Errorf("Only one group should exist, instead %v groups exist", groupQuantityJSON.Num)
129130
}
130131

132+
// restart pod by pod in the cluster and verify its ready and MarkLogic server is healthy
133+
testUtil.RestartPodAndVerify(t, false, []string{podZeroName, podOneName}, namespaceName, kubectlOptions, &tlsConfig)
134+
135+
// restart all pods in the cluster and verify its ready and MarkLogic server is healthy
136+
testUtil.RestartPodAndVerify(t, true, []string{podZeroName, podOneName}, namespaceName, kubectlOptions, &tlsConfig)
131137
}

test/e2e/path_based_test.go

Lines changed: 54 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package e2e
22

33
import (
4+
"crypto/tls"
45
"fmt"
56
"io"
67
"os"
@@ -13,6 +14,7 @@ import (
1314
"github.com/gruntwork-io/terratest/modules/k8s"
1415
"github.com/gruntwork-io/terratest/modules/random"
1516
"github.com/imroc/req/v3"
17+
"github.com/marklogic/marklogic-kubernetes/test/testUtil"
1618
"github.com/stretchr/testify/assert"
1719
"github.com/tidwall/gjson"
1820
)
@@ -44,7 +46,7 @@ func TestPathBasedRouting(t *testing.T) {
4446
options := &helm.Options{
4547
KubectlOptions: kubectlOptions,
4648
SetValues: map[string]string{
47-
"persistence.enabled": "false",
49+
"persistence.enabled": "true",
4850
"replicaCount": "3",
4951
"image.repository": imageRepo,
5052
"image.tag": imageTag,
@@ -68,11 +70,13 @@ func TestPathBasedRouting(t *testing.T) {
6870
releaseName := "test-path"
6971
helm.Install(t, options, helmChartPath, releaseName)
7072

71-
podName := releaseName + "-2"
73+
podZeroName := releaseName + "-0"
74+
podOneName := releaseName + "-1"
75+
podTwoName := releaseName + "-2"
7276
svcName := releaseName + "-haproxy"
7377

7478
// wait until the pod is in Ready status
75-
k8s.WaitUntilPodAvailable(t, kubectlOptions, podName, 15, 20*time.Second)
79+
k8s.WaitUntilPodAvailable(t, kubectlOptions, podTwoName, 15, 20*time.Second)
7680

7781
tunnel := k8s.NewTunnel(
7882
kubectlOptions, k8s.ResourceTypeService, svcName, 8080, 80)
@@ -145,6 +149,13 @@ func TestPathBasedRouting(t *testing.T) {
145149
t.Errorf("basic authentication is not configured for %s AppServer", appServers[i])
146150
}
147151
}
152+
153+
tlsConfig := tls.Config{}
154+
// restart 1 pod at a time in the cluster and verify its ready and MarkLogic server is healthy
155+
testUtil.RestartPodAndVerify(t, false, []string{podZeroName, podOneName, podTwoName}, namespaceName, kubectlOptions, &tlsConfig)
156+
157+
// restart all pods at once in the cluster and verify its ready and MarkLogic server is healthy
158+
testUtil.RestartPodAndVerify(t, true, []string{podZeroName, podOneName, podTwoName}, namespaceName, kubectlOptions, &tlsConfig)
148159
}
149160

150161
func TestPathBasedRoutAppServers(t *testing.T) {
@@ -159,9 +170,25 @@ func TestPathBasedRoutAppServers(t *testing.T) {
159170
namespaceName := "ml-" + strings.ToLower(random.UniqueId())
160171
kubectlOptions := k8s.NewKubectlOptions("", "", namespaceName)
161172

173+
imageRepo, repoPres := os.LookupEnv("dockerRepository")
174+
imageTag, tagPres := os.LookupEnv("dockerVersion")
175+
if !repoPres {
176+
imageRepo = "marklogicdb/marklogic-db"
177+
t.Logf("No imageRepo variable present, setting to default value: " + imageRepo)
178+
}
179+
180+
if !tagPres {
181+
imageTag = "latest-11"
182+
t.Logf("No imageTag variable present, setting to default value: " + imageTag)
183+
}
184+
162185
// Setup the args for helm install using custom values.yaml file
163186
options := &helm.Options{
164-
ValuesFiles: []string{"../test_data/values/tls_pbr_appser_values.yaml"},
187+
ValuesFiles: []string{"../test_data/values/tls_pbr_appser_values.yaml"},
188+
SetValues: map[string]string{
189+
"image.repository": imageRepo,
190+
"image.tag": imageTag,
191+
},
165192
KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
166193
}
167194

@@ -175,11 +202,12 @@ func TestPathBasedRoutAppServers(t *testing.T) {
175202
releaseName := "test-path"
176203
helm.Install(t, options, helmChartPath, releaseName)
177204

178-
podName := releaseName + "-1"
205+
podZeroName := releaseName + "-0"
206+
podOneName := releaseName + "-1"
179207
svcName := releaseName + "-haproxy"
180208

181209
// wait until the pod is in Ready status
182-
k8s.WaitUntilPodAvailable(t, kubectlOptions, podName, 15, 20*time.Second)
210+
k8s.WaitUntilPodAvailable(t, kubectlOptions, podOneName, 15, 20*time.Second)
183211

184212
tunnel := k8s.NewTunnel(
185213
kubectlOptions, k8s.ResourceTypeService, svcName, 8080, 80)
@@ -249,6 +277,13 @@ func TestPathBasedRoutAppServers(t *testing.T) {
249277
t.Fatalf(err.Error())
250278
}
251279
assert.Contains(t, string(body), "XDMP-MODNOTFOUND")
280+
281+
tlsConfig := tls.Config{}
282+
// restart 1 pod at a time in the cluster and verify its ready and MarkLogic server is healthy
283+
testUtil.RestartPodAndVerify(t, false, []string{podZeroName, podOneName}, namespaceName, kubectlOptions, &tlsConfig)
284+
285+
// restart all pods at once in the cluster and verify its ready and MarkLogic server is healthy
286+
testUtil.RestartPodAndVerify(t, true, []string{podZeroName, podOneName}, namespaceName, kubectlOptions, &tlsConfig)
252287
}
253288

254289
func TestPathBasedRoutingWithTLS(t *testing.T) {
@@ -278,7 +313,7 @@ func TestPathBasedRoutingWithTLS(t *testing.T) {
278313
options := &helm.Options{
279314
KubectlOptions: kubectlOptions,
280315
SetValues: map[string]string{
281-
"persistence.enabled": "false",
316+
"persistence.enabled": "true",
282317
"replicaCount": "3",
283318
"image.repository": imageRepo,
284319
"image.tag": imageTag,
@@ -303,11 +338,13 @@ func TestPathBasedRoutingWithTLS(t *testing.T) {
303338
releaseName := "test-pb-tls"
304339
helm.Install(t, options, helmChartPath, releaseName)
305340

306-
podName := releaseName + "-2"
341+
podZeroName := releaseName + "-0"
342+
podOneName := releaseName + "-1"
343+
podTwoName := releaseName + "-2"
307344
svcName := releaseName + "-haproxy"
308345

309346
// wait until the pod is in Ready status
310-
k8s.WaitUntilPodAvailable(t, kubectlOptions, podName, 10, 20*time.Second)
347+
k8s.WaitUntilPodAvailable(t, kubectlOptions, podTwoName, 10, 20*time.Second)
311348

312349
tunnel := k8s.NewTunnel(
313350
kubectlOptions, k8s.ResourceTypeService, svcName, 8080, 80)
@@ -377,4 +414,12 @@ func TestPathBasedRoutingWithTLS(t *testing.T) {
377414
t.Errorf("ssl is not enabled for %s AppServer", appServers[i])
378415
}
379416
}
417+
418+
tlsConfig := tls.Config{}
419+
420+
// restart 1 pod at a time in the cluster and verify its ready and MarkLogic server is healthy
421+
testUtil.RestartPodAndVerify(t, false, []string{podZeroName, podOneName, podTwoName}, namespaceName, kubectlOptions, &tlsConfig)
422+
423+
// restart all pods at once in the cluster and verify its ready and MarkLogic server is healthy
424+
testUtil.RestartPodAndVerify(t, true, []string{podZeroName, podOneName, podTwoName}, namespaceName, kubectlOptions, &tlsConfig)
380425
}

0 commit comments

Comments
 (0)