MLE-12244: fixes for failing upgrade tests on jenkins (#190)

Author: barkhachoithani

test/e2e/upgrade_test.go

@@ -3,7 +3,7 @@ package e2e
 import (
 	"crypto/tls"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -15,6 +15,7 @@ import (
 	http_helper "github.com/gruntwork-io/terratest/modules/http-helper"
 	"github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/gruntwork-io/terratest/modules/random"
+	"github.com/imroc/req/v3"
 	"github.com/stretchr/testify/assert"
 	"github.com/tidwall/gjson"
 	digestAuth "github.com/xinsnake/go-http-digest-auth-client"
@@ -64,6 +65,8 @@ func TestHelmUpgrade(t *testing.T) {
 	// save the generated password from first installation
 	secretName := releaseName + "-admin"
 	secret := k8s.GetSecret(t, kubectlOptions, secretName)
+	usernameArr := secret.Data["username"]
+	username := string(usernameArr)
 	passwordArr := secret.Data["password"]
 	passwordAfterInstall := string(passwordArr[:])
 
@@ -82,12 +85,21 @@ func TestHelmUpgrade(t *testing.T) {
 	helm.Upgrade(t, newOptions, helmChartPath, releaseName)
 
 	tlsConfig := tls.Config{}
-	podName := releaseName + "-1"
+	podOneName := releaseName + "-1"
+	podZeroName := releaseName + "-0"
 
 	// wait until the pod is in Ready status
-	k8s.WaitUntilPodAvailable(t, kubectlOptions, podName, 20, 20*time.Second)
+	k8s.WaitUntilPodAvailable(t, kubectlOptions, podOneName, 20, 20*time.Second)
+
+	t.Log("====Test password in secret should not change after upgrade====")
+	secret = k8s.GetSecret(t, kubectlOptions, secretName)
+	passwordArr = secret.Data["password"]
+	passwordAfterUpgrade := string(passwordArr[:])
+	assert.Equal(t, passwordAfterUpgrade, passwordAfterInstall)
+
 	tunnel := k8s.NewTunnel(
-		kubectlOptions, k8s.ResourceTypePod, podName, 7997, 7997)
+		kubectlOptions, k8s.ResourceTypePod, podZeroName, 7997, 7997)
+
 	defer tunnel.Close()
 	tunnel.ForwardPort(t)
 	endpoint := fmt.Sprintf("http://%s", tunnel.Endpoint())
@@ -97,20 +109,51 @@ func TestHelmUpgrade(t *testing.T) {
 		t,
 		endpoint,
 		&tlsConfig,
-		10,
-		15*time.Second,
+		15,
+		20*time.Second,
 		func(statusCode int, body string) bool {
 			return statusCode == 200
 		},
 	)
 
-	t.Log("====Test password in secret should not change after upgrade====")
-	secret = k8s.GetSecret(t, kubectlOptions, secretName)
-	passwordArr = secret.Data["password"]
-	passwordAfterUpgrade := string(passwordArr[:])
-	assert.Equal(t, passwordAfterUpgrade, passwordAfterInstall)
-}
+	tunnel8002 := k8s.NewTunnel(
+		kubectlOptions, k8s.ResourceTypePod, podZeroName, 8002, 8002)
+	defer tunnel8002.Close()
+	tunnel8002.ForwardPort(t)
+
+	hostsEndpoint := fmt.Sprintf("http://%s/manage/v2/hosts?view=status&format=json", tunnel8002.Endpoint())
+	t.Logf(`Endpoint: %s`, hostsEndpoint)
+
+	totalHosts := 1
+	client := req.C().
+		SetCommonDigestAuth(username, passwordAfterUpgrade).
+		SetCommonRetryCount(10).
+		SetCommonRetryFixedInterval(10 * time.Second)
+
+	resp, err := client.R().
+		AddRetryCondition(func(resp *req.Response, err error) bool {
+			body, err := io.ReadAll(resp.Body)
+			if err != nil {
+				t.Logf("error: %s", err.Error())
+			}
+			totalHosts = int(gjson.Get(string(body), `host-status-list.status-list-summary.total-hosts.value`).Num)
+			if totalHosts != 2 {
+				t.Log("Waiting for second host to join MarkLogic cluster")
+			}
+			return totalHosts != 2
+		}).
+		Get(hostsEndpoint)
 
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+	defer resp.Body.Close()
+
+	if totalHosts != 2 {
+		t.Errorf("Incorrect number of MarkLogic hosts found after helm upgrade")
+	}
+
+}
 func TestMLupgrade(t *testing.T) {
 	// Path to the helm chart we will test
 	helmChartPath, e := filepath.Abs("../../charts")
@@ -202,15 +245,15 @@ func TestMLupgrade(t *testing.T) {
 		t.Fatalf(err.Error())
 	}
 	defer resp.Body.Close()
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		t.Fatalf(err.Error())
 	}
 	mlVersionPattern := regexp.MustCompile(`(\d+\.\d+)`)
 	mlVersionResp := gjson.Get(string(body), `local-cluster-default.version`)
 	actualMlVersion := mlVersionPattern.FindStringSubmatch(mlVersionResp.Str)
 	expectedMlVersion := mlVersionPattern.FindStringSubmatch(imageTag)
-	//expectedMlVersion := strings.Split(imageTag, "-centos")[0]
+	// expectedMlVersion := strings.Split(imageTag, "-centos")[0]
 	// verify latest MarkLogic version after upgrade
 	assert.Equal(t, actualMlVersion, expectedMlVersion)
 }