MLE-13839: added changes for helm upgrade e2e test (#250) (#259)

* added changes for helm upgrade e2e test

Author: barkhachoithani

Jenkinsfile

@@ -158,7 +158,7 @@ pipeline {
         skipStagesAfterUnstable()
     }
     triggers {
-        parameterizedCron( env.BRANCH_NAME == 'develop' ? '''00 04 * * * % IMAGE_SCAN=true;HC_TESTS=true''' : '')
+        parameterizedCron( env.BRANCH_NAME == 'develop' ? '''00 04 * * * % IMAGE_SCAN=true;HELM_UPGRADE_TESTS=true;HC_TESTS=true''' : '')
     }
     environment {
         dockerRegistry = 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com'
@@ -174,6 +174,8 @@ pipeline {
         string(name: 'KUBERNETES_TEST_SELECTION', defaultValue: '...', description: 'Pick one test to run. (e.g. tls_test.go) ... will run all tests.', trim: true)
         booleanParam(name: 'HC_TESTS', defaultValue: false, description: 'Run Hub Central E2E UI tests (takes about 3 hours)')
         booleanParam(name: 'IMAGE_SCAN', defaultValue: false, description: 'Find and scan dependent Docker images for security vulnerabilities')
+        booleanParam(name: 'HELM_UPGRADE_TESTS', defaultValue: false, description: 'Run Helm upgrade in E2E tests (runs nightly on develop)')
+        string(name: 'InitialChartVersion', defaultValue: '1.1.2', description: 'Helm Chart Version to use for upgrade tests. (e.g. 1.1.2)', trim: true)
         string(name: 'emailList', defaultValue: emailList, description: 'List of email for build notification', trim: true)
     }
 
@@ -209,6 +211,16 @@ pipeline {
                 """
             }
         }
+        stage('Kubernetes-Run-Upgrade-Tests') {
+            when {
+                expression { return params.HELM_UPGRADE_TESTS }
+            }
+            steps {
+                sh """
+                    export MINIKUBE_HOME=/space; export KUBECONFIG=/space/.kube-config; export GOPATH=/space/go; export upgradeTest=true; export initialChartVersion=${params.InitialChartVersion}; make upgrade-test dockerImage=${dockerRepository}:${dockerVersion} prevDockerImage=${dockerRepository}:${prevDockerVersion} kubernetesVersion=${params.K8_VERSION} saveOutput=true minikubeMemory=20gb
+                """
+            }
+        }
         stage('Kubernetes-Run-HC-Tests') {
             when {
                 expression { return params.HC_TESTS }

makefile

@@ -192,6 +192,24 @@ template-test: prepare
 .PHONY: test
 test: template-test e2e-test
 
+#***************************************************************************
+# test
+#***************************************************************************
+## Run upgrade in e2e tests
+## Set following environment variables 
+## [upgradeTest] to true. Use `export upgradeTest=true`
+## [initialChartVersion] to a valid MarkLogic helm chart version for ex.: 1.1.2 to run upgrade tests. Use `export initialChartVersion=1.1.2`
+.PHONY: upgrade-test
+upgrade-test: prepare
+	@echo "=====upgradeTest env var for upgrade tests"
+	echo $(upgradeTest)
+
+	@echo "=====initialChartVersion env var for upgrade tests"
+	echo ${initialChartVersion}
+	
+	@echo "=====Running upgrades in e2e tests"
+	make e2e-test
+	
 #***************************************************************************
 # image-scan
 #***************************************************************************

test/e2e/admin_secrets_test.go

@@ -4,6 +4,7 @@ import (
 	"crypto/tls"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -15,10 +16,13 @@ import (
 )
 
 func TestMlAdminSecrets(t *testing.T) {
-	// Path to the helm chart we will test
-	helmChartPath, e := filepath.Abs("../../charts")
-	if e != nil {
-		t.Fatalf(e.Error())
+	var helmChartPath string
+	var initialChartVersion string
+	upgradeHelm, _ := os.LookupEnv("upgradeTest")
+	runUpgradeTest, err := strconv.ParseBool(upgradeHelm)
+	if runUpgradeTest {
+		initialChartVersion, _ = os.LookupEnv("initialChartVersion")
+		t.Logf("====Setting initial Helm chart version: %s", initialChartVersion)
 	}
 	imageRepo, repoPres := os.LookupEnv("dockerRepository")
 	imageTag, tagPres := os.LookupEnv("dockerVersion")
@@ -40,12 +44,13 @@ func TestMlAdminSecrets(t *testing.T) {
 		SetValues: map[string]string{
 			"persistence.enabled": "true",
 			"replicaCount":        "1",
-			"image.repository":    imageRepo,
-			"image.tag":           imageTag,
+			"image.repository":    "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi-rootless",
+			"image.tag":           "latest-11",
 			"auth.adminUsername":  "admin",
 			"auth.adminPassword":  "admin",
 			"auth.walletPassword": "admin",
 		},
+		Version: initialChartVersion,
 	}
 
 	t.Logf("====Creating namespace: " + namespaceName)
@@ -54,14 +59,51 @@ func TestMlAdminSecrets(t *testing.T) {
 	defer t.Logf("====Deleting namespace: " + namespaceName)
 	defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
 
+	// Path to the helm chart we will test
+	helmChartPath, err = filepath.Abs("../../charts")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	//add the helm chart repo and install the last helm chart release from repository
+	//to test and upgrade this chart to the latest one to be released
+	if runUpgradeTest {
+		helm.AddRepo(t, options, "marklogic", "https://marklogic.github.io/marklogic-kubernetes/")
+		defer helm.RemoveRepo(t, options, "marklogic")
+		helmChartPath = "marklogic/marklogic"
+	}
+
+	t.Logf("====Setting helm chart path to %s", helmChartPath)
 	t.Logf("====Installing Helm Chart")
 	releaseName := "test-ml-secrets"
-	helm.Install(t, options, helmChartPath, releaseName)
+	podName := testUtil.HelmInstall(t, options, releaseName, kubectlOptions, helmChartPath)
 
-	podName := releaseName + "-0"
 	// wait until the pod is in Ready status
 	k8s.WaitUntilPodAvailable(t, kubectlOptions, podName, 15, 15*time.Second)
 
+	if runUpgradeTest {
+		// create options for helm upgrade
+		upgradeOptionsMap := map[string]string{
+			"persistence.enabled":   "true",
+			"replicaCount":          "1",
+			"logCollection.enabled": "false",
+			"allowLongHostnames":    "true",
+		}
+
+		if strings.HasPrefix(initialChartVersion, "1.0") {
+			podName = releaseName + "-marklogic-0"
+			upgradeOptionsMap["useLegacyHostnames"] = "true"
+		}
+
+		//set helm options for upgrading helm chart version
+		helmUpgradeOptions := &helm.Options{
+			KubectlOptions: kubectlOptions,
+			SetValues:      upgradeOptionsMap,
+		}
+		t.Logf("UpgradeHelmTest is enabled. Running helm upgrade test")
+		testUtil.HelmUpgrade(t, helmUpgradeOptions, releaseName, kubectlOptions, []string{podName}, initialChartVersion)
+	}
+
 	// get corev1.Pod to get logs of a pod
 	pod := k8s.GetPod(t, kubectlOptions, podName)
 

test/e2e/backup_restore_test.go

@@ -6,10 +6,13 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/gruntwork-io/terratest/modules/helm"
 	"github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/gruntwork-io/terratest/modules/random"
 	"github.com/imroc/req/v3"
@@ -136,9 +139,16 @@ func RunRequests(client *req.Client, dbReq string, hostsEndpoint string) (string
 
 func TestMlDbBackupRestore(t *testing.T) {
 	// var resp *http.Response
-
+	var helmChartPath string
 	var err error
 	var podName string
+	var initialChartVersion string
+	upgradeHelm, _ := os.LookupEnv("upgradeTest")
+	runUpgradeTest, err := strconv.ParseBool(upgradeHelm)
+	if runUpgradeTest {
+		initialChartVersion, _ = os.LookupEnv("initialChartVersion")
+		t.Logf("====Setting initial Helm chart version: %s", initialChartVersion)
+	}
 	imageRepo, repoPres := os.LookupEnv("dockerRepository")
 	imageTag, tagPres := os.LookupEnv("dockerVersion")
 
@@ -155,37 +165,76 @@ func TestMlDbBackupRestore(t *testing.T) {
 	username := "admin"
 	password := "admin"
 
-	options := map[string]string{
-		"persistence.enabled":   "true",
-		"replicaCount":          "1",
-		"image.repository":      imageRepo,
-		"image.tag":             imageTag,
-		"auth.adminUsername":    username,
-		"auth.adminPassword":    password,
-		"logCollection.enabled": "false",
+	namespaceName := "ml-" + strings.ToLower(random.UniqueId())
+	kubectlOptions := k8s.NewKubectlOptions("", "", namespaceName)
+	options := &helm.Options{
+		KubectlOptions: kubectlOptions,
+		SetValues: map[string]string{
+			"persistence.enabled":   "true",
+			"replicaCount":          "1",
+			"image.repository":      imageRepo,
+			"image.tag":             imageTag,
+			"auth.adminUsername":    username,
+			"auth.adminPassword":    password,
+			"logCollection.enabled": "false",
+		},
+		Version: initialChartVersion,
 	}
 
 	t.Logf("====Installing Helm Chart")
 	releaseName := "bkuprestore"
 
-	namespaceName := "ml-" + strings.ToLower(random.UniqueId())
-	kubectlOptions := k8s.NewKubectlOptions("", "", namespaceName)
-
 	t.Logf("====Creating namespace: " + namespaceName)
 	k8s.CreateNamespace(t, kubectlOptions, namespaceName)
 
 	defer t.Logf("====Deleting namespace: " + namespaceName)
 	defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
 
-	podName = testUtil.HelmInstall(t, options, releaseName, kubectlOptions)
+	helmChartPath, err = filepath.Abs("../../charts")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	//add the helm chart repo and install the last helm chart release from repository
+	//to test and upgrade this chart to the latest one to be released
+	if runUpgradeTest {
+		helm.AddRepo(t, options, "marklogic", "https://marklogic.github.io/marklogic-kubernetes/")
+		defer helm.RemoveRepo(t, options, "marklogic")
+		helmChartPath = "marklogic/marklogic"
+	}
+
+	t.Logf("====Setting helm chart path to %s", helmChartPath)
+	t.Logf("====Installing Helm Chart")
+	podName = testUtil.HelmInstall(t, options, releaseName, kubectlOptions, helmChartPath)
 
 	t.Logf("====Describe pod for backup restore test")
 	k8s.RunKubectl(t, kubectlOptions, "describe", "pod", podName)
 
-	tlsConfig := tls.Config{}
 	// wait until the pod is in Ready status
 	k8s.WaitUntilPodAvailable(t, kubectlOptions, podName, 10, 15*time.Second)
 
+	if runUpgradeTest {
+		// create options for helm upgrade
+		upgradeOptionsMap := map[string]string{
+			"persistence.enabled":   "true",
+			"replicaCount":          "1",
+			"logCollection.enabled": "false",
+			"allowLongHostnames":    "true",
+		}
+		if strings.HasPrefix(initialChartVersion, "1.0") {
+			podName = releaseName + "-marklogic-0"
+			upgradeOptionsMap["useLegacyHostnames"] = "true"
+		}
+		//set helm options for upgrading helm chart version
+		helmUpgradeOptions := &helm.Options{
+			KubectlOptions: kubectlOptions,
+			SetValues:      upgradeOptionsMap,
+		}
+
+		t.Logf("UpgradeHelmTest is enabled. Running helm upgrade test")
+		testUtil.HelmUpgrade(t, helmUpgradeOptions, releaseName, kubectlOptions, []string{podName}, initialChartVersion)
+	}
+
 	//create backup directories and setup permissions
 	k8s.RunKubectl(t, kubectlOptions, "exec", podName, "--", "/bin/bash", "-c", "cd /tmp && mkdir backup && chmod 777 backup && mkdir backup/incrBackup && chmod 777 backup/incrBackup")
 
@@ -327,6 +376,7 @@ func TestMlDbBackupRestore(t *testing.T) {
 		t.Errorf("Both docs are restored")
 	}
 
+	tlsConfig := tls.Config{}
 	// restart pods in the cluster and verify its ready and MarkLogic server is healthy
 	testUtil.RestartPodAndVerify(t, false, []string{podName}, namespaceName, kubectlOptions, &tlsConfig)
 }