Merge pull request #83 from barkhachoithani/feature/CLD-706

CLD-706: Support upgradeStrategy for ML K8s

Author: barkhachoithani

Jenkinsfile

@@ -135,6 +135,8 @@ void pullImage() {
         sh """
             echo "\$docker_password" | docker login --username \$docker_user --password-stdin ${dockerRegistry}
             docker pull ${dockerRepository}:${dockerVersion}
+            docker pull ${dockerRepository}:${dockerVersion}
+            docker pull ${dockerRepository}:${prevDockerVersion}
         """
     }
 }
@@ -169,14 +171,18 @@ pipeline {
         dockerRegistry = 'ml-docker-dev.marklogic.com'
         dockerRepository = "${dockerRegistry}/marklogic/marklogic-server-centos"
         dockerVerDivider = getVersionDiv(params.ML_VERSION)
+        prevDockerVerDivider = getVersionDiv(params.PREV_ML_VERSION)
         dockerVersion = "${ML_VERSION}${dockerVerDivider}${timeStamp}-centos-${dockerReleaseVer}"
+        prevDockerVersion = "${PREV_ML_VERSION}${prevDockerVerDivider}${timeStamp}-centos-${prevDockerReleaseVer}"
     }
 
     parameters {
         string(name: 'emailList', defaultValue: emailList, description: 'List of email for build notification', trim: true)
         choice(name: 'ML_VERSION', choices: '11.0\n12.0\n10.0\n9.0', description: 'MarkLogic version. used to pick appropriate docker image')
         booleanParam(name: 'KUBERNETES_TESTS', defaultValue: true, description: 'Run kubernetes tests')
         string(name: 'dockerReleaseVer', defaultValue: '1.0.1', description: 'Current Docker version. (e.g. 1.0.1)', trim: true)
+        choice(name: 'PREV_ML_VERSION', choices: '10.0\n9.0', description: 'Previous MarkLogic version for MarkLogic upgrade tests')
+        string(name: 'prevDockerReleaseVer', defaultValue: '1.0.1', description: 'Previous Docker version for MarkLogic upgrade tests. (e.g. 1.0.1)', trim: true)
     }
 
     stages {
@@ -205,7 +211,7 @@ pipeline {
             steps {
                 sh """
                     export MINIKUBE_HOME=/space;
-                    make test dockerImage=${dockerRepository}:${dockerVersion} saveOutput=true
+                    make test dockerImage=${dockerRepository}:${dockerVersion} prevDockerImage=${dockerRepository}:${prevDockerVersion} saveOutput=true
                 """
             }
         }

README.md

@@ -516,6 +516,7 @@ This table describes the list of available parameters for Helm Chart.
 | `networkPolicy.customRules`      | Placeholder to specify selectors              | `{}`                               |
 | `networkPolicy.ports`      | Ports to which traffic is allowed              | `[8000, 8001, 8002]`                               |
 | `priorityClassName`      | Name of a PriortyClass defined to set pod priority        | `""`                               |
+| `updateStrategy`      | Update strategy for helm chart and app version updates        | `OnDelete`                               |
 
 # Known Issues and Limitations
 

charts/templates/statefulset.yaml

@@ -8,6 +8,8 @@ metadata:
 spec:
   serviceName: {{ include "marklogic.headlessServiceName" . }}
   replicas: {{ .Values.replicaCount }}
+  updateStrategy:
+    type: {{ .Values.updateStrategy.type }}
   selector:
     matchLabels:
       {{- include "marklogic.selectorLabels" . | nindent 6 }}

charts/values.yaml

@@ -3,6 +3,12 @@
 # Number of Marklogic nodes
 replicaCount: 1
 
+# update strategy for MarkLogic and Helm chart upgrades
+# It is recommended to use OnDelete updateStrategy as MarkLogic bootstrap host (pod-0) needs to be upgraded first in the cluster
+# and onDelete allows more control over the upgrade and recovery in case of failure.
+updateStrategy:  
+    type: OnDelete
+
 # Termination Grace Period
 terminationGracePeriod: 120
 

makefile

@@ -1,5 +1,5 @@
-dockerImage?=marklogic-centos/marklogic-server-centos:10-internal
-
+dockerImage?=ml-docker-dev.marklogic.com/marklogic/marklogic-server-centos:11.0.20230307-centos-1.0.2
+prevDockerImage?=ml-docker-dev.marklogic.com/marklogic/marklogic-server-centos:10.0-20230307-centos-1.0.2
 ## System requirement:
 ## - Go 
 ## 		- gotestsum (if you want to enable saveOutput for testing commands)
@@ -80,6 +80,7 @@ lint:
 ## Run all end to end tests
 ## Options:
 ## * [dockerImage] optional. default is marklogicdb/marklogic-db:latest. Example: dockerImage=marklogic-centos/marklogic-server-centos:10-internal
+## * [prevDockerImage] optional. used for marklogic upgrade tests 
 ## * [saveOutput] optional. Save the output to a xml file. Example: saveOutput=true
 .PHONY: e2e-test
  e2e-test: prepare
@@ -92,6 +93,9 @@ lint:
 	@echo "=====Loading marklogc image $(dockerImage) to minikube cluster"
 	minikube image load $(dockerImage)
 
+	@echo "=====Loading marklogc image $(prevDockerImage) to minikube cluster"
+	minikube image load $(prevDockerImage)
+
 	@echo "=====Running e2e tests"
 	$(if $(saveOutput),gotestsum --junitfile test/test_results/e2e-tests.xml ./test/e2e/... -count=1 -timeout 30m, go test -v -count=1 -timeout 30m ./test/e2e/...) 
 

test/e2e/upgrade_test.go

@@ -3,6 +3,7 @@ package e2e
 import (
 	"crypto/tls"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -14,6 +15,8 @@ import (
 	"github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/gruntwork-io/terratest/modules/random"
 	"github.com/stretchr/testify/assert"
+	"github.com/tidwall/gjson"
+	digestAuth "github.com/xinsnake/go-http-digest-auth-client"
 )
 
 func TestHelmUpgrade(t *testing.T) {
@@ -26,12 +29,12 @@ func TestHelmUpgrade(t *testing.T) {
 	imageTag, tagPres := os.LookupEnv("dockerVersion")
 
 	if !repoPres {
-		imageRepo = "marklogic-centos/marklogic-server-centos"
+		imageRepo = "ml-docker-dev.marklogic.com/marklogic/marklogic-server-centos"
 		t.Logf("No imageRepo variable present, setting to default value: " + imageRepo)
 	}
 
 	if !tagPres {
-		imageTag = "10-internal"
+		imageTag = "11.0.20230307-centos-1.0.2"
 		t.Logf("No imageTag variable present, setting to default value: " + imageTag)
 	}
 
@@ -106,3 +109,104 @@ func TestHelmUpgrade(t *testing.T) {
 	passwordAfterUpgrade := string(passwordArr[:])
 	assert.Equal(t, passwordAfterUpgrade, passwordAfterInstall)
 }
+
+func TestMLupgrade(t *testing.T) {
+	// Path to the helm chart we will test
+	helmChartPath, e := filepath.Abs("../../charts")
+	if e != nil {
+		t.Fatalf(e.Error())
+	}
+	imageRepo, repoPres := os.LookupEnv("dockerRepository")
+	imageTag, tagPres := os.LookupEnv("dockerVersion")
+	prevImageTag, prevTagPres := os.LookupEnv("dockerVersion")
+
+	if !repoPres {
+		imageRepo = "ml-docker-dev.marklogic.com/marklogic/marklogic-server-centos"
+		t.Logf("No imageRepo variable present, setting to default value: " + imageRepo)
+	}
+	if !tagPres {
+		imageTag = "11.0.20230307-centos-1.0.2"
+		t.Logf("No imageTag variable present, setting to default value: " + imageTag)
+	}
+	if !prevTagPres {
+		prevImageTag = "10.0-20230307-centos-1.0.2"
+		t.Logf("No imageTag variable present, setting to default value: " + prevImageTag)
+	}
+
+	username := "admin"
+	password := "admin"
+
+	namespaceName := "marklogic-" + strings.ToLower(random.UniqueId())
+	kubectlOptions := k8s.NewKubectlOptions("", "", namespaceName)
+	options := &helm.Options{
+		KubectlOptions: kubectlOptions,
+		SetValues: map[string]string{
+			"persistence.enabled": "false",
+			"replicaCount":        "1",
+			"updateStrategy.type": "OnDelete",
+			"image.repository":    imageRepo,
+			"image.tag":           prevImageTag,
+			"auth.adminUsername":  username,
+			"auth.adminPassword":  password,
+		},
+	}
+
+	t.Logf("====Creating namespace: " + namespaceName)
+	k8s.CreateNamespace(t, kubectlOptions, namespaceName)
+	defer t.Logf("====Deleting namespace: " + namespaceName)
+	defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
+
+	t.Logf("====Installing Helm Chart")
+	releaseName := "test-ml-upgrade"
+	helm.Install(t, options, helmChartPath, releaseName)
+
+	podName := releaseName + "-marklogic-0"
+
+	// wait until second pod is in Ready status
+	k8s.WaitUntilPodAvailable(t, kubectlOptions, podName, 20, 20*time.Second)
+
+	k8s.RunKubectl(t, kubectlOptions, "describe", "pod", podName)
+
+	newOptions := &helm.Options{
+		KubectlOptions: kubectlOptions,
+		SetValues: map[string]string{
+			"persistence.enabled":   "false",
+			"image.repository":      imageRepo,
+			"image.tag":             imageTag,
+			"logCollection.enabled": "false",
+		},
+	}
+
+	t.Logf("====Upgrading Helm Chart")
+	helm.Upgrade(t, newOptions, helmChartPath, releaseName)
+
+	// delete pods to allow upgrades
+	k8s.RunKubectl(t, kubectlOptions, "delete", "pod", podName)
+
+	// wait until pod is in Ready status with new configuration
+	k8s.WaitUntilPodAvailable(t, kubectlOptions, podName, 15, 30*time.Second)
+
+	tunnel := k8s.NewTunnel(
+		kubectlOptions, k8s.ResourceTypePod, podName, 8002, 8002)
+	defer tunnel.Close()
+	tunnel.ForwardPort(t)
+
+	clusterEndpoint := fmt.Sprintf("http://%s/manage/v2?format=json", tunnel.Endpoint())
+	t.Logf(`Endpoint: %s`, clusterEndpoint)
+
+	getMLversion := digestAuth.NewRequest(username, password, "GET", clusterEndpoint, "")
+
+	resp, err := getMLversion.Execute()
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+	mlVersion := gjson.Get(string(body), `local-cluster-default.version`)
+	expectedMlVersion := strings.Split(imageTag, "-centos")[0]
+	// verify latest MarkLogic version after upgrade
+	assert.Equal(t, mlVersion.Str, expectedMlVersion)
+}

test/template/upgrade_templ_test.go

@@ -0,0 +1,53 @@
+package template_test
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	appsv1 "k8s.io/api/apps/v1"
+
+	"github.com/gruntwork-io/terratest/modules/helm"
+	"github.com/gruntwork-io/terratest/modules/k8s"
+	"github.com/gruntwork-io/terratest/modules/random"
+)
+
+func TestChartTemplateUpgradeStrategy(t *testing.T) {
+	t.Parallel()
+
+	// Path to the helm chart we will test
+	helmChartPath, err := filepath.Abs("../../charts")
+	releaseName := "marklogic-upgrade-test"
+	t.Log(helmChartPath, releaseName)
+	require.NoError(t, err)
+
+	// Set up the namespace; confirm that the template renders the expected value for the namespace.
+	namespaceName := "marklogic-" + strings.ToLower(random.UniqueId())
+	t.Logf("Namespace: %s\n", namespaceName)
+
+	// Setup the args for helm install
+	options := &helm.Options{
+		SetValues: map[string]string{
+			"image.repository":    "marklogicdb/marklogic-db",
+			"image.tag":           "latest",
+			"persistence.enabled": "false",
+			"updateStrategy.type": "RollingUpdate",
+		},
+		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
+	}
+
+	// render the tempate
+	output := helm.RenderTemplate(t, options, helmChartPath, releaseName, []string{"templates/statefulset.yaml"})
+
+	var statefulset appsv1.StatefulSet
+	helm.UnmarshalK8SYaml(t, output, &statefulset)
+
+	// Verify the name and namespace matches
+	require.Equal(t, namespaceName, statefulset.Namespace)
+
+	// Verify the updateStrategy type set for upgrade
+	expectedUpgradeStrategy := "RollingUpdate"
+	actualUpgradeStrategy := statefulset.Spec.UpdateStrategy.Type
+	require.Equal(t, string(actualUpgradeStrategy), expectedUpgradeStrategy)
+}