MLE-4173: Kubernetes Huge Pages test automation (#167)

* e2e tests for huge pages implementation

Author: barkhachoithani

Jenkinsfile

@@ -219,6 +219,7 @@ pipeline {
         always {
             publishTestResults()
             sh '''
+	            sudo sysctl -w vm.nr_hugepages=0
                 export MINIKUBE_HOME=/space; export KUBECONFIG=/space/.kube-config; export GOPATH=/space/go; minikube delete --all --purge
                 docker system prune --force --filter "until=720h"
                 docker volume prune --force

makefile

@@ -100,8 +100,24 @@ e2e-test: prepare
 	@echo "=====Loading marklogc image $(prevDockerImage) to minikube cluster"
 	minikube image load $(prevDockerImage)
 
+	@echo "=====Setting hugepages values to 0 for e2e tests"
+	sudo sysctl -w vm.nr_hugepages=0
+
 	@echo "=====Running e2e tests"
-	$(if $(saveOutput),gotestsum --junitfile test/test_results/e2e-tests.xml ./test/e2e/... -count=1 -timeout 70m, go test -v -count=1 -timeout 70m ./test/e2e/...) 
+	$(if $(saveOutput),gotestsum --junitfile test/test_results/e2e-tests.xml ./test/e2e/... -count=1 -timeout 70m, go test -v -count=1 -timeout 70m ./test/e2e/...)
+
+	@echo "=====Setting hugepages value to 1280 for hugepages-e2e test"
+	sudo sysctl -w vm.nr_hugepages=1280
+
+	@echo "=====Restart minikube cluster"
+	minikube stop
+	minikube start
+
+	@echo "=====Running hugepages e2e test"
+	$(if $(saveOutput),gotestsum --junitfile test/test_results/hugePages-tests.xml ./test/hugePages/... -count=1 -timeout 70m, go test -v -count=1 -timeout 70m ./test/hugePages/...)
+
+	@echo "=====Resetting hugepages value to 0"
+	sudo sysctl -w vm.nr_hugepages=0
 
 	@echo "=====Delete minikube cluster"
 	minikube delete

test/hugePages/huge_pages_test.go

@@ -0,0 +1,111 @@
+package hugePages
+
+import (
+	"crypto/tls"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	http_helper "github.com/gruntwork-io/terratest/modules/http-helper"
+	"github.com/gruntwork-io/terratest/modules/k8s"
+	"github.com/gruntwork-io/terratest/modules/random"
+	"github.com/marklogic/marklogic-kubernetes/test/testUtil"
+	"github.com/stretchr/testify/assert"
+	digestAuth "github.com/xinsnake/go-http-digest-auth-client"
+)
+
+func TestHugePagesSettings(t *testing.T) {
+	// var resp *http.Response
+	var body []byte
+	var err error
+	var podName string
+	imageRepo, repoPres := os.LookupEnv("dockerRepository")
+	imageTag, tagPres := os.LookupEnv("dockerVersion")
+
+	if !repoPres {
+		imageRepo = "marklogicdb/marklogic-db"
+		t.Logf("No imageRepo variable present, setting to default value: " + imageRepo)
+	}
+
+	if !tagPres {
+		imageTag = "latest"
+		t.Logf("No imageTag variable present, setting to default value: " + imageTag)
+	}
+
+	username := "admin"
+	password := "admin"
+
+	options := map[string]string{
+		"persistence.enabled":            "false",
+		"replicaCount":                   "1",
+		"image.repository":               imageRepo,
+		"image.tag":                      imageTag,
+		"auth.adminUsername":             username,
+		"auth.adminPassword":             password,
+		"logCollection.enabled":          "false",
+		"hugepages.enabled":              "true",
+		"hugepages.mountPath":            "/dev/hugepages",
+		"resources.limits.hugepages-2Mi": "1Gi",
+		"resources.limits.memory":        "8Gi",
+		"resources.requests.memory":      "8Gi",
+	}
+	t.Logf("====Installing Helm Chart")
+	releaseName := "hugepages"
+
+	namespaceName := "ml-" + strings.ToLower(random.UniqueId())
+	kubectlOptions := k8s.NewKubectlOptions("", "", namespaceName)
+
+	t.Logf("====Creating namespace: " + namespaceName)
+	k8s.CreateNamespace(t, kubectlOptions, namespaceName)
+
+	defer t.Logf("====Deleting namespace: " + namespaceName)
+	defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
+
+	podName = testUtil.HelmInstall(t, options, releaseName, kubectlOptions)
+
+	t.Logf("====Describe pod for verifying huge pages")
+	k8s.RunKubectl(t, kubectlOptions, "describe", "pod", podName)
+
+	tlsConfig := tls.Config{}
+	// wait until the pod is in Ready status
+	k8s.WaitUntilPodAvailable(t, kubectlOptions, podName, 10, 15*time.Second)
+
+	tunnel7997 := k8s.NewTunnel(kubectlOptions, k8s.ResourceTypePod, podName, 7997, 7997)
+	defer tunnel7997.Close()
+	tunnel7997.ForwardPort(t)
+	endpoint7997 := fmt.Sprintf("http://%s", tunnel7997.Endpoint())
+
+	// verify if 7997 health check endpoint returns 200
+	http_helper.HttpGetWithRetryWithCustomValidation(
+		t,
+		endpoint7997,
+		&tlsConfig,
+		10,
+		15*time.Second,
+		func(statusCode int, body string) bool {
+			return statusCode == 200
+		},
+	)
+	tunnel8002 := k8s.NewTunnel(kubectlOptions, k8s.ResourceTypePod, podName, 8002, 8002)
+	defer tunnel8002.Close()
+	tunnel8002.ForwardPort(t)
+	endpointManage := fmt.Sprintf("http://%s/manage/v2/logs?format=text&filename=ErrorLog.txt", tunnel8002.Endpoint())
+	request := digestAuth.NewRequest(username, password, "GET", endpointManage, "")
+	response, err := request.Execute()
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+	defer response.Body.Close()
+	assert.Equal(t, 200, response.StatusCode)
+
+	body, err = io.ReadAll(response.Body)
+	t.Log(string(body))
+
+	// Verify if Huge pages are configured on the MarkLogic node
+	if !strings.Contains(string(body), "Linux Huge Pages: detected 1280") {
+		t.Errorf("Huge Pages not configured for the node")
+	}
+}

test/template/hugePg_template_test.go

@@ -0,0 +1,63 @@
+package template_test
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	appsv1 "k8s.io/api/apps/v1"
+
+	"github.com/gruntwork-io/terratest/modules/helm"
+	"github.com/gruntwork-io/terratest/modules/k8s"
+	"github.com/gruntwork-io/terratest/modules/random"
+)
+
+func TestChartTemplateHugePagesConfig(t *testing.T) {
+
+	// Path to the helm chart we will test
+	helmChartPath, err := filepath.Abs("../../charts")
+	releaseName := "hugepages"
+	t.Log(helmChartPath, releaseName)
+	require.NoError(t, err)
+
+	// Set up the namespace; confirm that the template renders the expected value for the namespace.
+	namespaceName := "ml-" + strings.ToLower(random.UniqueId())
+
+	// Setup the args for helm install
+	options := &helm.Options{
+		SetValues: map[string]string{
+			"image.repository":               "marklogicdb/marklogic-db",
+			"image.tag":                      "latest",
+			"persistence.enabled":            "true",
+			"logCollection.enabled":          "true",
+			"resources.limits.hugepages-2Mi": "1Gi",
+		},
+		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
+	}
+
+	// render the tempate
+	output := helm.RenderTemplate(t, options, helmChartPath, releaseName, []string{"templates/statefulset.yaml"})
+
+	var statefulset appsv1.StatefulSet
+	helm.UnmarshalK8SYaml(t, output, &statefulset)
+
+	// Verify the name and namespace matches
+	require.Equal(t, namespaceName, statefulset.Namespace)
+
+	expectedHugePages := "1Gi"
+
+	statefulSetContainers := statefulset.Spec.Template.Spec.Containers
+	resourceLimits := statefulSetContainers[0].Resources.Limits
+
+	var actualHugePages string
+	if value, exist := resourceLimits["hugepages-2Mi"]; exist {
+		t.Log("ActualHugePages: ", value.String())
+		actualHugePages = value.String()
+	} else {
+		t.Errorf("hugepages-2Mi not found")
+	}
+	// Verify the huge pages is configured
+	require.Equal(t, len(statefulSetContainers), 2)
+	require.Equal(t, actualHugePages, expectedHugePages)
+}