CLD-678 Automate HC tests in Kubernetes pipeline (#138)

* update email list
* Add HC e2e tests to the pipeline
* disable HC tests by default
* add cleanup
* fix minikube cleanup
* test specific tls version
* java fix
* update dependencies for Jenkins
* update log redirect
* enable full e2e suite
* change step sequence
* add comment about test duration

* - Add a new stage and option to execute HC e2e tests
 - Add a helper shell script to handle the execution and background tasks
 - Update notification email list
 - CLD-789 fix minikube cleanup step

* Add HC tests to nightly runs
* remove forgotten 'date'

* Add Romain to the mailing list

Author: vitalykorolev

Jenkinsfile

@@ -5,7 +5,7 @@
 @Library('[email protected]')
 import groovy.json.JsonSlurperClassic
 
-emailList = '[email protected], irosenba@marklogic.com, [email protected], [email protected], [email protected], [email protected]'
+emailList = 'vitaly.korolev@marklogic.com, [email protected], [email protected], [email protected], barkha.choithani@marklogic.com, romain.winieski@marklogic.com'
 gitCredID = '550650ab-ee92-4d31-a3f4-91a11d5388a3'
 JIRA_ID = ''
 JIRA_ID_PATTERN = /CLD-\d{3,4}/
@@ -164,7 +164,7 @@ pipeline {
         skipStagesAfterUnstable()
     }
     triggers {
-        parameterizedCron( env.BRANCH_NAME == 'develop' ? '''00 04 * * *''' : '')
+        parameterizedCron( env.BRANCH_NAME == 'develop' ? '''00 04 * * *%HC_TESTS=true''' : '')
     }
     environment {
         timeStamp = sh(returnStdout: true, script: "date +%Y%m%d -d '-5 hours'").trim()
@@ -180,6 +180,7 @@ pipeline {
         string(name: 'emailList', defaultValue: emailList, description: 'List of email for build notification', trim: true)
         choice(name: 'ML_VERSION', choices: '11.1\n12.0\n10.0\n9.0', description: 'MarkLogic version. used to pick appropriate docker image')
         booleanParam(name: 'KUBERNETES_TESTS', defaultValue: true, description: 'Run kubernetes tests')
+        booleanParam(name: 'HC_TESTS', defaultValue: false, description: 'Run Hub Central E2E UI tests (takes about 3 hours)')
         string(name: 'dockerReleaseVer', defaultValue: '1.0.2', description: 'Current Docker version. (e.g. 1.0.1)', trim: true)
         choice(name: 'PREV_ML_VERSION', choices: '10.0\n9.0', description: 'Previous MarkLogic version for MarkLogic upgrade tests')
         string(name: 'prevDockerReleaseVer', defaultValue: '1.0.2', description: 'Previous Docker version for MarkLogic upgrade tests. (e.g. 1.0.1)', trim: true)
@@ -211,11 +212,22 @@ pipeline {
             }
             steps {
                 sh """
-                    export MINIKUBE_HOME=/space;
+                    export MINIKUBE_HOME=/space
                     make test dockerImage=${dockerRepository}:${dockerVersion} prevDockerImage=${dockerRepository}:${prevDockerVersion} kubernetesVersion=${params.K8_VERSION} saveOutput=true
                 """
             }
         }
+        stage('Kubernetes-Run-HC-Tests') {
+            when {
+                expression { return params.HC_TESTS }
+            }
+            steps {
+                sh """
+                    export MINIKUBE_HOME=/space;
+                    make hc-test dockerImage=${dockerRepository}:${dockerVersion} kubernetesVersion=${params.K8_VERSION}
+                """
+            }
+        }
     }
 
     post {
@@ -224,7 +236,7 @@ pipeline {
                 docker system prune --force --filter "until=720h"
                 docker volume prune --force
                 docker image prune --force --all
-                minikube delete --all --purge
+                export MINIKUBE_HOME=/space; minikube delete --all --purge
             '''
             publishTestResults()
         }

makefile

@@ -1,5 +1,5 @@
-dockerImage?=ml-docker-dev.marklogic.com/marklogic/marklogic-server-centos:11.1.20230329-centos-1.0.2
-prevDockerImage?=ml-docker-dev.marklogic.com/marklogic/marklogic-server-centos:10.0-20230307-centos-1.0.2
+dockerImage?=ml-docker-dev.marklogic.com/marklogic/marklogic-server-centos:11.1.20230522-centos-1.0.2
+prevDockerImage?=ml-docker-dev.marklogic.com/marklogic/marklogic-server-centos:10.0-20230522-centos-1.0.2
 kubernetesVersion?=v1.25.8
 ## System requirement:
 ## - Go 
@@ -85,7 +85,7 @@ lint:
 ## * [kubernetesVersion] optional. Default is v1.25.8. Used for testing kubernetes version compatibility
 ## * [saveOutput] optional. Save the output to a xml file. Example: saveOutput=true
 .PHONY: e2e-test
- e2e-test: prepare
+e2e-test: prepare
 	@echo "=====Delete if there are existing minikube cluster"
 	minikube delete
 
@@ -104,6 +104,45 @@ lint:
 	@echo "=====Delete minikube cluster"
 	minikube delete
 
+#***************************************************************************
+# hc-test
+#***************************************************************************
+## Run all HC tests
+## Options:
+## * [some option] 
+.PHONY: hc-test
+hc-test: 
+ 	
+	@echo "=====Delete if there are existing minikube cluster"
+	minikube delete
+
+	@echo "=====Installing minikube cluster"
+	minikube start --driver=docker --kubernetes-version=$(kubernetesVersion) -n=1 --cpus 2 --memory 10000
+
+	@echo "=====Loading marklogc image $(dockerImage) to minikube cluster"
+	minikube image load $(dockerImage)
+
+	@echo "=====Deploy helm with a single MarkLogic node"
+	helm install hc charts --set auth.adminUsername=admin --set auth.adminPassword=admin --set persistence.enabled=false --wait
+	kubectl wait -l statefulset.kubernetes.io/pod-name=hc-marklogic-0 --for=condition=ready pod --timeout=30m
+
+	@echo "=====Clone Data Hub repository"
+	rm -rf marklogic-data-hub; git clone https://github.com/marklogic/marklogic-data-hub
+
+	@echo "=====Run HC tests with a shell script (~3 hours)"
+	./test/hc_e2e.sh
+
+	@echo "=====Finalize test report"
+	mkdir -p ./test/test_results
+	cp ./marklogic-data-hub/marklogic-data-hub-central/ui/e2e/results/* ./test/test_results/
+	rm -rf marklogic-data-hub
+
+	@echo "=====Uninstall helm"
+	helm uninstall hc
+
+	@echo "=====Delete minikube cluster"
+	minikube delete
+
 #***************************************************************************
 # template-test
 #***************************************************************************

test/hc_e2e.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+#This helper script will run E2E UI tests for Hub Central (https://github.com/marklogic/marklogic-data-hub/tree/develop/marklogic-data-hub-central/ui/e2e)
+
+#override java and node version for Jenkins
+[[ $USER = 'builder' ]] && { export PATH=/home/builder/java/jdk1.8.0_201/bin:/home/builder/nodeJs/node-v16.19.1-linux-x64/bin/:$PATH; unset JAVA_HOME; }
+
+echo "---- start port forwarding ----"
+kubectl port-forward hc-marklogic-0 8000 8001 8002 8010 8011 8013 &> /dev/null &
+forwarderPID=$!
+
+echo "---- configure environment ----"
+cd marklogic-data-hub/
+./gradlew clean build -x test
+./gradlew publishToMavenLocal -PskipWeb=true
+cd marklogic-data-hub-central/ui/e2e
+./setup.sh dhs=false mlHost=localhost mlSecurityUsername=admin mlSecurityPassword=admin
+
+echo "---- start Hub Central application ----"
+cd ../../..
+./gradlew bootRun -PhubUseLocalDefaults=true &
+bootRunPID=$!
+
+sleep 10
+
+echo "---- start UI sanity tests on HC ----"
+cd marklogic-data-hub-central/ui/e2e
+npm run cy:run --reporter junit --reporter-options "toConsole=false"
+
+echo "---- cleanup background processes ----"
+kill $bootRunPID $forwarderPID