DEVO-860 new jenkins (#169)

* expand jira id matching
* update git credentials
* fix jira id matching
* update docker repo
* switch to nightly docker images
* Switch to new internal docker repo
* Update docker version handling
* disable minikube emoji
* update resource requirements
* update minikube config scope
* test use of dedicated kubeconfig

Author: vitalykorolev

Jenkinsfile

@@ -6,9 +6,9 @@
 import groovy.json.JsonSlurperClassic
 
 emailList = '[email protected], [email protected], [email protected], [email protected], [email protected], [email protected]'
-gitCredID = '550650ab-ee92-4d31-a3f4-91a11d5388a3'
+gitCredID = 'marklogic-builder'
 JIRA_ID = ''
-JIRA_ID_PATTERN = /CLD-\d{3,4}/
+JIRA_ID_PATTERN = /(?i)(CLD|DEVO|QAINF|BUG|DBI)-\d{3,4}/
 LINT_OUTPUT = ''
 SCAN_OUTPUT = ''
 IMAGE_INFO = 0
@@ -31,7 +31,7 @@ void preBuildCheck() {
     if (env.CHANGE_ID) {
         if (prDraftCheck()) { sh 'exit 1' }
         if (getReviewState().equalsIgnoreCase('CHANGES_REQUESTED')) {
-            println(reviewState)
+            echo 'PR changes requested. (' + reviewState + ') Aborting.'
             sh 'exit 1'
         }
     }
@@ -51,11 +51,11 @@ def extractJiraID() {
         match = env.GIT_BRANCH =~ JIRA_ID_PATTERN
     }
     else {
-        echo 'Warning: Jira ticket number not detected.'
+        echo 'Warning: No Git title or branch available.'
         return ''
     }
     try {
-        return match[0]
+        return match[0][0]
     } catch (any) {
         echo 'Warning: Jira ticket number not detected.'
         return ''
@@ -131,7 +131,7 @@ void publishTestResults() {
 }
 
 void pullImage() {
-    withCredentials([usernamePassword(credentialsId: '8c2e0b38-9e97-4953-aa60-f2851bb70cc8', passwordVariable: 'docker_password', usernameVariable: 'docker_user')]) {
+    withCredentials([usernamePassword(credentialsId: 'builder-credentials-artifactory', passwordVariable: 'docker_password', usernameVariable: 'docker_user')]) {
         sh """
             echo "\$docker_password" | docker login --username \$docker_user --password-stdin ${dockerRegistry}
             docker pull ${dockerRepository}:${dockerVersion}
@@ -167,8 +167,9 @@ pipeline {
         parameterizedCron( env.BRANCH_NAME == 'develop' ? '''00 04 * * *''' : '')
     }
     environment {
-        timeStamp = sh(returnStdout: true, script: "date +%Y%m%d -d '-5 hours'").trim()
-        dockerRegistry = 'ml-docker-dev.marklogic.com'
+        //timeStamp = sh(returnStdout: true, script: "date +%Y%m%d -d '-5 hours'").trim()
+        timeStamp = 'nightly'
+        dockerRegistry = 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com'
         dockerRepository = "${dockerRegistry}/marklogic/marklogic-server-centos"
         dockerVerDivider = getVersionDiv(params.ML_VERSION)
         prevDockerVerDivider = getVersionDiv(params.PREV_ML_VERSION)
@@ -178,11 +179,11 @@ pipeline {
 
     parameters {
         string(name: 'emailList', defaultValue: emailList, description: 'List of email for build notification', trim: true)
-        choice(name: 'ML_VERSION', choices: '11.2\n12.0\n10.0\n9.0', description: 'MarkLogic version. used to pick appropriate docker image')
+        choice(name: 'ML_VERSION', choices: '11.2\n12.0\n10.0', description: 'MarkLogic version. used to pick appropriate docker image')
         booleanParam(name: 'KUBERNETES_TESTS', defaultValue: true, description: 'Run kubernetes tests')
         booleanParam(name: 'HC_TESTS', defaultValue: false, description: 'Run Hub Central E2E UI tests (takes about 3 hours)')
         string(name: 'dockerReleaseVer', defaultValue: '1.1.1', description: 'Current Docker version. (e.g. 1.0.1)', trim: true)
-        choice(name: 'PREV_ML_VERSION', choices: '10.0\n9.0', description: 'Previous MarkLogic version for MarkLogic upgrade tests')
+        choice(name: 'PREV_ML_VERSION', choices: '10.0\n9.0\n11.2', description: 'Previous MarkLogic version for MarkLogic upgrade tests')
         string(name: 'prevDockerReleaseVer', defaultValue: '1.1.1', description: 'Previous Docker version for MarkLogic upgrade tests. (e.g. 1.0.1)', trim: true)
         choice(name: 'K8_VERSION', choices: 'v1.25.8\nv1.26.3\nv1.24.12\nv1.23.17', description: 'Test Kubernetes version. (e.g. v1.25.8)')
     }
@@ -212,8 +213,7 @@ pipeline {
             }
             steps {
                 sh """
-                    export MINIKUBE_HOME=/space
-                    make test dockerImage=${dockerRepository}:${dockerVersion} prevDockerImage=${dockerRepository}:${prevDockerVersion} kubernetesVersion=${params.K8_VERSION} saveOutput=true
+                    export MINIKUBE_HOME=/space; export KUBECONFIG=/space/.kube-config; make test dockerImage=${dockerRepository}:${dockerVersion} prevDockerImage=${dockerRepository}:${prevDockerVersion} kubernetesVersion=${params.K8_VERSION} saveOutput=true
                 """
             }
         }
@@ -223,8 +223,7 @@ pipeline {
             }
             steps {
                 sh """
-                    export MINIKUBE_HOME=/space;
-                    make hc-test dockerImage=${dockerRepository}:${dockerVersion} kubernetesVersion=${params.K8_VERSION}
+                    export MINIKUBE_HOME=/space; export KUBECONFIG=/space/.kube-config; make hc-test dockerImage=${dockerRepository}:${dockerVersion} kubernetesVersion=${params.K8_VERSION}
                 """
             }
         }
@@ -237,7 +236,7 @@ pipeline {
                 docker system prune --force --filter "until=720h"
                 docker volume prune --force
                 docker image prune --force --all
-                export MINIKUBE_HOME=/space; minikube delete --all --purge
+                export MINIKUBE_HOME=/space; export KUBECONFIG=/space/.kube-config; minikube delete --all --purge
             '''
             sh "rm -rf $WORKSPACE/test/test_results/"
         }

makefile

@@ -1,5 +1,5 @@
-dockerImage?=ml-docker-dev.marklogic.com/marklogic/marklogic-server-centos:11.1.20230522-centos-1.0.2
-prevDockerImage?=ml-docker-dev.marklogic.com/marklogic/marklogic-server-centos:10.0-20230522-centos-1.0.2
+dockerImage?=ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-centos:11.1.20230522-centos-1.0.2
+prevDockerImage?=ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-centos:10.0-20230522-centos-1.0.2
 kubernetesVersion?=v1.25.8
 ## System requirement:
 ## - Go 
@@ -90,7 +90,8 @@ e2e-test: prepare
 	minikube delete
 
 	@echo "=====Installing minikube cluster"
-	minikube start --driver=docker --kubernetes-version=$(kubernetesVersion) -n=1 --cpus 2 --memory 10000
+	minikube start --driver=docker --kubernetes-version=$(kubernetesVersion) -n=1 --memory=16000mb --cpus=2
+
 
 	@echo "=====Loading marklogc image $(dockerImage) to minikube cluster"
 	minikube image load $(dockerImage)
@@ -115,7 +116,8 @@ hc-test:
 	minikube delete
 
 	@echo "=====Installing minikube cluster"
-	minikube start --driver=docker --kubernetes-version=$(kubernetesVersion) -n=1 --cpus 2 --memory 10000
+	minikube start --driver=docker --kubernetes-version=$(kubernetesVersion) -n=1 --memory=16000mb --cpus=2
+
 
 	@echo "=====Loading marklogc image $(dockerImage) to minikube cluster"
 	minikube image load $(dockerImage)

test/e2e/admin_secrets_test.go

@@ -22,12 +22,12 @@ func TestMlAdminSecrets(t *testing.T) {
 	imageTag, tagPres := os.LookupEnv("dockerVersion")
 
 	if !repoPres {
-		imageRepo = "ml-docker-dev.marklogic.com/marklogic/marklogic-server-centos"
+		imageRepo = "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-centos"
 		t.Logf("No imageRepo variable present, setting to default value: " + imageRepo)
 	}
 
 	if !tagPres {
-		imageTag = "11.0.20230307-centos-1.0.2"
+		imageTag = "11.0.nightly-centos-1.0.2"
 		t.Logf("No imageTag variable present, setting to default value: " + imageTag)
 	}
 

test/e2e/upgrade_test.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"testing"
 	"time"
@@ -29,12 +30,12 @@ func TestHelmUpgrade(t *testing.T) {
 	imageTag, tagPres := os.LookupEnv("dockerVersion")
 
 	if !repoPres {
-		imageRepo = "ml-docker-dev.marklogic.com/marklogic/marklogic-server-centos"
+		imageRepo = "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-centos"
 		t.Logf("No imageRepo variable present, setting to default value: " + imageRepo)
 	}
 
 	if !tagPres {
-		imageTag = "11.0.20230307-centos-1.0.2"
+		imageTag = "11.0.nightly-centos-1.0.2"
 		t.Logf("No imageTag variable present, setting to default value: " + imageTag)
 	}
 
@@ -121,15 +122,15 @@ func TestMLupgrade(t *testing.T) {
 	prevImageTag, prevTagPres := os.LookupEnv("dockerVersion")
 
 	if !repoPres {
-		imageRepo = "ml-docker-dev.marklogic.com/marklogic/marklogic-server-centos"
+		imageRepo = "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-centos"
 		t.Logf("No imageRepo variable present, setting to default value: " + imageRepo)
 	}
 	if !tagPres {
-		imageTag = "11.0.20230307-centos-1.0.2"
+		imageTag = "11.0.nightly-centos-1.0.2"
 		t.Logf("No imageTag variable present, setting to default value: " + imageTag)
 	}
 	if !prevTagPres {
-		prevImageTag = "10.0-20230307-centos-1.0.2"
+		prevImageTag = "10.0-nightly-centos-1.0.2"
 		t.Logf("No imageTag variable present, setting to default value: " + prevImageTag)
 	}
 
@@ -205,8 +206,11 @@ func TestMLupgrade(t *testing.T) {
 	if err != nil {
 		t.Fatalf(err.Error())
 	}
-	mlVersion := gjson.Get(string(body), `local-cluster-default.version`)
-	expectedMlVersion := strings.Split(imageTag, "-centos")[0]
+	mlVersionPattern := regexp.MustCompile(`(\d+\.\d+)`)
+	mlVersionResp := gjson.Get(string(body), `local-cluster-default.version`)
+	actualMlVersion := mlVersionPattern.FindStringSubmatch(mlVersionResp.Str)
+	expectedMlVersion := mlVersionPattern.FindStringSubmatch(imageTag)
+	//expectedMlVersion := strings.Split(imageTag, "-centos")[0]
 	// verify latest MarkLogic version after upgrade
-	assert.Equal(t, mlVersion.Str, expectedMlVersion)
+	assert.Equal(t, actualMlVersion, expectedMlVersion)
 }