Merge pull request #1066 from pohly/coverage-jenkins

WIP: code coverage

Author: pohly

Jenkinsfile

@@ -68,6 +68,15 @@ pipeline {
         TEST_ETCD_TMPFS = "${WORKSPACE}/_work/${env.CLUSTER}/data/pmem-csi-${env.CLUSTER}-master/etcd-tmpfs"
         TEST_ETCD_VOLUME = "${env.TEST_ETCD_TMPFS}/etcd-volume"
         TEST_ETCD_VOLUME_SIZE = "1073741824" // 1GB
+
+        // Tests that will get skipped when collecting coverage information.
+        //
+        // The operator itself installs without enabling coverage collection,
+        // so running those tests doesn't help us. The relevant test is
+        // "operator API".
+        //
+        // Testing with OLM doesn't add much additional coverage.
+        COVERAGE_SKIP = "[email protected]@Top.Level..olm"
     }
 
     stages {
@@ -183,7 +192,21 @@ pipeline {
                 stage('1.22') {
                     steps {
                         // Skip production, i.e. run testing.
-                        TestInVM("", "fedora", "", "1.22", "Top.Level..[[:alpha:]]*-production[[:space:]]")
+                        TestInVM("", "", "fedora", "", "1.22", "Top.Level..[[:alpha:]]*-production[[:space:]]", "")
+                    }
+                }
+
+                // When adding or removing coverage workers, update the "Code Coverage" step below!
+                stage('coverage-1.22') {
+                    when {
+		        beforeAgent true
+		        not { changeRequest() }
+                    }
+                    agent {
+                        label "pmem-csi"
+                    }
+                    steps {
+                        TestInVM("fedora-coverage-1.22", "coverage-", "fedora", "", "1.22", "", "${env.COVERAGE_SKIP}")
                     }
                 }
 
@@ -197,7 +220,7 @@ pipeline {
                         label "pmem-csi"
                     }
                     steps {
-                        TestInVM("fedora-1.21", "fedora", "", "1.21", "")
+                        TestInVM("fedora-1.21", "", "fedora", "", "1.21", "", "")
                     }
                 }
                 stage('1.20') {
@@ -209,7 +232,7 @@ pipeline {
                         label "pmem-csi"
                     }
                     steps {
-                        TestInVM("fedora-1.20", "fedora", "", "1.20", "")
+                        TestInVM("fedora-1.20", "", "fedora", "", "1.20", "", "")
                     }
                 }
                 stage('1.19') {
@@ -218,7 +241,19 @@ pipeline {
                     }
                     steps {
                         // Skip testing, i.e. run production.
-                        TestInVM("fedora-1.19", "fedora", "", "1.19",  "Top.Level..[[:alpha:]]*-testing[[:space:]]")
+                        TestInVM("fedora-1.19", "", "fedora", "", "1.19",  "Top.Level..[[:alpha:]]*-testing[[:space:]]", "")
+                    }
+                }
+                stage('coverage-1.19') {
+                    when {
+		        beforeAgent true
+		        not { changeRequest() }
+                    }
+                    agent {
+                        label "pmem-csi"
+                    }
+                    steps {
+                        TestInVM("fedora-coverage-1.19", "coverage-", "fedora", "", "1.19", "", "${env.COVERAGE_SKIP}")
                     }
                 }
             }
@@ -291,6 +326,43 @@ git push origin HEAD:master
                 sh "${RunInBuilder()} ${env.BUILD_CONTAINER} docker image push ${env.BUILD_IMAGE}"
             }
         }
+
+        // Merge and publish coverage data.
+        stage('Code Coverage') {
+            when {
+                not { changeRequest() }
+            }
+            steps {
+                // Restore <cluster>-coverage.out files.
+                unstash '1.22-coverage'
+                unstash '1.19-coverage'
+
+                // Merge and convert to Cobertura XML.
+                sh "${RunInBuilder()} ${env.BUILD_CONTAINER} make _work/gocovmerge _work/gocover-cobertura"
+                sh "${RunInBuilder()} ${env.BUILD_CONTAINER} _work/gocovmerge *-coverage.out >coverage.out"
+                sh "${RunInBuilder()} ${env.BUILD_CONTAINER} go tool cover -func coverage.out"
+                sh "${RunInBuilder()} ${env.BUILD_CONTAINER} _work/gocover-cobertura <coverage.out >coverage.xml"
+
+                // Simplify relative paths ("github.com/intel/pmem-csi/...").
+                // To view source code (https://stackoverflow.com/a/59951809):
+                // - Jenkins users must be logged in.
+                // - A job must complete successfully.
+                sh "sed -i -e 's;filename=\"github.com/intel/pmem-csi/;filename=\";g' coverage.xml"
+
+                // The relationship between "Code Coverage API Plugin" and "Cobertura" plugin is not clear
+                // (https://stackoverflow.com/questions/71133394/what-is-the-relationship-between-the-jenkins-cobertura-and-code-coverage-api).
+                //
+                // With just "Code Coverage API Plugin" installed, this here works, but doesn't show source code
+                // (old UI?):
+                // publishCoverage adapters: [cobertura(path: 'coverage.xml')], tag: 't'
+
+                // When both are installed, this here works (note the different coberura parameter!)
+                // and shows source code.
+                publishCoverage adapters: [cobertura(coberturaReportFile: 'coverage.xml')]
+
+                // There is also a "coberturaAdapter". That one hasn't been tested.
+            }
+        }
     }
 }
 
@@ -313,6 +385,7 @@ git push origin HEAD:master
 String RunInBuilder() {
     "\
     docker exec \
+    -i \
     -e CACHEBUST=${env.CACHEBUST} \
     -e 'BUILD_ARGS=--cache-from ${env.BUILD_IMAGE} --cache-from ${env.PMEM_CSI_IMAGE}' \
     -e DOCKER_CONFIG=${WORKSPACE}/_work/docker-config \
@@ -436,10 +509,13 @@ void RestoreEnv() {
         done"
 }
 
-void TestInVM(worker, distro, distroVersion, kubernetesVersion, skipIfPR) {
+void TestInVM(worker, coverage, distro, distroVersion, kubernetesVersion, skipIfPR, skipAlways) {
     if (worker) {
         RestoreEnv()
     }
+    if (coverage) {
+        sh "${RunInBuilder()} -e CLUSTER=${env.CLUSTER} ${env.BUILD_CONTAINER} make kustomize KUSTOMIZE_WITH_COVERAGE=true"
+    }
     try { timeout(unit: "HOURS", time: TestTimeoutHours()) {
         /*
         We have to run "make start" in the current directory
@@ -463,7 +539,7 @@ void TestInVM(worker, distro, distroVersion, kubernetesVersion, skipIfPR) {
         so for now we disable VMX with -vmx.
         */
         sh "#!/bin/bash\n \
-           echo Note: job output is filtered, see joblog-${BUILD_TAG}-test-${kubernetesVersion}.log artifact for full output. && \
+           echo Note: job output is filtered, see joblog-${BUILD_TAG}-test-${coverage}${kubernetesVersion}.log artifact for full output. && \
            set -o pipefail && \
            ( \
            loggers=; \
@@ -508,10 +584,10 @@ void TestInVM(worker, distro, distroVersion, kubernetesVersion, skipIfPR) {
                                  done | sed -e \"s/^/\$hostname: /\" ) & \
                                loggers=\"\$loggers \$!\"; \
                            done && \
-                           testrun=\$(echo '${distro}-${distroVersion}-${kubernetesVersion}' | sed -e s/--*/-/g | tr . _ ) && \
+                           testrun=\$(echo '${distro}-${distroVersion}-${coverage}${kubernetesVersion}' | sed -e s/--*/-/g | tr . _ ) && \
                            make test_e2e TEST_E2E_REPORT_DIR=${WORKSPACE}/build/reports.tmp/\$testrun \
-                                         TEST_E2E_SKIP=\$(if [ \"${env.CHANGE_ID}\" ] && [ \"${env.CHANGE_ID}\" != null ]; then echo \\\\[Slow\\\\]@${skipIfPR}; fi) \
-                           ') 2>&1 | tee joblog-${BUILD_TAG}-test-${kubernetesVersion}.log | grep --line-buffered -E -e 'checking for test|Passed|FAIL:|^ERROR' \
+                                         TEST_E2E_SKIP=${skipAlways}@\$(if [ \"${env.CHANGE_ID}\" ] && [ \"${env.CHANGE_ID}\" != null ]; then echo \\\\[Slow\\\\]@${skipIfPR}; fi) \
+                           ') 2>&1 | tee joblog-${BUILD_TAG}-test-${coverage}${kubernetesVersion}.log | grep --line-buffered -E -e 'checking for test|Passed|FAIL:|^ERROR' \
            "
     } } finally {
         echo "Writing cluster state and kubelet logs into files."
@@ -545,6 +621,20 @@ void TestInVM(worker, distro, distroVersion, kubernetesVersion, skipIfPR) {
            done'''
         archiveArtifacts('**/joblog-*')
         junit 'build/reports/**/*.xml'
+
+        if (coverage) {
+            // https://stackoverflow.com/questions/36918370/cobertura-code-coverage-report-for-jenkins-pipeline-jobs
+            // https://www.jenkins.io/doc/pipeline/steps/cobertura/
+            sh "${RunInBuilder()} -e CLUSTER=${env.CLUSTER} ${env.BUILD_CONTAINER} make _work/coverage/coverage.txt"
+            sh "cat _work/coverage/coverage.txt"
+
+            // https://plugins.jenkins.io/code-coverage-api/#plugin-content-reports-combining-support
+            // claims that different reports can be merged, but that didn't work in practice
+            // (two "coverage reports" listed in the job UI with the same URL and unmerged data from
+            // one worker). Therefore we stash the individual results and merge later.
+            sh "mv _work/coverage/coverage.out ${kubernetesVersion}-coverage.out"
+            stash includes: "${kubernetesVersion}-coverage.out", name: "${kubernetesVersion}-coverage"
+        }
     }
 }
 

Makefile

@@ -159,14 +159,23 @@ KUSTOMIZE_INPUT := $(shell [ ! -d deploy/kustomize ] || find deploy/kustomize -t
 # Output files and their corresponding kustomization, in the format <target .yaml>=<kustomization directory>
 KUSTOMIZE :=
 
+# Setting this to any non-empty value before "make kustomize" will enable
+# collection of coverage profiles in all deployments. This also sets up the
+# original deployment files under "deploy/nocoverage" for use as reference when
+# testing the operator.
+KUSTOMIZE_WITH_COVERAGE =
+ifneq "$(KUSTOMIZE_WITH_COVERAGE)" ""
+KUSTOMIZE_COVERAGE_SUFFIX = -coverage
+endif
+
 # For each supported Kubernetes version, we provide four different flavors.
 # The "testing" flavor of the generated files contains both
 # the loglevel changes but does not enable coverage data collection.
 KUSTOMIZE_KUBERNETES_OUTPUT = \
-    deploy/kubernetes-X.XX/pmem-csi-direct.yaml=deploy/kustomize/kubernetes-base-direct \
-    deploy/kubernetes-X.XX/pmem-csi-lvm.yaml=deploy/kustomize/kubernetes-base-lvm \
-    deploy/kubernetes-X.XX/pmem-csi-direct-testing.yaml=deploy/kustomize/kubernetes-base-direct-testing \
-    deploy/kubernetes-X.XX/pmem-csi-lvm-testing.yaml=deploy/kustomize/kubernetes-base-lvm-testing \
+    deploy/kubernetes-X.XX/pmem-csi-direct.yaml=deploy/kustomize/kubernetes-base-direct$(KUSTOMIZE_COVERAGE_SUFFIX) \
+    deploy/kubernetes-X.XX/pmem-csi-lvm.yaml=deploy/kustomize/kubernetes-base-lvm$(KUSTOMIZE_COVERAGE_SUFFIX) \
+    deploy/kubernetes-X.XX/pmem-csi-direct-testing.yaml=deploy/kustomize/kubernetes-base-direct-testing$(KUSTOMIZE_COVERAGE_SUFFIX) \
+    deploy/kubernetes-X.XX/pmem-csi-lvm-testing.yaml=deploy/kustomize/kubernetes-base-lvm-testing$(KUSTOMIZE_COVERAGE_SUFFIX) \
 
 # Kubernetes versions derived from kubernetes-base.
 #
@@ -199,14 +208,22 @@ KUSTOMIZE_OUTPUT := $(foreach item,$(KUSTOMIZE),$(firstword $(subst =, ,$(item))
 # right one.
 KUSTOMIZE_LOOKUP_KUSTOMIZATION = $(strip $(foreach item,$(KUSTOMIZE),$(if $(filter $(1)=%,$(item)),$(word 2,$(subst =, ,$(item))))))
 
+# This is a wrapper around KUSTOMIZE_LOOKUP_KUSTOMIZATION which
+# removes the -coverage suffix.
+KUSTOMIZE_LOOKUP_KUSTOMIZATION_NO_COVERAGE = $(subst -coverage,,$(call KUSTOMIZE_LOOKUP_KUSTOMIZATION,$(1)))
+
 # This function takes the kustomize binary and the name of an output
 # file as arguments and returns the command which produces that file
 # as stdout.
-KUSTOMIZE_INVOCATION = (echo '\# Generated with "make kustomize", do not edit!'; echo; $(1) build --load-restrictor LoadRestrictionsNone $(call KUSTOMIZE_LOOKUP_KUSTOMIZATION,$(2)))
+KUSTOMIZE_INVOCATION = (echo '\# Generated with "make kustomize", do not edit!'; echo; $(1) build --load-restrictor LoadRestrictionsNone $(call $(3),$(2)))
 
 $(KUSTOMIZE_OUTPUT): _work/kustomize $(KUSTOMIZE_INPUT)
 	mkdir -p ${@D}
-	$(call KUSTOMIZE_INVOCATION,$<,$@) >$@
+	$(call KUSTOMIZE_INVOCATION,$<,$@,KUSTOMIZE_LOOKUP_KUSTOMIZATION) >$@
+ifneq "$(KUSTOMIZE_WITH_COVERAGE)" ""
+	mkdir -p $(subst deploy/,deploy/nocoverage/,${@D})
+	$(call KUSTOMIZE_INVOCATION,$<,$@,KUSTOMIZE_LOOKUP_KUSTOMIZATION_NO_COVERAGE) >$(subst deploy/,deploy/nocoverage/,$@)
+endif
 	if echo "$@" | grep '/pmem-csi-' | grep -qv '\-operator'; then \
 		dir=$$(echo "$@" | tr - / | sed -e 's;kubernetes/;kubernetes-;' -e 's;/alpha/;-alpha/;'  -e 's;/distributed/;-distributed/;' -e 's/.yaml//' -e 's;/pmem/csi/;/;') && \
 		mkdir -p $$dir && \
@@ -215,9 +232,13 @@ $(KUSTOMIZE_OUTPUT): _work/kustomize $(KUSTOMIZE_INPUT)
 	fi
 
 kustomize: clean_kustomize_output $(KUSTOMIZE_OUTPUT)
+ifneq "$(KUSTOMIZE_WITH_COVERAGE)" ""
+	sed -i -e 's/embed kubernetes-/embed nocoverage kubernetes-/' deploy/yamls.go
+endif
 
 clean_kustomize_output:
-	rm -rf deploy/kubernetes-*
+	rm -rf deploy/kubernetes-* deploy/nocoverage
+	sed -i -e 's/embed nocoverage /embed /' deploy/yamls.go
 	rm -f $(KUSTOMIZE_OUTPUT)
 
 # Always re-generate the output files because "git rebase" might have
@@ -234,7 +255,7 @@ clean-kustomize:
 test: test-kustomize
 test-kustomize: $(addprefix test-kustomize-,$(KUSTOMIZE_OUTPUT))
 $(addprefix test-kustomize-,$(KUSTOMIZE_OUTPUT)): test-kustomize-%: _work/kustomize
-	@ if ! diff <($(call KUSTOMIZE_INVOCATION,$<,$*)) $*; then echo "$* was modified manually" && false; fi
+	@ if ! diff <($(call KUSTOMIZE_INVOCATION,$<,$*,KUSTOMIZE_LOOKUP_KUSTOMIZATION)) $*; then echo "$* was modified manually" && false; fi
 
 # Targets in the makefile can depend on check-go-version-<path to go binary>
 # to trigger a warning if the x.y version of that binary does not match

deploy/kustomize/kubernetes-1.21-direct-testing-coverage/kustomization.yaml

@@ -0,0 +1,13 @@
+bases:
+- ../kubernetes-base-direct-testing-coverage
+
+patches:
+- ../patches/driverinfo-storage-capacity-patch.yaml
+
+patchesJson6902:
+- target:
+    group: apps
+    version: v1
+    kind: DaemonSet
+    name: pmem-csi-intel-com-node
+  path: ../patches/external-provisioner-storage-capacity-patch.yaml

deploy/kustomize/kubernetes-1.21-lvm-testing-coverage/kustomization.yaml

@@ -0,0 +1,13 @@
+bases:
+- ../kubernetes-base-lvm-testing-coverage
+
+patches:
+- ../patches/driverinfo-storage-capacity-patch.yaml
+
+patchesJson6902:
+- target:
+    group: apps
+    version: v1
+    kind: DaemonSet
+    name: pmem-csi-intel-com-node
+  path: ../patches/external-provisioner-storage-capacity-patch.yaml

deploy/kustomize/kubernetes-base-direct-coverage/kustomization.yaml

@@ -1,5 +1,5 @@
 bases:
-- ../kubernetes-base-direct-testing/
+- ../kubernetes-base-direct/
 
 patchesJson6902:
 - target:

deploy/kustomize/kubernetes-base-direct-testing-coverage/kustomization.yaml

@@ -0,0 +1,21 @@
+bases:
+- ../kubernetes-base-direct-testing/
+
+patchesJson6902:
+- target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: pmem-csi-intel-com-controller
+  path: ../testing/controller-coverage-patch.yaml
+
+- target:
+    group: apps
+    version: v1
+    kind: DaemonSet
+    name: pmem-csi-intel-com-node
+  path: ../testing/node-coverage-patch.yaml
+
+images:
+- name: intel/pmem-csi-driver
+  newName: intel/pmem-csi-driver-test

deploy/kustomize/kubernetes-base-lvm-coverage/kustomization.yaml

@@ -1,5 +1,5 @@
 bases:
-- ../kubernetes-base-lvm-testing/
+- ../kubernetes-base-lvm/
 
 patchesJson6902:
 - target:

deploy/kustomize/kubernetes-base-lvm-testing-coverage/kustomization.yaml

@@ -0,0 +1,21 @@
+bases:
+- ../kubernetes-base-lvm-testing/
+
+patchesJson6902:
+- target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: pmem-csi-intel-com-controller
+  path: ../testing/controller-coverage-patch.yaml
+
+- target:
+    group: apps
+    version: v1
+    kind: DaemonSet
+    name: pmem-csi-intel-com-node
+  path: ../testing/node-coverage-patch.yaml
+
+images:
+- name: intel/pmem-csi-driver
+  newName: intel/pmem-csi-driver-test

deploy/yamls.go

@@ -16,7 +16,7 @@ import (
 	"github.com/intel/pmem-csi/pkg/version"
 )
 
-//go:embed kubernetes-*/*/pmem-csi.yaml
+//go:embed kubernetes-*/pmem-csi-*.yaml
 //go:embed kustomize/webhook/webhook.yaml
 //go:embed kustomize/scheduler/scheduler-service.yaml
 //go:embed kustomize/webhook/webhook-service.yaml
@@ -42,7 +42,7 @@ type YamlFile struct {
 
 var yamls []YamlFile
 
-var re = regexp.MustCompile(`^kubernetes-([0-9\.]*)([^/]*)/([^/]*)$`)
+var re = regexp.MustCompile(`^kubernetes-([0-9\.]*)/pmem-csi-(lvm|direct)(.*).yaml$`)
 
 func init() {
 	deployDir, err := assets.ReadDir(".")
@@ -71,7 +71,7 @@ func init() {
 				Name:       name,
 				Kubernetes: kubernetes,
 				Flavor:     parts[3],
-				DeviceMode: api.DeviceMode(parts[3]),
+				DeviceMode: api.DeviceMode(parts[2]),
 			})
 		}
 	}