test: collect coverage information also for "operator API"

Those tests cover a significant chunk of code, so we want to include that. The
problem was that the reference files for "deploy" get modified when enabling
coverage collection, which broke the API tests. Now we generate and embed a
second set of "nocoverage" files for those tests.

Author: pohly

Jenkinsfile

@@ -70,8 +70,13 @@ pipeline {
         TEST_ETCD_VOLUME_SIZE = "1073741824" // 1GB
 
         // Tests that will get skipped when collecting coverage information.
-        // The operator tests fail because the reference files get modified.
-        COVERAGE_SKIP = "[email protected]"
+        //
+        // The operator itself installs without enabling coverage collection,
+        // so running those tests doesn't help us. The relevant test is
+        // "operator API".
+        //
+        // Testing with OLM doesn't add much additional coverage.
+        COVERAGE_SKIP = "[email protected]@Top.Level..olm"
     }
 
     stages {

Makefile

@@ -160,7 +160,9 @@ KUSTOMIZE_INPUT := $(shell [ ! -d deploy/kustomize ] || find deploy/kustomize -t
 KUSTOMIZE :=
 
 # Setting this to any non-empty value before "make kustomize" will enable
-# collection of coverage profiles in all deployments.
+# collection of coverage profiles in all deployments. This also sets up the
+# original deployment files under "deploy/nocoverage" for use as reference when
+# testing the operator.
 KUSTOMIZE_WITH_COVERAGE =
 ifneq "$(KUSTOMIZE_WITH_COVERAGE)" ""
 KUSTOMIZE_COVERAGE_SUFFIX = -coverage
@@ -206,14 +208,22 @@ KUSTOMIZE_OUTPUT := $(foreach item,$(KUSTOMIZE),$(firstword $(subst =, ,$(item))
 # right one.
 KUSTOMIZE_LOOKUP_KUSTOMIZATION = $(strip $(foreach item,$(KUSTOMIZE),$(if $(filter $(1)=%,$(item)),$(word 2,$(subst =, ,$(item))))))
 
+# This is a wrapper around KUSTOMIZE_LOOKUP_KUSTOMIZATION which
+# removes the -coverage suffix.
+KUSTOMIZE_LOOKUP_KUSTOMIZATION_NO_COVERAGE = $(subst -coverage,,$(call KUSTOMIZE_LOOKUP_KUSTOMIZATION,$(1)))
+
 # This function takes the kustomize binary and the name of an output
 # file as arguments and returns the command which produces that file
 # as stdout.
-KUSTOMIZE_INVOCATION = (echo '\# Generated with "make kustomize", do not edit!'; echo; $(1) build --load-restrictor LoadRestrictionsNone $(call KUSTOMIZE_LOOKUP_KUSTOMIZATION,$(2)))
+KUSTOMIZE_INVOCATION = (echo '\# Generated with "make kustomize", do not edit!'; echo; $(1) build --load-restrictor LoadRestrictionsNone $(call $(3),$(2)))
 
 $(KUSTOMIZE_OUTPUT): _work/kustomize $(KUSTOMIZE_INPUT)
 	mkdir -p ${@D}
-	$(call KUSTOMIZE_INVOCATION,$<,$@) >$@
+	$(call KUSTOMIZE_INVOCATION,$<,$@,KUSTOMIZE_LOOKUP_KUSTOMIZATION) >$@
+ifneq "$(KUSTOMIZE_WITH_COVERAGE)" ""
+	mkdir -p $(subst deploy/,deploy/nocoverage/,${@D})
+	$(call KUSTOMIZE_INVOCATION,$<,$@,KUSTOMIZE_LOOKUP_KUSTOMIZATION_NO_COVERAGE) >$(subst deploy/,deploy/nocoverage/,$@)
+endif
 	if echo "$@" | grep '/pmem-csi-' | grep -qv '\-operator'; then \
 		dir=$$(echo "$@" | tr - / | sed -e 's;kubernetes/;kubernetes-;' -e 's;/alpha/;-alpha/;'  -e 's;/distributed/;-distributed/;' -e 's/.yaml//' -e 's;/pmem/csi/;/;') && \
 		mkdir -p $$dir && \
@@ -222,9 +232,13 @@ $(KUSTOMIZE_OUTPUT): _work/kustomize $(KUSTOMIZE_INPUT)
 	fi
 
 kustomize: clean_kustomize_output $(KUSTOMIZE_OUTPUT)
+ifneq "$(KUSTOMIZE_WITH_COVERAGE)" ""
+	sed -i -e 's/embed kubernetes-/embed nocoverage kubernetes-/' deploy/yamls.go
+endif
 
 clean_kustomize_output:
-	rm -rf deploy/kubernetes-*
+	rm -rf deploy/kubernetes-* deploy/nocoverage
+	sed -i -e 's/embed nocoverage /embed /' deploy/yamls.go
 	rm -f $(KUSTOMIZE_OUTPUT)
 
 # Always re-generate the output files because "git rebase" might have
@@ -241,7 +255,7 @@ clean-kustomize:
 test: test-kustomize
 test-kustomize: $(addprefix test-kustomize-,$(KUSTOMIZE_OUTPUT))
 $(addprefix test-kustomize-,$(KUSTOMIZE_OUTPUT)): test-kustomize-%: _work/kustomize
-	@ if ! diff <($(call KUSTOMIZE_INVOCATION,$<,$*)) $*; then echo "$* was modified manually" && false; fi
+	@ if ! diff <($(call KUSTOMIZE_INVOCATION,$<,$*,KUSTOMIZE_LOOKUP_KUSTOMIZATION)) $*; then echo "$* was modified manually" && false; fi
 
 # Targets in the makefile can depend on check-go-version-<path to go binary>
 # to trigger a warning if the x.y version of that binary does not match

deploy/yamls.go

@@ -16,7 +16,7 @@ import (
 	"github.com/intel/pmem-csi/pkg/version"
 )
 
-//go:embed kubernetes-*/*/pmem-csi.yaml
+//go:embed kubernetes-*/pmem-csi-*.yaml
 //go:embed kustomize/webhook/webhook.yaml
 //go:embed kustomize/scheduler/scheduler-service.yaml
 //go:embed kustomize/webhook/webhook-service.yaml
@@ -42,7 +42,7 @@ type YamlFile struct {
 
 var yamls []YamlFile
 
-var re = regexp.MustCompile(`^kubernetes-([0-9\.]*)([^/]*)/([^/]*)$`)
+var re = regexp.MustCompile(`^kubernetes-([0-9\.]*)/pmem-csi-(lvm|direct)(.*).yaml$`)
 
 func init() {
 	deployDir, err := assets.ReadDir(".")
@@ -71,7 +71,7 @@ func init() {
 				Name:       name,
 				Kubernetes: kubernetes,
 				Flavor:     parts[3],
-				DeviceMode: api.DeviceMode(parts[3]),
+				DeviceMode: api.DeviceMode(parts[2]),
 			})
 		}
 	}

pkg/deployments/load.go

@@ -361,17 +361,21 @@ func patchPodTemplate(obj *unstructured.Unstructured, deployment api.PmemCSIDepl
 }
 
 func yamlPath(kubernetes version.Version, deviceMode api.DeviceMode) string {
-	return fmt.Sprintf("kubernetes-%s/%s/pmem-csi.yaml", kubernetes, deviceMode)
+	return fmt.Sprintf("kubernetes-%s/pmem-csi-%s.yaml", kubernetes, deviceMode)
 }
 
 func loadYAML(path string,
 	patchYAML func(yaml *[]byte),
 	enabled func(obj *unstructured.Unstructured) bool,
 	patchUnstructured func(obj *unstructured.Unstructured)) ([]unstructured.Unstructured, error) {
-	// We load the builtin yaml files.
-	yaml, err := deploy.Asset(path)
+	// We load the builtin yaml files. If they exist, we prefer
+	// the version without the patched in coverage support.
+	yaml, err := deploy.Asset("nocoverage/" + path)
 	if err != nil {
-		return nil, fmt.Errorf("read reference yaml file: %w", err)
+		yaml, err = deploy.Asset(path)
+		if err != nil {
+			return nil, fmt.Errorf("read reference yaml file: %w", err)
+		}
 	}
 
 	// Split at the "---" separator before working on individual