Ensure mocked kubeversion constraint is sane and configurable in template rendering (#502)

Signed-off-by: Jose R. Gonzalez <[email protected]>

Author: komish

Makefile

@@ -34,10 +34,22 @@ fmt: install.gofumpt
 	${GOFUMPT} -l -w . 
 	git diff --exit-code
 
+
+# These values capture client-go's supported Kubernetes version and uses that to
+# inform some sane defaults for the chart-verifier CLI, particularly when faking server
+# interactions for things like template rendering. It's modeled after Helm.
+K8S_MODULES_VER=$(subst ., ,$(subst v,,$(shell go list -f '{{.Version}}' -m k8s.io/client-go)))
+K8S_MODULES_MAJOR_VER=$(shell echo $$(($(firstword $(K8S_MODULES_VER)) + 1)))
+K8S_MODULES_MINOR_VER=$(word 2,$(K8S_MODULES_VER))
+
+LDFLAGS :=
+LDFLAGS += -X github.com/redhat-certification/chart-verifier/cmd.CommitIDLong=$(COMMIT_ID_LONG)
+LDFLAGS += -X github.com/redhat-certification/chart-verifier/internal/chartverifier/checks.defaultMockedKubeVersionString=v$(K8S_MODULES_MAJOR_VER).$(K8S_MODULES_MINOR_VER)
+
 .PHONY: bin
 bin:
 	CGO_ENABLED=0 go build \
-		-ldflags "-X 'github.com/redhat-certification/chart-verifier/cmd.CommitIDLong=$(COMMIT_ID_LONG)'" \
+		-ldflags '$(LDFLAGS)' \
 		-o ./out/chart-verifier main.go
 
 .PHONY: lint

docs/helm-chart-checks.md

@@ -14,8 +14,6 @@ Helm chart checks are a set of checks against which the Red Hat Helm chart-verif
     - [Using the podman or docker command for Helm chart checks](#using-the-podman-or-docker-command-for-helm-chart-checks)
       - [Prerequisites](#prerequisites)
       - [Procedure](#procedure)
-    - [Check processing](#check-processing)
-    - [Chart testing timeouts](#chart-testing-timeouts)
   - [Signed charts](#signed-charts)
 
 ## Key features
@@ -513,6 +511,32 @@ Notes:
 - The [helm chart certification process](./helm-chart-submission.md#submission-of-helm-charts-for-red-hat-openShift-certification) uses default timeout values.
   - If a helm chart can only pass the chart testing check with modified timeouts a verifier report must be included in the chart submission.  
 
+## Images are Certified
+
+The **images-are-certified** check must be able to render your templates in
+order to extract image references. In that process, your chart's kubeVersion
+constraint will be evaluated against a "server" version. No server is required
+for this check, and Chart Verifier will mock out the server version information
+for you (as is also done via `helm template`).
+
+However, you may find that Chart Verifier mocks out server version that does not
+fall within your chart's constraints. In most cases, this will happen if you
+have an upper bound on your supported kubeVersion (e.g. `<= v1.30.0`).
+
+If this is the case, you can override the mocked version
+string to another valid semantic version that falls within your constraints.
+This is done using the `--set images-are-certified.kube-version=<yourversion>`
+flag.
+
+For example, to set the kube-version value for this check to `v1.29`:
+
+```shell
+    $ chart-verifier                                                  \
+        verify                                                        \
+        --enable images-are-certified                                 \
+        --set images-are-certified.kube-version=v1.29                 \
+        some-chart.tgz
+```
 
 ## Signed charts
 

internal/chartverifier/checks/chart-0.1.0-v3.mocked-kubeversion-constraint-mismatch.tgz

@@ -482,10 +482,34 @@ func downloadFile(fileURL *url.URL, directory string) (string, error) {
 	return filePath, nil
 }
 
+// defaultMockedKubeVersionString represents a baseline "kubeVersion" for
+// use in rendering templates without actually talking to a server.
+//
+// Rendering templates doesn't technically require a server, but the
+// rendering process used by upstream libraries still does a compatibility
+// check between Chart.yaml kubeVersion constraints and a mocked server
+// version.
+//
+// For this reason, we set this version expected to be "newer than all
+// versions" to satisfy a majority of cases. For everything else, users
+// should set `--set images-are-certified.kube-version=<version>` to match
+// their constraints.
+//
+// Our built binaries will default this to the current k8s version supported
+// by client.go.
+var defaultMockedKubeVersionString = "v99.99"
+
 func certifyImages(r Result, opts *CheckOptions, registry string) Result {
-	images, err := getImageReferences(opts.URI, opts.Values)
+	kubeVersionString := defaultMockedKubeVersionString
+
+	if userKubeVersion := opts.ViperConfig.GetString("kube-version"); userKubeVersion != "" {
+		kubeVersionString = userKubeVersion
+	}
+
+	images, err := getImageReferences(opts.URI, opts.Values, kubeVersionString)
 	if err != nil {
 		r.SetResult(false, fmt.Sprintf("%s : Failed to get images, error running helm template : %v", ImageCertifyFailed, err))
+		return r
 	}
 
 	if len(images) == 0 {

internal/chartverifier/checks/checks.go

@@ -403,6 +403,42 @@ func TestHelmLint(t *testing.T) {
 	}
 }
 
+func TestImageCertifyKubeVersionConstraints(t *testing.T) {
+	specificKubeVersion := "v1.21.0"
+	tests := []struct {
+		description               string
+		uri                       string
+		optionalKubeVersionString *string
+		outputIsValidFunc         func(s string) bool
+	}{
+		{
+			description: "KubeVersion mismatch between default and chart should fail to run image checks",
+			uri:         "chart-0.1.0-v3.mocked-kubeversion-constraint-mismatch.tgz",
+			outputIsValidFunc: func(s string) bool {
+				return strings.Contains(s, ImageCertifyFailed) && strings.Contains(s, "which is incompatible with Kubernetes")
+			},
+		},
+		{
+			description: "KubeVersion should align when --set is used to set a kubeversion",
+			uri:         "chart-0.1.0-v3.mocked-kubeversion-constraint-mismatch.tgz",
+			outputIsValidFunc: func(s string) bool {
+				return strings.Contains(s, ImageCertified)
+			},
+			optionalKubeVersionString: &specificKubeVersion,
+		},
+	}
+
+	for _, test := range tests {
+		v := viper.New()
+		if test.optionalKubeVersionString != nil {
+			v.Set("kube-version", *test.optionalKubeVersionString)
+		}
+		// error is ignored because this fn never returns an error.
+		result, _ := ImagesAreCertified_V1_1(&CheckOptions{URI: test.uri, ViperConfig: v, HelmEnvSettings: cli.New()})
+		require.True(t, test.outputIsValidFunc(result.Reason), test.description)
+	}
+}
+
 func TestImageCertify(t *testing.T) {
 	checkImages(t)
 }
@@ -415,6 +451,7 @@ func checkImages(t *testing.T) {
 		numErrors   int
 		numPasses   int
 		numSkips    int
+		numFailures int
 	}{
 		{
 			description: "chart-0.1.0-v3.valid.tgz check images passes",

internal/chartverifier/checks/checks_test.go

@@ -189,13 +189,34 @@ func IsChartNotFound(err error) bool {
 	return ok
 }
 
-func getImageReferences(chartURI string, vals map[string]interface{}) ([]string, error) {
+// getImageReferences renders the templates for chartURI and extracts
+// imageReferences from the template output, using vals as necessaary.
+//
+// Note that template rendering doesn't technically need a remote cluster, but
+// the chart's constraints are still validated against mocked cluster
+// information. For this reaosn, serverKubeVersionString must produce a valid
+// semantic version corresponding to a kubeVersion within the chart's
+// constraints as defined in Chart.yaml.
+func getImageReferences(chartURI string, vals map[string]interface{}, serverKubeVersionString string) ([]string, error) {
+	// We'll start with DefaultCapabilities, but we'll really only use the
+	// kubeVersion of this when rendering manifests because Helm replaces the
+	// action config's capabilities for client-only execution.
+	caps := chartutil.DefaultCapabilities.Copy()
+
+	kubeVersion, err := chartutil.ParseKubeVersion(serverKubeVersionString)
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", "unable to render manifests and extract images due to invalid kubeVersion in server capabilities", err)
+	}
+
+	caps.KubeVersion = *kubeVersion
+
 	actionConfig := &action.Configuration{
 		Releases:     nil,
 		KubeClient:   &kubefake.PrintingKubeClient{Out: io.Discard},
-		Capabilities: chartutil.DefaultCapabilities,
+		Capabilities: caps,
 		Log:          func(format string, v ...interface{}) {},
 	}
+
 	mem := driver.NewMemory()
 	mem.SetNamespace("TestNamespace")
 	actionConfig.Releases = storage.Init(mem)

internal/chartverifier/checks/helm.go

@@ -167,7 +167,7 @@ func TestTemplate(t *testing.T) {
 
 	for _, tc := range TestCases {
 		t.Run(tc.description, func(t *testing.T) {
-			images, err := getImageReferences(tc.uri, map[string]interface{}{})
+			images, err := getImageReferences(tc.uri, map[string]interface{}{}, defaultMockedKubeVersionString)
 			require.NoError(t, err)
 			require.Equal(t, len(images), len(tc.images))
 			for i := 0; i < len(tc.images); i++ {

internal/chartverifier/checks/helm_test.go

@@ -25,6 +25,13 @@ func RenderManifests(name string, url string, vals map[string]interface{}, conf
 	client.ClientOnly = !validate
 	emptyResponse := ""
 
+	// Must set the capabilities on *action.Install{}.KubeVersion directly
+	// because Helm will replace our capabilities with the defaults they
+	// configure.
+	if conf.Capabilities != nil {
+		client.KubeVersion = &conf.Capabilities.KubeVersion
+	}
+
 	name, chart, err := client.NameAndChart([]string{name, url})
 	if err != nil {
 		return emptyResponse, err