fixup! test(e2e): Use trivy-operator for CIS benchmark

Author: jimmidyson

.github/workflows/e2e.yml

@@ -105,10 +105,10 @@ jobs:
           RUN_CIS_BENCHMARK: ${{ inputs.run-cis-benchmark }}
 
       - name: Add job summary for CIS benchmark
-        if: inputs.run-cis-benchmark
+        if: failure() && inputs.run-cis-benchmark
         run: |
           echo "## CIS Benchmark" >>"${GITHUB_STEP_SUMMARY}"
-          cat test/e2e/cis-benchmark-report.txt >>"${GITHUB_STEP_SUMMARY}"
+          cat test/e2e/cis-benchmark-report.md >>"${GITHUB_STEP_SUMMARY}"
 
       - if: success() || failure() # always run even if the previous step fails
         name: Publish e2e test report

.gitignore

@@ -50,3 +50,4 @@ test/e2e/config/caren-envsubst.yaml
 hack/tools/fetch-images/fetch-images
 caren-images.txt
 hack/examples/release/*-cluster-class.yaml
+test/e2e/cis-benchmark-report.txt

test/e2e/quick_start_test.go

@@ -9,20 +9,24 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
-	"path/filepath"
 	"slices"
 	"strconv"
 	"strings"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/samber/lo"
+	"gopkg.in/yaml.v2"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/utils/ptr"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	capie2e "sigs.k8s.io/cluster-api/test/e2e"
 	capiframework "sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
 	"sigs.k8s.io/cluster-api/util"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
 	apivariables "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/variables"
@@ -313,29 +317,113 @@ var _ = Describe("Quick start", func() {
 												if os.Getenv("RUN_CIS_BENCHMARK") == "true" {
 													By("Running CIS benchmark against workload cluster")
 
-													trivyCmd := exec.Command( //nolint:gosec // Only used for testing so safe here.
-														"trivy",
-														"k8s",
-														"--compliance=k8s-cis-1.23",
-														"--disable-node-collector",
-														"--report=summary",
-														fmt.Sprintf(
-															"--output=%s",
-															filepath.Join(
-																os.Getenv("GIT_REPO_ROOT"),
-																"cis-benchmark-report.txt",
-															),
-														),
+													trivyInstallCmd := exec.Command( //nolint:gosec // Only used for testing so safe here.
+														"helm",
+														"upgrade",
+														"--install",
+														"trivy-operator",
+														"oci://ghcr.io/aquasecurity/helm-charts/trivy-operator",
+														"--namespace=trivy-system",
+														"--create-namespace",
+														"--wait",
+														"--wait-for-jobs",
+														"--values=testdata/trivy-operator-values.yaml",
 														fmt.Sprintf(
 															"--kubeconfig=%s",
 															workloadProxy.GetKubeconfigPath(),
 														),
 													)
+													trivyInstallCmd.Stdout = GinkgoWriter
+													trivyInstallCmd.Stderr = GinkgoWriter
+													Expect(
+														trivyInstallCmd.Run(),
+													).To(Succeed(), "trivy operator installation failed")
+
+													complianceReport := unstructured.Unstructured{}
+													complianceReport.SetGroupVersionKind(
+														schema.GroupVersionKind{
+															Group:   "aquasecurity.github.io",
+															Version: "v1alpha1",
+															Kind:    "ClusterComplianceReport",
+														},
+													)
+													complianceReport.SetName("k8s-cis-1.23")
+
+													var (
+														passed, failed int64
+														err            error
+													)
+													Eventually(func() (bool, error) {
+														Expect(
+															workloadClient.Get(
+																ctx,
+																ctrlclient.ObjectKeyFromObject(&complianceReport),
+																&complianceReport,
+															),
+														).To(Succeed(), "failed to get compliance report")
+
+														passed, _, err = unstructured.NestedInt64(
+															complianceReport.Object,
+															"status",
+															"summary",
+															"passCount",
+														)
+														if err != nil {
+															return false, fmt.Errorf(
+																"failed to get compliance report status: %w",
+																err,
+															)
+														}
+														failed, _, err = unstructured.NestedInt64(
+															complianceReport.Object,
+															"status",
+															"summary",
+															"failCount",
+														)
+														if err != nil {
+															return false, fmt.Errorf(
+																"failed to get compliance report status: %w",
+																err,
+															)
+														}
+
+														return passed+failed > 0, nil
+													}).WithTimeout(5*time.Minute).
+														WithPolling(time.Second).
+														Should(BeTrue(), "failed to wait for compliance report")
+
+													complianceStatus, _, err := unstructured.NestedMap(
+														complianceReport.Object,
+														"status",
+													)
+													Expect(err).ToNot(HaveOccurred(), "failed to get compliance status")
+
+													complianceReportFile, err := os.Create("cis-benchmark-report.md")
+													Expect(
+														err,
+													).ToNot(HaveOccurred(), "failed to create compliance report file")
+													defer complianceReportFile.Close()
+
+													_, err = complianceReportFile.WriteString(
+														"# CIS Benchmark Report\n\n```yaml\n",
+													)
+													Expect(
+														err,
+													).NotTo(HaveOccurred(), "failed to write compliance report header")
 
-													trivyCmd.Stdout = GinkgoWriter
-													trivyCmd.Stderr = GinkgoWriter
+													complianceReportEncoder := yaml.NewEncoder(complianceReportFile)
+													Expect(
+														complianceReportEncoder.Encode(complianceStatus),
+													).To(Succeed(), "failed to write compliance report")
+													Expect(
+														complianceReportEncoder.Close(),
+													).To(Succeed(), "failed to close compliance report encoder")
+													_, err = complianceReportFile.WriteString("\n```\n")
+													Expect(
+														err,
+													).NotTo(HaveOccurred(), "failed to write compliance report footer")
 
-													Expect(trivyCmd.Run()).To(Succeed(), "CIS benchmark failed")
+													Expect(failed).To(BeZero(), "CIS benchmark failed with errors")
 												}
 											},
 										}

test/e2e/testdata/trivy-operator-values.yaml

@@ -0,0 +1,8 @@
+# Copyright 2025 Nutanix. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+compliance:
+  specs: ["k8s-cis-1.23"]
+  reportType: all
+  failEntriesLimit: 1000
+  cron: '* * * * *'