Use envtest for CRD validation tests (#3948)

* Use envtest for cel validation instead of a KinD cluster

* Add CRD validation tests to gh actions

* Move manifest test to use envtest

* Remove the crd testing shell script

* Fix boilerplate of test file

Author: rikatz

.github/workflows/crd-validation.yml

@@ -0,0 +1,32 @@
+name: CRD Validation
+
+on:
+  pull_request:
+    types: [opened, edited, synchronize, reopened]
+
+# Remove all permissions from GITHUB_TOKEN except metadata.
+permissions: {}
+
+jobs:
+  crd-validation:
+    name: CEL
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # Available versions at https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/HEAD/envtest-releases.yaml
+        k8s_version: [v1.33.0, v1.32.0, v1.31.0, v1.30.3, v1.29.5]
+        crd_channel: [standard, experimental]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+        with:
+          persist-credentials: false
+      - name: Set up Go
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # tag=v5.5.0
+      - name: Run CRD Validation tests
+        env:
+          K8S_VERSION: ${{ matrix.k8s_version }}
+          CRD_CHANNEL: ${{ matrix.crd_channel }}
+        run: |
+          make CEL_TEST_K8S_VERSION="${K8S_VERSION}" CEL_TEST_CRD_CHANNEL="${CRD_CHANNEL}" test.crds-validation

Makefile

@@ -54,6 +54,10 @@ ROOT := $(abspath $(TOP))
 CONFORMANCE_FLAGS ?=
 GO_TEST_FLAGS ?=
 
+# Flags for CRD validation tests
+CEL_TEST_K8S_VERSION ?= 
+CEL_TEST_CRD_CHANNEL ?= standard
+
 all: generate vet fmt verify test
 
 # Run generators for protos, Deepcopy funcs, CRDs, and docs.
@@ -85,7 +89,8 @@ test:
 # Run tests for CRDs validation
 .PHONY: test.crds-validation
 test.crds-validation:
-	./hack/test-crds-validation.sh $(VERSION)
+	K8S_VERSION=$(CEL_TEST_K8S_VERSION) CRD_CHANNEL=$(CEL_TEST_CRD_CHANNEL) go test ${GO_TEST_FLAGS} -count=1 -timeout=120s --tags=$(CEL_TEST_CRD_CHANNEL) -v ./pkg/test/cel
+	K8S_VERSION=$(CEL_TEST_K8S_VERSION) CRD_CHANNEL=$(CEL_TEST_CRD_CHANNEL) go test ${GO_TEST_FLAGS} -count=1 -timeout=120s -v ./pkg/test/crd
 
 # Run conformance tests against controller implementation
 .PHONY: conformance

go.mod

@@ -28,6 +28,7 @@ require (
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/emicklei/go-restful/v3 v3.12.0 // indirect
 	github.com/evanphx/json-patch/v5 v5.9.11 // indirect

go.sum

@@ -6,8 +6,12 @@ github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZC
 github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -120,6 +124,14 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
+github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
+github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
 github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -209,6 +221,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
+gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
+gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 h1:e0AIkUUhxyBKh6ssZNrAMeqhA7RKUj42346d1y02i2g=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
 google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok=

hack/test-crds-validation.sh

@@ -19,7 +19,7 @@ package main
 import (
 	"fmt"
 	"os"
-	"path"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -29,34 +29,84 @@ import (
 	"sigs.k8s.io/gateway-api/apis/v1beta1"
 	apisxv1alpha1 "sigs.k8s.io/gateway-api/apisx/v1alpha1"
 
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
 )
 
 var k8sClient client.Client
 
 func TestMain(m *testing.M) {
+	scheme := runtime.NewScheme()
+	var restConfig *rest.Config
+	var testEnv *envtest.Environment
+	var err error
+
+	v1alpha3.Install(scheme)
+	v1alpha2.Install(scheme)
+	v1beta1.Install(scheme)
+	v1.Install(scheme)
+	apisxv1alpha1.Install(scheme)
+
+	// Add core APIs in case we refer secrets, services and configmaps
+	corev1.AddToScheme(scheme)
+
+	// If one wants to use a local cluster, a KUBECONFIG envvar should be passed,
+	// otherwise testenv will be used
 	kubeconfig := os.Getenv("KUBECONFIG")
-	if kubeconfig == "" {
-		kubeconfig = path.Join(os.Getenv("HOME"), ".kube/config")
+	if kubeconfig != "" {
+		restConfig, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
+		if err != nil {
+			panic(fmt.Sprintf("Failed to get restConfig from BuildConfigFromFlags: %v", err))
+		}
+	} else {
+		// The version used here MUST reflect the available versions at
+		// controller-runtime repo: https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/HEAD/envtest-releases.yaml
+		// If the envvar is not passed, the latest GA will be used
+		k8sVersion := os.Getenv("K8S_VERSION")
+
+		crdChannel := "standard"
+		if requestedCRDChannel, ok := os.LookupEnv("CRD_CHANNEL"); ok {
+			crdChannel = requestedCRDChannel
+		}
+
+		testEnv = &envtest.Environment{
+			Scheme:                      scheme,
+			ErrorIfCRDPathMissing:       true,
+			DownloadBinaryAssets:        true,
+			DownloadBinaryAssetsVersion: k8sVersion,
+			CRDInstallOptions: envtest.CRDInstallOptions{
+				Paths: []string{
+					filepath.Join("..", "..", "..", "config", "crd", crdChannel),
+				},
+				CleanUpAfterUse: true,
+			},
+		}
+
+		restConfig, err = testEnv.Start()
+		if err != nil {
+			panic(fmt.Sprintf("Error initializing test environment: %v", err))
+		}
 	}
 
-	restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
+	k8sClient, err = client.New(restConfig, client.Options{
+		Scheme: scheme,
+	})
 	if err != nil {
-		panic(fmt.Sprintf("Failed to get restConfig from BuildConfigFromFlags: %v", err))
+		panic(fmt.Sprintf("Error initializing Kubernetes client: %v", err))
 	}
 
-	k8sClient, err = client.New(restConfig, client.Options{})
-	if err != nil {
-		panic(fmt.Sprintf("Error initializing Kubernetes client: %v", err))
+	rc := m.Run()
+	if testEnv != nil {
+		if err := testEnv.Stop(); err != nil {
+			panic(fmt.Sprintf("error stopping test environment: %v", err))
+		}
 	}
-	v1alpha3.Install(k8sClient.Scheme())
-	v1alpha2.Install(k8sClient.Scheme())
-	v1beta1.Install(k8sClient.Scheme())
-	v1.Install(k8sClient.Scheme())
-	apisxv1alpha1.Install(k8sClient.Scheme())
 
-	os.Exit(m.Run())
+	os.Exit(rc)
 }
 
 func ptrTo[T any](a T) *T {