feat(disaggregatedset): Add hack/e2e-test.sh for Prow CI compatibility

hasB4K · hasB4K · commit 89cc3e9ec6a3 · 2026-03-17T12:12:54.000+01:00
Add a dedicated e2e test script that mirrors the LWS pattern for running
e2e tests in Prow CI. The script handles the full e2e lifecycle:

- Kind cluster creation/deletion with cleanup trap
- Docker image building and loading to Kind
- LWS controller installation from release manifests
- DisaggregatedSet operator deployment via kustomize
- Running ginkgo tests with junit output
- Log collection on cleanup for debugging

Update Makefile to call the hack script with all required environment
variables (KIND, KUBECTL, KUSTOMIZE, GINKGO, etc.) and add ginkgo tool.

Update e2e tests to skip redundant operations when run via hack script
(detected via LWS_INSTALL_SKIP=true), allowing tests to work both
standalone and in Prow CI.
diff --git a/Makefile b/Makefile
@@ -176,7 +176,7 @@ test-integration: manifests fmt vet envtest ginkgo ## Run integration tests.
 .PHONY: test-e2e
 test-e2e: kustomize manifests fmt vet envtest ginkgo kind-image-build
 	E2E_KIND_VERSION=$(E2E_KIND_VERSION) KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) KIND=$(KIND) KUBECTL=$(KUBECTL) KUSTOMIZE=$(KUSTOMIZE) GINKGO=$(GINKGO) USE_EXISTING_CLUSTER=$(USE_EXISTING_CLUSTER) IMAGE_TAG=$(IMG) ARTIFACTS=$(ARTIFACTS) ./hack/e2e-test.sh
-	$(MAKE) -C disaggregatedset test-e2e KIND=$(KIND)
+	$(MAKE) -C disaggregatedset test-e2e KIND=$(KIND) KUBECTL=$(KUBECTL) E2E_KIND_VERSION=$(E2E_KIND_VERSION) ARTIFACTS=$(ARTIFACTS)
 
 .PHONY: test-e2e-cert-manager
 test-e2e-cert-manager: kustomize manifests fmt vet envtest ginkgo kind-image-build
diff --git a/disaggregatedset/Makefile b/disaggregatedset/Makefile
@@ -65,32 +65,24 @@ test: manifests generate fmt vet setup-envtest ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
 
 # E2E tests using Kind cluster with LWS controller.
-# The test suite handles LWS installation automatically.
-# Skip LWS installation with: LWS_INSTALL_SKIP=true
-KIND_CLUSTER ?= disaggregatedset-test-e2e
+KIND_CLUSTER_NAME ?= disaggregatedset-test-e2e
+E2E_KIND_VERSION ?= kindest/node:v1.32.2
 E2E_IMG ?= controller:e2e
-
-.PHONY: setup-test-e2e
-setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist
-	@if [ ! -x "$(KIND)" ] && ! command -v $(KIND) >/dev/null 2>&1; then \
-		echo "Kind is not installed at '$(KIND)'. Please install Kind or pass KIND=/path/to/kind."; \
-		exit 1; \
-	fi
-	@case "$$($(KIND) get clusters)" in \
-		*"$(KIND_CLUSTER)"*) \
-			echo "Kind cluster '$(KIND_CLUSTER)' already exists. Skipping creation." ;; \
-		*) \
-			echo "Creating Kind cluster '$(KIND_CLUSTER)'..."; \
-			$(KIND) create cluster --name $(KIND_CLUSTER) ;; \
-	esac
+USE_EXISTING_CLUSTER ?= false
+ARTIFACTS ?= $(LOCALBIN)
+LWS_VERSION ?= v0.8.0
 
 .PHONY: test-e2e
-test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind.
-	IMG=$(E2E_IMG) KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -v -ginkgo.v -timeout 15m
+test-e2e: manifests generate kustomize ginkgo ## Run e2e tests using Kind cluster.
+	E2E_KIND_VERSION=$(E2E_KIND_VERSION) KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) KIND=$(KIND) \
+		KUBECTL=$(KUBECTL) KUSTOMIZE=$(KUSTOMIZE) GINKGO=$(GINKGO) \
+		USE_EXISTING_CLUSTER=$(USE_EXISTING_CLUSTER) IMAGE_TAG=$(E2E_IMG) \
+		ARTIFACTS=$(ARTIFACTS) LWS_VERSION=$(LWS_VERSION) \
+		./hack/e2e-test.sh
 
 .PHONY: cleanup-test-e2e
 cleanup-test-e2e: ## Tear down the Kind cluster used for e2e tests
-	@$(KIND) delete cluster --name $(KIND_CLUSTER)
+	@$(KIND) delete cluster --name $(KIND_CLUSTER_NAME)
 
 ##@ Kind Development Helpers
 
@@ -230,11 +222,13 @@ CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
 ENVTEST ?= $(LOCALBIN)/setup-envtest
 GOLANGCI_LINT = $(LOCALBIN)/golangci-lint
 HELMIFY ?= $(LOCALBIN)/helmify
+GINKGO ?= $(LOCALBIN)/ginkgo
 
 ## Tool Versions
 KUSTOMIZE_VERSION ?= v5.7.1
 CONTROLLER_TOOLS_VERSION ?= v0.19.0
 HELMIFY_VERSION ?= v0.4.18
+GINKGO_VERSION ?= v2.27.5
 
 #ENVTEST_VERSION is the version of controller-runtime release branch to fetch the envtest setup script (i.e. release-0.20)
 ENVTEST_VERSION ?= $(shell v='$(call gomodver,sigs.k8s.io/controller-runtime)'; \
@@ -280,6 +274,11 @@ helmify: $(HELMIFY) ## Download helmify locally if necessary.
 $(HELMIFY): $(LOCALBIN)
 	test -s "$(LOCALBIN)/helmify" || GOBIN="$(LOCALBIN)" go install github.com/arttor/helmify/cmd/helmify@$(HELMIFY_VERSION)
 
+.PHONY: ginkgo
+ginkgo: $(GINKGO) ## Download ginkgo locally if necessary.
+$(GINKGO): $(LOCALBIN)
+	test -s "$(LOCALBIN)/ginkgo" || GOBIN="$(LOCALBIN)" go install github.com/onsi/ginkgo/v2/ginkgo@$(GINKGO_VERSION)
+
 # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
 # $1 - target path with name of binary
 # $2 - package url which can be installed
diff --git a/disaggregatedset/hack/e2e-test.sh b/disaggregatedset/hack/e2e-test.sh
@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+export CWD=$(pwd)
+
+# LWS version to install
+LWS_VERSION=${LWS_VERSION:-"v0.8.0"}
+
+function cleanup {
+    if [ "$USE_EXISTING_CLUSTER" == 'false' ]
+    then
+        if [ ! -d "$ARTIFACTS" ]; then
+            mkdir -p "$ARTIFACTS"
+        fi
+        # Export disaggregatedset controller logs
+        $KUBECTL logs -n disaggregatedset-system deployment/disaggregatedset-controller-manager > "$ARTIFACTS"/disaggregatedset-controller-manager.log || true
+        $KUBECTL describe pods -n disaggregatedset-system > "$ARTIFACTS"/disaggregatedset-system-pods.log || true
+        # Export LWS controller logs
+        $KUBECTL logs -n lws-system deployment/lws-controller-manager > "$ARTIFACTS"/lws-controller-manager.log || true
+        $KUBECTL describe pods -n lws-system > "$ARTIFACTS"/lws-system-pods.log || true
+        # Export kind logs and delete cluster
+        $KIND export logs "$ARTIFACTS" || true
+        $KIND delete cluster --name $KIND_CLUSTER_NAME
+    fi
+    (cd $CWD/config/manager && $KUSTOMIZE edit set image controller=controller:latest)
+}
+
+function startup {
+    if [ "$USE_EXISTING_CLUSTER" == 'false' ]
+    then
+        if [ ! -d "$ARTIFACTS" ]; then
+            mkdir -p "$ARTIFACTS"
+        fi
+        $KIND create cluster --name $KIND_CLUSTER_NAME --image $E2E_KIND_VERSION --wait 1m
+        $KUBECTL get nodes > "$ARTIFACTS"/kind-nodes.log || true
+        $KUBECTL describe pods -n kube-system > "$ARTIFACTS"/kube-system-pods.log || true
+    fi
+}
+
+function docker_build {
+    echo "Building disaggregatedset operator image..."
+    docker build -t "$IMAGE_TAG" .
+}
+
+function kind_load {
+    echo "Loading operator image to Kind..."
+    $KIND load docker-image "$IMAGE_TAG" --name "$KIND_CLUSTER_NAME"
+}
+
+function install_lws {
+    echo "Installing LWS ${LWS_VERSION}..."
+    $KUBECTL apply --server-side -f "https://github.com/kubernetes-sigs/lws/releases/download/${LWS_VERSION}/manifests.yaml"
+    echo "Waiting for LWS controller to be ready..."
+    $KUBECTL -n lws-system wait --for=condition=available --timeout=5m deployment/lws-controller-manager
+}
+
+function deploy_operator {
+    echo "Deploying disaggregatedset operator..."
+    pushd "$CWD/config/manager"
+    $KUSTOMIZE edit set image controller="$IMAGE_TAG"
+    popd
+    $KUSTOMIZE build "$CWD/config/default" | $KUBECTL apply --server-side -f -
+    echo "Waiting for disaggregatedset operator to be ready..."
+    $KUBECTL -n disaggregatedset-system wait --for=condition=available --timeout=2m deployment/disaggregatedset-controller-manager
+}
+
+function run_tests {
+    echo "Running e2e tests..."
+    # Set LWS_INSTALL_SKIP since we already installed LWS
+    # Set KIND_CLUSTER for the test code (it expects KIND_CLUSTER, not KIND_CLUSTER_NAME)
+    # Use --tags=e2e to include files with //go:build e2e constraint
+    LWS_INSTALL_SKIP=true KIND_CLUSTER="$KIND_CLUSTER_NAME" $GINKGO --tags=e2e --junit-report=junit.xml --output-dir="$ARTIFACTS" -v "$CWD/test/e2e/..."
+}
+
+trap cleanup EXIT
+startup
+docker_build
+kind_load
+install_lws
+deploy_operator
+run_tests
diff --git a/disaggregatedset/test/e2e/e2e_suite_test.go b/disaggregatedset/test/e2e/e2e_suite_test.go
@@ -79,17 +79,19 @@ var _ = BeforeSuite(func() {
 	_, err = utils.Run(cmd)
 	ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to switch kubectl context to Kind cluster")
 
-	By("building the operator image")
-	cmd = exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectImage))
-	_, err = utils.Run(cmd)
-	ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the operator image")
+	// When run via hack/e2e-test.sh, the image is already built and loaded
+	// and the operator is already deployed. Skip these steps.
+	if !skipLWSInstall {
+		By("building the operator image")
+		cmd = exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectImage))
+		_, err = utils.Run(cmd)
+		ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the operator image")
 
-	By("loading the operator image to Kind")
-	err = utils.LoadImageToKindClusterWithName(projectImage)
-	ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the operator image into Kind")
+		By("loading the operator image to Kind")
+		err = utils.LoadImageToKindClusterWithName(projectImage)
+		ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the operator image into Kind")
 
-	// Install LWS if not skipped and not already installed
-	if !skipLWSInstall {
+		// Install LWS if not already installed
 		By("checking if LWS is already installed")
 		isLWSAlreadyInstalled = utils.IsLWSInstalled()
 		if !isLWSAlreadyInstalled {
@@ -98,24 +100,38 @@ var _ = BeforeSuite(func() {
 		} else {
 			_, _ = fmt.Fprintf(GinkgoWriter, "LWS is already installed, skipping installation\n")
 		}
+
+		By("installing DisaggDeployment CRDs")
+		cmd = exec.Command("make", "install")
+		_, err = utils.Run(cmd)
+		ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to install CRDs")
+	} else {
+		_, _ = fmt.Fprintf(GinkgoWriter, "LWS_INSTALL_SKIP=true: skipping image build, load, and CRD install\n")
 	}
 
 	By("waiting for LWS to be ready")
 	Expect(utils.WaitForLWSReady()).To(Succeed(), "LWS is not ready")
-
-	By("installing DisaggDeployment CRDs")
-	cmd = exec.Command("make", "install")
-	_, err = utils.Run(cmd)
-	ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to install CRDs")
 })
 
 var _ = AfterSuite(func() {
+	// When run via hack/e2e-test.sh (LWS_INSTALL_SKIP=true), cleanup is handled by the script
+	if skipLWSInstall {
+		_, _ = fmt.Fprintf(GinkgoWriter, "LWS_INSTALL_SKIP=true: skipping cleanup (handled by hack/e2e-test.sh)\n")
+		// Just restore context
+		if originalKubeContext != "" {
+			By("restoring original kubectl context")
+			cmd := exec.Command("kubectl", "config", "use-context", originalKubeContext)
+			_, _ = utils.Run(cmd)
+		}
+		return
+	}
+
 	By("uninstalling CRDs")
 	cmd := exec.Command("make", "uninstall")
 	_, _ = utils.Run(cmd)
 
 	// Only uninstall LWS if we installed it
-	if !skipLWSInstall && !isLWSAlreadyInstalled {
+	if !isLWSAlreadyInstalled {
 		_, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling LWS controller...\n")
 		utils.UninstallLWS()
 	}
diff --git a/disaggregatedset/test/e2e/e2e_test.go b/disaggregatedset/test/e2e/e2e_test.go
