unite cluster setup and github action of e2e-mnnvl with e2e

Author: shmuel-runai

.github/actions/e2e-setup/action.yaml

@@ -0,0 +1,73 @@
+git# /*
+# Copyright 2025 The Grove Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# */
+
+name: "E2E Setup"
+description: "Install all tools and dependencies needed to run E2E tests (Go, k3d, skaffold, helm, Python + deps)"
+
+inputs:
+  go-version:
+    description: "Go version to install"
+    required: false
+    default: "1.25.7"
+
+runs:
+  using: "composite"
+  steps:
+    # NVIDIA self-hosted runners don't have make installed by default
+    - name: Install build-essential for make
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt install build-essential -y
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: ${{ inputs.go-version }}
+
+    - name: Install k3d
+      shell: bash
+      run: |
+        curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
+        k3d version
+
+    - name: Install skaffold
+      shell: bash
+      run: |
+        curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64
+        sudo install skaffold /usr/local/bin/
+        skaffold version
+
+    - name: Install Helm
+      shell: bash
+      run: |
+        curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+        helm version
+
+    - name: Install Python3 and pip
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y python3 python3-pip
+
+    - name: Install Python dependencies
+      shell: bash
+      run: |
+        echo "Installing Python dependencies..."
+        pip3 install --break-system-packages -r operator/hack/e2e-cluster/requirements.txt
+
+        echo "Verifying Python dependencies..."
+        python3 -c "import docker; import sh; import typer; import pydantic; import rich; print('All dependencies installed successfully')"

.github/workflows/build-check-test.yaml

@@ -83,6 +83,11 @@ jobs:
 
   # E2E tests - only run after build, check, and test jobs succeed
   # Only triggered by changes to operator or .github folders
+  #
+  # Matrix entries can set:
+  #   test_name     (required) - name shown in the GitHub Actions UI
+  #   test_pattern  (optional) - Go test -run pattern (standard e2e tests)
+  #   make_target   (optional) - Makefile target, defaults to run-e2e-full
   e2e:
     needs: [test, build, check, changes]
     # Run on non-draft PRs (or draft PRs with 'run-e2e' label)
@@ -109,9 +114,11 @@ jobs:
             test_pattern: "^Test_TAS"
           - test_name: cert_management
             test_pattern: "^Test_CM"
+          - test_name: auto_mnnvl
+            make_target: "run-e2e-mnnvl-full"
     name: E2E - ${{ matrix.test_name }}
     steps:
-      # print runner specs so we have a record incase of failures
+      # print runner specs so we have a record in case of failures
       - name: Print runner specs
         run: |
           echo "CPUs: $(nproc)"
@@ -120,49 +127,12 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      # NVIDIA self-hosted runners don't have make installed by default
-      - name: Install build-essential for make
-        run: |
-          sudo apt-get update
-          sudo apt install build-essential -y
-
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: "1.25.7"
-
-      - name: Install k3d
-        run: |
-          curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
-          k3d version
-
-      - name: Install skaffold
-        run: |
-          curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64
-          sudo install skaffold /usr/local/bin/
-          skaffold version
-
-      - name: Install Helm
-        run: |
-          curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
-          helm version
-
-      - name: Install Python3 and pip
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y python3 python3-pip
-
-      - name: Install Python dependencies
-        run: |
-          echo "Installing Python dependencies..."
-          pip3 install --break-system-packages -r operator/hack/e2e-cluster/requirements.txt
-
-          echo "Verifying Python dependencies..."
-          python3 -c "import docker; import sh; import typer; import pydantic; import rich; print('All dependencies installed successfully')"
+      - name: E2E Setup
+        uses: ./.github/actions/e2e-setup
 
       - name: Run e2e tests - ${{ matrix.test_name }}
         run: |
-          make run-e2e-full TEST_PATTERN='${{ matrix.test_pattern }}'
+          make ${{ matrix.make_target || 'run-e2e-full' }} TEST_PATTERN='${{ matrix.test_pattern }}'
         working-directory: operator
 
       # The test code handles cleanup via Teardown(), but this step provides
@@ -184,70 +154,3 @@ jobs:
           path: operator/e2e-diagnostics/
           if-no-files-found: warn
           retention-days: 7
-
-  # E2E tests for autoMNNVL feature
-  # Runs all 4 configurations (supported/unsupported x enabled/disabled) sequentially
-  # Uses a separate cluster setup from the main e2e tests
-  e2e-mnnvl:
-    needs: [test, build, check, changes]
-    if: |
-      github.event_name == 'pull_request' &&
-      needs.changes.outputs.e2e-relevant == 'true' &&
-      (github.event.pull_request.draft == false || contains(github.event.pull_request.labels.*.name, 'run-e2e'))
-    runs-on: cpu-amd-m5-2xlarge
-    timeout-minutes: 60
-    name: E2E - auto_mnnvl
-    steps:
-      - name: Print runner specs
-        run: |
-          echo "CPUs: $(nproc)"
-          echo "RAM: $(free -h | awk '/^Mem:/ {print $2}')"
-
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Install build-essential for make
-        run: |
-          sudo apt-get update
-          sudo apt install build-essential -y
-
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: "1.25.7"
-
-      - name: Install k3d
-        run: |
-          curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
-          k3d version
-
-      - name: Install skaffold
-        run: |
-          curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64
-          sudo install skaffold /usr/local/bin/
-          skaffold version
-
-      - name: Install Helm
-        run: |
-          curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
-          helm version
-
-      - name: Run autoMNNVL e2e tests (all 4 configurations)
-        run: |
-          make run-e2e-mnnvl-full
-        working-directory: operator
-
-      - name: Cleanup k3d cluster
-        if: always()
-        working-directory: operator
-        run: |
-          make e2e-mnnvl-cluster-down || true
-
-      - name: Upload test logs on failure
-        if: failure()
-        uses: actions/upload-artifact@v4
-        with:
-          name: e2e-test-logs-auto_mnnvl
-          path: /tmp/mnnvl-e2e-results.log
-          if-no-files-found: warn
-          retention-days: 7

.gitignore

@@ -31,8 +31,10 @@ dev
 .vscode
 .idea
 .zed
+.local
 .DS_Store
 
+
 # hack tools binaries
 hack/tools/*
 operator/hack/tools/*
@@ -50,3 +52,6 @@ scheduler/bin/*
 
 # generated/copied chart resources
 operator/charts/crds/*
+
+# Python bytecode cache
+__pycache__/

operator/Makefile

@@ -111,9 +111,10 @@ run-e2e:
 
 # Create a k3d cluster for e2e testing (with Grove and Kai scheduler deployed)
 # This is optional - tests work with any Kubernetes cluster that has the required components
+# Pass E2E_CREATE_FLAGS to add CLI flags (e.g. --skip-kai --skip-topology --skip-prepull)
 .PHONY: e2e-cluster-up
 e2e-cluster-up:
-	@$(MODULE_HACK_DIR)/e2e-cluster/create-e2e-cluster.py
+	@$(MODULE_HACK_DIR)/e2e-cluster/create-e2e-cluster.py $(E2E_CREATE_FLAGS)
 
 # Delete the k3d e2e test cluster
 .PHONY: e2e-cluster-down
@@ -134,18 +135,24 @@ run-e2e-full: e2e-cluster-up
 	@$(MAKE) e2e-cluster-down
 
 # Run autoMNNVL e2e tests (all 4 configurations: supported/unsupported x enabled/disabled)
-# This creates a dedicated k3d cluster, runs all configurations sequentially, then cleans up.
-# Images are built with skaffold/ko as part of cluster setup (no Docker build required).
+# Creates a lightweight k3d cluster (2 workers, no Kai/topology), runs all configurations
+# sequentially via config-cluster.py, then cleans up. Uses the same e2e-cluster-up target
+# with MNNVL-specific env overrides and flags.
 # Usage: make run-e2e-mnnvl-full
 .PHONY: run-e2e-mnnvl-full
-run-e2e-mnnvl-full:
+run-e2e-mnnvl-full: export E2E_WORKER_NODES = 2
+run-e2e-mnnvl-full: E2E_CREATE_FLAGS = --skip-kai --skip-topology --skip-prepull
+run-e2e-mnnvl-full: e2e-cluster-up
+	@echo "> Pushing alpine image to local registry..."
+	@docker pull alpine:latest || true
+	@docker tag alpine:latest localhost:5001/alpine:latest
+	@docker push localhost:5001/alpine:latest
 	@echo "> Running autoMNNVL e2e tests (all 4 configurations)..."
-	@python3 $(MODULE_HACK_DIR)/e2e-autoMNNVL/run_autoMNNVL_e2e_all.py
+	@python3 $(MODULE_HACK_DIR)/e2e-autoMNNVL/run_autoMNNVL_e2e_all.py || \
+		(echo "Tests failed, cleaning up cluster..."; $(MAKE) e2e-cluster-down; exit 1)
+	@echo "> Tests passed, cleaning up cluster..."
+	@$(MAKE) e2e-cluster-down
 
-# Delete the autoMNNVL e2e test cluster
-.PHONY: e2e-mnnvl-cluster-down
-e2e-mnnvl-cluster-down:
-	@python3 $(MODULE_HACK_DIR)/e2e-autoMNNVL/setup_autoMNNVL_cluster.py --shutdown
 
 # Make targets for local development and testing
 # -------------------------------------------------------------

operator/e2e/tests/auto-mnnvl/shared_cases.go

@@ -26,6 +26,7 @@ import (
 	"github.com/ai-dynamo/grove/operator/internal/mnnvl"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -43,10 +44,16 @@ func testNoMNNVLArtifactsWhenDisabled(t *testing.T, tc testContext) {
 	// Wait a bit to ensure reconciliation has time to run
 	time.Sleep(10 * time.Second)
 
-	// Verify no ComputeDomain exists
-	cdName := fmt.Sprintf("%s-0", pcsName)
-	_, err = tc.dynamicClient.Resource(computeDomainGVR).Namespace(tc.namespace).Get(tc.ctx, cdName, metav1.GetOptions{})
-	assert.Error(t, err, "No ComputeDomain should be created when feature is disabled")
+	// Verify no ComputeDomain exists.
+	// If the CRD itself is not installed (unsupported scenario), the List call returns
+	// a NotFound error -- that also means zero ComputeDomains, which is what we want.
+	cdList, err := tc.dynamicClient.Resource(computeDomainGVR).Namespace(tc.namespace).List(tc.ctx, metav1.ListOptions{})
+	if k8serrors.IsNotFound(err) {
+		// CRD not installed → no ComputeDomains can exist, which is the expected state.
+	} else {
+		require.NoError(t, err, "Failed to list ComputeDomains")
+		assert.Empty(t, cdList.Items, "Expected 0 ComputeDomains when feature is disabled, got %d", len(cdList.Items))
+	}
 
 	// Verify PCSGs do not get auto-mnnvl annotation
 	pcsgNames := []string{