fix check issue

Author: shmuel-runai

.github/workflows/build-check-test.yaml

@@ -221,6 +221,12 @@ jobs:
           curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
           k3d version
 
+      - name: Install skaffold
+        run: |
+          curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64
+          sudo install skaffold /usr/local/bin/
+          skaffold version
+
       - name: Install Helm
         run: |
           curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash

operator/Makefile

@@ -135,15 +135,12 @@ run-e2e-full: e2e-cluster-up
 
 # Run autoMNNVL e2e tests (all 4 configurations: supported/unsupported x enabled/disabled)
 # This creates a dedicated k3d cluster, runs all configurations sequentially, then cleans up.
+# Images are built with skaffold/ko as part of cluster setup (no Docker build required).
 # Usage: make run-e2e-mnnvl-full
-#   make run-e2e-mnnvl-full              # Full run with build
-#   SKIP_BUILD=1 make run-e2e-mnnvl-full # Skip image build (reuse existing)
 .PHONY: run-e2e-mnnvl-full
 run-e2e-mnnvl-full:
-	@echo "> Preparing charts (copying CRDs)..."
-	@$(MODULE_HACK_DIR)/prepare-charts.sh
 	@echo "> Running autoMNNVL e2e tests (all 4 configurations)..."
-	@python3 $(MODULE_HACK_DIR)/e2e-autoMNNVL/run_autoMNNVL_e2e_all.py $(if $(SKIP_BUILD),--skip-build)
+	@python3 $(MODULE_HACK_DIR)/e2e-autoMNNVL/run_autoMNNVL_e2e_all.py
 
 # Delete the autoMNNVL e2e test cluster
 .PHONY: e2e-mnnvl-cluster-down

operator/hack/e2e-autoMNNVL/run_autoMNNVL_e2e.py

@@ -1,4 +1,20 @@
 #!/usr/bin/env python3
+# /*
+# Copyright 2026 The Grove Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# */
+
 """run_autoMNNVL_e2e.py - Run autoMNNVL e2e tests with a single configuration.
 
 This script sets up the cluster (via setup_autoMNNVL_cluster.py) and runs the e2e tests.

operator/hack/e2e-autoMNNVL/run_autoMNNVL_e2e_all.py

@@ -1,4 +1,20 @@
 #!/usr/bin/env python3
+# /*
+# Copyright 2026 The Grove Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# */
+
 """run_autoMNNVL_e2e_all.py - Run autoMNNVL e2e tests with all 4 configurations.
 
 This script runs the autoMNNVL e2e tests with all possible configurations:
@@ -7,18 +23,19 @@
   3. Feature enabled  + CRD unsupported (no fake GPU)
   4. Feature disabled + CRD unsupported (no fake GPU)
 
+Images are built with skaffold/ko and pushed to the k3d registry as part of
+cluster setup -- no Docker build is required.
+
 Usage: ./hack/e2e-autoMNNVL/run_autoMNNVL_e2e_all.py [options]
 
 Options:
-  --skip-build    Skip initial image build, reuse existing local images
   --keep-cluster  Keep cluster after all configs (no shutdown)
   --help          Show this help message
 """
 
 from __future__ import annotations
 
 import argparse
-import os
 import subprocess
 import sys
 from dataclasses import dataclass
@@ -74,22 +91,27 @@ class ConfigEntry:
 
 
 # The four configurations we test.
+#
+# Images are built with skaffold/ko as part of cluster setup (no upfront Docker
+# build). Configs that create a new cluster also build+push images; configs that
+# reuse the cluster skip the build since images are already in the registry.
+#
 # Config 3 uses --skip-operator-wait because the operator intentionally exits
 # (preflight failure) in this invalid configuration; the e2e test itself
 # validates the expected failure behaviour.
 CONFIGS: list[ConfigEntry] = [
     ConfigEntry(1, "Config1_SupportedAndEnabled",
                 "--with-fake-gpu", "--mnnvl-enabled",
-                ["--skip-build"]),
+                []),
     ConfigEntry(2, "Config2_SupportedButDisabled",
                 "--with-fake-gpu", "--mnnvl-disabled",
-                ["--skip-cluster-create"]),
+                ["--skip-cluster-create", "--skip-build"]),
     ConfigEntry(3, "Config3_UnsupportedButEnabled",
                 "--without-fake-gpu", "--mnnvl-enabled",
-                ["--skip-build", "--skip-operator-wait"]),
+                ["--skip-operator-wait"]),
     ConfigEntry(4, "Config4_UnsupportedAndDisabled",
                 "--without-fake-gpu", "--mnnvl-disabled",
-                ["--skip-cluster-create"]),
+                ["--skip-cluster-create", "--skip-build"]),
 ]
 
 # ---------------------------------------------------------------------------
@@ -100,30 +122,11 @@ def parse_args() -> argparse.Namespace:
         description="Run MNNVL e2e tests with all 4 configurations.",
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    parser.add_argument("--skip-build", action="store_true", default=False,
-                        help="Skip initial image build, reuse existing local images")
     parser.add_argument("--keep-cluster", action="store_true", default=False,
                         help="Keep cluster after all configs (no shutdown)")
     return parser.parse_args()
 
 
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-def build_images_once() -> None:
-    """Build images once at the start so individual configs can use --skip-build."""
-    log_info("Building images once for all configurations...")
-    subprocess.run("./hack/prepare-charts.sh", shell=True, check=True, cwd=OPERATOR_DIR)
-    subprocess.run(
-        "./hack/docker-build.sh",
-        shell=True, check=True, cwd=OPERATOR_DIR,
-        env={**os.environ,
-             "GOARCH": "amd64",
-             "PLATFORM": "linux/amd64",
-             "DOCKER_BUILD_ADDITIONAL_ARGS": "--load"},
-    )
-
-
 def run_config(cfg: ConfigEntry) -> bool:
     """Run a single configuration. Returns True on success."""
     log_header("==========================================")
@@ -203,14 +206,6 @@ def main() -> None:
     log_info("==========================================")
     print(flush=True)
 
-    os.chdir(OPERATOR_DIR)
-
-    # Build images once if not skipping
-    if not args.skip_build:
-        build_images_once()
-    else:
-        log_warning("Skipping initial build (--skip-build)")
-
     # Run all configurations
     results: dict[str, str] = {cfg.name: "NOT_RUN" for cfg in CONFIGS}
 

operator/hack/e2e-autoMNNVL/setup_autoMNNVL_cluster.py

@@ -1,4 +1,20 @@
 #!/usr/bin/env python3
+# /*
+# Copyright 2026 The Grove Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# */
+
 """setup_autoMNNVL_cluster.py - Set up a k3d cluster for autoMNNVL e2e testing.
 
 This script handles cluster creation, fake GPU operator installation,
@@ -13,8 +29,8 @@
   --without-fake-gpu       Skip fake GPU operator installation
   --mnnvl-enabled          Enable MNNVL feature in Grove (default)
   --mnnvl-disabled         Disable MNNVL feature in Grove
-  --build                  Build images with docker (default)
-  --skip-build             Skip image build, use existing local images
+  --build                  Build images with skaffold/ko (default)
+  --skip-build             Skip image build, reuse images already in registry
   --skip-operator-wait     Don't wait for operator pod readiness
   --image <tag>            Use existing image with specified tag
   --help                   Show this help message
@@ -27,6 +43,7 @@
 import subprocess
 import sys
 import time
+from datetime import datetime, timezone
 from pathlib import Path
 
 # ---------------------------------------------------------------------------
@@ -188,33 +205,44 @@ def uninstall_fake_gpu_operator() -> None:
 # ---------------------------------------------------------------------------
 # Build / push images
 # ---------------------------------------------------------------------------
-def build_images() -> None:
-    log_info("Building Grove operator images...")
+def build_and_push_images() -> None:
+    """Build Grove images with skaffold/ko and push to the local k3d registry.
+
+    Uses the same approach as the main e2e cluster setup (create-e2e-cluster.py):
+    skaffold build with the ko builder compiles Go binaries into OCI images and
+    pushes them directly to the registry -- no Docker build required.
+    """
+    log_info("Building and pushing Grove operator images with skaffold...")
     run("./hack/prepare-charts.sh", cwd=OPERATOR_DIR)
+
+    push_repo = f"localhost:{REGISTRY_PORT}"
+    build_date = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    env = {
+        "VERSION": IMAGE_TAG,
+        "LD_FLAGS": (
+            "-X github.com/ai-dynamo/grove/operator/internal/version.gitCommit=e2e-test-commit "
+            "-X github.com/ai-dynamo/grove/operator/internal/version.gitTreeState=clean "
+            f"-X github.com/ai-dynamo/grove/operator/internal/version.buildDate={build_date} "
+            "-X github.com/ai-dynamo/grove/operator/internal/version.gitVersion=mnnvl-e2e"
+        ),
+    }
+
+    log_info(f"Running skaffold build (push to {push_repo})...")
     run(
-        "./hack/docker-build.sh",
+        f"skaffold build --default-repo {push_repo}",
         cwd=OPERATOR_DIR,
-        env={
-            "GOARCH": "amd64",
-            "PLATFORM": "linux/amd64",
-            "DOCKER_BUILD_ADDITIONAL_ARGS": "--load",
-        },
+        env=env,
     )
 
+    log_success("Grove images built and pushed to registry")
 
-def push_images_to_registry() -> None:
-    log_info("Pushing images to local registry...")
-
-    run(f"docker tag grove-operator:latest localhost:{REGISTRY_PORT}/grove-operator:{IMAGE_TAG}")
-    run(f"docker tag grove-initc:latest localhost:{REGISTRY_PORT}/grove-initc:{IMAGE_TAG}")
-    run(f"docker push localhost:{REGISTRY_PORT}/grove-operator:{IMAGE_TAG}")
-    run(f"docker push localhost:{REGISTRY_PORT}/grove-initc:{IMAGE_TAG}")
-
-    # Also push alpine for test workloads
+    # Push alpine for test workloads (simple pull/tag/push, not a build)
+    log_info("Pushing alpine image for test workloads...")
+    run("docker pull alpine:latest", check=False)
     run(f"docker tag alpine:latest localhost:{REGISTRY_PORT}/alpine:latest")
     run(f"docker push localhost:{REGISTRY_PORT}/alpine:latest")
 
-    log_success("Images pushed to registry")
+    log_success("All images pushed to registry")
 
 
 # ---------------------------------------------------------------------------
@@ -320,9 +348,9 @@ def parse_args() -> argparse.Namespace:
 
     build_group = parser.add_mutually_exclusive_group()
     build_group.add_argument("--build", action="store_true", default=None,
-                             help="Build images with docker (default)")
+                             help="Build images with skaffold/ko (default)")
     build_group.add_argument("--skip-build", action="store_true", default=None,
-                             help="Skip image build, use existing local images")
+                             help="Skip image build, reuse images already in registry")
     build_group.add_argument("--image", metavar="TAG", default=None,
                              help="Use existing image with specified tag")
 
@@ -424,19 +452,12 @@ def main() -> None:
     else:
         uninstall_fake_gpu_operator()
 
-    # Step 3: Build images (if requested)
+    # Step 3: Build and push images with skaffold/ko
+    # Skaffold builds Go binaries with ko and pushes directly to the k3d registry.
     if cfg["build_images"]:
-        build_images()
-    else:
-        log_warning("Skipping image build (using existing images)")
-
-    # Step 3b: Push images to registry
-    # Always push when a new cluster was created (fresh registry has no images).
-    # When reusing an existing cluster, push only if we just built new images.
-    if not cfg["skip_cluster_create"] or cfg["build_images"]:
-        push_images_to_registry()
+        build_and_push_images()
     else:
-        log_info("Skipping image push (reusing existing cluster with existing images)")
+        log_warning("Skipping image build (images already in registry)")
 
     # Step 4: Install Grove operator
     install_grove_operator(