node-installer: add integration test and add special handlings for k3s

.github/workflows/action-node-installer.yaml

@@ -1,4 +1,4 @@
-name: Publish node-installer image
+name: Build, Test, and Publish node-installer image
 
 on:
   workflow_call:
@@ -9,57 +9,130 @@ on:
         required: true
 
 jobs:
-  # Note: assumes being called in a workflow where build has already run and
-  # required artifacts have been uploaded
-  publish:
+  build-and-test:
     permissions:
       contents: read
-      packages: write
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        distribution: [kind, minikube, microk8s, k3s]
+    outputs:
+      release_version: ${{ steps.set_version.outputs.RELEASE_VERSION }}
     steps:
       - uses: actions/checkout@v4
 
       - name: Set RELEASE_VERSION env var
+        id: set_version
         run: |
           if [[ "${{ startsWith(inputs.ref, 'refs/tags/v')}}" == "true" ]]; then
-            echo "RELEASE_VERSION=$(echo -n ${{ inputs.ref }} | cut -d '/' -f 3)" >> $GITHUB_ENV
+            RELEASE_VERSION=$(echo -n ${{ inputs.ref }} | cut -d '/' -f 3)
           else
-            echo "RELEASE_VERSION=$(date +%Y%m%d-%H%M%S)-g$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+            RELEASE_VERSION=$(date +%Y%m%d-%H%M%S)-g$(git rev-parse --short HEAD)
           fi
+          echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_ENV
+          echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_OUTPUT
 
       - uses: actions/download-artifact@v4
         with:
           path: _artifacts
 
-      # Setup buildx to build multiarch image: https://github.com/docker/build-push-action/blob/master/docs/advanced/multi-platform.md
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 
       - name: Setup buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Login to GitHub container registry
-        uses: docker/login-action@v3
+      - name: Extract musl artifacts into ./node-installer/.tmp/linux/(amd64|arm64) dir
+        run: |
+          mkdir -p ./node-installer/.tmp/linux/amd64
+          mkdir -p ./node-installer/.tmp/linux/arm64
+          for f in ./_artifacts/*/*-x86_64.tar.gz; do tar -xf $f --directory ./node-installer/.tmp/linux/amd64; done
+          for f in ./_artifacts/*/*-aarch64.tar.gz; do tar -xf $f --directory ./node-installer/.tmp/linux/arm64; done
+
+      - name: Build node-installer image for testing
+        run: make build-dev-installer-image
+        working-directory: node-installer
+
+      - uses: helm/[email protected]
+        if: matrix.distribution == 'kind'
         with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          install_only: true
+
+      - uses: medyagh/[email protected]
+        if: matrix.distribution == 'minikube'
+        with:
+          start: false
+          container-runtime: containerd
+
+      - uses: balchua/[email protected]
+        if: matrix.distribution == 'microk8s'
+
+      - name: Run KIND test
+        if: matrix.distribution == 'kind'
+        run: make test-kind
+        working-directory: node-installer
+
+      - name: Run MiniKube test
+        if: matrix.distribution == 'minikube'
+        run: make test-minikube
+        working-directory: node-installer
+
+      - name: Run MicroK8s test
+        if: matrix.distribution == 'microk8s'
+        run: make test-microk8s
+        working-directory: node-installer
+
+      - name: Run K3s test
+        if: matrix.distribution == 'k3s'
+        run: make test-k3s
+        working-directory: node-installer
+
+      - name: Collect k3s debug logs
+        if: matrix.distribution == 'k3s' && failure()
+        run: |
+          sudo k3s kubectl describe pods -n kwasm
+          sudo k3s kubectl describe pods
+
+  publish:
+    needs: build-and-test
+    permissions:
+      contents: read
+      packages: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/download-artifact@v4
+        with:
+          path: _artifacts
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Setup buildx
+        uses: docker/setup-buildx-action@v3
 
-      # Build and push node-installer image
-      # TODO: remove once https://github.com/spinkube/runtime-class-manager handles this
       - name: Extract musl artifacts into ./node-installer/.tmp/linux/(amd64|arm64) dir
         run: |
           mkdir -p ./node-installer/.tmp/linux/amd64
           mkdir -p ./node-installer/.tmp/linux/arm64
           for f in ./_artifacts/*/*-x86_64.tar.gz; do tar -xf $f --directory ./node-installer/.tmp/linux/amd64; done
           for f in ./_artifacts/*/*-aarch64.tar.gz; do tar -xf $f --directory ./node-installer/.tmp/linux/arm64; done
 
+      - name: Login to GitHub container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Build and push node-installer image
         uses: docker/build-push-action@v5
         with:
           push: true
           tags: |
-            ghcr.io/${{ github.repository }}/node-installer:${{ env.RELEASE_VERSION }}
+            ghcr.io/${{ github.repository }}/node-installer:${{ needs.build-and-test.outputs.release_version }}
           context: node-installer
           platforms: linux/amd64,linux/arm64
 

node-installer/Makefile

@@ -3,16 +3,32 @@ IMAGE_NAME ?= ghcr.io/spinkube/containerd-shim-spin/node-installer
 PLATFORM ?= linux/amd64
 ARCH ?= x86_64
 TARGET ?= $(ARCH)-unknown-linux-musl
+BINARY := ./.tmp/$(PLATFORM)/containerd-shim-spin-$(SPIN_VERSION)
 
 compile-musl:
-	make build-spin-cross-$(TARGET) -C ../
+	make build-cross-$(TARGET) -C ../
 
-move-musl-to-tmp: compile-musl
-	mkdir -p ./.tmp
-	cp ../../containerd-shim-spin/target/$(TARGET)/release/containerd-shim-spin-$(SPIN_VERSION) ./.tmp/
+$(BINARY): 
+	mkdir -p ./.tmp/$(PLATFORM)
+	$(MAKE) compile-musl
+	cp ../target/$(TARGET)/release/containerd-shim-spin-$(SPIN_VERSION) $(BINARY)
 
-build-multi-installer-image: move-musl-to-tmp
+build-multi-installer-image: $(BINARY)
 	docker buildx build -t $(IMAGE_NAME) --platform linux/amd64,linux/arm64 .
 
-build-dev-installer-image: move-musl-to-tmp
+build-dev-installer-image: $(BINARY)
 	docker buildx build -t $(IMAGE_NAME) --load --platform $(PLATFORM) .
+
+test-kind:
+	./tests/integration-test-kind.sh
+
+test-minikube:
+	./tests/integration-test-minikube.sh
+
+test-microk8s:
+	./tests/integration-test-microk8s.sh
+
+test-k3s:
+	./tests/integration-test-k3s.sh
+
+test-all: test-kind test-minikube test-microk8s test-k3s

node-installer/README.md

@@ -7,3 +7,19 @@ which also bundles other shims.
 
 The intention is for the [spinkube/runtime-class-manager](https://github.com/spinkube/runtime-class-manager)
 project to handle this concern in the future.
+
+## Integration Tests
+
+The project includes integration test scripts for different Kubernetes distributions in the `tests/` directory:
+
+1. Kind: `make test-kind`
+2. MiniKube: `make test-minikube`
+3. MicroK8s: `make test-microk8s`
+4. K3s: `make test-k3s`
+
+## Build the Image Locally
+
+```bash
+make build-dev-installer-image
+```
+

node-installer/script/installer.sh

@@ -39,13 +39,24 @@ mkdir -p $NODE_ROOT$KWASM_DIR/bin/
 cp /assets/containerd-shim-spin-v2 $NODE_ROOT$KWASM_DIR/bin/
 
 if ! grep -q spin $NODE_ROOT$CONTAINERD_CONF; then
-    echo '
+    if $IS_K3S; then
+        echo '
+[plugins."io.containerd.cri.v1.runtime".containerd.runtimes."spin"]
+    runtime_type = "'$KWASM_DIR'/bin/containerd-shim-spin-v2"
+' >> $NODE_ROOT$CONTAINERD_CONF
+    else
+        echo '
 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin]
     runtime_type = "'$KWASM_DIR'/bin/containerd-shim-spin-v2"
 ' >> $NODE_ROOT$CONTAINERD_CONF
+    fi
     rm -Rf $NODE_ROOT$KWASM_DIR/active
 fi
 
+if $IS_K3S; then
+    sed -i "s|runtime_type = \"io.containerd.spin.*\"|runtime_type = \"$KWASM_DIR/bin/containerd-shim-spin-v2\"|g" $NODE_ROOT$CONTAINERD_CONF
+fi
+
 if [ ! -f $NODE_ROOT$KWASM_DIR/active ]; then
     touch $NODE_ROOT$KWASM_DIR/active
     if $IS_MICROK8S; then

node-installer/tests/integration-test-k3s.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+set -euo pipefail
+
+: ${IMAGE_NAME:=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev}
+
+echo "Installing K3s..."
+curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable=traefik --write-kubeconfig-mode=644" sh -
+
+echo "Waiting for K3s to be ready..."
+sleep 10
+export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+until kubectl get nodes | grep -q " Ready"; do
+  echo "Waiting for node to be ready..."
+  sleep 5
+done
+
+echo "=== Step 2: Create namespace and deploy RuntimeClass ==="
+kubectl create namespace kwasm || true
+kubectl apply -f ./tests/workloads/runtime.yaml
+
+echo "=== Step 3: Build and deploy the KWasm node installer ==="
+if ! docker image inspect $IMAGE_NAME >/dev/null 2>&1; then
+  echo "Building node installer image..."
+  PLATFORM=$(uname -m)
+  if [ "$PLATFORM" = "x86_64" ]; then
+    PLATFORM="linux/amd64"
+    ARCH="x86_64"
+  elif [ "$PLATFORM" = "aarch64" ] || [ "$PLATFORM" = "arm64" ]; then
+    PLATFORM="linux/arm64"
+    ARCH="aarch64"
+  else
+    echo "Unsupported platform: $PLATFORM"
+    exit 1
+  fi
+
+  PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=$IMAGE_NAME make build-dev-installer-image
+fi
+
+echo "Loading node installer image into K3s..."
+docker save $IMAGE_NAME > node-installer.tar
+sudo k3s ctr images import node-installer.tar
+rm node-installer.tar
+
+NODE_NAME=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}')
+cp ./tests/workloads/kwasm-job.yml k3s-kwasm-job.yml
+sed -i "s/spin-test-control-plane-provision-kwasm/k3s-provision-kwasm/g" k3s-kwasm-job.yml
+sed -i "s/spin-test-control-plane-provision-kwasm-dev/k3s-provision-kwasm-dev/g" k3s-kwasm-job.yml
+sed -i "s/spin-test-control-plane/${NODE_NAME}/g" k3s-kwasm-job.yml
+
+echo "Applying KWasm node installer job..."
+kubectl apply -f ./k3s-kwasm-job.yml
+
+echo "Waiting for node installer job to complete..."
+kubectl wait -n kwasm --for=condition=Ready pod --selector=job-name=k3s-provision-kwasm --timeout=90s || true
+kubectl wait -n kwasm --for=jsonpath='{.status.phase}'=Succeeded pod --selector=job-name=k3s-provision-kwasm --timeout=60s
+
+if ! kubectl get pods -n kwasm | grep -q "k3s-provision-kwasm.*Completed"; then
+  echo "Node installer job failed!"
+  kubectl logs -n kwasm $(kubectl get pods -n kwasm -o name | grep k3s-provision-kwasm)
+  exit 1
+fi
+
+echo "=== Step 4: Apply the workload ==="
+sudo k3s ctr images pull ghcr.io/spinkube/containerd-shim-spin/examples/spin-rust-hello:v0.18.0
+kubectl apply -f ./tests/workloads/workload.yaml
+
+echo "Waiting for deployment to be ready..."
+kubectl wait --for=condition=Available deployment/wasm-spin --timeout=120s
+
+echo "Checking pod status..."
+kubectl get pods
+
+echo "=== Step 5: Test the workload ==="
+echo "Waiting for service to be ready..."
+sleep 10
+
+echo "Testing workload with curl..."
+kubectl port-forward svc/wasm-spin 8888:80 &
+FORWARD_PID=$!
+sleep 5
+
+MAX_RETRIES=3
+RETRY_COUNT=0
+SUCCESS=false
+
+while [ $RETRY_COUNT -lt $MAX_RETRIES ] && [ "$SUCCESS" = false ]; do
+  if curl -s http://localhost:8888/hello | grep -q "Hello world from Spin!"; then
+    SUCCESS=true
+    echo "Workload test successful!"
+  else
+    echo "Retrying in 3 seconds..."
+    sleep 3
+    RETRY_COUNT=$((RETRY_COUNT+1))
+  fi
+done
+
+kill $FORWARD_PID || true
+
+if [ "$SUCCESS" = true ]; then
+  echo "=== Integration Test Passed! ==="
+  sudo /usr/local/bin/k3s-uninstall.sh 
+  sudo rm -rf /etc/rancher/k3s
+  sudo rm -rf /var/lib/rancher/k3s
+  exit 0
+else
+  echo "=== Integration Test Failed! ==="
+  echo "Could not get a successful response from the workload."
+  kubectl describe pods
+  kubectl logs $(kubectl get pods -o name | grep wasm-spin)
+  sudo /usr/local/bin/k3s-uninstall.sh 
+  sudo rm -rf /etc/rancher/k3s
+  sudo rm -rf /var/lib/rancher/k3s
+  exit 1
+fi