From d00f040780a5c3965f453fc6453f3ab0ab65990e Mon Sep 17 00:00:00 2001 From: "Jiaxiao (mossaka) Zhou" Date: Thu, 13 Mar 2025 00:11:46 +0000 Subject: [PATCH 1/9] node-installer: add integration test - introduce a script to integration test the node-installer image - add a KWasm node installer job file for the integration test - modified the README to include instructions for building the image locally and running the integration tests Signed-off-by: Jiaxiao (mossaka) Zhou --- node-installer/.kind/kind-config.yaml | 9 +++ node-installer/Makefile | 7 +- node-installer/README.md | 17 +++++ node-installer/integration-test.sh | 98 +++++++++++++++++++++++++++ node-installer/kwasm-job.yml | 27 ++++++++ 5 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 node-installer/.kind/kind-config.yaml create mode 100755 node-installer/integration-test.sh create mode 100644 node-installer/kwasm-job.yml diff --git a/node-installer/.kind/kind-config.yaml b/node-installer/.kind/kind-config.yaml new file mode 100644 index 00000000..715752eb --- /dev/null +++ b/node-installer/.kind/kind-config.yaml @@ -0,0 +1,9 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: spin-test +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 80 + hostPort: 8080 + protocol: TCP diff --git a/node-installer/Makefile b/node-installer/Makefile index 68d08fd0..25dbc021 100644 --- a/node-installer/Makefile +++ b/node-installer/Makefile @@ -5,14 +5,17 @@ ARCH ?= x86_64 TARGET ?= $(ARCH)-unknown-linux-musl compile-musl: - make build-spin-cross-$(TARGET) -C ../ + make build-cross-$(TARGET) -C ../ move-musl-to-tmp: compile-musl mkdir -p ./.tmp - cp ../../containerd-shim-spin/target/$(TARGET)/release/containerd-shim-spin-$(SPIN_VERSION) ./.tmp/ + cp ../target/$(TARGET)/release/containerd-shim-spin-$(SPIN_VERSION) ./.tmp/$(PLATFORM)/ build-multi-installer-image: move-musl-to-tmp docker buildx build -t $(IMAGE_NAME) --platform linux/amd64,linux/arm64 . build-dev-installer-image: move-musl-to-tmp docker buildx build -t $(IMAGE_NAME) --load --platform $(PLATFORM) . + +test: + ./integration-test.sh diff --git a/node-installer/README.md b/node-installer/README.md index 957fcb72..7fa2fdca 100644 --- a/node-installer/README.md +++ b/node-installer/README.md @@ -7,3 +7,20 @@ which also bundles other shims. The intention is for the [spinkube/runtime-class-manager](https://github.com/spinkube/runtime-class-manager) project to handle this concern in the future. + +## Integration Test + +The `integration-test.sh` script is used to test the node-installer image. +It creates a kind cluster, applies the KWasm node installer job, and then tests +the workload. + +```bash +./integration-test.sh +``` + +## Build the Image Locally + +```bash +make build-dev-installer-image +``` + diff --git a/node-installer/integration-test.sh b/node-installer/integration-test.sh new file mode 100755 index 00000000..412a7ead --- /dev/null +++ b/node-installer/integration-test.sh @@ -0,0 +1,98 @@ +#!/bin/bash +set -euo pipefail + +echo "=== Step 1: Create a kind cluster ===" +if kind get clusters | grep -q "spin-test"; then + echo "Deleting existing cluster..." + kind delete cluster --name spin-test +fi + +echo "Creating kind cluster..." +kind create cluster --config .kind/kind-config.yaml +kubectl --context=kind-spin-test wait --for=condition=Ready nodes --all --timeout=90s + +echo "=== Step 2: Create namespace and deploy RuntimeClass ===" +kubectl --context=kind-spin-test create namespace kwasm || true +kubectl --context=kind-spin-test apply -f ../deployments/workloads/runtime.yaml + +echo "=== Step 3: Build and deploy the KWasm node installer ===" +if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:v0.18.0 >/dev/null 2>&1; then + echo "Building node installer image..." + PLATFORM=$(uname -m) + if [ "$PLATFORM" = "x86_64" ]; then + PLATFORM="linux/amd64" + ARCH="x86_64" + elif [ "$PLATFORM" = "aarch64" ] || [ "$PLATFORM" = "arm64" ]; then + PLATFORM="linux/arm64" + ARCH="aarch64" + else + echo "Unsupported platform: $PLATFORM" + exit 1 + fi + + PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev make build-dev-installer-image +fi + +echo "Loading node installer image into kind..." +kind load docker-image ghcr.io/spinkube/containerd-shim-spin/node-installer:dev --name spin-test + +echo "Applying KWasm node installer job..." +kubectl --context=kind-spin-test apply -f ./kwasm-job.yml + +echo "Waiting for node installer job to complete..." +kubectl --context=kind-spin-test wait -n kwasm --for=condition=Ready pod --selector=job-name=spin-test-control-plane-provision-kwasm --timeout=90s || true +kubectl --context=kind-spin-test wait -n kwasm --for=jsonpath='{.status.phase}'=Succeeded pod --selector=job-name=spin-test-control-plane-provision-kwasm --timeout=60s + +if ! kubectl --context=kind-spin-test get pods -n kwasm | grep -q "spin-test-control-plane-provision-kwasm.*Completed"; then + echo "Node installer job failed!" + kubectl --context=kind-spin-test logs -n kwasm $(kubectl --context=kind-spin-test get pods -n kwasm -o name | grep spin-test-control-plane-provision-kwasm) + exit 1 +fi + +echo "=== Step 4: Apply the workload ===" +kubectl --context=kind-spin-test apply -f ../deployments/workloads/workload.yaml + +echo "Waiting for deployment to be ready..." +kubectl --context=kind-spin-test wait --for=condition=Available deployment/wasm-spin --timeout=120s + +echo "Checking pod status..." +kubectl --context=kind-spin-test get pods + +echo "=== Step 5: Test the workload ===" +echo "Waiting for service to be ready..." +sleep 10 + +echo "Testing workload with curl..." +kubectl --context=kind-spin-test port-forward svc/wasm-spin 8888:80 & +FORWARD_PID=$! +sleep 5 + +MAX_RETRIES=3 +RETRY_COUNT=0 +SUCCESS=false + +while [ $RETRY_COUNT -lt $MAX_RETRIES ] && [ "$SUCCESS" = false ]; do + if curl -s http://localhost:8888/hello | grep -q "Hello world from Spin!"; then + SUCCESS=true + echo "Workload test successful!" + else + echo "Retrying in 3 seconds..." + sleep 3 + RETRY_COUNT=$((RETRY_COUNT+1)) + fi +done + +kill $FORWARD_PID + +if [ "$SUCCESS" = true ]; then + echo "=== Integration Test Passed! ===" + kind delete cluster --name spin-test + exit 0 +else + echo "=== Integration Test Failed! ===" + echo "Could not get a successful response from the workload." + kubectl --context=kind-spin-test describe pods + kubectl --context=kind-spin-test logs $(kubectl --context=kind-spin-test get pods -o name | grep wasm-spin) + kind delete cluster --name spin-test + exit 1 +fi \ No newline at end of file diff --git a/node-installer/kwasm-job.yml b/node-installer/kwasm-job.yml new file mode 100644 index 00000000..9ce112c9 --- /dev/null +++ b/node-installer/kwasm-job.yml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + job-name: spin-test-control-plane-provision-kwasm + name: spin-test-control-plane-provision-kwasm-dev + namespace: kwasm +spec: + containers: + - env: + - name: NODE_ROOT + value: /mnt/node-root + image: ghcr.io/spinkube/containerd-shim-spin/node-installer:dev + imagePullPolicy: IfNotPresent + name: kwasm-provision + securityContext: + privileged: true + volumeMounts: + - mountPath: /mnt/node-root + name: root-mount + hostPID: true + nodeName: spin-test-control-plane + restartPolicy: Never + volumes: + - hostPath: + path: / + name: root-mount \ No newline at end of file From aca6e023cf94148d6f5a60c19afdda6c84743d82 Mon Sep 17 00:00:00 2001 From: "Jiaxiao (mossaka) Zhou" Date: Thu, 13 Mar 2025 00:18:23 +0000 Subject: [PATCH 2/9] ci: Add integration test workflow for node-installer Signed-off-by: Jiaxiao (mossaka) Zhou --- .github/workflows/action-node-installer.yaml | 38 ++++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/.github/workflows/action-node-installer.yaml b/.github/workflows/action-node-installer.yaml index 9d603eee..80890c73 100644 --- a/.github/workflows/action-node-installer.yaml +++ b/.github/workflows/action-node-installer.yaml @@ -1,4 +1,4 @@ -name: Publish node-installer image +name: Build, Test, and Publish node-installer image on: workflow_call: @@ -11,7 +11,7 @@ on: jobs: # Note: assumes being called in a workflow where build has already run and # required artifacts have been uploaded - publish: + build-test-publish: permissions: contents: read packages: write @@ -38,15 +38,7 @@ jobs: - name: Setup buildx uses: docker/setup-buildx-action@v3 - - name: Login to GitHub container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # Build and push node-installer image - # TODO: remove once https://github.com/spinkube/runtime-class-manager handles this + # Build and extract node-installer image - name: Extract musl artifacts into ./node-installer/.tmp/linux/(amd64|arm64) dir run: | mkdir -p ./node-installer/.tmp/linux/amd64 @@ -54,6 +46,30 @@ jobs: for f in ./_artifacts/*/*-x86_64.tar.gz; do tar -xf $f --directory ./node-installer/.tmp/linux/amd64; done for f in ./_artifacts/*/*-aarch64.tar.gz; do tar -xf $f --directory ./node-installer/.tmp/linux/arm64; done + # Build local image for testing + - name: Build node-installer image for testing + run: make build-dev-installer-image + working-directory: node-installer + + # Install kind for integration testing + - name: Install kind + uses: helm/kind-action@v1.12.0 + with: + install_only: true + + # Run the integration test + - name: Run integration test + run: ./integration-test.sh + working-directory: node-installer + + # Login to registry and publish if tests pass + - name: Login to GitHub container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push node-installer image uses: docker/build-push-action@v5 with: From 54a1fafcac769d609115abb8641b471c7ca3ab71 Mon Sep 17 00:00:00 2001 From: "Jiaxiao (mossaka) Zhou" Date: Thu, 13 Mar 2025 00:31:42 +0000 Subject: [PATCH 3/9] node-installer: modify the makefile for binary handling Signed-off-by: Jiaxiao (mossaka) Zhou --- node-installer/Makefile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/node-installer/Makefile b/node-installer/Makefile index 25dbc021..b1a6a866 100644 --- a/node-installer/Makefile +++ b/node-installer/Makefile @@ -3,18 +3,20 @@ IMAGE_NAME ?= ghcr.io/spinkube/containerd-shim-spin/node-installer PLATFORM ?= linux/amd64 ARCH ?= x86_64 TARGET ?= $(ARCH)-unknown-linux-musl +BINARY := ./.tmp/$(PLATFORM)/containerd-shim-spin-$(SPIN_VERSION) compile-musl: make build-cross-$(TARGET) -C ../ -move-musl-to-tmp: compile-musl - mkdir -p ./.tmp - cp ../target/$(TARGET)/release/containerd-shim-spin-$(SPIN_VERSION) ./.tmp/$(PLATFORM)/ +$(BINARY): + mkdir -p ./.tmp/$(PLATFORM) + $(MAKE) compile-musl + cp ../target/$(TARGET)/release/containerd-shim-spin-$(SPIN_VERSION) $(BINARY) -build-multi-installer-image: move-musl-to-tmp +build-multi-installer-image: $(BINARY) docker buildx build -t $(IMAGE_NAME) --platform linux/amd64,linux/arm64 . -build-dev-installer-image: move-musl-to-tmp +build-dev-installer-image: $(BINARY) docker buildx build -t $(IMAGE_NAME) --load --platform $(PLATFORM) . test: From 63badf25e0ff2be01474d815d9266d855c8c7afc Mon Sep 17 00:00:00 2001 From: "Jiaxiao (mossaka) Zhou" Date: Thu, 13 Mar 2025 21:45:14 +0000 Subject: [PATCH 4/9] node-installer: add integration tests for multiple distributions This includes integration tests for Kind, MiniKube, MicroK8s, and K3s. Signed-off-by: Jiaxiao (mossaka) Zhou --- .github/workflows/action-node-installer.yaml | 83 ++++++++++--- node-installer/Makefile | 15 ++- node-installer/README.md | 13 +- node-installer/integration-test-k3s.sh | 111 +++++++++++++++++ ...ation-test.sh => integration-test-kind.sh} | 0 node-installer/integration-test-microk8s.sh | 116 ++++++++++++++++++ node-installer/integration-test-minikube.sh | 94 ++++++++++++++ 7 files changed, 405 insertions(+), 27 deletions(-) create mode 100755 node-installer/integration-test-k3s.sh rename node-installer/{integration-test.sh => integration-test-kind.sh} (100%) create mode 100755 node-installer/integration-test-microk8s.sh create mode 100755 node-installer/integration-test-minikube.sh diff --git a/.github/workflows/action-node-installer.yaml b/.github/workflows/action-node-installer.yaml index 80890c73..96b6100c 100644 --- a/.github/workflows/action-node-installer.yaml +++ b/.github/workflows/action-node-installer.yaml @@ -9,36 +9,39 @@ on: required: true jobs: - # Note: assumes being called in a workflow where build has already run and - # required artifacts have been uploaded - build-test-publish: + build-and-test: permissions: contents: read - packages: write runs-on: ubuntu-latest + strategy: + matrix: + distribution: [kind, minikube, microk8s, k3s] + outputs: + release_version: ${{ steps.set_version.outputs.RELEASE_VERSION }} steps: - uses: actions/checkout@v4 - name: Set RELEASE_VERSION env var + id: set_version run: | if [[ "${{ startsWith(inputs.ref, 'refs/tags/v')}}" == "true" ]]; then - echo "RELEASE_VERSION=$(echo -n ${{ inputs.ref }} | cut -d '/' -f 3)" >> $GITHUB_ENV + RELEASE_VERSION=$(echo -n ${{ inputs.ref }} | cut -d '/' -f 3) else - echo "RELEASE_VERSION=$(date +%Y%m%d-%H%M%S)-g$(git rev-parse --short HEAD)" >> $GITHUB_ENV + RELEASE_VERSION=$(date +%Y%m%d-%H%M%S)-g$(git rev-parse --short HEAD) fi + echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_ENV + echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_OUTPUT - uses: actions/download-artifact@v4 with: path: _artifacts - # Setup buildx to build multiarch image: https://github.com/docker/build-push-action/blob/master/docs/advanced/multi-platform.md - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Setup buildx uses: docker/setup-buildx-action@v3 - # Build and extract node-installer image - name: Extract musl artifacts into ./node-installer/.tmp/linux/(amd64|arm64) dir run: | mkdir -p ./node-installer/.tmp/linux/amd64 @@ -46,23 +49,67 @@ jobs: for f in ./_artifacts/*/*-x86_64.tar.gz; do tar -xf $f --directory ./node-installer/.tmp/linux/amd64; done for f in ./_artifacts/*/*-aarch64.tar.gz; do tar -xf $f --directory ./node-installer/.tmp/linux/arm64; done - # Build local image for testing - name: Build node-installer image for testing run: make build-dev-installer-image working-directory: node-installer - # Install kind for integration testing - - name: Install kind - uses: helm/kind-action@v1.12.0 + - uses: helm/kind-action@v1.12.0 + if: matrix.distribution == 'kind' with: install_only: true - - # Run the integration test - - name: Run integration test - run: ./integration-test.sh + + - uses: medyagh/setup-minikube@v0.0.19 + if: matrix.distribution == 'minikube' + + - uses: balchua/microk8s-actions@v0.4.3 + if: matrix.distribution == 'microk8s' + + - name: Run KIND test + if: matrix.distribution == 'kind' + run: make test-kind working-directory: node-installer - # Login to registry and publish if tests pass + - name: Run MiniKube test + if: matrix.distribution == 'minikube' + run: make test-minikube + working-directory: node-installer + + - name: Run MicroK8s test + if: matrix.distribution == 'microk8s' + run: make test-microk8s + working-directory: node-installer + + - name: Run K3s test + if: matrix.distribution == 'k3s' + run: make test-k3s + working-directory: node-installer + + publish: + needs: build-and-test + permissions: + contents: read + packages: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + path: _artifacts + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Setup buildx + uses: docker/setup-buildx-action@v3 + + - name: Extract musl artifacts into ./node-installer/.tmp/linux/(amd64|arm64) dir + run: | + mkdir -p ./node-installer/.tmp/linux/amd64 + mkdir -p ./node-installer/.tmp/linux/arm64 + for f in ./_artifacts/*/*-x86_64.tar.gz; do tar -xf $f --directory ./node-installer/.tmp/linux/amd64; done + for f in ./_artifacts/*/*-aarch64.tar.gz; do tar -xf $f --directory ./node-installer/.tmp/linux/arm64; done + - name: Login to GitHub container registry uses: docker/login-action@v3 with: @@ -75,7 +122,7 @@ jobs: with: push: true tags: | - ghcr.io/${{ github.repository }}/node-installer:${{ env.RELEASE_VERSION }} + ghcr.io/${{ github.repository }}/node-installer:${{ needs.build-and-test.outputs.release_version }} context: node-installer platforms: linux/amd64,linux/arm64 diff --git a/node-installer/Makefile b/node-installer/Makefile index b1a6a866..e242ed58 100644 --- a/node-installer/Makefile +++ b/node-installer/Makefile @@ -19,5 +19,16 @@ build-multi-installer-image: $(BINARY) build-dev-installer-image: $(BINARY) docker buildx build -t $(IMAGE_NAME) --load --platform $(PLATFORM) . -test: - ./integration-test.sh +test-kind: + ./integration-test-kind.sh + +test-minikube: + ./integration-test-minikube.sh + +test-microk8s: + ./integration-test-microk8s.sh + +test-k3s: + ./integration-test-k3s.sh + +test-all: test-kind test-minikube test-microk8s test-k3s diff --git a/node-installer/README.md b/node-installer/README.md index 7fa2fdca..39562435 100644 --- a/node-installer/README.md +++ b/node-installer/README.md @@ -8,15 +8,14 @@ which also bundles other shims. The intention is for the [spinkube/runtime-class-manager](https://github.com/spinkube/runtime-class-manager) project to handle this concern in the future. -## Integration Test +## Integration Tests -The `integration-test.sh` script is used to test the node-installer image. -It creates a kind cluster, applies the KWasm node installer job, and then tests -the workload. +The project includes integration test scripts for different Kubernetes distributions: -```bash -./integration-test.sh -``` +1. Kind: `make test-kind` +2. MiniKube: `make test-minikube` +3. MicroK8s: `make test-microk8s` +4. K3s: `make test-k3s` ## Build the Image Locally diff --git a/node-installer/integration-test-k3s.sh b/node-installer/integration-test-k3s.sh new file mode 100755 index 00000000..cc619197 --- /dev/null +++ b/node-installer/integration-test-k3s.sh @@ -0,0 +1,111 @@ +#!/bin/bash +set -euo pipefail + +echo "Installing K3s..." +curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable=traefik --write-kubeconfig-mode=644" sh - + +echo "Waiting for K3s to be ready..." +sleep 10 +export KUBECONFIG=/etc/rancher/k3s/k3s.yaml +until kubectl get nodes | grep -q " Ready"; do + echo "Waiting for node to be ready..." + sleep 5 +done + +echo "=== Step 2: Create namespace and deploy RuntimeClass ===" +kubectl create namespace kwasm || true +kubectl apply -f ../deployments/workloads/runtime.yaml + +echo "=== Step 3: Build and deploy the KWasm node installer ===" +if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:dev >/dev/null 2>&1; then + echo "Building node installer image..." + PLATFORM=$(uname -m) + if [ "$PLATFORM" = "x86_64" ]; then + PLATFORM="linux/amd64" + ARCH="x86_64" + elif [ "$PLATFORM" = "aarch64" ] || [ "$PLATFORM" = "arm64" ]; then + PLATFORM="linux/arm64" + ARCH="aarch64" + else + echo "Unsupported platform: $PLATFORM" + exit 1 + fi + + PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev make build-dev-installer-image +fi + +echo "Loading node installer image into K3s..." +docker save ghcr.io/spinkube/containerd-shim-spin/node-installer:dev > node-installer.tar +sudo k3s ctr images import node-installer.tar +rm node-installer.tar + +NODE_NAME=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') +cp kwasm-job.yml k3s-kwasm-job.yml +sed -i "s/spin-test-control-plane-provision-kwasm/k3s-provision-kwasm/g" k3s-kwasm-job.yml +sed -i "s/spin-test-control-plane-provision-kwasm-dev/k3s-provision-kwasm-dev/g" k3s-kwasm-job.yml +sed -i "s/spin-test-control-plane/${NODE_NAME}/g" k3s-kwasm-job.yml + +echo "Applying KWasm node installer job..." +kubectl apply -f ./k3s-kwasm-job.yml + +echo "Waiting for node installer job to complete..." +kubectl wait -n kwasm --for=condition=Ready pod --selector=job-name=k3s-provision-kwasm --timeout=90s || true +kubectl wait -n kwasm --for=jsonpath='{.status.phase}'=Succeeded pod --selector=job-name=k3s-provision-kwasm --timeout=60s + +if ! kubectl get pods -n kwasm | grep -q "k3s-provision-kwasm.*Completed"; then + echo "Node installer job failed!" + kubectl logs -n kwasm $(kubectl get pods -n kwasm -o name | grep k3s-provision-kwasm) + exit 1 +fi + +echo "=== Step 4: Apply the workload ===" +kubectl apply -f ../deployments/workloads/workload.yaml + +echo "Waiting for deployment to be ready..." +kubectl wait --for=condition=Available deployment/wasm-spin --timeout=120s + +echo "Checking pod status..." +kubectl get pods + +echo "=== Step 5: Test the workload ===" +echo "Waiting for service to be ready..." +sleep 10 + +echo "Testing workload with curl..." +kubectl port-forward svc/wasm-spin 8888:80 & +FORWARD_PID=$! +sleep 5 + +MAX_RETRIES=3 +RETRY_COUNT=0 +SUCCESS=false + +while [ $RETRY_COUNT -lt $MAX_RETRIES ] && [ "$SUCCESS" = false ]; do + if curl -s http://localhost:8888/hello | grep -q "Hello world from Spin!"; then + SUCCESS=true + echo "Workload test successful!" + else + echo "Retrying in 3 seconds..." + sleep 3 + RETRY_COUNT=$((RETRY_COUNT+1)) + fi +done + +kill $FORWARD_PID || true + +if [ "$SUCCESS" = true ]; then + echo "=== Integration Test Passed! ===" + sudo /usr/local/bin/k3s-uninstall.sh + sudo rm -rf /etc/rancher/k3s + sudo rm -rf /var/lib/rancher/k3s + exit 0 +else + echo "=== Integration Test Failed! ===" + echo "Could not get a successful response from the workload." + kubectl describe pods + kubectl logs $(kubectl get pods -o name | grep wasm-spin) + sudo /usr/local/bin/k3s-uninstall.sh + sudo rm -rf /etc/rancher/k3s + sudo rm -rf /var/lib/rancher/k3s + exit 1 +fi \ No newline at end of file diff --git a/node-installer/integration-test.sh b/node-installer/integration-test-kind.sh similarity index 100% rename from node-installer/integration-test.sh rename to node-installer/integration-test-kind.sh diff --git a/node-installer/integration-test-microk8s.sh b/node-installer/integration-test-microk8s.sh new file mode 100755 index 00000000..458809b8 --- /dev/null +++ b/node-installer/integration-test-microk8s.sh @@ -0,0 +1,116 @@ +#!/bin/bash +set -euo pipefail + +echo "=== Step 1: Setup MicroK8s ===" +if ! command -v microk8s >/dev/null 2>&1; then + echo "MicroK8s is not installed. Please install it first." + exit 1 +fi + +if ! microk8s status | grep -q "microk8s is running"; then + echo "Starting MicroK8s..." + microk8s start + sleep 10 +else + microk8s reset + sleep 10 +fi + +microk8s enable dns + +alias kubectl='microk8s kubectl' + +echo "=== Step 2: Create namespace and deploy RuntimeClass ===" +kubectl create namespace kwasm || true +kubectl apply -f ../deployments/workloads/runtime.yaml + +echo "=== Step 3: Build and deploy the KWasm node installer ===" +if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:dev >/dev/null 2>&1; then + echo "Building node installer image..." + PLATFORM=$(uname -m) + if [ "$PLATFORM" = "x86_64" ]; then + PLATFORM="linux/amd64" + ARCH="x86_64" + elif [ "$PLATFORM" = "aarch64" ] || [ "$PLATFORM" = "arm64" ]; then + PLATFORM="linux/arm64" + ARCH="aarch64" + else + echo "Unsupported platform: $PLATFORM" + exit 1 + fi + + PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev make build-dev-installer-image +fi + +echo "Loading node installer image into MicroK8s..." +docker save ghcr.io/spinkube/containerd-shim-spin/node-installer:dev > node-installer.tar +microk8s ctr image import node-installer.tar +rm node-installer.tar + +NODE_NAME=$(microk8s kubectl get nodes -o jsonpath='{.items[0].metadata.name}') +cp kwasm-job.yml microk8s-kwasm-job.yml +sed -i "s/spin-test-control-plane-provision-kwasm/microk8s-provision-kwasm/g" microk8s-kwasm-job.yml +sed -i "s/spin-test-control-plane-provision-kwasm-dev/microk8s-provision-kwasm-dev/g" microk8s-kwasm-job.yml +sed -i "s/spin-test-control-plane/${NODE_NAME}/g" microk8s-kwasm-job.yml + +echo "Applying KWasm node installer job..." +kubectl apply -f ./microk8s-kwasm-job.yml + +echo "Waiting for node installer job to complete..." +kubectl wait -n kwasm --for=condition=Ready pod --selector=job-name=microk8s-provision-kwasm --timeout=90s || true +kubectl wait -n kwasm --for=jsonpath='{.status.phase}'=Succeeded pod --selector=job-name=microk8s-provision-kwasm --timeout=60s + +if ! kubectl get pods -n kwasm | grep -q "microk8s-provision-kwasm.*Completed"; then + echo "Node installer job failed!" + kubectl logs -n kwasm $(kubectl get pods -n kwasm -o name | grep microk8s-provision-kwasm) + exit 1 +fi + +echo "=== Step 4: Apply the workload ===" +kubectl apply -f ../deployments/workloads/workload.yaml + +echo "Waiting for deployment to be ready..." +kubectl wait --for=condition=Available deployment/wasm-spin --timeout=120s + +echo "Checking pod status..." +kubectl get pods + +echo "=== Step 5: Test the workload ===" +echo "Waiting for service to be ready..." +sleep 10 + +microk8s enable ingress +sleep 5 + +echo "Testing workload with curl..." +kubectl port-forward svc/wasm-spin 8888:80 & +FORWARD_PID=$! +sleep 5 + +MAX_RETRIES=3 +RETRY_COUNT=0 +SUCCESS=false + +while [ $RETRY_COUNT -lt $MAX_RETRIES ] && [ "$SUCCESS" = false ]; do + if curl -s http://localhost:8888/hello | grep -q "Hello world from Spin!"; then + SUCCESS=true + echo "Workload test successful!" + else + echo "Retrying in 3 seconds..." + sleep 3 + RETRY_COUNT=$((RETRY_COUNT+1)) + fi +done + +kill $FORWARD_PID || true + +if [ "$SUCCESS" = true ]; then + echo "=== Integration Test Passed! ===" + exit 0 +else + echo "=== Integration Test Failed! ===" + echo "Could not get a successful response from the workload." + kubectl describe pods + kubectl logs $(kubectl get pods -o name | grep wasm-spin) + exit 1 +fi \ No newline at end of file diff --git a/node-installer/integration-test-minikube.sh b/node-installer/integration-test-minikube.sh new file mode 100755 index 00000000..514c38bc --- /dev/null +++ b/node-installer/integration-test-minikube.sh @@ -0,0 +1,94 @@ +#!/bin/bash +set -euo pipefail + +echo "=== Step 1: Create a MiniKube cluster ===" +minikube start -p spin-minikube --driver=docker --container-runtime=containerd + +echo "=== Step 2: Create namespace and deploy RuntimeClass ===" +kubectl create namespace kwasm || true +kubectl apply -f ../deployments/workloads/runtime.yaml + +echo "=== Step 3: Build and deploy the KWasm node installer ===" +if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:dev >/dev/null 2>&1; then + echo "Building node installer image..." + PLATFORM=$(uname -m) + if [ "$PLATFORM" = "x86_64" ]; then + PLATFORM="linux/amd64" + ARCH="x86_64" + elif [ "$PLATFORM" = "aarch64" ] || [ "$PLATFORM" = "arm64" ]; then + PLATFORM="linux/arm64" + ARCH="aarch64" + else + echo "Unsupported platform: $PLATFORM" + exit 1 + fi + + PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev make build-dev-installer-image +fi + +echo "Loading node installer image into MiniKube..." +minikube image load ghcr.io/spinkube/containerd-shim-spin/node-installer:dev -p spin-minikube + +NODE_NAME=$(kubectl get nodes --context=minikube -o jsonpath='{.items[0].metadata.name}') +cp kwasm-job.yml minikube-kwasm-job.yml +sed -i "s/spin-test-control-plane-provision-kwasm/spin-minikube-provision-kwasm/g" minikube-kwasm-job.yml +sed -i "s/spin-test-control-plane-provision-kwasm-dev/spin-minikube-provision-kwasm-dev/g" minikube-kwasm-job.yml +sed -i "s/spin-test-control-plane/${NODE_NAME}/g" minikube-kwasm-job.yml + +echo "Applying KWasm node installer job..." +kubectl apply -f ./minikube-kwasm-job.yml + +echo "Waiting for node installer job to complete..." +kubectl wait -n kwasm --for=condition=Ready pod --selector=job-name=spin-minikube-provision-kwasm --timeout=90s || true +kubectl wait -n kwasm --for=jsonpath='{.status.phase}'=Succeeded pod --selector=job-name=spin-minikube-provision-kwasm --timeout=60s + +if ! kubectl get pods -n kwasm | grep -q "spin-minikube-provision-kwasm.*Completed"; then + echo "Node installer job failed!" + kubectl logs -n kwasm $(kubectl get pods -n kwasm -o name | grep spin-minikube-provision-kwasm) + exit 1 +fi + +echo "=== Step 4: Apply the workload ===" +kubectl apply -f ../deployments/workloads/workload.yaml + +echo "Waiting for deployment to be ready..." +kubectl wait --for=condition=Available deployment/wasm-spin --timeout=120s + +echo "Checking pod status..." +kubectl get pods + +echo "=== Step 5: Test the workload ===" +echo "Waiting for service to be ready..." +sleep 10 + +echo "Testing workload with curl..." +minikube service wasm-spin --url -p spin-minikube > service_url.txt +SERVICE_URL=$(cat service_url.txt) + +MAX_RETRIES=3 +RETRY_COUNT=0 +SUCCESS=false + +while [ $RETRY_COUNT -lt $MAX_RETRIES ] && [ "$SUCCESS" = false ]; do + if curl -s $SERVICE_URL/hello | grep -q "Hello world from Spin!"; then + SUCCESS=true + echo "Workload test successful!" + else + echo "Retrying in 3 seconds..." + sleep 3 + RETRY_COUNT=$((RETRY_COUNT+1)) + fi +done + +if [ "$SUCCESS" = true ]; then + echo "=== Integration Test Passed! ===" + minikube delete -p spin-minikube + exit 0 +else + echo "=== Integration Test Failed! ===" + echo "Could not get a successful response from the workload." + kubectl describe pods + kubectl logs $(kubectl get pods -o name | grep wasm-spin) + minikube delete -p spin-minikube + exit 1 +fi \ No newline at end of file From c68b1c22b50b4fcfa09ff689db476460f1f6255f Mon Sep 17 00:00:00 2001 From: "Jiaxiao (mossaka) Zhou" Date: Thu, 13 Mar 2025 22:41:27 +0000 Subject: [PATCH 5/9] node-installer: make integration tests work Signed-off-by: Jiaxiao (mossaka) Zhou --- .github/workflows/action-node-installer.yaml | 10 ++++++ node-installer/integration-test-k3s.sh | 9 +++-- node-installer/integration-test-kind.sh | 8 +++-- node-installer/integration-test-microk8s.sh | 22 ++++++------ node-installer/integration-test-minikube.sh | 35 ++++++++++++-------- node-installer/k3s-kwasm-job.yml | 27 +++++++++++++++ node-installer/script/installer.sh | 13 +++++++- 7 files changed, 93 insertions(+), 31 deletions(-) create mode 100644 node-installer/k3s-kwasm-job.yml diff --git a/.github/workflows/action-node-installer.yaml b/.github/workflows/action-node-installer.yaml index 96b6100c..773c5539 100644 --- a/.github/workflows/action-node-installer.yaml +++ b/.github/workflows/action-node-installer.yaml @@ -14,6 +14,7 @@ jobs: contents: read runs-on: ubuntu-latest strategy: + fail-fast: false matrix: distribution: [kind, minikube, microk8s, k3s] outputs: @@ -60,6 +61,9 @@ jobs: - uses: medyagh/setup-minikube@v0.0.19 if: matrix.distribution == 'minikube' + with: + start: false + container-runtime: containerd - uses: balchua/microk8s-actions@v0.4.3 if: matrix.distribution == 'microk8s' @@ -83,6 +87,12 @@ jobs: if: matrix.distribution == 'k3s' run: make test-k3s working-directory: node-installer + + - name: Collect k3s debug logs + if: matrix.distribution == 'k3s' && failure() + run: | + sudo k3s kubectl describe pods -n kwasm + sudo k3s kubectl describe pods publish: needs: build-and-test diff --git a/node-installer/integration-test-k3s.sh b/node-installer/integration-test-k3s.sh index cc619197..1f407135 100755 --- a/node-installer/integration-test-k3s.sh +++ b/node-installer/integration-test-k3s.sh @@ -1,6 +1,8 @@ #!/bin/bash set -euo pipefail +: ${IMAGE_NAME:=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev} + echo "Installing K3s..." curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable=traefik --write-kubeconfig-mode=644" sh - @@ -17,7 +19,7 @@ kubectl create namespace kwasm || true kubectl apply -f ../deployments/workloads/runtime.yaml echo "=== Step 3: Build and deploy the KWasm node installer ===" -if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:dev >/dev/null 2>&1; then +if ! docker image inspect $IMAGE_NAME >/dev/null 2>&1; then echo "Building node installer image..." PLATFORM=$(uname -m) if [ "$PLATFORM" = "x86_64" ]; then @@ -31,11 +33,11 @@ if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:d exit 1 fi - PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev make build-dev-installer-image + PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=$IMAGE_NAME make build-dev-installer-image fi echo "Loading node installer image into K3s..." -docker save ghcr.io/spinkube/containerd-shim-spin/node-installer:dev > node-installer.tar +docker save $IMAGE_NAME > node-installer.tar sudo k3s ctr images import node-installer.tar rm node-installer.tar @@ -59,6 +61,7 @@ if ! kubectl get pods -n kwasm | grep -q "k3s-provision-kwasm.*Completed"; then fi echo "=== Step 4: Apply the workload ===" +sudo k3s ctr images pull ghcr.io/spinkube/containerd-shim-spin/examples/spin-rust-hello:v0.18.0 kubectl apply -f ../deployments/workloads/workload.yaml echo "Waiting for deployment to be ready..." diff --git a/node-installer/integration-test-kind.sh b/node-installer/integration-test-kind.sh index 412a7ead..55c7ea91 100755 --- a/node-installer/integration-test-kind.sh +++ b/node-installer/integration-test-kind.sh @@ -1,6 +1,8 @@ #!/bin/bash set -euo pipefail +: ${IMAGE_NAME:=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev} + echo "=== Step 1: Create a kind cluster ===" if kind get clusters | grep -q "spin-test"; then echo "Deleting existing cluster..." @@ -16,7 +18,7 @@ kubectl --context=kind-spin-test create namespace kwasm || true kubectl --context=kind-spin-test apply -f ../deployments/workloads/runtime.yaml echo "=== Step 3: Build and deploy the KWasm node installer ===" -if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:v0.18.0 >/dev/null 2>&1; then +if ! docker image inspect $IMAGE_NAME >/dev/null 2>&1; then echo "Building node installer image..." PLATFORM=$(uname -m) if [ "$PLATFORM" = "x86_64" ]; then @@ -30,11 +32,11 @@ if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:v exit 1 fi - PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev make build-dev-installer-image + PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=$IMAGE_NAME make build-dev-installer-image fi echo "Loading node installer image into kind..." -kind load docker-image ghcr.io/spinkube/containerd-shim-spin/node-installer:dev --name spin-test +kind load docker-image $IMAGE_NAME --name spin-test echo "Applying KWasm node installer job..." kubectl --context=kind-spin-test apply -f ./kwasm-job.yml diff --git a/node-installer/integration-test-microk8s.sh b/node-installer/integration-test-microk8s.sh index 458809b8..118ab5df 100755 --- a/node-installer/integration-test-microk8s.sh +++ b/node-installer/integration-test-microk8s.sh @@ -1,6 +1,8 @@ #!/bin/bash set -euo pipefail +: ${IMAGE_NAME:=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev} + echo "=== Step 1: Setup MicroK8s ===" if ! command -v microk8s >/dev/null 2>&1; then echo "MicroK8s is not installed. Please install it first." @@ -9,23 +11,23 @@ fi if ! microk8s status | grep -q "microk8s is running"; then echo "Starting MicroK8s..." - microk8s start + sudo microk8s start sleep 10 else - microk8s reset + sudo microk8s reset sleep 10 fi -microk8s enable dns +sudo microk8s enable dns -alias kubectl='microk8s kubectl' +alias kubectl='sudo microk8s kubectl' echo "=== Step 2: Create namespace and deploy RuntimeClass ===" kubectl create namespace kwasm || true kubectl apply -f ../deployments/workloads/runtime.yaml echo "=== Step 3: Build and deploy the KWasm node installer ===" -if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:dev >/dev/null 2>&1; then +if ! docker image inspect $IMAGE_NAME >/dev/null 2>&1; then echo "Building node installer image..." PLATFORM=$(uname -m) if [ "$PLATFORM" = "x86_64" ]; then @@ -39,15 +41,15 @@ if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:d exit 1 fi - PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev make build-dev-installer-image + PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=$IMAGE_NAME make build-dev-installer-image fi echo "Loading node installer image into MicroK8s..." -docker save ghcr.io/spinkube/containerd-shim-spin/node-installer:dev > node-installer.tar -microk8s ctr image import node-installer.tar +docker save $IMAGE_NAME > node-installer.tar +sudo microk8s ctr image import node-installer.tar rm node-installer.tar -NODE_NAME=$(microk8s kubectl get nodes -o jsonpath='{.items[0].metadata.name}') +NODE_NAME=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') cp kwasm-job.yml microk8s-kwasm-job.yml sed -i "s/spin-test-control-plane-provision-kwasm/microk8s-provision-kwasm/g" microk8s-kwasm-job.yml sed -i "s/spin-test-control-plane-provision-kwasm-dev/microk8s-provision-kwasm-dev/g" microk8s-kwasm-job.yml @@ -79,7 +81,7 @@ echo "=== Step 5: Test the workload ===" echo "Waiting for service to be ready..." sleep 10 -microk8s enable ingress +sudo microk8s enable ingress sleep 5 echo "Testing workload with curl..." diff --git a/node-installer/integration-test-minikube.sh b/node-installer/integration-test-minikube.sh index 514c38bc..e08570a4 100755 --- a/node-installer/integration-test-minikube.sh +++ b/node-installer/integration-test-minikube.sh @@ -1,15 +1,17 @@ #!/bin/bash set -euo pipefail +: ${IMAGE_NAME:=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev} + echo "=== Step 1: Create a MiniKube cluster ===" -minikube start -p spin-minikube --driver=docker --container-runtime=containerd +minikube start -p minikube --driver=docker --container-runtime=containerd echo "=== Step 2: Create namespace and deploy RuntimeClass ===" kubectl create namespace kwasm || true kubectl apply -f ../deployments/workloads/runtime.yaml echo "=== Step 3: Build and deploy the KWasm node installer ===" -if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:dev >/dev/null 2>&1; then +if ! docker image inspect $IMAGE_NAME >/dev/null 2>&1; then echo "Building node installer image..." PLATFORM=$(uname -m) if [ "$PLATFORM" = "x86_64" ]; then @@ -23,28 +25,28 @@ if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:d exit 1 fi - PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev make build-dev-installer-image + PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=$IMAGE_NAME make build-dev-installer-image fi echo "Loading node installer image into MiniKube..." -minikube image load ghcr.io/spinkube/containerd-shim-spin/node-installer:dev -p spin-minikube +minikube image load $IMAGE_NAME -p minikube NODE_NAME=$(kubectl get nodes --context=minikube -o jsonpath='{.items[0].metadata.name}') cp kwasm-job.yml minikube-kwasm-job.yml -sed -i "s/spin-test-control-plane-provision-kwasm/spin-minikube-provision-kwasm/g" minikube-kwasm-job.yml -sed -i "s/spin-test-control-plane-provision-kwasm-dev/spin-minikube-provision-kwasm-dev/g" minikube-kwasm-job.yml +sed -i "s/spin-test-control-plane-provision-kwasm/minikube-provision-kwasm/g" minikube-kwasm-job.yml +sed -i "s/spin-test-control-plane-provision-kwasm-dev/minikube-provision-kwasm-dev/g" minikube-kwasm-job.yml sed -i "s/spin-test-control-plane/${NODE_NAME}/g" minikube-kwasm-job.yml echo "Applying KWasm node installer job..." kubectl apply -f ./minikube-kwasm-job.yml echo "Waiting for node installer job to complete..." -kubectl wait -n kwasm --for=condition=Ready pod --selector=job-name=spin-minikube-provision-kwasm --timeout=90s || true -kubectl wait -n kwasm --for=jsonpath='{.status.phase}'=Succeeded pod --selector=job-name=spin-minikube-provision-kwasm --timeout=60s +kubectl wait -n kwasm --for=condition=Ready pod --selector=job-name=minikube-provision-kwasm --timeout=90s || true +kubectl wait -n kwasm --for=jsonpath='{.status.phase}'=Succeeded pod --selector=job-name=minikube-provision-kwasm --timeout=60s -if ! kubectl get pods -n kwasm | grep -q "spin-minikube-provision-kwasm.*Completed"; then +if ! kubectl get pods -n kwasm | grep -q "minikube-provision-kwasm.*Completed"; then echo "Node installer job failed!" - kubectl logs -n kwasm $(kubectl get pods -n kwasm -o name | grep spin-minikube-provision-kwasm) + kubectl logs -n kwasm $(kubectl get pods -n kwasm -o name | grep minikube-provision-kwasm) exit 1 fi @@ -62,9 +64,12 @@ echo "Waiting for service to be ready..." sleep 10 echo "Testing workload with curl..." -minikube service wasm-spin --url -p spin-minikube > service_url.txt -SERVICE_URL=$(cat service_url.txt) +PORT=8080 +kubectl port-forward service/wasm-spin $PORT:80 & +PORT_FORWARD_PID=$! +sleep 10 +SERVICE_URL="http://localhost:$PORT" MAX_RETRIES=3 RETRY_COUNT=0 SUCCESS=false @@ -80,15 +85,17 @@ while [ $RETRY_COUNT -lt $MAX_RETRIES ] && [ "$SUCCESS" = false ]; do fi done +kill $PORT_FORWARD_PID 2>/dev/null || true + if [ "$SUCCESS" = true ]; then echo "=== Integration Test Passed! ===" - minikube delete -p spin-minikube + minikube delete -p minikube exit 0 else echo "=== Integration Test Failed! ===" echo "Could not get a successful response from the workload." kubectl describe pods kubectl logs $(kubectl get pods -o name | grep wasm-spin) - minikube delete -p spin-minikube + minikube delete -p minikube exit 1 fi \ No newline at end of file diff --git a/node-installer/k3s-kwasm-job.yml b/node-installer/k3s-kwasm-job.yml new file mode 100644 index 00000000..2d741796 --- /dev/null +++ b/node-installer/k3s-kwasm-job.yml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + job-name: k3s-provision-kwasm + name: k3s-provision-kwasm-dev + namespace: kwasm +spec: + containers: + - env: + - name: NODE_ROOT + value: /mnt/node-root + image: ghcr.io/spinkube/containerd-shim-spin/node-installer:dev + imagePullPolicy: IfNotPresent + name: kwasm-provision + securityContext: + privileged: true + volumeMounts: + - mountPath: /mnt/node-root + name: root-mount + hostPID: true + nodeName: devbox-5 + restartPolicy: Never + volumes: + - hostPath: + path: / + name: root-mount \ No newline at end of file diff --git a/node-installer/script/installer.sh b/node-installer/script/installer.sh index b8786014..08f7ec0e 100644 --- a/node-installer/script/installer.sh +++ b/node-installer/script/installer.sh @@ -39,13 +39,24 @@ mkdir -p $NODE_ROOT$KWASM_DIR/bin/ cp /assets/containerd-shim-spin-v2 $NODE_ROOT$KWASM_DIR/bin/ if ! grep -q spin $NODE_ROOT$CONTAINERD_CONF; then - echo ' + if $IS_K3S; then + echo ' +[plugins."io.containerd.cri.v1.runtime".containerd.runtimes."spin"] + runtime_type = "'$KWASM_DIR'/bin/containerd-shim-spin-v2" +' >> $NODE_ROOT$CONTAINERD_CONF + else + echo ' [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin] runtime_type = "'$KWASM_DIR'/bin/containerd-shim-spin-v2" ' >> $NODE_ROOT$CONTAINERD_CONF + fi rm -Rf $NODE_ROOT$KWASM_DIR/active fi +if $IS_K3S; then + sed -i "s|runtime_type = \"io.containerd.spin.*\"|runtime_type = \"$KWASM_DIR/bin/containerd-shim-spin-v2\"|g" $NODE_ROOT$CONTAINERD_CONF +fi + if [ ! -f $NODE_ROOT$KWASM_DIR/active ]; then touch $NODE_ROOT$KWASM_DIR/active if $IS_MICROK8S; then From 336a8ef13ea208abe186fb1e4fa8bcb504226186 Mon Sep 17 00:00:00 2001 From: "Jiaxiao (mossaka) Zhou" Date: Thu, 20 Mar 2025 18:35:00 +0000 Subject: [PATCH 6/9] node-installer: remove accidentally checked in yaml file Signed-off-by: Jiaxiao (mossaka) Zhou --- node-installer/k3s-kwasm-job.yml | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 node-installer/k3s-kwasm-job.yml diff --git a/node-installer/k3s-kwasm-job.yml b/node-installer/k3s-kwasm-job.yml deleted file mode 100644 index 2d741796..00000000 --- a/node-installer/k3s-kwasm-job.yml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - labels: - job-name: k3s-provision-kwasm - name: k3s-provision-kwasm-dev - namespace: kwasm -spec: - containers: - - env: - - name: NODE_ROOT - value: /mnt/node-root - image: ghcr.io/spinkube/containerd-shim-spin/node-installer:dev - imagePullPolicy: IfNotPresent - name: kwasm-provision - securityContext: - privileged: true - volumeMounts: - - mountPath: /mnt/node-root - name: root-mount - hostPID: true - nodeName: devbox-5 - restartPolicy: Never - volumes: - - hostPath: - path: / - name: root-mount \ No newline at end of file From 768d6dc948ad1f477c17ce0f5b1aa8d9dd26883f Mon Sep 17 00:00:00 2001 From: "Jiaxiao (mossaka) Zhou" Date: Thu, 20 Mar 2025 18:35:23 +0000 Subject: [PATCH 7/9] node-installer: moved kind-config.yaml to inline in the script Signed-off-by: Jiaxiao (mossaka) Zhou --- node-installer/.kind/kind-config.yaml | 9 --------- node-installer/integration-test-kind.sh | 12 +++++++++++- 2 files changed, 11 insertions(+), 10 deletions(-) delete mode 100644 node-installer/.kind/kind-config.yaml diff --git a/node-installer/.kind/kind-config.yaml b/node-installer/.kind/kind-config.yaml deleted file mode 100644 index 715752eb..00000000 --- a/node-installer/.kind/kind-config.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -name: spin-test -nodes: -- role: control-plane - extraPortMappings: - - containerPort: 80 - hostPort: 8080 - protocol: TCP diff --git a/node-installer/integration-test-kind.sh b/node-installer/integration-test-kind.sh index 55c7ea91..f9898f60 100755 --- a/node-installer/integration-test-kind.sh +++ b/node-installer/integration-test-kind.sh @@ -10,7 +10,17 @@ if kind get clusters | grep -q "spin-test"; then fi echo "Creating kind cluster..." -kind create cluster --config .kind/kind-config.yaml +kind create cluster --config - << EOF +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: spin-test +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 80 + hostPort: 8080 + protocol: TCP +EOF kubectl --context=kind-spin-test wait --for=condition=Ready nodes --all --timeout=90s echo "=== Step 2: Create namespace and deploy RuntimeClass ===" From 9fee7492942dc0a3450f0947ef25a460367367e8 Mon Sep 17 00:00:00 2001 From: "Jiaxiao (mossaka) Zhou" Date: Thu, 20 Mar 2025 18:50:04 +0000 Subject: [PATCH 8/9] node-installer: move things around - move the testing scripts to the `node-installer/tests/` directory - move the kwasm-job.yml to the `node-installer/tests/workloads/` directory - symlink the workload.yaml and runtime.yaml from the root `deployments/workloads/` to the `node-installer/tests/workloads/` directory Signed-off-by: Jiaxiao (mossaka) Zhou --- node-installer/Makefile | 8 +++--- node-installer/README.md | 2 +- node-installer/k3s-kwasm-job.yml | 27 +++++++++++++++++++ .../{ => tests}/integration-test-k3s.sh | 6 ++--- .../{ => tests}/integration-test-kind.sh | 6 ++--- .../{ => tests}/integration-test-microk8s.sh | 6 ++--- .../{ => tests}/integration-test-minikube.sh | 6 ++--- .../{ => tests/workloads}/kwasm-job.yml | 0 node-installer/tests/workloads/runtime.yaml | 1 + node-installer/tests/workloads/workload.yaml | 1 + 10 files changed, 46 insertions(+), 17 deletions(-) create mode 100644 node-installer/k3s-kwasm-job.yml rename node-installer/{ => tests}/integration-test-k3s.sh (95%) rename node-installer/{ => tests}/integration-test-kind.sh (93%) rename node-installer/{ => tests}/integration-test-microk8s.sh (95%) rename node-installer/{ => tests}/integration-test-minikube.sh (95%) rename node-installer/{ => tests/workloads}/kwasm-job.yml (100%) create mode 120000 node-installer/tests/workloads/runtime.yaml create mode 120000 node-installer/tests/workloads/workload.yaml diff --git a/node-installer/Makefile b/node-installer/Makefile index e242ed58..51016d35 100644 --- a/node-installer/Makefile +++ b/node-installer/Makefile @@ -20,15 +20,15 @@ build-dev-installer-image: $(BINARY) docker buildx build -t $(IMAGE_NAME) --load --platform $(PLATFORM) . test-kind: - ./integration-test-kind.sh + ./tests/integration-test-kind.sh test-minikube: - ./integration-test-minikube.sh + ./tests/integration-test-minikube.sh test-microk8s: - ./integration-test-microk8s.sh + ./tests/integration-test-microk8s.sh test-k3s: - ./integration-test-k3s.sh + ./tests/integration-test-k3s.sh test-all: test-kind test-minikube test-microk8s test-k3s diff --git a/node-installer/README.md b/node-installer/README.md index 39562435..896faf57 100644 --- a/node-installer/README.md +++ b/node-installer/README.md @@ -10,7 +10,7 @@ project to handle this concern in the future. ## Integration Tests -The project includes integration test scripts for different Kubernetes distributions: +The project includes integration test scripts for different Kubernetes distributions in the `tests/` directory: 1. Kind: `make test-kind` 2. MiniKube: `make test-minikube` diff --git a/node-installer/k3s-kwasm-job.yml b/node-installer/k3s-kwasm-job.yml new file mode 100644 index 00000000..2d741796 --- /dev/null +++ b/node-installer/k3s-kwasm-job.yml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + job-name: k3s-provision-kwasm + name: k3s-provision-kwasm-dev + namespace: kwasm +spec: + containers: + - env: + - name: NODE_ROOT + value: /mnt/node-root + image: ghcr.io/spinkube/containerd-shim-spin/node-installer:dev + imagePullPolicy: IfNotPresent + name: kwasm-provision + securityContext: + privileged: true + volumeMounts: + - mountPath: /mnt/node-root + name: root-mount + hostPID: true + nodeName: devbox-5 + restartPolicy: Never + volumes: + - hostPath: + path: / + name: root-mount \ No newline at end of file diff --git a/node-installer/integration-test-k3s.sh b/node-installer/tests/integration-test-k3s.sh similarity index 95% rename from node-installer/integration-test-k3s.sh rename to node-installer/tests/integration-test-k3s.sh index 1f407135..4290c72f 100755 --- a/node-installer/integration-test-k3s.sh +++ b/node-installer/tests/integration-test-k3s.sh @@ -16,7 +16,7 @@ done echo "=== Step 2: Create namespace and deploy RuntimeClass ===" kubectl create namespace kwasm || true -kubectl apply -f ../deployments/workloads/runtime.yaml +kubectl apply -f ./tests/workloads/runtime.yaml echo "=== Step 3: Build and deploy the KWasm node installer ===" if ! docker image inspect $IMAGE_NAME >/dev/null 2>&1; then @@ -42,7 +42,7 @@ sudo k3s ctr images import node-installer.tar rm node-installer.tar NODE_NAME=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') -cp kwasm-job.yml k3s-kwasm-job.yml +cp ./tests/workloads/kwasm-job.yml k3s-kwasm-job.yml sed -i "s/spin-test-control-plane-provision-kwasm/k3s-provision-kwasm/g" k3s-kwasm-job.yml sed -i "s/spin-test-control-plane-provision-kwasm-dev/k3s-provision-kwasm-dev/g" k3s-kwasm-job.yml sed -i "s/spin-test-control-plane/${NODE_NAME}/g" k3s-kwasm-job.yml @@ -62,7 +62,7 @@ fi echo "=== Step 4: Apply the workload ===" sudo k3s ctr images pull ghcr.io/spinkube/containerd-shim-spin/examples/spin-rust-hello:v0.18.0 -kubectl apply -f ../deployments/workloads/workload.yaml +kubectl apply -f ./tests/workloads/workload.yaml echo "Waiting for deployment to be ready..." kubectl wait --for=condition=Available deployment/wasm-spin --timeout=120s diff --git a/node-installer/integration-test-kind.sh b/node-installer/tests/integration-test-kind.sh similarity index 93% rename from node-installer/integration-test-kind.sh rename to node-installer/tests/integration-test-kind.sh index f9898f60..51c5d50e 100755 --- a/node-installer/integration-test-kind.sh +++ b/node-installer/tests/integration-test-kind.sh @@ -25,7 +25,7 @@ kubectl --context=kind-spin-test wait --for=condition=Ready nodes --all --timeou echo "=== Step 2: Create namespace and deploy RuntimeClass ===" kubectl --context=kind-spin-test create namespace kwasm || true -kubectl --context=kind-spin-test apply -f ../deployments/workloads/runtime.yaml +kubectl --context=kind-spin-test apply -f ./tests/workloads/runtime.yaml echo "=== Step 3: Build and deploy the KWasm node installer ===" if ! docker image inspect $IMAGE_NAME >/dev/null 2>&1; then @@ -49,7 +49,7 @@ echo "Loading node installer image into kind..." kind load docker-image $IMAGE_NAME --name spin-test echo "Applying KWasm node installer job..." -kubectl --context=kind-spin-test apply -f ./kwasm-job.yml +kubectl --context=kind-spin-test apply -f ./tests/workloads/kwasm-job.yml echo "Waiting for node installer job to complete..." kubectl --context=kind-spin-test wait -n kwasm --for=condition=Ready pod --selector=job-name=spin-test-control-plane-provision-kwasm --timeout=90s || true @@ -62,7 +62,7 @@ if ! kubectl --context=kind-spin-test get pods -n kwasm | grep -q "spin-test-con fi echo "=== Step 4: Apply the workload ===" -kubectl --context=kind-spin-test apply -f ../deployments/workloads/workload.yaml +kubectl --context=kind-spin-test apply -f ./tests/workloads/workload.yaml echo "Waiting for deployment to be ready..." kubectl --context=kind-spin-test wait --for=condition=Available deployment/wasm-spin --timeout=120s diff --git a/node-installer/integration-test-microk8s.sh b/node-installer/tests/integration-test-microk8s.sh similarity index 95% rename from node-installer/integration-test-microk8s.sh rename to node-installer/tests/integration-test-microk8s.sh index 118ab5df..2618766e 100755 --- a/node-installer/integration-test-microk8s.sh +++ b/node-installer/tests/integration-test-microk8s.sh @@ -24,7 +24,7 @@ alias kubectl='sudo microk8s kubectl' echo "=== Step 2: Create namespace and deploy RuntimeClass ===" kubectl create namespace kwasm || true -kubectl apply -f ../deployments/workloads/runtime.yaml +kubectl apply -f ./tests/workloads/runtime.yaml echo "=== Step 3: Build and deploy the KWasm node installer ===" if ! docker image inspect $IMAGE_NAME >/dev/null 2>&1; then @@ -50,7 +50,7 @@ sudo microk8s ctr image import node-installer.tar rm node-installer.tar NODE_NAME=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') -cp kwasm-job.yml microk8s-kwasm-job.yml +cp ./tests/workloads/kwasm-job.yml microk8s-kwasm-job.yml sed -i "s/spin-test-control-plane-provision-kwasm/microk8s-provision-kwasm/g" microk8s-kwasm-job.yml sed -i "s/spin-test-control-plane-provision-kwasm-dev/microk8s-provision-kwasm-dev/g" microk8s-kwasm-job.yml sed -i "s/spin-test-control-plane/${NODE_NAME}/g" microk8s-kwasm-job.yml @@ -69,7 +69,7 @@ if ! kubectl get pods -n kwasm | grep -q "microk8s-provision-kwasm.*Completed"; fi echo "=== Step 4: Apply the workload ===" -kubectl apply -f ../deployments/workloads/workload.yaml +kubectl apply -f ./tests/workloads/workload.yaml echo "Waiting for deployment to be ready..." kubectl wait --for=condition=Available deployment/wasm-spin --timeout=120s diff --git a/node-installer/integration-test-minikube.sh b/node-installer/tests/integration-test-minikube.sh similarity index 95% rename from node-installer/integration-test-minikube.sh rename to node-installer/tests/integration-test-minikube.sh index e08570a4..8a0e81bd 100755 --- a/node-installer/integration-test-minikube.sh +++ b/node-installer/tests/integration-test-minikube.sh @@ -8,7 +8,7 @@ minikube start -p minikube --driver=docker --container-runtime=containerd echo "=== Step 2: Create namespace and deploy RuntimeClass ===" kubectl create namespace kwasm || true -kubectl apply -f ../deployments/workloads/runtime.yaml +kubectl apply -f ./tests/workloads/runtime.yaml echo "=== Step 3: Build and deploy the KWasm node installer ===" if ! docker image inspect $IMAGE_NAME >/dev/null 2>&1; then @@ -32,7 +32,7 @@ echo "Loading node installer image into MiniKube..." minikube image load $IMAGE_NAME -p minikube NODE_NAME=$(kubectl get nodes --context=minikube -o jsonpath='{.items[0].metadata.name}') -cp kwasm-job.yml minikube-kwasm-job.yml +cp ./tests/workloads/kwasm-job.yml minikube-kwasm-job.yml sed -i "s/spin-test-control-plane-provision-kwasm/minikube-provision-kwasm/g" minikube-kwasm-job.yml sed -i "s/spin-test-control-plane-provision-kwasm-dev/minikube-provision-kwasm-dev/g" minikube-kwasm-job.yml sed -i "s/spin-test-control-plane/${NODE_NAME}/g" minikube-kwasm-job.yml @@ -51,7 +51,7 @@ if ! kubectl get pods -n kwasm | grep -q "minikube-provision-kwasm.*Completed"; fi echo "=== Step 4: Apply the workload ===" -kubectl apply -f ../deployments/workloads/workload.yaml +kubectl apply -f ./tests/workloads/workload.yaml echo "Waiting for deployment to be ready..." kubectl wait --for=condition=Available deployment/wasm-spin --timeout=120s diff --git a/node-installer/kwasm-job.yml b/node-installer/tests/workloads/kwasm-job.yml similarity index 100% rename from node-installer/kwasm-job.yml rename to node-installer/tests/workloads/kwasm-job.yml diff --git a/node-installer/tests/workloads/runtime.yaml b/node-installer/tests/workloads/runtime.yaml new file mode 120000 index 00000000..669d18f6 --- /dev/null +++ b/node-installer/tests/workloads/runtime.yaml @@ -0,0 +1 @@ +../../../deployments/workloads/runtime.yaml \ No newline at end of file diff --git a/node-installer/tests/workloads/workload.yaml b/node-installer/tests/workloads/workload.yaml new file mode 120000 index 00000000..3d5475c1 --- /dev/null +++ b/node-installer/tests/workloads/workload.yaml @@ -0,0 +1 @@ +../../../deployments/workloads/workload.yaml \ No newline at end of file From ecbee0d2245372dfff446481a9bd10282c1f7231 Mon Sep 17 00:00:00 2001 From: "Jiaxiao (mossaka) Zhou" Date: Thu, 20 Mar 2025 18:53:07 +0000 Subject: [PATCH 9/9] node-installer: remove k3s-kwasm-job.yml file Signed-off-by: Jiaxiao (mossaka) Zhou --- node-installer/k3s-kwasm-job.yml | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 node-installer/k3s-kwasm-job.yml diff --git a/node-installer/k3s-kwasm-job.yml b/node-installer/k3s-kwasm-job.yml deleted file mode 100644 index 2d741796..00000000 --- a/node-installer/k3s-kwasm-job.yml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - labels: - job-name: k3s-provision-kwasm - name: k3s-provision-kwasm-dev - namespace: kwasm -spec: - containers: - - env: - - name: NODE_ROOT - value: /mnt/node-root - image: ghcr.io/spinkube/containerd-shim-spin/node-installer:dev - imagePullPolicy: IfNotPresent - name: kwasm-provision - securityContext: - privileged: true - volumeMounts: - - mountPath: /mnt/node-root - name: root-mount - hostPID: true - nodeName: devbox-5 - restartPolicy: Never - volumes: - - hostPath: - path: / - name: root-mount \ No newline at end of file