Skip to content

Commit e50bf66

Browse files
MossakaMossaka
committed
node-installer: add integration tests for multiple distributions
This includes integration tests for Kind, MiniKube, MicroK8s, and K3s. Signed-off-by: Jiaxiao (mossaka) Zhou <[email protected]>
1 parent 54a1faf commit e50bf66

File tree

7 files changed

+405
-27
lines changed

7 files changed

+405
-27
lines changed

‎.github/workflows/action-node-installer.yaml‎

Lines changed: 65 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -9,60 +9,107 @@ on:
99
required: true
1010

1111
jobs:
12-
# Note: assumes being called in a workflow where build has already run and
13-
# required artifacts have been uploaded
14-
build-test-publish:
12+
build-and-test:
1513
permissions:
1614
contents: read
17-
packages: write
1815
runs-on: ubuntu-latest
16+
strategy:
17+
matrix:
18+
distribution: [kind, minikube, microk8s, k3s]
19+
outputs:
20+
release_version: ${{ steps.set_version.outputs.RELEASE_VERSION }}
1921
steps:
2022
- uses: actions/checkout@v4
2123

2224
- name: Set RELEASE_VERSION env var
25+
id: set_version
2326
run: |
2427
if [[ "${{ startsWith(inputs.ref, 'refs/tags/v')}}" == "true" ]]; then
25-
echo "RELEASE_VERSION=$(echo -n ${{ inputs.ref }} | cut -d '/' -f 3)" >> $GITHUB_ENV
28+
RELEASE_VERSION=$(echo -n ${{ inputs.ref }} | cut -d '/' -f 3)
2629
else
27-
echo "RELEASE_VERSION=$(date +%Y%m%d-%H%M%S)-g$(git rev-parse --short HEAD)" >> $GITHUB_ENV
30+
RELEASE_VERSION=$(date +%Y%m%d-%H%M%S)-g$(git rev-parse --short HEAD)
2831
fi
32+
echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_ENV
33+
echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_OUTPUT
2934
3035
- uses: actions/download-artifact@v4
3136
with:
3237
path: _artifacts
3338

34-
# Setup buildx to build multiarch image: https://github.com/docker/build-push-action/blob/master/docs/advanced/multi-platform.md
3539
- name: Set up QEMU
3640
uses: docker/setup-qemu-action@v3
3741

3842
- name: Setup buildx
3943
uses: docker/setup-buildx-action@v3
4044

41-
# Build and extract node-installer image
4245
- name: Extract musl artifacts into ./node-installer/.tmp/linux/(amd64|arm64) dir
4346
run: |
4447
mkdir -p ./node-installer/.tmp/linux/amd64
4548
mkdir -p ./node-installer/.tmp/linux/arm64
4649
for f in ./_artifacts/*/*-x86_64.tar.gz; do tar -xf $f --directory ./node-installer/.tmp/linux/amd64; done
4750
for f in ./_artifacts/*/*-aarch64.tar.gz; do tar -xf $f --directory ./node-installer/.tmp/linux/arm64; done
4851
49-
# Build local image for testing
5052
- name: Build node-installer image for testing
5153
run: make build-dev-installer-image
5254
working-directory: node-installer
5355

54-
# Install kind for integration testing
55-
- name: Install kind
56-
uses: helm/[email protected]
56+
- uses: helm/[email protected]
57+
if: matrix.distribution == 'kind'
5758
with:
5859
install_only: true
59-
60-
# Run the integration test
61-
- name: Run integration test
62-
run: ./integration-test.sh
60+
61+
- uses: medyagh/[email protected]
62+
if: matrix.distribution == 'minikube'
63+
64+
- uses: balchua/[email protected]
65+
if: matrix.distribution == 'microk8s'
66+
67+
- name: Run KIND test
68+
if: matrix.distribution == 'kind'
69+
run: make test-kind
6370
working-directory: node-installer
6471

65-
# Login to registry and publish if tests pass
72+
- name: Run MiniKube test
73+
if: matrix.distribution == 'minikube'
74+
run: make test-minikube
75+
working-directory: node-installer
76+
77+
- name: Run MicroK8s test
78+
if: matrix.distribution == 'microk8s'
79+
run: make test-microk8s
80+
working-directory: node-installer
81+
82+
- name: Run K3s test
83+
if: matrix.distribution == 'k3s'
84+
run: make test-k3s
85+
working-directory: node-installer
86+
87+
publish:
88+
needs: build-and-test
89+
permissions:
90+
contents: read
91+
packages: write
92+
runs-on: ubuntu-latest
93+
steps:
94+
- uses: actions/checkout@v4
95+
96+
- uses: actions/download-artifact@v4
97+
with:
98+
path: _artifacts
99+
100+
- name: Set up QEMU
101+
uses: docker/setup-qemu-action@v3
102+
103+
- name: Setup buildx
104+
uses: docker/setup-buildx-action@v3
105+
106+
- name: Extract musl artifacts into ./node-installer/.tmp/linux/(amd64|arm64) dir
107+
run: |
108+
mkdir -p ./node-installer/.tmp/linux/amd64
109+
mkdir -p ./node-installer/.tmp/linux/arm64
110+
for f in ./_artifacts/*/*-x86_64.tar.gz; do tar -xf $f --directory ./node-installer/.tmp/linux/amd64; done
111+
for f in ./_artifacts/*/*-aarch64.tar.gz; do tar -xf $f --directory ./node-installer/.tmp/linux/arm64; done
112+
66113
- name: Login to GitHub container registry
67114
uses: docker/login-action@v3
68115
with:
@@ -75,7 +122,7 @@ jobs:
75122
with:
76123
push: true
77124
tags: |
78-
ghcr.io/${{ github.repository }}/node-installer:${{ env.RELEASE_VERSION }}
125+
ghcr.io/${{ github.repository }}/node-installer:${{ needs.build-and-test.outputs.release_version }}
79126
context: node-installer
80127
platforms: linux/amd64,linux/arm64
81128

‎node-installer/Makefile‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,16 @@ build-multi-installer-image: $(BINARY)
1919
build-dev-installer-image: $(BINARY)
2020
docker buildx build -t $(IMAGE_NAME) --load --platform $(PLATFORM) .
2121

22-
test:
23-
./integration-test.sh
22+
test-kind:
23+
./integration-test-kind.sh
24+
25+
test-minikube:
26+
./integration-test-minikube.sh
27+
28+
test-microk8s:
29+
./integration-test-microk8s.sh
30+
31+
test-k3s:
32+
./integration-test-k3s.sh
33+
34+
test-all: test-kind test-minikube test-microk8s test-k3s

‎node-installer/README.md‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,14 @@ which also bundles other shims.
88
The intention is for the [spinkube/runtime-class-manager](https://github.com/spinkube/runtime-class-manager)
99
project to handle this concern in the future.
1010

11-
## Integration Test
11+
## Integration Tests
1212

13-
The `integration-test.sh` script is used to test the node-installer image.
14-
It creates a kind cluster, applies the KWasm node installer job, and then tests
15-
the workload.
13+
The project includes integration test scripts for different Kubernetes distributions:
1614

17-
```bash
18-
./integration-test.sh
19-
```
15+
1. Kind: `make test-kind`
16+
2. MiniKube: `make test-minikube`
17+
3. MicroK8s: `make test-microk8s`
18+
4. K3s: `make test-k3s`
2019

2120
## Build the Image Locally
2221

‎node-installer/integration-test-k3s.sh‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
echo "Installing K3s..."
5+
curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable=traefik --write-kubeconfig-mode=644" sh -
6+
7+
echo "Waiting for K3s to be ready..."
8+
sleep 10
9+
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
10+
until kubectl get nodes | grep -q " Ready"; do
11+
echo "Waiting for node to be ready..."
12+
sleep 5
13+
done
14+
15+
echo "=== Step 2: Create namespace and deploy RuntimeClass ==="
16+
kubectl create namespace kwasm || true
17+
kubectl apply -f ../deployments/workloads/runtime.yaml
18+
19+
echo "=== Step 3: Build and deploy the KWasm node installer ==="
20+
if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:dev >/dev/null 2>&1; then
21+
echo "Building node installer image..."
22+
PLATFORM=$(uname -m)
23+
if [ "$PLATFORM" = "x86_64" ]; then
24+
PLATFORM="linux/amd64"
25+
ARCH="x86_64"
26+
elif [ "$PLATFORM" = "aarch64" ] || [ "$PLATFORM" = "arm64" ]; then
27+
PLATFORM="linux/arm64"
28+
ARCH="aarch64"
29+
else
30+
echo "Unsupported platform: $PLATFORM"
31+
exit 1
32+
fi
33+
34+
PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev make build-dev-installer-image
35+
fi
36+
37+
echo "Loading node installer image into K3s..."
38+
docker save ghcr.io/spinkube/containerd-shim-spin/node-installer:dev > node-installer.tar
39+
sudo k3s ctr images import node-installer.tar
40+
rm node-installer.tar
41+
42+
NODE_NAME=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}')
43+
cp kwasm-job.yml k3s-kwasm-job.yml
44+
sed -i "s/spin-test-control-plane-provision-kwasm/k3s-provision-kwasm/g" k3s-kwasm-job.yml
45+
sed -i "s/spin-test-control-plane-provision-kwasm-dev/k3s-provision-kwasm-dev/g" k3s-kwasm-job.yml
46+
sed -i "s/spin-test-control-plane/${NODE_NAME}/g" k3s-kwasm-job.yml
47+
48+
echo "Applying KWasm node installer job..."
49+
kubectl apply -f ./k3s-kwasm-job.yml
50+
51+
echo "Waiting for node installer job to complete..."
52+
kubectl wait -n kwasm --for=condition=Ready pod --selector=job-name=k3s-provision-kwasm --timeout=90s || true
53+
kubectl wait -n kwasm --for=jsonpath='{.status.phase}'=Succeeded pod --selector=job-name=k3s-provision-kwasm --timeout=60s
54+
55+
if ! kubectl get pods -n kwasm | grep -q "k3s-provision-kwasm.*Completed"; then
56+
echo "Node installer job failed!"
57+
kubectl logs -n kwasm $(kubectl get pods -n kwasm -o name | grep k3s-provision-kwasm)
58+
exit 1
59+
fi
60+
61+
echo "=== Step 4: Apply the workload ==="
62+
kubectl apply -f ../deployments/workloads/workload.yaml
63+
64+
echo "Waiting for deployment to be ready..."
65+
kubectl wait --for=condition=Available deployment/wasm-spin --timeout=120s
66+
67+
echo "Checking pod status..."
68+
kubectl get pods
69+
70+
echo "=== Step 5: Test the workload ==="
71+
echo "Waiting for service to be ready..."
72+
sleep 10
73+
74+
echo "Testing workload with curl..."
75+
kubectl port-forward svc/wasm-spin 8888:80 &
76+
FORWARD_PID=$!
77+
sleep 5
78+
79+
MAX_RETRIES=3
80+
RETRY_COUNT=0
81+
SUCCESS=false
82+
83+
while [ $RETRY_COUNT -lt $MAX_RETRIES ] && [ "$SUCCESS" = false ]; do
84+
if curl -s http://localhost:8888/hello | grep -q "Hello world from Spin!"; then
85+
SUCCESS=true
86+
echo "Workload test successful!"
87+
else
88+
echo "Retrying in 3 seconds..."
89+
sleep 3
90+
RETRY_COUNT=$((RETRY_COUNT+1))
91+
fi
92+
done
93+
94+
kill $FORWARD_PID || true
95+
96+
if [ "$SUCCESS" = true ]; then
97+
echo "=== Integration Test Passed! ==="
98+
sudo /usr/local/bin/k3s-uninstall.sh
99+
sudo rm -rf /etc/rancher/k3s
100+
sudo rm -rf /var/lib/rancher/k3s
101+
exit 0
102+
else
103+
echo "=== Integration Test Failed! ==="
104+
echo "Could not get a successful response from the workload."
105+
kubectl describe pods
106+
kubectl logs $(kubectl get pods -o name | grep wasm-spin)
107+
sudo /usr/local/bin/k3s-uninstall.sh
108+
sudo rm -rf /etc/rancher/k3s
109+
sudo rm -rf /var/lib/rancher/k3s
110+
exit 1
111+
fi

‎node-installer/integration-test.sh‎ renamed to ‎node-installer/integration-test-kind.sh‎

File renamed without changes.

0 commit comments

Comments
 (0)