Skip to content

Commit c68b1c2

Browse files
MossakaMossaka
committed
node-installer: make integration tests work
Signed-off-by: Jiaxiao (mossaka) Zhou <[email protected]>
1 parent 63badf2 commit c68b1c2

File tree

7 files changed

+93
-31
lines changed

7 files changed

+93
-31
lines changed

‎.github/workflows/action-node-installer.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ jobs:
1414
contents: read
1515
runs-on: ubuntu-latest
1616
strategy:
17+
fail-fast: false
1718
matrix:
1819
distribution: [kind, minikube, microk8s, k3s]
1920
outputs:
@@ -60,6 +61,9 @@ jobs:
6061

6162
- uses: medyagh/[email protected]
6263
if: matrix.distribution == 'minikube'
64+
with:
65+
start: false
66+
container-runtime: containerd
6367

6468
- uses: balchua/[email protected]
6569
if: matrix.distribution == 'microk8s'
@@ -83,6 +87,12 @@ jobs:
8387
if: matrix.distribution == 'k3s'
8488
run: make test-k3s
8589
working-directory: node-installer
90+
91+
- name: Collect k3s debug logs
92+
if: matrix.distribution == 'k3s' && failure()
93+
run: |
94+
sudo k3s kubectl describe pods -n kwasm
95+
sudo k3s kubectl describe pods
8696
8797
publish:
8898
needs: build-and-test

‎node-installer/integration-test-k3s.sh‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
set -euo pipefail
33

4+
: ${IMAGE_NAME:=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev}
5+
46
echo "Installing K3s..."
57
curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable=traefik --write-kubeconfig-mode=644" sh -
68

@@ -17,7 +19,7 @@ kubectl create namespace kwasm || true
1719
kubectl apply -f ../deployments/workloads/runtime.yaml
1820

1921
echo "=== Step 3: Build and deploy the KWasm node installer ==="
20-
if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:dev >/dev/null 2>&1; then
22+
if ! docker image inspect $IMAGE_NAME >/dev/null 2>&1; then
2123
echo "Building node installer image..."
2224
PLATFORM=$(uname -m)
2325
if [ "$PLATFORM" = "x86_64" ]; then
@@ -31,11 +33,11 @@ if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:d
3133
exit 1
3234
fi
3335

34-
PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev make build-dev-installer-image
36+
PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=$IMAGE_NAME make build-dev-installer-image
3537
fi
3638

3739
echo "Loading node installer image into K3s..."
38-
docker save ghcr.io/spinkube/containerd-shim-spin/node-installer:dev > node-installer.tar
40+
docker save $IMAGE_NAME > node-installer.tar
3941
sudo k3s ctr images import node-installer.tar
4042
rm node-installer.tar
4143

@@ -59,6 +61,7 @@ if ! kubectl get pods -n kwasm | grep -q "k3s-provision-kwasm.*Completed"; then
5961
fi
6062

6163
echo "=== Step 4: Apply the workload ==="
64+
sudo k3s ctr images pull ghcr.io/spinkube/containerd-shim-spin/examples/spin-rust-hello:v0.18.0
6265
kubectl apply -f ../deployments/workloads/workload.yaml
6366

6467
echo "Waiting for deployment to be ready..."

‎node-installer/integration-test-kind.sh‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
set -euo pipefail
33

4+
: ${IMAGE_NAME:=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev}
5+
46
echo "=== Step 1: Create a kind cluster ==="
57
if kind get clusters | grep -q "spin-test"; then
68
echo "Deleting existing cluster..."
@@ -16,7 +18,7 @@ kubectl --context=kind-spin-test create namespace kwasm || true
1618
kubectl --context=kind-spin-test apply -f ../deployments/workloads/runtime.yaml
1719

1820
echo "=== Step 3: Build and deploy the KWasm node installer ==="
19-
if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:v0.18.0 >/dev/null 2>&1; then
21+
if ! docker image inspect $IMAGE_NAME >/dev/null 2>&1; then
2022
echo "Building node installer image..."
2123
PLATFORM=$(uname -m)
2224
if [ "$PLATFORM" = "x86_64" ]; then
@@ -30,11 +32,11 @@ if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:v
3032
exit 1
3133
fi
3234

33-
PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev make build-dev-installer-image
35+
PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=$IMAGE_NAME make build-dev-installer-image
3436
fi
3537

3638
echo "Loading node installer image into kind..."
37-
kind load docker-image ghcr.io/spinkube/containerd-shim-spin/node-installer:dev --name spin-test
39+
kind load docker-image $IMAGE_NAME --name spin-test
3840

3941
echo "Applying KWasm node installer job..."
4042
kubectl --context=kind-spin-test apply -f ./kwasm-job.yml

‎node-installer/integration-test-microk8s.sh‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
22
set -euo pipefail
33

4+
: ${IMAGE_NAME:=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev}
5+
46
echo "=== Step 1: Setup MicroK8s ==="
57
if ! command -v microk8s >/dev/null 2>&1; then
68
echo "MicroK8s is not installed. Please install it first."
@@ -9,23 +11,23 @@ fi
911

1012
if ! microk8s status | grep -q "microk8s is running"; then
1113
echo "Starting MicroK8s..."
12-
microk8s start
14+
sudo microk8s start
1315
sleep 10
1416
else
15-
microk8s reset
17+
sudo microk8s reset
1618
sleep 10
1719
fi
1820

19-
microk8s enable dns
21+
sudo microk8s enable dns
2022

21-
alias kubectl='microk8s kubectl'
23+
alias kubectl='sudo microk8s kubectl'
2224

2325
echo "=== Step 2: Create namespace and deploy RuntimeClass ==="
2426
kubectl create namespace kwasm || true
2527
kubectl apply -f ../deployments/workloads/runtime.yaml
2628

2729
echo "=== Step 3: Build and deploy the KWasm node installer ==="
28-
if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:dev >/dev/null 2>&1; then
30+
if ! docker image inspect $IMAGE_NAME >/dev/null 2>&1; then
2931
echo "Building node installer image..."
3032
PLATFORM=$(uname -m)
3133
if [ "$PLATFORM" = "x86_64" ]; then
@@ -39,15 +41,15 @@ if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:d
3941
exit 1
4042
fi
4143

42-
PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev make build-dev-installer-image
44+
PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=$IMAGE_NAME make build-dev-installer-image
4345
fi
4446

4547
echo "Loading node installer image into MicroK8s..."
46-
docker save ghcr.io/spinkube/containerd-shim-spin/node-installer:dev > node-installer.tar
47-
microk8s ctr image import node-installer.tar
48+
docker save $IMAGE_NAME > node-installer.tar
49+
sudo microk8s ctr image import node-installer.tar
4850
rm node-installer.tar
4951

50-
NODE_NAME=$(microk8s kubectl get nodes -o jsonpath='{.items[0].metadata.name}')
52+
NODE_NAME=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}')
5153
cp kwasm-job.yml microk8s-kwasm-job.yml
5254
sed -i "s/spin-test-control-plane-provision-kwasm/microk8s-provision-kwasm/g" microk8s-kwasm-job.yml
5355
sed -i "s/spin-test-control-plane-provision-kwasm-dev/microk8s-provision-kwasm-dev/g" microk8s-kwasm-job.yml
@@ -79,7 +81,7 @@ echo "=== Step 5: Test the workload ==="
7981
echo "Waiting for service to be ready..."
8082
sleep 10
8183

82-
microk8s enable ingress
84+
sudo microk8s enable ingress
8385
sleep 5
8486

8587
echo "Testing workload with curl..."

‎node-installer/integration-test-minikube.sh‎

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
11
#!/bin/bash
22
set -euo pipefail
33

4+
: ${IMAGE_NAME:=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev}
5+
46
echo "=== Step 1: Create a MiniKube cluster ==="
5-
minikube start -p spin-minikube --driver=docker --container-runtime=containerd
7+
minikube start -p minikube --driver=docker --container-runtime=containerd
68

79
echo "=== Step 2: Create namespace and deploy RuntimeClass ==="
810
kubectl create namespace kwasm || true
911
kubectl apply -f ../deployments/workloads/runtime.yaml
1012

1113
echo "=== Step 3: Build and deploy the KWasm node installer ==="
12-
if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:dev >/dev/null 2>&1; then
14+
if ! docker image inspect $IMAGE_NAME >/dev/null 2>&1; then
1315
echo "Building node installer image..."
1416
PLATFORM=$(uname -m)
1517
if [ "$PLATFORM" = "x86_64" ]; then
@@ -23,28 +25,28 @@ if ! docker image inspect ghcr.io/spinkube/containerd-shim-spin/node-installer:d
2325
exit 1
2426
fi
2527

26-
PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev make build-dev-installer-image
28+
PLATFORM=$PLATFORM ARCH=$ARCH IMAGE_NAME=$IMAGE_NAME make build-dev-installer-image
2729
fi
2830

2931
echo "Loading node installer image into MiniKube..."
30-
minikube image load ghcr.io/spinkube/containerd-shim-spin/node-installer:dev -p spin-minikube
32+
minikube image load $IMAGE_NAME -p minikube
3133

3234
NODE_NAME=$(kubectl get nodes --context=minikube -o jsonpath='{.items[0].metadata.name}')
3335
cp kwasm-job.yml minikube-kwasm-job.yml
34-
sed -i "s/spin-test-control-plane-provision-kwasm/spin-minikube-provision-kwasm/g" minikube-kwasm-job.yml
35-
sed -i "s/spin-test-control-plane-provision-kwasm-dev/spin-minikube-provision-kwasm-dev/g" minikube-kwasm-job.yml
36+
sed -i "s/spin-test-control-plane-provision-kwasm/minikube-provision-kwasm/g" minikube-kwasm-job.yml
37+
sed -i "s/spin-test-control-plane-provision-kwasm-dev/minikube-provision-kwasm-dev/g" minikube-kwasm-job.yml
3638
sed -i "s/spin-test-control-plane/${NODE_NAME}/g" minikube-kwasm-job.yml
3739

3840
echo "Applying KWasm node installer job..."
3941
kubectl apply -f ./minikube-kwasm-job.yml
4042

4143
echo "Waiting for node installer job to complete..."
42-
kubectl wait -n kwasm --for=condition=Ready pod --selector=job-name=spin-minikube-provision-kwasm --timeout=90s || true
43-
kubectl wait -n kwasm --for=jsonpath='{.status.phase}'=Succeeded pod --selector=job-name=spin-minikube-provision-kwasm --timeout=60s
44+
kubectl wait -n kwasm --for=condition=Ready pod --selector=job-name=minikube-provision-kwasm --timeout=90s || true
45+
kubectl wait -n kwasm --for=jsonpath='{.status.phase}'=Succeeded pod --selector=job-name=minikube-provision-kwasm --timeout=60s
4446

45-
if ! kubectl get pods -n kwasm | grep -q "spin-minikube-provision-kwasm.*Completed"; then
47+
if ! kubectl get pods -n kwasm | grep -q "minikube-provision-kwasm.*Completed"; then
4648
echo "Node installer job failed!"
47-
kubectl logs -n kwasm $(kubectl get pods -n kwasm -o name | grep spin-minikube-provision-kwasm)
49+
kubectl logs -n kwasm $(kubectl get pods -n kwasm -o name | grep minikube-provision-kwasm)
4850
exit 1
4951
fi
5052

@@ -62,9 +64,12 @@ echo "Waiting for service to be ready..."
6264
sleep 10
6365

6466
echo "Testing workload with curl..."
65-
minikube service wasm-spin --url -p spin-minikube > service_url.txt
66-
SERVICE_URL=$(cat service_url.txt)
67+
PORT=8080
68+
kubectl port-forward service/wasm-spin $PORT:80 &
69+
PORT_FORWARD_PID=$!
70+
sleep 10
6771

72+
SERVICE_URL="http://localhost:$PORT"
6873
MAX_RETRIES=3
6974
RETRY_COUNT=0
7075
SUCCESS=false
@@ -80,15 +85,17 @@ while [ $RETRY_COUNT -lt $MAX_RETRIES ] && [ "$SUCCESS" = false ]; do
8085
fi
8186
done
8287

88+
kill $PORT_FORWARD_PID 2>/dev/null || true
89+
8390
if [ "$SUCCESS" = true ]; then
8491
echo "=== Integration Test Passed! ==="
85-
minikube delete -p spin-minikube
92+
minikube delete -p minikube
8693
exit 0
8794
else
8895
echo "=== Integration Test Failed! ==="
8996
echo "Could not get a successful response from the workload."
9097
kubectl describe pods
9198
kubectl logs $(kubectl get pods -o name | grep wasm-spin)
92-
minikube delete -p spin-minikube
99+
minikube delete -p minikube
93100
exit 1
94101
fi

‎node-installer/k3s-kwasm-job.yml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
apiVersion: v1
2+
kind: Pod
3+
metadata:
4+
labels:
5+
job-name: k3s-provision-kwasm
6+
name: k3s-provision-kwasm-dev
7+
namespace: kwasm
8+
spec:
9+
containers:
10+
- env:
11+
- name: NODE_ROOT
12+
value: /mnt/node-root
13+
image: ghcr.io/spinkube/containerd-shim-spin/node-installer:dev
14+
imagePullPolicy: IfNotPresent
15+
name: kwasm-provision
16+
securityContext:
17+
privileged: true
18+
volumeMounts:
19+
- mountPath: /mnt/node-root
20+
name: root-mount
21+
hostPID: true
22+
nodeName: devbox-5
23+
restartPolicy: Never
24+
volumes:
25+
- hostPath:
26+
path: /
27+
name: root-mount

‎node-installer/script/installer.sh‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,13 +39,24 @@ mkdir -p $NODE_ROOT$KWASM_DIR/bin/
3939
cp /assets/containerd-shim-spin-v2 $NODE_ROOT$KWASM_DIR/bin/
4040

4141
if ! grep -q spin $NODE_ROOT$CONTAINERD_CONF; then
42-
echo '
42+
if $IS_K3S; then
43+
echo '
44+
[plugins."io.containerd.cri.v1.runtime".containerd.runtimes."spin"]
45+
runtime_type = "'$KWASM_DIR'/bin/containerd-shim-spin-v2"
46+
' >> $NODE_ROOT$CONTAINERD_CONF
47+
else
48+
echo '
4349
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin]
4450
runtime_type = "'$KWASM_DIR'/bin/containerd-shim-spin-v2"
4551
' >> $NODE_ROOT$CONTAINERD_CONF
52+
fi
4653
rm -Rf $NODE_ROOT$KWASM_DIR/active
4754
fi
4855

56+
if $IS_K3S; then
57+
sed -i "s|runtime_type = \"io.containerd.spin.*\"|runtime_type = \"$KWASM_DIR/bin/containerd-shim-spin-v2\"|g" $NODE_ROOT$CONTAINERD_CONF
58+
fi
59+
4960
if [ ! -f $NODE_ROOT$KWASM_DIR/active ]; then
5061
touch $NODE_ROOT$KWASM_DIR/active
5162
if $IS_MICROK8S; then

0 commit comments

Comments
 (0)