Skip to content

Commit 63c33a3

Browse files
fidenciofidencio
committed
tests: Add a way to test what we ship
This commit is huge, I know it, and I am sorry about it. What we're doing here is basically: * Adding a new dependency that will be used for testing * Adding a script to update dependencies when needed * This script will automatically remove the testing dependency that we use, as there's no reason for that to be released / consumed outside of CI * Adding actions for setting up several types of k8s flavours where we should be able to run (as kata-containers does) * Add e2e tests to: * Deploy our chart * Start a pod * Ensure the pod is running with the correct runtime * Remove our chart Last but not least, we're adding a new action to ensure that our test dependency is never ever commited to our Chart.lock. Majority of the tests were vibe generated using Cursor AI based on what I personally wrote for kata-containers. Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>
1 parent 8a50a6a commit 63c33a3

File tree

17 files changed

+2018
-2
lines changed

17 files changed

+2018
-2
lines changed

‎.github/actions/install-chart/action.yaml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
name: Install Helm Chart
2+
description: Install the Confidential Containers Helm chart with various configurations
3+
4+
inputs:
5+
release-name:
6+
description: 'Helm release name'
7+
required: false
8+
default: 'coco'
9+
namespace:
10+
description: 'Kubernetes namespace'
11+
required: false
12+
default: 'kube-system'
13+
extra-args:
14+
description: 'Extra Helm install arguments (e.g., --set flags)'
15+
required: false
16+
default: ''
17+
values-file:
18+
description: 'Path to values file (optional)'
19+
required: false
20+
default: ''
21+
wait-timeout:
22+
description: 'Timeout for helm install --wait'
23+
required: false
24+
default: '15m'
25+
26+
outputs:
27+
installed:
28+
description: 'Whether installation succeeded'
29+
value: ${{ steps.install.outputs.result }}
30+
31+
runs:
32+
using: composite
33+
steps:
34+
- name: Update Helm dependencies
35+
shell: bash
36+
run: |
37+
echo "📦 Updating Helm dependencies..."
38+
helm dependency update
39+
echo "✅ Dependencies updated"
40+
41+
- name: Validate chart
42+
shell: bash
43+
run: |
44+
echo "🔍 Validating chart..."
45+
helm lint .
46+
echo "✅ Chart is valid"
47+
48+
- name: Install chart
49+
id: install
50+
shell: bash
51+
run: |
52+
echo "🚀 Installing chart: ${{ inputs.release-name }}"
53+
echo " Namespace: ${{ inputs.namespace }}"
54+
echo " Extra args: ${{ inputs.extra-args }}"
55+
if [ -n "${{ inputs.values-file }}" ]; then
56+
echo " Values file: ${{ inputs.values-file }}"
57+
fi
58+
59+
INSTALL_CMD="helm install ${{ inputs.release-name }} . \
60+
--namespace ${{ inputs.namespace }} \
61+
--create-namespace \
62+
--debug"
63+
64+
if [ -n "${{ inputs.values-file }}" ]; then
65+
INSTALL_CMD="$INSTALL_CMD -f ${{ inputs.values-file }}"
66+
fi
67+
68+
if [ -n "${{ inputs.extra-args }}" ]; then
69+
INSTALL_CMD="$INSTALL_CMD ${{ inputs.extra-args }}"
70+
fi
71+
72+
echo "Running: $INSTALL_CMD"
73+
74+
if eval $INSTALL_CMD; then
75+
echo "✅ Chart installed successfully"
76+
echo "result=success" >> $GITHUB_OUTPUT
77+
else
78+
echo "❌ Chart installation failed"
79+
echo "result=failed" >> $GITHUB_OUTPUT
80+
exit 1
81+
fi

‎.github/actions/run-test-pod/action.yaml‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
name: Run Test Pod
2+
description: Deploy and verify a test pod using Kata runtime
3+
4+
inputs:
5+
runtime-class:
6+
description: 'RuntimeClass to use for the test pod'
7+
required: true
8+
namespace:
9+
description: 'Kubernetes namespace for test pod'
10+
required: false
11+
default: 'default'
12+
pod-name:
13+
description: 'Name of the test pod'
14+
required: false
15+
default: 'kata-test-pod'
16+
timeout:
17+
description: 'Timeout for pod to become ready'
18+
required: false
19+
default: '5m'
20+
21+
outputs:
22+
pod-status:
23+
description: 'Final status of the test pod'
24+
value: ${{ steps.check-status.outputs.status }}
25+
26+
runs:
27+
using: composite
28+
steps:
29+
- name: Create test pod
30+
shell: bash
31+
run: |
32+
set -e # Exit on any error
33+
echo "🚀 Creating test pod with RuntimeClass: ${{ inputs.runtime-class }}"
34+
35+
cat > /tmp/test-pod.yaml <<EOF
36+
apiVersion: v1
37+
kind: Pod
38+
metadata:
39+
name: ${{ inputs.pod-name }}
40+
namespace: ${{ inputs.namespace }}
41+
labels:
42+
app: kata-test
43+
spec:
44+
runtimeClassName: ${{ inputs.runtime-class }}
45+
containers:
46+
- name: test
47+
image: docker.io/library/busybox:latest
48+
command: ['sh', '-c', 'echo "Hello from Kata Containers!" && sleep 30']
49+
restartPolicy: Never
50+
EOF
51+
52+
if ! kubectl apply -f /tmp/test-pod.yaml; then
53+
echo "❌ Failed to create test pod"
54+
exit 1
55+
fi
56+
echo "✅ Test pod created"
57+
58+
- name: Wait for pod
59+
shell: bash
60+
run: |
61+
set -e # Exit on any error
62+
echo "⏳ Waiting for pod to start (timeout: ${{ inputs.timeout }})..."
63+
64+
# Wait for pod to be scheduled
65+
SUCCESS=false
66+
for i in {1..30}; do
67+
POD_PHASE=$(kubectl get pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }} -o jsonpath='{.status.phase}' 2>/dev/null || echo "NotFound")
68+
echo " Attempt $i/30: Pod phase is: $POD_PHASE"
69+
70+
if [ "$POD_PHASE" = "Running" ] || [ "$POD_PHASE" = "Succeeded" ]; then
71+
echo "✅ Pod is in $POD_PHASE state"
72+
SUCCESS=true
73+
break
74+
elif [ "$POD_PHASE" = "Failed" ]; then
75+
echo "❌ Pod failed"
76+
kubectl describe pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }}
77+
exit 1
78+
fi
79+
80+
sleep 10
81+
done
82+
83+
if [ "$SUCCESS" = "false" ]; then
84+
echo "❌ Timeout: Pod did not reach Running/Succeeded state"
85+
kubectl describe pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }}
86+
exit 1
87+
fi
88+
89+
- name: Check pod status
90+
id: check-status
91+
shell: bash
92+
run: |
93+
set -e # Exit on any error
94+
echo "🔍 Checking final pod status..."
95+
96+
kubectl get pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }}
97+
98+
POD_PHASE=$(kubectl get pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }} -o jsonpath='{.status.phase}')
99+
100+
echo ""
101+
echo "Pod phase: $POD_PHASE"
102+
103+
if [ "$POD_PHASE" = "Running" ] || [ "$POD_PHASE" = "Succeeded" ]; then
104+
echo "✅ Pod reached $POD_PHASE state successfully"
105+
echo "status=success" >> $GITHUB_OUTPUT
106+
else
107+
echo "❌ Pod did not reach Running/Succeeded state (current: $POD_PHASE)"
108+
echo "status=failed" >> $GITHUB_OUTPUT
109+
kubectl describe pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }}
110+
exit 1
111+
fi
112+
113+
- name: Show pod details
114+
shell: bash
115+
run: |
116+
echo "📋 Pod details:"
117+
kubectl describe pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }}
118+
119+
- name: Show pod logs
120+
shell: bash
121+
run: |
122+
echo "📋 Pod logs:"
123+
kubectl logs ${{ inputs.pod-name }} -n ${{ inputs.namespace }} || echo "No logs available yet"
124+
125+
- name: Verify Kata runtime is used
126+
shell: bash
127+
run: |
128+
set -e # Exit on any error
129+
echo "🔍 Verifying Kata runtime is actually being used..."
130+
131+
# Get the node the pod is running on
132+
NODE=$(kubectl get pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }} -o jsonpath='{.spec.nodeName}')
133+
if [ -z "$NODE" ]; then
134+
echo "❌ Failed to get node name for pod"
135+
exit 1
136+
fi
137+
echo "Pod is running on node: $NODE"
138+
139+
# In kind, we can check the container runtime via docker
140+
if command -v docker &> /dev/null; then
141+
echo ""
142+
echo "Container processes on the node:"
143+
docker exec ${NODE} ps aux | grep -E "containerd|qemu" | head -10 || true
144+
fi
145+
146+
# Check RuntimeClass in pod spec
147+
RUNTIME_CLASS=$(kubectl get pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }} -o jsonpath='{.spec.runtimeClassName}')
148+
echo ""
149+
echo "Pod RuntimeClass: $RUNTIME_CLASS"
150+
echo "Expected RuntimeClass: ${{ inputs.runtime-class }}"
151+
152+
if [ "$RUNTIME_CLASS" = "${{ inputs.runtime-class }}" ]; then
153+
echo "✅ Pod is using the correct RuntimeClass"
154+
else
155+
echo "❌ Pod RuntimeClass mismatch!"
156+
echo " Expected: ${{ inputs.runtime-class }}"
157+
echo " Got: $RUNTIME_CLASS"
158+
kubectl describe pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }}
159+
exit 1
160+
fi
161+
162+
- name: Cleanup test pod
163+
if: always()
164+
shell: bash
165+
run: |
166+
echo "🗑️ Cleaning up test pod..."
167+
kubectl delete pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }} --ignore-not-found=true
168+
echo "✅ Test pod cleaned up"

‎.github/actions/setup-k8s-k0s/action.yaml‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
name: 'Setup K0s Kubernetes'
2+
description: 'Install K0s Kubernetes distribution'
3+
inputs:
4+
extra-params:
5+
description: 'Extra parameters to pass to k0s install'
6+
required: false
7+
default: ''
8+
runs:
9+
using: 'composite'
10+
steps:
11+
- name: Free up disk space
12+
shell: bash
13+
run: |
14+
echo "🧹 Removing unnecessary directories to free up disk space..."
15+
sudo rm -rf /usr/local/.ghcup
16+
sudo rm -rf /opt/hostedtoolcache/CodeQL
17+
sudo rm -rf /usr/local/lib/android
18+
sudo rm -rf /usr/share/dotnet
19+
sudo rm -rf /opt/ghc
20+
sudo rm -rf /usr/local/share/boost
21+
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
22+
sudo rm -rf /usr/lib/jvm
23+
sudo rm -rf /usr/share/swift
24+
sudo rm -rf /usr/local/share/powershell
25+
sudo rm -rf /usr/local/julia*
26+
sudo rm -rf /opt/az
27+
sudo rm -rf /usr/local/share/chromium
28+
sudo rm -rf /opt/microsoft
29+
sudo rm -rf /opt/google
30+
sudo rm -rf /usr/lib/firefox
31+
echo "✅ Disk space freed up"
32+
df -h / | grep -v Filesystem
33+
34+
- name: Install K0s
35+
shell: bash
36+
run: |
37+
echo "📦 Installing K0s..."
38+
curl -sSLf https://get.k0s.sh | sudo sh
39+
40+
# Install k0s controller
41+
sudo k0s install controller --single ${{ inputs.extra-params }}
42+
43+
# kube-router uses :8080 for metrics, which causes issues in k0s 1.30.0+
44+
# Change metricsPort to :9999 to avoid conflicts
45+
sudo mkdir -p /etc/k0s
46+
k0s config create | sudo tee /etc/k0s/k0s.yaml
47+
sudo sed -i -e "s/metricsPort: 8080/metricsPort: 9999/g" /etc/k0s/k0s.yaml
48+
49+
sudo k0s start
50+
51+
echo "⏳ Waiting for K0s to be ready..."
52+
sleep 120
53+
54+
- name: Setup kubectl
55+
shell: bash
56+
run: |
57+
echo "🔧 Setting up kubectl..."
58+
59+
# Download the kubectl binary into /usr/bin
60+
ARCH=$(uname -m)
61+
case "${ARCH}" in
62+
x86_64) ARCH="amd64" ;;
63+
aarch64) ARCH="arm64" ;;
64+
esac
65+
66+
kubectl_version=$(sudo k0s kubectl version 2>/dev/null | grep "Client Version" | sed -e 's/Client Version: //')
67+
sudo curl -fL --progress-bar -o /usr/bin/kubectl https://dl.k8s.io/release/"${kubectl_version}"/bin/linux/"${ARCH}"/kubectl
68+
sudo chmod +x /usr/bin/kubectl
69+
70+
mkdir -p ~/.kube
71+
sudo cp /var/lib/k0s/pki/admin.conf ~/.kube/config
72+
sudo chown "${USER}":"${USER}" ~/.kube/config
73+
74+
echo "✅ kubectl installed: $(kubectl version --client --short)"
75+
76+
- name: Verify cluster
77+
shell: bash
78+
run: |
79+
echo "🔍 Verifying K0s cluster..."
80+
kubectl get nodes
81+
kubectl get pods -A
82+
83+
# Wait for system pods to be ready (excluding completed/job pods)
84+
echo "⏳ Waiting for system pods..."
85+
# Wait only for Running pods (not Completed/Job pods)
86+
kubectl wait --for=condition=Ready pods --all -n kube-system --timeout=5m \
87+
--field-selector=status.phase!=Succeeded,status.phase!=Failed || true
88+
89+
# Verify all pods are either Running or Completed
90+
NOT_READY=$(kubectl get pods -n kube-system -o json | \
91+
jq -r '.items[] | select(.status.phase != "Running" and .status.phase != "Succeeded") | .metadata.name')
92+
93+
if [ -n "$NOT_READY" ]; then
94+
echo "❌ Some pods are not ready:"
95+
echo "$NOT_READY"
96+
kubectl get pods -A
97+
exit 1
98+
fi
99+
100+
echo "✅ K0s cluster is ready!"

0 commit comments

Comments
 (0)