Skip to content

Commit 2d1c2f2

Browse files
fidenciofidencio
committed
tests: Add a way to test what we ship
This commit is huge, I know it, and I am sorry about it. What we're doing here is basically: * Adding a new dependency that will be used for testing * Adding a script to update dependencies when needed * This script will automatically remove the testing dependency that we use, as there's no reason for that to be released / consumed outside of CI * Adding actions for setting up several types of k8s flavours where we should be able to run (as kata-containers does) * Add e2e tests to: * Deploy our chart * Start a pod * Ensure the pod is running with the correct runtime * Remove our chart Last but not least, we're adding a new action to ensure that our test dependency is never ever commited to our Chart.lock. Majority of the tests were vibe generated using Cursor AI based on what I personally wrote for kata-containers. Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>
1 parent e408136 commit 2d1c2f2

File tree

17 files changed

+1447
-2
lines changed

17 files changed

+1447
-2
lines changed

‎.github/actions/install-chart/action.yaml‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
name: Install Helm Chart
2+
description: Install the Confidential Containers Helm chart with various configurations
3+
4+
inputs:
5+
release-name:
6+
description: 'Helm release name'
7+
required: false
8+
default: 'coco'
9+
namespace:
10+
description: 'Kubernetes namespace'
11+
required: false
12+
default: 'kube-system'
13+
extra-args:
14+
description: 'Extra Helm install arguments (e.g., --set flags)'
15+
required: false
16+
default: ''
17+
values-file:
18+
description: 'Path to values file (optional)'
19+
required: false
20+
default: ''
21+
wait-timeout:
22+
description: 'Timeout for helm install --wait'
23+
required: false
24+
default: '15m'
25+
26+
outputs:
27+
installed:
28+
description: 'Whether installation succeeded'
29+
value: ${{ steps.install.outputs.result }}
30+
31+
runs:
32+
using: composite
33+
steps:
34+
- name: Update Helm dependencies
35+
shell: bash
36+
run: |
37+
echo "📦 Updating Helm dependencies..."
38+
helm dependency update
39+
echo "✅ Dependencies updated"
40+
41+
- name: Validate chart
42+
shell: bash
43+
run: |
44+
echo "🔍 Validating chart..."
45+
helm lint .
46+
echo "✅ Chart is valid"
47+
48+
- name: Install chart
49+
id: install
50+
shell: bash
51+
run: |
52+
echo "🚀 Installing chart: ${{ inputs.release-name }}"
53+
echo " Namespace: ${{ inputs.namespace }}"
54+
echo " Extra args: ${{ inputs.extra-args }}"
55+
if [ -n "${{ inputs.values-file }}" ]; then
56+
echo " Values file: ${{ inputs.values-file }}"
57+
fi
58+
59+
INSTALL_CMD="helm install ${{ inputs.release-name }} . \
60+
--namespace ${{ inputs.namespace }} \
61+
--create-namespace \
62+
--wait \
63+
--timeout ${{ inputs.wait-timeout }} \
64+
--debug"
65+
66+
if [ -n "${{ inputs.values-file }}" ]; then
67+
INSTALL_CMD="$INSTALL_CMD -f ${{ inputs.values-file }}"
68+
fi
69+
70+
if [ -n "${{ inputs.extra-args }}" ]; then
71+
INSTALL_CMD="$INSTALL_CMD ${{ inputs.extra-args }}"
72+
fi
73+
74+
echo "Running: $INSTALL_CMD"
75+
76+
if eval $INSTALL_CMD; then
77+
echo "✅ Chart installed successfully"
78+
echo "result=success" >> $GITHUB_OUTPUT
79+
else
80+
echo "❌ Chart installation failed"
81+
echo "result=failed" >> $GITHUB_OUTPUT
82+
exit 1
83+
fi
84+
85+
- name: Show installation status
86+
shell: bash
87+
run: |
88+
echo "📊 Installation status for ${{ inputs.release-name }}:"
89+
helm list -n ${{ inputs.namespace }}
90+
echo ""
91+
echo "📋 Kubernetes resources:"
92+
kubectl get all -n ${{ inputs.namespace }} -l app.kubernetes.io/instance=${{ inputs.release-name }}

‎.github/actions/run-test-pod/action.yaml‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
name: Run Test Pod
2+
description: Deploy and verify a test pod using Kata runtime
3+
4+
inputs:
5+
runtime-class:
6+
description: 'RuntimeClass to use for the test pod'
7+
required: true
8+
namespace:
9+
description: 'Kubernetes namespace for test pod'
10+
required: false
11+
default: 'default'
12+
pod-name:
13+
description: 'Name of the test pod'
14+
required: false
15+
default: 'kata-test-pod'
16+
timeout:
17+
description: 'Timeout for pod to become ready'
18+
required: false
19+
default: '5m'
20+
21+
outputs:
22+
pod-status:
23+
description: 'Final status of the test pod'
24+
value: ${{ steps.check-status.outputs.status }}
25+
26+
runs:
27+
using: composite
28+
steps:
29+
- name: Create test pod
30+
shell: bash
31+
run: |
32+
echo "🚀 Creating test pod with RuntimeClass: ${{ inputs.runtime-class }}"
33+
34+
cat > /tmp/test-pod.yaml <<EOF
35+
apiVersion: v1
36+
kind: Pod
37+
metadata:
38+
name: ${{ inputs.pod-name }}
39+
namespace: ${{ inputs.namespace }}
40+
labels:
41+
app: kata-test
42+
spec:
43+
runtimeClassName: ${{ inputs.runtime-class }}
44+
containers:
45+
- name: test
46+
image: docker.io/library/busybox:latest
47+
command: ['sh', '-c', 'echo "Hello from Kata Containers!" && sleep 30']
48+
restartPolicy: Never
49+
EOF
50+
51+
kubectl apply -f /tmp/test-pod.yaml
52+
echo "✅ Test pod created"
53+
54+
- name: Wait for pod
55+
shell: bash
56+
run: |
57+
echo "⏳ Waiting for pod to start (timeout: ${{ inputs.timeout }})..."
58+
59+
# Wait for pod to be scheduled
60+
for i in {1..30}; do
61+
POD_PHASE=$(kubectl get pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }} -o jsonpath='{.status.phase}' 2>/dev/null || echo "NotFound")
62+
echo " Attempt $i/30: Pod phase is: $POD_PHASE"
63+
64+
if [ "$POD_PHASE" = "Running" ] || [ "$POD_PHASE" = "Succeeded" ]; then
65+
echo "✅ Pod is in $POD_PHASE state"
66+
break
67+
elif [ "$POD_PHASE" = "Failed" ]; then
68+
echo "❌ Pod failed"
69+
break
70+
fi
71+
72+
sleep 10
73+
done
74+
75+
- name: Check pod status
76+
id: check-status
77+
shell: bash
78+
run: |
79+
echo "🔍 Checking final pod status..."
80+
81+
kubectl get pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }}
82+
83+
POD_PHASE=$(kubectl get pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }} -o jsonpath='{.status.phase}')
84+
85+
echo ""
86+
echo "Pod phase: $POD_PHASE"
87+
88+
if [ "$POD_PHASE" = "Running" ] || [ "$POD_PHASE" = "Succeeded" ]; then
89+
echo "✅ Pod reached $POD_PHASE state successfully"
90+
echo "status=success" >> $GITHUB_OUTPUT
91+
else
92+
echo "❌ Pod did not reach Running/Succeeded state (current: $POD_PHASE)"
93+
echo "status=failed" >> $GITHUB_OUTPUT
94+
fi
95+
96+
- name: Show pod details
97+
shell: bash
98+
run: |
99+
echo "📋 Pod details:"
100+
kubectl describe pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }}
101+
102+
- name: Show pod logs
103+
shell: bash
104+
run: |
105+
echo "📋 Pod logs:"
106+
kubectl logs ${{ inputs.pod-name }} -n ${{ inputs.namespace }} || echo "No logs available yet"
107+
108+
- name: Verify Kata runtime is used
109+
shell: bash
110+
run: |
111+
echo "🔍 Verifying Kata runtime is actually being used..."
112+
113+
# Get the node the pod is running on
114+
NODE=$(kubectl get pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }} -o jsonpath='{.spec.nodeName}')
115+
echo "Pod is running on node: $NODE"
116+
117+
# In kind, we can check the container runtime via docker
118+
if command -v docker &> /dev/null; then
119+
echo ""
120+
echo "Container processes on the node:"
121+
docker exec ${NODE} ps aux | grep -E "containerd|qemu" | head -10 || true
122+
fi
123+
124+
# Check RuntimeClass in pod spec
125+
RUNTIME_CLASS=$(kubectl get pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }} -o jsonpath='{.spec.runtimeClassName}')
126+
echo ""
127+
echo "Pod RuntimeClass: $RUNTIME_CLASS"
128+
129+
if [ "$RUNTIME_CLASS" = "${{ inputs.runtime-class }}" ]; then
130+
echo "✅ Pod is using the correct RuntimeClass"
131+
else
132+
echo "⚠️ Pod RuntimeClass mismatch"
133+
fi
134+
135+
- name: Cleanup test pod
136+
if: always()
137+
shell: bash
138+
run: |
139+
echo "🗑️ Cleaning up test pod..."
140+
kubectl delete pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }} --ignore-not-found=true
141+
echo "✅ Test pod cleaned up"

‎.github/actions/setup-k8s-k0s/action.yaml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
name: 'Setup K0s Kubernetes'
2+
description: 'Install K0s Kubernetes distribution'
3+
inputs:
4+
extra-params:
5+
description: 'Extra parameters to pass to k0s install'
6+
required: false
7+
default: ''
8+
runs:
9+
using: 'composite'
10+
steps:
11+
- name: Install K0s
12+
shell: bash
13+
run: |
14+
echo "📦 Installing K0s..."
15+
curl -sSLf https://get.k0s.sh | sudo sh
16+
17+
# Install k0s controller
18+
sudo k0s install controller --single ${{ inputs.extra-params }}
19+
20+
# kube-router uses :8080 for metrics, which causes issues in k0s 1.30.0+
21+
# Change metricsPort to :9999 to avoid conflicts
22+
sudo mkdir -p /etc/k0s
23+
k0s config create | sudo tee /etc/k0s/k0s.yaml
24+
sudo sed -i -e "s/metricsPort: 8080/metricsPort: 9999/g" /etc/k0s/k0s.yaml
25+
26+
sudo k0s start
27+
28+
echo "⏳ Waiting for K0s to be ready..."
29+
sleep 120
30+
31+
- name: Setup kubectl
32+
shell: bash
33+
run: |
34+
echo "🔧 Setting up kubectl..."
35+
36+
# Download the kubectl binary into /usr/bin
37+
ARCH=$(uname -m)
38+
case "${ARCH}" in
39+
x86_64) ARCH="amd64" ;;
40+
aarch64) ARCH="arm64" ;;
41+
esac
42+
43+
kubectl_version=$(sudo k0s kubectl version 2>/dev/null | grep "Client Version" | sed -e 's/Client Version: //')
44+
sudo curl -fL --progress-bar -o /usr/bin/kubectl https://dl.k8s.io/release/"${kubectl_version}"/bin/linux/"${ARCH}"/kubectl
45+
sudo chmod +x /usr/bin/kubectl
46+
47+
mkdir -p ~/.kube
48+
sudo cp /var/lib/k0s/pki/admin.conf ~/.kube/config
49+
sudo chown "${USER}":"${USER}" ~/.kube/config
50+
51+
echo "✅ kubectl installed: $(kubectl version --client --short)"
52+
53+
- name: Verify cluster
54+
shell: bash
55+
run: |
56+
echo "🔍 Verifying K0s cluster..."
57+
kubectl get nodes
58+
kubectl get pods -A
59+
60+
# Wait for system pods to be ready
61+
echo "⏳ Waiting for system pods..."
62+
kubectl wait --for=condition=Ready pods --all -n kube-system --timeout=5m || true
63+
64+
echo "✅ K0s cluster is ready!"

‎.github/actions/setup-k8s-k3s/action.yaml‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
name: 'Setup K3s Kubernetes'
2+
description: 'Install K3s Kubernetes distribution'
3+
inputs:
4+
extra-params:
5+
description: 'Extra parameters to pass to k3s installer'
6+
required: false
7+
default: ''
8+
runs:
9+
using: 'composite'
10+
steps:
11+
- name: Install K3s
12+
shell: bash
13+
run: |
14+
echo "📦 Installing K3s..."
15+
curl -sfL https://get.k3s.io | sh -s - --write-kubeconfig-mode 644 ${{ inputs.extra-params }}
16+
17+
echo "⏳ Waiting for K3s to be ready..."
18+
sleep 120
19+
20+
- name: Setup kubectl
21+
shell: bash
22+
run: |
23+
echo "🔧 Setting up kubectl..."
24+
25+
# Download the kubectl binary into /usr/bin and remove /usr/local/bin/kubectl
26+
# We need to do this to avoid hitting issues like:
27+
# error: open /etc/rancher/k3s/k3s.yaml.lock: permission denied
28+
# Which happens because k3s links `/usr/local/bin/kubectl` to `/usr/local/bin/k3s`,
29+
# and that does extra stuff that vanilla `kubectl` doesn't do.
30+
31+
ARCH=$(uname -m)
32+
case "${ARCH}" in
33+
x86_64) ARCH="amd64" ;;
34+
aarch64) ARCH="arm64" ;;
35+
esac
36+
37+
kubectl_version=$(/usr/local/bin/k3s kubectl version --client=true 2>/dev/null | grep "Client Version" | sed -e 's/Client Version: //' -e 's/+k3s[0-9]\+//')
38+
sudo curl -fL --progress-bar -o /usr/bin/kubectl https://dl.k8s.io/release/"${kubectl_version}"/bin/linux/"${ARCH}"/kubectl
39+
sudo chmod +x /usr/bin/kubectl
40+
sudo rm -rf /usr/local/bin/kubectl
41+
42+
mkdir -p ~/.kube
43+
cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
44+
45+
echo "✅ kubectl installed: $(kubectl version --client --short)"
46+
47+
- name: Verify cluster
48+
shell: bash
49+
run: |
50+
echo "🔍 Verifying K3s cluster..."
51+
kubectl get nodes
52+
kubectl get pods -A
53+
54+
# Wait for system pods to be ready
55+
echo "⏳ Waiting for system pods..."
56+
kubectl wait --for=condition=Ready pods --all -n kube-system --timeout=5m
57+
58+
echo "✅ K3s cluster is ready!"

0 commit comments

Comments
 (0)