Skip to content

Commit d5e2348

Browse files
fidenciofidencio
committed
tests: Add a way to test what we ship
This commit is huge, I know it, and I am sorry about it. What we're doing here is basically: * Adding a new dependency that will be used for testing * Adding a script to update dependencies when needed * This script will automatically remove the testing dependency that we use, as there's no reason for that to be released / consumed outside of CI * Adding actions for setting up several types of k8s flavours where we should be able to run (as kata-containers does) * Add e2e tests to: * Deploy our chart * Start a pod * Ensure the pod is running with the correct runtime * Remove our chart Last but not least, we're adding a new action to ensure that our test dependency is never ever commited to our Chart.lock. Majority of the tests were vibe generated using Cursor AI based on what I personally wrote for kata-containers. Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>
1 parent 8a50a6a commit d5e2348

File tree

17 files changed

+1738
-2
lines changed

17 files changed

+1738
-2
lines changed

‎.github/actions/install-chart/action.yaml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
name: Install Helm Chart
2+
description: Install the Confidential Containers Helm chart with various configurations
3+
4+
inputs:
5+
release-name:
6+
description: 'Helm release name'
7+
required: false
8+
default: 'coco'
9+
namespace:
10+
description: 'Kubernetes namespace'
11+
required: false
12+
default: 'kube-system'
13+
extra-args:
14+
description: 'Extra Helm install arguments (e.g., --set flags)'
15+
required: false
16+
default: ''
17+
values-file:
18+
description: 'Path to values file (optional)'
19+
required: false
20+
default: ''
21+
wait-timeout:
22+
description: 'Timeout for helm install --wait'
23+
required: false
24+
default: '15m'
25+
26+
outputs:
27+
installed:
28+
description: 'Whether installation succeeded'
29+
value: ${{ steps.install.outputs.result }}
30+
31+
runs:
32+
using: composite
33+
steps:
34+
- name: Update Helm dependencies
35+
shell: bash
36+
run: |
37+
echo "📦 Updating Helm dependencies..."
38+
helm dependency update
39+
echo "✅ Dependencies updated"
40+
41+
- name: Validate chart
42+
shell: bash
43+
run: |
44+
echo "🔍 Validating chart..."
45+
helm lint .
46+
echo "✅ Chart is valid"
47+
48+
- name: Install chart
49+
id: install
50+
shell: bash
51+
run: |
52+
echo "🚀 Installing chart: ${{ inputs.release-name }}"
53+
echo " Namespace: ${{ inputs.namespace }}"
54+
echo " Extra args: ${{ inputs.extra-args }}"
55+
if [ -n "${{ inputs.values-file }}" ]; then
56+
echo " Values file: ${{ inputs.values-file }}"
57+
fi
58+
59+
INSTALL_CMD="helm install ${{ inputs.release-name }} . \
60+
--namespace ${{ inputs.namespace }} \
61+
--create-namespace \
62+
--debug"
63+
64+
if [ -n "${{ inputs.values-file }}" ]; then
65+
INSTALL_CMD="$INSTALL_CMD -f ${{ inputs.values-file }}"
66+
fi
67+
68+
if [ -n "${{ inputs.extra-args }}" ]; then
69+
INSTALL_CMD="$INSTALL_CMD ${{ inputs.extra-args }}"
70+
fi
71+
72+
echo "Running: $INSTALL_CMD"
73+
74+
if eval $INSTALL_CMD; then
75+
echo "✅ Chart installed successfully"
76+
echo "result=success" >> $GITHUB_OUTPUT
77+
else
78+
echo "❌ Chart installation failed"
79+
echo "result=failed" >> $GITHUB_OUTPUT
80+
exit 1
81+
fi

‎.github/actions/run-test-pod/action.yaml‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
name: Run Test Pod
2+
description: Deploy and verify a test pod using Kata runtime
3+
4+
inputs:
5+
runtime-class:
6+
description: 'RuntimeClass to use for the test pod'
7+
required: true
8+
namespace:
9+
description: 'Kubernetes namespace for test pod'
10+
required: false
11+
default: 'default'
12+
pod-name:
13+
description: 'Name of the test pod'
14+
required: false
15+
default: 'kata-test-pod'
16+
timeout:
17+
description: 'Timeout for pod to become ready'
18+
required: false
19+
default: '5m'
20+
21+
outputs:
22+
pod-status:
23+
description: 'Final status of the test pod'
24+
value: ${{ steps.check-status.outputs.status }}
25+
26+
runs:
27+
using: composite
28+
steps:
29+
- name: Create test pod
30+
shell: bash
31+
run: |
32+
echo "🚀 Creating test pod with RuntimeClass: ${{ inputs.runtime-class }}"
33+
34+
cat > /tmp/test-pod.yaml <<EOF
35+
apiVersion: v1
36+
kind: Pod
37+
metadata:
38+
name: ${{ inputs.pod-name }}
39+
namespace: ${{ inputs.namespace }}
40+
labels:
41+
app: kata-test
42+
spec:
43+
runtimeClassName: ${{ inputs.runtime-class }}
44+
containers:
45+
- name: test
46+
image: docker.io/library/busybox:latest
47+
command: ['sh', '-c', 'echo "Hello from Kata Containers!" && sleep 30']
48+
restartPolicy: Never
49+
EOF
50+
51+
kubectl apply -f /tmp/test-pod.yaml
52+
echo "✅ Test pod created"
53+
54+
- name: Wait for pod
55+
shell: bash
56+
run: |
57+
echo "⏳ Waiting for pod to start (timeout: ${{ inputs.timeout }})..."
58+
59+
# Wait for pod to be scheduled
60+
for i in {1..30}; do
61+
POD_PHASE=$(kubectl get pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }} -o jsonpath='{.status.phase}' 2>/dev/null || echo "NotFound")
62+
echo " Attempt $i/30: Pod phase is: $POD_PHASE"
63+
64+
if [ "$POD_PHASE" = "Running" ] || [ "$POD_PHASE" = "Succeeded" ]; then
65+
echo "✅ Pod is in $POD_PHASE state"
66+
break
67+
elif [ "$POD_PHASE" = "Failed" ]; then
68+
echo "❌ Pod failed"
69+
break
70+
fi
71+
72+
sleep 10
73+
done
74+
75+
- name: Check pod status
76+
id: check-status
77+
shell: bash
78+
run: |
79+
echo "🔍 Checking final pod status..."
80+
81+
kubectl get pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }}
82+
83+
POD_PHASE=$(kubectl get pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }} -o jsonpath='{.status.phase}')
84+
85+
echo ""
86+
echo "Pod phase: $POD_PHASE"
87+
88+
if [ "$POD_PHASE" = "Running" ] || [ "$POD_PHASE" = "Succeeded" ]; then
89+
echo "✅ Pod reached $POD_PHASE state successfully"
90+
echo "status=success" >> $GITHUB_OUTPUT
91+
else
92+
echo "❌ Pod did not reach Running/Succeeded state (current: $POD_PHASE)"
93+
echo "status=failed" >> $GITHUB_OUTPUT
94+
fi
95+
96+
- name: Show pod details
97+
shell: bash
98+
run: |
99+
echo "📋 Pod details:"
100+
kubectl describe pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }}
101+
102+
- name: Show pod logs
103+
shell: bash
104+
run: |
105+
echo "📋 Pod logs:"
106+
kubectl logs ${{ inputs.pod-name }} -n ${{ inputs.namespace }} || echo "No logs available yet"
107+
108+
- name: Verify Kata runtime is used
109+
shell: bash
110+
run: |
111+
echo "🔍 Verifying Kata runtime is actually being used..."
112+
113+
# Get the node the pod is running on
114+
NODE=$(kubectl get pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }} -o jsonpath='{.spec.nodeName}')
115+
echo "Pod is running on node: $NODE"
116+
117+
# In kind, we can check the container runtime via docker
118+
if command -v docker &> /dev/null; then
119+
echo ""
120+
echo "Container processes on the node:"
121+
docker exec ${NODE} ps aux | grep -E "containerd|qemu" | head -10 || true
122+
fi
123+
124+
# Check RuntimeClass in pod spec
125+
RUNTIME_CLASS=$(kubectl get pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }} -o jsonpath='{.spec.runtimeClassName}')
126+
echo ""
127+
echo "Pod RuntimeClass: $RUNTIME_CLASS"
128+
129+
if [ "$RUNTIME_CLASS" = "${{ inputs.runtime-class }}" ]; then
130+
echo "✅ Pod is using the correct RuntimeClass"
131+
else
132+
echo "⚠️ Pod RuntimeClass mismatch"
133+
fi
134+
135+
- name: Cleanup test pod
136+
if: always()
137+
shell: bash
138+
run: |
139+
echo "🗑️ Cleaning up test pod..."
140+
kubectl delete pod ${{ inputs.pod-name }} -n ${{ inputs.namespace }} --ignore-not-found=true
141+
echo "✅ Test pod cleaned up"

‎.github/actions/setup-k8s-k0s/action.yaml‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
name: 'Setup K0s Kubernetes'
2+
description: 'Install K0s Kubernetes distribution'
3+
inputs:
4+
extra-params:
5+
description: 'Extra parameters to pass to k0s install'
6+
required: false
7+
default: ''
8+
runs:
9+
using: 'composite'
10+
steps:
11+
- name: Free up disk space
12+
shell: bash
13+
run: |
14+
echo "🧹 Removing unnecessary directories to free up disk space..."
15+
sudo rm -rf /usr/local/.ghcup
16+
sudo rm -rf /opt/hostedtoolcache/CodeQL
17+
sudo rm -rf /usr/local/lib/android
18+
sudo rm -rf /usr/share/dotnet
19+
sudo rm -rf /opt/ghc
20+
sudo rm -rf /usr/local/share/boost
21+
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
22+
sudo rm -rf /usr/lib/jvm
23+
sudo rm -rf /usr/share/swift
24+
sudo rm -rf /usr/local/share/powershell
25+
sudo rm -rf /usr/local/julia*
26+
sudo rm -rf /opt/az
27+
sudo rm -rf /usr/local/share/chromium
28+
sudo rm -rf /opt/microsoft
29+
sudo rm -rf /opt/google
30+
sudo rm -rf /usr/lib/firefox
31+
echo "✅ Disk space freed up"
32+
df -h / | grep -v Filesystem
33+
34+
- name: Install K0s
35+
shell: bash
36+
run: |
37+
echo "📦 Installing K0s..."
38+
curl -sSLf https://get.k0s.sh | sudo sh
39+
40+
# Install k0s controller
41+
sudo k0s install controller --single ${{ inputs.extra-params }}
42+
43+
# kube-router uses :8080 for metrics, which causes issues in k0s 1.30.0+
44+
# Change metricsPort to :9999 to avoid conflicts
45+
sudo mkdir -p /etc/k0s
46+
k0s config create | sudo tee /etc/k0s/k0s.yaml
47+
sudo sed -i -e "s/metricsPort: 8080/metricsPort: 9999/g" /etc/k0s/k0s.yaml
48+
49+
sudo k0s start
50+
51+
echo "⏳ Waiting for K0s to be ready..."
52+
sleep 120
53+
54+
- name: Setup kubectl
55+
shell: bash
56+
run: |
57+
echo "🔧 Setting up kubectl..."
58+
59+
# Download the kubectl binary into /usr/bin
60+
ARCH=$(uname -m)
61+
case "${ARCH}" in
62+
x86_64) ARCH="amd64" ;;
63+
aarch64) ARCH="arm64" ;;
64+
esac
65+
66+
kubectl_version=$(sudo k0s kubectl version 2>/dev/null | grep "Client Version" | sed -e 's/Client Version: //')
67+
sudo curl -fL --progress-bar -o /usr/bin/kubectl https://dl.k8s.io/release/"${kubectl_version}"/bin/linux/"${ARCH}"/kubectl
68+
sudo chmod +x /usr/bin/kubectl
69+
70+
mkdir -p ~/.kube
71+
sudo cp /var/lib/k0s/pki/admin.conf ~/.kube/config
72+
sudo chown "${USER}":"${USER}" ~/.kube/config
73+
74+
echo "✅ kubectl installed: $(kubectl version --client --short)"
75+
76+
- name: Verify cluster
77+
shell: bash
78+
run: |
79+
echo "🔍 Verifying K0s cluster..."
80+
kubectl get nodes
81+
kubectl get pods -A
82+
83+
# Wait for system pods to be ready
84+
echo "⏳ Waiting for system pods..."
85+
kubectl wait --for=condition=Ready pods --all -n kube-system --timeout=5m || true
86+
87+
echo "✅ K0s cluster is ready!"

‎.github/actions/setup-k8s-k3s/action.yaml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
name: 'Setup K3s Kubernetes'
2+
description: 'Install K3s Kubernetes distribution'
3+
inputs:
4+
extra-params:
5+
description: 'Extra parameters to pass to k3s installer'
6+
required: false
7+
default: ''
8+
runs:
9+
using: 'composite'
10+
steps:
11+
- name: Free up disk space
12+
shell: bash
13+
run: |
14+
echo "🧹 Removing unnecessary directories to free up disk space..."
15+
sudo rm -rf /usr/local/.ghcup
16+
sudo rm -rf /opt/hostedtoolcache/CodeQL
17+
sudo rm -rf /usr/local/lib/android
18+
sudo rm -rf /usr/share/dotnet
19+
sudo rm -rf /opt/ghc
20+
sudo rm -rf /usr/local/share/boost
21+
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
22+
sudo rm -rf /usr/lib/jvm
23+
sudo rm -rf /usr/share/swift
24+
sudo rm -rf /usr/local/share/powershell
25+
sudo rm -rf /usr/local/julia*
26+
sudo rm -rf /opt/az
27+
sudo rm -rf /usr/local/share/chromium
28+
sudo rm -rf /opt/microsoft
29+
sudo rm -rf /opt/google
30+
sudo rm -rf /usr/lib/firefox
31+
echo "✅ Disk space freed up"
32+
df -h / | grep -v Filesystem
33+
34+
- name: Install K3s
35+
shell: bash
36+
run: |
37+
echo "📦 Installing K3s..."
38+
curl -sfL https://get.k3s.io | sh -s - --write-kubeconfig-mode 644 ${{ inputs.extra-params }}
39+
40+
echo "⏳ Waiting for K3s to be ready..."
41+
sleep 120
42+
43+
- name: Setup kubectl
44+
shell: bash
45+
run: |
46+
echo "🔧 Setting up kubectl..."
47+
48+
# Download the kubectl binary into /usr/bin and remove /usr/local/bin/kubectl
49+
# We need to do this to avoid hitting issues like:
50+
# error: open /etc/rancher/k3s/k3s.yaml.lock: permission denied
51+
# Which happens because k3s links `/usr/local/bin/kubectl` to `/usr/local/bin/k3s`,
52+
# and that does extra stuff that vanilla `kubectl` doesn't do.
53+
54+
ARCH=$(uname -m)
55+
case "${ARCH}" in
56+
x86_64) ARCH="amd64" ;;
57+
aarch64) ARCH="arm64" ;;
58+
esac
59+
60+
kubectl_version=$(/usr/local/bin/k3s kubectl version --client=true 2>/dev/null | grep "Client Version" | sed -e 's/Client Version: //' -e 's/+k3s[0-9]\+//')
61+
sudo curl -fL --progress-bar -o /usr/bin/kubectl https://dl.k8s.io/release/"${kubectl_version}"/bin/linux/"${ARCH}"/kubectl
62+
sudo chmod +x /usr/bin/kubectl
63+
sudo rm -rf /usr/local/bin/kubectl
64+
65+
mkdir -p ~/.kube
66+
cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
67+
68+
echo "✅ kubectl installed: $(kubectl version --client --short)"
69+
70+
- name: Verify cluster
71+
shell: bash
72+
run: |
73+
echo "🔍 Verifying K3s cluster..."
74+
kubectl get nodes
75+
kubectl get pods -A
76+
77+
# Wait for system pods to be ready
78+
echo "⏳ Waiting for system pods..."
79+
kubectl wait --for=condition=Ready pods --all -n kube-system --timeout=5m
80+
81+
echo "✅ K3s cluster is ready!"

0 commit comments

Comments
 (0)