Merge pull request #289 from spinframework/add-systemd-cgroup

feat: add SystemCgroup option to containerd runtime options

Author: Mossaka

.github/workflows/action-node-installer.yaml

@@ -101,7 +101,15 @@ jobs:
         if: matrix.distribution == 'k3s' && failure()
         run: |
           sudo k3s kubectl describe pods -n kwasm
+          sudo k3s kubectl logs k3s-provision-kwasm-dev -n kwasm
           sudo k3s kubectl describe pods
+      
+      - name: Collect minikube debug logs
+        if: matrix.distribution == 'minikube' && failure()
+        run: |
+          kubectl describe pods -n kwasm
+          kubectl logs minikube-provision-kwasm-dev -n kwasm 
+          kubectl describe pods
 
   publish:
     needs: build-and-test

README.md

@@ -66,14 +66,18 @@ To carry out the installation step-by-step, do the following:
 
     ```toml
     [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.spin]
-    runtime_type = "io.containerd.spin.v2"
+      runtime_type = "io.containerd.spin.v2"
+    [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.spin.options]
+      SystemdCgroup = true
     ```
 
     Otherwise, add:
 
     ```toml
     [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin]
-    runtime_type = "io.containerd.spin.v2"
+      runtime_type = "io.containerd.spin.v2"
+    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin.options]
+      SystemdCgroup = true
     ```
 
     The [Node Installer script](./node-installer/script/installer.sh) that is used by the [`runtime-class-manager`](https://www.spinkube.dev/docs/topics/architecture/#runtime-class-manager) does this for you and is a good reference to understand the common paths to the containerd configuration file for popular Kubernetes distributions.

conformance-tests/README.md

@@ -19,6 +19,8 @@ Containerd must be configured to access the `containerd-shim-spin`:
     ```toml
     [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin]
         runtime_type = "/usr/bin/containerd-shim-spin-v2"
+    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin.options]
+        SystemdCgroup = true
     ```
 3. Restart containerd if it is running as a service
     ```sh

node-installer/script/installer.sh

@@ -11,6 +11,7 @@ IS_MICROK8S=false
 IS_K3S=false
 IS_RKE2_AGENT=false
 IS_K0S_WORKER=false
+
 if pgrep -f snap/microk8s > /dev/null; then
     CONTAINERD_CONF=/var/snap/microk8s/current/args/containerd-template.toml
     IS_MICROK8S=true
@@ -34,11 +35,40 @@ elif pgrep -f /var/lib/k0s/bin/kubelet > /dev/null; then
     touch $NODE_ROOT$CONTAINERD_CONF
 fi
 
+# If SYSTEMD_CGROUP is not set, default to true except for distros that do not default to systemd
+# TODO: detect k3d which defaults to cgroupfs
+if [ -z "${SYSTEMD_CGROUP+x}" ] && [ "$IS_MICROK8S" = "true" ]; then
+    SYSTEMD_CGROUP=false
+else
+    : "${SYSTEMD_CGROUP:=true}"
+fi
+
+# Install D-Bus if it's not available but systemd cgroups are requested
+if [ "$SYSTEMD_CGROUP" = "true" ]; then
+    if ! nsenter -m/$NODE_ROOT/proc/1/ns/mnt -- which dbus-daemon >/dev/null 2>&1; then
+        if nsenter -m/$NODE_ROOT/proc/1/ns/mnt -- which apt-get >/dev/null 2>&1; then
+            nsenter -m/$NODE_ROOT/proc/1/ns/mnt -- apt-get update -y
+            nsenter -m/$NODE_ROOT/proc/1/ns/mnt -- apt-get install -y dbus
+        elif nsenter -m/$NODE_ROOT/proc/1/ns/mnt -- which yum >/dev/null 2>&1; then
+            nsenter -m/$NODE_ROOT/proc/1/ns/mnt -- yum install -y dbus
+        elif nsenter -m/$NODE_ROOT/proc/1/ns/mnt -- which dnf >/dev/null 2>&1; then
+            nsenter -m/$NODE_ROOT/proc/1/ns/mnt -- dnf install -y dbus
+        elif nsenter -m/$NODE_ROOT/proc/1/ns/mnt -- which apk >/dev/null 2>&1; then
+            nsenter -m/$NODE_ROOT/proc/1/ns/mnt -- apk add dbus
+        else
+            echo "WARNING: Could not install D-Bus. No supported package manager found."
+            SYSTEMD_CGROUP=false
+            echo "SYSTEMD_CGROUP is now set to $SYSTEMD_CGROUP"
+        fi
+    fi
+fi
+
 mkdir -p $NODE_ROOT$KWASM_DIR/bin/
 
 cp /assets/containerd-shim-spin-v2 $NODE_ROOT$KWASM_DIR/bin/
 
 if ! grep -q spin $NODE_ROOT$CONTAINERD_CONF; then
+    echo "Adding Spin runtime to containerd"
     if $IS_K3S; then
         echo '
 [plugins."io.containerd.cri.v1.runtime".containerd.runtimes."spin"]
@@ -53,12 +83,36 @@ if ! grep -q spin $NODE_ROOT$CONTAINERD_CONF; then
     rm -Rf $NODE_ROOT$KWASM_DIR/active
 fi
 
+# Configure Spin runtime options if using systemd cgroups and no options are configured
+# TODO: this should allow for some options to already be configured and just additively 
+# configure SystemdCgroup
+if ! grep -q 'runtimes.spin.options' $NODE_ROOT$CONTAINERD_CONF && [ "$SYSTEMD_CGROUP" = "true" ]; then
+    echo "Setting SystemdCgroup to $SYSTEMD_CGROUP in Spin containerd configuration"
+    if $IS_K3S; then
+        echo '
+[plugins."io.containerd.cri.v1.runtime".containerd.runtimes.spin.options]
+    SystemdCgroup = '$SYSTEMD_CGROUP'
+' >> $NODE_ROOT$CONTAINERD_CONF
+    else
+        echo '
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.spin.options]
+    SystemdCgroup = '$SYSTEMD_CGROUP'
+' >> $NODE_ROOT$CONTAINERD_CONF
+    fi
+fi
+
 if $IS_K3S; then
     sed -i "s|runtime_type = \"io.containerd.spin.*\"|runtime_type = \"$KWASM_DIR/bin/containerd-shim-spin-v2\"|g" $NODE_ROOT$CONTAINERD_CONF
 fi
 
 if [ ! -f $NODE_ROOT$KWASM_DIR/active ]; then
     touch $NODE_ROOT$KWASM_DIR/active
+    
+    # Ensure D-Bus is running before restarting services if using systemd cgroups
+    if [ "$SYSTEMD_CGROUP" = "true" ]; then
+        nsenter -m/$NODE_ROOT/proc/1/ns/mnt -- systemctl restart dbus
+    fi
+    
     if $IS_MICROK8S; then
         nsenter -m/$NODE_ROOT/proc/1/ns/mnt -- systemctl restart snap.microk8s.daemon-containerd
     elif $IS_K3S; then

node-installer/tests/Dockerfile.minikube-custom

@@ -0,0 +1,14 @@
+FROM gcr.io/k8s-minikube/kicbase:v0.0.46@sha256:fd2d445ddcc33ebc5c6b68a17e6219ea207ce63c005095ea1525296da2d1a279
+
+RUN apt-get update -y || true && \
+    apt-get -y install wget curl apt-transport-https ca-certificates gnupg2 && \
+    
+    # Remove existing repository configurations to avoid conflicts
+    rm -f /etc/apt/sources.list.d/devel:kubic:*.list && \
+    
+    mkdir -p /etc/apt/keyrings && \
+    curl -fsSL "https://downloadcontent.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_22.04/Release.key" | gpg --dearmor > /etc/apt/keyrings/libcontainers-stable.gpg && \
+    curl -fsSL "https://downloadcontent.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.24/xUbuntu_22.04/Release.key" | gpg --dearmor > /etc/apt/keyrings/crio-stable.gpg && \
+    echo "deb [signed-by=/etc/apt/keyrings/libcontainers-stable.gpg] https://downloadcontent.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_22.04/ /" > /etc/apt/sources.list.d/devel-kubic-libcontainers-stable.list && \
+    echo "deb [signed-by=/etc/apt/keyrings/crio-stable.gpg] https://downloadcontent.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.24/xUbuntu_22.04/ /" > /etc/apt/sources.list.d/devel-kubic-libcontainers-crio-stable.list && \
+    apt-get update -y || true 

node-installer/tests/integration-test-k3s.sh

@@ -51,9 +51,16 @@ echo "Applying KWasm node installer job..."
 kubectl apply -f ./k3s-kwasm-job.yml
 
 echo "Waiting for node installer job to complete..."
-kubectl wait -n kwasm --for=condition=Ready pod --selector=job-name=k3s-provision-kwasm --timeout=90s || true
 kubectl wait -n kwasm --for=jsonpath='{.status.phase}'=Succeeded pod --selector=job-name=k3s-provision-kwasm --timeout=60s
 
+# Verify the SystemdCgroup is set to true
+if sudo cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml | grep -A2 "runtimes.spin.options" | grep -q "SystemdCgroup = true"; then
+  echo "SystemdCgroup is set to true"
+else
+  echo "SystemdCgroup is not set to true"
+  exit 1
+fi
+
 if ! kubectl get pods -n kwasm | grep -q "k3s-provision-kwasm.*Completed"; then
   echo "Node installer job failed!"
   kubectl logs -n kwasm $(kubectl get pods -n kwasm -o name | grep k3s-provision-kwasm)

node-installer/tests/integration-test-kind.sh

@@ -55,6 +55,14 @@ echo "Waiting for node installer job to complete..."
 kubectl --context=kind-spin-test wait -n kwasm --for=condition=Ready pod --selector=job-name=spin-test-control-plane-provision-kwasm --timeout=90s || true
 kubectl --context=kind-spin-test wait -n kwasm --for=jsonpath='{.status.phase}'=Succeeded pod --selector=job-name=spin-test-control-plane-provision-kwasm --timeout=60s
 
+# Verify the SystemdCgroup is set to true
+if docker exec spin-test-control-plane cat /etc/containerd/config.toml | grep -A5 "spin" | grep -q "SystemdCgroup = true"; then
+  echo "SystemdCgroup is set to true"
+else
+  echo "SystemdCgroup is not set to true"
+  exit 1
+fi
+
 if ! kubectl --context=kind-spin-test get pods -n kwasm | grep -q "spin-test-control-plane-provision-kwasm.*Completed"; then
   echo "Node installer job failed!"
   kubectl --context=kind-spin-test logs -n kwasm $(kubectl --context=kind-spin-test get pods -n kwasm -o name | grep spin-test-control-plane-provision-kwasm)

node-installer/tests/integration-test-microk8s.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 set -euo pipefail
+shopt -s expand_aliases
 
 : ${IMAGE_NAME:=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev}
 
@@ -12,12 +13,12 @@ fi
 if ! microk8s status | grep -q "microk8s is running"; then
   echo "Starting MicroK8s..."
   sudo microk8s start
-  sleep 10
 else
   sudo microk8s reset
-  sleep 10
 fi
 
+sudo microk8s status --wait-ready
+
 sudo microk8s enable dns
 
 alias kubectl='sudo microk8s kubectl'
@@ -59,9 +60,16 @@ echo "Applying KWasm node installer job..."
 kubectl apply -f ./microk8s-kwasm-job.yml
 
 echo "Waiting for node installer job to complete..."
-kubectl wait -n kwasm --for=condition=Ready pod --selector=job-name=microk8s-provision-kwasm --timeout=90s || true
 kubectl wait -n kwasm --for=jsonpath='{.status.phase}'=Succeeded pod --selector=job-name=microk8s-provision-kwasm --timeout=60s
 
+# Ensure the SystemdCgroup is not set to true
+if sudo cat /var/snap/microk8s/current/args/containerd.toml | grep -A5 "spin" | grep -q "SystemdCgroup = true"; then
+  echo "Failed: SystemdCgroup is set to true"
+  exit 1
+else
+  echo "Passed: SystemdCgroup is not set to true"
+fi
+
 if ! kubectl get pods -n kwasm | grep -q "microk8s-provision-kwasm.*Completed"; then
   echo "Node installer job failed!"
   kubectl logs -n kwasm $(kubectl get pods -n kwasm -o name | grep microk8s-provision-kwasm)
@@ -72,7 +80,11 @@ echo "=== Step 4: Apply the workload ==="
 kubectl apply -f ./tests/workloads/workload.yaml
 
 echo "Waiting for deployment to be ready..."
-kubectl wait --for=condition=Available deployment/wasm-spin --timeout=120s
+if ! kubectl wait --for=condition=Available deployment/wasm-spin --timeout=120s; then
+  echo "Deployment failed to become ready!"
+  kubectl describe deployment wasm-spin
+  exit 1
+fi
 
 echo "Checking pod status..."
 kubectl get pods

node-installer/tests/integration-test-minikube.sh

@@ -4,7 +4,8 @@ set -euo pipefail
 : ${IMAGE_NAME:=ghcr.io/spinkube/containerd-shim-spin/node-installer:dev}
 
 echo "=== Step 1: Create a MiniKube cluster ==="
-minikube start -p minikube --driver=docker --container-runtime=containerd
+docker build -t minikube-custom:v0.0.46-fixed -f ./tests/Dockerfile.minikube-custom .
+minikube start -p minikube --driver=docker --container-runtime=containerd --base-image="minikube-custom:v0.0.46-fixed"
 
 echo "=== Step 2: Create namespace and deploy RuntimeClass ==="
 kubectl create namespace kwasm || true
@@ -44,6 +45,14 @@ echo "Waiting for node installer job to complete..."
 kubectl wait -n kwasm --for=condition=Ready pod --selector=job-name=minikube-provision-kwasm --timeout=90s || true
 kubectl wait -n kwasm --for=jsonpath='{.status.phase}'=Succeeded pod --selector=job-name=minikube-provision-kwasm --timeout=60s
 
+# Verify the SystemdCgroup is set to true
+if docker exec $NODE_NAME cat /etc/containerd/config.toml | grep -A5 "spin" | grep -q "SystemdCgroup = true"; then
+  echo "SystemdCgroup is set to true"
+else
+  echo "SystemdCgroup is not set to true"
+  exit 1
+fi
+
 if ! kubectl get pods -n kwasm | grep -q "minikube-provision-kwasm.*Completed"; then
   echo "Node installer job failed!"
   kubectl logs -n kwasm $(kubectl get pods -n kwasm -o name | grep minikube-provision-kwasm)