Merge pull request containerd#9470 from lengrongfu/feat/add-e2e-for-systemdcgroup

add use systemd cgroup e2e

Author: dmcgowan

.github/workflows/ci.yml

@@ -219,6 +219,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [windows-2022, windows-2019]
+        cgroup_driver: [cgroupfs]
 
     defaults:
       run:
@@ -345,12 +346,14 @@ jobs:
       - name: CRI Integration Test
         env:
           TEST_IMAGE_LIST: ${{github.workspace}}/repolist.toml
+          CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
         run: |
           make cri-integration
 
       - name: cri-tools critest
         env:
           CRI_TEST_IMAGES: ${{github.workspace}}/cri-test-images.yaml
+          CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
         shell: powershell
         run: |
           Start-Process -FilePath containerd.exe -NoNewWindow -RedirectStandardError true -PassThru
@@ -367,7 +370,7 @@ jobs:
       - uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: TestResults ${{ matrix.os }}
+          name: TestResults ${{ matrix.os }} ${{ matrix.cgroup_driver }}
           path: |
             ${{github.workspace}}/*-junit.xml
             ${{github.workspace}}/*-gotest.json
@@ -388,6 +391,7 @@ jobs:
         os: [ubuntu-22.04, ubuntu-24.04, actuated-arm64-4cpu-16gb]
         exclude:
           - os: ${{ github.repository != 'containerd/containerd' && 'actuated-arm64-4cpu-16gb' }}
+        cgroup_driver: [cgroupfs, systemd]
 
     env:
       GOTEST: gotestsum --
@@ -476,7 +480,15 @@ jobs:
       - name: CRI Integration Test
         env:
           TEST_RUNTIME: ${{ matrix.runtime }}
+          CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
+          RUNC_FLAVOR: ${{ matrix.runc }}
         run: |
+          cat /sys/fs/cgroup/cgroup.controllers
+          systemctl status
+          [ "${RUNC_FLAVOR}" == "crun" ] && {
+             export XDG_RUNTIME_DIR=/run/user/$(id -u)
+          }
+          runc --version
           CONTAINERD_RUNTIME=$TEST_RUNTIME make cri-integration
 
       - name: Fix up for actuated
@@ -488,6 +500,7 @@ jobs:
       - name: cri-tools critest
         env:
           TEST_RUNTIME: ${{ matrix.runtime }}
+          CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
         run: |
           env
           sudo -E PATH=$PATH ./script/critest.sh "${{github.workspace}}/report"
@@ -510,7 +523,7 @@ jobs:
       - uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: TestResults ${{ matrix.runtime }} ${{matrix.runc}} ${{ matrix.os }}
+          name: TestResults ${{ matrix.runtime }} ${{matrix.runc}} ${{ matrix.os }} ${{ matrix.cgroup_driver }}
           path: |
             *-junit.xml
             *-gotest.json
@@ -534,9 +547,10 @@ jobs:
           # https://github.com/containerd/containerd/pull/10297
           - almalinux/8
           - rockylinux/[email protected]
+        cgroup_driver: [cgroupfs, systemd]
     env:
       BOX: ${{ matrix.box }}
-
+      CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
     steps:
       - name: Show the host info
         run: |
@@ -586,9 +600,9 @@ jobs:
       - name: test-integration
         run: sudo BOX=$BOX vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-integration
       - name: test-cri-integration
-        run: sudo BOX=$BOX vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri-integration
+        run: sudo BOX=$BOX CGROUP_DRIVER=$CGROUP_DRIVER vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri-integration
       - name: test-cri
-        run: sudo BOX=$BOX vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri
+        run: sudo BOX=$BOX CGROUP_DRIVER=$CGROUP_DRIVER vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri
 
   tests-cri-in-userns:
     name: "CRI-in-UserNS"
@@ -597,6 +611,11 @@ jobs:
     timeout-minutes: 40
     needs: [project, linters, protos, man]
 
+    strategy:
+      fail-fast: false
+      matrix:
+        cgroup_driver: [cgroupfs, systemd]
+
     steps:
       - uses: actions/checkout@v4
       - name: Set up cgroup v2 delegation
@@ -608,11 +627,35 @@ jobs:
           EOF
           sudo systemctl daemon-reload
       - name: Build cri-in-userns image
-        run: podman build --target cri-in-userns -t cri-in-userns -f ./contrib/Dockerfile.test .
+        env:
+          CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
+        run: |
+          if [ "$CGROUP_DRIVER" = "systemd" ];then
+            podman build --target cri-in-userns-systemd -t cri-in-userns-systemd -f ./contrib/Dockerfile.test .
+          else
+            podman build --target cri-in-userns -t cri-in-userns -f ./contrib/Dockerfile.test .
+          fi
+
       - name: Run cri-in-userns image
+        env:
+          CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
         # Rootless Podman is used for testing CRI-in-UserNS
         # (We could use rootless Docker or rootless nerdctl, but we are using Podman here because it is preinstalled)
-        run: podman run --rm --privileged cri-in-userns
+        run: |
+          if [ "$CGROUP_DRIVER" = "systemd" ];then
+              set +e
+              touch ./critest_exit_code.txt
+              podman run --rm --privileged --group-add keep-groups -v ./critest_exit_code.txt:/tmp/critest_exit_code.txt cri-in-userns-systemd
+              exit_code=`cat ./critest_exit_code.txt`
+              echo "exit_code:"$exit_code
+              if [ "$exit_code" -gt 0 ]; then
+                exit 1
+              else
+                exit 0
+              fi
+          else
+            podman run --rm --privileged cri-in-userns
+          fi
 
   tests-mac-os:
     name: MacOS unit tests

Vagrantfile

@@ -272,6 +272,7 @@ EOF
         'GOTESTSUM_JUNITFILE': ENV['GOTESTSUM_JUNITFILE'],
         'GOTESTSUM_JSONFILE': ENV['GOTESTSUM_JSONFILE'],
         'GITHUB_WORKSPACE': '',
+        'CGROUP_DRIVER': ENV['CGROUP_DRIVER'],
     }
     sh.inline = <<~SHELL
         #!/usr/bin/env bash
@@ -299,6 +300,7 @@ EOF
     sh.env = {
         'GOTEST': ENV['GOTEST'] || "go test",
         'REPORT_DIR': ENV['REPORT_DIR'],
+        'CGROUP_DRIVER': ENV['CGROUP_DRIVER'],
     }
     sh.inline = <<~SHELL
         #!/usr/bin/env bash

contrib/Dockerfile.test

@@ -27,6 +27,11 @@
 # "cri-in-userns": for running critest with "CRI-in-UserNS" mode; needs Rootless Docker/Podman/nerdctl:
 #   docker build -t containerd-test -f Dockerfile.test --target cri-in-userns ../
 #   docker run --privileged containerd-test
+#
+# "cri-in-userns-systemd": for running critest with "CRI-in-UserNS-Systemd" mode; needs Rootless Docker/Podman/nerdctl:
+#   docker build -t containerd-test -f Dockerfile.test --target cri-in-userns-systemd ../
+#   touch ./critest_exit_code.txt
+#   docker run --privileged --group-add keep-groups -v ./critest_exit_code.txt:/tmp/critest_exit_code.txt containerd-test
 # ------------------------------------------------------------------------------
 
 ARG GOLANG_VERSION=1.23.1
@@ -111,6 +116,7 @@ CMD ["script/critest.sh", "/tmp"]
 # Requires Rootless Docker/Podman/nerdctl with cgroup v2 delegation: https://rootlesscontaine.rs/getting-started/common/cgroup2/
 # (Rootless Docker/Podman/nerdctl prepares the UserNS, so we do not need to create UserNS by ourselves)
 FROM critest AS cri-in-userns
+ENV IS_SYSTEMD_CGROUP=false
 COPY contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml /etc/containerd/config.toml
 COPY contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh       /docker-entrypoint.sh
 ENTRYPOINT ["/docker-entrypoint.sh"]
@@ -119,6 +125,24 @@ ENTRYPOINT ["/docker-entrypoint.sh"]
 # Skip "should allow privilege escalation when (NoNewPrivis is) false": expected log "Effective uid: 0\n" (stream="stdout") not found in logs [{timestamp:{wall:974487519 ext:63761339984 loc:<nil>} stream:stdout log:Effective uid: 1000) }]
 CMD ["critest", "--ginkgo.skip=should support unsafe sysctls|should support safe sysctls|should allow privilege escalation when false"]
 
+# cri-in-userns-systemd stage is for testing "CRI-in-UserNS-Systemd", which should be used in conjunction with
+# "Kubelet-in-UserNS": https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2033-kubelet-in-userns-aka-rootless
+# This feature is mostly expected to be used for `kind` and `minikube`.
+#
+# Requires Rootless Docker/Podman/nerdctl with cgroup v2 delegation: https://rootlesscontaine.rs/getting-started/common/cgroup2/
+# (Rootless Docker/Podman/nerdctl prepares the UserNS, so we do not need to create UserNS by ourselves)
+FROM critest AS cri-in-userns-systemd
+ENV IS_SYSTEMD_CGROUP=true
+RUN apt-get update && apt-get install -y systemd sysvinit-core
+COPY contrib/Dockerfile.test.d/cri-in-userns/etc_containerd_config.toml /etc/containerd/config.toml
+COPY contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh       /docker-entrypoint.sh
+COPY contrib/Dockerfile.test.d/critest.sh /critest.sh
+# Skip "runtime should support unsafe sysctls": `container init caused:  write sysctl key fs.mqueue.msg_max: open /proc/sys/fs/mqueue/msg_max: permission denied`
+# Skip "runtime should support safe sysctls": `container init caused: write sysctl key kernel.shm_rmid_forced: open /proc/sys/kernel/shm_rmid_forced: permission denied`
+# Skip "should allow privilege escalation when (NoNewPrivis is) false": expected log "Effective uid: 0\n" (stream="stdout") not found in logs [{timestamp:{wall:974487519 ext:63761339984 loc:<nil>} stream:stdout log:Effective uid: 1000) }]
+ENTRYPOINT ["/critest.sh","start"]
+
+
 # Install proto3
 FROM golang AS proto3
 ARG DESTDIR=/build

contrib/Dockerfile.test.d/cri-in-userns/docker-entrypoint.sh

@@ -41,11 +41,23 @@ xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
 sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \
   > /sys/fs/cgroup/cgroup.subtree_control
 
+if [ ! -z "$IS_SYSTEMD_CGROUP" ] && [ "$IS_SYSTEMD_CGROUP" = true ];then
+  cat >> /etc/containerd/config.toml <<EOF
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+   SystemdCgroup = true
+EOF
+fi
+
 set -x
 echo >&2 "Running containerd in background"
 containerd &
 
 echo >&2 "Waiting for containerd"
 until ctr plugins list; do sleep 3; done
 
-exec "$@"
+if [ ! -z "$IS_SYSTEMD_CGROUP" ] && [ "$IS_SYSTEMD_CGROUP" = true ];then
+  critest "--ginkgo.skip=should support unsafe sysctls|should support safe sysctls|should allow privilege escalation when false"
+  /bin/bash /critest.sh exit
+else
+  exec "$@"
+fi

contrib/Dockerfile.test.d/critest.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+cat > /etc/systemd/system/critest.service << EOF
+[Unit]
+Description=critest script
+[Service]
+Type=simple
+Environment="IS_SYSTEMD_CGROUP=true"
+RemainAfterExit=yes
+ExecStart=/bin/bash /docker-entrypoint.sh
+StandardOutput=/dev/stdout
+StandardError=/dev/stderr
+[Install]
+WantedBy=default.target
+EOF
+
+
+function echo_exit_code() {
+    sleep 30
+    log_str=`systemctl status critest.service|grep "SUCCESS!"`
+    if [ -z "$log_str" ]; then
+        echo 1 > /tmp/critest_exit_code.txt
+        /bin/systemctl poweroff
+    fi
+    failed_count=$(echo "$log_str" | awk '{for(i=1;i<=NF;i++) if($i=="Failed") {print $(i-1); exit}}')
+    if [ "$failed_count" -gt 0 ]; then
+        echo 1 > /tmp/critest_exit_code.txt
+    else
+        echo 0 > /tmp/critest_exit_code.txt
+    fi
+    /bin/systemctl poweroff
+}
+
+function start(){
+  systemctl enable critest.service
+  journalctl -f &
+  exec /lib/systemd/systemd
+}
+
+case $1 in
+    start)
+      start
+      ;;
+    exit)
+      echo_exit_code
+      ;;
+esac

integration/nri_test.go

@@ -401,6 +401,7 @@ func TestNriLinuxCpusetAdjustmentUpdate(t *testing.T) {
 	)
 
 	t.Log("Test that NRI plugins can update linux cpusets of existing containers.")
+	t.Logf("availableCpuset values is %v", availableCpuset)
 
 	var (
 		out = t.TempDir()
@@ -423,10 +424,12 @@ func TestNriLinuxCpusetAdjustmentUpdate(t *testing.T) {
 					Type:        "bind",
 					Options:     []string{"bind"},
 				})
+				t.Logf("ctr0 availableCpuset values is %v", availableCpuset)
 				adjust.SetLinuxCPUSetCPUs(availableCpuset[0])
 			} else {
 				update = []*api.ContainerUpdate{{}}
 				update[0].SetContainerId(ctr0)
+				t.Logf("ctr1 availableCpuset values is %v", availableCpuset)
 				update[0].SetLinuxCPUSetCPUs(availableCpuset[1])
 			}
 			return adjust, update, nil

script/critest.sh

@@ -62,6 +62,14 @@ runtime_type = "${TEST_RUNTIME}"
 # This is safely ignored for kernel >= 5.19.
 slow_chown = true
 EOF
+
+if [ ! -z "$CGROUP_DRIVER" ] && [ "$CGROUP_DRIVER" = "systemd" ];then
+  cat >> ${BDIR}/config.toml <<EOF
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+   SystemdCgroup = true
+EOF
+fi
+
 ls /etc/cni/net.d
 
 /usr/local/bin/containerd \

script/test/utils.sh

@@ -167,6 +167,13 @@ version = 2
 EOF
 fi
 
+if [ $IS_WINDOWS -eq 0 ] && [ ! -z "$CGROUP_DRIVER" ] && [ "$CGROUP_DRIVER" = "systemd" ];then
+  cat >> ${CONTAINERD_CONFIG_FILE} << EOF
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+   SystemdCgroup = true
+EOF
+fi
+
 # CONTAINERD_TEST_SUFFIX is the suffix appended to the root/state directory used
 # by test containerd.
 CONTAINERD_TEST_SUFFIX=${CONTAINERD_TEST_SUFFIX:-"-test"}