ci: add test for rootful docker

This is important to run on multi-node

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

Author: vsoch

.github/workflows/main.yaml

@@ -20,16 +20,23 @@ jobs:
         include:
           - lima_template: template://ubuntu-24.04
             container_engine: docker
+            rootful: "false"
+          - lima_template: template://docker-rootful
+            container_engine: docker
+            rootful: "true"
           - lima_template: template://ubuntu-24.04
             container_engine: nerdctl
+            rootful: "false"
           - lima_template: template://centos-stream-9
             container_engine: podman
           - lima_template: template://fedora
             container_engine: podman
+            rootful: "false"
     uses: ./.github/workflows/reusable-multi-node.yaml
     with:
       lima_template: ${{ matrix.lima_template }}
       container_engine: ${{ matrix.container_engine }}
+      rootful: ${{ matrix.rootful }}
 
   # TODO: this test should create multiple instances of Usernetes on each of the hosts
   multi-node-custom-ports:

.github/workflows/reusable-multi-node.yaml

@@ -19,6 +19,10 @@ on:
         description: flannel vxlan port
         type: string
         default: "8472"
+      rootful:
+        description: use rootful mode for a container technology
+        type: string
+        default: "false"
       etcd_port:
         description: etcd service port
         type: string
@@ -41,6 +45,7 @@ jobs:
     env:
       LIMA_TEMPLATE: "${{ inputs.lima_template }}"
       CONTAINER_ENGINE: "${{ inputs.container_engine }}"
+      CONTAINER_ROOTFUL: "${{ inputs.rootful }}"
       PORT_KUBE_APISERVER: "${{ inputs.kube_apiserver_port }}"
       PORT_FLANNEL: "${{ inputs.flannel_port }}"
       PORT_KUBELET: "${{ inputs.kubelet_port }}"

hack/create-cluster-lima.sh

@@ -24,15 +24,15 @@ for host in host0 host1; do
 	# Set --plain to minimize Limaism
 	${LIMACTL} start --plain --network lima:user-v2 --name="${host}" ${LIMACTL_CREATE_ARGS} "${LIMA_TEMPLATE}"
 	${LIMACTL} copy -r "$(pwd)" "${host}:${guest_home}/usernetes"
-	${LIMACTL} shell "${host}" sudo CONTAINER_ENGINE="${CONTAINER_ENGINE}" "${guest_home}/usernetes/init-host/init-host.root.sh"
+	${LIMACTL} shell "${host}" sudo CONTAINER_ENGINE="${CONTAINER_ENGINE}" CONTAINER_ROOTFUL="${CONTAINER_ROOTFUL}" "${guest_home}/usernetes/init-host/init-host.root.sh"
 	# Terminate the current session so that the cgroup delegation takes an effect. This command exits with status 255 as SSH terminates.
 	${LIMACTL} shell "${host}" sudo loginctl terminate-user "${USER}" || true
 	${LIMACTL} shell "${host}" sudo loginctl enable-linger "${USER}"
 	if [ "${LOCKDOWN_SUDO}" = "1" ]; then
 		# Lockdown sudo to ensure rootless-ness
 		${LIMACTL} shell "${host}" sudo sh -euxc 'rm -rf /etc/sudoers.d/*-cloud-init-users'
 	fi
-	${LIMACTL} shell "${host}" CONTAINER_ENGINE="${CONTAINER_ENGINE}" "${guest_home}/usernetes/init-host/init-host.rootless.sh"
+	${LIMACTL} shell "${host}" CONTAINER_ENGINE="${CONTAINER_ENGINE}" CONTAINER_ROOTFUL="${CONTAINER_ROOTFUL}" "${guest_home}/usernetes/init-host/init-host.rootless.sh"
 done
 
 SERVICE_PORTS="PORT_KUBE_APISERVER=${PORT_KUBE_APISERVER} PORT_ETCD=${PORT_ETCD} PORT_FLANNEL=${PORT_FLANNEL} PORT_KUBELET=${PORT_KUBELET}"

init-host/init-host.root.sh

@@ -7,6 +7,7 @@ if [ "$(id -u)" != "0" ]; then
 fi
 
 : "${CONTAINER_ENGINE:=docker}"
+: "${CONTAINER_ROOTFUL:=false}"
 script_dir="$(dirname "$0")"
 
 if [ ! -e /etc/systemd/system/user@.service.d/delegate.conf ]; then
@@ -64,8 +65,12 @@ else
 	apt-get install -y git uidmap make jq
 fi
 
-case "${CONTAINER_ENGINE}" in
-"docker")
+setup_docker() {
+	if [ "${CONTAINER_ROOTFUL}" = "true" ]; then
+		echo "Preparing to run docker in default rootful mode."	
+		return
+	fi
+	echo "Preparing to run docker in rootless mode."	
 	if ! command -v dockerd-rootless-setuptool.sh >/dev/null 2>&1; then
 		if grep -q centos /etc/os-release; then
 			# Works with Rocky and Alma too
@@ -76,6 +81,11 @@ case "${CONTAINER_ENGINE}" in
 		fi
 	fi
 	systemctl disable --now docker
+}
+
+case "${CONTAINER_ENGINE}" in
+"docker")
+	setup_docker
 	;;
 "podman")
 	if ! command -v podman-compose >/dev/null 2>&1; then

init-host/init-host.rootless.sh

@@ -7,10 +7,23 @@ if [ "$(id -u)" == "0" ]; then
 fi
 
 : "${CONTAINER_ENGINE:=docker}"
+: "${CONTAINER_ROOTFUL:=false}"
 : "${XDG_CONFIG_HOME:=${HOME}/.config}"
+
+setup_docker_rootless() {
+	if [ "${CONTAINER_ROOTFUL}" = "true" ]; then
+		return
+	fi
+	dockerd-rootless-setuptool.sh install || (journalctl --user --since "10 min ago"; exit 1)
+}
+
 case "${CONTAINER_ENGINE}" in
 "docker")
-	dockerd-rootless-setuptool.sh install || (journalctl --user --since "10 min ago"; exit 1)
+	setup_docker_rootless
+	;;
+"docker-rootful")
+	echo "Skipping rootless install of docker"
+	CONTAINER_ENGINE="docker"
 	;;
 "nerdctl")
 	containerd-rootless-setuptool.sh install