Private cgroupns 2

.github/workflows/go.yaml

@@ -92,4 +92,27 @@ jobs:
       run: go test -timeout 5m -v ./pkg/...
 
     - name: Run tests (e2e)
+      continue-on-error: true
       run: go test -timeout 15m -v ./tests -args -image=${{ matrix.image }}
+
+    - name: Inspect docker info
+      run: docker info --format '{{json .}}'
+
+    - name: Inspect daemon json file
+      run: cat /etc/docker/daemon.json
+
+    - name: Collect Docker logs using journalctl
+      if: always()
+      run: |
+        sudo journalctl -u docker --no-pager --since "1 hour ago"
+
+    - name: Collect Docker container logs
+      if: always()
+      run: |
+        for container in $(docker ps -a -q); do
+          echo "Logs for container $container:"
+          docker logs $container || true
+        done
+        docker ps -a
+        docker images
+

cmd/bootloose/config_create.go

@@ -7,6 +7,8 @@ package bootloose
 import (
 	"fmt"
 	"os"
+	"path/filepath"
+	"strings"
 
 	"github.com/k0sproject/bootloose/pkg/cluster"
 	"github.com/k0sproject/bootloose/pkg/config"
@@ -17,6 +19,7 @@ import (
 type configCreateOptions struct {
 	override bool
 	config   config.Config
+	volumes  []string
 }
 
 func NewConfigCreateCommand() *cobra.Command {
@@ -50,6 +53,8 @@ func NewConfigCreateCommand() *cobra.Command {
 	containerCmd := &opts.config.Machines[0].Spec.Cmd
 	cmd.Flags().StringVarP(containerCmd, "cmd", "d", *containerCmd, "The command to execute on the container")
 
+	cmd.Flags().StringSliceVarP(&opts.volumes, "volume", "v", nil, "Volumes to mount in the container")
+
 	return cmd
 }
 
@@ -72,6 +77,45 @@ func (opts *configCreateOptions) create(cmd *cobra.Command, args []string) error
 	if configExists(cfgFile) && !opts.override {
 		return fmt.Errorf("configuration file at %s already exists", cfgFile)
 	}
+	for _, v := range opts.volumes {
+		volume, err := parseVolume(v)
+		if err != nil {
+			return err
+		}
+		for _, machine := range opts.config.Machines {
+			machine.Spec.Volumes = append(machine.Spec.Volumes, volume)
+		}
+	}
 	return cluster.Save(cfgFile)
 }
 
+// volume flags can be in the form of:
+// -v /host/path:/container/path (bind mount)
+// -v volume:/container/path (volume mount)
+// or contain the permissions field:
+// -v /host/path:/container/path:ro (bind mount (read only))
+// -v volume:/container/path:rw (volume mount (read write))
+func parseVolume(v string) (config.Volume, error) {
+	if v == "" {
+		return config.Volume{}, fmt.Errorf("empty volume value")
+	}
+	parts := strings.Split(v, ":")
+	if len(parts) < 2 || len(parts) > 3 {
+		return config.Volume{}, fmt.Errorf("invalid volume value: %v", v)
+	}
+
+	vol := config.Volume{}
+	if filepath.IsAbs(parts[0]) {
+		vol.Type = "bind"
+	} else {
+		vol.Type = "volume"
+	}
+
+	if len(parts) == 3 {
+		vol.ReadOnly = parts[2] == "ro"
+	}
+
+	vol.Source = parts[0]
+	vol.Destination = parts[1]
+	return vol, nil
+}

pkg/cluster/cluster.go

@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -86,7 +87,7 @@ func (c *Cluster) Save(path string) error {
 	if err != nil {
 		return err
 	}
-	return os.WriteFile(path, data, 0666)
+	return os.WriteFile(path, data, 0o666)
 }
 
 func f(format string, args ...interface{}) string {
@@ -124,7 +125,7 @@ func (c *Cluster) forEachMachine(do func(*Machine, int) error) error {
 	for _, template := range c.spec.Machines {
 		for i := 0; i < template.Count; i++ {
 			// machine name indexed with i
-			machine := c.machine(&template.Spec, i)
+			machine := c.machine(template.Spec, i)
 			// but to prevent port collision, we use machineIndex for the real machine creation
 			if err := do(machine, machineIndex); err != nil {
 				return err
@@ -143,7 +144,7 @@ func (c *Cluster) forSpecificMachines(do func(*Machine, int) error, machineNames
 	}
 	for _, template := range c.spec.Machines {
 		for i := 0; i < template.Count; i++ {
-			machine := c.machine(&template.Spec, i)
+			machine := c.machine(template.Spec, i)
 			_, ok := machineToStart[machine.name]
 			if ok {
 				if err := do(machine, i); err != nil {
@@ -192,10 +193,10 @@ func (c *Cluster) ensureSSHKey() error {
 	sshPubBytes, sshPrivBytes := ssh.MarshalAuthorizedKey(sshPub), pem.EncodeToMemory(privPEM)
 
 	// Save the key pair (unencrypted).
-	if err := os.WriteFile(path, sshPrivBytes, 0600); err != nil {
+	if err := os.WriteFile(path, sshPrivBytes, 0o600); err != nil {
 		return fmt.Errorf("failed to save private key: %w", err)
 	}
-	if err := os.WriteFile(path+".pub", sshPubBytes, 0644); err != nil {
+	if err := os.WriteFile(path+".pub", sshPubBytes, 0o644); err != nil {
 		return fmt.Errorf("failed to save public key: %w", err)
 	}
 
@@ -257,13 +258,15 @@ func (c *Cluster) CreateMachine(machine *Machine, i int) error {
 	}
 
 	runArgs := c.createMachineRunArgs(machine, name, i)
+	log.Infof("Create machine: image: %v runArgs: %+v cmd: %v", machine.spec.Image, runArgs, cmd)
 	_, err = docker.Create(machine.spec.Image,
 		runArgs,
 		[]string{cmd},
 	)
 	if err != nil {
 		return err
 	}
+	log.Infof("done that")
 
 	if len(machine.spec.Networks) > 1 {
 		for _, network := range machine.spec.Networks[1:] {
@@ -280,6 +283,7 @@ func (c *Cluster) CreateMachine(machine *Machine, i int) error {
 		}
 	}
 
+	log.Infof("starting container %v", name)
 	if err := docker.Start(name); err != nil {
 		return err
 	}
@@ -302,17 +306,64 @@ func (c *Cluster) createMachineRunArgs(machine *Machine, name string, i int) []s
 		"--label", "io.k0sproject.bootloose.cluster=" + c.spec.Cluster.Name,
 		"--name", name,
 		"--hostname", machine.Hostname(),
-		"--tmpfs", "/run",
-		"--tmpfs", "/run/lock",
+		"--tmpfs", "/run:rw,size=100m,mode=755",
+		"--tmpfs", "/run/lock:rw,size=100m,mode=755",
 		"--tmpfs", "/tmp:exec,mode=777",
 	}
 	if docker.CgroupVersion() == "2" {
-		runArgs = append(runArgs, "--cgroupns", "host",
-			"--cgroup-parent", "bootloose.slice",
-			"-v", "/sys/fs/cgroup:/sys/fs/cgroup:rw")
-
+		runArgs = append(runArgs, "--cgroupns", "private")
+
+		if !machine.spec.Privileged {
+			// Non-privileged containers will have their /sys/fs/cgroup folder
+			// mounted read-only, even when running in private cgroup
+			// namespaces. This is a bummer for init systems. Containers could
+			// probably remount the cgroup fs in read-write mode, but that would
+			// require CAP_SYS_ADMIN _and_ a custom logic in the container's
+			// entry point. Podman has `--security-opt unmask=/sys/fs/cgroup`,
+			// but that's not a thing for Docker. The only other way to get a
+			// writable cgroup fs inside the container is to explicitly mount
+			// it. Some references:
+			//   - https://github.com/moby/moby/issues/42275
+			//   - https://serverfault.com/a/1054414
+
+			// Docker will use cgroups like
+			//   <cgroup-parent>/docker-{{ContainerID}}.scope.
+			//
+			// Ideally, we could mount those to /sys/fs/cgroup inside the
+			// containers. But there's some chicken-and-egg problem, as we only
+			// know the container ID _after_ the container creation. As a
+			// duct-tape solution, we mount our own cgroup as the root, which is
+			// unrelated to the Docker-managed one:
+			//   <cgroup-parent>/cluster-{{ClusterID}}.scope/machine-{{MachineID}}.scope
+
+			// FIXME: How to clean this up? Especially when Docker is being run
+			// on a different machine?
+
+			// Just assume that the cgroup fs is mounted at its default
+			// location. We could try to figure this out via
+			// /proc/self/mountinfo, but it's really not worth the hassle.
+			const cgroupMountpoint = "/sys/fs/cgroup"
+
+			// Use this as the parent cgroup for everything. Note that if Docker
+			// uses the systemd cgroup driver, the cgroup name has to end with
+			// .slice. This is not a requirement for the cgroupfs driver; it
+			// won't care. Hence, just always use the .slice suffix, no matter
+			// if it's required or not.
+			const cgroupParent = "/actions_job/bootloose.slice"
+
+			cg := path.Join(
+				cgroupMountpoint, cgroupParent,
+				fmt.Sprintf("cluster-%s.scope", c.spec.Cluster.Name),
+				fmt.Sprintf("machine-%s.scope", name),
+			)
+
+			runArgs = append(runArgs,
+				"--cgroup-parent", cgroupParent,
+				"-v", fmt.Sprintf("%s:%s:rw", cg, cgroupMountpoint),
+			)
+		}
 	} else {
-		runArgs = append(runArgs, "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro")
+		runArgs = append(runArgs, "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro", "--privileged")
 	}
 
 	for _, volume := range machine.spec.Volumes {
@@ -342,10 +393,6 @@ func (c *Cluster) createMachineRunArgs(machine *Machine, name string, i int) []s
 		runArgs = append(runArgs, "-p", publish)
 	}
 
-	if machine.spec.Privileged {
-		runArgs = append(runArgs, "--privileged")
-	}
-
 	if len(machine.spec.Networks) > 0 {
 		network := machine.spec.Networks[0]
 		log.Infof("Connecting %s to the %s network...", name, network)
@@ -499,7 +546,7 @@ func (c *Cluster) gatherMachinesByCluster() (machines []*Machine) {
 	for _, template := range c.spec.Machines {
 		for i := 0; i < template.Count; i++ {
 			s := template.Spec
-			machine := c.machine(&s, i)
+			machine := c.machine(s, i)
 			machines = append(machines, machine)
 		}
 	}
@@ -626,7 +673,7 @@ func (c *Cluster) machineFromHostname(hostname string) (*Machine, error) {
 	for _, template := range c.spec.Machines {
 		for i := 0; i < template.Count; i++ {
 			if hostname == f(template.Spec.Name, i) {
-				return c.machine(&template.Spec, i), nil
+				return c.machine(template.Spec, i), nil
 			}
 		}
 	}

pkg/cluster/cluster_test.go

@@ -55,13 +55,13 @@ machines:
 	assert.Equal(t, uint16(22), portMapping.ContainerPort)
 	assert.Equal(t, uint16(2222), portMapping.HostPort)
 
-	machine0 := cluster.machine(&template.Spec, 0)
+	machine0 := cluster.machine(template.Spec, 0)
 	args0 := cluster.createMachineRunArgs(machine0, machine0.ContainerName(), 0)
 	i := indexOf("-p", args0)
 	assert.NotEqual(t, -1, i)
 	assert.Equal(t, "2222:22", args0[i+1])
 
-	machine1 := cluster.machine(&template.Spec, 1)
+	machine1 := cluster.machine(template.Spec, 1)
 	args1 := cluster.createMachineRunArgs(machine1, machine1.ContainerName(), 1)
 	i = indexOf("-p", args1)
 	assert.NotEqual(t, -1, i)
@@ -96,13 +96,12 @@ func TestCluster_EnsureSSHKeys(t *testing.T) {
 
 		privStat, err = os.Stat(keyPath)
 		if assert.NoError(t, err, "failed to stat private key file") {
-			assert.Equal(t, privStat.Mode().Perm(), os.FileMode(0600), "private key file has wrong permissions")
-
+			assert.Equal(t, privStat.Mode().Perm(), os.FileMode(0o600), "private key file has wrong permissions")
 		}
 
 		pubStat, err = os.Stat(keyPath + ".pub")
 		if assert.NoError(t, err, "failed to stat public key file") {
-			assert.Equal(t, pubStat.Mode().Perm(), os.FileMode(0644), "public key file has wrong permissions")
+			assert.Equal(t, pubStat.Mode().Perm(), os.FileMode(0o644), "public key file has wrong permissions")
 		}
 	})
 

pkg/config/cluster.go

@@ -30,8 +30,8 @@ func NewConfigFromFile(path string) (*Config, error) {
 
 // MachineReplicas are a number of machine following the same specification.
 type MachineReplicas struct {
-	Spec  Machine `json:"spec"`
-	Count int     `json:"count"`
+	Spec  *Machine `json:"spec"`
+	Count int      `json:"count"`
 }
 
 // Cluster is a set of Machines.
@@ -85,7 +85,7 @@ func DefaultConfig() Config {
 		Machines: []MachineReplicas{
 			{
 				Count: 1,
-				Spec: Machine{
+				Spec: &Machine{
 					Name:  "node%d",
 					Image: "quay.io/k0sproject/bootloose-ubuntu20.04",
 					PortMappings: []PortMapping{

pkg/config/get_test.go

@@ -10,9 +10,9 @@ func TestGetValueFromConfig(t *testing.T) {
 	config := Config{
 		Cluster: Cluster{Name: "clustername", PrivateKey: "privatekey"},
 		Machines: []MachineReplicas{
-			MachineReplicas{
+			{
 				Count: 3,
-				Spec: Machine{
+				Spec: &Machine{
 					Image:      "myImage",
 					Name:       "myName",
 					Privileged: true,
@@ -32,7 +32,7 @@ func TestGetValueFromConfig(t *testing.T) {
 			"cluster.name",
 			Config{
 				Cluster:  Cluster{Name: "clustername", PrivateKey: "privatekey"},
-				Machines: []MachineReplicas{MachineReplicas{Count: 3, Spec: Machine{}}},
+				Machines: []MachineReplicas{{Count: 3, Spec: &Machine{}}},
 			},
 			"clustername",
 		},

pkg/docker/create.go

@@ -26,6 +26,8 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
+	goexec "os/exec"
+
 	"github.com/k0sproject/bootloose/pkg/exec"
 )
 
@@ -37,6 +39,7 @@ func Create(image string, runArgs []string, containerArgs []string) (id string,
 	args = append(args, image)
 	args = append(args, containerArgs...)
 	cmd := exec.Command("docker", args...)
+	cmd.SetEnv("DOCKER_LOG_LEVEL=debug")
 	var stdout, stderr bytes.Buffer
 	cmd.SetStdout(&stdout)
 	cmd.SetStderr(&stderr)
@@ -60,5 +63,27 @@ func Create(image string, runArgs []string, containerArgs []string) (id string,
 	if !containerIDRegex.MatchString(outputLines[0]) {
 		return "", fmt.Errorf("failed to get container id, output did not match: %v", outputLines)
 	}
-	return outputLines[0], nil
+	containerID := outputLines[0]
+
+	// Check container status
+	statusCmd := goexec.Command("docker", "ps", "-a", "--filter", "id="+containerID, "--format", "{{.Status}}")
+	var statusOut bytes.Buffer
+	statusCmd.Stdout = &statusOut
+	if err := statusCmd.Run(); err != nil {
+		log.Printf("Error checking container status: %s", err)
+		return "", err
+	}
+	log.Printf("Container status: %s", statusOut.String())
+
+	// Capture container logs
+	logCmd := goexec.Command("docker", "logs", containerID)
+	var logOut bytes.Buffer
+	logCmd.Stdout = &logOut
+	if err := logCmd.Run(); err != nil {
+		log.Printf("Error capturing container logs: %s", err)
+		return "", err
+	}
+	log.Printf("Container logs: %s", logOut.String())
+
+	return containerID, nil
 }