Merge pull request #65 from germag/macos-switch-to-krunkit

Macos switch to krunkit

Author: germag

cmd/vmmon.go

@@ -0,0 +1,68 @@
+//go:build darwin
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"github.com/containers/podman-bootc/pkg/user"
+	"github.com/containers/podman-bootc/pkg/vm"
+
+	"github.com/spf13/cobra"
+)
+
+// We treat this command as internal, we don't expect the user to call
+// this directly. In the future this will be replaced by an external binary
+var (
+	monCmd = &cobra.Command{
+		Use:    "vmmon <ID> <username> <ssh identity> <ssh port>",
+		Hidden: true,
+		Args:   cobra.ExactArgs(4),
+		RunE:   doMon,
+	}
+	console bool
+)
+
+func init() {
+	RootCmd.AddCommand(monCmd)
+	runCmd.Flags().BoolVar(&console, "console", false, "Show boot console")
+}
+
+func doMon(_ *cobra.Command, args []string) error {
+	usr, err := user.NewUser()
+	if err != nil {
+		return err
+	}
+
+	ctx := context.Background()
+	fullImageId := args[0]
+	username := args[1]
+	sshIdentity := args[2]
+
+	sshPort, err := strconv.Atoi(args[3])
+	if err != nil {
+		return fmt.Errorf("invalid ssh port: %w", err)
+	}
+
+	cacheDir := filepath.Join(usr.CacheDir(), fullImageId)
+
+	// MacOS has a 104 bytes limit for a unix socket path
+	runDir := filepath.Join(usr.RunDir(), fullImageId[0:12])
+	if err := os.MkdirAll(runDir, os.ModePerm); err != nil {
+		return err
+	}
+
+	params := vm.MonitorParmeters{
+		CacheDir:    cacheDir,
+		RunDir:      runDir,
+		Username:    username,
+		SshIdentity: sshIdentity,
+		SshPort:     sshPort,
+	}
+
+	return vm.StartMonitor(ctx, params)
+}

go.mod

@@ -5,16 +5,17 @@ go 1.20
 require (
 	github.com/adrg/xdg v0.4.0
 	github.com/containers/common v0.58.1
+	github.com/containers/gvisor-tap-vsock v0.7.3
 	github.com/containers/podman/v5 v5.0.1
 	github.com/docker/go-units v0.5.0
 	github.com/gofrs/flock v0.8.1
 	github.com/onsi/ginkgo/v2 v2.17.1
 	github.com/onsi/gomega v1.32.0
-	github.com/opencontainers/runtime-spec v1.2.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.0
 	golang.org/x/crypto v0.22.0
 	golang.org/x/sys v0.19.0
+	golang.org/x/term v0.19.0
 	libvirt.org/go/libvirt v1.10002.0
 )
 
@@ -40,7 +41,6 @@ require (
 	github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
 	github.com/containerd/typeurl/v2 v2.1.1 // indirect
 	github.com/containers/buildah v1.35.3 // indirect
-	github.com/containers/gvisor-tap-vsock v0.7.3 // indirect
 	github.com/containers/image/v5 v5.30.0 // indirect
 	github.com/containers/libhvee v0.7.0 // indirect
 	github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect
@@ -134,6 +134,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/opencontainers/runc v1.1.12 // indirect
+	github.com/opencontainers/runtime-spec v1.2.0 // indirect
 	github.com/opencontainers/runtime-tools v0.9.1-0.20230914150019-408c51e934dc // indirect
 	github.com/opencontainers/selinux v1.11.0 // indirect
 	github.com/openshift/imagebuilder v1.2.6 // indirect
@@ -176,7 +177,6 @@ require (
 	golang.org/x/mod v0.15.0 // indirect
 	golang.org/x/net v0.22.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/term v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.18.0 // indirect

pkg/user/user.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"os/user"
 	"path/filepath"
+	"runtime"
 	"strconv"
 
 	"github.com/containers/podman-bootc/pkg/config"
@@ -57,6 +58,15 @@ func (u *User) DefaultIdentity() string {
 }
 
 func (u *User) RunDir() string {
+	if runtime.GOOS == "darwin" {
+		tmpDir, ok := os.LookupEnv("TMPDIR")
+		if !ok {
+			tmpDir = "/tmp"
+		}
+
+		return filepath.Join(tmpDir, config.ProjectName, "run")
+	}
+
 	return filepath.Join(xdg.RuntimeDir, config.ProjectName, "run")
 }
 

pkg/utils/file.go

@@ -3,8 +3,10 @@ package utils
 import (
 	"bytes"
 	"errors"
+	"fmt"
 	"os"
 	"strconv"
+	"time"
 )
 
 func ReadPidFile(pidFile string) (int, error) {
@@ -24,6 +26,15 @@ func ReadPidFile(pidFile string) (int, error) {
 	return int(pid), nil
 }
 
+func WritePidFile(pidFile string, pid int) error {
+	if pid < 1 {
+		// We might be running as PID 1 when running docker-in-docker,
+		// but 0 or negative PIDs are not acceptable.
+		return fmt.Errorf("invalid negative PID %d", pid)
+	}
+	return os.WriteFile(pidFile, []byte(strconv.Itoa(pid)), 0o644)
+}
+
 func FileExists(path string) (bool, error) {
 	_, err := os.Stat(path)
 	exists := false
@@ -35,3 +46,17 @@ func FileExists(path string) (bool, error) {
 	}
 	return exists, err
 }
+
+// WaitForFileWithBackoffs attempts to discover a file in maxBackoffs attempts
+func WaitForFileWithBackoffs(maxBackoffs int, backoff time.Duration, path string) error {
+	backoffWait := backoff
+	for i := 0; i < maxBackoffs; i++ {
+		e, _ := FileExists(path)
+		if e {
+			return nil
+		}
+		time.Sleep(backoffWait)
+		backoffWait *= 2
+	}
+	return fmt.Errorf("unable to find file at %q", path)
+}

pkg/utils/process.go

@@ -14,3 +14,13 @@ func IsProcessAlive(pid int) bool {
 	err = process.Signal(syscall.Signal(0))
 	return err == nil
 }
+
+// SendInterrupt sends SIGINT to pid
+func SendInterrupt(pid int) error {
+	process, err := os.FindProcess(pid)
+	if err != nil {
+		return err
+	}
+
+	return process.Signal(os.Interrupt)
+}

pkg/vm/gvproxy.go

@@ -0,0 +1,90 @@
+package vm
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/containers/podman-bootc/pkg/utils"
+
+	gvproxy "github.com/containers/gvisor-tap-vsock/pkg/types"
+	"github.com/sirupsen/logrus"
+)
+
+const gvproxyBinaryName = "gvproxy"
+
+type Vmm int
+
+const (
+	pidFileName = "gvproxy.pid"
+	socketFile  = "net.sock"
+	// How log we should wait for gvproxy to be ready
+	maxBackoffs = 5
+	backoff     = time.Millisecond * 200
+)
+
+type gvproxyParams struct {
+	SshPort int
+}
+
+type gvproxyDaemon struct {
+	socketPath string
+	pidFile    string
+	cmd        *exec.Cmd
+}
+
+func newGvproxy(ctx context.Context, binaryPath, rundir string, param gvproxyParams) *gvproxyDaemon {
+	socketPath := filepath.Join(rundir, socketFile)
+	pidFile := filepath.Join(rundir, pidFileName)
+
+	gvpCmd := gvproxy.NewGvproxyCommand()
+	gvpCmd.SSHPort = param.SshPort
+	gvpCmd.PidFile = pidFile
+	gvpCmd.AddVfkitSocket(fmt.Sprintf("unixgram://%s", socketPath))
+
+	cmdLine := gvpCmd.ToCmdline()
+	cmd := exec.CommandContext(ctx, binaryPath, cmdLine...)
+	logrus.Debugf("gvproxy command-line: %s %s", binaryPath, strings.Join(cmdLine, " "))
+
+	return &gvproxyDaemon{socketPath: socketPath, pidFile: pidFile, cmd: cmd}
+}
+
+// Start spawn the gvproxy daemon, killing any running daemon using the same unix socket file.
+// It blocks until the unix socket file exists, this does not guarantee that the socket will be
+// on listen state
+func (d *gvproxyDaemon) start() error {
+	cleanup(d.pidFile, d.socketPath)
+
+	if err := d.cmd.Start(); err != nil {
+		return fmt.Errorf("unable to start gvproxy: %w", err)
+	}
+
+	// this is racy, the socket file could exist but not be in the listen state yet
+	if err := utils.WaitForFileWithBackoffs(maxBackoffs, backoff, d.socketPath); err != nil {
+		return fmt.Errorf("waiting for gvproxy socket: %w", err)
+	}
+	return nil
+}
+
+func (d *gvproxyDaemon) stop() error {
+	return d.cmd.Cancel()
+}
+
+func (d *gvproxyDaemon) wait() error {
+	return d.cmd.Wait()
+}
+
+func cleanup(pidFile, socketPath string) {
+	// Let's kill any possible running daemon
+	pid, err := utils.ReadPidFile(pidFile)
+	if err == nil {
+		_ = utils.SendInterrupt(pid)
+	}
+
+	_ = os.Remove(pidFile)
+	_ = os.Remove(socketPath)
+}

pkg/vm/krunkit.go

@@ -0,0 +1,110 @@
+package vm
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"strconv"
+	"strings"
+
+	"github.com/containers/podman-bootc/pkg/utils"
+
+	"github.com/sirupsen/logrus"
+)
+
+const krunkitBinaryName = "krunkit"
+
+const defaultCpus = 4
+const defaultMemory = 2048
+
+type krunkitParams struct {
+	disk      string
+	netSocket string
+	oemString string
+	pidFile   string
+}
+
+type krunkit struct {
+	pidFile string
+	cmd     *exec.Cmd
+}
+
+func newKrunkit(ctx context.Context, binaryPath string, params krunkitParams) *krunkit {
+	cmdLine := newKrunkitCmdLine(defaultCpus, defaultMemory)
+	cmdLine.addRngDevice()
+	cmdLine.addBlockDevice(params.disk)
+	cmdLine.addNetworkDevice(params.netSocket)
+	cmdLine.addOemString(params.oemString)
+
+	cmdLineSlice := cmdLine.asSlice()
+	cmd := exec.CommandContext(ctx, binaryPath, cmdLineSlice...)
+	logrus.Debugf("krunkit command-line: %s %s", binaryPath, strings.Join(cmdLineSlice, " "))
+
+	return &krunkit{cmd: cmd, pidFile: params.pidFile}
+}
+
+func (k *krunkit) start() error {
+	if err := k.cmd.Start(); err != nil {
+		return fmt.Errorf("unable to start krunkit: %w", err)
+	}
+
+	if err := utils.WritePidFile(k.pidFile, k.cmd.Process.Pid); err != nil {
+		if err := k.cmd.Cancel(); err != nil {
+			logrus.Debugf("stopping krunkit: %v", err)
+		}
+		return fmt.Errorf("writing pid file %s: %w", k.pidFile, err)
+	}
+
+	return nil
+}
+
+func (k *krunkit) wait() error {
+	return k.cmd.Wait()
+}
+
+type krunkitCmdLine struct {
+	cpus      int
+	memory    int
+	devices   []string
+	oemString []string
+}
+
+func newKrunkitCmdLine(cpus int, memory int) *krunkitCmdLine {
+	return &krunkitCmdLine{cpus: cpus, memory: memory}
+}
+
+func (kc *krunkitCmdLine) addDevice(device string) {
+	kc.devices = append(kc.devices, device)
+}
+
+func (kc *krunkitCmdLine) addRngDevice() {
+	kc.addDevice("virtio-rng")
+}
+
+func (kc *krunkitCmdLine) addBlockDevice(diskAbsPath string) {
+	kc.addDevice(fmt.Sprintf("virtio-blk,path=%s", diskAbsPath))
+}
+
+func (kc *krunkitCmdLine) addNetworkDevice(socketAbsPath string) {
+	kc.addDevice(fmt.Sprintf("virtio-net,unixSocketPath=%s,mac=5a:94:ef:e4:0c:ee", socketAbsPath))
+}
+
+func (kc *krunkitCmdLine) addOemString(oemStr string) {
+	kc.oemString = append(kc.oemString, oemStr)
+}
+
+func (kc *krunkitCmdLine) asSlice() []string {
+	args := []string{}
+
+	args = append(args, "--cpus", strconv.Itoa(kc.cpus))
+	args = append(args, "--memory", strconv.Itoa(kc.memory))
+
+	for _, device := range kc.devices {
+		args = append(args, "--device", device)
+	}
+
+	for _, oemStr := range kc.oemString {
+		args = append(args, "--oem-string", oemStr)
+	}
+	return args
+}