Add a VM monitor for macOS

In order to switch from QEMU to krunkit we need to use gvproxy
to provide network support to the VM. We need to bound gvproxy
lifetime to krunkit lifetime, so there is not an "orphan"
daemon after krunkit finish.

We are using an "internal" command to monitor, in the future
this should be a different binary, but right now is the
easiest way. We save the pid fo krunkit instead of the monitor
because the `stop` command sends a SIGINT to that pid, and
we currently are capturing the signal at podman-bootc.

Signed-off-by: German Maglione <[email protected]>

Author: germag

cmd/vmmon.go

@@ -0,0 +1,68 @@
+//go:build darwin
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"github.com/containers/podman-bootc/pkg/user"
+	"github.com/containers/podman-bootc/pkg/vm"
+
+	"github.com/spf13/cobra"
+)
+
+// We treat this command as internal, we don't expect the user to call
+// this directly. In the future this will be replaced by an external binary
+var (
+	monCmd = &cobra.Command{
+		Use:    "vmmon <ID> <username> <ssh identity> <ssh port>",
+		Hidden: true,
+		Args:   cobra.ExactArgs(4),
+		RunE:   doMon,
+	}
+	console bool
+)
+
+func init() {
+	RootCmd.AddCommand(monCmd)
+	runCmd.Flags().BoolVar(&console, "console", false, "Show boot console")
+}
+
+func doMon(_ *cobra.Command, args []string) error {
+	usr, err := user.NewUser()
+	if err != nil {
+		return err
+	}
+
+	ctx := context.Background()
+	fullImageId := args[0]
+	username := args[1]
+	sshIdentity := args[2]
+
+	sshPort, err := strconv.Atoi(args[3])
+	if err != nil {
+		return fmt.Errorf("invalid ssh port: %w", err)
+	}
+
+	cacheDir := filepath.Join(usr.CacheDir(), fullImageId)
+
+	// MacOS has a 104 bytes limit for a unix socket path
+	runDir := filepath.Join(usr.RunDir(), fullImageId[0:12])
+	if err := os.MkdirAll(runDir, os.ModePerm); err != nil {
+		return err
+	}
+
+	params := vm.MonitorParmeters{
+		CacheDir:    cacheDir,
+		RunDir:      runDir,
+		Username:    username,
+		SshIdentity: sshIdentity,
+		SshPort:     sshPort,
+	}
+
+	return vm.StartMonitor(ctx, params)
+}

go.mod

@@ -5,16 +5,17 @@ go 1.20
 require (
 	github.com/adrg/xdg v0.4.0
 	github.com/containers/common v0.58.1
+	github.com/containers/gvisor-tap-vsock v0.7.3
 	github.com/containers/podman/v5 v5.0.1
 	github.com/docker/go-units v0.5.0
 	github.com/gofrs/flock v0.8.1
 	github.com/onsi/ginkgo/v2 v2.17.1
 	github.com/onsi/gomega v1.32.0
-	github.com/opencontainers/runtime-spec v1.2.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.0
 	golang.org/x/crypto v0.22.0
 	golang.org/x/sys v0.19.0
+	golang.org/x/term v0.19.0
 	libvirt.org/go/libvirt v1.10002.0
 )
 
@@ -40,7 +41,6 @@ require (
 	github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
 	github.com/containerd/typeurl/v2 v2.1.1 // indirect
 	github.com/containers/buildah v1.35.3 // indirect
-	github.com/containers/gvisor-tap-vsock v0.7.3 // indirect
 	github.com/containers/image/v5 v5.30.0 // indirect
 	github.com/containers/libhvee v0.7.0 // indirect
 	github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect
@@ -134,6 +134,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/opencontainers/runc v1.1.12 // indirect
+	github.com/opencontainers/runtime-spec v1.2.0 // indirect
 	github.com/opencontainers/runtime-tools v0.9.1-0.20230914150019-408c51e934dc // indirect
 	github.com/opencontainers/selinux v1.11.0 // indirect
 	github.com/openshift/imagebuilder v1.2.6 // indirect
@@ -176,7 +177,6 @@ require (
 	golang.org/x/mod v0.15.0 // indirect
 	golang.org/x/net v0.22.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/term v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.18.0 // indirect

pkg/vm/gvproxy.go

@@ -0,0 +1,90 @@
+package vm
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/containers/podman-bootc/pkg/utils"
+
+	gvproxy "github.com/containers/gvisor-tap-vsock/pkg/types"
+	"github.com/sirupsen/logrus"
+)
+
+const gvproxyBinaryName = "gvproxy"
+
+type Vmm int
+
+const (
+	pidFileName = "gvproxy.pid"
+	socketFile  = "net.sock"
+	// How log we should wait for gvproxy to be ready
+	maxBackoffs = 5
+	backoff     = time.Millisecond * 200
+)
+
+type gvproxyParams struct {
+	SshPort int
+}
+
+type gvproxyDaemon struct {
+	socketPath string
+	pidFile    string
+	cmd        *exec.Cmd
+}
+
+func newGvproxy(ctx context.Context, binaryPath, rundir string, param gvproxyParams) *gvproxyDaemon {
+	socketPath := filepath.Join(rundir, socketFile)
+	pidFile := filepath.Join(rundir, pidFileName)
+
+	gvpCmd := gvproxy.NewGvproxyCommand()
+	gvpCmd.SSHPort = param.SshPort
+	gvpCmd.PidFile = pidFile
+	gvpCmd.AddVfkitSocket(fmt.Sprintf("unixgram://%s", socketPath))
+
+	cmdLine := gvpCmd.ToCmdline()
+	cmd := exec.CommandContext(ctx, binaryPath, cmdLine...)
+	logrus.Debugf("gvproxy command-line: %s %s", binaryPath, strings.Join(cmdLine, " "))
+
+	return &gvproxyDaemon{socketPath: socketPath, pidFile: pidFile, cmd: cmd}
+}
+
+// Start spawn the gvproxy daemon, killing any running daemon using the same unix socket file.
+// It blocks until the unix socket file exists, this does not guarantee that the socket will be
+// on listen state
+func (d *gvproxyDaemon) start() error {
+	cleanup(d.pidFile, d.socketPath)
+
+	if err := d.cmd.Start(); err != nil {
+		return fmt.Errorf("unable to start gvproxy: %w", err)
+	}
+
+	// this is racy, the socket file could exist but not be in the listen state yet
+	if err := utils.WaitForFileWithBackoffs(maxBackoffs, backoff, d.socketPath); err != nil {
+		return fmt.Errorf("waiting for gvproxy socket: %w", err)
+	}
+	return nil
+}
+
+func (d *gvproxyDaemon) stop() error {
+	return d.cmd.Cancel()
+}
+
+func (d *gvproxyDaemon) wait() error {
+	return d.cmd.Wait()
+}
+
+func cleanup(pidFile, socketPath string) {
+	// Let's kill any possible running daemon
+	pid, err := utils.ReadPidFile(pidFile)
+	if err == nil {
+		_ = utils.SendInterrupt(pid)
+	}
+
+	_ = os.Remove(pidFile)
+	_ = os.Remove(socketPath)
+}

pkg/vm/krunkit.go

@@ -0,0 +1,110 @@
+package vm
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"strconv"
+	"strings"
+
+	"github.com/containers/podman-bootc/pkg/utils"
+
+	"github.com/sirupsen/logrus"
+)
+
+const krunkitBinaryName = "krunkit"
+
+const defaultCpus = 4
+const defaultMemory = 2048
+
+type krunkitParams struct {
+	disk      string
+	netSocket string
+	oemString string
+	pidFile   string
+}
+
+type krunkit struct {
+	pidFile string
+	cmd     *exec.Cmd
+}
+
+func newKrunkit(ctx context.Context, binaryPath string, params krunkitParams) *krunkit {
+	cmdLine := newKrunkitCmdLine(defaultCpus, defaultMemory)
+	cmdLine.addRngDevice()
+	cmdLine.addBlockDevice(params.disk)
+	cmdLine.addNetworkDevice(params.netSocket)
+	cmdLine.addOemString(params.oemString)
+
+	cmdLineSlice := cmdLine.asSlice()
+	cmd := exec.CommandContext(ctx, binaryPath, cmdLineSlice...)
+	logrus.Debugf("krunkit command-line: %s %s", binaryPath, strings.Join(cmdLineSlice, " "))
+
+	return &krunkit{cmd: cmd, pidFile: params.pidFile}
+}
+
+func (k *krunkit) start() error {
+	if err := k.cmd.Start(); err != nil {
+		return fmt.Errorf("unable to start krunkit: %w", err)
+	}
+
+	if err := utils.WritePidFile(k.pidFile, k.cmd.Process.Pid); err != nil {
+		if err := k.cmd.Cancel(); err != nil {
+			logrus.Debugf("stopping krunkit: %v", err)
+		}
+		return fmt.Errorf("writing pid file %s: %w", k.pidFile, err)
+	}
+
+	return nil
+}
+
+func (k *krunkit) wait() error {
+	return k.cmd.Wait()
+}
+
+type krunkitCmdLine struct {
+	cpus      int
+	memory    int
+	devices   []string
+	oemString []string
+}
+
+func newKrunkitCmdLine(cpus int, memory int) *krunkitCmdLine {
+	return &krunkitCmdLine{cpus: cpus, memory: memory}
+}
+
+func (kc *krunkitCmdLine) addDevice(device string) {
+	kc.devices = append(kc.devices, device)
+}
+
+func (kc *krunkitCmdLine) addRngDevice() {
+	kc.addDevice("virtio-rng")
+}
+
+func (kc *krunkitCmdLine) addBlockDevice(diskAbsPath string) {
+	kc.addDevice(fmt.Sprintf("virtio-blk,path=%s", diskAbsPath))
+}
+
+func (kc *krunkitCmdLine) addNetworkDevice(socketAbsPath string) {
+	kc.addDevice(fmt.Sprintf("virtio-net,unixSocketPath=%s,mac=5a:94:ef:e4:0c:ee", socketAbsPath))
+}
+
+func (kc *krunkitCmdLine) addOemString(oemStr string) {
+	kc.oemString = append(kc.oemString, oemStr)
+}
+
+func (kc *krunkitCmdLine) asSlice() []string {
+	args := []string{}
+
+	args = append(args, "--cpus", strconv.Itoa(kc.cpus))
+	args = append(args, "--memory", strconv.Itoa(kc.memory))
+
+	for _, device := range kc.devices {
+		args = append(args, "--device", device)
+	}
+
+	for _, oemStr := range kc.oemString {
+		args = append(args, "--oem-string", oemStr)
+	}
+	return args
+}