QEMU support (#48)

* Generate QEMU implementation

* Simplify to higher level apis

* Add otel integration

* arm64 configs

* Add review comment

* Package hint

* Add vsock dialer abstraction

* QMP connection pool

* Fix cleanup of vm

* Address review comments

* Add version checking to hypervisor interface

* Fix log naming

* fix indent

* Add startup warning

Author: sjmiller609

cmd/api/api/api_test.go

@@ -40,7 +40,7 @@ func newTestService(t *testing.T) *ApiService {
 	limits := instances.ResourceLimits{
 		MaxOverlaySize: 100 * 1024 * 1024 * 1024, // 100GB
 	}
-	instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, nil, nil)
+	instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil)
 
 	// Register cleanup for orphaned Cloud Hypervisor processes
 	t.Cleanup(func() {

cmd/api/api/cp.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/gorilla/websocket"
 	"github.com/onkernel/hypeman/lib/guest"
+	"github.com/onkernel/hypeman/lib/hypervisor"
 	"github.com/onkernel/hypeman/lib/instances"
 	"github.com/onkernel/hypeman/lib/logger"
 	mw "github.com/onkernel/hypeman/lib/middleware"
@@ -218,7 +219,13 @@ func (s *ApiService) CpHandler(w http.ResponseWriter, r *http.Request) {
 // handleCopyTo handles copying files from client to guest
 // Returns the number of bytes transferred and any error.
 func (s *ApiService) handleCopyTo(ctx context.Context, ws *websocket.Conn, inst *instances.Instance, req CpRequest) (int64, error) {
-	grpcConn, err := guest.GetOrCreateConnPublic(ctx, inst.VsockSocket)
+	// Create vsock dialer for this hypervisor type
+	dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID)
+	if err != nil {
+		return 0, fmt.Errorf("create vsock dialer: %w", err)
+	}
+
+	grpcConn, err := guest.GetOrCreateConn(ctx, dialer)
 	if err != nil {
 		return 0, fmt.Errorf("get grpc connection: %w", err)
 	}
@@ -322,7 +329,13 @@ func (s *ApiService) handleCopyTo(ctx context.Context, ws *websocket.Conn, inst
 // handleCopyFrom handles copying files from guest to client
 // Returns the number of bytes transferred and any error.
 func (s *ApiService) handleCopyFrom(ctx context.Context, ws *websocket.Conn, inst *instances.Instance, req CpRequest) (int64, error) {
-	grpcConn, err := guest.GetOrCreateConnPublic(ctx, inst.VsockSocket)
+	// Create vsock dialer for this hypervisor type
+	dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID)
+	if err != nil {
+		return 0, fmt.Errorf("create vsock dialer: %w", err)
+	}
+
+	grpcConn, err := guest.GetOrCreateConn(ctx, dialer)
 	if err != nil {
 		return 0, fmt.Errorf("get grpc connection: %w", err)
 	}
@@ -406,4 +419,3 @@ func (s *ApiService) handleCopyFrom(ctx context.Context, ws *websocket.Conn, ins
 	}
 	return bytesReceived, nil
 }
-

cmd/api/api/cp_test.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/onkernel/hypeman/lib/guest"
+	"github.com/onkernel/hypeman/lib/hypervisor"
 	"github.com/onkernel/hypeman/lib/oapi"
 	"github.com/onkernel/hypeman/lib/paths"
 	"github.com/onkernel/hypeman/lib/system"
@@ -109,10 +110,14 @@ func TestCpToAndFromInstance(t *testing.T) {
 	err = os.WriteFile(srcFile, []byte(testContent), 0644)
 	require.NoError(t, err)
 
+	// Create vsock dialer
+	dialer, err := hypervisor.NewVsockDialer(actualInst.HypervisorType, actualInst.VsockSocket, actualInst.VsockCID)
+	require.NoError(t, err)
+
 	// Test 1: Copy file TO instance
 	t.Log("Testing CopyToInstance...")
 	dstPath := "/tmp/copied-file.txt"
-	err = guest.CopyToInstance(ctx(), actualInst.VsockSocket, guest.CopyToInstanceOptions{
+	err = guest.CopyToInstance(ctx(), dialer, guest.CopyToInstanceOptions{
 		SrcPath: srcFile,
 		DstPath: dstPath,
 	})
@@ -121,7 +126,7 @@ func TestCpToAndFromInstance(t *testing.T) {
 	// Verify the file was copied by reading it back via exec
 	t.Log("Verifying file was copied via exec...")
 	var stdout, stderr outputBuffer
-	exit, err := guest.ExecIntoInstance(ctx(), actualInst.VsockSocket, guest.ExecOptions{
+	exit, err := guest.ExecIntoInstance(ctx(), dialer, guest.ExecOptions{
 		Command: []string{"cat", dstPath},
 		Stdout:  &stdout,
 		Stderr:  &stderr,
@@ -134,7 +139,7 @@ func TestCpToAndFromInstance(t *testing.T) {
 	// Test 2: Copy file FROM instance
 	t.Log("Testing CopyFromInstance...")
 	localDstDir := t.TempDir()
-	err = guest.CopyFromInstance(ctx(), actualInst.VsockSocket, guest.CopyFromInstanceOptions{
+	err = guest.CopyFromInstance(ctx(), dialer, guest.CopyFromInstanceOptions{
 		SrcPath: dstPath,
 		DstPath: localDstDir,
 	})
@@ -211,6 +216,10 @@ func TestCpDirectoryToInstance(t *testing.T) {
 	actualInst, err := svc.InstanceManager.GetInstance(ctx(), inst.Id)
 	require.NoError(t, err)
 
+	// Create vsock dialer
+	dialer, err := hypervisor.NewVsockDialer(actualInst.HypervisorType, actualInst.VsockSocket, actualInst.VsockCID)
+	require.NoError(t, err)
+
 	// Create a test directory structure
 	srcDir := filepath.Join(t.TempDir(), "testdir")
 	require.NoError(t, os.MkdirAll(filepath.Join(srcDir, "subdir"), 0755))
@@ -219,15 +228,15 @@ func TestCpDirectoryToInstance(t *testing.T) {
 
 	// Copy directory to instance
 	t.Log("Copying directory to instance...")
-	err = guest.CopyToInstance(ctx(), actualInst.VsockSocket, guest.CopyToInstanceOptions{
+	err = guest.CopyToInstance(ctx(), dialer, guest.CopyToInstanceOptions{
 		SrcPath: srcDir,
 		DstPath: "/tmp/testdir",
 	})
 	require.NoError(t, err)
 
 	// Verify files exist via exec
 	var stdout outputBuffer
-	exit, err := guest.ExecIntoInstance(ctx(), actualInst.VsockSocket, guest.ExecOptions{
+	exit, err := guest.ExecIntoInstance(ctx(), dialer, guest.ExecOptions{
 		Command: []string{"cat", "/tmp/testdir/file1.txt"},
 		Stdout:  &stdout,
 		TTY:     false,
@@ -237,7 +246,7 @@ func TestCpDirectoryToInstance(t *testing.T) {
 	assert.Equal(t, "file1 content", stdout.String())
 
 	stdout = outputBuffer{}
-	exit, err = guest.ExecIntoInstance(ctx(), actualInst.VsockSocket, guest.ExecOptions{
+	exit, err = guest.ExecIntoInstance(ctx(), dialer, guest.ExecOptions{
 		Command: []string{"cat", "/tmp/testdir/subdir/file2.txt"},
 		Stdout:  &stdout,
 		TTY:     false,
@@ -249,7 +258,7 @@ func TestCpDirectoryToInstance(t *testing.T) {
 	// Copy directory from instance
 	t.Log("Copying directory from instance...")
 	localDstDir := t.TempDir()
-	err = guest.CopyFromInstance(ctx(), actualInst.VsockSocket, guest.CopyFromInstanceOptions{
+	err = guest.CopyFromInstance(ctx(), dialer, guest.CopyFromInstanceOptions{
 		SrcPath: "/tmp/testdir",
 		DstPath: localDstDir,
 	})
@@ -266,4 +275,3 @@ func TestCpDirectoryToInstance(t *testing.T) {
 
 	t.Log("Directory cp tests passed!")
 }
-

cmd/api/api/exec.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/gorilla/websocket"
 	"github.com/onkernel/hypeman/lib/guest"
+	"github.com/onkernel/hypeman/lib/hypervisor"
 	"github.com/onkernel/hypeman/lib/instances"
 	"github.com/onkernel/hypeman/lib/logger"
 	mw "github.com/onkernel/hypeman/lib/middleware"
@@ -110,8 +111,17 @@ func (s *ApiService) ExecHandler(w http.ResponseWriter, r *http.Request) {
 	// Create WebSocket read/writer wrapper
 	wsConn := &wsReadWriter{ws: ws, ctx: ctx}
 
+	// Create vsock dialer for this hypervisor type
+	dialer, err := hypervisor.NewVsockDialer(hypervisor.Type(inst.HypervisorType), inst.VsockSocket, inst.VsockCID)
+	if err != nil {
+		log.ErrorContext(ctx, "failed to create vsock dialer", "error", err)
+		ws.WriteMessage(websocket.BinaryMessage, []byte(fmt.Sprintf("Error: %v\r\n", err)))
+		ws.WriteMessage(websocket.TextMessage, []byte(`{"exitCode":127}`))
+		return
+	}
+
 	// Execute via vsock
-	exit, err := guest.ExecIntoInstance(ctx, inst.VsockSocket, guest.ExecOptions{
+	exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{
 		Command: execReq.Command,
 		Stdin:   wsConn,
 		Stdout:  wsConn,

cmd/api/api/exec_test.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/onkernel/hypeman/lib/guest"
+	"github.com/onkernel/hypeman/lib/hypervisor"
 	"github.com/onkernel/hypeman/lib/instances"
 	"github.com/onkernel/hypeman/lib/oapi"
 	"github.com/onkernel/hypeman/lib/paths"
@@ -119,13 +120,16 @@ func TestExecInstanceNonTTY(t *testing.T) {
 	var stdout, stderr outputBuffer
 	var execErr error
 
+	dialer, err := hypervisor.NewVsockDialer(actualInst.HypervisorType, actualInst.VsockSocket, actualInst.VsockCID)
+	require.NoError(t, err)
+
 	t.Log("Testing exec command: whoami")
 	maxRetries := 10
 	for i := 0; i < maxRetries; i++ {
 		stdout = outputBuffer{}
 		stderr = outputBuffer{}
 
-		exit, execErr = guest.ExecIntoInstance(ctx(), actualInst.VsockSocket, guest.ExecOptions{
+		exit, execErr = guest.ExecIntoInstance(ctx(), dialer, guest.ExecOptions{
 			Command: []string{"/bin/sh", "-c", "whoami"},
 			Stdin:   nil,
 			Stdout:  &stdout,
@@ -250,9 +254,12 @@ func TestExecWithDebianMinimal(t *testing.T) {
 	assert.Contains(t, logs, "overlay-init: app exited with code", "App should have exited")
 
 	// Test exec commands work even though the main app (bash) has exited
+	dialer2, err := hypervisor.NewVsockDialer(actualInst.HypervisorType, actualInst.VsockSocket, actualInst.VsockCID)
+	require.NoError(t, err)
+
 	t.Log("Testing exec command: echo")
 	var stdout, stderr outputBuffer
-	exit, err := guest.ExecIntoInstance(ctx(), actualInst.VsockSocket, guest.ExecOptions{
+	exit, err := guest.ExecIntoInstance(ctx(), dialer2, guest.ExecOptions{
 		Command: []string{"echo", "hello from debian"},
 		Stdout:  &stdout,
 		Stderr:  &stderr,
@@ -266,7 +273,7 @@ func TestExecWithDebianMinimal(t *testing.T) {
 	// Verify we're actually in Debian
 	t.Log("Verifying OS release...")
 	stdout = outputBuffer{}
-	exit, err = guest.ExecIntoInstance(ctx(), actualInst.VsockSocket, guest.ExecOptions{
+	exit, err = guest.ExecIntoInstance(ctx(), dialer2, guest.ExecOptions{
 		Command: []string{"cat", "/etc/os-release"},
 		Stdout:  &stdout,
 		TTY:     false,

cmd/api/api/instances.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/c2h5oh/datasize"
 	"github.com/onkernel/hypeman/lib/guest"
+	"github.com/onkernel/hypeman/lib/hypervisor"
 	"github.com/onkernel/hypeman/lib/instances"
 	"github.com/onkernel/hypeman/lib/logger"
 	mw "github.com/onkernel/hypeman/lib/middleware"
@@ -137,6 +138,12 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 		}
 	}
 
+	// Convert hypervisor type from API enum to domain type
+	var hvType hypervisor.Type
+	if request.Body.Hypervisor != nil {
+		hvType = hypervisor.Type(*request.Body.Hypervisor)
+	}
+
 	domainReq := instances.CreateInstanceRequest{
 		Name:           request.Body.Name,
 		Image:          request.Body.Image,
@@ -148,6 +155,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 		NetworkEnabled: networkEnabled,
 		Devices:        deviceRefs,
 		Volumes:        volumes,
+		Hypervisor:     hvType,
 	}
 
 	inst, err := s.InstanceManager.CreateInstance(ctx, domainReq)
@@ -452,8 +460,17 @@ func (s *ApiService) StatInstancePath(ctx context.Context, request oapi.StatInst
 		}, nil
 	}
 
-	// Connect to guest agent
-	grpcConn, err := guest.GetOrCreateConnPublic(ctx, inst.VsockSocket)
+	// Create vsock dialer for this hypervisor type
+	dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID)
+	if err != nil {
+		log.ErrorContext(ctx, "failed to create vsock dialer", "error", err)
+		return oapi.StatInstancePath500JSONResponse{
+			Code:    "internal_error",
+			Message: "failed to create vsock dialer",
+		}, nil
+	}
+
+	grpcConn, err := guest.GetOrCreateConn(ctx, dialer)
 	if err != nil {
 		log.ErrorContext(ctx, "failed to get grpc connection", "error", err)
 		return oapi.StatInstancePath500JSONResponse{
@@ -537,6 +554,9 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance {
 		netObj.Mac = lo.ToPtr(inst.MAC)
 	}
 
+	// Convert hypervisor type
+	hvType := oapi.InstanceHypervisor(inst.HypervisorType)
+
 	oapiInst := oapi.Instance{
 		Id:          inst.Id,
 		Name:        inst.Name,
@@ -552,6 +572,7 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance {
 		StartedAt:   inst.StartedAt,
 		StoppedAt:   inst.StoppedAt,
 		HasSnapshot: lo.ToPtr(inst.HasSnapshot),
+		Hypervisor:  &hvType,
 	}
 
 	if len(inst.Env) > 0 {

cmd/api/config/config.go

@@ -101,6 +101,9 @@ type Config struct {
 
 	// Cloudflare configuration (if AcmeDnsProvider=cloudflare)
 	CloudflareApiToken string // Cloudflare API token
+
+	// Hypervisor configuration
+	DefaultHypervisor string // Default hypervisor type: "cloud-hypervisor" or "qemu"
 }
 
 // Load loads configuration from environment variables
@@ -163,6 +166,9 @@ func Load() *Config {
 
 		// Cloudflare configuration
 		CloudflareApiToken: getEnv("CLOUDFLARE_API_TOKEN", ""),
+
+		// Hypervisor configuration
+		DefaultHypervisor: getEnv("DEFAULT_HYPERVISOR", "cloud-hypervisor"),
 	}
 
 	return cfg

cmd/api/main.go

@@ -23,6 +23,7 @@ import (
 	"github.com/onkernel/hypeman/cmd/api/api"
 	"github.com/onkernel/hypeman/cmd/api/config"
 	"github.com/onkernel/hypeman/lib/guest"
+	"github.com/onkernel/hypeman/lib/hypervisor/qemu"
 	"github.com/onkernel/hypeman/lib/instances"
 	mw "github.com/onkernel/hypeman/lib/middleware"
 	"github.com/onkernel/hypeman/lib/oapi"
@@ -125,6 +126,11 @@ func run() error {
 	}
 	logger.Info("KVM access verified")
 
+	// Check if QEMU is available (optional - only warn if not present)
+	if _, err := (&qemu.Starter{}).GetBinaryPath(nil, ""); err != nil {
+		logger.Warn("QEMU not available - QEMU hypervisor will not work", "error", err)
+	}
+
 	// Validate log rotation config
 	var logMaxSize datasize.ByteSize
 	if err := logMaxSize.UnmarshalText([]byte(app.Config.LogMaxSize)); err != nil {

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500
 	github.com/creack/pty v1.1.24
 	github.com/cyphar/filepath-securejoin v0.6.1
+	github.com/digitalocean/go-qemu v0.0.0-20250212194115-ee9b0668d242
 	github.com/distribution/reference v0.6.0
 	github.com/getkin/kin-openapi v0.133.0
 	github.com/ghodss/yaml v1.0.0
@@ -58,6 +59,7 @@ require (
 	github.com/containerd/errdefs/pkg v0.3.0 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e // indirect
 	github.com/docker/cli v28.2.2+incompatible // indirect
 	github.com/docker/distribution v2.8.3+incompatible // indirect
 	github.com/docker/docker v28.2.2+incompatible // indirect