Qemu capabilities (#49)

* Add snap / restore for qemu

* Fix restore on qemu, test passing

* More observability

* Move sleeps

* Consolidated waiting

* Deduplicate arguments building

* Fix possible polling error in standby / resume

Author: sjmiller609

lib/hypervisor/cloudhypervisor/process.go

@@ -4,8 +4,10 @@ import (
 	"context"
 	"fmt"
 	"syscall"
+	"time"
 
 	"github.com/onkernel/hypeman/lib/hypervisor"
+	"github.com/onkernel/hypeman/lib/logger"
 	"github.com/onkernel/hypeman/lib/paths"
 	"github.com/onkernel/hypeman/lib/vmm"
 	"gvisor.dev/gvisor/pkg/cleanup"
@@ -100,17 +102,22 @@ func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, s
 // RestoreVM starts Cloud Hypervisor and restores VM state from a snapshot.
 // The VM is in paused state after restore; caller should call Resume() to continue execution.
 func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, socketPath string, snapshotPath string) (int, hypervisor.Hypervisor, error) {
+	log := logger.FromContext(ctx)
+	startTime := time.Now()
+
 	// Validate version
 	chVersion := vmm.CHVersion(version)
 	if !vmm.IsVersionSupported(chVersion) {
 		return 0, nil, fmt.Errorf("unsupported cloud-hypervisor version: %s", version)
 	}
 
 	// 1. Start the Cloud Hypervisor process
+	processStartTime := time.Now()
 	pid, err := vmm.StartProcess(ctx, p, chVersion, socketPath)
 	if err != nil {
 		return 0, nil, fmt.Errorf("start process: %w", err)
 	}
+	log.DebugContext(ctx, "CH process started", "pid", pid, "duration_ms", time.Since(processStartTime).Milliseconds())
 
 	// Setup cleanup to kill the process if subsequent steps fail
 	cu := cleanup.Make(func() {
@@ -125,6 +132,7 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string,
 	}
 
 	// 3. Restore from snapshot via HTTP API
+	restoreAPIStart := time.Now()
 	sourceURL := "file://" + snapshotPath
 	restoreConfig := vmm.RestoreConfig{
 		SourceUrl: sourceURL,
@@ -137,9 +145,11 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string,
 	if resp.StatusCode() != 204 {
 		return 0, nil, fmt.Errorf("restore failed with status %d: %s", resp.StatusCode(), string(resp.Body))
 	}
+	log.DebugContext(ctx, "CH restore API complete", "duration_ms", time.Since(restoreAPIStart).Milliseconds())
 
 	// Success - release cleanup to prevent killing the process
 	cu.Release()
+	log.DebugContext(ctx, "CH restore complete", "pid", pid, "total_duration_ms", time.Since(startTime).Milliseconds())
 	return pid, hv, nil
 }
 

lib/hypervisor/qemu/process.go

@@ -3,6 +3,7 @@ package qemu
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net"
 	"os"
@@ -14,10 +15,26 @@ import (
 	"time"
 
 	"github.com/onkernel/hypeman/lib/hypervisor"
+	"github.com/onkernel/hypeman/lib/logger"
 	"github.com/onkernel/hypeman/lib/paths"
 	"gvisor.dev/gvisor/pkg/cleanup"
 )
 
+// Timeout constants for QEMU operations
+const (
+	// socketWaitTimeout is how long to wait for QMP socket to become available after process start
+	socketWaitTimeout = 10 * time.Second
+
+	// migrationTimeout is how long to wait for migration to complete
+	migrationTimeout = 30 * time.Second
+
+	// socketPollInterval is how often to check if socket is ready
+	socketPollInterval = 50 * time.Millisecond
+
+	// socketDialTimeout is timeout for individual socket connection attempts
+	socketDialTimeout = 100 * time.Millisecond
+)
+
 func init() {
 	hypervisor.RegisterSocketName(hypervisor.TypeQEMU, "qemu.sock")
 }
@@ -88,31 +105,34 @@ func (s *Starter) GetVersion(p *paths.Paths) (string, error) {
 	return "", fmt.Errorf("could not parse QEMU version from: %s", string(output))
 }
 
-// StartVM launches QEMU with the VM configuration and returns a Hypervisor client.
-// QEMU receives all configuration via command-line arguments at process start.
-func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, socketPath string, config hypervisor.VMConfig) (int, hypervisor.Hypervisor, error) {
+// buildQMPArgs returns the base QMP socket arguments for QEMU.
+func buildQMPArgs(socketPath string) []string {
+	return []string{
+		"-chardev", fmt.Sprintf("socket,id=qmp,path=%s,server=on,wait=off", socketPath),
+		"-mon", "chardev=qmp,mode=control",
+	}
+}
+
+// startQEMUProcess handles the common QEMU process startup logic.
+// Returns the PID, hypervisor client, and a cleanup function.
+// The cleanup function must be called on error; call cleanup.Release() on success.
+func (s *Starter) startQEMUProcess(ctx context.Context, p *paths.Paths, version string, socketPath string, args []string) (int, *QEMU, *cleanup.Cleanup, error) {
+	log := logger.FromContext(ctx)
+
 	// Get binary path
 	binaryPath, err := s.GetBinaryPath(p, version)
 	if err != nil {
-		return 0, nil, fmt.Errorf("get binary: %w", err)
+		return 0, nil, nil, fmt.Errorf("get binary: %w", err)
 	}
 
 	// Check if socket is already in use
 	if isSocketInUse(socketPath) {
-		return 0, nil, fmt.Errorf("socket already in use, QEMU may be running at %s", socketPath)
+		return 0, nil, nil, fmt.Errorf("socket already in use, QEMU may be running at %s", socketPath)
 	}
 
 	// Remove stale socket if exists
 	os.Remove(socketPath)
 
-	// Build command arguments: QMP socket + VM configuration
-	args := []string{
-		"-chardev", fmt.Sprintf("socket,id=qmp,path=%s,server=on,wait=off", socketPath),
-		"-mon", "chardev=qmp,mode=control",
-	}
-	// Append VM configuration as command-line arguments
-	args = append(args, BuildArgs(config)...)
-
 	// Create command
 	cmd := exec.Command(binaryPath, args...)
 
@@ -125,7 +145,7 @@ func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, s
 	instanceDir := filepath.Dir(socketPath)
 	logsDir := filepath.Join(instanceDir, "logs")
 	if err := os.MkdirAll(logsDir, 0755); err != nil {
-		return 0, nil, fmt.Errorf("create logs directory: %w", err)
+		return 0, nil, nil, fmt.Errorf("create logs directory: %w", err)
 	}
 
 	vmmLogFile, err := os.OpenFile(
@@ -134,49 +154,148 @@ func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, s
 		0644,
 	)
 	if err != nil {
-		return 0, nil, fmt.Errorf("create vmm log: %w", err)
+		return 0, nil, nil, fmt.Errorf("create vmm log: %w", err)
 	}
 	defer vmmLogFile.Close()
 
 	cmd.Stdout = vmmLogFile
 	cmd.Stderr = vmmLogFile
 
+	processStartTime := time.Now()
 	if err := cmd.Start(); err != nil {
-		return 0, nil, fmt.Errorf("start qemu: %w", err)
+		return 0, nil, nil, fmt.Errorf("start qemu: %w", err)
 	}
 
 	pid := cmd.Process.Pid
+	log.DebugContext(ctx, "QEMU process started", "pid", pid, "duration_ms", time.Since(processStartTime).Milliseconds())
 
 	// Setup cleanup to kill the process if subsequent steps fail
 	cu := cleanup.Make(func() {
 		syscall.Kill(pid, syscall.SIGKILL)
 	})
-	defer cu.Clean()
 
 	// Wait for socket to be ready
-	if err := waitForSocket(socketPath, 10*time.Second); err != nil {
+	socketWaitStart := time.Now()
+	if err := waitForSocket(socketPath, socketWaitTimeout); err != nil {
+		cu.Clean()
 		vmmLogPath := filepath.Join(logsDir, "vmm.log")
 		if logData, readErr := os.ReadFile(vmmLogPath); readErr == nil && len(logData) > 0 {
-			return 0, nil, fmt.Errorf("%w; vmm.log: %s", err, string(logData))
+			return 0, nil, nil, fmt.Errorf("%w; vmm.log: %s", err, string(logData))
 		}
-		return 0, nil, err
+		return 0, nil, nil, err
 	}
+	log.DebugContext(ctx, "QMP socket ready", "duration_ms", time.Since(socketWaitStart).Milliseconds())
 
 	// Create QMP client
 	hv, err := New(socketPath)
 	if err != nil {
-		return 0, nil, fmt.Errorf("create client: %w", err)
+		cu.Clean()
+		return 0, nil, nil, fmt.Errorf("create client: %w", err)
+	}
+
+	return pid, hv, &cu, nil
+}
+
+// StartVM launches QEMU with the VM configuration and returns a Hypervisor client.
+// QEMU receives all configuration via command-line arguments at process start.
+func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, socketPath string, config hypervisor.VMConfig) (int, hypervisor.Hypervisor, error) {
+	log := logger.FromContext(ctx)
+
+	// Build command arguments: QMP socket + VM configuration
+	args := buildQMPArgs(socketPath)
+	args = append(args, BuildArgs(config)...)
+
+	pid, hv, cu, err := s.startQEMUProcess(ctx, p, version, socketPath, args)
+	if err != nil {
+		return 0, nil, err
+	}
+	defer cu.Clean()
+
+	// Save config for potential restore later
+	// QEMU migration files only contain memory state, not device config
+	instanceDir := filepath.Dir(socketPath)
+	if err := saveVMConfig(instanceDir, config); err != nil {
+		// Non-fatal - restore just won't work
+		log.WarnContext(ctx, "failed to save VM config for restore", "error", err)
 	}
 
-	// Success - release cleanup to prevent killing the process
 	cu.Release()
 	return pid, hv, nil
 }
 
 // RestoreVM starts QEMU and restores VM state from a snapshot.
-// Not yet implemented for QEMU.
+// The VM is in paused state after restore; caller should call Resume() to continue execution.
 func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, socketPath string, snapshotPath string) (int, hypervisor.Hypervisor, error) {
-	return 0, nil, fmt.Errorf("restore not supported by QEMU implementation")
+	log := logger.FromContext(ctx)
+	startTime := time.Now()
+
+	// Load saved VM config from snapshot directory
+	// QEMU requires exact same command-line args as when snapshot was taken
+	configLoadStart := time.Now()
+	config, err := loadVMConfig(snapshotPath)
+	if err != nil {
+		return 0, nil, fmt.Errorf("load vm config from snapshot: %w", err)
+	}
+	log.DebugContext(ctx, "loaded VM config from snapshot", "duration_ms", time.Since(configLoadStart).Milliseconds())
+
+	// Build command arguments: QMP socket + VM configuration + incoming migration
+	args := buildQMPArgs(socketPath)
+	args = append(args, BuildArgs(config)...)
+
+	// Add incoming migration flag to restore from snapshot
+	// The "file:" protocol is deprecated in QEMU 7.2+, use "exec:cat < path" instead
+	memoryFile := filepath.Join(snapshotPath, "memory")
+	incomingURI := "exec:cat < " + memoryFile
+	args = append(args, "-incoming", incomingURI)
+
+	pid, hv, cu, err := s.startQEMUProcess(ctx, p, version, socketPath, args)
+	if err != nil {
+		return 0, nil, err
+	}
+	defer cu.Clean()
+
+	// Wait for VM to be ready after loading migration data
+	// QEMU transitions from "inmigrate" to "paused" when loading completes
+	migrationWaitStart := time.Now()
+	if err := hv.client.WaitVMReady(ctx, migrationTimeout); err != nil {
+		return 0, nil, fmt.Errorf("wait for vm ready: %w", err)
+	}
+	log.DebugContext(ctx, "VM ready", "duration_ms", time.Since(migrationWaitStart).Milliseconds())
+
+	cu.Release()
+	log.DebugContext(ctx, "QEMU restore complete", "pid", pid, "total_duration_ms", time.Since(startTime).Milliseconds())
+	return pid, hv, nil
+}
+
+// vmConfigFile is the name of the file where VM config is saved for restore.
+const vmConfigFile = "qemu-config.json"
+
+// saveVMConfig saves the VM configuration to a file in the instance directory.
+// This is needed for QEMU restore since migration files only contain memory state.
+func saveVMConfig(instanceDir string, config hypervisor.VMConfig) error {
+	configPath := filepath.Join(instanceDir, vmConfigFile)
+	data, err := json.MarshalIndent(config, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal config: %w", err)
+	}
+	if err := os.WriteFile(configPath, data, 0644); err != nil {
+		return fmt.Errorf("write config: %w", err)
+	}
+	return nil
+}
+
+// loadVMConfig loads the VM configuration from the instance directory.
+func loadVMConfig(instanceDir string) (hypervisor.VMConfig, error) {
+	configPath := filepath.Join(instanceDir, vmConfigFile)
+	data, err := os.ReadFile(configPath)
+	if err != nil {
+		return hypervisor.VMConfig{}, fmt.Errorf("read config: %w", err)
+	}
+	var config hypervisor.VMConfig
+	if err := json.Unmarshal(data, &config); err != nil {
+		return hypervisor.VMConfig{}, fmt.Errorf("unmarshal config: %w", err)
+	}
+	return config, nil
 }
 
 // qemuBinaryName returns the QEMU binary name for the host architecture.
@@ -205,7 +324,7 @@ func qemuInstallHint() string {
 
 // isSocketInUse checks if a Unix socket is actively being used
 func isSocketInUse(socketPath string) bool {
-	conn, err := net.DialTimeout("unix", socketPath, 100*time.Millisecond)
+	conn, err := net.DialTimeout("unix", socketPath, socketDialTimeout)
 	if err != nil {
 		return false
 	}
@@ -217,12 +336,12 @@ func isSocketInUse(socketPath string) bool {
 func waitForSocket(socketPath string, timeout time.Duration) error {
 	deadline := time.Now().Add(timeout)
 	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("unix", socketPath, 100*time.Millisecond)
+		conn, err := net.DialTimeout("unix", socketPath, socketDialTimeout)
 		if err == nil {
 			conn.Close()
 			return nil
 		}
-		time.Sleep(50 * time.Millisecond)
+		time.Sleep(socketPollInterval)
 	}
 	return fmt.Errorf("timeout waiting for socket")
 }

lib/hypervisor/qemu/qemu.go

@@ -3,6 +3,8 @@ package qemu
 import (
 	"context"
 	"fmt"
+	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/digitalocean/go-qemu/qemu"
@@ -36,8 +38,8 @@ var _ hypervisor.Hypervisor = (*QEMU)(nil)
 // Capabilities returns the features supported by QEMU.
 func (q *QEMU) Capabilities() hypervisor.Capabilities {
 	return hypervisor.Capabilities{
-		SupportsSnapshot:       false, // Not implemented in first pass
-		SupportsHotplugMemory:  false, // Not implemented in first pass
+		SupportsSnapshot:       true,  // Uses QMP migrate file:// for snapshot
+		SupportsHotplugMemory:  false, // Not implemented - balloon not configured
 		SupportsPause:          true,
 		SupportsVsock:          true,
 		SupportsGPUPassthrough: true,
@@ -119,10 +121,40 @@ func (q *QEMU) Resume(ctx context.Context) error {
 	return nil
 }
 
-// Snapshot creates a VM snapshot.
-// Not implemented in first pass.
+// Snapshot creates a VM snapshot using QEMU's migrate-to-file mechanism.
+// The VM state is saved to destPath/memory file.
+// The VM config is copied to destPath for restore (QEMU requires exact arg match).
 func (q *QEMU) Snapshot(ctx context.Context, destPath string) error {
-	return fmt.Errorf("snapshot not supported by QEMU implementation")
+	// QEMU uses migrate to file for snapshots
+	// The "file:" protocol is deprecated in QEMU 7.2+, use "exec:cat > path" instead
+	memoryFile := destPath + "/memory"
+	uri := "exec:cat > " + memoryFile
+	if err := q.client.Migrate(uri); err != nil {
+		Remove(q.socketPath)
+		return fmt.Errorf("migrate: %w", err)
+	}
+
+	// Wait for migration to complete
+	if err := q.client.WaitMigration(ctx, migrationTimeout); err != nil {
+		Remove(q.socketPath)
+		return fmt.Errorf("wait migration: %w", err)
+	}
+
+	// Copy VM config from instance dir to snapshot dir
+	// QEMU restore requires exact same command-line args as when snapshot was taken
+	instanceDir := filepath.Dir(q.socketPath)
+	srcConfig := filepath.Join(instanceDir, vmConfigFile)
+	dstConfig := filepath.Join(destPath, vmConfigFile)
+
+	configData, err := os.ReadFile(srcConfig)
+	if err != nil {
+		return fmt.Errorf("read vm config for snapshot: %w", err)
+	}
+	if err := os.WriteFile(dstConfig, configData, 0644); err != nil {
+		return fmt.Errorf("write vm config to snapshot: %w", err)
+	}
+
+	return nil
 }
 
 // ResizeMemory changes the VM's memory allocation.