Add vGPU support

cmd/api/api/instances.go

@@ -190,6 +190,14 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 		hvType = hypervisor.Type(*request.Body.Hypervisor)
 	}
 
+	// Parse GPU configuration (vGPU mode)
+	var gpuConfig *instances.GPUConfig
+	if request.Body.Gpu != nil && request.Body.Gpu.Profile != nil && *request.Body.Gpu.Profile != "" {
+		gpuConfig = &instances.GPUConfig{
+			Profile: *request.Body.Gpu.Profile,
+		}
+	}
+
 	// Calculate default resource limits when not specified (0 = auto)
 	// Uses proportional allocation based on CPU: (vcpus / cpuCapacity) * resourceCapacity
 	if diskIOBps == 0 {
@@ -220,6 +228,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 		Devices:                  deviceRefs,
 		Volumes:                  volumes,
 		Hypervisor:               hvType,
+		GPU:                      gpuConfig,
 	}
 
 	inst, err := s.InstanceManager.CreateInstance(ctx, domainReq)
@@ -685,5 +694,13 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance {
 		oapiInst.Volumes = &oapiVolumes
 	}
 
+	// Convert GPU info
+	if inst.GPUProfile != "" {
+		oapiInst.Gpu = &oapi.InstanceGPU{
+			Profile:  lo.ToPtr(inst.GPUProfile),
+			MdevUuid: lo.ToPtr(inst.GPUMdevUUID),
+		}
+	}
+
 	return oapiInst
 }

cmd/api/api/resources.go

@@ -56,6 +56,12 @@ func (s *ApiService) GetResources(ctx context.Context, _ oapi.GetResourcesReques
 		})
 	}
 
+	// Add GPU status if available
+	if status.GPU != nil {
+		gpuStatus := convertGPUResourceStatus(status.GPU)
+		resp.Gpu = &gpuStatus
+	}
+
 	return oapi.GetResources200JSONResponse(resp), nil
 }
 
@@ -75,3 +81,38 @@ func convertResourceStatus(rs resources.ResourceStatus) oapi.ResourceStatus {
 		Source:         source,
 	}
 }
+
+func convertGPUResourceStatus(gs *resources.GPUResourceStatus) oapi.GPUResourceStatus {
+	result := oapi.GPUResourceStatus{
+		Mode:       oapi.GPUResourceStatusMode(gs.Mode),
+		TotalSlots: gs.TotalSlots,
+		UsedSlots:  gs.UsedSlots,
+	}
+
+	// Convert profiles (vGPU mode)
+	if len(gs.Profiles) > 0 {
+		profiles := make([]oapi.GPUProfile, len(gs.Profiles))
+		for i, p := range gs.Profiles {
+			profiles[i] = oapi.GPUProfile{
+				Name:          p.Name,
+				FramebufferMb: p.FramebufferMB,
+				Available:     p.Available,
+			}
+		}
+		result.Profiles = &profiles
+	}
+
+	// Convert devices (passthrough mode)
+	if len(gs.Devices) > 0 {
+		devices := make([]oapi.PassthroughDevice, len(gs.Devices))
+		for i, d := range gs.Devices {
+			devices[i] = oapi.PassthroughDevice{
+				Name:      d.Name,
+				Available: d.Available,
+			}
+		}
+		result.Devices = &devices
+	}
+
+	return result
+}

cmd/api/main.go

@@ -22,6 +22,7 @@ import (
 	"github.com/onkernel/hypeman"
 	"github.com/onkernel/hypeman/cmd/api/api"
 	"github.com/onkernel/hypeman/cmd/api/config"
+	"github.com/onkernel/hypeman/lib/devices"
 	"github.com/onkernel/hypeman/lib/guest"
 	"github.com/onkernel/hypeman/lib/hypervisor/qemu"
 	"github.com/onkernel/hypeman/lib/instances"
@@ -200,6 +201,26 @@ func run() error {
 		return fmt.Errorf("reconcile device state: %w", err)
 	}
 
+	// Reconcile mdev devices (clears orphaned vGPUs from crashed VMs)
+	// Build mdev info from instances - only destroys mdevs tracked by hypeman
+	logger.Info("Reconciling mdev devices...")
+	var mdevInfos []devices.MdevReconcileInfo
+	if allInstances != nil {
+		for _, inst := range allInstances {
+			if inst.GPUMdevUUID != "" {
+				mdevInfos = append(mdevInfos, devices.MdevReconcileInfo{
+					InstanceID: inst.Id,
+					MdevUUID:   inst.GPUMdevUUID,
+					IsRunning:  inst.State == instances.StateRunning || inst.State == instances.StateUnknown,
+				})
+			}
+		}
+	}
+	if err := devices.ReconcileMdevs(app.Ctx, mdevInfos); err != nil {
+		// Log but don't fail - mdev cleanup is best-effort
+		logger.Warn("failed to reconcile mdev devices", "error", err)
+	}
+
 	// Initialize ingress manager (starts Caddy daemon and DNS server for dynamic upstreams)
 	logger.Info("Initializing ingress manager...")
 	if err := app.IngressManager.Initialize(app.Ctx); err != nil {

integration/vgpu_test.go

@@ -0,0 +1,244 @@
+package integration
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/onkernel/hypeman/cmd/api/config"
+	"github.com/onkernel/hypeman/lib/devices"
+	"github.com/onkernel/hypeman/lib/guest"
+	"github.com/onkernel/hypeman/lib/hypervisor"
+	"github.com/onkernel/hypeman/lib/images"
+	"github.com/onkernel/hypeman/lib/instances"
+	"github.com/onkernel/hypeman/lib/network"
+	"github.com/onkernel/hypeman/lib/paths"
+	"github.com/onkernel/hypeman/lib/system"
+	"github.com/onkernel/hypeman/lib/volumes"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestVGPU is an integration test that verifies vGPU (SR-IOV mdev) support works.
+//
+// This test automatically detects vGPU availability and skips if:
+//   - No SR-IOV VFs are found in /sys/class/mdev_bus/
+//   - No vGPU profiles are available
+//   - Not running as root (required for mdev creation)
+//   - KVM is not available
+//
+// To run manually:
+//
+//	sudo go test -v -run TestVGPU -timeout 5m ./integration/...
+//
+// Note: This test verifies mdev creation and PCI device visibility inside the VM.
+// It does NOT test nvidia-smi or CUDA functionality since that requires NVIDIA
+// guest drivers pre-installed in the image.
+func TestVGPU(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	// Auto-detect vGPU availability - skip if prerequisites not met
+	skipReason, profile := checkVGPUTestPrerequisites()
+	if skipReason != "" {
+		t.Skip(skipReason)
+	}
+
+	t.Logf("vGPU test prerequisites met, using profile: %s", profile)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+
+	// Set up test environment
+	tmpDir := t.TempDir()
+	p := paths.New(tmpDir)
+
+	cfg := &config.Config{
+		DataDir:    tmpDir,
+		BridgeName: "vmbr0",
+		SubnetCIDR: "10.100.0.0/16",
+		DNSServer:  "1.1.1.1",
+	}
+
+	// Create managers
+	imageManager, err := images.NewManager(p, 1, nil)
+	require.NoError(t, err)
+
+	systemManager := system.NewManager(p)
+	networkManager := network.NewManager(p, cfg, nil)
+	deviceManager := devices.NewManager(p)
+	volumeManager := volumes.NewManager(p, 0, nil)
+
+	limits := instances.ResourceLimits{
+		MaxOverlaySize:       100 * 1024 * 1024 * 1024,
+		MaxVcpusPerInstance:  0,
+		MaxMemoryPerInstance: 0,
+		MaxTotalVcpus:        0,
+		MaxTotalMemory:       0,
+	}
+
+	instanceManager := instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil)
+
+	// Track instance ID for cleanup
+	var instanceID string
+
+	// Cleanup any orphaned instances and mdevs
+	t.Cleanup(func() {
+		if instanceID != "" {
+			t.Log("Cleanup: Deleting instance...")
+			instanceManager.DeleteInstance(ctx, instanceID)
+		}
+	})
+
+	// Step 1: Ensure system files (kernel, initrd)
+	t.Log("Step 1: Ensuring system files...")
+	err = systemManager.EnsureSystemFiles(ctx)
+	require.NoError(t, err)
+	t.Log("System files ready")
+
+	// Step 2: Pull alpine image (lightweight for testing)
+	imageName := "docker.io/library/alpine:latest"
+	t.Log("Step 2: Pulling alpine image...")
+	_, err = imageManager.CreateImage(ctx, images.CreateImageRequest{
+		Name: imageName,
+	})
+	require.NoError(t, err)
+
+	// Wait for image to be ready
+	t.Log("Waiting for image build...")
+	var img *images.Image
+	for i := 0; i < 120; i++ {
+		img, err = imageManager.GetImage(ctx, imageName)
+		if err == nil && img.Status == images.StatusReady {
+			break
+		}
+		if img != nil && img.Status == images.StatusFailed {
+			errMsg := "unknown"
+			if img.Error != nil {
+				errMsg = *img.Error
+			}
+			t.Fatalf("Image build failed: %s", errMsg)
+		}
+		time.Sleep(1 * time.Second)
+	}
+	require.NotNil(t, img, "Image should exist")
+	require.Equal(t, images.StatusReady, img.Status, "Image should be ready")
+	t.Log("Image ready")
+
+	// Step 3: Create instance with vGPU using QEMU hypervisor
+	// QEMU is required for vGPU/mdev passthrough with NVIDIA's vGPU manager
+	t.Log("Step 3: Creating instance with vGPU profile:", profile)
+	inst, err := instanceManager.CreateInstance(ctx, instances.CreateInstanceRequest{
+		Name:           "vgpu-test",
+		Image:          imageName,
+		Size:           2 * 1024 * 1024 * 1024, // 2GB
+		HotplugSize:    512 * 1024 * 1024,
+		OverlaySize:    1024 * 1024 * 1024,
+		Vcpus:          2,
+		NetworkEnabled: false, // No network needed for this test
+		Hypervisor:     hypervisor.TypeQEMU,
+		GPU: &instances.GPUConfig{
+			Profile: profile,
+		},
+	})
+	require.NoError(t, err)
+	instanceID = inst.Id
+	t.Logf("Instance created: %s", inst.Id)
+
+	// Verify mdev UUID was assigned
+	require.NotEmpty(t, inst.GPUMdevUUID, "Instance should have mdev UUID assigned")
+	t.Logf("mdev UUID: %s", inst.GPUMdevUUID)
+
+	// Step 4: Verify mdev was created in sysfs
+	t.Run("MdevCreated", func(t *testing.T) {
+		mdevPath := "/sys/bus/mdev/devices/" + inst.GPUMdevUUID
+		_, err := os.Stat(mdevPath)
+		assert.NoError(t, err, "mdev device should exist at %s", mdevPath)
+		t.Logf("mdev exists at: %s", mdevPath)
+	})
+
+	// Step 5: Wait for guest agent to be ready
+	t.Log("Step 5: Waiting for guest agent...")
+	err = waitForGuestAgent(ctx, instanceManager, inst.Id, 60*time.Second)
+	require.NoError(t, err, "guest agent should be ready")
+
+	// Step 6: Verify GPU is visible inside VM via PCI
+	t.Run("GPUVisibleInVM", func(t *testing.T) {
+		actualInst, err := instanceManager.GetInstance(ctx, inst.Id)
+		require.NoError(t, err)
+
+		dialer, err := hypervisor.NewVsockDialer(actualInst.HypervisorType, actualInst.VsockSocket, actualInst.VsockCID)
+		require.NoError(t, err)
+
+		// Check for NVIDIA vendor ID (0x10de) in guest PCI devices
+		var stdout, stderr bytes.Buffer
+		checkGPUCmd := "cat /sys/bus/pci/devices/*/vendor 2>/dev/null | grep -i 10de && echo 'NVIDIA_FOUND' || echo 'NO_NVIDIA'"
+
+		_, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{
+			Command: []string{"/bin/sh", "-c", checkGPUCmd},
+			Stdout:  &stdout,
+			Stderr:  &stderr,
+			TTY:     false,
+		})
+		require.NoError(t, err, "exec should work")
+
+		output := stdout.String()
+		t.Logf("GPU check output: %s", output)
+
+		assert.Contains(t, output, "NVIDIA_FOUND", "NVIDIA GPU (vendor 0x10de) should be visible in guest")
+	})
+
+	// Step 7: Check instance GPU info is correct
+	t.Run("InstanceGPUInfo", func(t *testing.T) {
+		actualInst, err := instanceManager.GetInstance(ctx, inst.Id)
+		require.NoError(t, err)
+
+		assert.Equal(t, profile, actualInst.GPUProfile, "GPU profile should match")
+		assert.NotEmpty(t, actualInst.GPUMdevUUID, "mdev UUID should be set")
+		t.Logf("Instance GPU: profile=%s, mdev=%s", actualInst.GPUProfile, actualInst.GPUMdevUUID)
+	})
+
+	t.Log("✅ vGPU test PASSED!")
+}
+
+// checkVGPUTestPrerequisites checks if vGPU test can run.
+// Returns (skipReason, profileName) - skipReason is empty if all prerequisites are met.
+func checkVGPUTestPrerequisites() (string, string) {
+	// Check KVM
+	if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) {
+		return "vGPU test requires /dev/kvm", ""
+	}
+
+	// Check for root (required for mdev creation via sysfs)
+	if os.Geteuid() != 0 {
+		return "vGPU test requires root (sudo) for mdev creation", ""
+	}
+
+	// Check for vGPU mode (SR-IOV VFs present)
+	mode := devices.DetectHostGPUMode()
+	if mode != devices.GPUModeVGPU {
+		return "vGPU test requires SR-IOV VFs in /sys/class/mdev_bus/", ""
+	}
+
+	// Check for available profiles
+	profiles, err := devices.ListGPUProfiles()
+	if err != nil {
+		return "vGPU test failed to list profiles: " + err.Error(), ""
+	}
+	if len(profiles) == 0 {
+		return "vGPU test requires at least one GPU profile", ""
+	}
+
+	// Find a profile with available instances
+	for _, p := range profiles {
+		if p.Available > 0 {
+			return "", p.Name
+		}
+	}
+
+	return "vGPU test requires at least one available VF (all VFs are in use)", ""
+}
+