feat(api): add devices API endpoints and documentation

Add REST API for device management and supporting documentation:

API endpoints:
- GET/POST /devices - List and register devices
- GET/DELETE /devices/{id} - Get and delete devices
- GET /devices/available - Discover passthrough-capable devices
- instances.go: Accept devices param in CreateInstance

Documentation:
- GPU.md: GPU passthrough architecture and driver injection
- README.md: Device management usage guide
- scripts/gpu-reset.sh: GPU reset utility

Tests and fixtures:
- gpu_e2e_test.go, gpu_inference_test.go, gpu_module_test.go
- testdata/ollama-cuda/ - CUDA test container

Also adds build-preview-cli Makefile target.

Author: Rafael Garcia

Makefile

@@ -1,5 +1,5 @@
 SHELL := /bin/bash
-.PHONY: oapi-generate generate-vmm-client generate-wire generate-all dev build test install-tools gen-jwt download-ch-binaries download-ch-spec ensure-ch-binaries build-caddy-binaries build-caddy ensure-caddy-binaries release-prep clean
+.PHONY: oapi-generate generate-vmm-client generate-wire generate-all dev build test install-tools gen-jwt download-ch-binaries download-ch-spec ensure-ch-binaries build-caddy-binaries build-caddy ensure-caddy-binaries build-preview-cli release-prep clean
 
 # Directory where local binaries will be installed
 BIN_DIR ?= $(CURDIR)/bin
@@ -168,6 +168,12 @@ build: ensure-ch-binaries ensure-caddy-binaries lib/system/exec_agent/exec-agent
 # Build all binaries
 build-all: build
 
+# Build preview CLI from stainless-sdks/hypeman-cli
+# Usage: make build-preview-cli                         - uses preview/<current-branch>
+#        make build-preview-cli CLI_BRANCH=preview/xyz  - uses specific branch
+build-preview-cli:
+	@./scripts/build-preview-cli.sh $(CLI_BRANCH)
+
 # Run in development mode with hot reload
 dev: ensure-ch-binaries ensure-caddy-binaries lib/system/exec_agent/exec-agent $(AIR)
 	@rm -f ./tmp/main

cmd/api/api/devices.go

@@ -0,0 +1,167 @@
+package api
+
+import (
+	"context"
+	"errors"
+
+	"github.com/onkernel/hypeman/lib/devices"
+	"github.com/onkernel/hypeman/lib/oapi"
+)
+
+// ListDevices returns all registered devices
+func (s *ApiService) ListDevices(ctx context.Context, request oapi.ListDevicesRequestObject) (oapi.ListDevicesResponseObject, error) {
+	deviceList, err := s.DeviceManager.ListDevices(ctx)
+	if err != nil {
+		return oapi.ListDevices500JSONResponse{
+			Code:    "internal_error",
+			Message: err.Error(),
+		}, nil
+	}
+
+	result := make([]oapi.Device, len(deviceList))
+	for i, d := range deviceList {
+		result[i] = deviceToOAPI(d)
+	}
+
+	return oapi.ListDevices200JSONResponse(result), nil
+}
+
+// ListAvailableDevices discovers passthrough-capable devices on the host
+func (s *ApiService) ListAvailableDevices(ctx context.Context, request oapi.ListAvailableDevicesRequestObject) (oapi.ListAvailableDevicesResponseObject, error) {
+	available, err := s.DeviceManager.ListAvailableDevices(ctx)
+	if err != nil {
+		return oapi.ListAvailableDevices500JSONResponse{
+			Code:    "internal_error",
+			Message: err.Error(),
+		}, nil
+	}
+
+	result := make([]oapi.AvailableDevice, len(available))
+	for i, d := range available {
+		result[i] = availableDeviceToOAPI(d)
+	}
+
+	return oapi.ListAvailableDevices200JSONResponse(result), nil
+}
+
+// CreateDevice registers a new device for passthrough
+func (s *ApiService) CreateDevice(ctx context.Context, request oapi.CreateDeviceRequestObject) (oapi.CreateDeviceResponseObject, error) {
+	var name string
+	if request.Body.Name != nil {
+		name = *request.Body.Name
+	}
+	req := devices.CreateDeviceRequest{
+		Name:       name,
+		PCIAddress: request.Body.PciAddress,
+	}
+
+	device, err := s.DeviceManager.CreateDevice(ctx, req)
+	if err != nil {
+		switch {
+		case errors.Is(err, devices.ErrInvalidName):
+			return oapi.CreateDevice400JSONResponse{
+				Code:    "invalid_name",
+				Message: err.Error(),
+			}, nil
+		case errors.Is(err, devices.ErrInvalidPCIAddress):
+			return oapi.CreateDevice400JSONResponse{
+				Code:    "invalid_pci_address",
+				Message: err.Error(),
+			}, nil
+		case errors.Is(err, devices.ErrDeviceNotFound):
+			return oapi.CreateDevice404JSONResponse{
+				Code:    "device_not_found",
+				Message: err.Error(),
+			}, nil
+		case errors.Is(err, devices.ErrAlreadyExists), errors.Is(err, devices.ErrNameExists):
+			return oapi.CreateDevice409JSONResponse{
+				Code:    "conflict",
+				Message: err.Error(),
+			}, nil
+		default:
+			return oapi.CreateDevice500JSONResponse{
+				Code:    "internal_error",
+				Message: err.Error(),
+			}, nil
+		}
+	}
+
+	return oapi.CreateDevice201JSONResponse(deviceToOAPI(*device)), nil
+}
+
+// GetDevice returns a device by ID or name
+func (s *ApiService) GetDevice(ctx context.Context, request oapi.GetDeviceRequestObject) (oapi.GetDeviceResponseObject, error) {
+	device, err := s.DeviceManager.GetDevice(ctx, request.Id)
+	if err != nil {
+		if errors.Is(err, devices.ErrNotFound) {
+			return oapi.GetDevice404JSONResponse{
+				Code:    "not_found",
+				Message: "device not found",
+			}, nil
+		}
+		return oapi.GetDevice500JSONResponse{
+			Code:    "internal_error",
+			Message: err.Error(),
+		}, nil
+	}
+
+	return oapi.GetDevice200JSONResponse(deviceToOAPI(*device)), nil
+}
+
+// DeleteDevice unregisters a device
+func (s *ApiService) DeleteDevice(ctx context.Context, request oapi.DeleteDeviceRequestObject) (oapi.DeleteDeviceResponseObject, error) {
+	err := s.DeviceManager.DeleteDevice(ctx, request.Id)
+	if err != nil {
+		switch {
+		case errors.Is(err, devices.ErrNotFound):
+			return oapi.DeleteDevice404JSONResponse{
+				Code:    "not_found",
+				Message: "device not found",
+			}, nil
+		case errors.Is(err, devices.ErrInUse):
+			return oapi.DeleteDevice409JSONResponse{
+				Code:    "in_use",
+				Message: "device is attached to an instance",
+			}, nil
+		default:
+			return oapi.DeleteDevice500JSONResponse{
+				Code:    "internal_error",
+				Message: err.Error(),
+			}, nil
+		}
+	}
+
+	return oapi.DeleteDevice204Response{}, nil
+}
+
+// Helper functions
+
+func deviceToOAPI(d devices.Device) oapi.Device {
+	deviceType := oapi.DeviceType(d.Type)
+	return oapi.Device{
+		Id:          d.Id,
+		Name:        &d.Name,
+		Type:        deviceType,
+		PciAddress:  d.PCIAddress,
+		VendorId:    d.VendorID,
+		DeviceId:    d.DeviceID,
+		IommuGroup:  d.IOMMUGroup,
+		BoundToVfio: d.BoundToVFIO,
+		AttachedTo:  d.AttachedTo,
+		CreatedAt:   d.CreatedAt,
+	}
+}
+
+func availableDeviceToOAPI(d devices.AvailableDevice) oapi.AvailableDevice {
+	return oapi.AvailableDevice{
+		PciAddress:    d.PCIAddress,
+		VendorId:      d.VendorID,
+		DeviceId:      d.DeviceID,
+		VendorName:    &d.VendorName,
+		DeviceName:    &d.DeviceName,
+		IommuGroup:    d.IOMMUGroup,
+		CurrentDriver: d.CurrentDriver,
+	}
+}
+
+

cmd/api/api/instances.go

@@ -96,6 +96,12 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 		networkEnabled = *request.Body.Network.Enabled
 	}
 
+	// Parse devices (GPU passthrough)
+	var deviceRefs []string
+	if request.Body.Devices != nil {
+		deviceRefs = *request.Body.Devices
+	}
+
 	// Parse volumes
 	var volumes []instances.VolumeAttachment
 	if request.Body.Volumes != nil {
@@ -139,6 +145,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 		Vcpus:          vcpus,
 		Env:            env,
 		NetworkEnabled: networkEnabled,
+		Devices:        deviceRefs,
 		Volumes:        volumes,
 	}
 

cmd/api/main.go

@@ -172,6 +172,18 @@ func run() error {
 	}
 	logger.Info("Network manager initialized")
 
+	// Reconcile device state (clears orphaned attachments from crashed VMs)
+	// Set up liveness checker so device reconciliation can accurately detect orphaned attachments
+	logger.Info("Reconciling device state...")
+	livenessChecker := instances.NewLivenessChecker(app.InstanceManager)
+	if livenessChecker != nil {
+		app.DeviceManager.SetLivenessChecker(livenessChecker)
+	}
+	if err := app.DeviceManager.ReconcileDevices(app.Ctx); err != nil {
+		logger.Error("failed to reconcile device state", "error", err)
+		return fmt.Errorf("reconcile device state: %w", err)
+	}
+
 	// Initialize ingress manager (starts Caddy daemon and DNS server for dynamic upstreams)
 	logger.Info("Initializing ingress manager...")
 	if err := app.IngressManager.Initialize(app.Ctx); err != nil {

go.mod

@@ -41,30 +41,37 @@ require (
 	go.opentelemetry.io/otel/trace v1.38.0
 	golang.org/x/sync v0.17.0
 	golang.org/x/sys v0.38.0
-	golang.org/x/term v0.37.0
 	google.golang.org/grpc v1.77.0
 	google.golang.org/protobuf v1.36.10
 	gvisor.dev/gvisor v0.0.0-20251125014920-fc40e232ff54
 )
 
 require (
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
 	github.com/apex/log v1.9.0 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
+	github.com/containerd/errdefs v1.0.0 // indirect
+	github.com/containerd/errdefs/pkg v0.3.0 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/docker/cli v28.2.2+incompatible // indirect
 	github.com/docker/distribution v2.8.3+incompatible // indirect
+	github.com/docker/docker v28.2.2+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.9.3 // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/go-test/deep v1.1.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
@@ -74,6 +81,8 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mdlayher/socket v0.5.1 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/sys/sequential v0.6.0 // indirect
 	github.com/moby/sys/user v0.4.0 // indirect
 	github.com/moby/sys/userns v0.1.0 // indirect
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
@@ -92,6 +101,7 @@ require (
 	github.com/vishvananda/netns v0.0.5 // indirect
 	github.com/woodsbury/decimal128 v1.3.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
 	go.opentelemetry.io/otel/log v0.14.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.7.1 // indirect