Merge remote-tracking branch 'origin/main' into qemu

Author: sjmiller609

.gitignore

@@ -24,3 +24,6 @@ lib/system/exec_agent/exec-agent
 # Envoy binaries
 lib/ingress/binaries/**
 dist/**
+
+# UTM VM - downloaded ISO files
+scripts/utm/images/

DEVELOPMENT.md

@@ -4,6 +4,8 @@ This document covers development setup, configuration, and contributing to Hypem
 
 ## Prerequisites
 
+> **macOS Users:** Hypeman requires KVM, which is only available on Linux. See [scripts/utm/README.md](scripts/utm/README.md) for instructions on setting up a Linux VM with nested virtualization on Apple Silicon Macs.
+
 **Go 1.25.4+**, **KVM**, **erofs-utils**, **dnsmasq**
 
 ```bash
@@ -13,17 +15,19 @@ dnsmasq --version
 ```
 
 **Install on Debian/Ubuntu:**
+
 ```bash
 sudo apt-get install erofs-utils dnsmasq
 ```
 
 **KVM Access:** User must be in `kvm` group for VM access:
+
 ```bash
 sudo usermod -aG kvm $USER
 # Log out and back in, or use: newgrp kvm
 ```
 
-**Network Capabilities:** 
+**Network Capabilities:**
 
 Before running or testing Hypeman, ensure IPv4 forwarding is enabled:
 
@@ -39,6 +43,7 @@ sudo sysctl -p
 **Why:** Required for routing traffic between VM network and external network.
 
 The hypeman binary needs network administration capabilities to create bridges and TAP devices:
+
 ```bash
 # After building, grant network capabilities
 sudo setcap 'cap_net_admin,cap_net_bind_service=+eip' /path/to/hypeman
@@ -78,34 +83,34 @@ root  hard  nofile  65536
 
 Hypeman can be configured using the following environment variables:
 
-| Variable | Description | Default |
-|----------|-------------|---------|
-| `PORT` | HTTP server port | `8080` |
-| `DATA_DIR` | Directory for storing VM images, volumes, and other data | `/var/lib/hypeman` |
-| `BRIDGE_NAME` | Name of the network bridge for VM networking | `vmbr0` |
-| `SUBNET_CIDR` | CIDR notation for the VM network subnet (gateway derived automatically) | `10.100.0.0/16` |
-| `UPLINK_INTERFACE` | Host network interface to use for VM internet access | _(auto-detect)_ |
-| `JWT_SECRET` | Secret key for JWT authentication (required for production) | _(empty)_ |
-| `DNS_SERVER` | DNS server IP address for VMs | `1.1.1.1` |
-| `MAX_CONCURRENT_BUILDS` | Maximum number of concurrent image builds | `1` |
-| `MAX_OVERLAY_SIZE` | Maximum size for overlay filesystem | `100GB` |
-| `ENV` | Deployment environment (filters telemetry, e.g. your name for dev) | `unset` |
-| `OTEL_ENABLED` | Enable OpenTelemetry traces/metrics | `false` |
-| `OTEL_ENDPOINT` | OTLP gRPC endpoint | `127.0.0.1:4317` |
-| `OTEL_SERVICE_INSTANCE_ID` | Instance ID for telemetry (differentiates multiple servers) | hostname |
-| `LOG_LEVEL` | Default log level (debug, info, warn, error) | `info` |
-| `LOG_LEVEL_<SUBSYSTEM>` | Per-subsystem log level (API, IMAGES, INSTANCES, NETWORK, VOLUMES, VMM, SYSTEM, EXEC, CADDY) | inherits default |
-| `CADDY_LISTEN_ADDRESS` | Address for Caddy ingress listeners | `0.0.0.0` |
-| `CADDY_ADMIN_ADDRESS` | Address for Caddy admin API | `127.0.0.1` |
-| `CADDY_ADMIN_PORT` | Port for Caddy admin API | `2019` |
-| `CADDY_STOP_ON_SHUTDOWN` | Stop Caddy when hypeman shuts down (set to `true` for dev) | `false` |
-| `ACME_EMAIL` | Email for ACME certificate registration (required for TLS ingresses) | _(empty)_ |
-| `ACME_DNS_PROVIDER` | DNS provider for ACME challenges: `cloudflare` | _(empty)_ |
-| `ACME_CA` | ACME CA URL (empty = Let's Encrypt production) | _(empty)_ |
-| `TLS_ALLOWED_DOMAINS` | Comma-separated allowed domains for TLS (e.g., `*.example.com,api.other.com`) | _(empty)_ |
-| `DNS_PROPAGATION_TIMEOUT` | Max time to wait for DNS propagation (e.g., `2m`) | _(empty)_ |
-| `DNS_RESOLVERS` | Comma-separated DNS resolvers for propagation checking | _(empty)_ |
-| `CLOUDFLARE_API_TOKEN` | Cloudflare API token (when using `cloudflare` provider) | _(empty)_ |
+| Variable                   | Description                                                                                  | Default            |
+| -------------------------- | -------------------------------------------------------------------------------------------- | ------------------ |
+| `PORT`                     | HTTP server port                                                                             | `8080`             |
+| `DATA_DIR`                 | Directory for storing VM images, volumes, and other data                                     | `/var/lib/hypeman` |
+| `BRIDGE_NAME`              | Name of the network bridge for VM networking                                                 | `vmbr0`            |
+| `SUBNET_CIDR`              | CIDR notation for the VM network subnet (gateway derived automatically)                      | `10.100.0.0/16`    |
+| `UPLINK_INTERFACE`         | Host network interface to use for VM internet access                                         | _(auto-detect)_    |
+| `JWT_SECRET`               | Secret key for JWT authentication (required for production)                                  | _(empty)_          |
+| `DNS_SERVER`               | DNS server IP address for VMs                                                                | `1.1.1.1`          |
+| `MAX_CONCURRENT_BUILDS`    | Maximum number of concurrent image builds                                                    | `1`                |
+| `MAX_OVERLAY_SIZE`         | Maximum size for overlay filesystem                                                          | `100GB`            |
+| `ENV`                      | Deployment environment (filters telemetry, e.g. your name for dev)                           | `unset`            |
+| `OTEL_ENABLED`             | Enable OpenTelemetry traces/metrics                                                          | `false`            |
+| `OTEL_ENDPOINT`            | OTLP gRPC endpoint                                                                           | `127.0.0.1:4317`   |
+| `OTEL_SERVICE_INSTANCE_ID` | Instance ID for telemetry (differentiates multiple servers)                                  | hostname           |
+| `LOG_LEVEL`                | Default log level (debug, info, warn, error)                                                 | `info`             |
+| `LOG_LEVEL_<SUBSYSTEM>`    | Per-subsystem log level (API, IMAGES, INSTANCES, NETWORK, VOLUMES, VMM, SYSTEM, EXEC, CADDY) | inherits default   |
+| `CADDY_LISTEN_ADDRESS`     | Address for Caddy ingress listeners                                                          | `0.0.0.0`          |
+| `CADDY_ADMIN_ADDRESS`      | Address for Caddy admin API                                                                  | `127.0.0.1`        |
+| `CADDY_ADMIN_PORT`         | Port for Caddy admin API                                                                     | `2019`             |
+| `CADDY_STOP_ON_SHUTDOWN`   | Stop Caddy when hypeman shuts down (set to `true` for dev)                                   | `false`            |
+| `ACME_EMAIL`               | Email for ACME certificate registration (required for TLS ingresses)                         | _(empty)_          |
+| `ACME_DNS_PROVIDER`        | DNS provider for ACME challenges: `cloudflare`                                               | _(empty)_          |
+| `ACME_CA`                  | ACME CA URL (empty = Let's Encrypt production)                                               | _(empty)_          |
+| `TLS_ALLOWED_DOMAINS`      | Comma-separated allowed domains for TLS (e.g., `*.example.com,api.other.com`)                | _(empty)_          |
+| `DNS_PROPAGATION_TIMEOUT`  | Max time to wait for DNS propagation (e.g., `2m`)                                            | _(empty)_          |
+| `DNS_RESOLVERS`            | Comma-separated DNS resolvers for propagation checking                                       | _(empty)_          |
+| `CLOUDFLARE_API_TOKEN`     | Cloudflare API token (when using `cloudflare` provider)                                      | _(empty)_          |
 
 **Important: Subnet Configuration**
 
@@ -114,10 +119,12 @@ The default subnet `10.100.0.0/16` is chosen to avoid common conflicts. Hypeman
 If you need a different subnet, set `SUBNET_CIDR` in your environment. The gateway is automatically derived as the first IP in the subnet (e.g., `10.100.0.0/16` → `10.100.0.1`).
 
 **Alternative subnets if needed:**
+
 - `172.30.0.0/16` - Private range between common Docker (172.17.x.x) and cloud provider (172.31.x.x) ranges
 - `10.200.0.0/16` - Another private range option
 
 **Example:**
+
 ```bash
 # In your .env file
 SUBNET_CIDR=172.30.0.0/16
@@ -128,23 +135,30 @@ SUBNET_CIDR=172.30.0.0/16
 `UPLINK_INTERFACE` tells Hypeman which host interface to use for routing VM traffic to the outside world (for iptables MASQUERADE rules). On many hosts this is `eth0`, but laptops and more complex setups often use Wi‑Fi or other names.
 
 **Quick way to discover it:**
+
 ```bash
 # Ask the kernel which interface is used to reach the internet
 ip route get 1.1.1.1
 ```
+
 Look for the `dev` field in the output, for example:
+
 ```text
 1.1.1.1 via 192.168.12.1 dev wlp2s0 src 192.168.12.98
 ```
+
 In this case, `wlp2s0` is the uplink interface, so you would set:
+
 ```bash
 UPLINK_INTERFACE=wlp2s0
 ```
 
 You can also inspect all routes:
+
 ```bash
 ip route show
 ```
+
 Pick the interface used by the default route (usually the line starting with `default`). Avoid using local bridges like `docker0`, `br-...`, `virbr0`, or `vmbr0` as the uplink; those are typically internal virtual networks, not your actual internet-facing interface.
 
 ### TLS Ingress (HTTPS)
@@ -154,6 +168,7 @@ Hypeman uses Caddy with automatic ACME certificates for TLS termination. Certifi
 To enable TLS ingresses:
 
 1. Configure ACME credentials in your `.env`:
+
 ```bash
 # Required for any TLS ingress
 [email protected]
@@ -164,6 +179,7 @@ CLOUDFLARE_API_TOKEN=your-api-token
 ```
 
 2. Create an ingress with TLS enabled:
+
 ```bash
 curl -X POST http://localhost:8080/v1/ingresses \
   -H "Content-Type: application/json" \
@@ -199,6 +215,7 @@ sudo chown $USER:$USER /var/lib/hypeman
 ### Dockerhub login
 
 Requires Docker Hub authentication to avoid rate limits when running the tests:
+
 ```bash
 docker login
 ```
@@ -214,14 +231,17 @@ make build
 ## Running the Server
 
 1. Generate a JWT token for testing (optional):
+
 ```bash
 make gen-jwt
 ```
 
 2. Start the server with hot-reload for development:
+
 ```bash
 make dev
 ```
+
 The server will start on port 8080 (configurable via `PORT` environment variable).
 
 ### Local OpenTelemetry (optional)
@@ -232,15 +252,19 @@ To collect traces and metrics locally, run the Grafana LGTM stack (Loki, Grafana
 # Start Grafana LGTM (UI at http://localhost:3000, login: admin/admin)
 # Note, if you are developing on a shared server, you can use the same LGTM stack as your peer(s)
 # You will be able to sort your metrics, traces, and logs using the ENV configuration (see below)
+BIND=127.0.0.1
+# YOLO=1  # Uncomment to expose ports externally
+if [ -n "$YOLO" ]; then BIND=0.0.0.0; fi
+
 docker run -d --name lgtm \
-  -p 127.0.0.1:3000:3000 \
-  -p 127.0.0.1:4317:4317 \
-  -p 127.0.0.1:4318:4318 \
-  -p 127.0.0.1:9090:9090 \
-  -p 127.0.0.1:4040:4040 \
+  -p $BIND:3000:3000 \
+  -p $BIND:4317:4317 \
+  -p $BIND:4318:4318 \
+  -p $BIND:9090:9090 \
+  -p $BIND:4040:4040 \
   grafana/otel-lgtm:latest
 
-# If developing on a remote server, forward the port to your local machine:
+# If developing on a remote server, forward the port to your local machine (or YOLO):
 # ssh -L 3001:localhost:3000 your-server  (then open http://localhost:3001)
 
 # Enable OTel in .env (set ENV to your name to filter your telemetry)
@@ -254,6 +278,7 @@ make dev
 Open http://localhost:3000 to view traces (Tempo), metrics (Mimir), and logs (Loki) in Grafana.
 
 **Import the Hypeman dashboard:**
+
 1. Go to Dashboards → New → Import
 2. Upload `dashboards/hypeman.json` or paste its contents
 3. Select the Prometheus datasource and click Import

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-chi/chi/v5 v5.2.3
 	github.com/golang-jwt/jwt/v5 v5.3.0
+	github.com/golang/protobuf v1.5.4
 	github.com/google/go-containerregistry v0.20.6
 	github.com/google/wire v0.7.0
 	github.com/gorilla/websocket v1.5.3
@@ -43,7 +44,6 @@ require (
 	golang.org/x/sync v0.17.0
 	golang.org/x/sys v0.38.0
 	google.golang.org/grpc v1.77.0
-	google.golang.org/protobuf v1.36.10
 	gvisor.dev/gvisor v0.0.0-20251125014920-fc40e232ff54
 )
 
@@ -114,6 +114,7 @@ require (
 	golang.org/x/tools v0.37.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect
+	google.golang.org/protobuf v1.36.10 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gotest.tools/v3 v3.5.2 // indirect

lib/images/oci.go

@@ -4,10 +4,12 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"runtime"
 	"strings"
 
 	"github.com/google/go-containerregistry/pkg/authn"
 	"github.com/google/go-containerregistry/pkg/name"
+	gcr "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/empty"
 	"github.com/google/go-containerregistry/pkg/v1/layout"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
@@ -60,24 +62,42 @@ func newOCIClient(cacheDir string) (*ociClient, error) {
 	return &ociClient{cacheDir: cacheDir}, nil
 }
 
+// currentPlatform returns the platform for the current host
+func currentPlatform() gcr.Platform {
+	return gcr.Platform{
+		Architecture: runtime.GOARCH,
+		OS:           runtime.GOOS,
+	}
+}
+
 // inspectManifest synchronously inspects a remote image to get its digest
 // without pulling the image. This is used for upfront digest discovery.
+// For multi-arch images, it returns the platform-specific manifest digest
+// (matching the current host platform) rather than the manifest index digest.
 func (c *ociClient) inspectManifest(ctx context.Context, imageRef string) (string, error) {
 	ref, err := name.ParseReference(imageRef)
 	if err != nil {
 		return "", fmt.Errorf("parse image reference: %w", err)
 	}
 
-	// Use system authentication (reads from ~/.docker/config.json, etc.)
-	// Default retry: only on network errors, max ~1.3s total
-	descriptor, err := remote.Head(ref,
+	// Use remote.Image with platform filtering to get the platform-specific digest.
+	// For multi-arch images, this resolves the manifest index to the correct platform.
+	// This matches what pullToOCILayout does to ensure cache key consistency.
+	// Note: remote.Image is lazy - it only fetches the manifest, not layer blobs.
+	img, err := remote.Image(ref,
 		remote.WithContext(ctx),
-		remote.WithAuthFromKeychain(authn.DefaultKeychain))
+		remote.WithAuthFromKeychain(authn.DefaultKeychain),
+		remote.WithPlatform(currentPlatform()))
 	if err != nil {
 		return "", fmt.Errorf("fetch manifest: %w", wrapRegistryError(err))
 	}
 
-	return descriptor.Digest.String(), nil
+	digest, err := img.Digest()
+	if err != nil {
+		return "", fmt.Errorf("get image digest: %w", err)
+	}
+
+	return digest.String(), nil
 }
 
 // pullResult contains the metadata and digest from pulling an image
@@ -126,9 +146,11 @@ func (c *ociClient) pullToOCILayout(ctx context.Context, imageRef, layoutTag str
 
 	// Use system authentication (reads from ~/.docker/config.json, etc.)
 	// Default retry: only on network errors, max ~1.3s total
+	// WithPlatform ensures we pull the correct architecture for multi-arch images
 	img, err := remote.Image(ref,
 		remote.WithContext(ctx),
-		remote.WithAuthFromKeychain(authn.DefaultKeychain))
+		remote.WithAuthFromKeychain(authn.DefaultKeychain),
+		remote.WithPlatform(currentPlatform()))
 	if err != nil {
 		// Rate limits fail here immediately (429 is not retried by default)
 		return fmt.Errorf("fetch image manifest: %w", wrapRegistryError(err))

lib/system/versions.go

@@ -74,5 +74,8 @@ func GetArch() string {
 	if arch == "amd64" {
 		return "x86_64"
 	}
+	if arch == "arm64" {
+		return "aarch64"
+	}
 	return arch
 }

lib/vmm/binaries.go

@@ -30,6 +30,8 @@ func ExtractBinary(p *paths.Paths, version CHVersion) (string, error) {
 	arch := runtime.GOARCH
 	if arch == "amd64" {
 		arch = "x86_64"
+	} else if arch == "arm64" {
+		arch = "aarch64"
 	}
 
 	embeddedPath := fmt.Sprintf("binaries/cloud-hypervisor/%s/%s/cloud-hypervisor", version, arch)