Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions .air.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
root = "."
testdata_dir = "testdata"
tmp_dir = "tmp"

[build]
args_bin = []
bin = "./tmp/main"
cmd = "go build -o ./tmp/main ./cmd/api"
delay = 1000
exclude_dir = ["assets", "tmp", "vendor", "testdata", "bin", "scripts", "data", "kernel"]
exclude_file = []
exclude_regex = ["_test.go"]
exclude_unchanged = false
follow_symlink = false
full_bin = ""
include_dir = []
include_ext = ["go", "tpl", "tmpl", "html", "yaml"]
include_file = []
kill_delay = "0s"
log = "build-errors.log"
poll = false
poll_interval = 0
rerun = false
rerun_delay = 500
send_interrupt = false
stop_on_error = false

[color]
app = ""
build = "yellow"
main = "magenta"
runner = "green"
watcher = "cyan"

[log]
main_only = false
time = false

[misc]
clean_on_exit = false

[screen]
clear_on_rebuild = false
keep_scroll = true

1 change: 1 addition & 0 deletions .env.example
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
JWT_SECRET='your-secret-key-here'
25 changes: 25 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
name: Test

on:
push: {}

jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4

- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: '1.25'

- name: Install dependencies
run: go mod download

- name: Run tests
run: make test

- name: Build
run: make build

6 changes: 6 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,9 @@ initramfs-overlay/**
data/**
*.raw
*.sock
kernel
kernel/**
bin/**
.env
tmp
tmp/**
60 changes: 60 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
SHELL := /bin/bash
.PHONY: oapi-generate generate-wire generate-all dev build test install-tools

# Directory where local binaries will be installed
BIN_DIR ?= $(CURDIR)/bin

$(BIN_DIR):
mkdir -p $(BIN_DIR)

# Local binary paths
OAPI_CODEGEN ?= $(BIN_DIR)/oapi-codegen
AIR ?= $(BIN_DIR)/air
WIRE ?= $(BIN_DIR)/wire

# Install oapi-codegen
$(OAPI_CODEGEN): | $(BIN_DIR)
GOBIN=$(BIN_DIR) go install github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen@latest

# Install air for hot reload
$(AIR): | $(BIN_DIR)
GOBIN=$(BIN_DIR) go install github.com/air-verse/air@latest

# Install wire for dependency injection
$(WIRE): | $(BIN_DIR)
GOBIN=$(BIN_DIR) go install github.com/google/wire/cmd/wire@latest

install-tools: $(OAPI_CODEGEN) $(AIR) $(WIRE)

# Generate Go code from OpenAPI spec
oapi-generate: $(OAPI_CODEGEN)
@echo "Generating Go code from OpenAPI spec..."
$(OAPI_CODEGEN) -config ./oapi-codegen.yaml ./openapi.yaml
@echo "Formatting generated code..."
go fmt ./lib/oapi/oapi.go

# Generate wire dependency injection code
generate-wire: $(WIRE)
@echo "Generating wire code..."
cd ./cmd/api && $(WIRE)

# Generate all code
generate-all: oapi-generate generate-wire

# Build the binary
build: | $(BIN_DIR)
go build -o $(BIN_DIR)/hypeman ./cmd/api

# Run in development mode with hot reload
dev: $(AIR)
$(AIR) -c .air.toml

# Run tests
test:
go test -v -timeout 30s ./...

# Clean generated files and binaries
clean:
rm -rf $(BIN_DIR)
rm -f lib/oapi/oapi.go

115 changes: 29 additions & 86 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,125 +1,68 @@
# Cloud Hypervisor POC
# Hypeman

Proof of concept for running 10 Chromium VMs simultaneously using Cloud Hypervisor with disk-based overlays, config disks, networking isolation, and standby/restore functionality.
[![Test](https://github.com/onkernel/hypeman/actions/workflows/test.yml/badge.svg)](https://github.com/onkernel/hypeman/actions/workflows/test.yml)

## Prerequisites
Run containerized workloads in VMs, powered by [Cloud Hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor).

Install cloud-hypervisor by [installing the pre-built binaries](https://www.cloudhypervisor.org/docs/prologue/quick-start/#use-pre-built-binaries). Make sure `ch-remote` and `cloud-hypervisor` are in path.
## Getting Started

```bash
ch-remote --version
cloud-hypervisor --version
```

Tested with version `v48.0.0`

Note: Requires `kernel-images-private` cloned to home directory with `iproute2` installed in the Chromium headful image.

Also, `lsof` and `lz4` needs to be installed on the host

```
sudo apt-get install -y lsof lz4
```

## Setup

Build kernel, initrd, and rootfs with config disk support:
### Prerequisites

**Cloud Hypervisor** - [Installation guide](https://www.cloudhypervisor.org/docs/prologue/quick-start/#use-pre-built-binaries)
```bash
./scripts/build-initrd.sh
```

This creates:
- `data/system/vmlinux` - Linux kernel
- `data/system/initrd` - BusyBox init with disk-based overlay
- `data/images/chromium-headful/v1/rootfs.ext4` - Chromium rootfs (read-only, shared)

Configure host network with bridge and guest isolation:

```bash
./scripts/setup-host-network.sh
```

Create 10 VM configurations (IPs 192.168.100.10-19, isolated TAP devices, overlay disks, config disks):

```bash
./scripts/setup-vms.sh
cloud-hypervisor --version # Verify
ch-remote --version
```

## Running VMs

Start all 10 VMs:

**containerd** - [Installation guide](https://github.com/containerd/containerd/blob/main/docs/getting-started.md)
```bash
./scripts/start-all-vms.sh
containerd --version # Verify
```

Check VM status:
**Go 1.25.4+** and **KVM**

```bash
./scripts/list-vms.sh
```

View VM logs:
### Configuration

```bash
./scripts/logs-vm.sh <vm-id> # Show last 100 lines
./scripts/logs-vm.sh <vm-id> -f # Follow logs
cp .env.example .env
# Edit .env and set JWT_SECRET
```

SSH into a VM:
### Build

```bash
./scripts/ssh-vm.sh <vm-id> # Password: root
make build
```
### Running the Server

Stop a VM:

Start the server with hot-reload for development:
```bash
./scripts/stop-vm.sh <vm-id>
./scripts/stop-all-vms.sh # Stop all
make dev
```
The server will start on port 8080 (configurable via `PORT` environment variable).

## Standby / Restore

Standby a VM (pause, snapshot, delete VMM):

```bash
./scripts/standby-vm.sh <vm-id>
```

Restore a VM from snapshot:
### Testing

```bash
./scripts/restore-vm.sh <vm-id>
make test
```

## Networking
### Code Generation

Enable port forwarding for WebRTC access (localhost:8080-8089 → guest VMs):
After modifying `openapi.yaml`, regenerate the Go code:

```bash
./scripts/setup-port-forwarding.sh
make oapi-generate
```

Connect to a VM:
After modifying dependency injection in `cmd/api/wire.go` or `lib/providers/providers.go`, regenerate wire code:

```bash
./scripts/connect-guest.sh <vm-id>
make generate-wire
```

## Volumes

Create a persistent volume:
Or generate everything at once:

```bash
./scripts/create-volume.sh <vol-id> <size-gb>
make generate-all
```

## Architecture

- **Disk-based overlay**: Each VM has a 50GB sparse overlay disk on `/dev/vdb` (faster restore than tmpfs)
- **Config disk**: Each VM has a config disk on `/dev/vdc` with VM-specific settings (IP, MAC, envs)
- **Guest isolation**: VMs cannot communicate with each other (iptables + bridge_slave isolation)
- **Serial logging**: All VM output captured to `data/guests/guest-N/logs/console.log`
- **Shared rootfs**: Single read-only rootfs image shared across all VMs
35 changes: 35 additions & 0 deletions cmd/api/api/api.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package api

import (
"github.com/onkernel/hypeman/cmd/api/config"
"github.com/onkernel/hypeman/lib/images"
"github.com/onkernel/hypeman/lib/instances"
"github.com/onkernel/hypeman/lib/oapi"
"github.com/onkernel/hypeman/lib/volumes"
)

// ApiService implements the oapi.StrictServerInterface
type ApiService struct {
Config *config.Config
ImageManager images.Manager
InstanceManager instances.Manager
VolumeManager volumes.Manager
}

var _ oapi.StrictServerInterface = (*ApiService)(nil)

// New creates a new ApiService
func New(
config *config.Config,
imageManager images.Manager,
instanceManager instances.Manager,
volumeManager volumes.Manager,
) *ApiService {
return &ApiService{
Config: config,
ImageManager: imageManager,
InstanceManager: instanceManager,
VolumeManager: volumeManager,
}
}

30 changes: 30 additions & 0 deletions cmd/api/api/api_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package api

import (
"context"
"testing"

"github.com/onkernel/hypeman/cmd/api/config"
"github.com/onkernel/hypeman/lib/images"
"github.com/onkernel/hypeman/lib/instances"
"github.com/onkernel/hypeman/lib/volumes"
)

// newTestService creates an ApiService for testing with temporary data directory
func newTestService(t *testing.T) *ApiService {
cfg := &config.Config{
DataDir: t.TempDir(),
}

return &ApiService{
Config: cfg,
ImageManager: images.NewManager(cfg.DataDir),
InstanceManager: instances.NewManager(cfg.DataDir),
VolumeManager: volumes.NewManager(cfg.DataDir),
}
}

func ctx() context.Context {
return context.Background()
}

15 changes: 15 additions & 0 deletions cmd/api/api/health.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package api

import (
"context"

"github.com/onkernel/hypeman/lib/oapi"
)

// GetHealth implements health check endpoint
func (s *ApiService) GetHealth(ctx context.Context, request oapi.GetHealthRequestObject) (oapi.GetHealthResponseObject, error) {
return oapi.GetHealth200JSONResponse{
Status: oapi.Ok,
}, nil
}

Loading