diff --git a/.gitignore b/.gitignore index fd299f8..d64bbf0 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ lib/system/exec_agent/exec-agent # Envoy binaries lib/ingress/binaries/** dist/** + +# UTM VM - downloaded ISO files +scripts/utm/images/ diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 487193f..85a14f8 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -4,6 +4,8 @@ This document covers development setup, configuration, and contributing to Hypem ## Prerequisites +> **macOS Users:** Hypeman requires KVM, which is only available on Linux. See [scripts/utm/README.md](scripts/utm/README.md) for instructions on setting up a Linux VM with nested virtualization on Apple Silicon Macs. + **Go 1.25.4+**, **KVM**, **erofs-utils**, **dnsmasq** ```bash @@ -13,17 +15,19 @@ dnsmasq --version ``` **Install on Debian/Ubuntu:** + ```bash sudo apt-get install erofs-utils dnsmasq ``` **KVM Access:** User must be in `kvm` group for VM access: + ```bash sudo usermod -aG kvm $USER # Log out and back in, or use: newgrp kvm ``` -**Network Capabilities:** +**Network Capabilities:** Before running or testing Hypeman, ensure IPv4 forwarding is enabled: @@ -39,6 +43,7 @@ sudo sysctl -p **Why:** Required for routing traffic between VM network and external network. The hypeman binary needs network administration capabilities to create bridges and TAP devices: + ```bash # After building, grant network capabilities sudo setcap 'cap_net_admin,cap_net_bind_service=+eip' /path/to/hypeman @@ -78,34 +83,34 @@ root hard nofile 65536 Hypeman can be configured using the following environment variables: -| Variable | Description | Default | -|----------|-------------|---------| -| `PORT` | HTTP server port | `8080` | -| `DATA_DIR` | Directory for storing VM images, volumes, and other data | `/var/lib/hypeman` | -| `BRIDGE_NAME` | Name of the network bridge for VM networking | `vmbr0` | -| `SUBNET_CIDR` | CIDR notation for the VM network subnet (gateway derived automatically) | `10.100.0.0/16` | -| `UPLINK_INTERFACE` | Host network interface to use for VM internet access | _(auto-detect)_ | -| `JWT_SECRET` | Secret key for JWT authentication (required for production) | _(empty)_ | -| `DNS_SERVER` | DNS server IP address for VMs | `1.1.1.1` | -| `MAX_CONCURRENT_BUILDS` | Maximum number of concurrent image builds | `1` | -| `MAX_OVERLAY_SIZE` | Maximum size for overlay filesystem | `100GB` | -| `ENV` | Deployment environment (filters telemetry, e.g. your name for dev) | `unset` | -| `OTEL_ENABLED` | Enable OpenTelemetry traces/metrics | `false` | -| `OTEL_ENDPOINT` | OTLP gRPC endpoint | `127.0.0.1:4317` | -| `OTEL_SERVICE_INSTANCE_ID` | Instance ID for telemetry (differentiates multiple servers) | hostname | -| `LOG_LEVEL` | Default log level (debug, info, warn, error) | `info` | -| `LOG_LEVEL_` | Per-subsystem log level (API, IMAGES, INSTANCES, NETWORK, VOLUMES, VMM, SYSTEM, EXEC, CADDY) | inherits default | -| `CADDY_LISTEN_ADDRESS` | Address for Caddy ingress listeners | `0.0.0.0` | -| `CADDY_ADMIN_ADDRESS` | Address for Caddy admin API | `127.0.0.1` | -| `CADDY_ADMIN_PORT` | Port for Caddy admin API | `2019` | -| `CADDY_STOP_ON_SHUTDOWN` | Stop Caddy when hypeman shuts down (set to `true` for dev) | `false` | -| `ACME_EMAIL` | Email for ACME certificate registration (required for TLS ingresses) | _(empty)_ | -| `ACME_DNS_PROVIDER` | DNS provider for ACME challenges: `cloudflare` | _(empty)_ | -| `ACME_CA` | ACME CA URL (empty = Let's Encrypt production) | _(empty)_ | -| `TLS_ALLOWED_DOMAINS` | Comma-separated allowed domains for TLS (e.g., `*.example.com,api.other.com`) | _(empty)_ | -| `DNS_PROPAGATION_TIMEOUT` | Max time to wait for DNS propagation (e.g., `2m`) | _(empty)_ | -| `DNS_RESOLVERS` | Comma-separated DNS resolvers for propagation checking | _(empty)_ | -| `CLOUDFLARE_API_TOKEN` | Cloudflare API token (when using `cloudflare` provider) | _(empty)_ | +| Variable | Description | Default | +| -------------------------- | -------------------------------------------------------------------------------------------- | ------------------ | +| `PORT` | HTTP server port | `8080` | +| `DATA_DIR` | Directory for storing VM images, volumes, and other data | `/var/lib/hypeman` | +| `BRIDGE_NAME` | Name of the network bridge for VM networking | `vmbr0` | +| `SUBNET_CIDR` | CIDR notation for the VM network subnet (gateway derived automatically) | `10.100.0.0/16` | +| `UPLINK_INTERFACE` | Host network interface to use for VM internet access | _(auto-detect)_ | +| `JWT_SECRET` | Secret key for JWT authentication (required for production) | _(empty)_ | +| `DNS_SERVER` | DNS server IP address for VMs | `1.1.1.1` | +| `MAX_CONCURRENT_BUILDS` | Maximum number of concurrent image builds | `1` | +| `MAX_OVERLAY_SIZE` | Maximum size for overlay filesystem | `100GB` | +| `ENV` | Deployment environment (filters telemetry, e.g. your name for dev) | `unset` | +| `OTEL_ENABLED` | Enable OpenTelemetry traces/metrics | `false` | +| `OTEL_ENDPOINT` | OTLP gRPC endpoint | `127.0.0.1:4317` | +| `OTEL_SERVICE_INSTANCE_ID` | Instance ID for telemetry (differentiates multiple servers) | hostname | +| `LOG_LEVEL` | Default log level (debug, info, warn, error) | `info` | +| `LOG_LEVEL_` | Per-subsystem log level (API, IMAGES, INSTANCES, NETWORK, VOLUMES, VMM, SYSTEM, EXEC, CADDY) | inherits default | +| `CADDY_LISTEN_ADDRESS` | Address for Caddy ingress listeners | `0.0.0.0` | +| `CADDY_ADMIN_ADDRESS` | Address for Caddy admin API | `127.0.0.1` | +| `CADDY_ADMIN_PORT` | Port for Caddy admin API | `2019` | +| `CADDY_STOP_ON_SHUTDOWN` | Stop Caddy when hypeman shuts down (set to `true` for dev) | `false` | +| `ACME_EMAIL` | Email for ACME certificate registration (required for TLS ingresses) | _(empty)_ | +| `ACME_DNS_PROVIDER` | DNS provider for ACME challenges: `cloudflare` | _(empty)_ | +| `ACME_CA` | ACME CA URL (empty = Let's Encrypt production) | _(empty)_ | +| `TLS_ALLOWED_DOMAINS` | Comma-separated allowed domains for TLS (e.g., `*.example.com,api.other.com`) | _(empty)_ | +| `DNS_PROPAGATION_TIMEOUT` | Max time to wait for DNS propagation (e.g., `2m`) | _(empty)_ | +| `DNS_RESOLVERS` | Comma-separated DNS resolvers for propagation checking | _(empty)_ | +| `CLOUDFLARE_API_TOKEN` | Cloudflare API token (when using `cloudflare` provider) | _(empty)_ | **Important: Subnet Configuration** @@ -114,10 +119,12 @@ The default subnet `10.100.0.0/16` is chosen to avoid common conflicts. Hypeman If you need a different subnet, set `SUBNET_CIDR` in your environment. The gateway is automatically derived as the first IP in the subnet (e.g., `10.100.0.0/16` → `10.100.0.1`). **Alternative subnets if needed:** + - `172.30.0.0/16` - Private range between common Docker (172.17.x.x) and cloud provider (172.31.x.x) ranges - `10.200.0.0/16` - Another private range option **Example:** + ```bash # In your .env file SUBNET_CIDR=172.30.0.0/16 @@ -128,23 +135,30 @@ SUBNET_CIDR=172.30.0.0/16 `UPLINK_INTERFACE` tells Hypeman which host interface to use for routing VM traffic to the outside world (for iptables MASQUERADE rules). On many hosts this is `eth0`, but laptops and more complex setups often use Wi‑Fi or other names. **Quick way to discover it:** + ```bash # Ask the kernel which interface is used to reach the internet ip route get 1.1.1.1 ``` + Look for the `dev` field in the output, for example: + ```text 1.1.1.1 via 192.168.12.1 dev wlp2s0 src 192.168.12.98 ``` + In this case, `wlp2s0` is the uplink interface, so you would set: + ```bash UPLINK_INTERFACE=wlp2s0 ``` You can also inspect all routes: + ```bash ip route show ``` + Pick the interface used by the default route (usually the line starting with `default`). Avoid using local bridges like `docker0`, `br-...`, `virbr0`, or `vmbr0` as the uplink; those are typically internal virtual networks, not your actual internet-facing interface. ### TLS Ingress (HTTPS) @@ -154,6 +168,7 @@ Hypeman uses Caddy with automatic ACME certificates for TLS termination. Certifi To enable TLS ingresses: 1. Configure ACME credentials in your `.env`: + ```bash # Required for any TLS ingress ACME_EMAIL=admin@example.com @@ -164,6 +179,7 @@ CLOUDFLARE_API_TOKEN=your-api-token ``` 2. Create an ingress with TLS enabled: + ```bash curl -X POST http://localhost:8080/v1/ingresses \ -H "Content-Type: application/json" \ @@ -199,6 +215,7 @@ sudo chown $USER:$USER /var/lib/hypeman ### Dockerhub login Requires Docker Hub authentication to avoid rate limits when running the tests: + ```bash docker login ``` @@ -214,14 +231,17 @@ make build ## Running the Server 1. Generate a JWT token for testing (optional): + ```bash make gen-jwt ``` 2. Start the server with hot-reload for development: + ```bash make dev ``` + The server will start on port 8080 (configurable via `PORT` environment variable). ### Local OpenTelemetry (optional) @@ -232,15 +252,19 @@ To collect traces and metrics locally, run the Grafana LGTM stack (Loki, Grafana # Start Grafana LGTM (UI at http://localhost:3000, login: admin/admin) # Note, if you are developing on a shared server, you can use the same LGTM stack as your peer(s) # You will be able to sort your metrics, traces, and logs using the ENV configuration (see below) +BIND=127.0.0.1 +# YOLO=1 # Uncomment to expose ports externally +if [ -n "$YOLO" ]; then BIND=0.0.0.0; fi + docker run -d --name lgtm \ - -p 127.0.0.1:3000:3000 \ - -p 127.0.0.1:4317:4317 \ - -p 127.0.0.1:4318:4318 \ - -p 127.0.0.1:9090:9090 \ - -p 127.0.0.1:4040:4040 \ + -p $BIND:3000:3000 \ + -p $BIND:4317:4317 \ + -p $BIND:4318:4318 \ + -p $BIND:9090:9090 \ + -p $BIND:4040:4040 \ grafana/otel-lgtm:latest -# If developing on a remote server, forward the port to your local machine: +# If developing on a remote server, forward the port to your local machine (or YOLO): # ssh -L 3001:localhost:3000 your-server (then open http://localhost:3001) # Enable OTel in .env (set ENV to your name to filter your telemetry) @@ -254,6 +278,7 @@ make dev Open http://localhost:3000 to view traces (Tempo), metrics (Mimir), and logs (Loki) in Grafana. **Import the Hypeman dashboard:** + 1. Go to Dashboards → New → Import 2. Upload `dashboards/hypeman.json` or paste its contents 3. Select the Prometheus datasource and click Import diff --git a/scripts/utm/README.md b/scripts/utm/README.md new file mode 100644 index 0000000..13923fa --- /dev/null +++ b/scripts/utm/README.md @@ -0,0 +1,155 @@ +# UTM VM for Hypeman Development + +Run Hypeman in a UTM virtual machine with nested KVM virtualization on Apple Silicon Macs. + +> **Note:** In the future, this dev environment setup may be automated using Ansible or similar tooling. + +## Requirements + +- **macOS 15 (Sequoia)** or later +- **Apple M3 or newer** chip (required for nested virtualization) +- **UTM 4.6+**: `brew install --cask utm` + +## Setup + +### 1. Download Ubuntu Server ISO + +Download Ubuntu Server for ARM from [Ubuntu's website](https://ubuntu.com/download/server/arm). + +Or use the helper script: + +```bash +./download-iso.sh +``` + +### 2. Create VM in UTM + +Follow the [official UTM Ubuntu guide](https://docs.getutm.app/guides/ubuntu/): + +1. Open UTM and click **+** to **Create a New Virtual Machine** +1. Start: select **Virtualize** +1. Operating System: select **Linux** +1. Hardware: Set RAM to **8192 MiB** and CPU cores to **4** +1. Linux: check the box to **Use Apple Virtualization**. **Boot from ISO** should be the default. Click **Browse** and select the Ubuntu Server ISO. +1. Storage: Set disk size to **100 GiB** +1. Shared Directory: skip this. The `bootstrap-dev-environment.sh` script will clone Hypeman onto the VM. +1. Summary: Name the VM **Hypeman** and click **Save** + +**Important:** Before starting the VM: + +- Right-click **HypemanDev** → **Edit** +- Go to **System** → Check **Use Hypervisor** (enables nested virtualization) +- Save + +### 3. Install Ubuntu Server + +Start the VM and follow the Ubuntu installer. You can select the defaults for everything except the following: + +1. Profile configuration: set up with your name, a hostname like `hypeman` and a linux username and password you can remember. +1. **Install OpenSSH Server** ← Important to select this and then **Import SSH key** → **from GitHub** → Enter your GitHub username to import your public keys. The result is that, assuming your host machine has a private key that is uploaded to GitHub, you can re-use this for sshing into the VM. + +Once installation is complete, select **Reboot now**. + +### 4. Find VM IP Address + +After reboot, the VM console will ask you to log in with the Linux username and password you set up. +Once you do that and have a shell, run: + +```bash +ip addr +``` + +Look for an IP like `192.168.x.x` on the `enp0s1` interface. + +Open a terminal window on your host machine and export it: `export HYPEMAN_VM_IP=192.168.x.x` + +### 5. SSH into the VM + +From your Mac terminal: + +```bash +ssh -A -i ~/.ssh/id_ed25519 yourusername@${HYPEMAN_VM_IP} +``` + +### 6. Configure SSH (Recommended) + +Add this to your `~/.ssh/config`, replacing `192.168.x.x` with your VM's IP and `yourusername` with your Ubuntu username: + +``` +Host hypeman + HostName 192.168.x.x + User yourusername + IdentityFile ~/.ssh/id_ed25519 + ForwardAgent yes +``` + +Now you can simply run: + +```bash +ssh hypeman +``` + +## Setting up the VM for Hypeman Development + +Copy the `bootstrap-dev-environment.sh` script into the VM and run it: + +```bash +scp bootstrap-dev-environment.sh hypeman: +ssh hypeman +./bootstrap-dev-environment.sh +``` + +This installs Go, erofs-utils, dnsmasq, and other dependencies needed for Hypeman. + +It also clones the main Hypeman repositories into ~/code. + +## Development Workflow + +### Cursor Remote-SSH + +Cursor can run on your host machine and connect to the VM to edit files and run commands. + +1. Open Command Palette (`Cmd+Shift+P`) +2. Run **Remote-SSH: Connect to Host...** +3. Type **hypeman** (from your SSH config) +4. Open the `~/code/hypeman` directory. + +### Run Hypeman + +Follow the directions in [../../DEVELOPMENT.md](../../DEVELOPMENT.md). + +You might want to set up your host machine's `/etc/hosts` file to resolve `hypeman.local` to your VM's IP: + +``` +192.168.x.x hypeman.local +``` + +This will let you, for example, run the LGTM stack in the VM (according to the instructions in [../../DEVELOPMENT.md](../../DEVELOPMENT.md)) and then on your host machine, open http://hypeman.local:3000 to view the Grafana dashboard. + +It will also let you test ingresses with subdomain routing. + +### Hypeman CLI on host -> VM + +The Hypeman CLI can be installed on your host machine and used to interact with the VM. + +1. Install the CLI on your host machine (or build from source): + +```bash +brew install onkernel/tap/hypeman +``` + +2. Set up the API key on your host machine: + +```bash +# in the vm, generate a token: +make gen-jwt + +# on the host machine +export HYPEMAN_API_KEY="" +export HYPEMAN_BASE_URL="http://hypemman.local:8080" + +# then try out some commands +hypeman pull debian:13-slim +hypeman run --name test-debian debian:13-slim +hypeman exec test-debian -- uname -a +``` diff --git a/scripts/utm/bootstrap-dev-environment.sh b/scripts/utm/bootstrap-dev-environment.sh new file mode 100755 index 0000000..b426288 --- /dev/null +++ b/scripts/utm/bootstrap-dev-environment.sh @@ -0,0 +1,137 @@ +#!/bin/bash +# +# Provision the VM with Hypeman dependencies +# +# Run this script ON the VM (after SSHing in), or from your Mac: +# ssh hypeman 'bash -s' < bootstrap-dev-environment.sh +# + +set -e + +echo "=== Provisioning Hypeman Development Environment ===" +echo "" + +# Update packages +echo "[1/10] Updating packages..." +sudo apt update +sudo apt upgrade -y + +# Install dependencies +echo "[2/10] Installing dependencies..." +sudo apt install -y \ + build-essential \ + git \ + curl \ + wget \ + make \ + erofs-utils \ + dnsmasq \ + libcap2-bin \ + cpu-checker + +# Install GitHub CLI +echo "[3/10] Installing GitHub CLI..." +if command -v gh &>/dev/null; then + echo "GitHub CLI already installed: $(gh --version | head -1)" +else + sudo mkdir -p -m 755 /etc/apt/keyrings + out=$(mktemp) + wget -nv -O"$out" https://cli.github.com/packages/githubcli-archive-keyring.gpg + cat "$out" | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null + sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg + sudo mkdir -p -m 755 /etc/apt/sources.list.d + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null + sudo apt update + sudo apt install gh -y + rm -f "$out" + echo "GitHub CLI installed: $(gh --version | head -1)" +fi + +# Install Go +echo "[4/10] Installing Go..." +if go version 2>/dev/null | grep -q "go1.2"; then + echo "Go already installed: $(go version)" +else + GO_VERSION="1.25.4" + curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-arm64.tar.gz" | sudo tar -C /usr/local -xzf - + echo 'export PATH=$PATH:/usr/local/go/bin' >> ~/.bashrc + export PATH=$PATH:/usr/local/go/bin + echo "Go ${GO_VERSION} installed" +fi + +# Configure passwordless sudo +echo "[5/10] Configuring passwordless sudo..." +SUDOERS_FILE="/etc/sudoers.d/$(whoami)-nopasswd" +if [ -f "$SUDOERS_FILE" ]; then + echo "Passwordless sudo already configured" +else + echo "$(whoami) ALL=(ALL) NOPASSWD:ALL" | sudo tee "$SUDOERS_FILE" > /dev/null + sudo chmod 440 "$SUDOERS_FILE" + echo "Passwordless sudo configured for $(whoami)" +fi + +# Enable IP forwarding +echo "[6/10] Enabling IP forwarding..." +sudo sysctl -w net.ipv4.ip_forward=1 +grep -q 'net.ipv4.ip_forward=1' /etc/sysctl.conf || echo 'net.ipv4.ip_forward=1' | sudo tee -a /etc/sysctl.conf + +# Configure KVM access +echo "[7/10] Configuring KVM..." +sudo chmod 666 /dev/kvm 2>/dev/null || true +sudo usermod -aG kvm "$(whoami)" 2>/dev/null || true + +# Create Hypeman data directory +echo "[8/10] Creating Hypeman data directory..." +sudo mkdir -p /var/lib/hypeman +sudo chown "$(whoami):$(whoami)" /var/lib/hypeman +echo "Created /var/lib/hypeman (owned by $(whoami))" + +# Verify KVM +echo "[9/10] Verifying KVM..." +if kvm-ok 2>&1 | grep -q "can be used"; then + echo "✓ KVM acceleration is available" +else + echo "⚠ KVM may not be available (nested virtualization requires M3+ and macOS 15+)" +fi + +# Clone onkernel repositories +echo "[10/10] Cloning onkernel repositories..." +mkdir -p ~/code + +REPOS=( + "onkernel/hypeman" + "onkernel/hypeman-ts" + "onkernel/hypeman-go" + "onkernel/hypeman-cli" + "onkernel/linux" +) + +# Check GitHub authentication status +if gh auth status 2>&1 | grep -q "You are not logged into any GitHub hosts"; then + echo " GitHub CLI not authenticated, logging in..." + gh auth login +fi + +for repo in "${REPOS[@]}"; do + repo_name="${repo#*/}" + target_dir=~/code/"$repo_name" + if [ -d "$target_dir" ]; then + echo " $repo_name already exists, skipping..." + else + echo " Cloning $repo..." + gh repo clone "$repo" "$target_dir" + fi +done + +echo "" +echo "=== Provisioning Complete ===" +echo "" +echo "Go version: $(go version 2>/dev/null || echo 'reload shell with: source ~/.bashrc')" +echo "GitHub CLI: $(gh --version 2>/dev/null | head -1 || echo 'installed')" +echo "" +echo "Repositories cloned to ~/code/:" +ls -1 ~/code/ 2>/dev/null | sed 's/^/ - /' +echo "" +echo "Next steps:" +echo " 1. cd ~/code/hypeman && make dev" +echo "" diff --git a/scripts/utm/download-iso.sh b/scripts/utm/download-iso.sh new file mode 100755 index 0000000..39df5a9 --- /dev/null +++ b/scripts/utm/download-iso.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# Download Ubuntu Server ARM64 ISO +# + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +IMAGES_DIR="${SCRIPT_DIR}/images" + +UBUNTU_VERSION="24.04.3" +UBUNTU_ISO_URL="https://cdimage.ubuntu.com/releases/24.04/release/ubuntu-${UBUNTU_VERSION}-live-server-arm64.iso" +UBUNTU_ISO="${IMAGES_DIR}/ubuntu-${UBUNTU_VERSION}-live-server-arm64.iso" + +mkdir -p "${IMAGES_DIR}" + +if [[ -f "${UBUNTU_ISO}" ]]; then + echo "ISO already exists: ${UBUNTU_ISO}" + echo "Size: $(du -h "${UBUNTU_ISO}" | cut -f1)" + exit 0 +fi + +echo "Downloading Ubuntu Server ${UBUNTU_VERSION} ARM64..." +echo "URL: ${UBUNTU_ISO_URL}" +echo "This will take a few minutes (~3GB)..." +echo "" + +curl -L -o "${UBUNTU_ISO}" --progress-bar "${UBUNTU_ISO_URL}" + +echo "" +echo "Download complete!" +echo "ISO: ${UBUNTU_ISO}" +echo "Size: $(du -h "${UBUNTU_ISO}" | cut -f1)" +echo "" +echo "Next: Create the VM in UTM following the instructions in README.md" + + +