Skip to content

Commit dac68f1

Browse files
sjmiller609sjmiller609
committed
Automatically install headers
1 parent fdc6d6a commit dac68f1

File tree

5 files changed

+241
-50
lines changed

5 files changed

+241
-50
lines changed

‎lib/system/README.md‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,16 @@ via `INIT_MODE` in the config disk.
7979

8080
**Result:** OCI images require **zero modifications** - no `/init` script needed!
8181

82+
## Kernel Headers
83+
84+
Kernel headers are bundled in the initrd and automatically installed at boot, enabling DKMS to build out-of-tree kernel modules (e.g., NVIDIA vGPU drivers).
85+
86+
**Why:** Guest images come with headers for their native kernel (e.g., Ubuntu's 5.15), but hypeman VMs run a custom kernel. Without matching headers, DKMS cannot compile drivers.
87+
88+
**How:** The initrd includes `kernel-headers.tar.gz` from the same release as the kernel. At boot, init extracts headers to `/usr/src/linux-headers-{version}/`, creates the `/lib/modules/{version}/build` symlink, and removes mismatched headers from the guest image.
89+
90+
**Result:** Guests can `apt install nvidia-driver-xxx` and DKMS builds modules for the running kernel automatically.
91+
8292
## Kernel Sources
8393

8494
Kernels downloaded from kernel/linux releases (Cloud Hypervisor-optimized fork):
@@ -94,7 +104,7 @@ Example URLs:
94104
2. **Add guest-agent binary** (embedded, runs in guest for exec/shell)
95105
3. **Add init.sh wrapper** (mounts /proc, /sys, /dev before Go runtime)
96106
4. **Add init binary** (embedded Go binary, runs as PID 1)
97-
5. **Add NVIDIA modules** (optional, for GPU passthrough)
107+
5. **Add kernel headers tarball** (downloaded from release, for DKMS)
98108
6. **Package as cpio** (initramfs format, pure Go - no shell tools required)
99109

100110
## Adding New Versions
@@ -148,7 +158,7 @@ go test ./lib/system/...
148158
| File | Size | Purpose |
149159
|------|------|---------|
150160
| kernel/*/vmlinux | ~70MB | Cloud Hypervisor optimized kernel |
151-
| initrd/*/initrd | ~5-10MB | Alpine base + Go init binary + guest-agent |
161+
| initrd/*/initrd | ~20MB | Alpine base + init binary + guest-agent + kernel headers |
152162

153163
Files downloaded/built once per version, reused for all instances using that version.
154164

@@ -161,7 +171,7 @@ lib/system/init/
161171
mount.go # Mount operations (overlay, bind mounts)
162172
config.go # Parse config disk
163173
network.go # Network configuration
164-
drivers.go # GPU driver loading
174+
headers.go # Kernel headers setup for DKMS
165175
volumes.go # Volume mounting
166176
mode_exec.go # Exec mode: chroot, run entrypoint, wait on guest-agent
167177
mode_systemd.go # Systemd mode: chroot + exec /sbin/init

‎lib/system/init/headers.go‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"os/exec"
7+
"path/filepath"
8+
"strings"
9+
"syscall"
10+
)
11+
12+
const (
13+
// Paths in the overlay filesystem
14+
newrootLibModules = "/overlay/newroot/lib/modules"
15+
newrootUsrSrc = "/overlay/newroot/usr/src"
16+
headersTarball = "/kernel-headers.tar.gz"
17+
)
18+
19+
// setupKernelHeaders installs kernel headers and cleans up mismatched headers from the guest image.
20+
// This enables DKMS to build out-of-tree kernel modules (e.g., NVIDIA vGPU drivers).
21+
func setupKernelHeaders(log *Logger) error {
22+
// Get running kernel version
23+
var uname syscall.Utsname
24+
if err := syscall.Uname(&uname); err != nil {
25+
return fmt.Errorf("uname: %w", err)
26+
}
27+
runningKernel := int8ArrayToString(uname.Release[:])
28+
log.Info("headers", "running kernel: "+runningKernel)
29+
30+
// Check if headers tarball exists in initrd
31+
if _, err := os.Stat(headersTarball); os.IsNotExist(err) {
32+
log.Info("headers", "no kernel headers tarball found, skipping")
33+
return nil
34+
}
35+
36+
// Clean up mismatched kernel modules directories
37+
if err := cleanupMismatchedModules(log, runningKernel); err != nil {
38+
log.Info("headers", "warning: failed to cleanup mismatched modules: "+err.Error())
39+
// Non-fatal, continue
40+
}
41+
42+
// Clean up mismatched kernel headers directories
43+
if err := cleanupMismatchedHeaders(log, runningKernel); err != nil {
44+
log.Info("headers", "warning: failed to cleanup mismatched headers: "+err.Error())
45+
// Non-fatal, continue
46+
}
47+
48+
// Create target directories
49+
headersDir := filepath.Join(newrootUsrSrc, "linux-headers-"+runningKernel)
50+
modulesDir := filepath.Join(newrootLibModules, runningKernel)
51+
52+
if err := os.MkdirAll(headersDir, 0755); err != nil {
53+
return fmt.Errorf("mkdir headers dir: %w", err)
54+
}
55+
if err := os.MkdirAll(modulesDir, 0755); err != nil {
56+
return fmt.Errorf("mkdir modules dir: %w", err)
57+
}
58+
59+
// Extract headers tarball
60+
if err := extractTarGz(headersTarball, headersDir); err != nil {
61+
return fmt.Errorf("extract headers: %w", err)
62+
}
63+
log.Info("headers", "extracted kernel headers to "+headersDir)
64+
65+
// Create build symlink
66+
buildLink := filepath.Join(modulesDir, "build")
67+
os.Remove(buildLink) // Remove if exists
68+
// Use absolute path for symlink target (will be correct after chroot)
69+
symlinkTarget := "/usr/src/linux-headers-" + runningKernel
70+
if err := os.Symlink(symlinkTarget, buildLink); err != nil {
71+
return fmt.Errorf("create build symlink: %w", err)
72+
}
73+
log.Info("headers", "created build symlink")
74+
75+
return nil
76+
}
77+
78+
// cleanupMismatchedModules removes /lib/modules/* directories that don't match the running kernel
79+
func cleanupMismatchedModules(log *Logger, runningKernel string) error {
80+
entries, err := os.ReadDir(newrootLibModules)
81+
if err != nil {
82+
if os.IsNotExist(err) {
83+
return nil // No modules directory, nothing to clean
84+
}
85+
return err
86+
}
87+
88+
for _, entry := range entries {
89+
if !entry.IsDir() {
90+
continue
91+
}
92+
if entry.Name() != runningKernel {
93+
path := filepath.Join(newrootLibModules, entry.Name())
94+
log.Info("headers", "removing mismatched modules: "+entry.Name())
95+
if err := os.RemoveAll(path); err != nil {
96+
return fmt.Errorf("remove %s: %w", path, err)
97+
}
98+
}
99+
}
100+
101+
return nil
102+
}
103+
104+
// cleanupMismatchedHeaders removes /usr/src/linux-headers-* directories that don't match the running kernel
105+
func cleanupMismatchedHeaders(log *Logger, runningKernel string) error {
106+
entries, err := os.ReadDir(newrootUsrSrc)
107+
if err != nil {
108+
if os.IsNotExist(err) {
109+
return nil // No usr/src directory, nothing to clean
110+
}
111+
return err
112+
}
113+
114+
expectedName := "linux-headers-" + runningKernel
115+
116+
for _, entry := range entries {
117+
if !entry.IsDir() {
118+
continue
119+
}
120+
// Remove any linux-headers-* directory that doesn't match
121+
if strings.HasPrefix(entry.Name(), "linux-headers-") && entry.Name() != expectedName {
122+
path := filepath.Join(newrootUsrSrc, entry.Name())
123+
log.Info("headers", "removing mismatched headers: "+entry.Name())
124+
if err := os.RemoveAll(path); err != nil {
125+
return fmt.Errorf("remove %s: %w", path, err)
126+
}
127+
}
128+
}
129+
130+
return nil
131+
}
132+
133+
// extractTarGz extracts a .tar.gz file to a destination directory
134+
func extractTarGz(tarball, destDir string) error {
135+
// Use tar command since it's available in Alpine
136+
cmd := exec.Command("/bin/tar", "-xzf", tarball, "-C", destDir)
137+
if output, err := cmd.CombinedOutput(); err != nil {
138+
return fmt.Errorf("tar: %s: %s", err, output)
139+
}
140+
return nil
141+
}
142+
143+
// int8ArrayToString converts a null-terminated int8 array (from syscall) to a Go string
144+
func int8ArrayToString(arr []int8) string {
145+
var buf []byte
146+
for _, b := range arr {
147+
if b == 0 {
148+
break
149+
}
150+
buf = append(buf, byte(b))
151+
}
152+
return string(buf)
153+
}

‎lib/system/init/main.go‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,13 @@ func main() {
6666
// Continue anyway - exec will still work, just no remote access
6767
}
6868

69-
// Phase 8: Mode-specific execution
69+
// Phase 8: Setup kernel headers for DKMS
70+
if err := setupKernelHeaders(log); err != nil {
71+
log.Error("headers", "failed to setup kernel headers", err)
72+
// Continue anyway - only needed for DKMS module building
73+
}
74+
75+
// Phase 9: Mode-specific execution
7076
if cfg.InitMode == "systemd" {
7177
log.Info("mode", "entering systemd mode")
7278
runSystemdMode(log, cfg)

‎lib/system/initrd.go‎

Lines changed: 55 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import (
55
"crypto/sha256"
66
"encoding/hex"
77
"fmt"
8+
"io"
9+
"net/http"
810
"os"
911
"path/filepath"
1012
"strconv"
@@ -68,6 +70,11 @@ func (m *manager) buildInitrd(ctx context.Context, arch string) (string, error)
6870
return "", fmt.Errorf("write init binary: %w", err)
6971
}
7072

73+
// Download and add kernel headers tarball (for DKMS support)
74+
if err := downloadKernelHeaders(arch, rootfsDir); err != nil {
75+
return "", fmt.Errorf("download kernel headers: %w", err)
76+
}
77+
7178
// Generate timestamp for this build
7279
timestamp := strconv.FormatInt(time.Now().Unix(), 10)
7380

@@ -141,11 +148,57 @@ func (m *manager) isInitrdStale(initrdPath, arch string) bool {
141148
return string(storedHash) != currentHash
142149
}
143150

144-
// computeInitrdHash computes a hash of the embedded binaries
145-
func computeInitrdHash(_ string) string {
151+
// computeInitrdHash computes a hash of the embedded binaries and header URL
152+
func computeInitrdHash(arch string) string {
146153
h := sha256.New()
147154
h.Write(GuestAgentBinary)
148155
h.Write(InitBinary)
149156
h.Write(InitWrapper)
157+
// Include kernel header URL in hash so initrd rebuilds when headers change
158+
if url, ok := KernelHeaderURLs[DefaultKernelVersion][arch]; ok {
159+
h.Write([]byte(url))
160+
}
150161
return hex.EncodeToString(h.Sum(nil))[:16]
151162
}
163+
164+
// downloadKernelHeaders downloads kernel headers tarball and adds it to the initrd rootfs
165+
func downloadKernelHeaders(arch, rootfsDir string) error {
166+
url, ok := KernelHeaderURLs[DefaultKernelVersion][arch]
167+
if !ok {
168+
// No headers available for this arch, skip (non-fatal)
169+
return nil
170+
}
171+
172+
destPath := filepath.Join(rootfsDir, "kernel-headers.tar.gz")
173+
174+
// Download headers (GitHub releases return 302 redirects)
175+
client := &http.Client{
176+
CheckRedirect: func(req *http.Request, via []*http.Request) error {
177+
return nil // Follow redirects
178+
},
179+
}
180+
181+
resp, err := client.Get(url)
182+
if err != nil {
183+
return fmt.Errorf("http get: %w", err)
184+
}
185+
defer resp.Body.Close()
186+
187+
if resp.StatusCode != 200 {
188+
return fmt.Errorf("download failed with status %d from %s", resp.StatusCode, url)
189+
}
190+
191+
// Create output file
192+
outFile, err := os.Create(destPath)
193+
if err != nil {
194+
return fmt.Errorf("create file: %w", err)
195+
}
196+
defer outFile.Close()
197+
198+
// Copy content
199+
if _, err = io.Copy(outFile, resp.Body); err != nil {
200+
return fmt.Errorf("write file: %w", err)
201+
}
202+
203+
return nil
204+
}

‎lib/system/versions.go‎

Lines changed: 13 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -6,66 +6,35 @@ import "runtime"
66
type KernelVersion string
77

88
const (
9-
// Kernel versions from kernel/linux releases
10-
Kernel_202511182 KernelVersion = "ch-6.12.8-kernel-1-202511182"
11-
Kernel_20251211 KernelVersion = "ch-6.12.8-kernel-1.1-20251211"
12-
Kernel_20251213 KernelVersion = "ch-6.12.8-kernel-1.2-20251213" // NVIDIA module + driver lib support + networking configs
9+
// Kernel_202601152 is the current kernel version with vGPU support
10+
Kernel_202601152 KernelVersion = "ch-6.12.8-kernel-1.3-202601152"
1311
)
1412

1513
var (
1614
// DefaultKernelVersion is the kernel version used for new instances
17-
DefaultKernelVersion = Kernel_20251213
15+
DefaultKernelVersion = Kernel_202601152
1816

1917
// SupportedKernelVersions lists all supported kernel versions
2018
SupportedKernelVersions = []KernelVersion{
21-
Kernel_202511182,
22-
Kernel_20251211,
23-
Kernel_20251213,
24-
// Add future versions here
19+
Kernel_202601152,
2520
}
2621
)
2722

2823
// KernelDownloadURLs maps kernel versions and architectures to download URLs
2924
var KernelDownloadURLs = map[KernelVersion]map[string]string{
30-
Kernel_202511182: {
31-
"x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1-202511182/vmlinux-x86_64",
32-
"aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1-202511182/Image-arm64",
25+
Kernel_202601152: {
26+
"x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.3-202601152/vmlinux-x86_64",
27+
"aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.3-202601152/Image-arm64",
3328
},
34-
Kernel_20251211: {
35-
"x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.1-20251211/vmlinux-x86_64",
36-
"aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.1-20251211/Image-arm64",
37-
},
38-
Kernel_20251213: {
39-
"x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.2-20251213/vmlinux-x86_64",
40-
"aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.2-20251213/Image-arm64",
41-
},
42-
// Add future versions here
4329
}
4430

45-
// NvidiaModuleURLs maps kernel versions and architectures to NVIDIA module tarball URLs
46-
// These tarballs contain pre-built NVIDIA kernel modules that match the kernel version
47-
var NvidiaModuleURLs = map[KernelVersion]map[string]string{
48-
Kernel_20251213: {
49-
"x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.2-20251213/nvidia-modules-x86_64.tar.gz",
50-
// Note: NVIDIA open-gpu-kernel-modules does not support arm64 yet
31+
// KernelHeaderURLs maps kernel versions and architectures to kernel header tarball URLs
32+
// These tarballs contain kernel headers needed for DKMS to build out-of-tree modules (e.g., NVIDIA vGPU drivers)
33+
var KernelHeaderURLs = map[KernelVersion]map[string]string{
34+
Kernel_202601152: {
35+
"x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.3-202601152/kernel-headers-x86_64.tar.gz",
36+
"aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.3-202601152/kernel-headers-aarch64.tar.gz",
5137
},
52-
// Kernel_202511182 and Kernel_20251211 do not have NVIDIA modules (pre-module-support kernels)
53-
}
54-
55-
// NvidiaDriverLibURLs maps kernel versions and architectures to driver library tarball URLs
56-
// These tarballs contain userspace NVIDIA libraries (libcuda.so, libnvidia-ml.so, etc.)
57-
// that match the kernel modules and are injected into containers at boot time.
58-
// See lib/devices/GPU.md for documentation on driver injection.
59-
var NvidiaDriverLibURLs = map[KernelVersion]map[string]string{
60-
Kernel_20251213: {
61-
"x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.2-20251213/nvidia-driver-libs-x86_64.tar.gz",
62-
},
63-
}
64-
65-
// NvidiaDriverVersion tracks the NVIDIA driver version bundled with each kernel
66-
var NvidiaDriverVersion = map[KernelVersion]string{
67-
Kernel_20251213: "570.86.16",
68-
// Kernel_202511182 and Kernel_20251211 do not have NVIDIA modules
6938
}
7039

7140
// GetArch returns the architecture string for the current platform

0 commit comments

Comments
 (0)