Merge pull request #1633 from afbjorklund/armv7l

AkihiroSuda · web-flow · commit 1eb50f9c972e · 2023-06-21T07:40:43.000+09:00
Add support for armv7l architecture
diff --git a/Makefile b/Makefile
@@ -49,6 +49,7 @@ binaries: clean \
 	_output/bin/kubectl.lima \
 	_output/share/lima/lima-guestagent.Linux-x86_64 \
 	_output/share/lima/lima-guestagent.Linux-aarch64 \
+	_output/share/lima/lima-guestagent.Linux-armv7l \
 	_output/share/lima/lima-guestagent.Linux-riscv64
 	cp -aL examples _output/share/lima
 	mkdir -p _output/share/doc/lima
@@ -108,6 +109,11 @@ _output/share/lima/lima-guestagent.Linux-aarch64:
 	GOOS=linux GOARCH=arm64 CGO_ENABLED=0 $(GO_BUILD) -o $@ ./cmd/lima-guestagent
 	chmod 644 $@
 
+.PHONY: _output/share/lima/lima-guestagent.Linux-armv7l
+_output/share/lima/lima-guestagent.Linux-armv7l:
+	GOOS=linux GOARCH=arm GOARM=7 CGO_ENABLED=0 $(GO_BUILD) -o $@ ./cmd/lima-guestagent
+	chmod 644 $@
+
 .PHONY: _output/share/lima/lima-guestagent.Linux-riscv64
 _output/share/lima/lima-guestagent.Linux-riscv64:
 	GOOS=linux GOARCH=riscv64 CGO_ENABLED=0 $(GO_BUILD) -o $@ ./cmd/lima-guestagent
diff --git a/docs/experimental.md b/docs/experimental.md
@@ -9,6 +9,7 @@ The following features are experimental and subject to change:
 - `video.display: vnc` and relevant configuration (`video.vnc.display`)
 - `mode: user-v2` in `networks.yml` and relevant configuration in `lima.yaml`
 - `audio.device`
+- `arch: armv7l`
 
 The following commands are experimental and subject to change:
 
diff --git a/docs/internal.md b/docs/internal.md
@@ -116,6 +116,9 @@ The directory contains the following files:
 - `$QEMU_SYSTEM_AARCH64`: path of `qemu-system-aarch64`
   - Default: `qemu-system-aarch64` in `$PATH`
 
+- `$QEMU_SYSTEM_ARM`: path of `qemu-system-arm`
+  - Default: `qemu-system-arm` in `$PATH`
+
 ## `cidata.iso`
 `cidata.iso` contains the following files:
 
diff --git a/examples/README.md b/examples/README.md
@@ -46,6 +46,7 @@ Optional feature enablers:
 - [`vmnet.yaml`](./vmnet.yaml): ⭐enable [`vmnet.framework`](../docs/network.md)
 - [`experimental/9p.yaml`](experimental/9p.yaml): [experimental] use 9p mount type
 - [`experimental/virtiofs-linux.yaml`](experimental/9p.yaml): [experimental] use virtiofs mount type for Linux
+- [`experimental/armv7l.yaml`](experimental/armv7l.yaml): [experimental] ARMv7
 - [`experimental/riscv64.yaml`](experimental/riscv64.yaml): [experimental] RISC-V
 - [`experimental/net-user-v2.yaml`](experimental/net-user-v2.yaml): [experimental] user-v2 network
   to enable VM-to-VM communication without root privilege
diff --git a/examples/default.yaml b/examples/default.yaml
@@ -237,6 +237,8 @@ containerd:
 cpuType:
   # 🟢 Builtin default: "cortex-a72" (or "host" when running on aarch64 host)
   aarch64: null
+  # 🟢 Builtin default: "cortex-a7" (or "host" when running on armv7l host)
+  armv7l: null
   # 🟢 Builtin default: "qemu64" (or "host,-pdpe1gb" when running on x86_64 host)
   x86_64: null
 
diff --git a/examples/experimental/armv7l.yaml b/examples/experimental/armv7l.yaml
@@ -0,0 +1,22 @@
+# This example requires Lima v0.7.0 or later.
+
+arch: "armv7l"
+images:
+# Try to use release-yyyyMMdd image if available. Note that release-yyyyMMdd will be removed after several months.
+- location: "https://cloud-images.ubuntu.com/releases/22.04/release-20230518/ubuntu-22.04-server-cloudimg-armhf.img"
+  arch: "armv7l"
+  digest: "sha256:f9baf349e9f4bb4acc6cbd14dc82aa2aa514fd76e06c74678e5453f0e6717fd2"
+# Fallback to the latest release image.
+# Hint: run `limactl prune` to invalidate the cache
+- location: "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-armhf.img"
+  arch: "armv7l"
+
+mounts:
+- location: "~"
+- location: "/tmp/lima"
+  writable: true
+
+# We do not have arm-v7 binaries of containerd
+containerd:
+  system: false
+  user: false
diff --git a/pkg/limayaml/defaults.go b/pkg/limayaml/defaults.go
@@ -21,6 +21,8 @@ import (
 	"github.com/lima-vm/lima/pkg/store/filenames"
 	"github.com/sirupsen/logrus"
 	"github.com/xorcare/pointer"
+
+	"golang.org/x/sys/cpu"
 )
 
 const (
@@ -51,6 +53,7 @@ func defaultContainerdArchives() []File {
 			Arch:     AARCH64,
 			Digest:   "sha256:589dabd962d936b29fd377dcddbb49c07d1c4c27dd4b402bc4b6b20287fe9c37",
 		},
+		// No arm-v7
 		// No riscv64
 	}
 }
@@ -151,6 +154,7 @@ func FillDefault(y, d, o *LimaYAML, filePath string) {
 
 	cpuType := map[Arch]string{
 		AARCH64: "cortex-a72",
+		ARMV7L:  "cortex-a7",
 		// Since https://github.com/lima-vm/lima/pull/494, we use qemu64 cpu for better emulation of x86_64.
 		X8664:   "qemu64",
 		RISCV64: "rv64", // FIXME: what is the right choice for riscv64?
@@ -749,12 +753,35 @@ func FillCopyToHostDefaults(rule *CopyToHost, instDir string) {
 	}
 }
 
+func goarm() int {
+	if runtime.GOOS != "linux" {
+		return 0
+	}
+	if runtime.GOARCH != "arm" {
+		return 0
+	}
+	if cpu.ARM.HasVFPv3 {
+		return 7
+	}
+	if cpu.ARM.HasVFP {
+		return 6
+	}
+	return 5 // default
+}
+
 func NewArch(arch string) Arch {
 	switch arch {
 	case "amd64":
 		return X8664
 	case "arm64":
 		return AARCH64
+	case "arm":
+		arm := goarm()
+		if arm == 7 {
+			return ARMV7L
+		}
+		logrus.Warnf("Unknown arm: %d", arm)
+		return arch
 	case "riscv64":
 		return RISCV64
 	default:
@@ -818,8 +845,9 @@ func HasMaxCPU() bool {
 func IsNativeArch(arch Arch) bool {
 	nativeX8664 := arch == X8664 && runtime.GOARCH == "amd64"
 	nativeAARCH64 := arch == AARCH64 && runtime.GOARCH == "arm64"
+	nativeARMV7L := arch == ARMV7L && runtime.GOARCH == "arm" && goarm() == 7
 	nativeRISCV64 := arch == RISCV64 && runtime.GOARCH == "riscv64"
-	return nativeX8664 || nativeAARCH64 || nativeRISCV64
+	return nativeX8664 || nativeAARCH64 || nativeARMV7L || nativeRISCV64
 }
 
 func unique(s []string) []string {
diff --git a/pkg/limayaml/defaults_test.go b/pkg/limayaml/defaults_test.go
@@ -32,6 +32,14 @@ func TestFillDefault(t *testing.T) {
 		arch = X8664
 	case "arm64":
 		arch = AARCH64
+	case "arm":
+		if runtime.GOOS != "linux" {
+			t.Skipf("unsupported GOOS: %s", runtime.GOOS)
+		}
+		if arm := goarm(); arm < 7 {
+			t.Skipf("unsupported GOARM: %d", arm)
+		}
+		arch = ARMV7L
 	case "riscv64":
 		arch = RISCV64
 	default:
@@ -56,6 +64,7 @@ func TestFillDefault(t *testing.T) {
 		Arch:   pointer.String(arch),
 		CPUType: map[Arch]string{
 			AARCH64: "cortex-a72",
+			ARMV7L:  "cortex-a7",
 			X8664:   "qemu64",
 			RISCV64: "rv64",
 		},
@@ -259,6 +268,7 @@ func TestFillDefault(t *testing.T) {
 		Arch:   pointer.String("unknown"),
 		CPUType: map[Arch]string{
 			AARCH64: "arm64",
+			ARMV7L:  "armhf",
 			X8664:   "amd64",
 			RISCV64: "riscv64",
 		},
@@ -434,6 +444,7 @@ func TestFillDefault(t *testing.T) {
 		Arch:   pointer.String(arch),
 		CPUType: map[Arch]string{
 			AARCH64: "uber-arm",
+			ARMV7L:  "armv8",
 			X8664:   "pentium",
 			RISCV64: "sifive-u54",
 		},
diff --git a/pkg/limayaml/limayaml.go b/pkg/limayaml/limayaml.go
@@ -45,6 +45,7 @@ type VMType = string
 const (
 	X8664   Arch = "x86_64"
 	AARCH64 Arch = "aarch64"
+	ARMV7L  Arch = "armv7l"
 	RISCV64 Arch = "riscv64"
 
 	REVSSHFS MountType = "reverse-sshfs"
diff --git a/pkg/limayaml/validate.go b/pkg/limayaml/validate.go
@@ -26,9 +26,9 @@ func validateFileObject(f File, fieldName string) error {
 		// f.Location does NOT need to be accessible, so we do NOT check os.Stat(f.Location)
 	}
 	switch f.Arch {
-	case X8664, AARCH64, RISCV64:
+	case X8664, AARCH64, ARMV7L, RISCV64:
 	default:
-		return fmt.Errorf("field `arch` must be %q, %q, or %q; got %q", X8664, AARCH64, RISCV64, f.Arch)
+		return fmt.Errorf("field `arch` must be %q, %q, %q, or %q; got %q", X8664, AARCH64, ARMV7L, RISCV64, f.Arch)
 	}
 	if f.Digest != "" {
 		if !f.Digest.Algorithm().Available() {
@@ -43,9 +43,9 @@ func validateFileObject(f File, fieldName string) error {
 
 func Validate(y LimaYAML, warn bool) error {
 	switch *y.Arch {
-	case X8664, AARCH64, RISCV64:
+	case X8664, AARCH64, ARMV7L, RISCV64:
 	default:
-		return fmt.Errorf("field `arch` must be %q, %q, or %q; got %q", X8664, AARCH64, RISCV64, *y.Arch)
+		return fmt.Errorf("field `arch` must be %q, %q, %q or %q; got %q", X8664, AARCH64, ARMV7L, RISCV64, *y.Arch)
 	}
 
 	switch *y.VMType {
@@ -91,7 +91,7 @@ func Validate(y LimaYAML, warn bool) error {
 
 	for arch := range y.CPUType {
 		switch arch {
-		case AARCH64, X8664, RISCV64:
+		case AARCH64, X8664, ARMV7L, RISCV64:
 			// these are the only supported architectures
 		default:
 			return fmt.Errorf("field `cpuType` uses unsupported arch %q", arch)
diff --git a/pkg/qemu/qemu.go b/pkg/qemu/qemu.go
@@ -290,12 +290,16 @@ type features struct {
 	// e.g. "Supported machines are:\nakita...\n...virt-6.2...\n...virt-7.0...\n...\n"
 	// Not machine-readable, but checking strings.Contains() should be fine.
 	MachineHelp []byte
+	// CPUHelp is the output of `qemu-system-x86_64 -cpu help`
+	// e.g. "Available CPUs:\n...\nx86 base...\nx86 host...\n...\n"
+	// Not machine-readable, but checking strings.Contains() should be fine.
+	CPUHelp []byte
 
 	// VersionGEQ7 is true when the QEMU version seems v7.0.0 or later
 	VersionGEQ7 bool
 }
 
-func inspectFeatures(exe string) (*features, error) {
+func inspectFeatures(exe string, machine string) (*features, error) {
 	var (
 		f      features
 		stdout bytes.Buffer
@@ -338,6 +342,19 @@ func inspectFeatures(exe string) (*features, error) {
 	}
 	f.VersionGEQ7 = strings.Contains(string(f.MachineHelp), "-7.0")
 
+	// Avoid error: "No machine specified, and there is no default"
+	cmd = exec.Command(exe, "-cpu", "help", "-machine", machine)
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		logrus.Warnf("failed to run %v: stdout=%q, stderr=%q", cmd.Args, stdout.String(), stderr.String())
+	} else {
+		f.CPUHelp = stdout.Bytes()
+		if len(f.CPUHelp) == 0 {
+			f.CPUHelp = stderr.Bytes()
+		}
+	}
+
 	return &f, nil
 }
 
@@ -442,14 +459,22 @@ func adjustMemBytesDarwinARM64HVF(memBytes int64, accel string, features *featur
 	return memBytes
 }
 
+// qemuMachine returns string to use for -machine
+func qemuMachine(arch limayaml.Arch) string {
+	if arch == limayaml.X8664 {
+		return "q35"
+	}
+	return "virt"
+}
+
 func Cmdline(cfg Config) (string, []string, error) {
 	y := cfg.LimaYAML
 	exe, args, err := getExe(*y.Arch)
 	if err != nil {
 		return "", nil, err
 	}
 
-	features, err := inspectFeatures(exe)
+	features, err := inspectFeatures(exe, qemuMachine(*y.Arch))
 	if err != nil {
 		return "", nil, err
 	}
@@ -495,8 +520,12 @@ func Cmdline(cfg Config) (string, []string, error) {
 			}
 		}
 	}
+	if !strings.Contains(string(features.CPUHelp), strings.Split(cpu, ",")[0]) {
+		return "", nil, fmt.Errorf("cpu %q is not supported by %s", cpu, exe)
+	}
 	args = appendArgsIfNoConflict(args, "-cpu", cpu)
 
+	// Machine
 	switch *y.Arch {
 	case limayaml.X8664:
 		if strings.HasPrefix(cpu, "qemu64") && runtime.GOOS != "windows" {
@@ -530,6 +559,9 @@ func Cmdline(cfg Config) (string, []string, error) {
 	case limayaml.RISCV64:
 		machine := "virt,accel=" + accel
 		args = appendArgsIfNoConflict(args, "-machine", machine)
+	case limayaml.ARMV7L:
+		machine := "virt,accel=" + accel
+		args = appendArgsIfNoConflict(args, "-machine", machine)
 	}
 
 	// SMP
@@ -538,7 +570,7 @@ func Cmdline(cfg Config) (string, []string, error) {
 
 	// Firmware
 	legacyBIOS := *y.Firmware.LegacyBIOS
-	if legacyBIOS && *y.Arch != limayaml.X8664 {
+	if legacyBIOS && *y.Arch != limayaml.X8664 && *y.Arch != limayaml.ARMV7L {
 		logrus.Warnf("field `firmware.legacyBIOS` is not supported for architecture %q, ignoring", *y.Arch)
 		legacyBIOS = false
 	}
@@ -756,12 +788,18 @@ func Cmdline(cfg Config) (string, []string, error) {
 		args = append(args, "-device", "virtio-keyboard-pci")
 		args = append(args, "-device", "virtio-mouse-pci")
 		args = append(args, "-device", "qemu-xhci,id=usb-bus")
-	default:
+	case limayaml.AARCH64:
 		// QEMU does not seem to support virtio-vga for aarch64
 		args = append(args, "-vga", "none", "-device", "ramfb")
 		args = append(args, "-device", "qemu-xhci,id=usb-bus")
 		args = append(args, "-device", "usb-kbd,bus=usb-bus.0")
 		args = append(args, "-device", "usb-mouse,bus=usb-bus.0")
+	case limayaml.ARMV7L:
+		// QEMU does not seem to support virtio-vga for arm
+		args = append(args, "-vga", "cirrus", "-device", "cirrus-vga")
+		args = append(args, "-device", "qemu-xhci,id=usb-bus")
+		args = append(args, "-device", "usb-kbd,bus=usb-bus.0")
+		args = append(args, "-device", "usb-mouse,bus=usb-bus.0")
 	}
 
 	// Parallel
@@ -928,10 +966,18 @@ func VirtiofsdCmdline(cfg Config, mountIndex int) ([]string, error) {
 	}, nil
 }
 
+// qemuArch returns the arch string used by qemu
+func qemuArch(arch limayaml.Arch) string {
+	if arch == limayaml.ARMV7L {
+		return "arm"
+	}
+	return arch
+}
+
 func getExe(arch limayaml.Arch) (string, []string, error) {
-	exeBase := "qemu-system-" + arch
+	exeBase := "qemu-system-" + qemuArch(arch)
 	var args []string
-	envK := "QEMU_SYSTEM_" + strings.ToUpper(arch)
+	envK := "QEMU_SYSTEM_" + strings.ToUpper(qemuArch(arch))
 	if envV := os.Getenv(envK); envV != "" {
 		ss, err := shellwords.Parse(envV)
 		if err != nil {
@@ -992,7 +1038,7 @@ func getQemuVersion(qemuExe string) (*semver.Version, error) {
 
 func getFirmware(qemuExe string, arch limayaml.Arch) (string, error) {
 	switch arch {
-	case limayaml.X8664, limayaml.AARCH64:
+	case limayaml.X8664, limayaml.AARCH64, limayaml.ARMV7L:
 	default:
 		return "", fmt.Errorf("unexpected architecture: %q", arch)
 	}
@@ -1006,7 +1052,7 @@ func getFirmware(qemuExe string, arch limayaml.Arch) (string, error) {
 	localDir := filepath.Dir(binDir)                             // "/usr/local"
 	userLocalDir := filepath.Join(currentUser.HomeDir, ".local") // "$HOME/.local"
 
-	relativePath := fmt.Sprintf("share/qemu/edk2-%s-code.fd", arch)
+	relativePath := fmt.Sprintf("share/qemu/edk2-%s-code.fd", qemuArch(arch))
 	candidates := []string{
 		filepath.Join(userLocalDir, relativePath), // XDG-like
 		filepath.Join(localDir, relativePath),     // macOS (homebrew)
@@ -1025,6 +1071,9 @@ func getFirmware(qemuExe string, arch limayaml.Arch) (string, error) {
 		candidates = append(candidates, "/usr/share/AAVMF/AAVMF_CODE.fd")
 		// Debian package "qemu-efi-aarch64" (unpadded, backwards compatibility)
 		candidates = append(candidates, "/usr/share/qemu-efi-aarch64/QEMU_EFI.fd")
+	case limayaml.ARMV7L:
+		// Debian package "qemu-efi-arm"
+		candidates = append(candidates, "/usr/share/AAVMF/AAVMF32_CODE.fd")
 	}
 
 	logrus.Debugf("firmware candidates = %v", candidates)
