WIP: android userspace

Author: Gulshan Singh

Makefile

@@ -8,7 +8,7 @@ SCRIPT_DIR := $(ROOT_DIR)/scripts
 CONFIG_DIR := $(ROOT_DIR)/config
 CLANG_DIR ?= $(ROOT_DIR)/toolchain/clang
 
-CLANG_URL := https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.6/clang+llvm-15.0.6-x86_64-linux-gnu-ubuntu-18.04.tar.xz
+CLANG_URL := https://github.com/llvm/llvm-project/releases/download/llvmorg-17.0.6/clang+llvm-17.0.6-x86_64-linux-gnu-ubuntu-22.04.tar.xz
 
 SHARED_DIR := $(ROOT_DIR)/shared
 
@@ -41,7 +41,7 @@ endif
 
 
 .PHONY: default
-default: linux tools-vm
+default: linux linux_modules tools-vm
 
 .PHONY: clean
 clean: linux_clean tools-vm_clean
@@ -151,6 +151,8 @@ else
   LINUX_DEFCONFIG ?= defconfig
 endif
 
+TRUSTY_LINUX_CONFIG_FRAGMENT := $(ROOT_DIR)/trusty/external/linux/arch/arm64/configs/trusty_qemu_defconfig.fragment
+
 LINUX_CONFIG_FRAGMENT ?= $(CONFIG_DIR)/config.fragment
 LINUX_OUT_MODULES_DEP := $(LINUX_OUT)/modules_install.stamp
 LINUX_MODULES_INSTALL_PATH := $(LINUX_OUT)/modules_install
@@ -448,17 +450,43 @@ trusty $(ATF_BL1): | $(TRUSTY_SRC)
 	$(BEAR_CMD) $(TRUSTY_SRC)/trusty/vendor/google/aosp/scripts/build.py \
 		--build-root $(TRUSTY_BUILD_ROOT) \
 		--skip-tests $(TRUSTY_TARGET)
-	$(MAKE) trusty_bl33
+	$(MAKE) trusty-bl33
 
-.PHONY: trusty_bl33
-trusty_bl33 $(ATF_BL33): $(UBOOT_BIN)
+.PHONY: trusty-bl33
+trusty-bl33 $(ATF_BL33): $(UBOOT_BIN)
 	rm $(ATF_BL33)
 	cp $(UBOOT_BIN) $(ATF_BL33)
 
 .PHONY: trusty_clean
 trusty_clean:
 	rm -rf $(TRUSTY_OUT)
 
+##
+## Android
+##
+
+# TODO: We shouldn't depend on Trusty for this
+TRUSTY_PREBUILT_IMAGE_DIR := $(TRUSTY_SRC)/trusty/prebuilts/aosp/android/out/target/product/trusty/
+SYSTEM_IMG := $(TRUSTY_PREBUILT_IMAGE_DIR)/system.img
+VENDOR_IMG := $(TRUSTY_PREBUILT_IMAGE_DIR)/vendor.img
+USERDATA_IMG := $(TRUSTY_PREBUILT_IMAGE_DIR)/userdata.img
+
+ANDROID_USERSPACE ?= 0
+
+DTC := dtc
+
+ANDROID_DTS := $(OUT_DIR)/android.dts
+ANDROID_DTB := $(OUT_DIR)/android.dtb
+QEMU_DTB := $(OUT_DIR)/qemu.dtb
+QEMU_DTS := $(OUT_DIR)/qemu.dts
+
+.PHONY: android_dtb
+android-dtb $(ANDROID_DTB):
+	QEMU_EXTRA_ARGS="-M dumpdtb=$(QEMU_DTB)" $(MAKE) run
+	$(DTC) -I dtb -O dts $(QEMU_DTB) > $(QEMU_DTS)
+	cat $(QEMU_DTS) $(ATF_DIR)/firmware.android.dts > $(ANDROID_DTS)
+	$(DTC) -I dts -O dtb $(ANDROID_DTS) > $(ANDROID_DTB)
+
 ##
 ## Run QEMU
 ##
@@ -479,7 +507,13 @@ KASLR ?= 0
 UBOOT ?= 0
 TRUSTY ?= 0
 
-QEMU_KERNEL_CMDLINE := selinux=0
+ifeq ($(ANDROID_USERSPACE),1)
+  SELINUX ?= 1
+else
+  SELINUX ?= 0
+endif
+
+QEMU_KERNEL_CMDLINE := selinux=$(SELINUX)
 
 QEMU_ARGS := \
   -m $(MEM) \
@@ -488,17 +522,30 @@ QEMU_ARGS := \
   -no-reboot \
   -kernel $(QEMU_KERNEL_IMAGE) \
   -echr $(ECHR) \
-  -netdev user,id=eth0,hostfwd=tcp::7777-:7777,hostfwd=tcp::2222-:22,hostfwd=tcp::2223-:23 -device virtio-net-pci,netdev=eth0 \
-  -virtfs local,security_model=mapped-xattr,path=$(SHARED_DIR),mount_tag=shared \
   $(QEMU_EXTRA_ARGS)
 
+ifneq ($(TRUSTY),1)
+  QEMU_ARGS += -netdev user,id=eth0,hostfwd=tcp::7777-:7777,hostfwd=tcp::2222-:22,hostfwd=tcp::2223-:23 -device virtio-net-pci,netdev=eth0
+  QEMU_ARGS += -virtfs local,security_model=mapped-xattr,path=$(SHARED_DIR),mount_tag=shared
+  #QEMU_ARGS += -virtfs local,security_model=mapped-xattr,path=$(LINUX_MODULES_INSTALL_PATH)/lib/modules,mount_tag=modules
+endif
+
 ifeq ($(TRUSTY),1)
   QEMU_ARGS += -bios $(ATF_BL1)
 else ifeq ($(UBOOT),1)
   QEMU_ARGS += -bios $(UBOOT_BIN)
 endif
 
-ifneq ($(INITRD),)
+# TODO: check that ROOTFS and ANDROID_USERSPACE aren't both passed at the same time?
+ifeq ($(ANDROID_USERSPACE),1)
+  QEMU_ARGS += \
+    -device virtio-blk,drive=vda -drive file=$(SYSTEM_IMG),index=0,if=none,id=vda,format=raw \
+    -device virtio-blk,drive=vdb -drive file=$(VENDOR_IMG),index=1,if=none,id=vdb,format=raw \
+    -device virtio-blk,drive=vdc -drive file=$(USERDATA_IMG),index=2,if=none,id=vdc,format=raw \
+    -device virtio-net,netdev=adbnet0 -netdev user,id=adbnet0,hostfwd=tcp::5554-:5554,hostfwd=tcp::5555-:5555
+  QEMU_KERNEL_CMDLINE += root=$(ROOT) $(RW) kvm-arm.mode=protected earlyprintk androidboot.hardware=qemu_trusty trusty-log.log_ratelimit_interval=0 trusty-log.log_to_dmesg=always
+  QEMU_ARGS += -dtb $(ANDROID_DTB)
+else ifneq ($(INITRD),)
   ifeq ($(INITRD),1)
     INITRD := $(CPIO_FILE)
   endif
@@ -529,9 +576,10 @@ else ifeq ($(ARCH),i386)
 else
   ifeq ($(TRUSTY),1)
     # Trusty needs to use its own build of QEMU which has some custom patches
-    QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/qemu-system-aarch64
+    # QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/qemu-system-aarch64
+    QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/aarch64-softmmu/qemu-system-aarch64
   else
-	QEMU_BIN ?= qemu-system-aarch64
+    QEMU_BIN ?= qemu-system-aarch64
   endif
 
   QEMU_KERNEL_CMDLINE += console=ttyAMA0
@@ -554,6 +602,11 @@ endif
 
 QEMU_ARGS += -append "$(QEMU_KERNEL_CMDLINE) $(QEMU_EXTRA_KERNEL_CMDLINE)"
 
+# Force using the Trusty kernel image for now, there are boot errors with other images
+ifeq ($(TRUSTY),1)
+  QEMU_KERNEL_IMAGE := $(TRUSTY_OUT)/linux-build/arch/arm64/boot/Image
+endif
+
 RUN_DEPS := $(QEMU_KERNEL_IMAGE)
 RUN_DIR := $(ROOT_DIR)
 
@@ -567,6 +620,11 @@ ifeq ($(TRUSTY),1)
   RUN_DIR := $(ATF_DIR)
 endif
 
+ifeq ($(ANDROID_USERSPACE),1)
+  # We need this device tree blob to mount /vendor
+  RUN_DEPS += $(ANDROID_DTB)
+endif
+
 .PHONY: run
 run: $(RUN_DEPS) | $(SHARED_DIR)
 	@echo "$(GREEN)Running QEMU, press 'ctrl-a x' to quit $(NC)"