Merge branch 'main' into grpc-targets

Author: qwattash

pycheribuild/config/chericonfig.py

@@ -123,6 +123,11 @@ def clang_march_flag(self) -> str:
         return self.value[1]
 
 
+class RiscvFloatAbi(Enum):
+    SOFT = "soft"
+    HARD = "hard"
+
+
 def _default_arm_none_eabi_prefix(c: "CheriConfig", _):
     # see if the local install exists:
     default_path = c.output_root / c.local_arm_none_eabi_toolchain_relpath
@@ -319,7 +324,14 @@ def __init__(self, loader, action_class: "type[CheribuildActionEnum]") -> None:
             default=MipsFloatAbi.SOFT,
             type=MipsFloatAbi,
             group=loader.cross_compile_options_group,
-            help="The floating point ABI to use for building MIPS+CHERI programs",
+            help="Whether to use soft or hard float ABIs when targeting MIPS",
+        )
+        self.riscv_float_abi = loader.add_option(
+            "riscv-float-abi",
+            default=RiscvFloatAbi.HARD,
+            type=RiscvFloatAbi,
+            group=loader.cross_compile_options_group,
+            help="Whether to use soft or hard float ABIs when targeting RISC-V",
         )
         self.aarch64_fp_and_simd_options = loader.add_option(
             "aarch64-fp-and-simd-options",

pycheribuild/config/compilation_targets.py

@@ -149,15 +149,18 @@ def default_install_dir(self, install_dir: DefaultInstallDir) -> Path:
 
     @property
     def c_compiler(self) -> Path:
-        return self._compiler_dir / "clang"
+        override = self.project.custom_c_compiler
+        return self._compiler_dir / "clang" if override is None else override
 
     @property
     def cxx_compiler(self) -> Path:
-        return self._compiler_dir / "clang++"
+        override = self.project.custom_cxx_compiler
+        return self._compiler_dir / "clang++" if override is None else override
 
     @property
     def c_preprocessor(self) -> Path:
-        return self._compiler_dir / "clang-cpp"
+        override = self.project.custom_c_preprocessor
+        return self._compiler_dir / "clang-cpp" if override is None else override
 
     @property
     def linker(self) -> Path:
@@ -203,10 +206,10 @@ def essential_compiler_and_linker_flags_impl(
         default_flags_only=False,
     ):
         assert xtarget is not None
-        if softfloat is None:
-            softfloat = cls.uses_softfloat_by_default(xtarget)
         config = instance.config
         project = instance.project
+        if softfloat is None:
+            softfloat = project.uses_softfloat_by_default()
         # noinspection PyProtectedMember
         if perform_sanity_checks and not project._setup_called:
             project.fatal(
@@ -241,19 +244,18 @@ def essential_compiler_and_linker_flags_impl(
             result.append("-integrated-as")
             result.append("-G0")  # no small objects in GOT optimization
             # Floating point ABI:
+            float_abi = MipsFloatAbi.SOFT if softfloat else MipsFloatAbi.HARD
+            result.append(float_abi.clang_float_flag())
             if cls.is_baremetal() or cls.is_rtems():
-                # The baremetal driver doesn't add -fPIC for CHERI
                 if xtarget.is_cheri_purecap([CPUArchitecture.MIPS64]):
+                    assert float_abi == MipsFloatAbi.SOFT  # For now use soft-float to avoid compiler crashes
+                    # The baremetal driver doesn't add -fPIC for CHERI
                     result.append("-fPIC")
-                    # For now use soft-float to avoid compiler crashes
-                    result.append(MipsFloatAbi.SOFT.clang_float_flag())
                 else:
                     # We don't have a softfloat library baremetal so always compile hard-float
-                    result.append(MipsFloatAbi.HARD.clang_float_flag())
+                    assert float_abi == MipsFloatAbi.HARD  # For now use soft-float to avoid compiler crashes
                     result.append("-fno-pic")
                     result.append("-mno-abicalls")
-            else:
-                result.append(config.mips_float_abi.clang_float_flag())
 
             # CPU flags (currently always BERI):
             if cls.is_cheribsd():
@@ -340,10 +342,6 @@ def get_riscv_abi(cls, xtarget: CrossCompileTarget, *, softfloat: bool) -> str:
             abi += "d"
         return abi
 
-    @classmethod
-    def uses_softfloat_by_default(cls, xtarget: "CrossCompileTarget"):
-        return False
-
 
 class FreeBSDTargetInfo(_ClangBasedTargetInfo):
     shortname: str = "FreeBSD"
@@ -843,11 +841,6 @@ def localbase(self) -> Path:
     def is_baremetal(cls):
         return True
 
-    @classmethod
-    def uses_softfloat_by_default(cls, xtarget: "CrossCompileTarget"):
-        # Note: RISC-V Baremetal/FreeRTOS currently only supports softfloat
-        return xtarget.is_riscv(include_purecap=True)
-
 
 class NewlibBaremetalTargetInfo(BaremetalClangTargetInfo):
     shortname = "Newlib"
@@ -928,10 +921,6 @@ def cmake_extra_toolchain_file_code(self) -> str:
 set(CMAKE_DL_LIBS "")
 """
 
-    @classmethod
-    def uses_softfloat_by_default(cls, xtarget: "CrossCompileTarget"):
-        return False
-
     @classmethod
     def essential_compiler_and_linker_flags_impl(cls, *args, xtarget: "CrossCompileTarget", **kwargs) -> "list[str]":
         result = super().essential_compiler_and_linker_flags_impl(*args, xtarget=xtarget, **kwargs)
@@ -952,7 +941,10 @@ def essential_compiler_and_linker_flags_impl(cls, *args, xtarget: "CrossCompileT
     @property
     def sysroot_dir(self) -> Path:
         sysroot_dir = self.config.sysroot_output_root / self.config.default_cheri_sdk_directory_name
-        return sysroot_dir / "picolibc" / self.target.get_rootfs_target().generic_arch_suffix
+        result = sysroot_dir / "picolibc" / self.target.get_rootfs_target().generic_arch_suffix
+        if self.target.is_riscv(include_purecap=True):
+            result /= self.get_riscv_abi(self.target, softfloat=self.project.uses_softfloat_by_default())
+        return result
 
     @classmethod
     def _get_compiler_project(cls) -> "type[BuildLLVMInterface]":

pycheribuild/config/config_loader_base.py

@@ -65,8 +65,8 @@ class ConfigLoaderBase(ABC):
     # will be set later...
     _cheri_config: ConfigBase
 
-    option_handles: "typing.ClassVar[dict[str, ConfigOptionHandle]]" = {}
-    _json: "typing.ClassVar[dict[str, _LoadedConfigValue]]" = {}
+    option_handles: "dict[str, ConfigOptionHandle]" = {}
+    _json: "dict[str, _LoadedConfigValue]" = {}
     is_completing_arguments: bool = "_ARGCOMPLETE" in os.environ
     is_generating_readme: bool = "_GENERATING_README" in os.environ
     is_running_unit_tests: bool = False

pycheribuild/config/target_info.py

@@ -37,7 +37,7 @@
 from pathlib import Path
 from typing import ClassVar, Optional
 
-from .chericonfig import AArch64FloatSimdOptions, CheriConfig, MipsFloatAbi
+from .chericonfig import AArch64FloatSimdOptions, CheriConfig, MipsFloatAbi, RiscvFloatAbi
 from ..filesystemutils import FileSystemUtils
 from ..processutils import CompilerInfo, get_compiler_info
 from ..utils import OSInfo, fatal_error, final, status_update, warning_message
@@ -180,6 +180,8 @@ def __init__(self, config):
         super().__init__(config)
         self._init_called = False
 
+    def uses_softfloat_by_default(self) -> bool: ...
+
     def get_compiler_info(self, compiler: Path) -> CompilerInfo:
         return get_compiler_info(compiler, config=self.config)
 
@@ -603,21 +605,18 @@ def default_install_dir(self, install_dir: DefaultInstallDir) -> Path:
 
     @property
     def c_compiler(self) -> Path:
-        if self.project.custom_c_compiler is not None:
-            return self.project.custom_c_compiler
-        return self.host_c_compiler(self.config)
+        override = self.project.custom_c_compiler
+        return self.host_c_compiler(self.config) if override is None else override
 
     @property
     def cxx_compiler(self) -> Path:
-        if self.project.custom_cxx_compiler is not None:
-            return self.project.custom_cxx_compiler
-        return self.host_cxx_compiler(self.config)
+        override = self.project.custom_cxx_compiler
+        return self.host_cxx_compiler(self.config) if override is None else override
 
     @property
     def c_preprocessor(self) -> Path:
-        if self.project.custom_c_preprocessor is not None:
-            return self.project.custom_c_preprocessor
-        return self.host_c_preprocessor(self.config)
+        override = self.project.custom_c_preprocessor
+        return self.host_c_preprocessor(self.config) if override is None else override
 
     @property
     def linker(self) -> Path:
@@ -933,6 +932,8 @@ def cheri_config_suffix(self, config: CheriConfig) -> str:
                     result += "-subobject-nodebug"
         if self.is_mips(include_purecap=True) and config.mips_float_abi == MipsFloatAbi.HARD:
             result += "-hardfloat"
+        if self.is_riscv(include_purecap=True) and config.riscv_float_abi == RiscvFloatAbi.SOFT:
+            result += "-softfloat"
         if self.is_aarch64(include_purecap=True):
             if config.aarch64_fp_and_simd_options != AArch64FloatSimdOptions.DEFAULT:
                 result += config.aarch64_fp_and_simd_options.config_suffix()

pycheribuild/mtree.py

@@ -28,7 +28,7 @@
 # SUCH DAMAGE.
 #
 
-import collections
+import collections.abc
 import fnmatch
 import io
 import os
@@ -190,31 +190,34 @@ def _glob(self, patfrags: "list[str]", prefix: MtreePath, *, case_sensitive=Fals
 
     def glob(self, pattern: str, *, case_sensitive=False) -> Iterator[MtreePath]:
         if len(pattern) == 0:
-            raise StopIteration()  # pytype does not like plain "return"
+            return iter([])
         head, tail = os.path.split(pattern)
         patfrags = [tail]
         while head:
             head, tail = os.path.split(head)
             patfrags.insert(0, tail)
         return self._glob(patfrags, MtreePath(), case_sensitive=case_sensitive)
 
-    def _walk(self, top, prefix):
+    def _walk(self, top, prefix) -> "Iterator[tuple[MtreePath, list[str], list[str]]]":
         split = self._split_key(top)
         if split is not None:
-            return self.children[split[0]]._walk(split[1], prefix / split[0])
+            if split[0] in self.children:
+                yield from self.children[split[0]]._walk(split[1], prefix / split[0])
+            return
         if self.entry is not None and self.entry.attributes["type"] != "dir":
-            raise StopIteration()  # pytype does not like plain "return"
-        files = []
-        dirs = []
+            return
+        files: "list[tuple[str, MtreeSubtree]]" = []
+        dirs: "list[tuple[str, MtreeSubtree]]" = []
         for k, v in self.children.items():
             if v.entry is not None and v.entry.attributes["type"] != "dir":
                 files.append((k, v))
             else:
                 dirs.append((k, v))
-        yield (prefix, list([k for k, _ in dirs]), list([k for k, _ in files]))
-        return iter([v._walk(MtreePath(), prefix) for _, v in dirs])
+        yield prefix, list([k for k, _ in dirs]), list([k for k, _ in files])
+        for _, v in dirs:
+            yield from v._walk(MtreePath(), prefix)
 
-    def walk(self, top):
+    def walk(self, top) -> "Iterator[tuple[MtreePath, list[str], list[str]]]":
         return self._walk(top, MtreePath())
 
 
@@ -315,6 +318,7 @@ def add_file(
             if symlink_dest is not None:
                 mode = "0755"
             else:
+                assert file is not None
                 mode = self.infer_mode_string(file, False)
         mode = self._ensure_mtree_mode_fmt(mode)
         mtree_path = self._ensure_mtree_path_fmt(path_in_image)
@@ -401,15 +405,16 @@ def add_from_mtree(self, mtree_file: "MtreeFile", path: "Union[PurePath, str]",
         assert not path.startswith("/")
         path = path.rstrip("/")  # remove trailing slashes
         mtree_path = self._ensure_mtree_path_fmt(path)
-        if mtree_path in self._mtree:
+        if self.get(mtree_path) is not None:
             return
-        if mtree_path not in mtree_file._mtree:
+        subtree = mtree_file.get(mtree_path)
+        if subtree is None:
             fatal_error("Could not find " + str(mtree_path) + " in source mtree", pretend=True)
             return
         parent = mtree_path.parent
         if parent != mtree_path:
             self.add_from_mtree(mtree_file, parent, print_status=print_status)
-        attribs = mtree_file.get(mtree_path).attributes
+        attribs = subtree.attributes
         entry = MtreeEntry(mtree_path, attribs)
         if print_status:
             if "link" in attribs:
@@ -465,15 +470,15 @@ def write(self, output: "Union[io.StringIO,Path,typing.IO]", *, pretend):
             output.write("\n")
         output.write("# END\n")
 
-    def get(self, key):
+    def get(self, key) -> Optional[MtreeEntry]:
         return self._mtree.get(key)
 
     @property
-    def root(self):
+    def root(self) -> MtreeSubtree:
         return self._mtree
 
     def glob(self, pattern: str, *, case_sensitive=False) -> Iterator[MtreePath]:
         return self._mtree.glob(pattern, case_sensitive=case_sensitive)
 
-    def walk(self, top):
+    def walk(self, top) -> "Iterator[tuple[MtreePath, list[str], list[str]]]":
         return self._mtree.walk(top)

pycheribuild/projects/build_qemu.py

@@ -241,7 +241,6 @@ def setup(self):
                 "-Werror=implicit-function-declaration",
                 "-Werror=incompatible-pointer-types",
                 "-Werror=format",
-                "-Werror=return-type",
                 "-Wno-sign-compare",
                 "-Wno-unused-parameter",
                 "-Wno-missing-field-initializers",
@@ -316,6 +315,25 @@ def setup(self):
                 "--make=" + self.make_args.command,
             ],
         )
+        python_bin = sys.executable
+        python_search_path = self.config.dollar_path_with_other_tools
+        # Python from a venv cannot be used for QEMU builds, use the base installation instead.
+        if sys.prefix != sys.base_prefix:
+            python_bin = sys.executable.replace(sys.prefix, sys.base_prefix)
+            python_search_path = python_search_path.replace(sys.prefix, sys.base_prefix)
+        py3_version = get_program_version(
+            Path(python_bin),
+            config=self.config,
+            regex=re.compile(rb"Python\s+(\d+)\.(\d+)\.?(\d+)?"),
+        )
+        # QEMU tests are not compatible with 3.12 yet, try to use an older version in that case
+        if py3_version >= (3, 12, 0):
+            for minor_version in (11, 10, 9):
+                found = shutil.which(f"python3.{minor_version}", path=python_search_path)
+                if found:
+                    python_bin = found
+                    break
+        self.configure_args.append(f"--python={python_bin}")
 
         if self.config.create_compilation_db:
             self.make_args.set(V=1)  # Otherwise bear can't parse the compiler output
@@ -506,25 +524,6 @@ def setup(self):
         # Linux/BSD-user is not supported for CHERI (yet)
         self.configure_args.append("--disable-bsd-user")
         self.configure_args.append("--disable-linux-user")
-        python_bin = sys.executable
-        python_search_path = self.config.dollar_path_with_other_tools
-        # Python from a venv cannot be used for QEMU builds, use the base installation instead.
-        if sys.prefix != sys.base_prefix:
-            python_bin = sys.executable.replace(sys.prefix, sys.base_prefix)
-            python_search_path = python_search_path.replace(sys.prefix, sys.base_prefix)
-        py3_version = get_program_version(
-            Path(python_bin),
-            config=self.config,
-            regex=re.compile(rb"Python\s+(\d+)\.(\d+)\.?(\d+)?"),
-        )
-        # QEMU tests are not compatible with 3.12 yet, try to use an older version in that case
-        if py3_version >= (3, 12, 0):
-            for minor_version in (11, 10, 9):
-                found = shutil.which(f"python3.{minor_version}", path=python_search_path)
-                if found:
-                    python_bin = found
-                    break
-        self.configure_args.append(f"--python={python_bin}")
 
         # TODO: tests:
         # noinspection PyUnreachableCode

pycheribuild/projects/cross/freertos.py

@@ -53,6 +53,9 @@ class BuildFreeRTOS(CrossCompileAutotoolsProject):
     )
     default_install_dir = DefaultInstallDir.ROOTFS_LOCALBASE
 
+    def uses_softfloat_by_default(self) -> bool:
+        return True  # FreeRTOS currently only supports soft-float
+
     # FreeRTOS Demos to build
     supported_freertos_demos = [
         # Generic/simple (CHERI-)RISC-V Demo that runs main_blinky on simulators