kbuild: Add tuxmake backend support

Add tuxmake as an alternative build backend, enabled with the
USE_TUXMAKE=1 environment variable.

Refactor _parse_fragments() to return a list of fragment file paths
instead of a count and update _merge_frags() to accept this list.
Add _build_with_tuxmake() method with basic tuxmake invocation.
Only call _merge_frags() for the make backend since tuxmake handles
fragments via --kconfig-add, added in the next commit.

Reviewed-by: Ben Copeland <ben.copeland@linaro.org>
Signed-off-by: Anders Roxell <anders.roxell@linaro.org>

Author: roxell

kernelci/kbuild.py

@@ -166,6 +166,9 @@ def __init__(self, node=None, jobname=None, params=None, jsonobj=None, apiconfig
             if isinstance(self._defconfig, str) and '+' in self._defconfig:
                 self._defconfig = self._defconfig.split('+')
             self._backend = params.get('backend', 'make')
+            # Support USE_TUXMAKE environment variable for backward compatibility
+            if os.environ.get('USE_TUXMAKE') == '1':
+                self._backend = 'tuxmake'
             self._fragments = params['fragments']
             self._fragment_configs = fragment_configs or {}
             if 'coverage' in self._fragments:
@@ -313,20 +316,21 @@ def init_steps(self):
         # set environment variables
         self.addcomment("Set environment variables")
         self.addcmd("export ARCH=" + self._arch)
-        if self._cross_compile:
-            self.addcmd("export CROSS_COMPILE=" + self._cross_compile)
-        if self._cross_compile_compat:
-            self.addcmd("export CROSS_COMPILE_COMPAT=" +
-                        self._cross_compile_compat)
-        self.addcmd("export INSTALL_MOD_PATH=_modules_")
-        self.addcmd("export INSTALL_MOD_STRIP=1")
-        self.addcmd("export INSTALL_DTBS_PATH=_dtbs_")
-        self.addcmd("export CC=" + self._compiler)
-        self.addcmd("export HOSTCC=" + self._compiler)
-        # if self._compiler start with clang- we need to set env vars
-        if self._compiler.startswith("clang-"):
-            # LLVM=1, can be suffix with version in future, like -14
-            self.addcmd("export LLVM=1")
+        if self._backend != 'tuxmake':
+            if self._cross_compile:
+                self.addcmd("export CROSS_COMPILE=" + self._cross_compile)
+            if self._cross_compile_compat:
+                self.addcmd("export CROSS_COMPILE_COMPAT=" +
+                            self._cross_compile_compat)
+            self.addcmd("export INSTALL_MOD_PATH=_modules_")
+            self.addcmd("export INSTALL_MOD_STRIP=1")
+            self.addcmd("export INSTALL_DTBS_PATH=_dtbs_")
+            self.addcmd("export CC=" + self._compiler)
+            self.addcmd("export HOSTCC=" + self._compiler)
+            # if self._compiler start with clang- we need to set env vars
+            if self._compiler.startswith("clang-"):
+                # LLVM=1, can be suffix with version in future, like -14
+                self.addcmd("export LLVM=1")
         # set -x for echo
         self._steps.append("set -x")
         # touch build.log
@@ -339,21 +343,22 @@ def init_steps(self):
         # but keep pid, so i can kill it later
         self._artifacts.append("build.log")
         self._artifacts.append("build.sh")
-        if not self._dtbs_check:
-            self._artifacts.append("build_kimage.log")
-            self._artifacts.append("build_kimage_stderr.log")
-            self._artifacts.append("build_modules.log")
-            self._artifacts.append("build_modules_stderr.log")
-            if self._kfselftest:
-                self._artifacts.append("build_kselftest.log")
-                self._artifacts.append("build_kselftest_stderr.log")
-            # disable DTBS for some archs
-            if self._arch not in DTBS_DISABLED:
-                self._artifacts.append("build_dtbs.log")
-                self._artifacts.append("build_dtbs_stderr.log")
-        else:
-            self._artifacts.append("build_dtbs_check.log")
-            self._artifacts.append("build_dtbs_check_stderr.log")
+        if self._backend != 'tuxmake':
+            if not self._dtbs_check:
+                self._artifacts.append("build_kimage.log")
+                self._artifacts.append("build_kimage_stderr.log")
+                self._artifacts.append("build_modules.log")
+                self._artifacts.append("build_modules_stderr.log")
+                if self._kfselftest:
+                    self._artifacts.append("build_kselftest.log")
+                    self._artifacts.append("build_kselftest_stderr.log")
+                # disable DTBS for some archs
+                if self._arch not in DTBS_DISABLED:
+                    self._artifacts.append("build_dtbs.log")
+                    self._artifacts.append("build_dtbs_stderr.log")
+            else:
+                self._artifacts.append("build_dtbs_check.log")
+                self._artifacts.append("build_dtbs_check_stderr.log")
         self._artifacts.append("metadata.json")
         # download tarball
         self.addcomment("Download tarball")
@@ -567,12 +572,19 @@ def add_fragment(self, fragname):
         return self.extract_config(frag)
 
     def _parse_fragments(self, firmware=False):
-        """ Parse fragments kbuild config and create config fragments """
-        num = 0
-        for fragment in self._fragments:
+        """ Parse fragments kbuild config and create config fragments
+
+        Returns:
+            list: List of fragment file paths
+        """
+        fragment_files = []
+
+        for idx, fragment in enumerate(self._fragments):
             content = ''
+            fragment_name = fragment
+
             if fragment.startswith("cros://"):
-                (content, fragment) = self._getcrosfragment(fragment)
+                (content, fragment_name) = self._getcrosfragment(fragment)
             elif fragment.startswith("cip://"):
                 content = self._getcipfragment(fragment)
             elif fragment.startswith("CONFIG_"):
@@ -581,27 +593,37 @@ def _parse_fragments(self, firmware=False):
                 # Use fragment configs passed from scheduler
                 content = self.add_fragment(fragment)
 
-            fragfile = os.path.join(self._fragments_dir, f"{num}.config")
+            fragfile = os.path.join(self._fragments_dir, f"{idx}.config")
             with open(fragfile, 'w') as f:
                 f.write(content)
+
+            fragment_files.append(fragfile)
+
             # add fragment to artifacts but relative to artifacts dir
             frag_rel = os.path.relpath(fragfile, self._af_dir)
-            self._config_full += '+' + fragment
+            self._config_full += '+' + fragment_name
             self._artifacts.append(frag_rel)
-            num += 1
+
         if firmware:
             content = 'CONFIG_EXTRA_FIRMWARE_DIR="'+self._firmware_dir+'"\n'
-            fragfile = os.path.join(self._fragments_dir, f"{num}.config")
+            fragfile = os.path.join(self._fragments_dir, f"{len(self._fragments)}.config")
             with open(fragfile, 'w') as f:
                 f.write(content)
+
+            fragment_files.append(fragfile)
+
             # add fragment to artifacts but relative to artifacts dir
             frag_rel = os.path.relpath(fragfile, self._af_dir)
             self._artifacts.append(frag_rel)
-            num += 1
-        return num
 
-    def _merge_frags(self, fragnum):
-        """ Merge config fragments to .config """
+        return fragment_files
+
+    def _merge_frags(self, fragment_files):
+        """ Merge config fragments to .config
+
+        Args:
+            fragment_files: List of fragment file paths to merge
+        """
         self.startjob("config_defconfig")
         self.addcmd("cd " + self._srcdir)
         if isinstance(self._defconfig, str) and self._defconfig.startswith('cros://'):
@@ -625,9 +647,8 @@ def _merge_frags(self, fragnum):
                 self._config_full = defconfigs + self._config_full
         # fragments
         self.startjob("config_fragments")
-        for i in range(0, fragnum):
-            self.addcmd("./scripts/kconfig/merge_config.sh" +
-                        f" -m .config {self._fragments_dir}/{i}.config")
+        for fragfile in fragment_files:
+            self.addcmd(f"./scripts/kconfig/merge_config.sh -m .config {fragfile}")
         # TODO: olddefconfig should be optional/configurable
         # TODO: log all warnings/errors of olddefconfig to separate file
         self.addcmd("make olddefconfig")
@@ -638,9 +659,14 @@ def _merge_frags(self, fragnum):
     def _generate_script(self):
         """ Generate shell script for complete build """
         print("Generating shell script")
-        fragnum = self._parse_fragments(firmware=True)
-        self._merge_frags(fragnum)
-        self._build_with_make()
+        self._fragment_files = self._parse_fragments(firmware=True)
+
+        if self._backend == 'tuxmake':
+            self._build_with_tuxmake()
+        else:
+            self._merge_frags(self._fragment_files)
+            self._build_with_make()
+
         self._write_metadata()
         # terminate all active jobs
         self.startjob(None)
@@ -700,6 +726,29 @@ def _build_with_make(self):
         else:
             self._build_dtbs_check()
 
+    def _build_with_tuxmake(self):
+        """ Build kernel using tuxmake """
+        self.startjob("build_tuxmake")
+        self.addcmd("cd " + self._srcdir)
+
+        tuxmake_cmd = (
+            f"tuxmake --runtime=null "
+            f"--target-arch={self._arch} "
+            f"--toolchain={self._compiler} "
+            f"kernel modules"
+        )
+
+        print(f"Building with tuxmake: {tuxmake_cmd}")
+        self.addcmd(tuxmake_cmd)
+
+        # tuxmake outputs 'config' (no dot), rename to '.config' for consistency
+        self.addcmd(
+            f"[ -f {self._af_dir}/config ] && "
+            f"mv {self._af_dir}/config {self._af_dir}/.config"
+        )
+
+        self.addcmd("cd ..")
+
     def _build_kernel(self):
         """ Add kernel build steps """
         self.startjob("build_kernel")