Add TuxMake as an alternative backend for kernel builds

config/docker/fragment/kernelci.jinja2

@@ -19,6 +19,9 @@ RUN cp -R config /etc/kernelci/
 WORKDIR /root
 RUN rm -rf /tmp/kernelci-core
 
+# Install tuxmake kernel build tool
+RUN pip3 install --break-system-packages --no-cache-dir tuxmake==1.34.0
+
 # Set up kernelci user
 RUN useradd kernelci -u 1000 -d /home/kernelci -s /bin/bash
 RUN mkdir -p /home/kernelci

kernelci/kbuild.py

@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 #
 # Copyright (C) 2023 Collabora Limited
 # Author: Denys Fedoryshchenko <[email protected]>
@@ -44,12 +44,12 @@
 from typing import Dict, Tuple, List
 
 CIP_CONFIG_URL = \
     "https://gitlab.com/cip-project/cip-kernel/cip-kernel-config/-/raw/master/{branch}/{config}"  # noqa
 CROS_CONFIG_URL = \
     "https://chromium.googlesource.com/chromiumos/third_party/kernel/+archive/refs/heads/{branch}/chromeos/config.tar.gz"  # noqa
 FW_GIT = "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git"  # noqa
 
 # TODO: find a way to automatically fetch this information
 LATEST_LTS_MAJOR = 6
 LATEST_LTS_MINOR = 12
 
@@ -106,19 +106,19 @@
 REDIR = ' > >(tee {}) 2> >(tee {} >&1)'
 
 
 def _download_file_inner(url, file_path):
     try:
         r = requests.get(url, stream=True, timeout=60)
     except requests.exceptions.RequestException as e:
         print(f"[_download_file_inner] Error: {e}")
         return False
     except requests.exceptions.Timeout as e:
         print(f"[_download_file_inner] Timeout: {e}")
         return False
     except requests.exceptions.ConnectionError as e:
         print(f"[_download_file_inner] Connection error: {e}")
         return False
     except Exception as e:
         print(f"[_download_file_inner] Exception: {e}")
         return False
     if r.status_code == 200:
@@ -143,13 +143,13 @@
     return False
 
 
 class KBuild():
     '''
     Build class that represents kernel build
     if node, jobname and params are provided, create new build object
     if jsonobj is provided, load class from serialized json
     '''
     def __init__(self, node=None, jobname=None, params=None, jsonobj=None, apiconfig=None,
                  fragment_configs=None):
         # Retrieve and store API token for future use
         self._api_token = os.environ.get('KCI_API_TOKEN')
@@ -165,6 +165,10 @@
             # if defconfig contains '+', it means it is a list
             if isinstance(self._defconfig, str) and '+' in self._defconfig:
                 self._defconfig = self._defconfig.split('+')
+            self._backend = params.get('backend', 'make')
+            # Support USE_TUXMAKE environment variable for backward compatibility
+            if os.environ.get('USE_TUXMAKE') == '1':
+                self._backend = 'tuxmake'
             self._fragments = params['fragments']
             self._fragment_configs = fragment_configs or {}
             if 'coverage' in self._fragments:
@@ -220,6 +224,7 @@
             self._defconfig = jsonobj['defconfig']
             self._fragments = jsonobj['fragments']
             self._fragment_configs = jsonobj.get('fragment_configs', {})
+            self._backend = jsonobj.get('backend', 'make')
             self._cross_compile = jsonobj['cross_compile']
             self._cross_compile_compat = jsonobj['cross_compile_compat']
             self._steps = jsonobj['steps']
@@ -565,12 +570,19 @@
         return self.extract_config(frag)
 
     def _parse_fragments(self, firmware=False):
-        """ Parse fragments kbuild config and create config fragments """
-        num = 0
-        for fragment in self._fragments:
+        """ Parse fragments kbuild config and create config fragments
+
+        Returns:
+            list: List of fragment file paths
+        """
+        fragment_files = []
+
+        for idx, fragment in enumerate(self._fragments):
             content = ''
+            fragment_name = fragment
+
             if fragment.startswith("cros://"):
-                (content, fragment) = self._getcrosfragment(fragment)
+                (content, fragment_name) = self._getcrosfragment(fragment)
             elif fragment.startswith("cip://"):
                 content = self._getcipfragment(fragment)
             elif fragment.startswith("CONFIG_"):
@@ -579,27 +591,50 @@
                 # Use fragment configs passed from scheduler
                 content = self.add_fragment(fragment)
 
-            fragfile = os.path.join(self._fragments_dir, f"{num}.config")
+            if not content:
+                print(f"[_parse_fragments] WARNING: Fragment {fragment} has no content")
+                continue
+
+            fragfile = os.path.join(self._fragments_dir, f"{idx}.config")
             with open(fragfile, 'w') as f:
                 f.write(content)
+
+            if not os.path.exists(fragfile):
+                print(f"[_parse_fragments] ERROR: Failed to create fragment file: {fragfile}")
+                continue
+
+            config_count = len([line for line in content.split('\n') if line.strip()])
+            print(f"[_parse_fragments] Created {fragfile} ({config_count} configs)")
+
+            fragment_files.append(fragfile)
+
             # add fragment to artifacts but relative to artifacts dir
             frag_rel = os.path.relpath(fragfile, self._af_dir)
-            self._config_full += '+' + fragment
+            self._config_full += '+' + fragment_name
             self._artifacts.append(frag_rel)
-            num += 1
+
         if firmware:
             content = 'CONFIG_EXTRA_FIRMWARE_DIR="'+self._firmware_dir+'"\n'
-            fragfile = os.path.join(self._fragments_dir, f"{num}.config")
+            fragfile = os.path.join(self._fragments_dir, f"{len(self._fragments)}.config")
             with open(fragfile, 'w') as f:
                 f.write(content)
-            # add fragment to artifacts but relative to artifacts dir
-            frag_rel = os.path.relpath(fragfile, self._af_dir)
-            self._artifacts.append(frag_rel)
-            num += 1
-        return num
 
-    def _merge_frags(self, fragnum):
-        """ Merge config fragments to .config """
+            if os.path.exists(fragfile):
+                fragment_files.append(fragfile)
+                frag_rel = os.path.relpath(fragfile, self._af_dir)
+                self._artifacts.append(frag_rel)
+            else:
+                print("[_parse_fragments] ERROR: Failed to create firmware fragment")
+
+        print(f"[_parse_fragments] Created {len(fragment_files)} fragment files")
+        return fragment_files
+
+    def _merge_frags(self, fragment_files):
+        """ Merge config fragments to .config
+
+        Args:
+            fragment_files: List of fragment file paths to merge
+        """
         self.startjob("config_defconfig")
         self.addcmd("cd " + self._srcdir)
         if isinstance(self._defconfig, str) and self._defconfig.startswith('cros://'):
@@ -623,11 +658,10 @@
                 self._config_full = defconfigs + self._config_full
         # fragments
         self.startjob("config_fragments")
-        for i in range(0, fragnum):
-            self.addcmd("./scripts/kconfig/merge_config.sh" +
-                        f" -m .config {self._fragments_dir}/{i}.config")
+        for fragfile in fragment_files:
+            self.addcmd(f"./scripts/kconfig/merge_config.sh -m .config {fragfile}")
         # TODO: olddefconfig should be optional/configurable
         # TODO: log all warnings/errors of olddefconfig to separate file
         self.addcmd("make olddefconfig")
         self.addcmd(f"cp .config {self._af_dir}/")
         self.addcmd("cd ..")
@@ -635,30 +669,15 @@
 
     def _generate_script(self):
         """ Generate shell script for complete build """
-        # TODO(nuclearcat): Fetch firmware only if needed
         print("Generating shell script")
-        fragnum = self._parse_fragments(firmware=True)
-        self._merge_frags(fragnum)
-        if not self._dtbs_check:
-            # TODO: verify if CONFIG_EXTRA_FIRMWARE have any files
-            # We can check that if fragments have CONFIG_EXTRA_FIRMWARE
-            self._fetch_firmware()
-            self._build_kernel()
-            self._build_modules()
-            if self._kfselftest:
-                self._build_kselftest()
-            if self._arch not in DTBS_DISABLED:
-                self._build_dtbs()
-            self._package_kimage()
-            self._package_modules()
-            if self._coverage:
-                self._package_coverage()
-            if self._kfselftest:
-                self._package_kselftest()
-            if self._arch not in DTBS_DISABLED:
-                self._package_dtbs()
+        self._fragment_files = self._parse_fragments(firmware=True)
+
+        if self._backend == 'tuxmake':
+            self._build_with_tuxmake()
         else:
-            self._build_dtbs_check()
+            self._merge_frags(self._fragment_files)
+            self._build_with_make()
+
         self._write_metadata()
         # terminate all active jobs
         self.startjob(None)
@@ -697,6 +716,113 @@
         # copy to artifacts dir
         os.system(f"cp {filename} {self._af_dir}/build.sh")
 
+    def _build_with_make(self):
+        """ Build kernel using make """
+        if not self._dtbs_check:
+            self._fetch_firmware()
+            self._build_kernel()
+            self._build_modules()
+            if self._kfselftest:
+                self._build_kselftest()
+            if self._arch not in DTBS_DISABLED:
+                self._build_dtbs()
+            self._package_kimage()
+            self._package_modules()
+            if self._coverage:
+                self._package_coverage()
+            if self._kfselftest:
+                self._package_kselftest()
+            if self._arch not in DTBS_DISABLED:
+                self._package_dtbs()
+        else:
+            self._build_dtbs_check()
+
+    def _build_with_tuxmake(self):
+        """ Build kernel using tuxmake with native fragment support """
+        print("[_build_with_tuxmake] Starting tuxmake build")
+
+        if not hasattr(self, '_fragment_files'):
+            print("[_build_with_tuxmake] ERROR: No fragment files available")
+            self._fragment_files = []
+
+        print(f"[_build_with_tuxmake] Using {len(self._fragment_files)} fragment files")
+
+        # Handle multiple defconfigs - tuxmake only supports one
+        if isinstance(self._defconfig, list):
+            if len(self._defconfig) > 1:
+                raise ValueError(
+                    f"TuxMake backend does not support multiple defconfigs: "
+                    f"{self._defconfig}. Use backend=make or specify a single defconfig."
+                )
+            defconfig = self._defconfig[0]
+            self._config_full = self._defconfig[0] + self._config_full
+        elif isinstance(self._defconfig, str):
+            defconfig = self._defconfig
+            self._config_full = self._defconfig + self._config_full
+        else:
+            defconfig = 'defconfig'
+            print("[_build_with_tuxmake] WARNING: No defconfig specified, using 'defconfig'")
+
+        # Check for unsupported kselftest
+        if self._kfselftest:
+            print("[_build_with_tuxmake] WARNING: kselftest is not supported with "
+                  "tuxmake backend, skipping. Use backend=make for kselftest builds.")
+
+        # Fetch firmware only for normal builds, not dtbs_check
+        if not self._dtbs_check:
+            self._fetch_firmware()
+
+        self.startjob("build_tuxmake")
+        self.addcmd("cd " + self._srcdir)
+
+        use_kconfig_flag = True
+
+        # Handle ChromeOS defconfig
+        if defconfig.startswith('cros://'):
+            print(f"[_build_with_tuxmake] Handling ChromeOS defconfig: {defconfig}")
+            dotconfig = os.path.join(self._srcdir, ".config")
+            content, defconfig_name = self._getcrosfragment(defconfig)
+            with open(dotconfig, 'w') as f:
+                f.write(content)
+            self.addcmd("make olddefconfig")
+            use_kconfig_flag = False
+
+        cmd_parts = [
+            "tuxmake --runtime=null",
+            f"--target-arch={self._arch}",
+            f"--toolchain={self._compiler}",
+            f"--output-dir={self._af_dir}",
+        ]
+
+        if use_kconfig_flag:
+            cmd_parts.append(f"--kconfig={defconfig}")
+
+        for fragfile in self._fragment_files:
+            if os.path.exists(fragfile):
+                cmd_parts.append(f"--kconfig-add={fragfile}")
+                print(f"[_build_with_tuxmake] Adding fragment: {os.path.basename(fragfile)}")
+            else:
+                print(f"[_build_with_tuxmake] WARNING: Fragment file not found: {fragfile}")
+
+        # Build targets depend on mode
+        if self._dtbs_check:
+            # dtbs_check mode: run ONLY dtbs_check, like make backend
+            targets = ["dtbs_check"]
+        else:
+            # Normal build: kernel, modules, plus dtbs if arch supports it
+            targets = ["kernel", "modules"]
+            if self._arch not in DTBS_DISABLED:
+                targets.append("dtbs")
+        cmd_parts.append(" ".join(targets))
+        print(f"[_build_with_tuxmake] Building targets: {' '.join(targets)}")
+
+        tuxmake_cmd = " ".join(cmd_parts)
+        print(f"[_build_with_tuxmake] Command: {tuxmake_cmd}")
+        print(f"[_build_with_tuxmake] Output directory: {self._af_dir}")
+        self.addcmd(tuxmake_cmd)
+
+        self.addcmd("cd ..")
+
     def _build_kernel(self):
         """ Add kernel build steps """
         self.startjob("build_kernel")
@@ -859,13 +985,18 @@
         metadata['build']['fragments'] = self._fragments
         metadata['build']['srcdir'] = self._srcdir
         metadata['build']['config_full'] = self._config_full
+        metadata['build']['backend'] = self._backend
 
         with open(os.path.join(self._af_dir, "metadata.json"), 'w') as f:
             json.dump(metadata, f, indent=4)
 
     def serialize(self, filename):
-        """ Serialize class to json """
-        # TODO(nuclearcat): Implement to_json method?
+        """ Serialize class to json
+
+        Note: Uses __dict__ to serialize all instance attributes (including
+        _backend, _arch, etc). The from_json() method strips underscore
+        prefixes when loading, so _backend becomes 'backend' in jsonobj.
+        """
         data = json.dumps(self, default=lambda o: o.__dict__,
                           sort_keys=True, indent=4)
         with open(filename, 'w') as f:
@@ -939,9 +1070,19 @@
 
         # Prepare all artifacts for upload
         upload_tasks = []
-        for artifact in self._artifacts:
-            artifact_path = os.path.join(self._af_dir, artifact)
-            upload_tasks.append((artifact, artifact_path))
+        if self._backend == 'tuxmake':
+            # For TuxMake, upload everything in artifacts directory
+            print("[_upload_artifacts] TuxMake backend: discovering files in artifacts dir")
+            for root, dirs, files in os.walk(self._af_dir):
+                for file in files:
+                    file_rel = os.path.relpath(os.path.join(root, file), self._af_dir)
+                    artifact_path = os.path.join(self._af_dir, file_rel)
+                    upload_tasks.append((file_rel, artifact_path))
+        else:
+            # For make backend, upload only listed artifacts
+            for artifact in self._artifacts:
+                artifact_path = os.path.join(self._af_dir, artifact)
+                upload_tasks.append((artifact, artifact_path))
 
         # Function to handle a single artifact upload
         # args: (artifact, artifact_path)
@@ -980,12 +1121,12 @@
                     os.unlink(upload_path)
 
                 print(f"[_upload_artifacts] Uploaded {artifact} to {stored_url}")
                 return artifact, stored_url, None
             except Exception as e:
                 print(f"[_upload_artifacts] Error uploading {artifact}: {e}")
                 if compressed_file and os.path.exists(upload_path):
                     os.unlink(upload_path)
                 return artifact, None, str(e)
 
         # Process uploads in parallel
         max_workers = min(10, len(upload_tasks))  # Limit concurrent uploads