Staging PR 3037

Author: Staging script

config/docker/fragment/kernelci.jinja2

@@ -18,6 +18,9 @@ RUN cp -R config /etc/kernelci/
 WORKDIR /root
 RUN rm -rf /tmp/kernelci-core
 
+# Install tuxmake kernel build tool
+RUN pip3 install --break-system-packages --no-cache-dir tuxmake==1.34.0
+
 # Set up kernelci user
 RUN useradd kernelci -u 1000 -d /home/kernelci -s /bin/bash
 RUN mkdir -p /home/kernelci

kernelci/kbuild.py

@@ -165,6 +165,10 @@ def __init__(self, node=None, jobname=None, params=None, jsonobj=None, apiconfig
             # if defconfig contains '+', it means it is a list
             if isinstance(self._defconfig, str) and '+' in self._defconfig:
                 self._defconfig = self._defconfig.split('+')
+            self._backend = params.get('backend', 'make')
+            # Support USE_TUXMAKE environment variable for backward compatibility
+            if os.environ.get('USE_TUXMAKE') == '1':
+                self._backend = 'tuxmake'
             self._fragments = params['fragments']
             self._fragment_configs = fragment_configs or {}
             if 'coverage' in self._fragments:
@@ -220,6 +224,7 @@ def __init__(self, node=None, jobname=None, params=None, jsonobj=None, apiconfig
             self._defconfig = jsonobj['defconfig']
             self._fragments = jsonobj['fragments']
             self._fragment_configs = jsonobj.get('fragment_configs', {})
+            self._backend = jsonobj.get('backend', 'make')
             self._cross_compile = jsonobj['cross_compile']
             self._cross_compile_compat = jsonobj['cross_compile_compat']
             self._steps = jsonobj['steps']
@@ -565,12 +570,19 @@ def add_fragment(self, fragname):
         return self.extract_config(frag)
 
     def _parse_fragments(self, firmware=False):
-        """ Parse fragments kbuild config and create config fragments """
-        num = 0
-        for fragment in self._fragments:
+        """ Parse fragments kbuild config and create config fragments
+
+        Returns:
+            list: List of fragment file paths
+        """
+        fragment_files = []
+
+        for idx, fragment in enumerate(self._fragments):
             content = ''
+            fragment_name = fragment
+
             if fragment.startswith("cros://"):
-                (content, fragment) = self._getcrosfragment(fragment)
+                (content, fragment_name) = self._getcrosfragment(fragment)
             elif fragment.startswith("cip://"):
                 content = self._getcipfragment(fragment)
             elif fragment.startswith("CONFIG_"):
@@ -579,27 +591,50 @@ def _parse_fragments(self, firmware=False):
                 # Use fragment configs passed from scheduler
                 content = self.add_fragment(fragment)
 
-            fragfile = os.path.join(self._fragments_dir, f"{num}.config")
+            if not content:
+                print(f"[_parse_fragments] WARNING: Fragment {fragment} has no content")
+                continue
+
+            fragfile = os.path.join(self._fragments_dir, f"{idx}.config")
             with open(fragfile, 'w') as f:
                 f.write(content)
+
+            if not os.path.exists(fragfile):
+                print(f"[_parse_fragments] ERROR: Failed to create fragment file: {fragfile}")
+                continue
+
+            config_count = len([line for line in content.split('\n') if line.strip()])
+            print(f"[_parse_fragments] Created {fragfile} ({config_count} configs)")
+
+            fragment_files.append(fragfile)
+
             # add fragment to artifacts but relative to artifacts dir
             frag_rel = os.path.relpath(fragfile, self._af_dir)
-            self._config_full += '+' + fragment
+            self._config_full += '+' + fragment_name
             self._artifacts.append(frag_rel)
-            num += 1
+
         if firmware:
             content = 'CONFIG_EXTRA_FIRMWARE_DIR="'+self._firmware_dir+'"\n'
-            fragfile = os.path.join(self._fragments_dir, f"{num}.config")
+            fragfile = os.path.join(self._fragments_dir, f"{len(self._fragments)}.config")
             with open(fragfile, 'w') as f:
                 f.write(content)
-            # add fragment to artifacts but relative to artifacts dir
-            frag_rel = os.path.relpath(fragfile, self._af_dir)
-            self._artifacts.append(frag_rel)
-            num += 1
-        return num
 
-    def _merge_frags(self, fragnum):
-        """ Merge config fragments to .config """
+            if os.path.exists(fragfile):
+                fragment_files.append(fragfile)
+                frag_rel = os.path.relpath(fragfile, self._af_dir)
+                self._artifacts.append(frag_rel)
+            else:
+                print("[_parse_fragments] ERROR: Failed to create firmware fragment")
+
+        print(f"[_parse_fragments] Created {len(fragment_files)} fragment files")
+        return fragment_files
+
+    def _merge_frags(self, fragment_files):
+        """ Merge config fragments to .config
+
+        Args:
+            fragment_files: List of fragment file paths to merge
+        """
         self.startjob("config_defconfig")
         self.addcmd("cd " + self._srcdir)
         if isinstance(self._defconfig, str) and self._defconfig.startswith('cros://'):
@@ -623,9 +658,8 @@ def _merge_frags(self, fragnum):
                 self._config_full = defconfigs + self._config_full
         # fragments
         self.startjob("config_fragments")
-        for i in range(0, fragnum):
-            self.addcmd("./scripts/kconfig/merge_config.sh" +
-                        f" -m .config {self._fragments_dir}/{i}.config")
+        for fragfile in fragment_files:
+            self.addcmd(f"./scripts/kconfig/merge_config.sh -m .config {fragfile}")
         # TODO: olddefconfig should be optional/configurable
         # TODO: log all warnings/errors of olddefconfig to separate file
         self.addcmd("make olddefconfig")
@@ -635,30 +669,15 @@ def _merge_frags(self, fragnum):
 
     def _generate_script(self):
         """ Generate shell script for complete build """
-        # TODO(nuclearcat): Fetch firmware only if needed
         print("Generating shell script")
-        fragnum = self._parse_fragments(firmware=True)
-        self._merge_frags(fragnum)
-        if not self._dtbs_check:
-            # TODO: verify if CONFIG_EXTRA_FIRMWARE have any files
-            # We can check that if fragments have CONFIG_EXTRA_FIRMWARE
-            self._fetch_firmware()
-            self._build_kernel()
-            self._build_modules()
-            if self._kfselftest:
-                self._build_kselftest()
-            if self._arch not in DTBS_DISABLED:
-                self._build_dtbs()
-            self._package_kimage()
-            self._package_modules()
-            if self._coverage:
-                self._package_coverage()
-            if self._kfselftest:
-                self._package_kselftest()
-            if self._arch not in DTBS_DISABLED:
-                self._package_dtbs()
+        self._fragment_files = self._parse_fragments(firmware=True)
+
+        if self._backend == 'tuxmake':
+            self._build_with_tuxmake()
         else:
-            self._build_dtbs_check()
+            self._merge_frags(self._fragment_files)
+            self._build_with_make()
+
         self._write_metadata()
         # terminate all active jobs
         self.startjob(None)
@@ -697,6 +716,110 @@ def write_script(self, filename):
         # copy to artifacts dir
         os.system(f"cp {filename} {self._af_dir}/build.sh")
 
+    def _build_with_make(self):
+        """ Build kernel using make """
+        if not self._dtbs_check:
+            self._fetch_firmware()
+            self._build_kernel()
+            self._build_modules()
+            if self._kfselftest:
+                self._build_kselftest()
+            if self._arch not in DTBS_DISABLED:
+                self._build_dtbs()
+            self._package_kimage()
+            self._package_modules()
+            if self._coverage:
+                self._package_coverage()
+            if self._kfselftest:
+                self._package_kselftest()
+            if self._arch not in DTBS_DISABLED:
+                self._package_dtbs()
+        else:
+            self._build_dtbs_check()
+
+    def _build_with_tuxmake(self):
+        """ Build kernel using tuxmake with native fragment support """
+        print("[_build_with_tuxmake] Starting tuxmake build")
+
+        if not hasattr(self, '_fragment_files'):
+            print("[_build_with_tuxmake] ERROR: No fragment files available")
+            self._fragment_files = []
+
+        print(f"[_build_with_tuxmake] Using {len(self._fragment_files)} fragment files")
+
+        # Handle multiple defconfigs - tuxmake only supports one
+        if isinstance(self._defconfig, list):
+            if len(self._defconfig) > 1:
+                raise ValueError(
+                    f"TuxMake backend does not support multiple defconfigs: "
+                    f"{self._defconfig}. Use backend=make or specify a single defconfig."
+                )
+            defconfig = self._defconfig[0]
+            self._config_full = self._defconfig[0] + self._config_full
+        elif isinstance(self._defconfig, str):
+            defconfig = self._defconfig
+            self._config_full = self._defconfig + self._config_full
+        else:
+            defconfig = 'defconfig'
+            print("[_build_with_tuxmake] WARNING: No defconfig specified, using 'defconfig'")
+
+        # Fetch firmware only for normal builds, not dtbs_check
+        if not self._dtbs_check:
+            self._fetch_firmware()
+
+        self.startjob("build_tuxmake")
+        self.addcmd("cd " + self._srcdir)
+
+        use_kconfig_flag = True
+
+        # Handle ChromeOS defconfig
+        if defconfig.startswith('cros://'):
+            print(f"[_build_with_tuxmake] Handling ChromeOS defconfig: {defconfig}")
+            dotconfig = os.path.join(self._srcdir, ".config")
+            content, defconfig_name = self._getcrosfragment(defconfig)
+            with open(dotconfig, 'w') as f:
+                f.write(content)
+            self.addcmd("make olddefconfig")
+            use_kconfig_flag = False
+
+        cmd_parts = [
+            "tuxmake --runtime=null",
+            f"--target-arch={self._arch}",
+            f"--toolchain={self._compiler}",
+            f"--output-dir={self._af_dir}",
+        ]
+
+        if use_kconfig_flag:
+            cmd_parts.append(f"--kconfig={defconfig}")
+
+        for fragfile in self._fragment_files:
+            if os.path.exists(fragfile):
+                cmd_parts.append(f"--kconfig-add={fragfile}")
+                print(f"[_build_with_tuxmake] Adding fragment: {os.path.basename(fragfile)}")
+            else:
+                print(f"[_build_with_tuxmake] WARNING: Fragment file not found: {fragfile}")
+
+        # Build targets depend on mode
+        if self._dtbs_check:
+            # dtbs_check mode: run ONLY dtbs_check, like make backend
+            targets = ["dtbs_check"]
+        else:
+            # Normal build: kernel, modules, plus dtbs if arch supports it
+            targets = ["kernel", "modules"]
+            if self._arch not in DTBS_DISABLED:
+                targets.append("dtbs")
+            if self._kfselftest:
+                targets.append("kselftest")
+        cmd_parts.append(" ".join(targets))
+        print(f"[_build_with_tuxmake] Building targets: {' '.join(targets)}")
+
+        tuxmake_cmd = " ".join(cmd_parts)
+        print(f"[_build_with_tuxmake] Command: {tuxmake_cmd}")
+        print(f"[_build_with_tuxmake] Output directory: {self._af_dir}")
+        self.addcmd(tuxmake_cmd)
+
+        self.addcmd("cd ..")
+
     def _build_kernel(self):
         """ Add kernel build steps """
         self.startjob("build_kernel")
@@ -859,13 +982,18 @@ def _write_metadata(self):
         metadata['build']['fragments'] = self._fragments
         metadata['build']['srcdir'] = self._srcdir
         metadata['build']['config_full'] = self._config_full
+        metadata['build']['backend'] = self._backend
 
         with open(os.path.join(self._af_dir, "metadata.json"), 'w') as f:
             json.dump(metadata, f, indent=4)
 
     def serialize(self, filename):
-        """ Serialize class to json """
-        # TODO(nuclearcat): Implement to_json method?
+        """ Serialize class to json
+
+        Note: Uses __dict__ to serialize all instance attributes (including
+        _backend, _arch, etc). The from_json() method strips underscore
+        prefixes when loading, so _backend becomes 'backend' in jsonobj.
+        """
         data = json.dumps(self, default=lambda o: o.__dict__,
                           sort_keys=True, indent=4)
         with open(filename, 'w') as f:
@@ -939,9 +1067,19 @@ def upload_artifacts(self):
 
         # Prepare all artifacts for upload
         upload_tasks = []
-        for artifact in self._artifacts:
-            artifact_path = os.path.join(self._af_dir, artifact)
-            upload_tasks.append((artifact, artifact_path))
+        if self._backend == 'tuxmake':
+            # For TuxMake, upload everything in artifacts directory
+            print("[_upload_artifacts] TuxMake backend: discovering files in artifacts dir")
+            for root, dirs, files in os.walk(self._af_dir):
+                for file in files:
+                    file_rel = os.path.relpath(os.path.join(root, file), self._af_dir)
+                    artifact_path = os.path.join(self._af_dir, file_rel)
+                    upload_tasks.append((file_rel, artifact_path))
+        else:
+            # For make backend, upload only listed artifacts
+            for artifact in self._artifacts:
+                artifact_path = os.path.join(self._af_dir, artifact)
+                upload_tasks.append((artifact, artifact_path))
 
         # Function to handle a single artifact upload
         # args: (artifact, artifact_path)