Add basic support for building Rust

.github/workflows/build.yml

@@ -12,6 +12,8 @@ jobs:
       run: bash ci.sh deps
     - name: Build LLVM
       run: bash ci.sh llvm
+    - name: Build Rust
+      run: bash ci.sh rust
     - name: Build binutils
       run: bash ci.sh binutils
     - name: Build kernel

README.md

@@ -37,6 +37,7 @@ These scripts have been tested in a Docker image of the following distributions
               lld \
               make \
               ninja-build \
+              pkg-config \
               python3-dev \
               texinfo \
               u-boot-tools \
@@ -145,6 +146,19 @@ bfd plugin: LLVM gold plugin has failed to create LTO module: Unknown attribute
 
 Having a standalone copy of binutils (ideally in the same folder at the LLVM toolchain so that only one `PATH` modification is needed) works around this without any adverse side effects. Another workaround is bind mounting the new `LLVMgold.so` to `/usr/lib/LLVMgold.so`.
 
+## build-rust.py
+
+By default, `./build-rust.py` will clone Rust and build it using an LLVM previously built by `./build-llvm.py`, e.g.:
+
+```sh
+./build-llvm.py
+./build-rust.py
+```
+
+This script does not apply any Rust-specific patches to LLVM.
+
+Run `./build-rust.py -h` for more options and information.
+
 ## Contributing
 
 This repository openly welcomes pull requests! There are a few presubmit checks that run to make sure the code stays consistently formatted and free of bugs.

build-llvm.py

@@ -193,7 +193,7 @@
                     By default, the script will clone the llvm-project into the tc-build repo. If you have
                     another LLVM checkout that you would like to work out of, pass it to this parameter.
                     This can either be an absolute or relative path. Implies '--no-update'. When this
-                    option is supplied, '--ref' and '--use-good-revison' do nothing, as the script does
+                    option is supplied, '--ref' and '--use-good-revision' do nothing, as the script does
                     not manipulate a repository it does not own.
 
                     '''),

build-rust.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+# pylint: disable=invalid-name
+
+from argparse import ArgumentParser, RawTextHelpFormatter
+from pathlib import Path
+import textwrap
+import time
+
+import tc_build.utils
+
+from tc_build.rust import RustBuilder, RustSourceManager
+
+# This is a known good revision of Rust for building the kernel
+GOOD_REVISION = '69b3959afec9b5468d5de15133b199553f6e55d2'
+
+parser = ArgumentParser(formatter_class=RawTextHelpFormatter)
+clone_options = parser.add_mutually_exclusive_group()
+
+parser.add_argument('--debug',
+                    help=textwrap.dedent('''\
+                    Build a debug compiler and standard library. This enables debug assertions,
+                    debug logging, overflow checks and debug info. The debug assertions and overflow
+                    checks can help catch issues when compiling.
+
+                    '''),
+                    action='store_true')
+parser.add_argument('-b',
+                    '--build-folder',
+                    help=textwrap.dedent('''\
+                    By default, the script will create a "build/rust" folder in the same folder as this
+                    script and build each requested stage within that containing folder. To change the
+                    location of the containing build folder, pass it to this parameter. This can be either
+                    an absolute or relative path. If it is provided, then a custom LLVM install folder
+                    needs to be provided as well to prevent mistakes.
+
+                    '''),
+                    type=str)
+parser.add_argument('-i',
+                    '--install-folder',
+                    help=textwrap.dedent('''\
+                    By default, the script will leave the toolchain in its build folder. To install it
+                    outside the build folder for persistent use, pass the installation location that you
+                    desire to this parameter. This can be either an absolute or relative path.
+
+                    '''),
+                    type=str)
+parser.add_argument('-l',
+                    '--llvm-install-folder',
+                    help=textwrap.dedent('''\
+                    By default, the script will try to use a built LLVM by './build-llvm.py'. To use
+                    another LLVM installation (perhaps from './build-llvm.py --install-folder'), pass
+                    it to this parameter.
+
+                    '''),
+                    type=str)
+parser.add_argument('-R',
+                    '--rust-folder',
+                    help=textwrap.dedent('''\
+                    By default, the script will clone the Rust project into the tc-build repo. If you have
+                    another Rust checkout that you would like to work out of, pass it to this parameter.
+                    This can either be an absolute or relative path. Implies '--no-update'. When this
+                    option is supplied, '--ref' and '--use-good-revision' do nothing, as the script does
+                    not manipulate a repository it does not own.
+
+                    '''),
+                    type=str)
+parser.add_argument('-n',
+                    '--no-update',
+                    help=textwrap.dedent('''\
+                    By default, the script always updates the Rust repo before building. This prevents
+                    that, which can be helpful during something like bisecting or manually managing the
+                    repo to pin it to a particular revision.
+
+                    '''),
+                    action='store_true')
+parser.add_argument('-r',
+                    '--ref',
+                    help=textwrap.dedent('''\
+                    By default, the script builds the main branch (tip of tree) of Rust. If you would
+                    like to build an older branch, use this parameter. This may be helpful in tracking
+                    down an older bug to properly bisect. This value is just passed along to 'git checkout'
+                    so it can be a branch name, tag name, or hash. This will have no effect if
+                    '--rust-folder' is provided, as the script does not manipulate a repository that it
+                    does not own.
+
+                    '''),
+                    default='master',
+                    type=str)
+parser.add_argument('--show-build-commands',
+                    help=textwrap.dedent('''\
+                    By default, the script only shows the output of the comands it is running. When this option
+                    is enabled, the invocations of the build tools will be shown to help with reproducing
+                    issues outside of the script.
+
+                    '''),
+                    action='store_true')
+clone_options.add_argument('--use-good-revision',
+                           help=textwrap.dedent('''\
+                    By default, the script updates Rust to the latest tip of tree revision, which may at times be
+                    broken or not work right. With this option, it will checkout a known good revision of Rust
+                    that builds and works properly. If you use this option often, please remember to update the
+                    script as the known good revision will change. This option may work best with a matching good
+                    revision used to build LLVM by './build-llvm.py'.
+
+                           '''),
+                           action='store_const',
+                           const=GOOD_REVISION,
+                           dest='ref')
+parser.add_argument('--vendor-string',
+                    help=textwrap.dedent('''\
+                    Add this value to the Rust version string (like "rustc ... (ClangBuiltLinux)"). Useful when
+                    reverting or applying patches on top of upstream Rust to differentiate a toolchain built
+                    with this script from upstream Rust or to distinguish a toolchain built with this script
+                    from the system's Rust. Defaults to ClangBuiltLinux, can be set to an empty string to
+                    override this and have no vendor in the version string.
+
+                    '''),
+                    type=str,
+                    default='ClangBuiltLinux')
+args = parser.parse_args()
+
+# Start tracking time that the script takes
+script_start = time.time()
+
+# Folder validation
+tc_build_folder = Path(__file__).resolve().parent
+src_folder = Path(tc_build_folder, 'src')
+
+if args.build_folder:
+    build_folder = Path(args.build_folder).resolve()
+
+    if not args.llvm_install_folder:
+        raise RuntimeError(
+            'Build folder customized, but no custom LLVM install folder provided -- this is likely a mistake. Provide both if you want to build in a custom folder?'
+        )
+else:
+    build_folder = Path(tc_build_folder, 'build/rust')
+
+if args.llvm_install_folder:
+    llvm_install_folder = Path(args.llvm_install_folder).resolve()
+else:
+    llvm_install_folder = Path(tc_build_folder, 'build/llvm/final')
+
+# Validate and configure Rust source
+if args.rust_folder:
+    if not (rust_folder := Path(args.rust_folder).resolve()).exists():
+        raise RuntimeError(f"Provided Rust folder ('{args.rust_folder}') does not exist?")
+else:
+    rust_folder = Path(src_folder, 'rust')
+rust_source = RustSourceManager(rust_folder)
+rust_source.download(args.ref)
+if not (args.rust_folder or args.no_update):
+    rust_source.update(args.ref)
+
+# Build Rust
+tc_build.utils.print_header('Building Rust')
+
+# Final build
+final = RustBuilder()
+final.folders.source = rust_folder
+final.folders.build = Path(build_folder, 'final')
+final.folders.install = Path(args.install_folder).resolve() if args.install_folder else None
+final.llvm_install_folder = llvm_install_folder
+final.debug = args.debug
+final.vendor_string = args.vendor_string
+final.show_commands = args.show_build_commands
+
+final.configure()
+final.build()
+final.show_install_info()
+
+print(f"Script duration: {tc_build.utils.get_duration(script_start)}")

ci.sh

@@ -9,7 +9,7 @@ set -eu
 function parse_parameters() {
     while (($#)); do
         case $1 in
-            all | binutils | deps | kernel | llvm) action=$1 ;;
+            all | binutils | deps | kernel | llvm | rust) action=$1 ;;
             *) exit 33 ;;
         esac
         shift
@@ -19,6 +19,7 @@ function parse_parameters() {
 function do_all() {
     do_deps
     do_llvm
+    do_rust
     do_binutils
     do_kernel
 }
@@ -54,6 +55,7 @@ function do_deps() {
         lld \
         make \
         ninja-build \
+        pkg-config \
         python3 \
         texinfo \
         xz-utils \
@@ -111,5 +113,13 @@ function do_llvm() {
         "${extra_args[@]}"
 }
 
+function do_rust() {
+    "$base"/build-rust.py \
+        --debug \
+        --llvm-install-folder "$install" \
+        --install-folder "$install" \
+        --show-build-commands
+}
+
 parse_parameters "$@"
 do_"${action:=all}"

tc_build/builder.py

@@ -32,6 +32,12 @@ def clean_build_folder(self):
             else:
                 self.folders.build.unlink()
 
+    def make_build_folder(self):
+        if not self.folders.build:
+            raise RuntimeError('No build folder set?')
+
+        self.folders.build.mkdir(parents=True)
+
     def run_cmd(self, cmd, capture_output=False, cwd=None):
         if self.show_commands:
             # Acts sort of like 'set -x' in bash

tc_build/llvm.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-import contextlib
 import os
 from pathlib import Path
 import platform
@@ -10,6 +9,7 @@
 import time
 
 from tc_build.builder import Builder
+from tc_build.source import GitSourceManager
 import tc_build.utils
 
 LLVM_VER_FOR_RUNTIMES = 20
@@ -243,9 +243,9 @@ def configure(self):
         # disable the optimization when compiler-rt is enabled and there is an
         # install directory. Ideally thin archives should still be usable for
         # non-compiler-rt projects.
-        if not (self.folders.install and self.project_is_enabled('compiler-rt')):
-            self.cmake_defines['CMAKE_CXX_ARCHIVE_CREATE'] = '<CMAKE_AR> DqcT <TARGET> <OBJECTS>'
-        self.cmake_defines['CMAKE_CXX_ARCHIVE_FINISH'] = 'true'
+        #if not (self.folders.install and self.project_is_enabled('compiler-rt')):
+        #    self.cmake_defines['CMAKE_CXX_ARCHIVE_CREATE'] = '<CMAKE_AR> DqcT <TARGET> <OBJECTS>'
+        #self.cmake_defines['CMAKE_CXX_ARCHIVE_FINISH'] = 'true'
 
         if self.tools.ranlib:
             self.cmake_defines['CMAKE_RANLIB'] = self.tools.ranlib
@@ -430,7 +430,11 @@ def configure(self):
 
         self.set_llvm_major_version()
 
-        distribution_components = []
+        distribution_components = [
+            'llvm-config',
+            'llvm-headers',
+            'llvm-libraries',
+        ]
         runtime_distribution_components = []
         if llvm_build_tools:
             distribution_components += [
@@ -572,10 +576,13 @@ class LLVMSlimInstrumentedBuilder(LLVMInstrumentedBuilder, LLVMSlimBuilder):
     pass
 
 
-class LLVMSourceManager:
+class LLVMSourceManager(GitSourceManager):
 
     def __init__(self, repo):
-        self.repo = repo
+        super().__init__(repo)
+
+        self._pretty_name = 'LLVM'
+        self._repo_url = 'https://github.com/llvm/llvm-project.git'
 
     def default_projects(self):
         return ['clang', 'compiler-rt', 'lld', 'polly']
@@ -591,57 +598,3 @@ def default_targets(self):
             targets.append('LoongArch')
 
         return targets
-
-    def download(self, ref, shallow=False):
-        if self.repo.exists():
-            return
-
-        tc_build.utils.print_header('Downloading LLVM')
-
-        git_clone = ['git', 'clone']
-        if shallow:
-            git_clone.append('--depth=1')
-            if ref != 'main':
-                git_clone.append('--no-single-branch')
-        git_clone += ['https://github.com/llvm/llvm-project', self.repo]
-
-        subprocess.run(git_clone, check=True)
-
-        self.git(['checkout', ref])
-
-    def git(self, cmd, capture_output=False):
-        return subprocess.run(['git', *cmd],
-                              capture_output=capture_output,
-                              check=True,
-                              cwd=self.repo,
-                              text=True)
-
-    def git_capture(self, cmd):
-        return self.git(cmd, capture_output=True).stdout.strip()
-
-    def is_shallow(self):
-        git_dir = self.git_capture(['rev-parse', '--git-dir'])
-        return Path(git_dir, 'shallow').exists()
-
-    def ref_exists(self, ref):
-        try:
-            self.git(['show-branch', ref])
-        except subprocess.CalledProcessError:
-            return False
-        return True
-
-    def update(self, ref):
-        tc_build.utils.print_header('Updating LLVM')
-
-        self.git(['fetch', 'origin'])
-
-        if self.is_shallow() and not self.ref_exists(ref):
-            raise RuntimeError(f"Repo is shallow and supplied ref ('{ref}') does not exist!")
-
-        self.git(['checkout', ref])
-
-        local_ref = None
-        with contextlib.suppress(subprocess.CalledProcessError):
-            local_ref = self.git_capture(['symbolic-ref', '-q', 'HEAD'])
-        if local_ref and local_ref.startswith('refs/heads/'):
-            self.git(['pull', '--rebase', 'origin', local_ref.replace('refs/heads/', '')])