Skip to content

Commit 30dafdc

Browse files
authored
Merge pull request #299 from ojeda/rust
Add basic support for building Rust
2 parents 5f6c171 + 7d293fd commit 30dafdc

File tree

9 files changed

+517
-119
lines changed

9 files changed

+517
-119
lines changed

.github/workflows/build.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ jobs:
1212
run: bash ci.sh deps
1313
- name: Build LLVM
1414
run: bash ci.sh llvm
15+
- name: Build Rust
16+
run: bash ci.sh rust
1517
- name: Build binutils
1618
run: bash ci.sh binutils
1719
- name: Build kernel

README.md

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ These scripts have been tested in a Docker image of the following distributions
3737
lld \
3838
make \
3939
ninja-build \
40+
pkg-config \
4041
python3-dev \
4142
texinfo \
4243
u-boot-tools \
@@ -184,6 +185,18 @@ bfd plugin: LLVM gold plugin has failed to create LTO module: Unknown attribute
184185

185186
Having a standalone copy of binutils (ideally in the same folder at the LLVM toolchain so that only one `PATH` modification is needed) works around this without any adverse side effects. Another workaround is bind mounting the new `LLVMgold.so` to `/usr/lib/LLVMgold.so`.
186187

188+
## build-rust.py
189+
190+
By default, `./build-rust.py` will clone Rust and build it using an LLVM previously built by `./build-llvm.py`, e.g.:
191+
192+
```sh
193+
./build-llvm.py && ./build-rust.py
194+
```
195+
196+
This script does not apply any Rust-specific patches to LLVM.
197+
198+
Run `./build-rust.py -h` for more options and information.
199+
187200
## Contributing
188201

189202
This repository openly welcomes pull requests! There are a few presubmit checks that run to make sure the code stays consistently formatted and free of bugs.

build-llvm.py

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
import tc_build.utils
1111

12-
from tc_build.llvm import LLVMBootstrapBuilder, LLVMBuilder, LLVMInstrumentedBuilder, LLVMSlimBuilder, LLVMSlimInstrumentedBuilder, LLVMSourceManager
12+
from tc_build.llvm import LLVMBootstrapBuilder, LLVMBuilder, LLVMInstrumentedBuilder, LLVMSlimBuilder, LLVMSlimInstrumentedBuilder, LLVMSourceManager, VALID_DISTRIBUTION_PROFILES
1313
from tc_build.kernel import KernelBuilder, LinuxSourceManager, LLVMKernelBuilder
1414
from tc_build.tools import HostTools, StageTools
1515

@@ -158,6 +158,28 @@
158158
159159
'''),
160160
nargs='+')
161+
parser.add_argument('--distribution-profile',
162+
help=textwrap.dedent('''\
163+
Smartly set value of LLVM_DISTRIBUTION_COMPONENTS for final build stage. Use in
164+
combination with
165+
166+
--build-targets distribution
167+
--install-targets distribution
168+
169+
to generate a smaller toolchain installation.
170+
171+
Accepts the following values:
172+
173+
none - do not set LLVM_DISTRIBUTION_COMPONENTS at all
174+
bootstrap - components needed to build LLVM itself
175+
kernel - components needed to build the Linux kernel
176+
rust - components needed to build the Rust toolchain via build-rust.py
177+
178+
The default is 'none' when '--full-toolchain' is enabled, 'kernel' if not.
179+
180+
'''),
181+
type=str,
182+
choices=VALID_DISTRIBUTION_PROFILES)
161183
parser.add_argument('-f',
162184
'--full-toolchain',
163185
help=textwrap.dedent('''\
@@ -201,7 +223,7 @@
201223
By default, the script will clone the llvm-project into the tc-build repo. If you have
202224
another LLVM checkout that you would like to work out of, pass it to this parameter.
203225
This can either be an absolute or relative path. Implies '--no-update'. When this
204-
option is supplied, '--ref' and '--use-good-revison' do nothing, as the script does
226+
option is supplied, '--ref' and '--use-good-revision' do nothing, as the script does
205227
not manipulate a repository it does not own.
206228
207229
'''),
@@ -690,6 +712,8 @@
690712
final.build_targets = args.build_targets
691713
final.check_targets = args.check_targets
692714
final.cmake_defines.update(common_cmake_defines)
715+
if args.distribution_profile:
716+
final.distribution_profile = args.distribution_profile
693717
final.folders.build = Path(build_folder, 'final')
694718
final.folders.install = Path(args.install_folder).resolve() if args.install_folder else None
695719
final.install_targets = args.install_targets

build-rust.py

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
#!/usr/bin/env python3
2+
# pylint: disable=invalid-name
3+
4+
from argparse import ArgumentParser, RawTextHelpFormatter
5+
from pathlib import Path
6+
import textwrap
7+
import time
8+
9+
import tc_build.utils
10+
11+
from tc_build.rust import RustBuilder, RustSourceManager
12+
13+
# This is a known good revision of Rust for building the kernel
14+
GOOD_REVISION = '69b3959afec9b5468d5de15133b199553f6e55d2'
15+
16+
parser = ArgumentParser(formatter_class=RawTextHelpFormatter)
17+
clone_options = parser.add_mutually_exclusive_group()
18+
19+
parser.add_argument('--debug',
20+
help=textwrap.dedent('''\
21+
Build a debug compiler and standard library. This enables debug assertions,
22+
debug logging, overflow checks and debug info. The debug assertions and overflow
23+
checks can help catch issues when compiling.
24+
25+
'''),
26+
action='store_true')
27+
parser.add_argument('-b',
28+
'--build-folder',
29+
help=textwrap.dedent('''\
30+
By default, the script will create a "build/rust" folder in the same folder as this
31+
script and build each requested stage within that containing folder. To change the
32+
location of the containing build folder, pass it to this parameter. This can be either
33+
an absolute or relative path. If it is provided, then a custom LLVM install folder
34+
needs to be provided as well to prevent mistakes.
35+
36+
'''),
37+
type=str)
38+
parser.add_argument('-i',
39+
'--install-folder',
40+
help=textwrap.dedent('''\
41+
By default, the script will leave the toolchain in its build folder. To install it
42+
outside the build folder for persistent use, pass the installation location that you
43+
desire to this parameter. This can be either an absolute or relative path.
44+
45+
'''),
46+
type=str)
47+
parser.add_argument('-l',
48+
'--llvm-install-folder',
49+
help=textwrap.dedent('''\
50+
By default, the script will try to use a built LLVM by './build-llvm.py'. To use
51+
another LLVM installation (perhaps from './build-llvm.py --install-folder'), pass
52+
it to this parameter.
53+
54+
'''),
55+
type=str)
56+
parser.add_argument('-R',
57+
'--rust-folder',
58+
help=textwrap.dedent('''\
59+
By default, the script will clone the Rust project into the tc-build repo. If you have
60+
another Rust checkout that you would like to work out of, pass it to this parameter.
61+
This can either be an absolute or relative path. Implies '--no-update'. When this
62+
option is supplied, '--ref' and '--use-good-revision' do nothing, as the script does
63+
not manipulate a repository it does not own.
64+
65+
'''),
66+
type=str)
67+
parser.add_argument('-n',
68+
'--no-update',
69+
help=textwrap.dedent('''\
70+
By default, the script always updates the Rust repo before building. This prevents
71+
that, which can be helpful during something like bisecting or manually managing the
72+
repo to pin it to a particular revision.
73+
74+
'''),
75+
action='store_true')
76+
parser.add_argument('-r',
77+
'--ref',
78+
help=textwrap.dedent('''\
79+
By default, the script builds the main branch (tip of tree) of Rust. If you would
80+
like to build an older branch, use this parameter. This may be helpful in tracking
81+
down an older bug to properly bisect. This value is just passed along to 'git checkout'
82+
so it can be a branch name, tag name, or hash. This will have no effect if
83+
'--rust-folder' is provided, as the script does not manipulate a repository that it
84+
does not own.
85+
86+
'''),
87+
default='main',
88+
type=str)
89+
parser.add_argument('--show-build-commands',
90+
help=textwrap.dedent('''\
91+
By default, the script only shows the output of the comands it is running. When this option
92+
is enabled, the invocations of the build tools will be shown to help with reproducing
93+
issues outside of the script.
94+
95+
'''),
96+
action='store_true')
97+
clone_options.add_argument('--use-good-revision',
98+
help=textwrap.dedent('''\
99+
By default, the script updates Rust to the latest tip of tree revision, which may at times be
100+
broken or not work right. With this option, it will checkout a known good revision of Rust
101+
that builds and works properly. If you use this option often, please remember to update the
102+
script as the known good revision will change. This option may work best with a matching good
103+
revision used to build LLVM by './build-llvm.py'.
104+
105+
'''),
106+
action='store_const',
107+
const=GOOD_REVISION,
108+
dest='ref')
109+
parser.add_argument('--vendor-string',
110+
help=textwrap.dedent('''\
111+
Add this value to the Rust version string (like "rustc ... (ClangBuiltLinux)"). Useful when
112+
reverting or applying patches on top of upstream Rust to differentiate a toolchain built
113+
with this script from upstream Rust or to distinguish a toolchain built with this script
114+
from the system's Rust. Defaults to ClangBuiltLinux, can be set to an empty string to
115+
override this and have no vendor in the version string.
116+
117+
'''),
118+
type=str,
119+
default='ClangBuiltLinux')
120+
args = parser.parse_args()
121+
122+
# Start tracking time that the script takes
123+
script_start = time.time()
124+
125+
# Folder validation
126+
tc_build_folder = Path(__file__).resolve().parent
127+
src_folder = Path(tc_build_folder, 'src')
128+
129+
if args.build_folder:
130+
build_folder = Path(args.build_folder).resolve()
131+
132+
if not args.llvm_install_folder:
133+
raise RuntimeError(
134+
'Build folder customized, but no custom LLVM install folder provided -- this is likely a mistake. Provide both if you want to build in a custom folder?'
135+
)
136+
else:
137+
build_folder = Path(tc_build_folder, 'build/rust')
138+
139+
if args.llvm_install_folder:
140+
llvm_install_folder = Path(args.llvm_install_folder).resolve()
141+
else:
142+
llvm_install_folder = Path(tc_build_folder, 'build/llvm/final')
143+
144+
# Validate and configure Rust source
145+
if args.rust_folder:
146+
if not (rust_folder := Path(args.rust_folder).resolve()).exists():
147+
raise RuntimeError(f"Provided Rust folder ('{args.rust_folder}') does not exist?")
148+
else:
149+
rust_folder = Path(src_folder, 'rust')
150+
rust_source = RustSourceManager(rust_folder)
151+
rust_source.download(args.ref)
152+
if not (args.rust_folder or args.no_update):
153+
rust_source.update(args.ref)
154+
155+
# Build Rust
156+
tc_build.utils.print_header('Building Rust')
157+
158+
# Final build
159+
final = RustBuilder()
160+
final.folders.source = rust_folder
161+
final.folders.build = Path(build_folder, 'final')
162+
final.folders.install = Path(args.install_folder).resolve() if args.install_folder else None
163+
final.llvm_install_folder = llvm_install_folder
164+
final.debug = args.debug
165+
final.vendor_string = args.vendor_string
166+
final.show_commands = args.show_build_commands
167+
168+
final.configure()
169+
final.build()
170+
final.show_install_info()
171+
172+
print(f"Script duration: {tc_build.utils.get_duration(script_start)}")

ci.sh

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ set -eu
99
function parse_parameters() {
1010
while (($#)); do
1111
case $1 in
12-
all | binutils | deps | kernel | llvm) action=$1 ;;
12+
all | binutils | deps | kernel | llvm | rust) action=$1 ;;
1313
*) exit 33 ;;
1414
esac
1515
shift
@@ -19,6 +19,7 @@ function parse_parameters() {
1919
function do_all() {
2020
do_deps
2121
do_llvm
22+
do_rust
2223
do_binutils
2324
do_kernel
2425
}
@@ -54,6 +55,7 @@ function do_deps() {
5455
lld \
5556
make \
5657
ninja-build \
58+
pkg-config \
5759
python3 \
5860
texinfo \
5961
xz-utils \
@@ -100,6 +102,7 @@ function do_llvm() {
100102
--build-stage1-only \
101103
--build-target distribution \
102104
--check-targets clang lld llvm \
105+
--distribution-profile rust \
103106
--install-folder "$install" \
104107
--install-target distribution \
105108
--projects clang lld \
@@ -111,5 +114,13 @@ function do_llvm() {
111114
"${extra_args[@]}"
112115
}
113116

117+
function do_rust() {
118+
"$base"/build-rust.py \
119+
--debug \
120+
--llvm-install-folder "$install" \
121+
--install-folder "$install" \
122+
--show-build-commands
123+
}
124+
114125
parse_parameters "$@"
115126
do_"${action:=all}"

tc_build/builder.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,12 @@ def clean_build_folder(self):
3232
else:
3333
self.folders.build.unlink()
3434

35+
def make_build_folder(self):
36+
if not self.folders.build:
37+
raise RuntimeError('No build folder set?')
38+
39+
self.folders.build.mkdir(parents=True)
40+
3541
def run_cmd(self, cmd, capture_output=False, cwd=None):
3642
if self.show_commands:
3743
# Acts sort of like 'set -x' in bash

0 commit comments

Comments
 (0)