From d7a8b53e582839003f809da013b9b556e3997122 Mon Sep 17 00:00:00 2001 From: Michal R Date: Thu, 27 Nov 2025 11:14:49 +0100 Subject: [PATCH 01/11] Drop LLVM 19 support It requires supporting Rust down to 1.86.0, which excludes us from using new stabilized features like chained `if let`, `cast_signed()` or `OsStr::display()`. Rust stable from 1.87.0 to 1.90.0 uses LLVM 20, it's enough of a variety even for package maintainers who don't follow latest releases. --- .github/workflows/ci.yml | 12 ++++-------- Cargo.lock | 15 --------------- Cargo.toml | 7 ------- src/bin/bpf-linker.rs | 6 +----- src/lib.rs | 3 --- src/llvm/types/di.rs | 21 +++++++++------------ src/llvm/types/ir.rs | 6 +----- 7 files changed, 15 insertions(+), 55 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28b9bf45..17dec166 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,26 +52,22 @@ jobs: fail-fast: false matrix: include: - - rust: 1.86.0 - llvm-version: 19 - llvm-from: apt - exclude-features: default,llvm-20,llvm-21,rust-llvm-20,rust-llvm-21 - rust: 1.89.0 llvm-version: 20 llvm-from: apt - exclude-features: default,llvm-19,llvm-21,rust-llvm-19,rust-llvm-21 + exclude-features: default,llvm-21,rust-llvm-21 - rust: beta llvm-version: 21 llvm-from: apt - exclude-features: default,llvm-19,llvm-20,rust-llvm-19,rust-llvm-20 + exclude-features: default,llvm-20,rust-llvm-20 - rust: nightly llvm-version: 21 llvm-from: apt - exclude-features: llvm-19,llvm-20,rust-llvm-19,rust-llvm-20 + exclude-features: llvm-20,rust-llvm-20 - rust: nightly llvm-version: 21 llvm-from: source - exclude-features: llvm-19,llvm-20,rust-llvm-19,rust-llvm-20 + exclude-features: llvm-20,rust-llvm-20 name: rustc=${{ matrix.rust }} llvm-version=${{ matrix.llvm-version }} llvm-from=${{ matrix.llvm-from }} needs: llvm diff --git a/Cargo.lock b/Cargo.lock index 23dc38cb..76396778 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -113,7 +113,6 @@ dependencies = [ "compiletest_rs", "gimli", "libc", - "llvm-sys 191.0.0", "llvm-sys 201.0.1", "llvm-sys 211.0.0", "log", @@ -620,20 +619,6 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" -[[package]] -name = "llvm-sys" -version = "191.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "893cddf1adf0354b93411e413553dd4daf5c43195d73f1acfa1e394bdd371456" -dependencies = [ - "anyhow", - "cc", - "lazy_static", - "libc", - "regex-lite", - "semver", -] - [[package]] name = "llvm-sys" version = "201.0.1" diff --git a/Cargo.toml b/Cargo.toml index 955178f3..1fd5967f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,6 @@ ar = { version = "0.9.0" } aya-rustc-llvm-proxy = { version = "0.9.5", optional = true } gimli = { version = "0.32.0" } libc = { version = "0.2.174" } -llvm-sys-19 = { package = "llvm-sys", features = ["disable-alltargets-init"], version = "191.0.0", optional = true } llvm-sys-20 = { package = "llvm-sys", features = ["disable-alltargets-init"], version = "201.0.1", optional = true } llvm-sys-21 = { package = "llvm-sys", features = ["disable-alltargets-init"], version = "211.0.0", optional = true } log = { version = "0.4.27" } @@ -48,14 +47,8 @@ workspace = true name = "bpf-linker" [features] -llvm-19 = ["dep:llvm-sys-19"] llvm-20 = ["dep:llvm-sys-20"] llvm-21 = ["dep:llvm-sys-21"] -rust-llvm-19 = [ - "dep:aya-rustc-llvm-proxy", - "llvm-19", - "llvm-sys-19/no-llvm-linking", -] rust-llvm-20 = [ "dep:aya-rustc-llvm-proxy", "llvm-20", diff --git a/src/bin/bpf-linker.rs b/src/bin/bpf-linker.rs index d8e828b5..70263d14 100644 --- a/src/bin/bpf-linker.rs +++ b/src/bin/bpf-linker.rs @@ -8,11 +8,7 @@ use std::{ str::FromStr, }; -#[cfg(any( - feature = "rust-llvm-19", - feature = "rust-llvm-20", - feature = "rust-llvm-21" -))] +#[cfg(any(feature = "rust-llvm-20", feature = "rust-llvm-21"))] use aya_rustc_llvm_proxy as _; use bpf_linker::{Cpu, Linker, LinkerInput, LinkerOptions, OptLevel, OutputType}; use clap::{ diff --git a/src/lib.rs b/src/lib.rs index db29365f..8efab920 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,13 +30,10 @@ macro_rules! assert_unique_used_features { } assert_unique_used_features! { - "llvm-19", "llvm-20", "llvm-21" } -#[cfg(feature = "llvm-19")] -pub extern crate llvm_sys_19 as llvm_sys; #[cfg(feature = "llvm-20")] pub extern crate llvm_sys_20 as llvm_sys; #[cfg(feature = "llvm-21")] diff --git a/src/llvm/types/di.rs b/src/llvm/types/di.rs index ce0211b9..5e9dedca 100644 --- a/src/llvm/types/di.rs +++ b/src/llvm/types/di.rs @@ -151,12 +151,12 @@ impl<'ctx> From> for DIType<'ctx> { #[repr(u32)] enum DIDerivedTypeOperand { /// [`DIType`] representing a base type of the given derived type. - /// Reference in [LLVM 19-20][llvm-19] and [LLVM 21][llvm-21]. + /// Reference in [LLVM 20][llvm-20] and [LLVM 21][llvm-21]. /// - /// [llvm-19]: https://github.com/llvm/llvm-project/blob/llvmorg-19.1.7/llvm/include/llvm/IR/DebugInfoMetadata.h#L1084 + /// [llvm-20]: https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/llvm/include/llvm/IR/DebugInfoMetadata.h#L1106 /// [llvm-21]: https://github.com/llvm/llvm-project/blob/llvmorg-21.1.0-rc3/llvm/include/llvm/IR/DebugInfoMetadata.h#L1386 /// - #[cfg(any(feature = "llvm-19", feature = "llvm-20"))] + #[cfg(feature = "llvm-20")] BaseType = 3, #[cfg(feature = "llvm-21")] BaseType = 5, @@ -219,12 +219,12 @@ impl DIDerivedType<'_> { /// correspond to the operand indices within metadata nodes. #[repr(u32)] enum DICompositeTypeOperand { - /// Elements of the composite type. Reference in [LLVM 19-20][llvm-19] and + /// Elements of the composite type. Reference in [LLVM 20][llvm-20] and /// [LLVM 21][llvm-21]. /// - /// [llvm-19]: https://github.com/llvm/llvm-project/blob/llvmorg-19.1.7/llvm/include/llvm/IR/DebugInfoMetadata.h#L1299 + /// [llvm-20]: https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/llvm/include/llvm/IR/DebugInfoMetadata.h#L1332 /// [llvm-21]: https://github.com/llvm/llvm-project/blob/llvmorg-21.1.0-rc3/llvm/include/llvm/IR/DebugInfoMetadata.h#L1813 - #[cfg(any(feature = "llvm-19", feature = "llvm-20"))] + #[cfg(feature = "llvm-20")] Elements = 4, #[cfg(feature = "llvm-21")] Elements = 6, @@ -260,10 +260,6 @@ impl DICompositeType<'_> { /// Returns an iterator over elements (struct fields, enum variants, etc.) /// of the composite type. - #[expect( - clippy::cast_sign_loss, - reason = "replace as u32 with cast_unsigned when we no longer support LLVM 19" - )] pub(crate) fn elements(&self) -> impl Iterator> { let elements = unsafe { LLVMGetOperand(self.value_ref, DICompositeTypeOperand::Elements as u32) }; @@ -273,8 +269,9 @@ impl DICompositeType<'_> { unsafe { LLVMGetNumOperands(elements) } }; - (0..operands) - .map(move |i| unsafe { Metadata::from_value_ref(LLVMGetOperand(elements, i as u32)) }) + (0..operands).map(move |i| unsafe { + Metadata::from_value_ref(LLVMGetOperand(elements, i.cast_unsigned())) + }) } /// Returns the name of the composite type. diff --git a/src/llvm/types/ir.rs b/src/llvm/types/ir.rs index b790f72e..02e41a09 100644 --- a/src/llvm/types/ir.rs +++ b/src/llvm/types/ir.rs @@ -85,10 +85,6 @@ impl Value<'_> { MetadataEntries::new(value) } - #[expect( - clippy::cast_sign_loss, - reason = "replace as u32 with cast_unsigned when we no longer support LLVM 19" - )] pub(crate) fn operands(&self) -> Option> { let value = match self { Value::MDNode(node) => Some(node.value_ref), @@ -98,7 +94,7 @@ impl Value<'_> { }; value.map(|value| unsafe { - (0..LLVMGetNumOperands(value)).map(move |i| LLVMGetOperand(value, i as u32)) + (0..LLVMGetNumOperands(value)).map(move |i| LLVMGetOperand(value, i.cast_unsigned())) }) } } From 0ee605290d2cbe0b79f2dd3ce736e490bc76af37 Mon Sep 17 00:00:00 2001 From: Michal R Date: Thu, 27 Nov 2025 17:31:37 +0100 Subject: [PATCH 02/11] ci: Do not install `gcc-multilib` Current libbpf works just fine without it. Furthermore, that package is not available for non x86_64 architectures. --- .github/workflows/ci.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17dec166..bac4223d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,19 +93,15 @@ jobs: - name: Install btfdump run: cargo install btfdump - - name: Install prerequisites + - name: Add clang to PATH # ubuntu-22.04 comes with clang 13-15[0]; support for signed and 64bit # enum values was added in clang 15[1] which isn't in `$PATH`. # - # gcc-multilib provides at least which is referenced by libbpf. - # # [0] https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md # # [1] https://github.com/llvm/llvm-project/commit/dc1c43d run: | set -euxo pipefail - sudo apt update - sudo apt -y install gcc-multilib echo /usr/lib/llvm-15/bin >> $GITHUB_PATH - name: Install LLVM From 77dc51d1aad960d097fb6027a6f4c061681d2516 Mon Sep 17 00:00:00 2001 From: Michal R Date: Mon, 17 Nov 2025 09:44:42 +0100 Subject: [PATCH 03/11] ci: Do not overuse `include` To make the configuration matrix less repetitive, move the toolchain parameters directly under `matrix`, letting GitHub actions produce combinations. Use `include` only to specify the parameters of a job using LLVM from sources. That will let us to add more platforms LLVM installation sources and installation sources with less effort. --- .github/workflows/ci.yml | 48 +++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bac4223d..d3a03bee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,37 +51,38 @@ jobs: strategy: fail-fast: false matrix: - include: + toolchain: - rust: 1.89.0 - llvm-version: 20 - llvm-from: apt + llvm: 20 exclude-features: default,llvm-21,rust-llvm-21 - rust: beta - llvm-version: 21 - llvm-from: apt + llvm: 21 exclude-features: default,llvm-20,rust-llvm-20 - rust: nightly - llvm-version: 21 - llvm-from: apt + llvm: 21 exclude-features: llvm-20,rust-llvm-20 - - rust: nightly - llvm-version: 21 + llvm-from: + - packages + include: + - toolchain: + rust: nightly + llvm: 21 + exclude-features: llvm-20,rust-llvm-20 llvm-from: source - exclude-features: llvm-20,rust-llvm-20 - name: rustc=${{ matrix.rust }} llvm-version=${{ matrix.llvm-version }} llvm-from=${{ matrix.llvm-from }} + name: rustc=${{ matrix.toolchain.rust }} llvm-version=${{ matrix.toolchain.llvm }} llvm-from=${{ matrix.llvm-from }} needs: llvm env: RUST_BACKTRACE: full - LLVM_FEATURES: llvm-${{ matrix.llvm-version }},llvm-sys-${{ matrix.llvm-version }}/force-dynamic + LLVM_FEATURES: llvm-${{ matrix.toolchain.llvm }},llvm-sys-${{ matrix.llvm-version }}/force-dynamic steps: - uses: actions/checkout@v6 - - name: Install Rust ${{ matrix.rust }} + - name: Install Rust ${{ matrix.toolchain.rust }} uses: dtolnay/rust-toolchain@master with: - toolchain: ${{ matrix.rust }} + toolchain: ${{ matrix.toolchain.rust }} components: rust-src - name: Check (default features, no system LLVM) @@ -105,11 +106,11 @@ jobs: echo /usr/lib/llvm-15/bin >> $GITHUB_PATH - name: Install LLVM - if: matrix.llvm-from == 'apt' + if: matrix.llvm-from == 'packages' run: | set -euxo pipefail wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc - echo -e deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${{ matrix.llvm-version }} main | sudo tee /etc/apt/sources.list.d/llvm.list + echo -e deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${{ matrix.toolchain.llvm }} main | sudo tee /etc/apt/sources.list.d/llvm.list sudo apt update # TODO(vadorovsky): Remove the requirement of libpolly. @@ -127,6 +128,7 @@ jobs: # ditch the system LLVM option. sudo apt -y install llvm-${{ matrix.llvm-version }}-dev libpolly-${{ matrix.llvm-version }}-dev echo /usr/lib/llvm-${{ matrix.llvm-version }}/bin >> $GITHUB_PATH + echo /usr/lib/llvm-${{ matrix.toolchain.llvm }}/bin >> $GITHUB_PATH - name: Restore LLVM if: matrix.llvm-from == 'source' @@ -158,13 +160,13 @@ jobs: - name: Check run: | cargo hack check --feature-powerset \ - --exclude-features ${{ matrix.exclude-features }} \ + --exclude-features ${{ matrix.toolchain.exclude-features }} \ --features ${{ env.LLVM_FEATURES }} - name: Build run: | cargo hack build --feature-powerset \ - --exclude-features ${{ matrix.exclude-features }} \ + --exclude-features ${{ matrix.toolchain.exclude-features }} \ --features ${{ env.LLVM_FEATURES }} # Toolchains provided by rustup include standard library artifacts @@ -178,7 +180,7 @@ jobs: - name: Test (sysroot built on demand) run: | RUSTC_BOOTSTRAP=1 cargo hack test --feature-powerset \ - --exclude-features ${{ matrix.exclude-features }} \ + --exclude-features ${{ matrix.toolchain.exclude-features }} \ --features ${{ env.LLVM_FEATURES }} # To make things easier for package maintainers, the step of building a @@ -202,22 +204,22 @@ jobs: --target bpfel-unknown-none BPFEL_SYSROOT_DIR="$BPFEL_SYSROOT_DIR" cargo hack test --feature-powerset \ - --exclude-features ${{ matrix.exclude-features }} \ + --exclude-features ${{ matrix.toolchain.exclude-features }} \ --features ${{ env.LLVM_FEATURES }} - uses: actions/checkout@v6 - if: matrix.rust == 'nightly' + if: matrix.toolchain.rust == 'nightly' with: repository: aya-rs/aya path: aya submodules: recursive - name: Install - if: matrix.rust == 'nightly' + if: matrix.toolchain.rust == 'nightly' run: cargo install --path . --no-default-features --features ${{ env.LLVM_FEATURES }} - name: Run aya integration tests - if: matrix.rust == 'nightly' + if: matrix.toolchain.rust == 'nightly' working-directory: aya run: cargo xtask integration-test local From 9bd855ef80bcc41a84ab3cdcda48302bc854719a Mon Sep 17 00:00:00 2001 From: Michal R Date: Sat, 22 Nov 2025 09:43:16 +0100 Subject: [PATCH 04/11] ci: Extract disk usage report as an action, make it work on macOS It's useful for release jobs as well. Extract it to a separate action. Use `-d` instead of `--max-depth` to make it compatible with the BSD implementation of `du` that's used on macOS. Do not check the APT directory on macOS. --- .github/actions/report-disk-usage/action.yml | 30 ++++++++++++++++++++ .github/workflows/ci.yml | 22 +------------- 2 files changed, 31 insertions(+), 21 deletions(-) create mode 100644 .github/actions/report-disk-usage/action.yml diff --git a/.github/actions/report-disk-usage/action.yml b/.github/actions/report-disk-usage/action.yml new file mode 100644 index 00000000..f76405db --- /dev/null +++ b/.github/actions/report-disk-usage/action.yml @@ -0,0 +1,30 @@ +name: Report disk usage +description: Print disk usage for key directories on the runner +runs: + using: composite + steps: + - name: Report disk usage + shell: bash + run: | + set -euo pipefail + + printf "\n== Filesystems ==\n" + df -h + + report_dir() { + local dir="$1" + local depth="$2" + + [[ -d "$dir" ]] || return + + printf "\n== %s ==\n" "$dir" + sudo -n du -x -h -d "$depth" "$dir" 2>/dev/null | sort -h -r | head -n 20 + } + + report_dir "${{ github.workspace }}" 2 + report_dir "$HOME/.cargo" 2 + report_dir "$HOME/.rustup" 1 + report_dir /tmp 1 + if [[ "${RUNNER_OS:-}" == "Linux" ]]; then + report_dir /var/cache/apt/archives 1 + fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3a03bee..763eac3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -225,24 +225,4 @@ jobs: - name: Report disk usage if: ${{ always() }} - run: | - set -euo pipefail - - printf "\n== Filesystems ==\n" - df -h - - report_dir() { - local dir="$1" - local depth="$2" - - [[ -d "$dir" ]] || return - - printf "\n== %s ==\n" "$dir" - sudo -n du -x -h --max-depth="$depth" "$dir" 2>/dev/null | sort -h -r | head -n 20 - } - - report_dir "${{ github.workspace }}" 2 - report_dir "$HOME/.cargo" 2 - report_dir "$HOME/.rustup" 1 - report_dir /tmp 1 - report_dir /var/cache/apt/archives 1 + uses: ./.github/actions/report-disk-usage From fec538acc4760e2a02921b77da34873b51050579 Mon Sep 17 00:00:00 2001 From: Michal R Date: Thu, 20 Nov 2025 09:09:01 +0100 Subject: [PATCH 05/11] build: Link LLVM our own way llvm-sys' build.rs[0] uses the output of `llvm-config` binary[1] to figure out where libLLVM libraries live and which additional system libraries to link. There are several problems with that approach. The first one is the fact that the host system has to execute `llvm-config`, which is not always possible. For instance, binaries in LLVM tarballs provided by Rust community for `*-musl` targets are linked dynamically to musl, which makes it impossible to execute on distros with GNU userland. At the same time, libraries in the same tarball are linked statically to musl (and dynamically to libstdc++), which means that it's perfectly fine to use them even in GNU userland. Executing binaries also makes cross-compilation hard. Avoiding these problems by omitting the build.rs of llvm-sys was suggested by upstream[2]. The second problem is that `llvm-config` includes all system libraries that were used for all LLVM subprojects that were built together. Given that the most of package providers build the whole LLVM monorepo once and then split binary artifacts into different packages, there are dependencies we don't need, like libpolly (which gives hard time to our users, see #29). The system libraries we need are: C++ standard library and compression libraries (zlib and/or zstd, depends on the build). Furthermore, llvm-sys hard codes the system library paths, making assumptions about different operating systems[3]. We avoid that by asking the C compiler about the library paths, which is more reliable. Last but not least, llvm-sys differentiates between `force-{dynamic,static}` and `prefer-{dynamic,static}` features, giving uncertainity about the linkage. There is also no direct control over how other system libraries are linked. Our build.rs builds everything dynamically by default and honors two features: - `llvm-link-static`, triggering static linkage of libLLVM. - `llvm-deps-link-static`, triggering static linkage of libraries that libLLVM depends on (zlib, zstd). `llvm-deps-link-static` requires `llvm-link-static` - when libLLVM is linked dynamically, static linkage of dependencies makes no sense. Shared libLLVM libraries should come with `DT_NEEDED` referencing all necessary dependencies. What's great is that these entries are limited to C++ stdlib and compression libraries and libpolly is not included. [0] https://gitlab.com/taricorp/llvm-sys.rs/-/blob/e2e64ff03193cd1b548942ac71ec46baa2ad17d8/build.rs [1] https://gitlab.com/taricorp/llvm-sys.rs/-/blob/e2e64ff03193cd1b548942ac71ec46baa2ad17d8/build.rs#L224-231 [2] https://gitlab.com/taricorp/llvm-sys.rs/-/merge_requests/49#note_2130852482 [3] https://gitlab.com/taricorp/llvm-sys.rs/-/blob/e2e64ff03193cd1b548942ac71ec46baa2ad17d8/build.rs#L388-427 Fixes: #29 --- .github/workflows/ci.yml | 28 +- .github/workflows/release.yml | 2 +- Cargo.lock | 10 + Cargo.toml | 34 ++- README.md | 4 +- build.rs | 534 ++++++++++++++++++++++++++++++++++ 6 files changed, 585 insertions(+), 27 deletions(-) create mode 100644 build.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 763eac3d..c55b7640 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: components: clippy, rust-src - name: Run clippy - run: cargo clippy --features llvm-21,llvm-sys-21/no-llvm-linking --all-targets --workspace -- --deny warnings + run: cargo clippy --features llvm-21,no-llvm-linking --all-targets --workspace -- --deny warnings lint-nightly: runs-on: ubuntu-latest @@ -74,7 +74,8 @@ jobs: env: RUST_BACKTRACE: full - LLVM_FEATURES: llvm-${{ matrix.toolchain.llvm }},llvm-sys-${{ matrix.llvm-version }}/force-dynamic + LLVM_FEATURES: llvm-${{ matrix.toolchain.llvm }} + LLVM_EXCLUDE_FEATURES: llvm-link-static,no-llvm-linking steps: - uses: actions/checkout@v6 @@ -113,20 +114,7 @@ jobs: echo -e deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${{ matrix.toolchain.llvm }} main | sudo tee /etc/apt/sources.list.d/llvm.list sudo apt update - # TODO(vadorovsky): Remove the requirement of libpolly. - # - # Packages from apt.llvm.org are being built all at once, with one - # cmake build with superset of options, then different binaries and - # libraries are being included in different packages. - # - # That results in `llvm-config --libname --link-static` mentioning - # libpolly, even if it's not installed. The output of that command is - # being used in build.rs of llvm-sys, so building llvm-sys on such - # system is complaining about lack of libpolly. - # - # Hopefully that nightmare goes away once we switch to binstalls and - # ditch the system LLVM option. - sudo apt -y install llvm-${{ matrix.llvm-version }}-dev libpolly-${{ matrix.llvm-version }}-dev + sudo apt -y install llvm-${{ matrix.llvm-version }}-dev echo /usr/lib/llvm-${{ matrix.llvm-version }}/bin >> $GITHUB_PATH echo /usr/lib/llvm-${{ matrix.toolchain.llvm }}/bin >> $GITHUB_PATH @@ -160,13 +148,13 @@ jobs: - name: Check run: | cargo hack check --feature-powerset \ - --exclude-features ${{ matrix.toolchain.exclude-features }} \ + --exclude-features ${{ env.LLVM_EXCLUDE_FEATURES }},${{ matrix.toolchain.exclude-features }} \ --features ${{ env.LLVM_FEATURES }} - name: Build run: | cargo hack build --feature-powerset \ - --exclude-features ${{ matrix.toolchain.exclude-features }} \ + --exclude-features ${{ env.LLVM_EXCLUDE_FEATURES }},${{ matrix.toolchain.exclude-features }} \ --features ${{ env.LLVM_FEATURES }} # Toolchains provided by rustup include standard library artifacts @@ -180,7 +168,7 @@ jobs: - name: Test (sysroot built on demand) run: | RUSTC_BOOTSTRAP=1 cargo hack test --feature-powerset \ - --exclude-features ${{ matrix.toolchain.exclude-features }} \ + --exclude-features ${{ env.LLVM_EXCLUDE_FEATURES }},${{ matrix.toolchain.exclude-features }} \ --features ${{ env.LLVM_FEATURES }} # To make things easier for package maintainers, the step of building a @@ -204,7 +192,7 @@ jobs: --target bpfel-unknown-none BPFEL_SYSROOT_DIR="$BPFEL_SYSROOT_DIR" cargo hack test --feature-powerset \ - --exclude-features ${{ matrix.toolchain.exclude-features }} \ + --exclude-features ${{ env.LLVM_EXCLUDE_FEATURES }},${{ matrix.toolchain.exclude-features }} \ --features ${{ env.LLVM_FEATURES }} - uses: actions/checkout@v6 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3472eefc..5fd02665 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,6 +33,6 @@ jobs: - uses: taiki-e/upload-rust-binary-action@v1 with: bin: bpf-linker - features: llvm-sys/force-static + features: llvm-link-static env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Cargo.lock b/Cargo.lock index 76396778..1842aeb8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,6 +116,7 @@ dependencies = [ "llvm-sys 201.0.1", "llvm-sys 211.0.0", "log", + "object", "regex", "rustc-build-sysroot", "thiserror 2.0.17", @@ -711,6 +712,15 @@ dependencies = [ "libc", ] +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.21.3" diff --git a/Cargo.toml b/Cargo.toml index 1fd5967f..f7710238 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,12 +28,28 @@ ar = { version = "0.9.0" } aya-rustc-llvm-proxy = { version = "0.9.5", optional = true } gimli = { version = "0.32.0" } libc = { version = "0.2.174" } -llvm-sys-20 = { package = "llvm-sys", features = ["disable-alltargets-init"], version = "201.0.1", optional = true } -llvm-sys-21 = { package = "llvm-sys", features = ["disable-alltargets-init"], version = "211.0.0", optional = true } +llvm-sys-20 = { package = "llvm-sys", features = [ + "disable-alltargets-init", + "no-llvm-linking", +], version = "201.0.1", optional = true } +llvm-sys-21 = { package = "llvm-sys", features = [ + "disable-alltargets-init", + "no-llvm-linking", +], version = "211.0.0", optional = true } log = { version = "0.4.27" } thiserror = { version = "2.0.12" } tracing = "0.1" +[build-dependencies] +anyhow = { workspace = true } +object = { version = "0.37", default-features = false, features = [ + "archive", + "elf", + "macho", + "read_core", + "unaligned", +] } + [dev-dependencies] compiletest_rs = { version = "0.11.0" } regex = { version = "1.11.1", default-features = false } @@ -49,16 +65,26 @@ name = "bpf-linker" [features] llvm-20 = ["dep:llvm-sys-20"] llvm-21 = ["dep:llvm-sys-21"] + +# Use libLLVM shared library provided by Rust. Works only on +# x86_64-unknown-linux-gnu. rust-llvm-20 = [ "dep:aya-rustc-llvm-proxy", "llvm-20", - "llvm-sys-20/no-llvm-linking", + "no-llvm-linking", ] rust-llvm-21 = [ "dep:aya-rustc-llvm-proxy", "llvm-21", - "llvm-sys-21/no-llvm-linking", + "no-llvm-linking", ] + +# Link LLVM and its dependencies statically. When not enabled, they're linked +# dynamically. +llvm-link-static = [] +# Skip LLVM linking. +no-llvm-linking = [] + default = [ "llvm-21", "rust-llvm-21", diff --git a/README.md b/README.md index 64b9dd74..07e406d2 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,8 @@ cargo install bpf-linker #### System packages -On Debian based distributions you need to install the `llvm-21-dev`, `libclang-21-dev` -and `libpolly-21-dev` packages, from the official LLVM repo at https://apt.llvm.org. +On Debian based distributions you need to install the `llvm-21-dev` and `libclang-21-dev` +packages, from the official LLVM repo at https://apt.llvm.org. You may need to build LLVM from source if a recent version is not available through any package manager that supports your platform. diff --git a/build.rs b/build.rs new file mode 100644 index 00000000..a7366c59 --- /dev/null +++ b/build.rs @@ -0,0 +1,534 @@ +use std::{ + borrow::Cow, + env, + ffi::{OsStr, OsString}, + fmt::Display, + fs, + io::{self, Write as _}, + iter, + os::unix::ffi::OsStrExt as _, + path::{Path, PathBuf}, + process::Command, +}; + +use anyhow::{Context as _, anyhow}; +use object::{Object as _, ObjectSymbol as _, read::archive::ArchiveFile}; + +macro_rules! write_bytes { + ($dst:expr, $($bytes:expr),* $(,)?) => { + { + let result = (|| -> anyhow::Result<()> { + $( + $dst.write_all($bytes.as_ref()).with_context(|| { + format!( + "failed to write bytes to stdout: {}", + OsStr::from_bytes($bytes.as_ref()).display() + ) + })?; + )* + Ok(()) + })(); + + result + } + }; +} + +enum Cxxstdlibs<'a> { + EnvVar(OsString), + Single(&'static [u8]), + Multiple(&'a [&'static [u8]]), +} + +impl Cxxstdlibs<'_> { + /// Detects which standard C++ library to link. + fn new() -> Self { + match env::var_os("CXXSTDLIB") { + Some(cxxstdlib) => Self::EnvVar(cxxstdlib), + None => { + if cfg!(target_os = "linux") { + // Default to GNU libstdc++ on Linux. Can be overwritten through + // `CXXSTDLIB` variable on distributions using LLVM as default + // toolchain. + Self::Single(b"stdc++") + } else if cfg!(target_os = "macos") { + // Default to LLVM libc++ on macOS, where LLVM is the default + // toolchain. + if cfg!(feature = "llvm-link-static") { + // Static LLVM libc++ has two files - libc++.a and libc++abi.a. + Self::Multiple(&[b"c++", b"c++abi"]) + } else { + // Shared LLVM libc++ has one file. + Self::Single(b"c++") + } + } else { + // Fall back to GNU libstdc++ on all other platforms. Again, + // can be overwritten through `CXXSTDLIB`. + Self::Single(b"stdc++") + } + } + } + } + + fn iter(&self) -> impl Iterator { + match self { + Self::EnvVar(p) => CxxstdlibsIter::Parsed(p.as_bytes().split(|b| *b == b',')), + Self::Single(s) => { + CxxstdlibsIter::Single(iter::once( + // Coerce `&&[u8]` to `&[u8]`. + *s, + )) + } + Self::Multiple(m) => CxxstdlibsIter::Multiple( + m.iter() + // Coerce `&&[u8]` to `&[u8]`. + .copied(), + ), + } + } + + fn iter_static_filenames(&self) -> impl Iterator { + self.iter().map(|lib| { + let mut filename = OsString::from("lib"); + filename.push(OsStr::from_bytes(lib)); + filename.push(".a"); + filename + }) + } +} + +impl Display for Cxxstdlibs<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut libs = self.iter(); + if let Some(first) = libs.next() { + OsStr::from_bytes(first).display().fmt(f)?; + } + for lib in libs { + write!(f, ", ")?; + OsStr::from_bytes(lib).display().fmt(f)?; + } + Ok(()) + } +} + +enum CxxstdlibsIter { + Parsed(P), + Single(S), + Multiple(M), +} + +impl<'a, P, S, M> Iterator for CxxstdlibsIter +where + P: Iterator, + S: Iterator, + M: Iterator, +{ + type Item = &'a [u8]; + + fn next(&mut self) -> Option { + match self { + Self::Parsed(p) => p.next(), + Self::Single(s) => s.next(), + Self::Multiple(m) => m.next(), + } + } +} + +/// Checks whether the given environment variable `env_var` exists and if yes, +/// emits its content as a search path for the linker. +/// +/// Returns a boolean indicating whether the variable was found +fn emit_search_path_if_defined( + stdout: &mut io::StdoutLock<'_>, + env_var: &str, +) -> anyhow::Result { + match env::var_os(env_var) { + Some(path) => { + write_bytes!( + stdout, + "cargo:rustc-link-search=", + path.as_os_str().as_bytes(), + "\n", + )?; + Ok(true) + } + None => Ok(false), + } +} + +/// Points cargo to static library names of LLVM and dependencies needed by +/// LLVM. +/// +/// That includes figuring out which libraries LLVM depends on: +/// +/// - Standard C++ library (GNU stdc++ or LLVM libc++). +/// - zlib (optional). +/// - zstd (optional). +/// +/// It's necessary, because static libraries have no equivalent of `DT_NEEDED` +/// entries. They come with undefined symbols that must be filled in by other +/// libraries at link time. Since static archives do not explicitly express +/// which additional libraries are required, we have to determine that set +/// ourselves using the undefined symbols, and instruct Cargo to link them. +fn link_llvm_static(stdout: &mut io::StdoutLock<'_>, llvm_lib_dir: &Path) -> anyhow::Result<()> { + // Link the library files found inside the directory. + let dir_entries = fs::read_dir(llvm_lib_dir) + .with_context(|| format!("failed to read directory {}", llvm_lib_dir.display()))?; + for entry in dir_entries { + let entry = entry.with_context(|| { + format!( + "failed to read entry of the directory {}", + llvm_lib_dir.display() + ) + })?; + let file_name = entry.file_name(); + let file_name = file_name.as_bytes(); + let Some(trimmed) = file_name + .strip_prefix(b"libLLVM") + .and_then(|name| name.strip_suffix(b".a")) + else { + continue; + }; + + write_bytes!(stdout, "cargo:rustc-link-lib=static=LLVM", trimmed, "\n")?; + } + + // Static libraries have no metadata indicating a dependency on other + // libraries. Given that zlib and zstd might or might be not enabled in + // different LLVM builds, check whether libLLVMSupport references their + // symbols. + let (mut needs_zlib, mut needs_zstd) = (false, false); + let llvm_support = llvm_lib_dir.join("libLLVMSupport.a"); + let data = fs::read(&llvm_support) + .with_context(|| format!("failed to read library {}", llvm_support.display()))?; + let archive = ArchiveFile::parse(data.as_slice()) + .with_context(|| format!("failed to parse library archive {}", llvm_support.display()))?; + 'outer: for member in archive.members() { + let member = member.with_context(|| { + format!( + "failed to process the member of library archive {}", + llvm_support.display() + ) + })?; + let member_data = member.data(data.as_slice()).with_context(|| { + format!( + "failed to read data of static library archive member {}", + OsStr::from_bytes(member.name()).display() + ) + })?; + let obj = object::File::parse(member_data).with_context(|| { + format!( + "failed to parse data of static library archive member {} as object file", + OsStr::from_bytes(member.name()).display() + ) + })?; + for symbol in obj.symbols() { + if symbol.is_undefined() { + let sym_name = symbol.name().with_context(|| { + format!( + "failed to retrieve the symbol name in static library archive member {}", + OsStr::from_bytes(member.name()).display() + ) + })?; + if sym_name.contains("crc32") { + needs_zlib = true; + } else if sym_name.contains("ZSTD") { + needs_zstd = true; + } + if needs_zlib && needs_zstd { + break 'outer; + } + } + } + } + + let cxxstdlibs = Cxxstdlibs::new(); + + // Find directories with static libraries we're interested in: + // - C++ standard library + // - zlib (if needed) + // - zstd (if needed) + + // Check whether custom paths were provided. If yes, point the linker to + // them. + let cxxstdlib_found = emit_search_path_if_defined(stdout, "CXXSTDLIB_PATH")?; + let zlib_found = if needs_zlib { + emit_search_path_if_defined(stdout, "ZLIB_PATH")? + } else { + false + }; + let zstd_found = if needs_zstd { + emit_search_path_if_defined(stdout, "LIBZSTD_PATH")? + } else { + false + }; + + if !cxxstdlib_found || (needs_zlib && !zlib_found) || (needs_zstd && !zstd_found) { + // Unfortunately, Rust/cargo don't look for static libraries in system + // directories, like C compilers do, so we had to implement the logic + // of searching for them ourselves. + + // Use C compiler to retrieve the system library paths. + let cc = match env::var_os("CC") { + Some(cc) => Cow::Owned(cc), + None => Cow::Borrowed(OsStr::from_bytes(b"cc")), + }; + let cc_output = Command::new(&cc) + .arg("-print-search-dirs") + .output() + .with_context(|| format!("failed to run `{} -print-search-dirs`", cc.display()))?; + if !cc_output.status.success() { + anyhow::bail!( + "`{} -print-search-dirs` failed with status: {}", + cc.display(), + cc_output.status + ); + } + let cc_stdout = cc_output.stdout; + let ld_paths = cc_stdout + .split(|&b| b == b'\n') + .find_map(|line| line.strip_prefix(b"libraries: =")) + .ok_or_else(|| { + anyhow!( + "failed to find library paths in the output of `{} -print-search-dirs`: {}", + cc.display(), + OsStr::from_bytes(&cc_stdout).display() + ) + })?; + let ld_paths = OsStr::from_bytes(ld_paths); + + // Find directories with static libraries we're interested in: + // - C++ standard library + // - zlib (if needed) + // - zstd (if needed) + let mut cxxstdlib_paths = if !cxxstdlib_found { + Some(Vec::new()) + } else { + None + }; + let mut zlib_paths = if needs_zlib && !zlib_found { + Some(Vec::new()) + } else { + None + }; + let mut zstd_paths = if needs_zstd && !zstd_found { + Some(Vec::new()) + } else { + None + }; + for ld_path in env::split_paths(ld_paths) { + let mut found_any = false; + if !cxxstdlib_found && let Some(ref mut cxxstdlib_paths) = cxxstdlib_paths { + for cxxstdlib in cxxstdlibs.iter_static_filenames() { + let cxxstdlib_path = ld_path.join(cxxstdlib); + if cxxstdlib_path.try_exists().with_context(|| { + format!("failed to inspect the file {}", cxxstdlib_path.display(),) + })? { + cxxstdlib_paths.push(cxxstdlib_path); + found_any = true; + } + } + } + if needs_zlib + && !zlib_found + && let Some(ref mut zlib_paths) = zlib_paths + { + let zlib_path = ld_path.join("libz.a"); + if zlib_path.try_exists().with_context(|| { + format!("failed to inspect the file {}", zlib_path.display()) + })? { + zlib_paths.push(zlib_path); + found_any = true; + } + } + if needs_zstd + && !zstd_found + && let Some(ref mut zstd_paths) = zstd_paths + { + let zstd_path = ld_path.join("libzstd.a"); + if zstd_path.try_exists().with_context(|| { + format!("failed to inspect the file {}", zstd_path.display()) + })? { + zstd_paths.push(zstd_path); + found_any = true; + } + } + if found_any { + write_bytes!( + io::stdout(), + "cargo:rustc-link-search=", + ld_path.as_os_str().as_bytes(), + "\n" + )?; + } + } + + fn check_library( + stdout: &mut io::StdoutLock<'_>, + ld_paths: &OsStr, + library: S, + paths: Option>, + ) -> anyhow::Result<()> { + if let Some(paths) = paths { + match paths.as_slice() { + [] => { + anyhow::bail!( + "could not find {library} in any of the following directories: {}", + ld_paths.display() + ); + } + [_] => {} + paths => { + write!( + stdout, + "cargo:warning={library} was found in multiple locations: " + )?; + let mut paths = paths.iter(); + if let Some(first) = paths.next() { + write_bytes!(stdout, first.as_os_str().as_bytes())?; + } + for path in paths { + write_bytes!(stdout, ", ", path.as_os_str().as_bytes())?; + } + write_bytes!(stdout, "\n")?; + } + } + } + Ok(()) + } + check_library(stdout, ld_paths, &cxxstdlibs, cxxstdlib_paths)?; + check_library(stdout, ld_paths, "libz.a", zlib_paths)?; + check_library(stdout, ld_paths, "libzstd.a", zstd_paths)?; + } + + for cxxstdlib in cxxstdlibs.iter() { + write_bytes!(stdout, "cargo:rustc-link-lib=static=", cxxstdlib, "\n")?; + } + if needs_zlib { + write_bytes!(stdout, "cargo:rustc-link-lib=static=z\n")?; + } + if needs_zstd { + write_bytes!(stdout, "cargo:rustc-link-lib=static=zstd\n")?; + } + + Ok(()) +} + +/// Points cargo to shared library name of LLVM. +/// +/// Unlike [`link_llvm_static`], it does not require explicit search for +/// dependencies, since shared libraries contain `DT_NEEDED` entries that +/// specify the names of libaries that the dynamic linker should link +/// beforehand. +fn link_llvm_dynamic(stdout: &mut io::StdoutLock<'_>, llvm_lib_dir: &Path) -> anyhow::Result<()> { + #[cfg(target_os = "macos")] + const DYLIB_EXT: &[u8] = b".dylib"; + #[cfg(not(target_os = "macos"))] + const DYLIB_EXT: &[u8] = b".so"; + + let dir_entries = fs::read_dir(llvm_lib_dir).with_context(|| { + format!( + "failed to read entry of the directory {}", + llvm_lib_dir.display() + ) + })?; + let mut found = false; + for entry in dir_entries { + let entry = entry.with_context(|| { + format!( + "failed to read entry of the directory {}", + llvm_lib_dir.display() + ) + })?; + let file_name = entry.file_name(); + let file_name = file_name.as_bytes(); + if let Some(trimmed) = file_name + .strip_prefix(b"libLLVM") + .and_then(|name| name.strip_suffix(DYLIB_EXT)) + { + write_bytes!(stdout, "cargo:rustc-link-lib=dylib=LLVM", trimmed, "\n")?; + found = true; + break; + }; + } + if !found { + anyhow::bail!( + "could not find dynamic libLLVM in the directory {}", + llvm_lib_dir.display() + ); + } + + Ok(()) +} + +/// Points cargo to the path containing LLVM libraries and to the LLVM library +/// files. +fn main() -> anyhow::Result<()> { + let link_fn = if cfg!(feature = "no-llvm-linking") { + if cfg!(feature = "llvm-link-static") { + anyhow::bail!("`no-llvm-linking` and `llvm-link-static` are mutually exclusive"); + } + return Ok(()); + } else if cfg!(feature = "llvm-link-static") { + link_llvm_static + } else { + link_llvm_dynamic + }; + + let stdout = io::stdout(); + let mut stdout = stdout.lock(); + + // If `LLVM_PREFIX` variable is not provided, find the directory with LLVM + // libraries by assuming it's `lib/` inside a prefix where `llvm-config` + // lives. + const LLVM_PREFIX: &str = "LLVM_PREFIX"; + const PATH: &str = "PATH"; + let (var_name, paths_os) = env::var_os(LLVM_PREFIX) + .map(|mut p| { + p.push("/bin"); + (LLVM_PREFIX, p) + }) + .or_else(|| env::var_os(PATH).map(|p| (PATH, p))) + .ok_or_else(|| anyhow!("neither {LLVM_PREFIX} nor {PATH} is set"))?; + let llvm_config = env::split_paths(&paths_os) + .find_map(|dir| { + let candidate = Path::new(&dir).join("llvm-config"); + candidate + .try_exists() + .with_context(|| format!("failed to inspect the file {}", candidate.display())) + .map(|exists| exists.then_some(candidate)) + .transpose() + }) + .transpose()? + .ok_or_else(|| { + anyhow!( + "could not find llvm-config in directories specified by environment +variable `{var_name}` {}", + paths_os.display() + ) + })?; + let llvm_lib_dir = llvm_config + .parent() + .and_then(|p| p.parent()) + .ok_or_else(|| { + anyhow!( + "llvm-config location has no parent: {}", + llvm_config.display() + ) + })? + .join("lib"); + let llvm_lib_dir = fs::canonicalize(&llvm_lib_dir).with_context(|| { + format!( + "failed to canonicalize LLVM lib directory {}", + llvm_lib_dir.display() + ) + })?; + write_bytes!( + stdout, + b"cargo:rustc-link-search=", + llvm_lib_dir.as_os_str().as_bytes(), + "\n", + )?; + + link_fn(&mut stdout, &llvm_lib_dir) +} From 0501b6c14eaf0c09764b21175c1ef48e920129f0 Mon Sep 17 00:00:00 2001 From: Michal R Date: Tue, 18 Nov 2025 08:47:40 +0100 Subject: [PATCH 06/11] ci: Bump Rust versions to 1.90.0 and stable (currently 1.91.0) 1.90.0 is the latest version that uses LLVM 20. Use it instead of 1.89.0. Always use the latest stable, to be able to catch updates in CI. --- .github/workflows/ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c55b7640..2d87dd92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,9 +52,12 @@ jobs: fail-fast: false matrix: toolchain: - - rust: 1.89.0 + - rust: 1.90.0 llvm: 20 - exclude-features: default,llvm-21,rust-llvm-21 + exclude-features: default,llvm-19,llvm-21,rust-llvm-19,rust-llvm-21 + - rust: stable + llvm: 21 + exclude-features: default,llvm-19,llvm-20,rust-llvm-19,rust-llvm-20 - rust: beta llvm: 21 exclude-features: default,llvm-20,rust-llvm-20 From 79c97284cdc3f0602ad3771d7110eaf7c63542a5 Mon Sep 17 00:00:00 2001 From: Michal R Date: Fri, 14 Nov 2025 16:22:26 +0100 Subject: [PATCH 07/11] ci: Run on Linux arm64 --- .github/workflows/ci.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d87dd92..770331ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: build: # We don't use ubuntu-latest because we care about the apt packages available. - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.platform.os }} strategy: fail-fast: false matrix: @@ -64,15 +64,21 @@ jobs: - rust: nightly llvm: 21 exclude-features: llvm-20,rust-llvm-20 + platform: + - os: ubuntu-22.04 + - os: ubuntu-22.04-arm llvm-from: - packages include: + # Currently we build LLVM from source only for Linux x86_64. - toolchain: rust: nightly llvm: 21 exclude-features: llvm-20,rust-llvm-20 + platform: + os: ubuntu-22.04 llvm-from: source - name: rustc=${{ matrix.toolchain.rust }} llvm-version=${{ matrix.toolchain.llvm }} llvm-from=${{ matrix.llvm-from }} + name: os=${{ matrix.platform.os }} rustc=${{ matrix.toolchain.rust }} llvm-version=${{ matrix.toolchain.llvm }} llvm-from=${{ matrix.llvm-from }} needs: llvm env: From cb5bda0a1a4a8f9f6f83f7008773089789eeb7dd Mon Sep 17 00:00:00 2001 From: Michal R Date: Mon, 17 Nov 2025 10:06:18 +0100 Subject: [PATCH 08/11] ci: Run on macOS (ARM, Intel) --- .github/workflows/ci.yml | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 770331ac..0dab2388 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,8 @@ jobs: llvm: 21 exclude-features: llvm-20,rust-llvm-20 platform: + - os: macos-latest + - os: macos-15-intel - os: ubuntu-22.04 - os: ubuntu-22.04-arm llvm-from: @@ -105,6 +107,7 @@ jobs: run: cargo install btfdump - name: Add clang to PATH + if: runner.os == 'Linux' # ubuntu-22.04 comes with clang 13-15[0]; support for signed and 64bit # enum values was added in clang 15[1] which isn't in `$PATH`. # @@ -115,8 +118,8 @@ jobs: set -euxo pipefail echo /usr/lib/llvm-15/bin >> $GITHUB_PATH - - name: Install LLVM - if: matrix.llvm-from == 'packages' + - name: Install LLVM (Linux, packages) + if: matrix.llvm-from == 'packages' && runner.os == 'Linux' run: | set -euxo pipefail wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc @@ -127,6 +130,14 @@ jobs: echo /usr/lib/llvm-${{ matrix.llvm-version }}/bin >> $GITHUB_PATH echo /usr/lib/llvm-${{ matrix.toolchain.llvm }}/bin >> $GITHUB_PATH + - name: Install LLVM (macOS, packages) + if: matrix.llvm-from == 'packages' && runner.os == 'macOS' + run: | + set -euxo pipefail + brew install llvm@${{ matrix.toolchain.llvm }} + echo $(brew --prefix llvm@${{ matrix.toolchain.llvm }})/bin >> $GITHUB_PATH + echo "DYLD_LIBRARY_PATH=$(brew --prefix llvm@${{ matrix.toolchain.llvm }})/lib" >> $GITHUB_ENV + - name: Restore LLVM if: matrix.llvm-from == 'source' uses: actions/cache/restore@v4 @@ -205,18 +216,18 @@ jobs: --features ${{ env.LLVM_FEATURES }} - uses: actions/checkout@v6 - if: matrix.toolchain.rust == 'nightly' + if: runner.os == 'Linux' && matrix.toolchain.rust == 'nightly' with: repository: aya-rs/aya path: aya submodules: recursive - name: Install - if: matrix.toolchain.rust == 'nightly' + if: runner.os == 'Linux' && matrix.toolchain.rust == 'nightly' run: cargo install --path . --no-default-features --features ${{ env.LLVM_FEATURES }} - name: Run aya integration tests - if: matrix.toolchain.rust == 'nightly' + if: runner.os == 'Linux' && matrix.toolchain.rust == 'nightly' working-directory: aya run: cargo xtask integration-test local From 43c8ac31abdbf8986ae6f0ffeeceab29ad792b55 Mon Sep 17 00:00:00 2001 From: Michal R Date: Wed, 19 Nov 2025 17:55:10 +0100 Subject: [PATCH 09/11] xtask: Add `rustc-llvm-commit` subcommand Add a subcommand that prints the github.com/rust-lang/rust commit from master that corresponds to an LLVM update for the current Rust toolchain. The goal is to make downloading LLVM artifacts from Rust CI easier. --- Cargo.lock | 730 +++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 4 +- xtask/Cargo.toml | 2 + xtask/src/main.rs | 120 +++++++- 4 files changed, 848 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1842aeb8..95e5bf02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -73,6 +73,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d67af77d68a931ecd5cbd8a3b5987d63a1d1d1278f7f6a60ae33db485cdebb69" +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.0" @@ -96,6 +102,12 @@ dependencies = [ "syn", ] +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitflags" version = "2.9.1" @@ -127,6 +139,18 @@ dependencies = [ "which", ] +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + +[[package]] +name = "bytes" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" + [[package]] name = "camino" version = "1.1.10" @@ -191,6 +215,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "clap" version = "4.5.53" @@ -349,7 +379,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -376,6 +406,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -385,6 +421,56 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "getopts" version = "0.2.23" @@ -401,8 +487,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.1+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -412,9 +500,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", ] [[package]] @@ -446,6 +536,108 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08" +[[package]] +name = "http" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e9a2a24dc5c6821e71a7030e1e14b7b632acac55c40e9d2e082c621261bb56" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + [[package]] name = "icu_collections" version = "2.0.0" @@ -563,6 +755,22 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -575,6 +783,16 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "js-sys" +version = "0.3.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b011eec8cc36da2aab2d5cff675ec18454fad408585853910a202391cf9f8e65" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -654,6 +872,12 @@ version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "matchers" version = "0.2.0" @@ -669,6 +893,17 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "mio" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.61.2", +] + [[package]] name = "miow" version = "0.6.0" @@ -754,6 +989,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "potential_utf" version = "0.1.2" @@ -769,6 +1010,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "prettyplease" version = "0.2.33" @@ -788,6 +1038,61 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.17", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +dependencies = [ + "bytes", + "getrandom 0.3.3", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.17", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.40" @@ -803,6 +1108,35 @@ version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.3", +] + [[package]] name = "redox_syscall" version = "0.5.12" @@ -858,6 +1192,60 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "reqwest" +version = "0.12.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rustc-build-sysroot" version = "0.5.11" @@ -871,6 +1259,12 @@ dependencies = [ "walkdir", ] +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + [[package]] name = "rustc_version" version = "0.4.1" @@ -905,6 +1299,41 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "rustls" +version = "0.23.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94182ad936a0c91c324cd46c6511b9510ed16af436d7b5bab34beab0afd55f7a" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.21" @@ -937,10 +1366,11 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ + "serde_core", "serde_derive", ] @@ -965,11 +1395,20 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + [[package]] name = "serde_derive" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", @@ -997,6 +1436,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1012,12 +1463,28 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + [[package]] name = "smallvec" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "socket2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -1030,6 +1497,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.102" @@ -1041,6 +1514,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -1180,6 +1662,45 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "toml" version = "0.8.23" @@ -1222,6 +1743,51 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.41" @@ -1307,6 +1873,12 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typeid" version = "1.0.3" @@ -1331,6 +1903,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.7" @@ -1371,6 +1949,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -1386,6 +1973,93 @@ dependencies = [ "wit-bindgen-rt", ] +[[package]] +name = "wasm-bindgen" +version = "0.2.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "551f88106c6d5e7ccc7cd9a16f312dd3b5d36ea8b4954304657d5dfba115d4a0" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04264334509e04a7bf8690f2384ef5265f05143a4bff3889ab7a3269adab59c2" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "420bc339d9f322e562942d52e115d57e950d12d88983a14c79b86859ee6c7ebc" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76f218a38c84bcb33c25ec7059b07847d465ce0e0a76b995e134a45adcb6af76" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a1f95c0d03a47f4ae1f7a64643a6bb97465d9b740f0fa8f90ea33915c99a9a1" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "which" version = "8.0.0" @@ -1462,6 +2136,24 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -1684,7 +2376,9 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", + "reqwest", "rustc-build-sysroot", + "serde", "walkdir", ] @@ -1712,6 +2406,26 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zerofrom" version = "0.1.6" @@ -1733,6 +2447,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + [[package]] name = "zerotrie" version = "0.2.2" diff --git a/Cargo.toml b/Cargo.toml index f7710238..699def2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -99,9 +99,11 @@ edition = "2024" [workspace.dependencies] # cli deps anyhow = { version = "1.0.100", default-features = false } -clap = { version = "4.5.53", features = ["derive"] } +clap = { version = "4.5.53", features = ["derive", "env"] } # dev deps +reqwest = { version = "0.12.24", default-features = false } rustc-build-sysroot = { version = "0.5.11", default-features = false } +serde = { version = "1.0.228", default-features = false } walkdir = { version = "2.5.0", default-features = false } [workspace.lints.clippy] diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index e3210b0e..e0b72139 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -6,7 +6,9 @@ edition.workspace = true [dependencies] anyhow = { workspace = true } clap = { workspace = true } +reqwest = { workspace = true, features = ["blocking", "json", "rustls-tls"] } rustc-build-sysroot = { workspace = true } +serde = { workspace = true, features = ["derive"] } walkdir = { workspace = true } [lints] diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 7d77c00f..7a882ef3 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,7 +1,12 @@ -use std::{ffi::OsString, fs, path::PathBuf, process::Command}; +use std::{env, ffi::OsString, fs, path::PathBuf, process::Command}; -use anyhow::{Context as _, Result}; +use anyhow::{Context as _, Result, anyhow}; +use reqwest::{ + blocking::Client, + header::{ACCEPT, AUTHORIZATION, HeaderMap, USER_AGENT}, +}; use rustc_build_sysroot::{BuildMode, SysrootConfig, SysrootStatus}; +use serde::Deserialize; use walkdir::WalkDir; #[derive(Clone, clap::ValueEnum)] @@ -44,6 +49,13 @@ struct BuildLlvm { install_prefix: PathBuf, } +#[derive(clap::Args)] +struct RustcLlvmCommitOptions { + /// GitHub token used for API requests. Reads from `GITHUB_TOKEN` when unset. + #[arg(long = "github-token", env = "GITHUB_TOKEN")] + github_token: String, +} + #[derive(clap::Subcommand)] enum XtaskSubcommand { /// Builds the Rust standard library for the given target in the current @@ -51,6 +63,9 @@ enum XtaskSubcommand { BuildStd(BuildStd), /// Manages and builds LLVM. BuildLlvm(BuildLlvm), + /// Finds the commit in github.com/rust-lang/rust that can be used for + /// downloading LLVM for the current Rust toolchain. + RustcLlvmCommit(RustcLlvmCommitOptions), } /// Additional build commands for bpf-linker. @@ -180,10 +195,111 @@ fn build_llvm(options: BuildLlvm) -> Result<()> { Ok(()) } +#[derive(Deserialize)] +struct SearchIssuesResponse { + items: Vec, +} + +#[derive(Deserialize)] +struct IssueItem { + number: u64, + title: String, +} + +#[derive(Deserialize)] +struct PullRequest { + merge_commit_sha: Option, +} + +/// Returns the LLVM version used by the current Rust toolchain. +fn get_llvm_version(toolchain: Option) -> Result { + let mut rustc_cmd = Command::new("rustc"); + if let Some(toolchain) = toolchain.filter(|toolchain| !toolchain.is_empty()) { + let mut toolchain_arg = OsString::new(); + toolchain_arg.push(toolchain); + let _cmd = rustc_cmd.arg(toolchain_arg); + } + let _cmd = rustc_cmd.args(["--version", "--verbose"]); + let output = rustc_cmd + .output() + .with_context(|| format!("failed to run {rustc_cmd:?}"))?; + + if !output.status.success() { + return Err(anyhow!( + "{rustc_cmd:?} failed with status {}", + output.status + )); + } + + let stdout = String::from_utf8(output.stdout).context("rustc output was not valid UTF-8")?; + + for line in stdout.lines() { + if let Some(rest) = line.strip_prefix("LLVM version: ") { + return Ok(rest.trim().to_string()); + } + } + + Err(anyhow!( + "could not find `LLVM version:` line in {rustc_cmd:?} output" + )) +} + +/// Finds a commit in the [Rust GitHub repository][rust-repo] that corresponds +/// to an update of LLVM and can be used to download libLLVM from Rust CI. +/// +/// [rust-repo]: https://github.com/rust-lang/rust +fn rustc_llvm_commit(options: RustcLlvmCommitOptions) -> Result<()> { + let RustcLlvmCommitOptions { github_token } = options; + let toolchain = env::var_os("RUSTUP_TOOLCHAIN"); + let llvm_version = get_llvm_version(toolchain)?; + + let headers: HeaderMap = [ + (USER_AGENT, "bpf-linker-xtask/0.1".parse().unwrap()), + (ACCEPT, "application/vnd.github+json".parse().unwrap()), + ( + AUTHORIZATION, + format!("Bearer {github_token}").parse().unwrap(), + ), + ] + .into_iter() + .collect(); + let client = Client::builder().default_headers(headers).build()?; + + let query = + format!(r#"repo:rust-lang/rust is:pr is:closed in:title "Update LLVM to {llvm_version}""#); + let resp = client + .get("https://api.github.com/search/issues") + .query(&[("q", query)]) + .send()? + .error_for_status()?; + + let body: SearchIssuesResponse = resp.json()?; + let pr = body + .items + .into_iter() + .find(|item| item.title == format!("Update LLVM to {llvm_version}")) + .ok_or_else(|| { + anyhow!("Could not find an LLVM bump PR titled \"Update LLVM to {llvm_version}\"") + })?; + let pr_number = pr.number; + + let url = format!("https://api.github.com/repos/rust-lang/rust/pulls/{pr_number}"); + let resp = client.get(url).send()?.error_for_status()?; + let pr: PullRequest = resp.json()?; + + let bors_sha = pr + .merge_commit_sha + .ok_or_else(|| anyhow!("PR #{pr_number} has no merge_commit_sha"))?; + println!("{bors_sha}"); + + Ok(()) +} + fn main() -> Result<()> { let CommandLine { subcommand } = clap::Parser::parse(); match subcommand { XtaskSubcommand::BuildStd(options) => build_std(options), XtaskSubcommand::BuildLlvm(options) => build_llvm(options), + XtaskSubcommand::RustcLlvmCommit(options) => rustc_llvm_commit(options), } } From a7fb4a847745ba898870f945b5f481ef35fc64ec Mon Sep 17 00:00:00 2001 From: Michal R Date: Fri, 21 Nov 2025 21:28:14 +0100 Subject: [PATCH 10/11] ci: Test with LLVM downloaded from Rust CI, test static linking We are planning to use libLLVM from Rust CI to produce static binaries. This change makes sure that tests work with Rust's libLLVM artifacts and libLLVM linked statically (from any source). Rust CI ships only one flavor of libLLVM, shared or static, per target. Linux GNU targets come with shared ones. Apple and Linux musl targets come with dynamic ones. To save disk space, run `cargo hack --feature-powerset` only when shared libraries are used. Otherwise, run cargo with a single feature set. --- .github/workflows/ci.yml | 157 +++++++++++++++++++++++++++++++++------ 1 file changed, 133 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0dab2388..8b8fe7d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,12 +65,22 @@ jobs: llvm: 21 exclude-features: llvm-20,rust-llvm-20 platform: + # Rust CI ships only one flavor of libLLVM, dynamic or static, per + # target. Linux GNU targets come with dynamic ones. Apple and Linux + # musl targets come with static ones. - os: macos-latest + static-target: aarch64-apple-darwin - os: macos-15-intel + static-target: x86_64-apple-darwin - os: ubuntu-22.04 + dynamic-target: x86_64-unknown-linux-gnu + static-target: x86_64-unknown-linux-musl - os: ubuntu-22.04-arm + dynamic-target: aarch64-unknown-linux-gnu + static-target: aarch64-unknown-linux-musl llvm-from: - packages + - rust-ci include: # Currently we build LLVM from source only for Linux x86_64. - toolchain: @@ -85,8 +95,15 @@ jobs: env: RUST_BACKTRACE: full - LLVM_FEATURES: llvm-${{ matrix.toolchain.llvm }} - LLVM_EXCLUDE_FEATURES: llvm-link-static,no-llvm-linking + # Features that have to be included for dynamic linking. + LLVM_FEATURES_DYNAMIC: llvm-${{ matrix.toolchain.llvm }} + # Features that have to be included for static linking. + LLVM_FEATURES_STATIC: llvm-${{ matrix.toolchain.llvm }},llvm-link-static + # Features that have to be excluded when running `cargo hack --feature-powerset` + # and intending to link dynamically. + LLVM_EXCLUDE_FEATURES_DYNAMIC: llvm-link-static,no-llvm-linking + RUSTC_LLVM_INSTALL_DIR_DYNAMIC: /tmp/rustc-llvm-dynamic + RUSTC_LLVM_INSTALL_DIR_STATIC: /tmp/rustc-llvm-static steps: - uses: actions/checkout@v6 @@ -118,16 +135,19 @@ jobs: set -euxo pipefail echo /usr/lib/llvm-15/bin >> $GITHUB_PATH - - name: Install LLVM (Linux, packages) - if: matrix.llvm-from == 'packages' && runner.os == 'Linux' + - name: Add LLVM APT repository + if: runner.os == 'Linux' run: | set -euxo pipefail wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc echo -e deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${{ matrix.toolchain.llvm }} main | sudo tee /etc/apt/sources.list.d/llvm.list - sudo apt update - sudo apt -y install llvm-${{ matrix.llvm-version }}-dev - echo /usr/lib/llvm-${{ matrix.llvm-version }}/bin >> $GITHUB_PATH + + - name: Install LLVM (Linux, packages) + if: matrix.llvm-from == 'packages' && runner.os == 'Linux' + run: | + set -euxo pipefail + sudo apt -y install llvm-${{ matrix.toolchain.llvm }}-dev echo /usr/lib/llvm-${{ matrix.toolchain.llvm }}/bin >> $GITHUB_PATH - name: Install LLVM (macOS, packages) @@ -138,7 +158,39 @@ jobs: echo $(brew --prefix llvm@${{ matrix.toolchain.llvm }})/bin >> $GITHUB_PATH echo "DYLD_LIBRARY_PATH=$(brew --prefix llvm@${{ matrix.toolchain.llvm }})/lib" >> $GITHUB_ENV - - name: Restore LLVM + - name: Install LLVM from Rust CI + if: matrix.llvm-from == 'rust-ci' + run: | + set -euxo pipefail + mkdir -p $RUSTC_LLVM_INSTALL_DIR_DYNAMIC $RUSTC_LLVM_INSTALL_DIR_STATIC + rustc_sha=$(cargo xtask rustc-llvm-commit --github-token "${{ secrets.GITHUB_TOKEN }}") + download_llvm() { + local target=$1 + local install_dir=$2 + wget -q -O - "https://ci-artifacts.rust-lang.org/rustc-builds/$rustc_sha/rust-dev-nightly-$target.tar.xz" | \ + tar -xJ --strip-components 2 -C $install_dir + } + if [[ -n "${{ matrix.platform['dynamic-target'] }}" ]]; then + download_llvm \ + "${{ matrix.platform['dynamic-target'] }}" \ + ${RUSTC_LLVM_INSTALL_DIR_DYNAMIC} + echo "LD_LIBRARY_PATH=${RUSTC_LLVM_INSTALL_DIR_DYNAMIC}/lib" >> $GITHUB_ENV + # We start with steps that use dynamic linking. Add llvm-config + # associated with dynamic target to `PATH`. + echo "${RUSTC_LLVM_INSTALL_DIR_DYNAMIC}/bin" >> $GITHUB_PATH + fi + if [[ -n "${{ matrix.platform['static-target'] }}" ]]; then + download_llvm \ + "${{ matrix.platform['static-target'] }}" \ + ${RUSTC_LLVM_INSTALL_DIR_STATIC} + if [[ "${{ runner.os }}" == "Linux" ]]; then + # `FileCheck` binary shipped in musl tarballs is linked dynamically + # to musl, we can't execute it on Ubuntu. + rm -f "${RUSTC_LLVM_INSTALL_DIR_STATIC}/bin/FileCheck" + fi + fi + + - name: Restore LLVM from GitHub Actions if: matrix.llvm-from == 'source' uses: actions/cache/restore@v4 with: @@ -165,17 +217,19 @@ jobs: - uses: taiki-e/install-action@cargo-hack - - name: Check + - name: Check (dynamic linking, feature powerset) + if: matrix.platform.dynamic-target || matrix.llvm-from != 'rust-ci' run: | - cargo hack check --feature-powerset \ - --exclude-features ${{ env.LLVM_EXCLUDE_FEATURES }},${{ matrix.toolchain.exclude-features }} \ - --features ${{ env.LLVM_FEATURES }} + cargo hack check --feature-powerset --exclude-features \ + ${{ env.LLVM_EXCLUDE_FEATURES_DYNAMIC }},${{ matrix.toolchain.exclude-features }} \ + --features ${{ env.LLVM_FEATURES_DYNAMIC }} - - name: Build + - name: Build (dynamic linking, feature powerset) + if: matrix.platform.dynamic-target || matrix.llvm-from != 'rust-ci' run: | - cargo hack build --feature-powerset \ - --exclude-features ${{ env.LLVM_EXCLUDE_FEATURES }},${{ matrix.toolchain.exclude-features }} \ - --features ${{ env.LLVM_FEATURES }} + cargo hack build --feature-powerset --exclude-features \ + ${{ env.LLVM_EXCLUDE_FEATURES_DYNAMIC }},${{ matrix.toolchain.exclude-features }} \ + --features ${{ env.LLVM_FEATURES_DYNAMIC }} # Toolchains provided by rustup include standard library artifacts # only for Tier 1 targets, which do not include BPF targets. @@ -185,11 +239,13 @@ jobs: # running compiler tests. # # `RUSTC_BOOTSTRAP` is needed to use `rustc-build-sysroot` on stable Rust. - - name: Test (sysroot built on demand) + - name: Test (sysroot built on demand, dynamic linking) + if: matrix.platform.dynamic-target || matrix.llvm-from != 'rust-ci' run: | RUSTC_BOOTSTRAP=1 cargo hack test --feature-powerset \ - --exclude-features ${{ env.LLVM_EXCLUDE_FEATURES }},${{ matrix.toolchain.exclude-features }} \ - --features ${{ env.LLVM_FEATURES }} + --exclude-features \ + ${{ env.LLVM_EXCLUDE_FEATURES_DYNAMIC }},${{ matrix.toolchain.exclude-features }} \ + --features ${{ env.LLVM_FEATURES_DYNAMIC }} # To make things easier for package maintainers, the step of building a # custom sysroot can be skipped by setting the `BPFEL_SYSROOT_DIR` @@ -200,7 +256,7 @@ jobs: # # `RUSTC_BOOTSTRAP` is needed to make `xtask build-std` work on stable # Rust. - - name: Test (prebuilt BPF standard library) + - name: Build BPF standard library run: | set -euxo pipefail @@ -211,9 +267,13 @@ jobs: --sysroot-dir "$BPFEL_SYSROOT_DIR" \ --target bpfel-unknown-none - BPFEL_SYSROOT_DIR="$BPFEL_SYSROOT_DIR" cargo hack test --feature-powerset \ - --exclude-features ${{ env.LLVM_EXCLUDE_FEATURES }},${{ matrix.toolchain.exclude-features }} \ - --features ${{ env.LLVM_FEATURES }} + - name: Test (prebuilt BPF standard libary, dynamic linking) + if: matrix.platform.dynamic-target || matrix.llvm-from != 'rust-ci' + run: | + BPFEL_SYSROOT_DIR="${{ github.workspace }}/bpf-sysroot" \ + cargo hack test --feature-powerset --exclude-features \ + ${{ env.LLVM_EXCLUDE_FEATURES_DYNAMIC }},${{ matrix.toolchain.exclude-features }} \ + --features ${{ env.LLVM_FEATURES_DYNAMIC }} - uses: actions/checkout@v6 if: runner.os == 'Linux' && matrix.toolchain.rust == 'nightly' @@ -224,13 +284,62 @@ jobs: - name: Install if: runner.os == 'Linux' && matrix.toolchain.rust == 'nightly' - run: cargo install --path . --no-default-features --features ${{ env.LLVM_FEATURES }} + run: | + cargo install --path . --no-default-features --features \ + ${{ env.LLVM_FEATURES_DYNAMIC }} - name: Run aya integration tests if: runner.os == 'Linux' && matrix.toolchain.rust == 'nightly' working-directory: aya run: cargo xtask integration-test local + - name: Prepare for static linking (LLVM from Rust CI) + if: matrix.llvm-from == 'rust-ci' + run: | + echo "${RUSTC_LLVM_INSTALL_DIR_STATIC}/bin" >> $GITHUB_PATH + + - name: Install static libraries (macOS) + if: runner.os == 'macOS' + # macOS does not provide any static libraries. Homebrew does provide + # them, but in custom paths that the system-wide clang is not aware of. + # Point build.rs to them by setting environment variables. + # + # We install llvm package only for libc++. + # + # libLLVM from homebrew requires zstd. + run: | + brew install llvm zlib + echo "CXXSTDLIB_PATH=$(brew --prefix llvm)/lib/c++" >> $GITHUB_ENV + echo "ZLIB_PATH=$(brew --prefix zlib)/lib" >> $GITHUB_ENV + if [[ "${{ matrix.llvm-from }}" == "packages" ]]; then + brew install zstd + echo "LIBZSTD_PATH=$(brew --prefix zstd)/lib" >> $GITHUB_ENV + fi + + - name: Check (static linking, single feature set) + # Static linking in combination with `cargo hack --feature-powerset` + # (multiple builds) increases the disk usage massively. Therefore we + # perform all static builds with only one fixed feature set. + run: | + cargo check --no-default-features --features \ + ${{ env.LLVM_FEATURES_STATIC }} + + - name: Build (static linking, single feature set) + run: | + cargo build --no-default-features --features \ + ${{ env.LLVM_FEATURES_STATIC }} + + - name: Test (sysroot built on demand, static linking) + run: | + RUSTC_BOOTSTRAP=1 cargo test --no-default-features --features \ + ${{ env.LLVM_FEATURES_STATIC }} + + - name: Test (prebuilt BPF standard library, static linking) + run: | + BPFEL_SYSROOT_DIR="${{ github.workspace }}/bpf-sysroot" \ + cargo test --no-default-features --features \ + ${{ env.LLVM_FEATURES_STATIC }} + - name: Report disk usage if: ${{ always() }} uses: ./.github/actions/report-disk-usage From bbdcbcb84d1ebdf58beb60e79bb649c1347b57de Mon Sep 17 00:00:00 2001 From: Michal R Date: Fri, 14 Nov 2025 13:04:27 +0100 Subject: [PATCH 11/11] ci/release: Use LLVM from Rust CI, support macOS and Linux ARM - Use LLVM from Rust CI instead of LLVM built from source. - Support for macOS and Linux ARM. - Add the disk usage reports. - To make sure that the job does not regress, execute it in a "dry run" mode for each pull request. --- .github/workflows/release.yml | 62 +++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5fd02665..bd1976bf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,38 +1,64 @@ name: Release on: + pull_request: + release: types: [published] permissions: {} jobs: - llvm: - uses: ./.github/workflows/llvm.yml - upload-bins: - # TODO: Build for macos someday. - runs-on: ubuntu-22.04 - needs: llvm + runs-on: ${{ matrix.platform.os }} + strategy: + matrix: + platform: + - os: macos-latest + target: aarch64-apple-darwin + - os: macos-15-intel + target: x86_64-apple-darwin + - os: ubuntu-22.04 + target: x86_64-unknown-linux-musl + - os: ubuntu-22.04-arm + target: aarch64-unknown-linux-musl + env: + RUST_CI_LLVM_INSTALL_DIR: /tmp/rustc-llvm steps: - - name: Restore LLVM - uses: actions/cache/restore@v4 - with: - path: llvm-install - key: ${{ needs.llvm.outputs.cache-key }} - fail-on-cache-miss: true + - uses: actions/checkout@v6 + - uses: Swatinem/rust-cache@v2 - - name: Add LLVM to PATH + - name: Install static libraries (macOS) + if: runner.os == 'macOS' + # macOS does not provide any static libraries. Homebrew does provide + # them, but in custom paths that the system-wide clang is not aware of. + # Point build.rs to them by setting environment variables. + # + # We install llvm package only for libc++. run: | - echo "${{ github.workspace }}/llvm-install/bin" >> $GITHUB_PATH - echo "$PATH" + brew install llvm zlib + echo "CXXSTDLIB_PATH=$(brew --prefix llvm)/lib/c++" >> $GITHUB_ENV + echo "ZLIB_PATH=$(brew --prefix zlib)/lib" >> $GITHUB_ENV - - uses: actions/checkout@v6 - - uses: Swatinem/rust-cache@v2 + - name: Install LLVM from Rust CI + run: | + set -euxo pipefail + sudo mkdir -p $RUST_CI_LLVM_INSTALL_DIR + rustc_date="$(curl -s https://static.rust-lang.org/dist/channel-rust-nightly-date.txt)" + rustc_sha="$(curl -s "https://static.rust-lang.org/dist/$rustc_date/channel-rust-nightly-git-commit-hash.txt")" + wget -q -O - "https://ci-artifacts.rust-lang.org/rustc-builds/$rustc_sha/rust-dev-nightly-${{ matrix.platform.target }}.tar.xz" | \ + sudo tar -xJ --strip-components 2 -C $RUST_CI_LLVM_INSTALL_DIR + echo "${RUST_CI_LLVM_INSTALL_DIR}/bin" >> $GITHUB_PATH - uses: taiki-e/upload-rust-binary-action@v1 with: bin: bpf-linker - features: llvm-link-static + features: llvm-21,llvm-link-static + no-default-features: true + dry-run: ${{ github.event_name != 'release' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Report disk usage + if: ${{ always() }} + uses: ./.github/actions/report-disk-usage