diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index db1a3281..efcac997 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -10,33 +10,80 @@ on: env: CARGO_TERM_COLOR: always RUST_LOG: info + PARTITIONS: 8 jobs: + # verify that we are able to rebuild all benchmarks; the results here + # are not used in the CI pipeline but this verifies that changes have + # not regressed our ability to rebuild benchmarks. rebuild: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Install nightly - uses: actions-rs/toolchain@v1 - with: - toolchain: nightly + - uses: actions/checkout@v4 + - run: rustup update - name: Rebuild benchmarks - run: benchmarks/build-all.sh 5 + run: benchmarks/build-all.sh - run: + build-wasmtime: runs-on: ubuntu-latest + strategy: + matrix: + # for now, limit this a bit + revision: ["main", "v25.0.0"] + env: + REVISION: ${{ matrix.revision }} steps: - - uses: actions/checkout@v2 - - name: Install nightly - uses: actions-rs/toolchain@v1 + - uses: actions/checkout@v4 + - run: rustup update + - name: Build Engine + run: | + pushd engines/wasmtime + rustc build.rs + ./build + popd + - name: Upload Wasmtime Shared Library + uses: actions/upload-artifact@v4 with: - toolchain: nightly - - name: Run benchmarks - run: benchmarks/run-all.sh + name: wasmtime-${{ matrix.revision }} + path: engines/wasmtime/libengine.so + + benchmark: + runs-on: ubuntu-latest + needs: build-wasmtime + strategy: + matrix: + partition: ["00", "01", "02", "03", "04", "05", "06", "07"] + env: + SPLIT_PREFIX: "benchmarks/split-" + PARTITION_SUITE: "benchmarks/split-${{ matrix.partition }}.suite" + steps: + - uses: actions/checkout@v4 + - name: Downloads All Artifacts (root) + uses: actions/download-artifact@v4 + with: + path: artifacts + - run: ls -al . artifacts/ + - run: rustup update + - run: chmod +x artifacts/*/*.so + - name: Partition Benchmark Suite + run: | + benchmarks/split-suite.sh benchmarks/all.suite ${PARTITIONS} "${SPLIT_PREFIX}" + + - name: Output Running Benchmarks + run: cat "${PARTITION_SUITE}" + + - name: Benchmark Partition ${{ matrix.partition }} + run: | + cargo run benchmark \ + --engine=artifacts/wasmtime-main/libengine.so \ + --engine=artifacts/wasmtime-v25.0.0/libengine.so \ + --processes=4 \ + --iterations-per-process=1 \ + -- "${PARTITION_SUITE}" check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Check for missing paths in `all.suite` run: benchmarks/check-incomplete-suite.sh diff --git a/.github/workflows/benchmarks_native.yaml b/.github/workflows/benchmarks_native.yaml index 20a7c85e..cbfd1203 100644 --- a/.github/workflows/benchmarks_native.yaml +++ b/.github/workflows/benchmarks_native.yaml @@ -11,10 +11,7 @@ jobs: name: Build and run native benchmarks runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Install nightly - uses: actions-rs/toolchain@v1 - with: - toolchain: nightly + - uses: actions/checkout@v4 + - run: rustup update nightly - name: Rebuild and run native benchmarks using Docker run: benchmarks/build-all-native.sh --run diff --git a/.github/workflows/sightglass.yml b/.github/workflows/sightglass.yml index 8c47e37f..bf5ec114 100644 --- a/.github/workflows/sightglass.yml +++ b/.github/workflows/sightglass.yml @@ -19,13 +19,10 @@ jobs: format: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: true - - name: Install stable - uses: actions-rs/toolchain@v1 - with: - toolchain: stable + - run: rustup update - run: rustup component add rustfmt - run: cargo fmt --all -- --check @@ -35,15 +32,10 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] steps: - - uses: actions/checkout@v2 - - - name: Install nightly - uses: actions-rs/toolchain@v1 - with: - toolchain: nightly - + - uses: actions/checkout@v4 + - run: rustup update nightly - name: Download Cached Wasmtime engine - uses: actions/cache@v2 + uses: actions/cache@v4 id: wasmtime-cache with: path: engines/wasmtime/* diff --git a/benchmarks/split-suite.sh b/benchmarks/split-suite.sh new file mode 100755 index 00000000..d2a4321b --- /dev/null +++ b/benchmarks/split-suite.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env sh +# +# This script splits a test suite into N roughly +# equally sized test suites using the provided +# output prefix. +# +# Usage: +# ./ci-split.sh +# +# The naming of the split files will being in the form generated +# by `split` which is `.suite` where num is in %02d +# format; for example, a prefix of "benchmarks/split-" will yield +# files with names like benchmarks/split-07.suite. + +set -e + +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " + exit 1 +fi + +SUITE="$1" +NUMBER_SPLITS="$2" +OUTPUT_PREFIX="$3" + +ceil_div() { + echo "($1 + $2 - 1) / $2" | bc +} + +count_lines() { + wc -l "$1" | awk '{print $1}' +} + +tmpdir="$(mktemp -d)" +tmp_suite="${tmpdir}/input.suite" +grep -v '^\#' "${SUITE}" > "${tmp_suite}" +line_count=$(count_lines "${tmp_suite}") +lines_per_split=$(ceil_div "$line_count" "$NUMBER_SPLITS") +split -d -l "$lines_per_split" "${tmp_suite}" "${OUTPUT_PREFIX}" +rm -rf "$tmpdir" + +echo "Generated Splits:" +for f in "${OUTPUT_PREFIX}"*; do + # suite files must end with ".suite" to work correctly + # with sigtglass, so do the rename as split isn't + # able to do it directly. + suited="${f}.suite" + mv "$f" "${suited}" + echo "- ${suited}" +done