Grab Bag of CI Enhancements (#284)

* Grab Bag of CI Enhancements

This first set of changes includes the following:
- Removes outdated github actions recipes that are no
  longer functioning.
- Updates the benchmark flow to complete much more quickly by:
  1. Don't try to run 10 processes concurrently; this seems to be
     what previously caused some benchmarks to run for hours.
  2. Just do 1 iteration (for PR flow); nightly runs will be added
     later to get more complete results.
  3. Partition the test suite to run in 8 parallel jobs.
- Build wasmtime main and compare with version 25 (chosen
  arbitrarily) for test runs.  I suspect this will be a discussion
  point but this was somewhere to start.  Matrixing in other versions
  is very straightforward.

* ci: Extract suite splitting to script

* ci: re-enable the rebuild job

* fixup! ci: Extract suite splitting to script

Author: posborne

.github/workflows/benchmarks.yml

@@ -10,33 +10,80 @@ on:
 env:
   CARGO_TERM_COLOR: always
   RUST_LOG: info
+  PARTITIONS: 8
 
 jobs:
+  # verify that we are able to rebuild all benchmarks; the results here
+  # are not used in the CI pipeline but this verifies that changes have
+  # not regressed our ability to rebuild benchmarks.
   rebuild:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Install nightly
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: nightly
+    - uses: actions/checkout@v4
+    - run: rustup update
     - name: Rebuild benchmarks
-      run: benchmarks/build-all.sh 5
+      run: benchmarks/build-all.sh
 
-  run:
+  build-wasmtime:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # for now, limit this a bit
+        revision: ["main", "v25.0.0"]
+    env:
+      REVISION: ${{ matrix.revision }}
     steps:
-    - uses: actions/checkout@v2
-    - name: Install nightly
-      uses: actions-rs/toolchain@v1
+    - uses: actions/checkout@v4
+    - run: rustup update
+    - name: Build Engine
+      run: |
+        pushd engines/wasmtime
+        rustc build.rs
+        ./build
+        popd
+    - name: Upload Wasmtime Shared Library
+      uses: actions/upload-artifact@v4
       with:
-        toolchain: nightly
-    - name: Run benchmarks
-      run: benchmarks/run-all.sh
+        name: wasmtime-${{ matrix.revision }}
+        path: engines/wasmtime/libengine.so
+
+  benchmark:
+    runs-on: ubuntu-latest
+    needs: build-wasmtime
+    strategy:
+      matrix:
+        partition: ["00", "01", "02", "03", "04", "05", "06", "07"]
+    env:
+      SPLIT_PREFIX: "benchmarks/split-"
+      PARTITION_SUITE: "benchmarks/split-${{ matrix.partition }}.suite"
+    steps:
+    - uses: actions/checkout@v4
+    - name: Downloads All Artifacts (root)
+      uses: actions/download-artifact@v4
+      with:
+        path: artifacts
+    - run: ls -al . artifacts/
+    - run: rustup update
+    - run: chmod +x artifacts/*/*.so
+    - name: Partition Benchmark Suite
+      run: |
+        benchmarks/split-suite.sh benchmarks/all.suite ${PARTITIONS} "${SPLIT_PREFIX}"
+
+    - name: Output Running Benchmarks
+      run: cat "${PARTITION_SUITE}"
+
+    - name: Benchmark Partition ${{ matrix.partition }}
+      run: |
+        cargo run benchmark \
+        --engine=artifacts/wasmtime-main/libengine.so \
+        --engine=artifacts/wasmtime-v25.0.0/libengine.so \
+        --processes=4 \
+        --iterations-per-process=1 \
+        -- "${PARTITION_SUITE}"
 
   check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Check for missing paths in `all.suite`
         run: benchmarks/check-incomplete-suite.sh

.github/workflows/benchmarks_native.yaml

@@ -11,10 +11,7 @@ jobs:
     name: Build and run native benchmarks
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Install nightly
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: nightly
+    - uses: actions/checkout@v4
+    - run: rustup update nightly
     - name: Rebuild and run native benchmarks using Docker
       run: benchmarks/build-all-native.sh --run

.github/workflows/sightglass.yml

@@ -19,13 +19,10 @@ jobs:
   format:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - name: Install stable
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: stable
+    - run: rustup update
     - run: rustup component add rustfmt
     - run: cargo fmt --all -- --check
 
@@ -35,15 +32,10 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
     steps:
-    - uses: actions/checkout@v2
-
-    - name: Install nightly
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: nightly
-
+    - uses: actions/checkout@v4
+    - run: rustup update nightly
     - name: Download Cached Wasmtime engine
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       id: wasmtime-cache
       with:
         path: engines/wasmtime/*

benchmarks/split-suite.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env sh
+#
+# This script splits a test suite into N roughly
+# equally sized test suites using the provided
+# output prefix.
+#
+# Usage:
+#     ./ci-split.sh <input.suite> <N> <output-prefix>
+#
+# The naming of the split files will being in the form generated
+# by `split` which is `<prefix><num>.suite` where num is in %02d
+# format; for example, a prefix of "benchmarks/split-" will yield
+# files with names like benchmarks/split-07.suite.
+
+set -e
+
+if [ "$#" -ne 3 ]; then
+    echo "Usage: $0 <suite> <number splits> <output prefix>"
+    exit 1
+fi
+
+SUITE="$1"
+NUMBER_SPLITS="$2"
+OUTPUT_PREFIX="$3"
+
+ceil_div() {
+    echo "($1 + $2 - 1) / $2" | bc
+}
+
+count_lines() {
+    wc -l "$1" | awk '{print $1}'
+}
+
+tmpdir="$(mktemp -d)"
+tmp_suite="${tmpdir}/input.suite"
+grep -v '^\#' "${SUITE}" > "${tmp_suite}"
+line_count=$(count_lines "${tmp_suite}")
+lines_per_split=$(ceil_div "$line_count" "$NUMBER_SPLITS")
+split -d -l "$lines_per_split" "${tmp_suite}" "${OUTPUT_PREFIX}"
+rm -rf "$tmpdir"
+
+echo "Generated Splits:"
+for f in "${OUTPUT_PREFIX}"*; do
+    # suite files must end with ".suite" to work correctly
+    # with sigtglass, so do the rename as split isn't
+    # able to do it directly.
+    suited="${f}.suite"
+    mv "$f" "${suited}"
+    echo "- ${suited}"
+done