diff --git a/.github/actions/setup-rust-2/action.yml b/.github/actions/setup-rust-2/action.yml new file mode 100644 index 00000000000..2bc11137dc9 --- /dev/null +++ b/.github/actions/setup-rust-2/action.yml @@ -0,0 +1,40 @@ +name: "Setup Rust (Custom AMI)" +description: "Lightweight Rust setup for custom AMI runners with pre-installed toolchain" + +inputs: + cache-suffix: + description: "optional suffix for cache key to isolate builds with different RUSTFLAGS (e.g. 'sanitizer')" + required: false + default: "" + targets: + description: "optional targets override (for cache key compatibility)" + required: false + timestamp: + description: "Timestamp cache with cargo sweep" + default: "false" + +runs: + using: "composite" + steps: + - name: Get Rust version for cache key + id: rust-version + shell: bash + run: | + # Get a short hash of the rustc version (matches dtolnay/rust-toolchain cachekey format) + RUSTC_VERSION=$(rustc --version) + CACHEKEY=$(echo "$RUSTC_VERSION" | sha256sum | cut -c1-20) + echo "cachekey=$CACHEKEY" >> $GITHUB_OUTPUT + + - name: Rust Dependency Cache + uses: Swatinem/rust-cache@v2 + with: +# save-if: ${{ github.ref_name == 'develop' }} + shared-key: "rust-cache-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.rust-version.outputs.cachekey }}-${{ inputs.targets }}${{ inputs.cache-suffix && format('-{0}', inputs.cache-suffix) || '' }}" + + # Note: sccache is configured by runs-on/action with sccache: s3 in the workflow + # Do NOT add mozilla-actions/sccache-action here as it would override S3 config with GH cache + + - name: Timestamp Cache + shell: bash + if: ${{ inputs.timestamp == 'true' && github.ref_name == 'develop' }} + run: cargo sweep --stamp diff --git a/.github/runs-on.yml b/.github/runs-on.yml index 7970bf75e17..63cb2753423 100644 --- a/.github/runs-on.yml +++ b/.github/runs-on.yml @@ -1,11 +1,28 @@ +# Custom AMIs for Vortex CI runners +# These AMIs are automatically rebuilt every 25 days by the ami-prebuild.yml workflow +# to keep the GitHub Actions runner agent up to date (required to be <30 days old). +# AMIs are deprecated after 25 days via Packer's deprecate_at setting. +# +# AMI naming pattern: vortex-ci-{arch}-{timestamp} +# Built with: .github/actions/build-ami and .github/packer/vortex-ci.pkr.hcl +runners: + rust-amd64-medium: + image: vortex-ci-amd64 + cpu: 8 + family: [m7i, m7i-flex, m7a] + rust-amd64-large: + image: vortex-ci-amd64 + cpu: 16 + family: [m7i, m7i-flex, m7a] + images: vortex-ci-amd64: platform: "linux" arch: "x64" - name: "vortex-ci-*" + name: "vortex-ci-x64-*" owner: "375504701696" vortex-ci-arm64: platform: "linux" arch: "arm64" - name: "vortex-ci-*" + name: "vortex-ci-arm64-*" owner: "375504701696" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f5c521c17e..fce755ccd88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,9 +49,7 @@ jobs: name: "Python (lint)" runs-on: - runs-on=${{ github.run_id }} - - family=m7i+m7i-flex+m7a - - cpu=16 - - image=ubuntu24-full-x64 + - runner=rust-amd64-large - extras=s3-cache - tag=python-lint timeout-minutes: 120 @@ -60,9 +58,7 @@ jobs: with: sccache: s3 - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: ./.github/actions/setup-rust-2 - name: Install uv uses: spiraldb/actions/.github/actions/setup-uv@0.18.5 with: @@ -81,9 +77,7 @@ jobs: name: "Python (test)" runs-on: - runs-on=${{ github.run_id }} - - family=m7i+m7i-flex+m7a - - cpu=16 - - image=ubuntu24-full-x64 + - runner=rust-amd64-large - extras=s3-cache - tag=python-test timeout-minutes: 120 @@ -94,9 +88,7 @@ jobs: with: sccache: s3 - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: ./.github/actions/setup-rust-2 - name: Install uv uses: spiraldb/actions/.github/actions/setup-uv@0.18.5 with: @@ -164,9 +156,7 @@ jobs: timeout-minutes: 120 runs-on: - runs-on=${{ github.run_id }} - - family=m7i+m7i-flex+m7a - - cpu=8 - - image=ubuntu24-full-x64 + - runner=rust-amd64-medium - extras=s3-cache - tag=rust-docs steps: @@ -174,9 +164,7 @@ jobs: with: sccache: s3 - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: ./.github/actions/setup-rust-2 - name: Docs run: | RUSTDOCFLAGS="-D warnings" cargo doc --no-deps @@ -188,9 +176,7 @@ jobs: timeout-minutes: 120 runs-on: - runs-on=${{ github.run_id }} - - family=m7i+m7i-flex+m7a - - cpu=8 - - image=ubuntu24-full-x64 + - runner=rust-amd64-medium - extras=s3-cache - tag=${{ matrix.config.name }} env: @@ -220,34 +206,20 @@ jobs: with: sccache: s3 - uses: actions/checkout@v6 - - id: setup-rust - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - timestamp: "true" - - name: Install wasm32 target - if: ${{ matrix.config.target == 'wasm32-unknown-unknown' }} - run: rustup target add wasm32-unknown-unknown - - name: Install cargo-hack - uses: taiki-e/install-action@cargo-hack + - uses: ./.github/actions/setup-rust-2 - name: Rust Build (${{matrix.config.name}}) run: ${{matrix.config.env.rustflags}} cargo hack ${{matrix.config.command}} --locked ${{matrix.config.args}} --ignore-private - name: "Make sure no files changed after build" run: | git status --porcelain test -z "$(git status --porcelain)" - - name: Prune cache - if: ${{ github.ref_name == 'develop' && steps.setup-rust.outputs.deps-cache-hit != 'true' }} - run: cargo sweep --file check-min-deps: name: "Check build with minimal dependencies" timeout-minutes: 120 runs-on: - runs-on=${{ github.run_id }} - - family=m7i+m7i-flex+m7a - - cpu=8 - - image=ubuntu24-full-x64 + - runner=rust-amd64-medium - extras=s3-cache - tag=rust-min-deps steps: @@ -255,11 +227,7 @@ jobs: with: sccache: s3 - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: taiki-e/install-action@cargo-hack - - uses: taiki-e/install-action@cargo-minimal-versions + - uses: ./.github/actions/setup-rust-2 - run: cargo minimal-versions check --direct --workspace --ignore-private rust-lint: @@ -267,9 +235,7 @@ jobs: timeout-minutes: 120 runs-on: - runs-on=${{ github.run_id }} - - family=m7i+m7i-flex+m7a - - cpu=16 - - image=ubuntu24-full-x64 + - runner=rust-amd64-large - extras=s3-cache - tag=rust-lint steps: @@ -277,12 +243,7 @@ jobs: with: sccache: s3 - uses: actions/checkout@v6 - - id: setup-rust - uses: ./.github/actions/setup-rust - with: - toolchain: nightly - repo-token: ${{ secrets.GITHUB_TOKEN }} - timestamp: "true" + - uses: ./.github/actions/setup-rust-2 - name: Rust Lint - Format run: cargo +nightly fmt --all --check - name: Rustc check @@ -293,18 +254,13 @@ jobs: run: cargo clippy --locked --all-features --all-targets -- -D warnings - name: Rust Lint - Clippy Default Features run: cargo clippy --locked --all-targets -- -D warnings - - name: Prune cache - if: ${{ github.ref_name == 'develop' && steps.setup-rust.outputs.deps-cache-hit != 'true' }} - run: cargo sweep --file rust-lint-no-default: name: "Rust (lint, no default)" timeout-minutes: 120 runs-on: - runs-on=${{ github.run_id }} - - family=m7i+m7i-flex+m7a - - cpu=16 - - image=ubuntu24-full-x64 + - runner=rust-amd64-large - extras=s3-cache - tag=rust-lint-no-default steps: @@ -312,21 +268,12 @@ jobs: with: sccache: s3 - uses: actions/checkout@v6 - - id: setup-rust - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - timestamp: "true" - - name: Install cargo-hack - uses: taiki-e/install-action@cargo-hack + - uses: ./.github/actions/setup-rust-2 - name: Rust Lint - Clippy No Default Features shell: bash run: | # https://spiraldb.slack.com/archives/C07BV3GKAJ2/p1732736281946729 cargo hack clippy --no-default-features -- -D warnings - - name: Prune cache - if: ${{ github.ref_name == 'develop' && steps.setup-rust.outputs.deps-cache-hit != 'true' }} - run: cargo sweep --file rust-semver: name: "Rust (semver checks)" @@ -334,9 +281,7 @@ jobs: if: github.ref != 'refs/heads/develop' runs-on: - runs-on=${{ github.run_id }} - - family=m7i+m7i-flex+m7a - - cpu=16 - - image=ubuntu24-full-x64 + - runner=rust-amd64-large - extras=s3-cache - tag=rust-semver steps: @@ -347,9 +292,7 @@ jobs: with: fetch-depth: 0 fetch-tags: true - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: ./.github/actions/setup-rust-2 # We have to set the correct Cargo.toml versions so semver checks uses the previous release. - name: Latest Tag @@ -358,9 +301,7 @@ jobs: LATEST_TAG=$(git describe --tags --abbrev=0) echo "tag=$LATEST_TAG" >> $GITHUB_OUTPUT - name: Cargo Set Version - run: | - cargo install cargo-edit - cargo set-version --workspace ${{ steps.latest-tag.outputs.tag }} + run: cargo set-version --workspace ${{ steps.latest-tag.outputs.tag }} - name: Check semver uses: obi1kenobi/cargo-semver-checks-action@v2 @@ -378,9 +319,7 @@ jobs: - ffi runs-on: - runs-on=${{ github.run_id }} - - family=m7i+m7i-flex+m7a - - cpu=16 - - image=ubuntu24-full-x64 + - runner=rust-amd64-large - extras=s3-cache - tag=rust-coverage - tag=suite-${{ matrix.suite }} @@ -394,17 +333,7 @@ jobs: with: sccache: s3 - uses: actions/checkout@v6 - - id: setup-rust - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - timestamp: "true" - - name: Install grcov - uses: taiki-e/install-action@grcov - - name: Install nextest - uses: taiki-e/install-action@v2 - with: - tool: nextest + - uses: ./.github/actions/setup-rust-2 - name: Rust Tests if: ${{ matrix.suite == 'tests' }} env: @@ -422,8 +351,6 @@ jobs: run: | cargo build -p vortex-ffi cargo run -p vortex-ffi --example hello_vortex - - name: Install llvm-tools-preview - run: rustup component add llvm-tools-preview - name: Generate coverage report run: | grcov . --binary-path target/debug/ -s . -t lcov --llvm --ignore-not-existing \ @@ -443,18 +370,13 @@ jobs: disable_search: true flags: ${{ matrix.suite }} use_oidc: true - - name: Prune cache - if: ${{ github.ref_name == 'develop' && steps.setup-rust.outputs.deps-cache-hit != 'true' }} - run: cargo sweep --file rust-test: name: "Rust tests (sanitizer)" timeout-minutes: 120 runs-on: - runs-on=${{ github.run_id }} - - family=m7i+m7i-flex+m7a - - cpu=16 - - image=ubuntu24-full-x64 + - runner=rust-amd64-large - extras=s3-cache - tag=rust-test-sanitizer env: @@ -472,27 +394,7 @@ jobs: with: sccache: s3 - uses: actions/checkout@v6 - - name: Install llvm - uses: aminya/setup-cpp@v1 - with: - compiler: llvm - cache-tools: true - - id: setup-rust - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - toolchain: nightly - components: "rust-src, rustfmt, clippy, llvm-tools-preview" - cache-suffix: "sanitizer" - timestamp: "true" - - name: Install build dependencies - run: | - sudo apt-get update - sudo apt-get install -y ninja-build cmake - - name: Install nextest - uses: taiki-e/install-action@v2 - with: - tool: nextest + - uses: ./.github/actions/setup-rust-2 - name: Rust Tests env: RUSTFLAGS: "-A warnings -Zsanitizer=address -Zsanitizer=leak --cfg disable_loom --cfg vortex_nightly -C debuginfo=2 -C opt-level=0 -C strip=none" @@ -507,9 +409,6 @@ jobs: --no-fail-fast \ --target x86_64-unknown-linux-gnu \ --verbose - - name: Prune cache - if: ${{ github.ref_name == 'develop' && steps.setup-rust.outputs.deps-cache-hit != 'true' }} - run: cargo sweep --file rust-test-other: name: "Rust tests (${{ matrix.os }})" @@ -569,9 +468,7 @@ jobs: name: "Java" runs-on: - runs-on=${{ github.run_id }} - - family=m7i+m7i-flex+m7a - - cpu=16 - - image=ubuntu24-full-x64 + - runner=rust-amd64-large - extras=s3-cache - tag=java timeout-minutes: 120 @@ -580,13 +477,11 @@ jobs: with: sccache: s3 - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-rust-2 - uses: actions/setup-java@v5 with: distribution: "corretto" java-version: "17" - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - run: ./gradlew test --parallel working-directory: ./java @@ -653,16 +548,15 @@ jobs: timeout-minutes: 120 runs-on: - runs-on=${{ github.run_id }} - - family=m7i+m7i-flex+m7a - - cpu=8 - - image=ubuntu24-full-x64 + - runner=rust-amd64-medium - extras=s3-cache - tag=cxx-build steps: - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-rust + - uses: runs-on/action@v2 with: - repo-token: ${{ secrets.GITHUB_TOKEN }} + sccache: s3 + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-rust-2 - name: Build and run C++ unit tests working-directory: vortex-cxx/ run: |