Bump version #5
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} | |
| on: | |
| push: | |
| branches: ["develop"] | |
| pull_request: {} | |
| workflow_dispatch: {} | |
| permissions: | |
| actions: read | |
| contents: read | |
| checks: write # audit-check creates checks | |
| issues: write # audit-check creates issues | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RUST_BACKTRACE: 1 | |
| jobs: | |
| lint-toml: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: spiraldb/actions/.github/actions/[email protected] | |
| validate-workflow-yaml: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Validate YAML file | |
| run: | | |
| # Lint the workflows and yamllint's configuration file. | |
| yamllint \ | |
| --strict \ | |
| -c yamllint-config.yaml \ | |
| yamllint-config.yaml \ | |
| .github/ | |
| python-lint: | |
| name: "Python (lint)" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Install uv | |
| uses: spiraldb/actions/.github/actions/[email protected] | |
| with: | |
| sync: false | |
| prune-cache: false | |
| - name: Python Lint - Format | |
| run: uv run ruff format --check . | |
| - name: Python Lint - Ruff | |
| run: uv run ruff check . | |
| - name: Python Lint - PyRight | |
| run: uv run basedpyright vortex-python | |
| python-test: | |
| name: "Python (test)" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| env: | |
| RUST_LOG: "info,uv=debug" | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./.github/actions/setup-rust | |
| - name: Install uv | |
| uses: spiraldb/actions/.github/actions/[email protected] | |
| with: | |
| sync: false | |
| prune-cache: false | |
| - name: Pytest - Vortex | |
| run: | | |
| uv run --all-packages pytest --benchmark-disable test/ | |
| working-directory: vortex-python/ | |
| - name: Pytest Benchmarks - Vortex | |
| run: | | |
| uv run --all-packages pytest --benchmark-only benchmark/ | |
| working-directory: vortex-python/ | |
| - name: Doctest - PyVortex | |
| run: | | |
| uv run --all-packages make doctest | |
| working-directory: docs/ | |
| - name: Ensure docs build - PyVortex | |
| run: | | |
| uv run --all-packages make html | |
| working-directory: docs/ | |
| - uses: mlugg/setup-zig@v2 | |
| - name: Ensure wheel and sdist can be built on Linux - PyVortex | |
| shell: bash | |
| run: | | |
| echo "Clearing wheel target directory" | |
| rm -rf ../target/wheels/ | |
| uv venv | |
| uv pip install "maturin[zig]" | |
| uv tool run maturin build --interpreter python3.11 --zig | |
| uv tool run maturin build --interpreter python3.11 --zig --sdist | |
| file_count=$(ls -1 ../target/wheels/ | wc -l) | |
| if [[ $file_count -ne 2 ]]; then | |
| echo "Unexpected number of files detected ${file_count}:" | |
| ls ../target/wheels/ | |
| exit 1 | |
| else | |
| echo "Generated two files" | |
| fi | |
| working-directory: vortex-python/ | |
| rust-docs: | |
| name: "Rust (docs)" | |
| timeout-minutes: 120 | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - family=m7i+m7i-flex+m7a | |
| - cpu=8 | |
| - image=ubuntu24-full-x64 | |
| - extras=s3-cache | |
| - tag=rust-docs | |
| steps: | |
| - uses: runs-on/action@v2 | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v5 | |
| - uses: ./.github/actions/setup-rust | |
| - name: Docs | |
| run: | | |
| RUSTDOCFLAGS="-D warnings" cargo doc --no-deps | |
| # nextest doesn't support doc tests, so we run it here | |
| cargo test --doc --workspace --all-features --exclude vortex-cxx --exclude vortex-jni --exclude vortex-ffi --exclude xtask --no-fail-fast | |
| build-rust: | |
| name: "Rust build (${{matrix.config.name}})" | |
| timeout-minutes: 120 | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - family=m7i+m7i-flex+m7a | |
| - cpu=8 | |
| - image=ubuntu24-full-x64 | |
| - extras=s3-cache | |
| - tag=${{ matrix.config.name }} | |
| env: | |
| # disable lints for build, they will be caught in Rust lint job. | |
| RUSTFLAGS: "-A warnings" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: | |
| - name: "all-features" | |
| command: "build" | |
| args: "--all-features --all-targets" | |
| - name: "default features" | |
| command: "build" | |
| args: "--all-targets" | |
| - name: "with tokio dispatcher" | |
| command: "build" | |
| args: "--no-default-features --features tokio --all-targets --ignore-unknown-features" | |
| - name: "with compio dispatcher" | |
| command: "build" | |
| args: "--no-default-features --features compio --all-targets --ignore-unknown-features" | |
| - name: "with tokio+compio dispatcher" | |
| command: "build" | |
| args: "--no-default-features --features tokio,compio --all-targets --ignore-unknown-features" | |
| - name: "wasm32 with default features" | |
| command: "build" | |
| target: wasm32-unknown-unknown | |
| env: | |
| rustflags: "RUSTFLAGS='-A warnings --cfg getrandom_backend=\"wasm_js\"'" | |
| args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd" | |
| steps: | |
| - uses: runs-on/action@v2 | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v5 | |
| - uses: ./.github/actions/setup-rust | |
| - name: Install wasm32 target | |
| if: ${{ matrix.config.target == 'wasm32-unknown-unknown' }} | |
| run: rustup target add wasm32-unknown-unknown | |
| - name: Install cargo-hack | |
| uses: taiki-e/install-action@cargo-hack | |
| - name: Rust Build (${{matrix.config.name}}) | |
| run: ${{matrix.config.env.rustflags}} cargo hack ${{matrix.config.command}} --locked ${{matrix.config.args}} --ignore-private | |
| - name: "Make sure no files changed after build" | |
| run: | | |
| git status --porcelain | |
| test -z "$(git status --porcelain)" | |
| check-min-deps: | |
| name: "Check build with minimal dependencies" | |
| timeout-minutes: 120 | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - family=m7i+m7i-flex+m7a | |
| - cpu=8 | |
| - image=ubuntu24-full-x64 | |
| - extras=s3-cache | |
| - tag=rust-min-deps | |
| steps: | |
| - uses: runs-on/action@v2 | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v5 | |
| - uses: ./.github/actions/setup-rust | |
| - uses: taiki-e/install-action@cargo-hack | |
| - uses: taiki-e/install-action@cargo-minimal-versions | |
| - run: cargo minimal-versions check --direct --workspace --ignore-private | |
| rust-lint: | |
| name: "Rust (lint)" | |
| timeout-minutes: 120 | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - family=m7i+m7i-flex+m7a | |
| - cpu=16 | |
| - image=ubuntu24-full-x64 | |
| - extras=s3-cache | |
| - tag=rust-lint | |
| steps: | |
| - uses: runs-on/action@v2 | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v5 | |
| - uses: ./.github/actions/setup-rust | |
| with: | |
| toolchain: nightly | |
| - name: Rust Lint - Format | |
| run: cargo +nightly fmt --all --check | |
| - name: Rustc check | |
| run: cargo check --locked --all-features --all-targets | |
| - name: Rust Lint - Clippy All Features | |
| run: cargo clippy --locked --all-features --all-targets -- -D warnings | |
| - name: Rust Lint - Clippy Default Features | |
| run: cargo clippy --locked --all-targets -- -D warnings | |
| rust-lint-no-default: | |
| name: "Rust (lint, no default)" | |
| timeout-minutes: 120 | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - family=m7i+m7i-flex+m7a | |
| - cpu=16 | |
| - image=ubuntu24-full-x64 | |
| - extras=s3-cache | |
| - tag=rust-lint-no-default | |
| steps: | |
| - uses: runs-on/action@v2 | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v5 | |
| - uses: ./.github/actions/setup-rust | |
| - name: Install cargo-hack | |
| uses: taiki-e/install-action@cargo-hack | |
| - name: Rust Lint - Clippy No Default Features | |
| shell: bash | |
| run: | | |
| # https://spiraldb.slack.com/archives/C07BV3GKAJ2/p1732736281946729 | |
| cargo hack clippy --no-default-features | |
| rust-coverage: | |
| name: "Rust tests (coverage)" | |
| timeout-minutes: 120 | |
| permissions: | |
| id-token: write | |
| strategy: | |
| matrix: | |
| suite: | |
| - tests | |
| - tpc-h | |
| - ffi | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - family=m7i+m7i-flex+m7a | |
| - cpu=8 | |
| - image=ubuntu24-full-x64 | |
| - extras=s3-cache | |
| - tag=rust-coverage | |
| - tag=suite-${{ matrix.suite }} | |
| env: | |
| RUSTFLAGS: "-Cinstrument-coverage -A warnings" | |
| CARGO_INCREMENTAL: 0 # Disable incremental compilation to get accurate coverage | |
| LLVM_PROFILE_FILE: "target/coverage/vortex-%p-%m.profraw" | |
| GRCOV_OUTPUT_FILE: "target/coverage/vortex.lcov" | |
| steps: | |
| - uses: runs-on/action@v2 | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v5 | |
| - uses: ./.github/actions/setup-rust | |
| - name: Install grcov | |
| uses: taiki-e/install-action@grcov | |
| - name: Install nextest | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: [email protected] | |
| - name: Rust Tests | |
| if: ${{ matrix.suite == 'tests' }} | |
| run: | | |
| cargo +nightly nextest run --locked --workspace --all-features --no-fail-fast | |
| - name: Run TPC-H | |
| if: ${{ matrix.suite == 'tpc-h' }} | |
| run: | | |
| cargo run --bin query_bench -- tpch -i2 --scale-factor 0.1 | |
| - name: Run FFI Example | |
| if: ${{ matrix.suite == 'ffi' }} | |
| run: | | |
| cargo build -p vortex-ffi | |
| cargo run -p vortex-ffi --example hello_vortex | |
| - name: Generate coverage report | |
| run: | | |
| rustup component add llvm-tools-preview | |
| grcov . --binary-path target/debug/ -s . -t lcov --llvm --ignore-not-existing \ | |
| --ignore '../*' --ignore '/*' --ignore 'fuzz/*' --ignore 'bench-vortex/*' \ | |
| --ignore 'home/*' --ignore 'xtask/*' --ignore 'target/*' --ignore 'vortex-error/*' \ | |
| --ignore 'vortex-python/*' --ignore 'vortex-jni/*' --ignore 'vortex-flatbuffers/*' \ | |
| --ignore 'vortex-proto/*' --ignore 'vortex-tui/*' --ignore 'vortex-datafusion/examples/*' \ | |
| --ignore 'vortex-ffi/examples/*' --ignore '*/arbitrary/*' --ignore '*/arbitrary.rs' --ignore 'vortex-cxx/*' \ | |
| -o ${{ env.GRCOV_OUTPUT_FILE }} | |
| - name: Codecov | |
| uses: codecov/codecov-action@v5 | |
| with: | |
| name: run-${{ matrix.suite }} | |
| files: ${{ env.GRCOV_OUTPUT_FILE }} | |
| disable_search: true | |
| flags: ${{ matrix.suite }} | |
| use_oidc: true | |
| rust-test: | |
| name: "Rust tests (sanitizer)" | |
| timeout-minutes: 120 | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - family=m7i+m7i-flex+m7a | |
| - cpu=8 | |
| - image=ubuntu24-full-x64 | |
| - extras=s3-cache | |
| - tag=rust-test-sanitizer | |
| env: | |
| # Add debug symbols and enable ASAN/LSAN with better output | |
| RUSTFLAGS: "-A warnings -Zsanitizer=address -Zsanitizer=leak --cfg disable_loom -C debuginfo=2 -C opt-level=0 -C strip=none" | |
| ASAN_OPTIONS: "symbolize=1:print_stats=1:check_initialization_order=1:detect_leaks=1:halt_on_error=0:verbosity=1:leak_check_at_exit=1" | |
| LSAN_OPTIONS: "verbosity=1:report_objects=1" | |
| ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" | |
| # Keep frame pointers for better stack traces | |
| CARGO_PROFILE_DEV_DEBUG: "true" | |
| CARGO_PROFILE_TEST_DEBUG: "true" | |
| steps: | |
| - uses: runs-on/action@v2 | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v5 | |
| - name: Install llvm | |
| uses: aminya/setup-cpp@v1 | |
| with: | |
| compiler: llvm | |
| - uses: ./.github/actions/setup-rust | |
| with: | |
| toolchain: nightly | |
| components: "rust-src, rustfmt, clippy, llvm-tools-preview" | |
| - name: Install nextest | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: [email protected] | |
| - name: Rust Tests | |
| run: | | |
| # Build with full debug info first (helps with caching) | |
| cargo +nightly build --locked --workspace --all-features --target x86_64-unknown-linux-gnu | |
| # Run tests with sanitizers and debug output | |
| cargo +nightly nextest run \ | |
| --locked \ | |
| --workspace \ | |
| --all-features \ | |
| --no-fail-fast \ | |
| --target x86_64-unknown-linux-gnu \ | |
| --verbose | |
| build-java: | |
| name: "Java" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: actions/setup-java@v5 | |
| with: | |
| distribution: "corretto" | |
| java-version: "17" | |
| - uses: ./.github/actions/setup-rust | |
| - run: ./gradlew test --parallel | |
| working-directory: ./java | |
| bench-codspeed: | |
| strategy: | |
| matrix: | |
| shard: [1, 2] | |
| name: "Benchmark with Codspeed (Shard #${{ matrix.shard }})" | |
| timeout-minutes: 120 | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - family=c6id.8xlarge | |
| - extras=s3-cache | |
| - image=ubuntu24-full-x64 | |
| - tag=bench-codspeed | |
| steps: | |
| - uses: runs-on/action@v2 | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v5 | |
| - uses: ./.github/actions/setup-rust | |
| - name: Install Codspeed | |
| shell: bash | |
| run: cargo install --force cargo-codspeed --locked | |
| - name: Build benchmarks (shard 1) | |
| env: | |
| RUSTFLAGS: "-C target-feature=+avx2" | |
| if: ${{ matrix.shard == 1 }} | |
| run: | | |
| cargo codspeed build --features test-harness \ | |
| -p vortex-buffer \ | |
| -p vortex-array \ | |
| -p vortex \ | |
| -p vortex-fastlanes \ | |
| --profile bench | |
| - name: Build benchmarks (shard 2) | |
| env: | |
| RUSTFLAGS: "-C target-feature=+avx2" | |
| if: ${{ matrix.shard == 2 }} | |
| run: | | |
| cargo codspeed build --features test-harness \ | |
| --exclude bench-vortex \ | |
| --exclude vortex-datafusion \ | |
| --exclude vortex-duckdb \ | |
| --exclude vortex-fuzz \ | |
| --exclude vortex-python \ | |
| --exclude vortex-tui \ | |
| --exclude xtask \ | |
| --exclude vortex-buffer \ | |
| --exclude vortex-array \ | |
| --exclude vortex \ | |
| --exclude vortex-fastlanes \ | |
| --workspace \ | |
| --profile bench | |
| - name: Run benchmarks | |
| uses: CodSpeedHQ/action@v4 | |
| with: | |
| run: cargo codspeed run | |
| token: ${{ secrets.CODSPEED_TOKEN }} | |
| mode: "instrumentation" | |
| license-check-and-audit-check: | |
| name: License Check and Audit Check | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| strategy: | |
| matrix: | |
| checks: | |
| - advisories | |
| - bans licenses sources | |
| # Prevent sudden announcement of a new advisory from failing ci: | |
| continue-on-error: ${{ matrix.checks == 'advisories' }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: EmbarkStudios/cargo-deny-action@v2 | |
| with: | |
| command: check ${{ matrix.checks }} | |
| cxx-test: | |
| name: "C++ build" | |
| timeout-minutes: 120 | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - family=m7i+m7i-flex+m7a | |
| - cpu=8 | |
| - image=ubuntu24-full-x64 | |
| - extras=s3-cache | |
| - tag=cxx-build | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./.github/actions/setup-rust | |
| - name: Build and run C++ unit tests | |
| working-directory: vortex-cxx/ | |
| run: | | |
| mkdir -p build | |
| cd build | |
| cmake -DVORTEX_ENABLE_TESTING=ON -DVORTEX_ENABLE_ASAN=ON .. | |
| cmake --build . --parallel $(nproc) | |
| ctest -V | |
| - name: Build and run the example in release mode | |
| working-directory: vortex-cxx/examples | |
| run: | | |
| mkdir -p build | |
| cd build | |
| cmake -DCMAKE_BUILD_TYPE=Release .. | |
| cmake --build . --parallel $(nproc) | |
| ./hello-vortex ../goldenfiles/example.vortex | |
| wasm-integration: | |
| name: "wasm-integration" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./.github/actions/setup-rust | |
| with: | |
| toolchain: nightly-2025-06-26 | |
| targets: "wasm32-wasip1" | |
| components: "rust-src" | |
| - name: Setup Wasmer | |
| uses: wasmerio/[email protected] | |
| # there is a compiler bug in nightly (but not in nightly-2025-06-26) | |
| - run: cargo +nightly-2025-06-26 -Zbuild-std=panic_abort,std build --target wasm32-wasip1 | |
| working-directory: ./wasm-test | |
| - run: wasmer run ./target/wasm32-wasip1/debug/wasm-test.wasm | |
| working-directory: ./wasm-test | |
| miri: | |
| name: "Rust tests (miri)" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| env: | |
| MIRIFLAGS: -Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-disable-isolation -Zmiri-env-forward=RUST_BACKTRACE | |
| RUSTFLAGS: "-A warnings" | |
| RUST_BACKTRACE: full | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./.github/actions/setup-rust | |
| with: | |
| toolchain: nightly | |
| components: "rust-src, rustfmt, clippy, miri" | |
| - uses: taiki-e/install-action@v2 | |
| with: | |
| tool: [email protected] | |
| - name: Run Miri | |
| run: cargo +nightly miri nextest run --no-fail-fast -p vortex-buffer -p vortex-ffi | |
| generated-files: | |
| name: "Check generated source files are up to date" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: ./.github/actions/setup-rust | |
| with: | |
| toolchain: nightly | |
| - uses: ./.github/actions/setup-flatc | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: "regenerate all .fbs/.proto Rust code" | |
| run: | | |
| cargo xtask generate-fbs | |
| cargo xtask generate-proto | |
| - name: "regenerate FFI header file" | |
| run: | | |
| cargo +nightly build -p vortex-ffi | |
| - name: "Make sure no files changed after regenerating" | |
| run: | | |
| git status --porcelain | |
| test -z "$(git status --porcelain)" | |
| - name: "Checkout develop flatbuffers" | |
| working-directory: vortex-flatbuffers/ | |
| run: | | |
| cp -R flatbuffers flatbuffers.HEAD | |
| git fetch origin develop --depth 1 | |
| git checkout origin/develop -- flatbuffers | |
| - name: "Verify flatbuffer back-compat" | |
| working-directory: vortex-flatbuffers/ | |
| run: | | |
| find flatbuffers/ -type f -name "*.fbs" | sed 's/^flatbuffers\///' | xargs -I{} -n1 flatc -I flatbuffers.HEAD --conform-includes flatbuffers --conform flatbuffers/{} flatbuffers.HEAD/{} |