chore[array]: iterative execute method p1 (#6860) #26725
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| # Concurrency control: | |
| # - PRs: new commits on a feature branch will cancel in-progress (outdated) runs. | |
| # - Push to develop: runs queue sequentially, never cancelled. This allows us to have benchmarks | |
| # run on every commit for our benchmarks website. | |
| # - `workflow_dispatch`: groups by branch and queues if run on develop. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} | |
| on: | |
| push: | |
| branches: [develop] | |
| pull_request: { } | |
| workflow_dispatch: { } | |
| permissions: | |
| actions: read | |
| contents: read | |
| checks: write # audit-check creates checks | |
| issues: write # audit-check creates issues | |
| env: | |
| CARGO_TERM_COLOR: auto | |
| RUST_BACKTRACE: 1 | |
| NIGHTLY_TOOLCHAIN: nightly-2026-02-05 | |
| jobs: | |
| lint-toml: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: spiraldb/actions/.github/actions/lint-toml@0.18.5 | |
| validate-workflow-yaml: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Validate YAML file | |
| run: | | |
| # Lint the workflows and yamllint's configuration file. | |
| yamllint \ | |
| --strict \ | |
| -c .yamllint.yaml \ | |
| .github/ | |
| python-lint: | |
| name: "Python (lint)" | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=python-lint', github.run_id) | |
| || 'ubuntu-latest' }} | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| # Use uvx for ruff to avoid building the Rust extension (saves ~4.5 min) | |
| - name: Python Lint - Format | |
| run: uvx ruff format --check . | |
| - name: Python Lint - Ruff | |
| run: uvx ruff check . | |
| # PyRight needs the project for type information, so use uv run | |
| - name: Python Lint - PyRight | |
| env: | |
| MATURIN_PEP517_ARGS: "--profile dev" | |
| run: uv run basedpyright vortex-python | |
| python-test: | |
| name: "Python (test)" | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=python-test', github.run_id) | |
| || 'ubuntu-latest' }} | |
| timeout-minutes: 40 | |
| env: | |
| RUST_LOG: "info,uv=debug" | |
| MATURIN_PEP517_ARGS: "--profile dev" | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Pytest - Vortex | |
| run: | | |
| uv run --all-packages pytest --benchmark-disable -n auto test/ | |
| working-directory: vortex-python/ | |
| - name: Setup benchmark environment | |
| run: sudo bash scripts/setup-benchmark.sh | |
| - name: Pytest Benchmarks - Vortex | |
| run: | | |
| bash ../scripts/bench-taskset.sh uv run --all-packages pytest --benchmark-only benchmark/ | |
| working-directory: vortex-python/ | |
| - name: Doctest - PyVortex | |
| run: | | |
| uv run --all-packages make doctest | |
| working-directory: docs/ | |
| - name: Ensure docs build - PyVortex | |
| run: | | |
| uv run --all-packages make html | |
| working-directory: docs/ | |
| python-wheel-build: | |
| name: "Python (wheel build)" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Rust Dependency Cache | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| save-if: ${{ github.ref_name == 'develop' }} | |
| - uses: ./.github/actions/setup-rust | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| enable-sccache: "false" | |
| - uses: mlugg/setup-zig@v2.2.1 | |
| - name: Install uv | |
| uses: spiraldb/actions/.github/actions/setup-uv@0.18.5 | |
| with: | |
| sync: false | |
| prune-cache: false | |
| - name: Ensure wheel and sdist can be built on Linux - PyVortex | |
| shell: bash | |
| run: | | |
| echo "Clearing wheel target directory" | |
| rm -rf ../target/wheels/ | |
| uv venv | |
| uv tool run maturin@1.10 build --interpreter python3.11 --zig | |
| uv tool run maturin@1.10 build --interpreter python3.11 --zig --sdist | |
| file_count=$(ls -1 ../target/wheels/ | wc -l) | |
| if [[ $file_count -ne 2 ]]; then | |
| echo "Unexpected number of files detected ${file_count}:" | |
| ls ../target/wheels/ | |
| exit 1 | |
| else | |
| echo "Generated two files" | |
| fi | |
| working-directory: vortex-python/ | |
| rust-docs: | |
| name: "Rust (docs)" | |
| timeout-minutes: 40 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-small/image=ubuntu24-full-x64-pre-v2/tag=rust-docs', github.run_id) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Docs | |
| run: | | |
| RUSTDOCFLAGS="-D warnings" cargo doc --no-deps | |
| # nextest doesn't support doc tests, so we run it here | |
| cargo test --doc --workspace --all-features --exclude vortex-cxx --exclude vortex-jni --exclude vortex-ffi --exclude xtask --no-fail-fast | |
| build-rust: | |
| name: "Rust build (${{matrix.config.name}})" | |
| timeout-minutes: 40 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner={1}/image=ubuntu24-full-x64-pre-v2/tag={2}', github.run_id, matrix.config.runner, matrix.config.name) | |
| || 'ubuntu-latest' }} | |
| env: | |
| # disable lints for build, they will be caught in Rust lint job. | |
| RUSTFLAGS: "-A warnings" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: | |
| - name: "all-features" | |
| runner: amd64-large | |
| args: "--all-features --all-targets" | |
| - name: "default features" | |
| runner: amd64-large | |
| args: "--all-targets" | |
| - name: "with tokio dispatcher" | |
| runner: amd64-small | |
| # Only build the crates that have the tokio features, not re-building other crates with no-default-features | |
| args: "--no-default-features --features tokio --all-targets -p vortex -p vortex-io -p vortex-file -p vortex-layout" | |
| - name: "wasm32 with default features" | |
| runner: amd64-medium | |
| target: wasm32-unknown-unknown | |
| env: | |
| rustflags: "RUSTFLAGS='-A warnings --cfg getrandom_backend=\"wasm_js\"'" | |
| args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-cuda --exclude vortex-cub --exclude vortex-nvcomp --exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd --exclude vortex-test-e2e-cuda --exclude vortex-sqllogictest" | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Install wasm32 target | |
| if: ${{ matrix.config.target == 'wasm32-unknown-unknown' }} | |
| run: rustup target add wasm32-unknown-unknown | |
| - uses: ./.github/actions/check-rebuild | |
| with: | |
| command: "${{matrix.config.env.rustflags}} cargo hack build --locked ${{matrix.config.args}} --ignore-private" | |
| - name: "Make sure no files changed after build" | |
| run: | | |
| git status --porcelain | |
| test -z "$(git status --porcelain)" | |
| check-min-deps: | |
| name: "Check build with minimal dependencies" | |
| timeout-minutes: 40 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-min-deps', github.run_id) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - run: cargo minimal-versions check --direct --workspace --ignore-private | |
| rust-lint: | |
| name: "Rust (lint)" | |
| timeout-minutes: 40 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=rust-lint', github.run_id) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Install nightly for fmt | |
| run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rustfmt | |
| - name: Rust Lint - Format | |
| run: cargo +$NIGHTLY_TOOLCHAIN fmt --all --check | |
| - name: Rustc check | |
| run: RUSTFLAGS="-D warnings" cargo check --locked --all-features --all-targets | |
| - name: Rustc check (release) | |
| run: RUSTFLAGS="-D warnings" cargo check --locked --all-features --all-targets --release | |
| - name: Rust Lint - Clippy All Features | |
| run: cargo clippy --locked --all-features --all-targets -- -D warnings | |
| - name: Rust Lint - Clippy Default Features | |
| run: cargo clippy --locked --all-targets -- -D warnings | |
| cpp-lint: | |
| name: "C/C++ (lint)" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: C/C++ Lint - clang-format | |
| run: | | |
| git ls-files vortex-cuda vortex-cxx vortex-duckdb vortex-ffi \ | |
| | grep -E '\.(cpp|hpp|cu|cuh|h)$' \ | |
| | grep -v 'kernels/src/bit_unpack_.*\.cu$' \ | |
| | xargs clang-format --dry-run --Werror --style=file | |
| rust-lint-no-default: | |
| name: "Rust (lint, no default)" | |
| timeout-minutes: 40 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-lint-no-default', github.run_id) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Rust Lint - Clippy No Default Features | |
| shell: bash | |
| run: | | |
| cargo hack --no-dev-deps --ignore-private clippy --no-default-features -- -D warnings | |
| public-api: | |
| name: "Public API lock files" | |
| timeout-minutes: 40 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-xsmall/image=ubuntu24-full-x64-pre-v2/tag=public-api', github.run_id) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Install nightly for public-api | |
| run: rustup toolchain install $NIGHTLY_TOOLCHAIN | |
| - name: Regenerate public API lock files | |
| run: cargo +$NIGHTLY_TOOLCHAIN xtask public-api | |
| - name: Verify lock files are up to date | |
| run: | | |
| if ! git diff --quiet '**/public-api.lock'; then | |
| git diff -U0 '**/public-api.lock' | |
| echo "" | |
| echo "" | |
| echo "Changed files:" | |
| git diff --name-only '**/public-api.lock' | |
| echo "" | |
| echo "" | |
| echo "Public API lock files are out of date." | |
| echo "Run './scripts/public-api.sh' locally to" | |
| echo "regenerate them, then commit the changes." | |
| exit 1 | |
| fi | |
| rust-coverage: | |
| name: "Rust tests (coverage) (${{ matrix.suite }})" | |
| timeout-minutes: 40 | |
| permissions: | |
| id-token: write | |
| strategy: | |
| matrix: | |
| include: | |
| - suite: tests | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=rust-coverage-suite-{1}', github.run_id, matrix.suite) | |
| || 'ubuntu-latest' }} | |
| env: | |
| RUSTFLAGS: "-Cinstrument-coverage -A warnings" | |
| CARGO_INCREMENTAL: 0 # Disable incremental compilation to get accurate coverage | |
| LLVM_PROFILE_FILE: "target/coverage/vortex-%p-%m.profraw" | |
| GRCOV_OUTPUT_FILE: "target/coverage/vortex.lcov" | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Rust Tests | |
| if: ${{ matrix.suite == 'tests' }} | |
| run: | | |
| cargo nextest run --locked --workspace --all-features --no-fail-fast | |
| - name: Generate coverage report | |
| run: | | |
| grcov . --binary-path target/debug/ -s . -t lcov --llvm --ignore-not-existing \ | |
| --threads $(nproc) \ | |
| --ignore '../*' --ignore '/*' --ignore 'fuzz/*' --ignore 'vortex-bench/*' \ | |
| --ignore 'home/*' --ignore 'xtask/*' --ignore 'target/*' --ignore 'vortex-error/*' \ | |
| --ignore 'vortex-python/*' --ignore 'vortex-jni/*' --ignore 'vortex-flatbuffers/*' \ | |
| --ignore 'vortex-proto/*' --ignore 'vortex-tui/*' --ignore 'vortex-datafusion/examples/*' \ | |
| --ignore 'vortex-ffi/examples/*' --ignore '*/arbitrary/*' --ignore '*/arbitrary.rs' --ignore 'vortex-cxx/*' \ | |
| --ignore benchmarks/* --ignore 'vortex-test/*' \ | |
| -o ${{ env.GRCOV_OUTPUT_FILE }} | |
| - name: Codecov | |
| uses: codecov/codecov-action@v5 | |
| with: | |
| name: run-${{ matrix.suite }} | |
| files: ${{ env.GRCOV_OUTPUT_FILE }} | |
| disable_search: true | |
| flags: ${{ matrix.suite }} | |
| use_oidc: true | |
| rust-test: | |
| name: "Rust tests (sanitizer)" | |
| timeout-minutes: 40 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-test-sanitizer', github.run_id) | |
| || 'ubuntu-latest' }} | |
| env: | |
| # Add debug symbols and enable ASAN/LSAN with better output | |
| ASAN_OPTIONS: "symbolize=1:print_stats=1:check_initialization_order=1:detect_leaks=1:halt_on_error=0:verbosity=1:leak_check_at_exit=1" | |
| LSAN_OPTIONS: "verbosity=1:report_objects=1" | |
| ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" | |
| # Link against DuckDB debug build | |
| VX_DUCKDB_DEBUG: "1" | |
| # Keep frame pointers for better stack traces | |
| CARGO_PROFILE_DEV_DEBUG: "true" | |
| CARGO_PROFILE_TEST_DEBUG: "true" | |
| # Skip slow tests that are too expensive under sanitizer | |
| VORTEX_SKIP_SLOW_TESTS: "1" | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Install nightly for sanitizer | |
| run: | | |
| rustup toolchain install $NIGHTLY_TOOLCHAIN | |
| rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview | |
| - name: Rust Tests | |
| env: | |
| RUSTFLAGS: "-A warnings -Zsanitizer=address -Zsanitizer=leak --cfg disable_loom --cfg vortex_nightly -C debuginfo=2 -C opt-level=0 -C strip=none" | |
| run: | | |
| # Build with full debug info first (helps with caching) | |
| cargo +$NIGHTLY_TOOLCHAIN build --locked --all-features \ | |
| --target x86_64-unknown-linux-gnu \ | |
| -p vortex-buffer -p vortex-ffi -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array | |
| # Run tests with sanitizers and debug output | |
| cargo +$NIGHTLY_TOOLCHAIN nextest run \ | |
| --locked \ | |
| --all-features \ | |
| --no-fail-fast \ | |
| --target x86_64-unknown-linux-gnu \ | |
| -p vortex-buffer -p vortex-ffi -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array | |
| cuda-build-lint: | |
| if: github.repository == 'vortex-data/vortex' | |
| name: "CUDA build & lint" | |
| timeout-minutes: 40 | |
| runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-build | |
| steps: | |
| - uses: runs-on/action@v2 | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-rust | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: ./.github/actions/check-rebuild | |
| with: | |
| command: >- | |
| cargo build --locked --all-features --all-targets | |
| -p vortex-cuda -p vortex-cub -p vortex-nvcomp | |
| -p gpu-scan-cli -p vortex-test-e2e-cuda | |
| - name: Clippy CUDA crates | |
| run: | | |
| cargo clippy --locked --all-features --all-targets \ | |
| -p vortex-cuda \ | |
| -p vortex-cub \ | |
| -p vortex-nvcomp \ | |
| -p gpu-scan-cli \ | |
| -p vortex-test-e2e-cuda \ | |
| -- -D warnings | |
| cuda-test: | |
| if: github.repository == 'vortex-data/vortex' | |
| name: "CUDA tests" | |
| timeout-minutes: 30 | |
| runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-tests | |
| steps: | |
| - uses: runs-on/action@v2 | |
| with: | |
| sccache: s3 | |
| - name: Display NVIDIA SMI details | |
| run: | | |
| nvidia-smi | |
| nvidia-smi -L | |
| nvidia-smi -q -d Memory | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-rust | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install nextest | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: nextest | |
| - name: Rust Tests | |
| env: | |
| FLAT_LAYOUT_INLINE_ARRAY_NODE: true | |
| run: | | |
| cargo nextest run \ | |
| --locked \ | |
| -p vortex-file \ | |
| -p vortex-cuda \ | |
| -p vortex-cub \ | |
| -p vortex-nvcomp \ | |
| -p vortex-test-e2e-cuda \ | |
| --all-features \ | |
| --no-fail-fast \ | |
| --target x86_64-unknown-linux-gnu \ | |
| --verbose | |
| cuda-test-sanitizer: | |
| if: github.repository == 'vortex-data/vortex' | |
| name: "CUDA tests (${{ matrix.sanitizer }})" | |
| timeout-minutes: 30 | |
| runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-sanitizer | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - sanitizer: memcheck | |
| runner_flags: "--tool memcheck --leak-check=full --error-exitcode 1" | |
| # TODO(joe): try to re-enable racecheck, it is hanging in CI. | |
| # - sanitizer: racecheck | |
| # runner_flags: "--tool racecheck --error-exitcode 1" | |
| - sanitizer: synccheck | |
| runner_flags: "--tool synccheck --error-exitcode 1" | |
| - sanitizer: initcheck | |
| runner_flags: "--tool initcheck --error-exitcode 1" | |
| steps: | |
| - uses: runs-on/action@v2 | |
| with: | |
| sccache: s3 | |
| - name: Display NVIDIA SMI details | |
| run: | | |
| nvidia-smi | |
| nvidia-smi -L | |
| nvidia-smi -q -d Memory | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-rust | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build tests | |
| run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu --no-run | |
| - name: "CUDA - ${{ matrix.sanitizer }}" | |
| env: | |
| CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: "compute-sanitizer ${{ matrix.runner_flags }}" | |
| run: cargo test --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu | |
| cuda-test-cudf: | |
| if: github.repository == 'vortex-data/vortex' | |
| name: "CUDA tests (cudf)" | |
| timeout-minutes: 30 | |
| runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-cudf | |
| steps: | |
| - uses: runs-on/action@v2 | |
| with: | |
| sccache: s3 | |
| - name: Display NVIDIA SMI details | |
| run: | | |
| nvidia-smi | |
| nvidia-smi -L | |
| nvidia-smi -q -d Memory | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-rust | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build cudf test library | |
| run: cargo build --locked -p vortex-test-e2e-cuda --target x86_64-unknown-linux-gnu | |
| - name: Download and run cudf-test-harness | |
| run: | | |
| curl -fsSL https://github.com/vortex-data/cudf-test-harness/releases/latest/download/cudf-test-harness-x86_64.tar.gz | tar -xz | |
| cd cudf-test-harness-x86_64 | |
| compute-sanitizer --tool memcheck --error-exitcode 1 ./cudf-test-harness check $GITHUB_WORKSPACE/target/x86_64-unknown-linux-gnu/debug/libvortex_test_e2e_cuda.so | |
| rust-test-other: | |
| name: "Rust tests (${{ matrix.os }})" | |
| timeout-minutes: 40 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: windows-x64 | |
| runner: runs-on=${{ github.run_id }}/pool=windows-x64-pre | |
| fallback_runner: windows-latest | |
| - os: linux-arm64 | |
| runner: runs-on=${{ github.run_id }}/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/tag=rust-test-linux-arm64 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && matrix.runner | |
| || matrix.fallback_runner }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - name: Setup (Windows) | |
| if: matrix.os == 'windows-x64' | |
| run: | | |
| $flags = '-C debuginfo=0' | |
| echo "RUSTFLAGS=$flags" >> $env:GITHUB_ENV | |
| echo "C:\rust\cargo\bin" >> $env:GITHUB_PATH | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Rust Tests (Windows) | |
| if: matrix.os == 'windows-x64' | |
| run: | | |
| cargo nextest run --locked --workspace --all-features --no-fail-fast ` | |
| --exclude vortex-bench --exclude vortex-python --exclude vortex-duckdb ` | |
| --exclude vortex-fuzz --exclude vortex-cuda --exclude vortex-nvcomp ` | |
| --exclude vortex-cub --exclude vortex-test-e2e-cuda --exclude duckdb-bench ` | |
| --exclude lance-bench --exclude datafusion-bench --exclude random-access-bench ` | |
| --exclude compress-bench --exclude xtask --exclude vortex-datafusion ` | |
| --exclude gpu-scan-cli --exclude vortex-sqllogictest | |
| - name: Rust Tests (Other) | |
| if: matrix.os != 'windows-x64' | |
| run: | | |
| cargo nextest run --locked --workspace --all-features --no-fail-fast --exclude vortex-bench --exclude xtask --exclude vortex-sqllogictest | |
| - uses: ./.github/actions/check-rebuild | |
| if: matrix.os != 'windows-x64' | |
| with: | |
| command: "cargo test --locked --workspace --all-features --no-run --exclude vortex-bench --exclude xtask --exclude vortex-sqllogictest" | |
| - name: Alert incident.io | |
| if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/develop' | |
| uses: ./.github/actions/alert-incident-io | |
| with: | |
| api-key: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }} | |
| alert-title: "Rust tests (${{ matrix.os }}) failed on develop" | |
| deduplication-key: ci-rust-test-${{ matrix.os }}-failure | |
| build-java: | |
| name: "Java" | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=java', github.run_id) | |
| || 'ubuntu-latest' }} | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - run: ./gradlew test --parallel | |
| working-directory: ./java | |
| bench-codspeed: | |
| strategy: | |
| matrix: | |
| include: | |
| - { shard: 1, name: "Core foundation", packages: "vortex-buffer vortex-error" } | |
| - { shard: 2, name: "Arrays", packages: "vortex-array", features: "--features _test-harness" } | |
| - { shard: 3, name: "Main library", packages: "vortex" } | |
| - { shard: 4, name: "Encodings 1", packages: "vortex-alp vortex-bytebool vortex-datetime-parts" } | |
| - { shard: 5, name: "Encodings 2", packages: "vortex-decimal-byte-parts vortex-fastlanes vortex-fsst", features: "--features _test-harness" } | |
| - { shard: 6, name: "Encodings 3", packages: "vortex-pco vortex-runend vortex-sequence" } | |
| - { shard: 7, name: "Encodings 4", packages: "vortex-sparse vortex-zigzag vortex-zstd" } | |
| - { shard: 8, name: "Storage formats", packages: "vortex-flatbuffers vortex-proto vortex-btrblocks" } | |
| name: "Benchmark with Codspeed (Shard #${{ matrix.shard }})" | |
| timeout-minutes: 40 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=bench-codspeed-{1}', github.run_id, matrix.shard) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - name: Setup benchmark environment | |
| run: sudo bash scripts/setup-benchmark.sh | |
| - uses: ./.github/actions/setup-prebuild | |
| - uses: ./.github/actions/system-info | |
| - name: Install Codspeed | |
| uses: taiki-e/cache-cargo-install-action@1ce845d7672f06bdf703007779598583b6c62f61 | |
| with: | |
| tool: cargo-codspeed | |
| - name: Build benchmarks | |
| env: | |
| RUSTFLAGS: "-C target-feature=+avx2 -C debug-assertions=yes" | |
| run: cargo codspeed build ${{ matrix.features }} $(printf -- '-p %s ' ${{ matrix.packages }}) --profile bench | |
| - name: Run benchmarks | |
| uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab | |
| with: | |
| run: bash scripts/bench-taskset.sh cargo codspeed run | |
| token: ${{ secrets.CODSPEED_TOKEN }} | |
| mode: "simulation" | |
| license-check-and-audit-check: | |
| name: License Check and Audit Check | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 40 | |
| strategy: | |
| matrix: | |
| checks: | |
| - advisories | |
| - bans licenses sources | |
| # Prevent sudden announcement of a new advisory from failing ci: | |
| continue-on-error: ${{ matrix.checks == 'advisories' }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: EmbarkStudios/cargo-deny-action@v2 | |
| with: | |
| command: check ${{ matrix.checks }} | |
| cxx-test: | |
| name: "C++ build" | |
| timeout-minutes: 40 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=cxx-build', github.run_id) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Build and run C++ unit tests | |
| run: | | |
| mkdir -p vortex-cxx/build | |
| cmake -S vortex-cxx -B vortex-cxx/build -DVORTEX_ENABLE_TESTING=ON -DVORTEX_ENABLE_ASAN=ON | |
| cmake --build vortex-cxx/build --parallel $(nproc) | |
| ctest --test-dir vortex-cxx/build -j $(nproc) -V | |
| - name: Build and run the example in release mode | |
| run: | | |
| cmake -S vortex-cxx/examples -B vortex-cxx/examples/build -DCMAKE_BUILD_TYPE=Release | |
| cmake --build vortex-cxx/examples/build --parallel $(nproc) | |
| vortex-cxx/examples/build/hello-vortex vortex-cxx/examples/goldenfiles/example.vortex | |
| - uses: ./.github/actions/check-rebuild | |
| with: | |
| command: "cargo build --locked -p vortex-cxx --lib" | |
| sqllogic-test: | |
| name: "SQL logic tests" | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=sql-logic-test', github.run_id) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Run sqllogictest tests | |
| run: | | |
| ./vortex-sqllogictest/slt/tpch/generate_data.sh | |
| cargo test -p vortex-sqllogictest --test sqllogictests | |
| - uses: ./.github/actions/check-rebuild | |
| with: | |
| command: "cargo test -p vortex-sqllogictest --test sqllogictests --no-run" | |
| wasm-integration: | |
| name: "wasm-integration" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-rust | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| targets: "wasm32-wasip1" | |
| - name: Setup Wasmer | |
| shell: bash | |
| run: | | |
| curl https://get.wasmer.io -sSfL | sh | |
| echo "$HOME/.wasmer/bin" >> $GITHUB_PATH | |
| - run: cargo build --target wasm32-wasip1 | |
| working-directory: ./wasm-test | |
| - run: wasmer run ./target/wasm32-wasip1/debug/wasm-test.wasm | |
| working-directory: ./wasm-test | |
| miri: | |
| name: "Rust tests (miri)" | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-miri', github.run_id) | |
| || 'ubuntu-latest' }} | |
| timeout-minutes: 40 | |
| env: | |
| MIRIFLAGS: -Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-disable-isolation -Zmiri-env-forward=RUST_BACKTRACE | |
| RUSTFLAGS: "-A warnings --cfg vortex_nightly" | |
| RUST_BACKTRACE: full | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Install nightly with miri | |
| run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rust-src,rustfmt,clippy,miri | |
| - name: Run Miri | |
| run: cargo +$NIGHTLY_TOOLCHAIN miri nextest run --no-fail-fast -p vortex-buffer -p vortex-ffi | |
| generated-files: | |
| name: "Check generated source files are up to date" | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=generated-files', github.run_id) | |
| || 'ubuntu-latest' }} | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Install nightly for cbindgen macro expansion | |
| run: rustup toolchain install $NIGHTLY_TOOLCHAIN | |
| - name: "regenerate all .fbs/.proto Rust code" | |
| run: | | |
| cargo xtask generate-fbs | |
| cargo xtask generate-proto | |
| - name: "regenerate FFI header file" | |
| run: | | |
| cargo +$NIGHTLY_TOOLCHAIN build -p vortex-ffi | |
| - name: "Make sure no files changed after regenerating" | |
| run: | | |
| git status --porcelain | |
| test -z "$(git status --porcelain)" | |
| - name: "Checkout develop flatbuffers" | |
| working-directory: vortex-flatbuffers/ | |
| run: | | |
| cp -R flatbuffers flatbuffers.HEAD | |
| git fetch origin develop --depth 1 | |
| git checkout origin/develop -- flatbuffers | |
| - name: "Verify flatbuffer back-compat" | |
| working-directory: vortex-flatbuffers/ | |
| run: | | |
| find flatbuffers/ -type f -name "*.fbs" | sed 's/^flatbuffers\///' | xargs -I{} -n1 flatc -I flatbuffers.HEAD --conform-includes flatbuffers --conform flatbuffers/{} flatbuffers.HEAD/{} | |
| ffi-c-test: | |
| name: "C API test build" | |
| timeout-minutes: 10 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=cxx-build', github.run_id) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: "regenerate FFI header file" | |
| run: | | |
| cargo +$NIGHTLY_TOOLCHAIN build -p vortex-ffi | |
| - name: Build and run C++ unit tests | |
| run: | | |
| cd vortex-ffi | |
| mkdir build | |
| cmake -Bbuild | |
| cmake --build build -j $(nproc) | |
| ctest --test-dir build -j $(nproc) | |
| check-java-publish-build: | |
| runs-on: ${{ matrix.target.runs-on }} | |
| container: | |
| image: "ubuntu:20.04" | |
| timeout-minutes: 40 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: | |
| - { os: ubuntu, runs-on: "ubuntu-24.04-arm", target: aarch64-unknown-linux-gnu } | |
| - { os: ubuntu, runs-on: "ubuntu-24.04", target: x86_64-unknown-linux-gnu } | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/prepare-java-linux | |
| - uses: actions/setup-java@v5 | |
| with: | |
| distribution: "corretto" | |
| java-version: "17" | |
| - uses: ./.github/actions/setup-rust | |
| with: | |
| targets: ${{ matrix.target.target }} | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| enable-sccache: "false" | |
| - run: cargo build --package vortex-jni | |
| compat-check: | |
| name: "Compat check" | |
| uses: ./.github/workflows/compat-validation.yml | |
| with: | |
| mode: last | |
| rust-publish-dry-run: | |
| name: "Rust publish dry-run" | |
| timeout-minutes: 40 | |
| runs-on: >- | |
| ${{ github.repository == 'vortex-data/vortex' | |
| && format('runs-on={0}/runner=amd64-xsmall/image=ubuntu24-full-x64-pre-v2/tag=rust-publish-dry-run', github.run_id) | |
| || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@v2 | |
| if: github.repository == 'vortex-data/vortex' | |
| with: | |
| sccache: s3 | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/setup-prebuild | |
| - name: Install cargo-edit | |
| uses: taiki-e/cache-cargo-install-action@1ce845d7672f06bdf703007779598583b6c62f61 | |
| with: | |
| tool: cargo-edit | |
| - name: Set Version | |
| run: | | |
| # This is mostly a dummy version, we don't actually publish anything but it cannot exist in crates.io | |
| cargo set-version --workspace 0.100000.0 | |
| - name: Test Release | |
| run: | | |
| cargo publish --dry-run --no-verify --allow-dirty --workspace |