bug[vortex-array]: take nullability chunked array not respects validity #13338

Workflow file for this run

	name: CI

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }}

	on:
	push:
	branches: [ "develop" ]
	pull_request: { }
	workflow_dispatch: { }

	permissions:
	actions: read
	contents: read
	checks: write # audit-check creates checks
	issues: write # audit-check creates issues

	env:
	CARGO_TERM_COLOR: always
	RUST_BACKTRACE: 1

	jobs:
	lint-toml:
	runs-on: ubuntu-latest
	timeout-minutes: 120
	steps:
	- uses: actions/checkout@v4
	- uses: spiraldb/actions/.github/actions/lint-toml@0.10.0

	validate-workflow-yaml:
	runs-on: ubuntu-latest
	timeout-minutes: 120
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: "recursive"
	- name: Validate YAML file
	run: \|
	# Lint the workflows and yamllint's configuration file.
	yamllint \
	-c yamllint-config.yaml \
	yamllint-config.yaml \
	.github/workflows

	python-lint:
	name: "Python (lint)"
	runs-on: ubuntu-latest
	timeout-minutes: 120
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: "recursive"
	- name: Install uv
	uses: spiraldb/actions/.github/actions/setup-uv@0.11.0
	with:
	sync: false
	- name: Python Lint - Format
	run: uv run ruff format --check .
	- name: Python Lint - Ruff
	run: uv run ruff check .

	python-test:
	name: "Python (test)"
	runs-on: ubuntu-latest
	timeout-minutes: 120
	env:
	RUST_LOG: "info,uv=debug"
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: "recursive"
	- uses: ./.github/actions/cleanup
	- uses: rui314/setup-mold@v1
	- uses: ./.github/actions/setup-rust
	- name: Install uv
	uses: spiraldb/actions/.github/actions/setup-uv@0.11.0
	with:
	sync: false

	# Ensure the stub files are up-to-date with the public API of the native module.
	- name: Stubtest - Vortex
	run: \|
	uv run --isolated --all-packages stubtest vortex._lib --allowlist python/vortex/_lib/mypy.allowlist
	working-directory: vortex-python/

	- name: Pytest - Vortex
	run: \|
	uv run --isolated --all-packages pytest --benchmark-disable test/
	working-directory: vortex-python/

	- name: Doctest - PyVortex
	run: \|
	uv run --isolated --all-packages make doctest
	working-directory: docs/

	- name: Ensure docs build - PyVortex
	run: \|
	uv run --isolated --all-packages make html
	working-directory: docs/

	- uses: mlugg/setup-zig@v2
	- name: Ensure wheel and sdist can be built on Linux - PyVortex
	shell: bash
	run: \|
	echo "Clearing wheel target directory"
	rm -rf ../target/wheels/

	uv venv
	uv pip install "maturin[zig]"
	uv tool run maturin build --interpreter python3.11 --zig
	uv tool run maturin build --interpreter python3.11 --zig --sdist

	file_count=$(ls -1 ../target/wheels/ \| wc -l)

	if [[ $file_count -ne 2 ]]; then
	echo "Unexpected number of files detected ${file_count}:"
	ls ../target/wheels/
	exit 1
	else
	echo "Generated two files"
	fi
	working-directory: vortex-python/

	rust-docs:
	name: "Rust (docs)"
	timeout-minutes: 120
	runs-on:
	- runs-on=${{ github.run_id }}
	- family=m7i+m7i-flex+m7a
	- cpu=8
	- image=ubuntu24-full-x64
	- tag=rust-docs
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: "recursive"
	- uses: ./.github/actions/setup-rust
	- name: Docs
	run: \|
	cargo doc --no-deps
	# nextest doesn't support doc tests, so we run it here
	cargo test --doc --workspace --all-features --exclude vortex-python --exclude vortex-jni --exclude vortex-ffi --exclude xtask --no-fail-fast

	build-rust:
	name: "Rust build (${{matrix.config.name}})"
	timeout-minutes: 120
	runs-on:
	- runs-on=${{ github.run_id }}
	- family=m7i+m7i-flex+m7a
	- cpu=8
	- image=ubuntu24-full-x64
	- tag=${{ matrix.config.name }}

	env:
	# disable lints for build, they will be caught in Rust lint job.
	RUSTFLAGS: '-A warnings'
	strategy:
	fail-fast: false
	matrix:
	config:
	- name: "all-features"
	command: "build"
	args: "--all-features --all-targets"
	- name: "default features"
	command: "build"
	args: "--all-targets"
	- name: "with tokio dispatcher"
	command: "build"
	args: "--no-default-features --features tokio --all-targets --ignore-unknown-features"
	- name: "with compio dispatcher"
	command: "build"
	args: "--no-default-features --features compio --all-targets --ignore-unknown-features"
	- name: "with tokio+compio dispatcher"
	command: "build"
	args: "--no-default-features --features tokio,compio --all-targets --ignore-unknown-features"
	- name: "wasm32 with default features"
	command: "build"
	target: wasm32-unknown-unknown
	env:
	rustflags: 'RUSTFLAGS="-A warnings --cfg getrandom_backend=\"wasm_js\""'
	args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-datafusion --exclude vortex-duckdb-ext --exclude vortex-tui --exclude vortex-zstd"
	steps:
	- uses: runs-on/action@v1
	- uses: actions/checkout@v4
	with:
	submodules: "recursive"
	- uses: rui314/setup-mold@v1
	- uses: ./.github/actions/setup-c++
	- name: Install wasm32 target
	if: ${{ matrix.config.target == 'wasm32-unknown-unknown' }}
	run: rustup target add wasm32-unknown-unknown
	- name: Install cargo-hack
	uses: taiki-e/install-action@cargo-hack
	- name: Rust Build (${{matrix.config.name}})
	run: ${{matrix.config.env.rustflags}} cargo hack ${{matrix.config.command}} --locked ${{matrix.config.args}} --ignore-private

	build-duckdb:
	name: "DuckDB extension build"
	timeout-minutes: 120
	runs-on:
	- runs-on=${{ github.run_id }}
	- family=m7i+m7i-flex+m7a
	- cpu=16
	- image=ubuntu24-full-x64
	- tag=build-duckdb

	steps:
	- uses: runs-on/action@v1
	- uses: actions/checkout@v4
	with:
	submodules: "recursive"
	- name: Install Ninja
	run: sudo apt-get update && sudo apt-get install -y ninja-build

	- name: DuckDB extension build
	run: GEN=ninja make debug
	working-directory: ${{ github.workspace }}/duckdb-vortex

	- name: DuckDB extension unit tests
	run: ./vortex_tests
	working-directory: ${{ github.workspace }}/duckdb-vortex/build/debug/extension/vortex/test

	- name: DuckDB sql unit tests
	run: make test_debug
	working-directory: ${{ github.workspace }}/duckdb-vortex

	check-min-deps:
	name: "Check build with minimal dependencies"
	timeout-minutes: 120
	runs-on:
	- runs-on=${{ github.run_id }}
	- family=m7i+m7i-flex+m7a
	- cpu=8
	- image=ubuntu24-full-x64
	- tag=rust-min-deps
	steps:
	- uses: runs-on/action@v1
	- uses: actions/checkout@v4
	with:
	submodules: "recursive"
	- uses: rui314/setup-mold@v1
	- uses: ./.github/actions/setup-c++
	- uses: taiki-e/install-action@cargo-hack
	- uses: taiki-e/install-action@cargo-minimal-versions
	- run: cargo minimal-versions check --direct --workspace --ignore-private

	rust-lint:
	name: "Rust (lint)"
	timeout-minutes: 120
	runs-on:
	- runs-on=${{ github.run_id }}
	- family=m7i+m7i-flex+m7a
	- cpu=16
	- image=ubuntu24-full-x64
	- tag=rust-lint
	steps:
	- uses: runs-on/action@v1
	- uses: actions/checkout@v4
	with:
	submodules: "recursive"
	- uses: rui314/setup-mold@v1
	- name: Rust Lint - Format
	run: cargo fmt --all --check
	- name: Rustc check
	run: cargo check --locked --all-features --all-targets
	- name: Rust Lint - Clippy All Features
	run: cargo clippy --locked --all-features --all-targets
	- name: Rust Lint - Clippy Default Features
	run: cargo clippy --locked --all-targets

	rust-lint-no-default:
	name: "Rust (lint, no default)"
	timeout-minutes: 120
	runs-on:
	- runs-on=${{ github.run_id }}
	- family=m7i+m7i-flex+m7a
	- cpu=16
	- image=ubuntu24-full-x64
	- tag=rust-lint
	steps:
	- uses: runs-on/action@v1
	- uses: actions/checkout@v4
	with:
	submodules: "recursive"
	- uses: rui314/setup-mold@v1
	- name: Install cargo-hack
	uses: taiki-e/install-action@cargo-hack
	- name: Rust Lint - Clippy No Default Features
	shell: bash
	run: \|
	# https://spiraldb.slack.com/archives/C07BV3GKAJ2/p1732736281946729
	cargo hack clippy --no-default-features

	rust-test:
	name: "Rust (tests)"
	timeout-minutes: 120
	runs-on:
	- runs-on=${{ github.run_id }}
	- family=m7i+m7i-flex+m7a
	- cpu=8
	- image=ubuntu24-full-x64
	- tag=rust-test
	steps:
	# Disable lints when running tests. They will be caught by the Rust lint job.
	- name: Setup environment (On develop)
	shell: bash
	if: github.ref == 'refs/heads/develop'
	run: \|
	echo "RUSTFLAGS=-Cinstrument-coverage -A warnings">> $GITHUB_ENV
	# Disable incremental compilation to get accurate coverage
	echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV
	echo "LLVM_PROFILE_FILE=target/coverage/vortex-%p-%m.profraw" >> $GITHUB_ENV
	echo "GRCOV_OUTPUT_FILE=target/coverage/vortex.lcov" >> $GITHUB_ENV
	- name: Setup environment
	shell: bash
	if: github.ref != 'refs/heads/develop'
	run: \|
	echo "RUSTFLAGS=-A warnings" >> $GITHUB_ENV
	- uses: actions/checkout@v4
	with:
	submodules: "recursive"
	- uses: ./.github/actions/cleanup
	- uses: rui314/setup-mold@v1
	- uses: ./.github/actions/setup-c++
	- name: Install grcov
	if: github.ref == 'refs/heads/develop'
	uses: taiki-e/install-action@grcov
	- name: Install nextest
	uses: taiki-e/install-action@v2
	with:
	tool: nextest@0.9.98
	- name: Rust Tests
	# vortex-duckdb-ext currently fails linking for cargo test targets.
	run: \|
	cargo nextest run --locked --workspace --all-features --no-fail-fast

	- name: Generate coverage report
	if: github.ref == 'refs/heads/develop'
	run: \|
	rustup component add llvm-tools-preview
	grcov . --binary-path target/debug/ -s . -t lcov --llvm --ignore-not-existing \
	--ignore '../' --ignore '/' --ignore 'bench-vortex/' --ignore 'fuzz/' \
	--ignore 'home/' --ignore 'xtask/' --ignore 'target/*'\
	-o ${{ env.GRCOV_OUTPUT_FILE }}
	- name: Coveralls
	uses: coverallsapp/github-action@v2
	if: github.ref == 'refs/heads/develop'
	with:
	file: ${{ env.GRCOV_OUTPUT_FILE }}

	build-java:
	name: "Java"
	runs-on: ubuntu-latest
	timeout-minutes: 120
	steps:
	- name: Checkout sources
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	submodules: "recursive"
	- uses: ./.github/actions/cleanup
	- uses: ./.github/actions/setup-rust
	# Build vortex-jni shared library, and copy into JAR.
	- name: Build vortex-jni
	run: \|
	cargo build -p vortex-jni
	mkdir -p java/vortex-jni/src/main/resources/native/linux-amd64
	cargo xtask java-test-files
	cp target/debug/libvortex_jni.so java/vortex-jni/src/main/resources/native/linux-amd64
	- run: ./gradlew test
	working-directory: ./java

	bench-codspeed:
	name: Benchmark with Codspeed
	timeout-minutes: 120
	runs-on:
	- runs-on=${{ github.run_id }}
	- family=c6id.8xlarge
	- spot=false
	- image=ubuntu24-full-x64

	steps:
	- uses: runs-on/action@v1
	- uses: actions/checkout@v4
	with:
	submodules: "recursive"
	# rustup is pre-installed on the ubuntu24-full-x64 image.

	- name: Install Codspeed
	shell: bash
	run: cargo install --force cargo-codspeed --locked

	- name: Build benchmarks
	env:
	RUSTFLAGS: "-C target-feature=+avx2"
	run: \|
	cargo codspeed build --features test-harness \
	--exclude bench-vortex \
	--exclude vortex-datafusion \
	--exclude vortex-duckdb-ext \
	--exclude vortex-fuzz \
	--exclude vortex-python \
	--exclude vortex-tui \
	--exclude xtask \
	--workspace \
	--profile bench

	- name: Run benchmarks
	uses: CodSpeedHQ/action@v3
	with:
	run: cargo codspeed run
	runner-version: 3.4.1-beta.1
	token: ${{ secrets.CODSPEED_TOKEN }}

	license-check-and-audit-check:
	name: License Check and Audit Check
	runs-on: ubuntu-latest
	timeout-minutes: 120
	strategy:
	matrix:
	checks:
	- advisories
	- bans licenses sources
	# Prevent sudden announcement of a new advisory from failing ci:
	continue-on-error: ${{ matrix.checks == 'advisories' }}
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: "recursive"
	- uses: EmbarkStudios/cargo-deny-action@v2
	with:
	command: check ${{ matrix.checks }}

	wasm-integration:
	name: "wasm-integration"
	runs-on: ubuntu-latest
	timeout-minutes: 120
	steps:
	- uses: rui314/setup-mold@v1
	- uses: actions/checkout@v4
	- uses: ./.github/actions/setup-rust
	with:
	targets: "wasm32-wasip1"
	- name: Setup Wasmer
	uses: wasmerio/setup-wasmer@v3.1
	- run: cargo -Zbuild-std=panic_abort,std build --target wasm32-wasip1
	working-directory: ./wasm-test
	- run: wasmer run ./target/wasm32-wasip1/debug/wasm-test.wasm
	working-directory: ./wasm-test

	miri:
	name: "miri"
	runs-on: ubuntu-latest
	timeout-minutes: 120
	env:
	MIRIFLAGS: -Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-backtrace=full -Zmiri-disable-isolation
	RUSTFLAGS: "-A warnings"
	steps:
	- uses: rui314/setup-mold@v1
	- uses: actions/checkout@v4
	with:
	submodules: "recursive"
	- uses: ./.github/actions/setup-rust
	- uses: taiki-e/install-action@v2
	with:
	tool: nextest@0.9.98
	- name: Run all tests with Miri
	run: cargo miri nextest run --no-fail-fast --workspace --exclude vortex-file --exclude vortex-layout --exclude vortex-sampling-compressor --exclude vortex-fsst --exclude vortex-array --exclude vortex-dtype --exclude vortex-expr --exclude vortex-scalar --exclude vortex-duckdb --exclude vortex-duckdb-ext
	# For now, we only run Miri against known "fiddly" crates.
	if: false
	- name: Run Miri
	run: cargo miri nextest run --no-fail-fast -p vortex-buffer -p vortex-ffi

	generated-files:
	name: "Check generated proto/fbs files are up to date"
	runs-on: ubuntu-latest
	timeout-minutes: 120
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: "recursive"
	- uses: ./.github/actions/cleanup
	- uses: rui314/setup-mold@v1
	- uses: ./.github/actions/setup-rust
	- uses: ./.github/actions/setup-flatc
	- name: Install Protoc
	uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- name: "regenerate all .fbs/.proto Rust code"
	run: \|
	cargo xtask generate-fbs
	cargo xtask generate-proto
	- name: "Make sure no files changed after regenerating"
	run: \|
	git status --porcelain
	test -z "$(git status --porcelain)"

	- name: "Checkout develop flatbuffers"
	working-directory: vortex-flatbuffers/
	run: \|
	cp -R flatbuffers flatbuffers.HEAD
	git fetch origin develop --depth 1
	git checkout origin/develop -- flatbuffers
	- name: "Verify flatbuffer back-compat"
	working-directory: vortex-flatbuffers/
	run: \|
	find flatbuffers/ -type f -name "*.fbs" \| sed 's/^flatbuffers\///' \| xargs -I{} -n1 flatc -I flatbuffers.HEAD --conform-includes flatbuffers --conform flatbuffers/{} flatbuffers.HEAD/{}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

bug[vortex-array]: take nullability chunked array not respects validity #13338

Workflow file

bug[vortex-array]: take nullability chunked array not respects validity #13338

Uh oh!

Workflow file for this run