Rust

feat: capture more metrics in distributed_query (#1353) #3292

Workflow file for this run

	# Licensed to the Apache Software Foundation (ASF) under one
	# or more contributor license agreements. See the NOTICE file
	# distributed with this work for additional information
	# regarding copyright ownership. The ASF licenses this file
	# to you under the Apache License, Version 2.0 (the
	# "License"); you may not use this file except in compliance
	# with the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing,
	# software distributed under the License is distributed on an
	# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	# KIND, either express or implied. See the License for the
	# specific language governing permissions and limitations
	# under the License.

	name: Rust

	concurrency:
	group: ${{ github.repository }}-${{ github.head_ref \|\| github.sha }}-${{ github.workflow }}
	cancel-in-progress: true

	on:
	push:
	paths-ignore:
	- "docs/**"
	- "**.md"
	- ".github/ISSUE_TEMPLATE/**"
	- ".github/pull_request_template.md"
	pull_request:
	paths-ignore:
	- "docs/**"
	- "**.md"
	- ".github/ISSUE_TEMPLATE/**"
	- ".github/pull_request_template.md"
	# manual trigger
	# https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
	workflow_dispatch:

	jobs:
	# Check crate compiles and base cargo check passes
	linux-build-lib:
	name: linux build test
	runs-on: ubuntu-latest
	container:
	image: amd64/rust
	steps:
	- uses: actions/checkout@v5
	with:
	submodules: true
	fetch-depth: 1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	- name: Prepare cargo build
	run: \|
	# Adding `--locked` here to assert that the `Cargo.lock` file is up to
	# date with the manifest. When this fails, please make sure to commit
	# the changes to `Cargo.lock` after building with the updated manifest.
	cargo check --profile ci --workspace --all-targets --locked

	linux-test:
	name: test linux crates
	runs-on: ubuntu-latest
	container:
	image: amd64/rust
	steps:
	- uses: actions/checkout@v5
	with:
	submodules: true
	fetch-depth: 1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	- name: Run tests
	run: \|
	export PATH=$PATH:$HOME/d/protoc/bin
	export ARROW_TEST_DATA=$(pwd)/testing/data
	export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
	cargo test --profile ci --features=testcontainers
	env:
	CARGO_HOME: "/github/home/.cargo"
	CARGO_TARGET_DIR: "/github/home/target"

	# Run `cargo doc` to ensure the rustdoc is clean
	linux-rustdoc:
	name: cargo doc
	needs: linux-build-lib
	runs-on: ubuntu-latest
	container:
	image: amd64/rust
	steps:
	- uses: actions/checkout@v5
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: stable
	- name: Run cargo doc
	run: ci/scripts/rust_docs.sh

	linux-build-workspace:
	name: check linux workspace
	runs-on: ubuntu-latest
	container:
	image: amd64/rust
	steps:
	- uses: actions/checkout@v5
	with:
	submodules: true
	fetch-depth: 1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	- name: Try to compile when `--no-default-features` is selected
	run: \|
	export PATH=$PATH:$HOME/d/protoc/bin
	export ARROW_TEST_DATA=$(pwd)/testing/data
	export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
	cargo check --profile ci -p ballista-scheduler -p ballista-executor -p ballista-core -p ballista --no-default-features --locked
	env:
	CARGO_HOME: "/github/home/.cargo"
	CARGO_TARGET_DIR: "/github/home/target"

	ballista-test:
	name: test linux balista
	runs-on: ubuntu-latest
	container:
	image: amd64/rust
	steps:
	- uses: actions/checkout@v5
	with:
	submodules: true
	fetch-depth: 1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	- name: Run Ballista tests
	run: \|
	export PATH=$PATH:$HOME/d/protoc/bin
	export ARROW_TEST_DATA=$(pwd)/testing/data
	export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
	cd ballista
	# Ensure also compiles in standalone mode
	cargo test --profile ci --no-default-features --features standalone --locked
	env:
	CARGO_HOME: "/github/home/.cargo"
	CARGO_TARGET_DIR: "/github/home/target"

	windows-test:
	name: windows test
	runs-on: windows-latest
	steps:
	- uses: actions/checkout@v5
	with:
	submodules: true
	fetch-depth: 1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-windows-builder
	- name: Run tests
	shell: bash
	run: \|
	export PATH=$PATH:$HOME/d/protoc/bin
	export ARROW_TEST_DATA=$(pwd)/testing/data
	export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
	export TMP=$(pwd)/.temp
	mkdir -p "$TMP"
	cargo test --profile ci --locked

	macos-test:
	name: macos test
	runs-on: macos-latest
	steps:
	- uses: actions/checkout@v5
	with:
	submodules: true
	fetch-depth: 1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-macos-builder
	- name: Run tests
	shell: bash
	run: \|
	export PATH=$PATH:$HOME/d/protoc/bin
	export ARROW_TEST_DATA=$(pwd)/testing/data
	export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
	cargo test --profile ci --locked

	# verify that the benchmark queries return the correct results
	verify-benchmark-results:
	name: verify benchmark results (amd64)
	runs-on: ubuntu-latest
	container:
	image: amd64/rust
	steps:
	- uses: actions/checkout@v5
	with:
	submodules: true
	fetch-depth: 1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	- name: Verify that benchmark queries return expected results
	run: \|
	cargo test --package ballista-benchmarks --profile release-nonlto --features=ci --locked -- --test-threads=1

	lint:
	name: Lint
	runs-on: ubuntu-latest
	container:
	image: amd64/rust
	steps:
	- uses: actions/checkout@v5
	- name: Setup toolchain
	run: \|
	rustup toolchain install stable
	rustup default stable
	rustup component add rustfmt
	- name: Run
	run: ci/scripts/rust_fmt.sh
	env:
	CARGO_HOME: "/github/home/.cargo"
	CARGO_TARGET_DIR: "/github/home/target"

	clippy:
	name: Clippy
	runs-on: ubuntu-latest
	strategy:
	matrix:
	arch: [amd64]
	rust: [stable]
	container:
	image: ${{ matrix.arch }}/rust
	env:
	# Disable full debug symbol generation to speed up CI build and keep memory down
	# "1" means line tables only, which is useful for panic tracebacks.
	RUSTFLAGS: "-C debuginfo=1"
	steps:
	- uses: actions/checkout@v5
	with:
	submodules: true
	fetch-depth: 1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: ${{ matrix.rust }}
	- name: Install Clippy
	run: \|
	rustup component add clippy
	- name: Run clippy
	run: ci/scripts/rust_clippy.sh
	env:
	CARGO_HOME: "/github/home/.cargo"
	CARGO_TARGET_DIR: "/github/home/target"

	cargo-toml-formatting-checks:
	name: Check Cargo.toml formatting
	runs-on: ubuntu-latest
	strategy:
	matrix:
	arch: [amd64]
	rust: [stable]
	container:
	image: ${{ matrix.arch }}/rust
	env:
	# Disable full debug symbol generation to speed up CI build and keep memory down
	# "1" means line tables only, which is useful for panic tracebacks.
	RUSTFLAGS: "-C debuginfo=1"
	steps:
	- uses: actions/checkout@v5
	with:
	submodules: true
	fetch-depth: 1
	- name: Setup Rust toolchain
	uses: ./.github/actions/setup-builder
	with:
	rust-version: ${{ matrix.rust }}
	- name: Install cargo-tomlfmt
	run: \|
	which cargo-tomlfmt \|\| cargo install cargo-tomlfmt
	env:
	CARGO_HOME: "/github/home/.cargo"
	CARGO_TARGET_DIR: "/github/home/target"
	- name: Check Cargo.toml formatting
	run: \|
	# if you encounter error, try rerun the command below, finally run 'git diff' to
	# check which Cargo.toml introduces formatting violation
	#
	# ignore ./Cargo.toml because putting workspaces in multi-line lists make it easy to read
	ci/scripts/rust_toml_fmt.sh
	if test -f "./Cargo.toml.bak"; then
	echo "cargo tomlfmt found format violations"
	exit 1
	fi
	env:
	CARGO_HOME: "/github/home/.cargo"
	CARGO_TARGET_DIR: "/github/home/target"
	# Coverage job was failing. https://github.com/apache/arrow-datafusion/issues/590 tracks re-instating it

	# coverage:
	# name: Coverage
	# runs-on: ubuntu-latest
	# strategy:
	# matrix:
	# arch: [amd64]
	# rust: [stable]
	# steps:
	# - uses: actions/checkout@v5
	# with:
	# submodules: true
	# - name: Cache Cargo
	# uses: actions/cache@v4
	# with:
	# path: /home/runner/.cargo
	# # this key is not equal because the user is different than on a container (runner vs github)
	# key: cargo-coverage-cache-
	# - name: Cache Rust dependencies
	# uses: actions/cache@v4
	# with:
	# path: /home/runner/target
	# # this key is not equal because coverage uses different compilation flags.
	# key: ${{ runner.os }}-${{ matrix.arch }}-target-coverage-cache-${{ matrix.rust }}-
	# - name: Run coverage
	# run: \|
	# export ARROW_TEST_DATA=$(pwd)/testing/data
	# export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data

	# # 2020-11-15: There is a cargo-tarpaulin regression in 0.17.0
	# # see https://github.com/xd009642/tarpaulin/issues/618
	# cargo install --version 0.16.0 cargo-tarpaulin
	# cargo tarpaulin --out Xml
	# env:
	# CARGO_HOME: "/home/runner/.cargo"
	# CARGO_TARGET_DIR: "/home/runner/target"
	# - name: Report coverage
	# continue-on-error: true
	# run: bash <(curl -s https://codecov.io/bash)

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

feat: capture more metrics in distributed_query (#1353) #3292

Workflow file

feat: capture more metrics in distributed_query (#1353) #3292

Uh oh!

Workflow file for this run