build.yml

name: build

on: [push, pull_request, merge_group]

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1

# Supply chain security: all cargo commands that resolve dependencies use --locked to
# enforce the committed Cargo.lock. This prevents CI from silently resolving a newer
# (potentially compromised) dependency version. If Cargo.lock is out of sync with
# Cargo.toml, the build fails immediately. Any dependency change must be an explicit,
# reviewable update to Cargo.lock in the PR. Commands that skip --locked: cargo fmt
# (no dep resolution), cargo msrv verify/show (wrapper tool), cargo miri setup (tooling).
#
# Swatinem/rust-cache caches the cargo registry and target directory (~450MB per job).
# save-if restricts cache writes to main pushes only. PRs read from main's cache but
# never write their own entries.
#
# The key insight: Cargo.lock changes infrequently, so main's cache key almost always
# matches. PRs download and compile zero dependencies on cache hit. By only writing on
# main, we keep main's cache entries alive (no LRU eviction from PR churn), and every
# PR benefits from them.
#
# Without this, GHA's ref-scoped caching works against us: each PR writes ~6.3GB of
# cache entries (14 jobs x ~450MB) that only that PR can read. A handful of active PRs
# fills the 10GB cache budget, LRU evicts main's shared entries, and every subsequent
# PR compiles from scratch.
#
# The save-if condition checks both event_name == 'push' and ref == main because
# pull_request_target events set github.ref to the base branch (main), not the PR
# branch. Without the event_name check, those workflows would write cache entries on
# every PR.
#
# Note: actions-rust-lang/setup-rust-toolchain has built-in Swatinem/rust-cache that
# writes on every run with no save-if support. We disable it with cache: false and
# manage caching explicitly via the Swatinem/rust-cache steps below.

jobs:
  format:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      - name: Install minimal stable with rustfmt
        uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
        with:
          cache: false
          components: rustfmt
      - name: format
        run: cargo fmt -- --check

  msrv:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      - name: Install minimal stable and cargo msrv
        uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
        with:
          cache: false
      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
        with:
          save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
      - uses: taiki-e/install-action@7bc99eee1f1b8902a125006cf790a1f4c8461e63 # v2.69.8
        with:
          tool: cargo-msrv
      - name: verify-msrv
        run: |
          cargo msrv --path kernel/ verify --all-features
          cargo msrv --path derive-macros/ verify --all-features
          cargo msrv --path ffi/ verify --all-features
          cargo msrv --path ffi-proc-macros/ verify --all-features
  msrv-run-tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      - name: Install minimal stable and cargo msrv
        uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
        with:
          cache: false
      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
        with:
          save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
      - uses: taiki-e/install-action@7bc99eee1f1b8902a125006cf790a1f4c8461e63 # v2.69.8
        with:
          tool: cargo-msrv
      - uses: taiki-e/install-action@98ec31d284eb962f41c14065e9391a955aa810cf # nextest
      - name: Get rust-version from Cargo.toml
        id: rust-version
        run: echo "RUST_VERSION=$(cargo msrv show --path kernel/ --output-format minimal)" >> $GITHUB_ENV
      - name: Install specified rust version
        uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
        with:
          cache: false
          toolchain: ${{ env.RUST_VERSION }}
      - name: run tests
        run: |
          pushd kernel
          echo "Testing with $(cargo msrv show --output-format minimal)"
          cargo +$(cargo msrv show --output-format minimal) nextest run --locked
  docs:
    runs-on: ubuntu-latest
    env:
      RUSTDOCFLAGS: -D warnings
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      - name: Install minimal stable
        uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
        with:
          cache: false
      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
        with:
          save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
      - name: build docs
        run: cargo doc --locked --workspace --all-features --no-deps

  # When we run cargo { build, clippy } --no-default-features, we want to build/lint the kernel to
  # ensure that we can build the kernel without any features enabled. Unfortunately, due to how
  # cargo resolves features, if we have a workspace member that depends on the kernel with features
  # enabled, the kernel will be compiled with those features (even if we specify
  # --no-default-features).
  #
  # To cope with this, we split build/clippy --no-default-features into two runs:
  # 1. build/clippy all packages that depend on the kernel with some features enabled:
  #    - acceptance
  #    - test_utils
  #    - feature_tests
  #    (and examples)
  #    - inspect-table
  #    - read-table-changes
  #    - read-table-multi-threaded
  #    - read-table-single-threaded
  # 2. build/clippy all packages that only have no-feature kernel dependency
  #    - delta_kernel
  #    - delta_kernel_derive
  #    - delta_kernel_ffi
  #    - delta_kernel_ffi_macros
  build:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os:
          - macOS-latest
          - ubuntu-latest
          - windows-latest
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      - name: Install minimal stable with clippy
        uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
        with:
          cache: false
          components: clippy
      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
        with:
          save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
      - name: build and lint with clippy
        run: cargo clippy --locked --benches --tests --all-features -- -D warnings
      - name: lint without default features - packages which depend on kernel with features enabled
        run: cargo clippy --locked --workspace --no-default-features --exclude delta_kernel --exclude delta_kernel_ffi --exclude delta_kernel_derive --exclude delta_kernel_ffi_macros -- -D warnings
      - name: lint without default features - packages which don't depend on kernel with features enabled
        run: cargo clippy --locked --no-default-features --package delta_kernel --package delta_kernel_ffi --package delta_kernel_derive --package delta_kernel_ffi_macros -- -D warnings
      - name: check kernel builds with default-engine-native-tls
        run: cargo build --locked -p feature_tests --features default-engine-native-tls
      - name: test native-tls backend has no crypto provider conflict
        run: cargo test --locked -p feature_tests --features default-engine-native-tls
      - name: check kernel builds with default-engine-rustls
        run: cargo build --locked -p feature_tests --features default-engine-rustls
      - name: test rustls TLS backend feature-tests
        run: cargo test --locked -p feature_tests --features default-engine-rustls
  test:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os:
          - macOS-latest
          - ubuntu-latest
          - windows-latest
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
        id: filter
        with:
          filters: |
            ffi:
              - 'ffi/src/handle.rs'
              - 'ffi-proc-macros/**'
      - name: Install minimal stable with clippy and rustfmt
        uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
        with:
          cache: false
      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
        with:
          save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
      - uses: taiki-e/install-action@98ec31d284eb962f41c14065e9391a955aa810cf # nextest
      - name: test
        run: cargo nextest run --locked --workspace --all-features -E 'not test(read_table_version_hdfs) and not test(invalid_handle_code)'
      - name: trybuild tests
        if: steps.filter.outputs.ffi == 'true'
        run: cargo test --locked --package delta_kernel_ffi --features internal-api -- invalid_handle_code

  ffi_test:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os:
          - macOS-latest
          - ubuntu-latest
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      - name: Setup cmake
        uses: jwlawson/actions-setup-cmake@0d6a7d60b009d01c9e7523be22153ff8f19460d3 # v2.2.0
        with:
          cmake-version: "3.30.x"
      - name: Install arrow-glib-linux
        run: |
          if [ "$RUNNER_OS" == "Linux" ]; then
              sudo apt update
              sudo apt install -y -V ca-certificates lsb-release wget
              wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
              wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb.sha512
              sha512sum -c apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb.sha512 || exit 1
              sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
              sudo apt update
              sudo apt install -y -V libarrow-dev # For C++
              sudo apt install -y -V libarrow-glib-dev # For GLib (C)
              sudo apt install -y -V valgrind # For memory leak test
           fi
      - name: Install arrow-glib-macOS
        if: runner.os == 'macOS'
        uses: ./.github/actions/use-homebrew-tools
        with:
          tools: "apache-arrow apache-arrow-glib"
      - name: Install minimal stable
        uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
        with:
          cache: false
      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
        with:
          save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
      - name: Set output on fail
        run: echo "CTEST_OUTPUT_ON_FAILURE=1" >> "$GITHUB_ENV"
      - name: Build kernel
        run: |
          pushd acceptance
          cargo build --locked
          popd
          pushd ffi
          cargo build --locked --features default-engine-rustls,test-ffi,tracing,delta-kernel-unity-catalog
          popd
      - name: build and run read-table test
        run: |
          pushd ffi/examples/read-table
          mkdir build
          pushd build
          cmake ..
          make
          make test
      - name: build and run visit-expression test
        run: |
          pushd ffi/examples/visit-expression
          mkdir build
          pushd build
          cmake ..
          make
          make test
      - name: build and run delta-kernel-unity-catalog-ffi test
        run: |
          pushd ffi/examples/delta-kernel-unity-catalog-example
          mkdir build
          pushd build
          cmake ..
          make
          make test
  miri:
    name: "Miri (shard ${{ matrix.partition }}/3)"
    runs-on: ubuntu-latest
    strategy:
      matrix:
        partition: [1, 2, 3]
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      - name: Install Miri
        run: |
          rustup toolchain install nightly --component miri
          rustup override set nightly
          cargo miri setup
      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
        with:
          save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
      - uses: taiki-e/install-action@98ec31d284eb962f41c14065e9391a955aa810cf # nextest
      - name: Test with Miri
        run: |
          pushd ffi
          MIRIFLAGS=-Zmiri-disable-isolation cargo miri nextest run --locked --features default-engine-rustls,delta-kernel-unity-catalog --partition slice:${{ matrix.partition }}/3

  coverage:
    runs-on: ubuntu-latest
    env:
      CARGO_TERM_COLOR: always
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      - name: Install rust
        uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
        with:
          cache: false
      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
        with:
          save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
      - name: Install cargo-llvm-cov
        uses: taiki-e/install-action@2d15d02e710b40b6332201aba6af30d595b5cd96 # cargo-llvm-cov
      - name: Generate code coverage
        run: cargo llvm-cov --locked --all-features --workspace --codecov --output-path codecov.json -- --skip read_table_version_hdfs
      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5.5.3
        with:
          files: codecov.json
          fail_ci_if_error: true
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}