chore(rust): separate adbc_core and adbc_datafusion arrow version requirements (#3017)

Context #2739
Closes #2524

Since datafusion tends to require newer versions of arrow than
adbc_core, the version of arrow used by adbc_core will be bumped by
adbc_datafusion when compiling the entire workspace.

This PullRequest separates the arrow requirements for adbc_datafusion
and others.
Since the newer arrow that meets adbc_datafusion requirements will be
used when the entire workspace is compiled, a job to test other than
adbc_datafuion by generating a file that locks the lower requirements on
CI is added.
It make sure adbc_core will work with the older arrow.

Author: eitsupi

.github/workflows/rust.yml

@@ -46,8 +46,16 @@ jobs:
       matrix:
         # TODO(alexandreyc): add `windows-latest`
         # See: https://github.com/apache/arrow-adbc/pull/1803#issuecomment-2117669300
-        os: [macos-13, macos-latest, ubuntu-latest]
-    name: "Rust ${{ matrix.os }}"
+        os:
+          - macos-13
+          - macos-latest
+          - ubuntu-latest
+        minimal-versions:
+          - false
+        include:
+          - os: ubuntu-latest
+            minimal-versions: true # Test can be built with older arrow except adbc_datafusion
+    name: Rust ${{ matrix.os }} ${{ matrix.minimal-versions && '(minimal versions for adbc_core)' || '' }}
     runs-on: ${{ matrix.os }}
     env:
       CARGO_INCREMENTAL: 0
@@ -57,6 +65,12 @@ jobs:
         with:
           fetch-depth: 0
           persist-credentials: false
+      - name: Cargo update for minimal (arrow) version
+        if: ${{ matrix.minimal-versions }}
+        working-directory: rust
+        run: |
+          rustup toolchain install nightly
+          cargo +nightly generate-lockfile -Z direct-minimal-versions
       - name: Use stable Rust
         id: rust
         run: |
@@ -120,27 +134,28 @@ jobs:
       - name: Set search dir for Snowflake Go lib
         run: echo "ADBC_SNOWFLAKE_GO_LIB_DIR=${{ github.workspace }}/local/lib" >> "$GITHUB_ENV"
       - name: Clippy
+        if: ${{ ! matrix.minimal-versions }}
         working-directory: rust
         run: |
           cargo clippy --workspace --all-targets --all-features --locked -- -Dwarnings
       - name: Test
         working-directory: rust
         run: |
-          cargo test --workspace --all-targets --all-features
+          cargo test --all-targets --all-features --workspace ${{ matrix.minimal-versions && '--exclude adbc_datafusion' || '' }}
         # env:
         # ADBC_SNOWFLAKE_TESTS: 1
         # ADBC_SNOWFLAKE_URI: ${{ secrets.SNOWFLAKE_URI }}
         # ADBC_SNOWFLAKE_SQL_DB: ADBC_TESTING
       - name: Doctests
         working-directory: rust
         run: |
-          cargo test --workspace --doc --all-features
+          cargo test --doc --all-features --workspace ${{ matrix.minimal-versions && '--exclude adbc_datafusion' || '' }}
       - name: Check docs
         working-directory: rust
         run: |
-          cargo doc --workspace --all-features
+          cargo doc --all-features --workspace ${{ matrix.minimal-versions && '--exclude adbc_datafusion' || '' }}
       - name: Verify MSRV (Minimum Supported Rust Version)
-        if: matrix.os == 'ubuntu-latest'
+        if: matrix.os == 'ubuntu-latest' && ! matrix.minimal-versions
         working-directory: rust
         run: |
           # Install cargo-msrv

rust/Cargo.toml

@@ -35,9 +35,9 @@ categories = ["database"]
 
 [workspace.dependencies]
 adbc_core = { path = "./core", version = "0.19.0" }
-arrow-array = { version = "55.1.0", default-features = false, features = [
+arrow-array = { version = ">=53.1.0, <56", default-features = false, features = [
     "ffi",
 ] }
-arrow-buffer = { version = "55.0.0", default-features = false }
-arrow-schema = { version = "55.0.0", default-features = false }
-arrow-select = { version = "55.1.0", default-features = false }
+arrow-buffer = { version = ">=53.1.0, <56", default-features = false }
+arrow-schema = { version = ">=53.1.0, <56", default-features = false }
+arrow-select = { version = ">=53.1.0, <56", default-features = false }

rust/driver/datafusion/Cargo.toml

@@ -32,16 +32,16 @@ categories.workspace = true
 
 [dependencies]
 adbc_core.workspace = true
-arrow-array.workspace = true
-arrow-buffer.workspace = true
-arrow-schema.workspace = true
+arrow-array = "55.0.0"
+arrow-buffer = "55.0.0"
+arrow-schema = "55.0.0"
 datafusion = "47.0.0"
 datafusion-substrait = "47.0.0"
 tokio = { version = "1.45", features = ["rt-multi-thread"] }
 prost = "0.13.5"
 
 [dev-dependencies]
-arrow-select.workspace = true
+arrow-select = "55.0.0"
 
 [lib]
 crate-type = ["lib", "cdylib"]