diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index 6ca095328af1..f86d8afdbd15 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -31,11 +31,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Arrow - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: apache/arrow - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.8 - name: Install Archery and Crossbow dependencies @@ -54,7 +54,7 @@ jobs: if: startsWith(github.event.comment.body, '@github-actions rebase') runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: r-lib/actions/pr-fetch@master with: repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 8bb35f1ef871..13bfce259f1a 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -24,13 +24,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Arrow - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: apache/arrow submodules: true fetch-depth: 0 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.8 - name: Setup Archery @@ -43,9 +43,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: 3.8 - name: Audit licenses @@ -55,8 +55,8 @@ jobs: name: Use prettier to check formatting of documents runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: node-version: "14" - name: Prettier check diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index 94396c7327a9..7b1ba27bc72d 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -29,7 +29,7 @@ jobs: name: Process runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Assign GitHub labels if: | diff --git a/.github/workflows/python_build.yml b/.github/workflows/python_build.yml index 39ac1374a4c6..3ccbba9cdbe8 100644 --- a/.github/workflows/python_build.yml +++ b/.github/workflows/python_build.yml @@ -31,9 +31,9 @@ jobs: python-version: ["3.6", "3.7", "3.8", "3.9"] os: [macos-latest, windows-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -59,7 +59,7 @@ jobs: run: find ./python/target/wheels/ - name: Archive wheels - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: dist path: python/target/wheels/* @@ -68,11 +68,11 @@ jobs: name: Manylinux runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Build wheels run: docker run --rm -v $(pwd):/io konstin2/maturin build --release --manylinux - name: Archive wheels - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: dist path: python/target/wheels/* @@ -82,7 +82,7 @@ jobs: needs: [build-manylinux, build-python-mac-win] runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v4 - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@master with: diff --git a/.github/workflows/python_test.yaml b/.github/workflows/python_test.yaml index e208b6f8ea32..a3e66f66d645 100644 --- a/.github/workflows/python_test.yaml +++ b/.github/workflows/python_test.yaml @@ -23,23 +23,23 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup Rust toolchain run: | rustup toolchain install nightly-2022-11-03 rustup default nightly-2022-11-03 rustup component add rustfmt - name: Cache Cargo - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: /home/runner/.cargo key: cargo-maturin-cache- - name: Cache Rust dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: /home/runner/target key: target-maturin-cache- - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: python-version: "3.9" - name: Install Python dependencies diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index fe60361f4f7d..a1573af6281b 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -38,16 +38,16 @@ jobs: # "1" means line tables only, which is useful for panic tracebacks. RUSTFLAGS: "-C debuginfo=1" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Cache Cargo - uses: actions/cache@v2 + uses: actions/cache@v4 with: # these represent dependencies downloaded by cargo # and thus do not depend on the OS, arch nor rust version. path: /github/home/.cargo key: cargo-cache- - name: Cache Rust dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: # these represent compiled steps of both dependencies and arrow # and thus are specific for a particular OS, arch and rust version. @@ -81,17 +81,17 @@ jobs: # "1" means line tables only, which is useful for panic tracebacks. RUSTFLAGS: "-C debuginfo=1" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: true - name: Cache Cargo - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: /github/home/.cargo # this key equals the ones on `linux-build-lib` for re-use key: cargo-cache- - name: Cache Rust dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: /github/home/target # this key equals the ones on `linux-build-lib` for re-use @@ -145,10 +145,10 @@ jobs: # --health-timeout 5s # --health-retries 5 # steps: -# - uses: actions/checkout@v2 +# - uses: actions/checkout@v4 # with: # submodules: true -# - uses: actions/setup-python@v2 +# - uses: actions/setup-python@v5 # with: # python-version: "3.8" # - name: Install Python dependencies @@ -201,7 +201,7 @@ jobs: os: [macos-latest] rust: [nightly-2024-01-29] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: true # TODO: this won't cache anything, which is expensive. Setup this action @@ -227,7 +227,7 @@ jobs: container: image: amd64/rust steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup toolchain run: | rustup toolchain install nightly-2024-01-29 @@ -251,17 +251,17 @@ jobs: # # "1" means line tables only, which is useful for panic tracebacks. # RUSTFLAGS: "-C debuginfo=1" # steps: -# - uses: actions/checkout@v2 +# - uses: actions/checkout@v4 # with: # submodules: true # - name: Cache Cargo -# uses: actions/cache@v2 +# uses: actions/cache@v4 # with: # path: /github/home/.cargo # # this key equals the ones on `linux-build-lib` for re-use # key: cargo-cache- # - name: Cache Rust dependencies -# uses: actions/cache@v2 +# uses: actions/cache@v4 # with: # path: /github/home/target # # this key equals the ones on `linux-build-lib` for re-use @@ -286,10 +286,10 @@ jobs: # arch: [amd64] # rust: [nightly-2021-07-04] # steps: -# - uses: actions/checkout@v2 +# - uses: actions/checkout@v4 # with: # submodules: true -# - uses: actions/cache@v2 +# - uses: actions/cache@v4 # with: # path: | # ~/.cargo/registry @@ -322,17 +322,17 @@ jobs: # arch: [amd64] # rust: [nightly-2021-07-04] # steps: - # - uses: actions/checkout@v2 + # - uses: actions/checkout@v4 # with: # submodules: true # - name: Cache Cargo - # uses: actions/cache@v2 + # uses: actions/cache@v4 # with: # path: /home/runner/.cargo # # this key is not equal because the user is different than on a container (runner vs github) # key: cargo-coverage-cache- # - name: Cache Rust dependencies - # uses: actions/cache@v2 + # uses: actions/cache@v4 # with: # path: /home/runner/target # # this key is not equal because coverage uses different compilation flags. diff --git a/.gitignore b/.gitignore index 31bdf49ce43e..fd18755ba60a 100644 --- a/.gitignore +++ b/.gitignore @@ -91,3 +91,5 @@ rusty-tags.vi .vscode venv/* +.claude +CLAUDE.md diff --git a/Cargo.toml b/Cargo.toml index da3e4c812554..8f92cd2588c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ # under the License. [workspace] +resolver = "2" members = [ "datafusion", "datafusion-cli", diff --git a/datafusion/src/cube_ext/util.rs b/datafusion/src/cube_ext/util.rs index fd3894d602e6..0470669a8b1e 100644 --- a/datafusion/src/cube_ext/util.rs +++ b/datafusion/src/cube_ext/util.rs @@ -255,7 +255,7 @@ macro_rules! cube_match_scalar { ScalarValue::TimestampSecond(v) => ($matcher!($($arg ,)* v, TimestampSecondBuilder)), ScalarValue::IntervalYearMonth(v) => ($matcher!($($arg ,)* v, IntervalYearMonthBuilder)), ScalarValue::IntervalDayTime(v) => ($matcher!($($arg ,)* v, IntervalDayTimeBuilder)), - ScalarValue::List(v, box dt) => ($matcher!($($arg ,)* v, dt, ListBuilder)), + ScalarValue::List(v, dt) => ($matcher!($($arg ,)* v, &**dt, ListBuilder)), ScalarValue::Binary(v) => ($matcher!($($arg ,)* v, BinaryBuilder)), ScalarValue::LargeBinary(v) => ($matcher!($($arg ,)* v, LargeBinaryBuilder)), } diff --git a/datafusion/src/datasource/parquet.rs b/datafusion/src/datasource/parquet.rs index d47da5a5d43f..0e6634fb5528 100644 --- a/datafusion/src/datasource/parquet.rs +++ b/datafusion/src/datasource/parquet.rs @@ -150,7 +150,6 @@ mod tests { use super::*; use arrow::array::{ BinaryArray, BooleanArray, Float32Array, Float64Array, Int32Array, - TimestampNanosecondArray, }; use arrow::record_batch::RecordBatch; use futures::StreamExt; diff --git a/datafusion/src/execution/context.rs b/datafusion/src/execution/context.rs index dab909a2d77e..26d87ac0c7ca 100644 --- a/datafusion/src/execution/context.rs +++ b/datafusion/src/execution/context.rs @@ -559,7 +559,7 @@ impl ExecutionContext { let handle: JoinHandle> = task::spawn(async move { stream .map(|batch| writer.write(&batch?)) - .try_collect() + .try_collect::<()>() .await .map_err(DataFusionError::from)?; writer.close().map_err(DataFusionError::from).map(|_| ()) diff --git a/datafusion/src/lib.rs b/datafusion/src/lib.rs index d470c0acd987..46761e6e8098 100644 --- a/datafusion/src/lib.rs +++ b/datafusion/src/lib.rs @@ -1,4 +1,3 @@ -#![feature(box_patterns)] // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information diff --git a/datafusion/src/physical_plan/hash_aggregate.rs b/datafusion/src/physical_plan/hash_aggregate.rs index d549855b937c..9a1f32d900b0 100644 --- a/datafusion/src/physical_plan/hash_aggregate.rs +++ b/datafusion/src/physical_plan/hash_aggregate.rs @@ -1371,10 +1371,10 @@ pub(crate) fn append_value(b: &mut dyn ArrayBuilder, v: &ScalarValue) -> Result< .expect("invalid list builder"); let vs = match $list { None => return Ok(b.append(false)?), - Some(box vs) => vs, + Some(vs) => vs, }; let values_builder = b.values(); - for v in vs { + for v in &**vs { append_value(values_builder, v)?; } Ok(b.append(true)?) diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 000000000000..8318010e59bd --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.88.0" +components = ["rustfmt", "clippy"] +profile = "minimal" \ No newline at end of file