Skip to content

Commit 457f579

Browse files
alambalamb
committed
Merge remote-tracking branch 'apache/main' into fix-predicate-pushdown-sa
2 parents 5ec3c15 + 76a7789 commit 457f579

File tree

168 files changed

+7033
-3433
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

168 files changed

+7033
-3433
lines changed

‎.asf.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ notifications:
2424
commits: commits@datafusion.apache.org
2525
issues: github@datafusion.apache.org
2626
pullrequests: github@datafusion.apache.org
27+
discussions: github@datafusion.apache.org
2728
jira_options: link label worklog
2829
github:
2930
description: "Apache DataFusion SQL Query Engine"
@@ -44,6 +45,7 @@ github:
4445
rebase: false
4546
features:
4647
issues: true
48+
discussions: true
4749
protected_branches:
4850
main:
4951
required_pull_request_reviews:

‎.github/workflows/rust.yml‎

Lines changed: 110 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -82,8 +82,13 @@ jobs:
8282
uses: ./.github/actions/setup-builder
8383
with:
8484
rust-version: stable
85+
- name: Check datafusion-common (default features)
86+
run: cargo check --profile ci --all-targets -p datafusion-common
87+
#
88+
# Note: Only check libraries (not --all-targets) to cover end user APIs
89+
#
8590
- name: Check datafusion-common (no-default-features)
86-
run: cargo check --profile ci --all-targets --no-default-features -p datafusion-common
91+
run: cargo check --profile ci --no-default-features -p datafusion-common
8792
# Note: don't check other feature flags as datafusion-common is not typically used standalone
8893

8994
# Check datafusion-substrait features
@@ -102,16 +107,21 @@ jobs:
102107
uses: ./.github/actions/setup-builder
103108
with:
104109
rust-version: stable
110+
- name: Check datafusion-substrait (default features)
111+
run: cargo check --profile ci --all-targets -p datafusion-substrait
112+
#
113+
# Note: Only check libraries (not --all-targets) to cover end user APIs
114+
#
105115
- name: Check datafusion-substrait (no-default-features)
106-
run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait
116+
run: cargo check --profile ci --no-default-features -p datafusion-substrait
107117
- name: Check datafusion-substrait (physical)
108-
run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait --features=physical
118+
run: cargo check --profile ci --no-default-features -p datafusion-substrait --features=physical
109119
- name: Install cmake
110120
run: |
111121
# note the builder setup runs apt-get update / installs protobuf compiler
112122
apt-get install -y cmake
113123
- name: Check datafusion-substrait (protoc)
114-
run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait --features=protoc
124+
run: cargo check --profile ci --no-default-features -p datafusion-substrait --features=protoc
115125

116126
# Check datafusion-proto features
117127
#
@@ -129,22 +139,27 @@ jobs:
129139
uses: ./.github/actions/setup-builder
130140
with:
131141
rust-version: stable
142+
- name: Check datafusion-proto (default features)
143+
run: cargo check --profile ci --all-targets -p datafusion-proto
144+
#
145+
# Note: Only check libraries (not --all-targets) to cover end user APIs
146+
#
132147
- name: Check datafusion-proto (no-default-features)
133-
run: cargo check --profile ci --all-targets --no-default-features -p datafusion-proto
148+
run: cargo check --profile ci --no-default-features -p datafusion-proto
134149
- name: Check datafusion-proto (json)
135-
run: cargo check --profile ci --all-targets --no-default-features -p datafusion-proto --features=json
150+
run: cargo check --profile ci --no-default-features -p datafusion-proto --features=json
136151
- name: Check datafusion-proto (parquet)
137-
run: cargo check --profile ci --all-targets --no-default-features -p datafusion-proto --features=parquet
152+
run: cargo check --profile ci --no-default-features -p datafusion-proto --features=parquet
138153
- name: Check datafusion-proto (avro)
139-
run: cargo check --profile ci --all-targets --no-default-features -p datafusion-proto --features=avro
154+
run: cargo check --profile ci --no-default-features -p datafusion-proto --features=avro
140155

141156

142157
# Check datafusion crate features
143158
#
144159
# Ensure via `cargo check` that the crate can be built with a
145160
# subset of the features packages enabled.
146161
linux-cargo-check-datafusion:
147-
name: cargo check datafusion
162+
name: cargo check datafusion features
148163
needs: linux-build-lib
149164
runs-on: ubuntu-latest
150165
container:
@@ -155,38 +170,54 @@ jobs:
155170
uses: ./.github/actions/setup-builder
156171
with:
157172
rust-version: stable
173+
- name: Check datafusion (default features)
174+
run: cargo check --profile ci --all-targets -p datafusion
175+
#
176+
# Note: Only check libraries (not --all-targets) to cover end user APIs
177+
#
158178
- name: Check datafusion (no-default-features)
159-
# Some of the test binaries require the parquet feature still
160-
#run: cargo check --all-targets --no-default-features -p datafusion
161179
run: cargo check --profile ci --no-default-features -p datafusion
162-
163180
- name: Check datafusion (nested_expressions)
164-
run: cargo check --profile ci --no-default-features --features=nested_expressions -p datafusion
165-
166-
- name: Check datafusion (crypto)
167-
run: cargo check --profile ci --no-default-features --features=crypto_expressions -p datafusion
168-
181+
run: cargo check --profile ci --no-default-features -p datafusion --features=nested_expressions
182+
- name: Check datafusion (array_expressions)
183+
run: cargo check --profile ci --no-default-features -p datafusion --features=array_expressions
184+
- name: Check datafusion (avro)
185+
run: cargo check --profile ci --no-default-features -p datafusion --features=avro
186+
- name: Check datafusion (backtrace)
187+
run: cargo check --profile ci --no-default-features -p datafusion --features=backtrace
188+
- name: Check datafusion (compression)
189+
run: cargo check --profile ci --no-default-features -p datafusion --features=compression
190+
- name: Check datafusion (crypto_expressions)
191+
run: cargo check --profile ci --no-default-features -p datafusion --features=crypto_expressions
169192
- name: Check datafusion (datetime_expressions)
170-
run: cargo check --profile ci --no-default-features --features=datetime_expressions -p datafusion
171-
193+
run: cargo check --profile ci --no-default-features -p datafusion --features=datetime_expressions
172194
- name: Check datafusion (encoding_expressions)
173-
run: cargo check --profile ci --no-default-features --features=encoding_expressions -p datafusion
174-
195+
run: cargo check --profile ci --no-default-features -p datafusion --features=encoding_expressions
196+
- name: Check datafusion (force_hash_collisions)
197+
run: cargo check --profile ci --no-default-features -p datafusion --features=force_hash_collisions
175198
- name: Check datafusion (math_expressions)
176-
run: cargo check --profile ci --no-default-features --features=math_expressions -p datafusion
177-
199+
run: cargo check --profile ci --no-default-features -p datafusion --features=math_expressions
200+
- name: Check datafusion (parquet)
201+
run: cargo check --profile ci --no-default-features -p datafusion --features=parquet
202+
- name: Check datafusion (pyarrow)
203+
run: cargo check --profile ci --no-default-features -p datafusion --features=pyarrow
178204
- name: Check datafusion (regex_expressions)
179-
run: cargo check --profile ci --no-default-features --features=regex_expressions -p datafusion
180-
205+
run: cargo check --profile ci --no-default-features -p datafusion --features=regex_expressions
206+
- name: Check datafusion (recursive_protection)
207+
run: cargo check --profile ci --no-default-features -p datafusion --features=recursive_protection
208+
- name: Check datafusion (serde)
209+
run: cargo check --profile ci --no-default-features -p datafusion --features=serde
181210
- name: Check datafusion (string_expressions)
182-
run: cargo check --profile ci --no-default-features --features=string_expressions -p datafusion
211+
run: cargo check --profile ci --no-default-features -p datafusion --features=string_expressions
212+
- name: Check datafusion (unicode_expressions)
213+
run: cargo check --profile ci --no-default-features -p datafusion --features=unicode_expressions
183214

184215
# Check datafusion-functions crate features
185216
#
186217
# Ensure via `cargo check` that the crate can be built with a
187218
# subset of the features packages enabled.
188219
linux-cargo-check-datafusion-functions:
189-
name: cargo check functions
220+
name: cargo check datafusion-functions features
190221
needs: linux-build-lib
191222
runs-on: ubuntu-latest
192223
container:
@@ -197,28 +228,32 @@ jobs:
197228
uses: ./.github/actions/setup-builder
198229
with:
199230
rust-version: stable
231+
- name: Check datafusion-functions (default features)
232+
run: cargo check --profile ci --all-targets -p datafusion-functions
233+
#
234+
# Note: Only check libraries (not --all-targets) to cover end user APIs
235+
#
200236
- name: Check datafusion-functions (no-default-features)
201-
run: cargo check --profile ci --all-targets --no-default-features -p datafusion-functions
202-
203-
- name: Check datafusion-functions (crypto)
204-
run: cargo check --profile ci --all-targets --no-default-features --features=crypto_expressions -p datafusion-functions
205-
237+
run: cargo check --profile ci --no-default-features -p datafusion-functions
238+
# Fails due https://github.com/apache/datafusion/issues/15207
239+
#- name: Check datafusion-functions (core_expressions)
240+
# run: cargo check --profile ci --no-default-features -p datafusion-functions --features=core_expressions
241+
- name: Check datafusion-functions (crypto_expressions)
242+
run: cargo check --profile ci --no-default-features -p datafusion-functions --features=crypto_expressions
206243
- name: Check datafusion-functions (datetime_expressions)
207-
run: cargo check --profile ci --all-targets --no-default-features --features=datetime_expressions -p datafusion-functions
208-
244+
run: cargo check --profile ci --no-default-features -p datafusion-functions --features=datetime_expressions
209245
- name: Check datafusion-functions (encoding_expressions)
210-
run: cargo check --profile ci --all-targets --no-default-features --features=encoding_expressions -p datafusion-functions
211-
246+
run: cargo check --profile ci --no-default-features -p datafusion-functions --features=encoding_expressions
212247
- name: Check datafusion-functions (math_expressions)
213-
run: cargo check --profile ci --all-targets --no-default-features --features=math_expressions -p datafusion-functions
214-
248+
run: cargo check --profile ci --no-default-features -p datafusion-functions --features=math_expressions
215249
- name: Check datafusion-functions (regex_expressions)
216-
run: cargo check --profile ci --all-targets --no-default-features --features=regex_expressions -p datafusion-functions
217-
250+
run: cargo check --profile ci --no-default-features -p datafusion-functions --features=regex_expressions
218251
- name: Check datafusion-functions (string_expressions)
219-
run: cargo check --profile ci --all-targets --no-default-features --features=string_expressions -p datafusion-functions
252+
run: cargo check --profile ci --no-default-features -p datafusion-functions --features=string_expressions
253+
- name: Check datafusion-functions (unicode_expressions)
254+
run: cargo check --profile ci --no-default-features -p datafusion-functions --features=unicode_expressions
220255

221-
# Run tests
256+
# Library and integration tests
222257
linux-test:
223258
name: cargo test (amd64)
224259
needs: linux-build-lib
@@ -232,6 +267,36 @@ jobs:
232267
run: rustup toolchain install stable
233268
- name: Install Protobuf Compiler
234269
run: sudo apt-get install -y protobuf-compiler
270+
- name: Run tests (excluding doctests and datafusion-cli)
271+
env:
272+
RUST_BACKTRACE: 1
273+
run: |
274+
cargo test \
275+
--profile ci \
276+
--exclude datafusion-examples \
277+
--exclude ffi_example_table_provider \
278+
--exclude datafusion-benchmarks \
279+
--exclude datafusion-cli \
280+
--workspace \
281+
--lib \
282+
--tests \
283+
--bins \
284+
--features serde,avro,json,backtrace,integration-tests
285+
- name: Verify Working Directory Clean
286+
run: git diff --exit-code
287+
288+
# datafusion-cli tests
289+
linux-test-datafusion-cli:
290+
name: cargo test datafusion-cli (amd64)
291+
needs: linux-build-lib
292+
runs-on: ubuntu-latest
293+
steps:
294+
- uses: actions/checkout@v4
295+
with:
296+
submodules: true
297+
fetch-depth: 1
298+
- name: Setup Rust toolchain
299+
run: rustup toolchain install stable
235300
- name: Setup Minio - S3-compatible storage
236301
run: |
237302
docker run -d --name minio-container \
@@ -252,13 +317,14 @@ jobs:
252317
AWS_SECRET_ACCESS_KEY: TEST-DataFusionPassword
253318
TEST_STORAGE_INTEGRATION: 1
254319
AWS_ALLOW_HTTP: true
255-
run: cargo test --profile ci --exclude datafusion-examples --exclude ffi_example_table_provider --exclude datafusion-benchmarks --workspace --lib --tests --bins --features avro,json,backtrace,integration-tests
320+
run: cargo test --profile ci -p datafusion-cli --lib --tests --bins
256321
- name: Verify Working Directory Clean
257322
run: git diff --exit-code
258323
- name: Minio Output
259324
if: ${{ !cancelled() }}
260325
run: docker logs minio-container
261326

327+
262328
linux-test-example:
263329
name: cargo examples (amd64)
264330
needs: linux-build-lib
@@ -460,7 +526,7 @@ jobs:
460526
uses: ./.github/actions/setup-macos-aarch64-builder
461527
- name: Run tests (excluding doctests)
462528
shell: bash
463-
run: cargo test --profile ci --lib --tests --bins --features avro,json,backtrace,integration-tests
529+
run: cargo test --profile ci --exclude datafusion-cli --workspace --lib --tests --bins --features avro,json,backtrace,integration-tests
464530

465531
test-datafusion-pyarrow:
466532
name: cargo test pyarrow (amd64)

0 commit comments

Comments
 (0)