Skip to content

Commit 1318172

Browse files
alambalamb
authored
Merge branch 'main' into alamb/forward_port_52.3.0
2 parents 9bc3d61 + 4b022c0 commit 1318172

File tree

48 files changed

+1587
-366
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+1587
-366
lines changed

‎.github/workflows/audit.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,4 +48,4 @@ jobs:
4848
- name: Run audit check
4949
# Note: you can ignore specific RUSTSEC issues using the `--ignore` flag ,for example:
5050
# run: cargo audit --ignore RUSTSEC-2026-0001
51-
run: cargo audit --ignore RUSTSEC-2024-0436
51+
run: cargo audit --ignore RUSTSEC-2024-0436 --ignore RUSTSEC-2024-0014

‎Cargo.lock‎

Lines changed: 13 additions & 11 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/core/Cargo.toml‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,6 @@ extended_tests = []
114114
arrow = { workspace = true }
115115
arrow-schema = { workspace = true, features = ["canonical_extension_types"] }
116116
async-trait = { workspace = true }
117-
bytes = { workspace = true }
118117
bzip2 = { workspace = true, optional = true }
119118
chrono = { workspace = true }
120119
datafusion-catalog = { workspace = true }
@@ -151,8 +150,6 @@ log = { workspace = true }
151150
object_store = { workspace = true }
152151
parking_lot = { workspace = true }
153152
parquet = { workspace = true, optional = true, default-features = true }
154-
rand = { workspace = true }
155-
regex = { workspace = true }
156153
serde = { version = "1.0", default-features = false, features = ["derive"], optional = true }
157154
sqlparser = { workspace = true, optional = true }
158155
tempfile = { workspace = true }
@@ -171,6 +168,7 @@ datafusion-functions-window-common = { workspace = true }
171168
datafusion-macros = { workspace = true }
172169
datafusion-physical-optimizer = { workspace = true }
173170
doc-comment = { workspace = true }
171+
bytes = { workspace = true }
174172
env_logger = { workspace = true }
175173
glob = { workspace = true }
176174
insta = { workspace = true }

‎datafusion/core/src/datasource/listing/mod.rs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@
2121
mod table;
2222
pub use datafusion_catalog_listing::helpers;
2323
pub use datafusion_catalog_listing::{ListingOptions, ListingTable, ListingTableConfig};
24-
pub use datafusion_datasource::{
25-
FileRange, ListingTableUrl, PartitionedFile, PartitionedFileStream,
26-
};
24+
// Keep for backwards compatibility until removed
25+
#[expect(deprecated)]
26+
pub use datafusion_datasource::PartitionedFileStream;
27+
pub use datafusion_datasource::{FileRange, ListingTableUrl, PartitionedFile};
2728
pub use table::ListingTableConfigExt;

‎datafusion/core/tests/dataframe/mod.rs‎

Lines changed: 17 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -3001,24 +3001,22 @@ async fn test_count_wildcard_on_sort() -> Result<()> {
30013001
assert_snapshot!(
30023002
pretty_format_batches(&sql_results).unwrap(),
30033003
@r"
3004-
+---------------+------------------------------------------------------------------------------------------------------------+
3005-
| plan_type | plan |
3006-
+---------------+------------------------------------------------------------------------------------------------------------+
3007-
| logical_plan | Projection: t1.b, count(*) |
3008-
| | Sort: count(Int64(1)) AS count(*) AS count(*) ASC NULLS LAST |
3009-
| | Projection: t1.b, count(Int64(1)) AS count(*), count(Int64(1)) |
3010-
| | Aggregate: groupBy=[[t1.b]], aggr=[[count(Int64(1))]] |
3011-
| | TableScan: t1 projection=[b] |
3012-
| physical_plan | ProjectionExec: expr=[b@0 as b, count(*)@1 as count(*)] |
3013-
| | SortPreservingMergeExec: [count(Int64(1))@2 ASC NULLS LAST] |
3014-
| | SortExec: expr=[count(*)@1 ASC NULLS LAST], preserve_partitioning=[true] |
3015-
| | ProjectionExec: expr=[b@0 as b, count(Int64(1))@1 as count(*), count(Int64(1))@1 as count(Int64(1))] |
3016-
| | AggregateExec: mode=FinalPartitioned, gby=[b@0 as b], aggr=[count(Int64(1))] |
3017-
| | RepartitionExec: partitioning=Hash([b@0], 4), input_partitions=1 |
3018-
| | AggregateExec: mode=Partial, gby=[b@0 as b], aggr=[count(Int64(1))] |
3019-
| | DataSourceExec: partitions=1, partition_sizes=[1] |
3020-
| | |
3021-
+---------------+------------------------------------------------------------------------------------------------------------+
3004+
+---------------+------------------------------------------------------------------------------------+
3005+
| plan_type | plan |
3006+
+---------------+------------------------------------------------------------------------------------+
3007+
| logical_plan | Sort: count(*) AS count(*) ASC NULLS LAST |
3008+
| | Projection: t1.b, count(Int64(1)) AS count(*) |
3009+
| | Aggregate: groupBy=[[t1.b]], aggr=[[count(Int64(1))]] |
3010+
| | TableScan: t1 projection=[b] |
3011+
| physical_plan | SortPreservingMergeExec: [count(*)@1 ASC NULLS LAST] |
3012+
| | SortExec: expr=[count(*)@1 ASC NULLS LAST], preserve_partitioning=[true] |
3013+
| | ProjectionExec: expr=[b@0 as b, count(Int64(1))@1 as count(*)] |
3014+
| | AggregateExec: mode=FinalPartitioned, gby=[b@0 as b], aggr=[count(Int64(1))] |
3015+
| | RepartitionExec: partitioning=Hash([b@0], 4), input_partitions=1 |
3016+
| | AggregateExec: mode=Partial, gby=[b@0 as b], aggr=[count(Int64(1))] |
3017+
| | DataSourceExec: partitions=1, partition_sizes=[1] |
3018+
| | |
3019+
+---------------+------------------------------------------------------------------------------------+
30223020
"
30233021
);
30243022

@@ -3028,7 +3026,7 @@ async fn test_count_wildcard_on_sort() -> Result<()> {
30283026
+---------------+----------------------------------------------------------------------------+
30293027
| plan_type | plan |
30303028
+---------------+----------------------------------------------------------------------------+
3031-
| logical_plan | Sort: count(*) ASC NULLS LAST |
3029+
| logical_plan | Sort: count(*) AS count(*) ASC NULLS LAST |
30323030
| | Aggregate: groupBy=[[t1.b]], aggr=[[count(Int64(1)) AS count(*)]] |
30333031
| | TableScan: t1 projection=[b] |
30343032
| physical_plan | SortPreservingMergeExec: [count(*)@1 ASC NULLS LAST] |

‎datafusion/datasource-json/Cargo.toml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@ datafusion-physical-plan = { workspace = true }
4444
datafusion-session = { workspace = true }
4545
futures = { workspace = true }
4646
object_store = { workspace = true }
47-
serde_json = { workspace = true }
4847
tokio = { workspace = true }
4948
tokio-stream = { workspace = true, features = ["sync"] }
5049

@@ -54,6 +53,9 @@ tokio-stream = { workspace = true, features = ["sync"] }
5453
[lints]
5554
workspace = true
5655

56+
[dev-dependencies]
57+
serde_json = { workspace = true }
58+
5759
[lib]
5860
name = "datafusion_datasource_json"
5961
path = "src/mod.rs"

‎datafusion/datasource-parquet/Cargo.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ tokio = { workspace = true }
5858
[dev-dependencies]
5959
chrono = { workspace = true }
6060
criterion = { workspace = true }
61+
datafusion-functions = { workspace = true }
6162
datafusion-functions-nested = { workspace = true }
6263
tempfile = { workspace = true }
6364

@@ -81,3 +82,7 @@ parquet_encryption = [
8182
[[bench]]
8283
name = "parquet_nested_filter_pushdown"
8384
harness = false
85+
86+
[[bench]]
87+
name = "parquet_struct_filter_pushdown"
88+
harness = false

0 commit comments

Comments
 (0)