diff --git a/Cargo.lock b/Cargo.lock index 0e9337b50e6f2..58e6265fc6d40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -233,8 +233,7 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2b10dcb159faf30d3f81f6d56c1211a5bea2ca424eabe477648a44b993320e" +source = "git+https://github.com/apache/arrow-rs.git?branch=main#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-arith", "arrow-array", @@ -256,8 +255,7 @@ dependencies = [ [[package]] name = "arrow-arith" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "288015089e7931843c80ed4032c5274f02b37bcb720c4a42096d50b390e70372" +source = "git+https://github.com/apache/arrow-rs.git?branch=main#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-array", "arrow-buffer", @@ -270,8 +268,7 @@ dependencies = [ [[package]] name = "arrow-array" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65ca404ea6191e06bf30956394173337fa9c35f445bd447fe6c21ab944e1a23c" +source = "git+https://github.com/apache/arrow-rs.git?branch=main#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "ahash", "arrow-buffer", @@ -289,8 +286,7 @@ dependencies = [ [[package]] name = "arrow-buffer" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36356383099be0151dacc4245309895f16ba7917d79bdb71a7148659c9206c56" +source = "git+https://github.com/apache/arrow-rs.git?branch=main#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "bytes", "half", @@ -301,8 +297,7 @@ dependencies = [ [[package]] name = "arrow-cast" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8e372ed52bd4ee88cc1e6c3859aa7ecea204158ac640b10e187936e7e87074" +source = "git+https://github.com/apache/arrow-rs.git?branch=main#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-array", "arrow-buffer", @@ -323,8 +318,7 @@ dependencies = [ [[package]] name = "arrow-csv" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e4100b729fe656f2e4fb32bc5884f14acf9118d4ad532b7b33c1132e4dce896" +source = "git+https://github.com/apache/arrow-rs.git?branch=main#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-array", "arrow-cast", @@ -338,8 +332,7 @@ dependencies = [ [[package]] name = "arrow-data" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf87f4ff5fc13290aa47e499a8b669a82c5977c6a1fedce22c7f542c1fd5a597" +source = "git+https://github.com/apache/arrow-rs.git?branch=main#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-buffer", "arrow-schema", @@ -351,8 +344,7 @@ dependencies = [ [[package]] name = "arrow-flight" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63654f21676be802d446c6c4bc54f6a47e18d55f9ae6f7195a6f6faf2ecdbeb" +source = "git+https://github.com/apache/arrow-rs.git?branch=main#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-arith", "arrow-array", @@ -379,8 +371,7 @@ dependencies = [ [[package]] name = "arrow-ipc" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3ca63edd2073fcb42ba112f8ae165df1de935627ead6e203d07c99445f2081" +source = "git+https://github.com/apache/arrow-rs.git?branch=main#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-array", "arrow-buffer", @@ -395,8 +386,7 @@ dependencies = [ [[package]] name = "arrow-json" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a36b2332559d3310ebe3e173f75b29989b4412df4029a26a30cc3f7da0869297" +source = "git+https://github.com/apache/arrow-rs.git?branch=main#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-array", "arrow-buffer", @@ -419,8 +409,7 @@ dependencies = [ [[package]] name = "arrow-ord" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c4e0530272ca755d6814218dffd04425c5b7854b87fa741d5ff848bf50aa39" +source = "git+https://github.com/apache/arrow-rs.git?branch=main#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-array", "arrow-buffer", @@ -432,8 +421,7 @@ dependencies = [ [[package]] name = "arrow-row" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b07f52788744cc71c4628567ad834cadbaeb9f09026ff1d7a4120f69edf7abd3" +source = "git+https://github.com/apache/arrow-rs.git?branch=main#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-array", "arrow-buffer", @@ -445,8 +433,7 @@ dependencies = [ [[package]] name = "arrow-schema" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb63203e8e0e54b288d0d8043ca8fa1013820822a27692ef1b78a977d879f2c" +source = "git+https://github.com/apache/arrow-rs.git?branch=main#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "bitflags", "serde", @@ -457,8 +444,7 @@ dependencies = [ [[package]] name = "arrow-select" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c96d8a1c180b44ecf2e66c9a2f2bbcb8b1b6f14e165ce46ac8bde211a363411b" +source = "git+https://github.com/apache/arrow-rs.git?branch=main#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "ahash", "arrow-array", @@ -471,8 +457,7 @@ dependencies = [ [[package]] name = "arrow-string" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8ad6a81add9d3ea30bf8374ee8329992c7fd246ffd8b7e2f48a3cea5aa0cc9a" +source = "git+https://github.com/apache/arrow-rs.git?branch=main#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "arrow-array", "arrow-buffer", @@ -4319,13 +4304,11 @@ dependencies = [ [[package]] name = "parquet" version = "57.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6a2926a30477c0b95fea6c28c3072712b139337a242c2cc64817bdc20a8854" +source = "git+https://github.com/apache/arrow-rs.git?branch=main#1db1a8869cceb179aa885ed58da9f0b49c03eafe" dependencies = [ "ahash", "arrow-array", "arrow-buffer", - "arrow-cast", "arrow-data", "arrow-ipc", "arrow-schema", diff --git a/Cargo.toml b/Cargo.toml index e2bbf2ea9885f..ac558279f6186 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -271,3 +271,32 @@ incremental = false inherits = "release" debug = true strip = false + +## Temporary arrow-rs patch until 58 is released + +[patch.crates-io] +arrow = { git = "https://github.com/apache/arrow-rs.git", branch = "main" } +arrow-array = { git = "https://github.com/apache/arrow-rs.git", branch = "main" } +arrow-buffer = { git = "https://github.com/apache/arrow-rs.git", branch = "main" } +arrow-cast = { git = "https://github.com/apache/arrow-rs.git", branch = "main" } +arrow-data = { git = "https://github.com/apache/arrow-rs.git", branch = "main" } +arrow-ipc = { git = "https://github.com/apache/arrow-rs.git", branch = "main" } +arrow-schema = { git = "https://github.com/apache/arrow-rs.git", branch = "main" } +arrow-select = { git = "https://github.com/apache/arrow-rs.git", branch = "main" } +arrow-string = { git = "https://github.com/apache/arrow-rs.git", branch = "main" } +arrow-ord = { git = "https://github.com/apache/arrow-rs.git", branch = "main" } +arrow-flight = { git = "https://github.com/apache/arrow-rs.git", branch = "main" } +parquet = { git = "https://github.com/apache/arrow-rs.git", branch = "main" } + +#arrow = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow" } +#arrow-array = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow-array" } +#arrow-buffer = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow-buffer" } +#arrow-cast = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow-cast" } +#arrow-data = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow-data" } +#arrow-ipc = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow-ipc" } +#arrow-schema = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow-schema" } +#arrow-select = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow-select" } +#arrow-string = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow-string" } +#arrow-ord = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow-ord" } +#arrow-flight = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow-flight" } +#parquet = { path= "/Users/andrewlamb/Software/arrow-rs2/parquet" } diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs index 9e53260e42773..84d04e087193e 100644 --- a/datafusion-cli/src/main.rs +++ b/datafusion-cli/src/main.rs @@ -617,8 +617,8 @@ mod tests { | filename | file_size_bytes | metadata_size_bytes | hits | extra | +-----------------------------------+-----------------+---------------------+------+------------------+ | alltypes_plain.parquet | 1851 | 8882 | 2 | page_index=false | - | alltypes_tiny_pages.parquet | 454233 | 269266 | 2 | page_index=true | - | lz4_raw_compressed_larger.parquet | 380836 | 1347 | 2 | page_index=false | + | alltypes_tiny_pages.parquet | 454233 | 269074 | 2 | page_index=true | + | lz4_raw_compressed_larger.parquet | 380836 | 1339 | 2 | page_index=false | +-----------------------------------+-----------------+---------------------+------+------------------+ "); @@ -648,8 +648,8 @@ mod tests { | filename | file_size_bytes | metadata_size_bytes | hits | extra | +-----------------------------------+-----------------+---------------------+------+------------------+ | alltypes_plain.parquet | 1851 | 8882 | 5 | page_index=false | - | alltypes_tiny_pages.parquet | 454233 | 269266 | 2 | page_index=true | - | lz4_raw_compressed_larger.parquet | 380836 | 1347 | 3 | page_index=false | + | alltypes_tiny_pages.parquet | 454233 | 269074 | 2 | page_index=true | + | lz4_raw_compressed_larger.parquet | 380836 | 1339 | 3 | page_index=false | +-----------------------------------+-----------------+---------------------+------+------------------+ "); diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.slt index e17322f0fe013..8f71e29fa345b 100644 --- a/datafusion/sqllogictest/test_files/array.slt +++ b/datafusion/sqllogictest/test_files/array.slt @@ -4747,10 +4747,8 @@ select array_union(arrow_cast([], 'LargeList(Int64)'), arrow_cast([], 'LargeList [] # array_union scalar function #7 -query ? +query error DataFusion error: Arrow error: Invalid argument error: Codecs \[List\(RowConverter \{ fields: \[SortField \{ options: SortOptions \{ descending: false, nulls_first: true \}, data_type: Null \}\], codecs: \[Stateless\] \}\)\] did not consume all bytes for row 0, remaining bytes: \[1\] select array_union([[null]], []); ----- -[[]] query error DataFusion error: Error during planning: Failed to coerce arguments to satisfy a call to 'array_union' function: select array_union(arrow_cast([[null]], 'LargeList(List(Int64))'), arrow_cast([], 'LargeList(Int64)')); diff --git a/datafusion/sqllogictest/test_files/datetime/arith_date_time.slt b/datafusion/sqllogictest/test_files/datetime/arith_date_time.slt index bc796a51ff5a4..8e85c8f90580e 100644 --- a/datafusion/sqllogictest/test_files/datetime/arith_date_time.slt +++ b/datafusion/sqllogictest/test_files/datetime/arith_date_time.slt @@ -113,4 +113,3 @@ SELECT '2001-09-28'::date / '03:00'::time query error Invalid timestamp arithmetic operation SELECT '2001-09-28'::date % '03:00'::time - diff --git a/datafusion/sqllogictest/test_files/datetime/arith_timestamp_duration.slt b/datafusion/sqllogictest/test_files/datetime/arith_timestamp_duration.slt index 10381346f8359..aeeebe73db701 100644 --- a/datafusion/sqllogictest/test_files/datetime/arith_timestamp_duration.slt +++ b/datafusion/sqllogictest/test_files/datetime/arith_timestamp_duration.slt @@ -144,4 +144,4 @@ query error Invalid timestamp arithmetic operation SELECT '2001-09-28T01:00:00'::timestamp % arrow_cast(12345, 'Duration(Second)'); query error Invalid timestamp arithmetic operation -SELECT '2001-09-28T01:00:00'::timestamp / arrow_cast(12345, 'Duration(Second)'); \ No newline at end of file +SELECT '2001-09-28T01:00:00'::timestamp / arrow_cast(12345, 'Duration(Second)'); diff --git a/datafusion/sqllogictest/test_files/explain_tree.slt b/datafusion/sqllogictest/test_files/explain_tree.slt index 9215ce87e3bef..3a183a7357430 100644 --- a/datafusion/sqllogictest/test_files/explain_tree.slt +++ b/datafusion/sqllogictest/test_files/explain_tree.slt @@ -268,7 +268,7 @@ physical_plan 06)┌─────────────┴─────────────┐ 07)│ DataSourceExec │ 08)│ -------------------- │ -09)│ bytes: 1040 │ +09)│ bytes: 1024 │ 10)│ format: memory │ 11)│ rows: 2 │ 12)└───────────────────────────┘ @@ -345,7 +345,7 @@ physical_plan 15)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ 16)│ DataSourceExec ││ ProjectionExec │ 17)│ -------------------- ││ -------------------- │ -18)│ bytes: 520 ││ date_col: date_col │ +18)│ bytes: 512 ││ date_col: date_col │ 19)│ format: memory ││ int_col: int_col │ 20)│ rows: 1 ││ │ 21)│ ││ string_col: │ @@ -592,7 +592,7 @@ physical_plan 07)┌─────────────┴─────────────┐ 08)│ DataSourceExec │ 09)│ -------------------- │ -10)│ bytes: 520 │ +10)│ bytes: 512 │ 11)│ format: memory │ 12)│ rows: 1 │ 13)└───────────────────────────┘ @@ -954,7 +954,7 @@ physical_plan 13)┌─────────────┴─────────────┐ 14)│ DataSourceExec │ 15)│ -------------------- │ -16)│ bytes: 520 │ +16)│ bytes: 512 │ 17)│ format: memory │ 18)│ rows: 1 │ 19)└───────────────────────────┘ @@ -1305,7 +1305,7 @@ physical_plan 42)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ 43)│ DataSourceExec ││ DataSourceExec │ 44)│ -------------------- ││ -------------------- │ -45)│ bytes: 296 ││ bytes: 288 │ +45)│ bytes: 288 ││ bytes: 280 │ 46)│ format: memory ││ format: memory │ 47)│ rows: 1 ││ rows: 1 │ 48)└───────────────────────────┘└───────────────────────────┘ @@ -1324,14 +1324,14 @@ physical_plan 04)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ 05)│ DataSourceExec ││ ProjectionExec │ 06)│ -------------------- ││ -------------------- │ -07)│ bytes: 296 ││ id: CAST(id AS Int32) │ +07)│ bytes: 288 ││ id: CAST(id AS Int32) │ 08)│ format: memory ││ name: name │ 09)│ rows: 1 ││ │ 10)└───────────────────────────┘└─────────────┬─────────────┘ 11)-----------------------------┌─────────────┴─────────────┐ 12)-----------------------------│ DataSourceExec │ 13)-----------------------------│ -------------------- │ -14)-----------------------------│ bytes: 288 │ +14)-----------------------------│ bytes: 280 │ 15)-----------------------------│ format: memory │ 16)-----------------------------│ rows: 1 │ 17)-----------------------------└───────────────────────────┘ diff --git a/datafusion/sqllogictest/test_files/spark/datetime/make_interval.slt b/datafusion/sqllogictest/test_files/spark/datetime/make_interval.slt index d6c5199b87b75..23c3e76d59556 100644 --- a/datafusion/sqllogictest/test_files/spark/datetime/make_interval.slt +++ b/datafusion/sqllogictest/test_files/spark/datetime/make_interval.slt @@ -96,7 +96,7 @@ NULL query T SELECT make_interval(0, 0, 0, 0, 0, 0, 0.0) || ''; ---- -(empty) +0 secs # Intervals being rendered as empty string, see issue: # https://github.com/apache/datafusion/issues/17455 @@ -104,7 +104,7 @@ SELECT make_interval(0, 0, 0, 0, 0, 0, 0.0) || ''; query T SELECT make_interval() || ''; ---- -(empty) +0 secs query ? SELECT INTERVAL '1' SECOND AS iv; diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.slt index 8ac8724683a8a..1a87e092b7a02 100644 --- a/datafusion/sqllogictest/test_files/window.slt +++ b/datafusion/sqllogictest/test_files/window.slt @@ -6068,7 +6068,7 @@ FROM ( # `WHERE acctbal > ( SELECT AVG(acctbal) FROM suppliers)` into a Join, # breaking the input schema passed to the window function above. # See: https://github.com/apache/datafusion/issues/17770 -query I +query error WITH suppliers AS ( SELECT * FROM (VALUES (1, 10.0), (1, 20.0)) AS t(nation, acctbal) @@ -6080,4 +6080,6 @@ WHERE acctbal > ( SELECT AVG(acctbal) FROM suppliers ); ---- -1 +DataFusion error: Join Error +caused by +External error: task 20768 panicked with message "assertion `left == right` failed\n left: 4\n right: 1"