diff --git a/Cargo.lock b/Cargo.lock index aa30a87e8..c940019ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -350,6 +350,7 @@ dependencies = [ "arrow-schema", "flatbuffers", "lz4_flex", + "zstd", ] [[package]] @@ -1377,6 +1378,15 @@ dependencies = [ "bzip2-sys", ] +[[package]] +name = "bzip2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea8dcd42434048e4f7a304411d9273a411f647446c1234a65ce0554923f4cff" +dependencies = [ + "libbz2-rs-sys", +] + [[package]] name = "bzip2-sys" version = "0.1.13+1.0.8" @@ -1892,16 +1902,16 @@ dependencies = [ [[package]] name = "datafusion" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a11e19a7ccc5bb979c95c1dceef663eab39c9061b3bbf8d1937faf0f03bf41f" +checksum = "69dfeda1633bf8ec75b068d9f6c27cdc392ffcf5ff83128d5dbab65b73c1fd02" dependencies = [ "arrow", "arrow-ipc", "arrow-schema", "async-trait", "bytes", - "bzip2 0.5.2", + "bzip2 0.6.0", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -1929,6 +1939,7 @@ dependencies = [ "datafusion-sql", "flate2", "futures", + "hex", "itertools 0.14.0", "log", "object_store", @@ -1947,9 +1958,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94985e67cab97b1099db2a7af11f31a45008b282aba921c1e1d35327c212ec18" +checksum = "2848fd1e85e2953116dab9cc2eb109214b0888d7bbd2230e30c07f1794f642c0" dependencies = [ "arrow", "async-trait", @@ -1973,9 +1984,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e002df133bdb7b0b9b429d89a69aa77b35caeadee4498b2ce1c7c23a99516988" +checksum = "051a1634628c2d1296d4e326823e7536640d87a118966cdaff069b68821ad53b" dependencies = [ "arrow", "async-trait", @@ -1996,9 +2007,9 @@ dependencies = [ [[package]] name = "datafusion-cli" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85591b54815b0c9d3fbf3b0425b75e9fc49ef73e6886d6c19d622986e2551b53" +checksum = "5966ac4973bf66cf5c189046b1b29de90db9ba059ce7193137dd089e277495de" dependencies = [ "arrow", "async-trait", @@ -2009,6 +2020,7 @@ dependencies = [ "dirs", "env_logger", "futures", + "log", "mimalloc", "object_store", "parking_lot", @@ -2021,17 +2033,19 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13242fc58fd753787b0a538e5ae77d356cb9d0656fa85a591a33c5f106267f6" +checksum = "765e4ad4ef7a4500e389a3f1e738791b71ff4c29fd00912c2f541d62b25da096" dependencies = [ "ahash 0.8.12", "apache-avro 0.17.0", "arrow", "arrow-ipc", "base64 0.22.1", + "chrono", "half", "hashbrown 0.14.5", + "hex", "indexmap 2.11.0", "libc", "log", @@ -2046,9 +2060,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2239f964e95c3a5d6b4a8cde07e646de8995c1396a7fd62c6e784f5341db499" +checksum = "40a2ae8393051ce25d232a6065c4558ab5a535c9637d5373bacfd464ac88ea12" dependencies = [ "futures", "log", @@ -2057,15 +2071,15 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cf792579bc8bf07d1b2f68c2d5382f8a63679cce8fbebfd4ba95742b6e08864" +checksum = "90cd841a77f378bc1a5c4a1c37345e1885a9203b008203f9f4b3a769729bf330" dependencies = [ "arrow", "async-compression", "async-trait", "bytes", - "bzip2 0.5.2", + "bzip2 0.6.0", "chrono", "datafusion-common", "datafusion-common-runtime", @@ -2093,9 +2107,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4de733d231abb0fba663ff60fd37bf7171fa8b2e46e8a99e41362001821d116e" +checksum = "3cba1696aa919da9517d29164d45f5902d6cc281f718e8d3bfe98bd52cd1142c" dependencies = [ "apache-avro 0.17.0", "arrow", @@ -2118,9 +2132,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc114f9a1415174f3e8d2719c371fc72092ef2195a7955404cfe6b2ba29a706" +checksum = "77f4a2c64939c6f0dd15b246723a699fa30d59d0133eb36a86e8ff8c6e2a8dc6" dependencies = [ "arrow", "async-trait", @@ -2143,9 +2157,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d88dd5e215c420a52362b9988ecd4cefd71081b730663d4f7d886f706111fc75" +checksum = "11387aaf931b2993ad9273c63ddca33f05aef7d02df9b70fb757429b4b71cdae" dependencies = [ "arrow", "async-trait", @@ -2168,9 +2182,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33692acdd1fbe75280d14f4676fe43f39e9cb36296df56575aa2cac9a819e4cf" +checksum = "028f430c5185120bf806347848b8d8acd9823f4038875b3820eeefa35f2bb4a2" dependencies = [ "arrow", "async-trait", @@ -2186,8 +2200,10 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-optimizer", "datafusion-physical-plan", + "datafusion-pruning", "datafusion-session", "futures", + "hex", "itertools 0.14.0", "log", "object_store", @@ -2199,15 +2215,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0e7b648387b0c1937b83cb328533c06c923799e73a9e3750b762667f32662c0" +checksum = "8ff336d1d755399753a9e4fbab001180e346fc8bfa063a97f1214b82274c00f8" [[package]] name = "datafusion-execution" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9609d83d52ff8315283c6dad3b97566e877d8f366fab4c3297742f33dcd636c7" +checksum = "042ea192757d1b2d7dcf71643e7ff33f6542c7704f00228d8b85b40003fd8e0f" dependencies = [ "arrow", "dashmap", @@ -2224,11 +2240,12 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e75230cd67f650ef0399eb00f54d4a073698f2c0262948298e5299fc7324da63" +checksum = "025222545d6d7fab71e2ae2b356526a1df67a2872222cbae7535e557a42abd2e" dependencies = [ "arrow", + "async-trait", "chrono", "datafusion-common", "datafusion-doc", @@ -2245,9 +2262,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70fafb3a045ed6c49cfca0cd090f62cf871ca6326cc3355cb0aaf1260fa760b6" +checksum = "9d5c267104849d5fa6d81cf5ba88f35ecd58727729c5eb84066c25227b644ae2" dependencies = [ "arrow", "datafusion-common", @@ -2258,9 +2275,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf9a9cf655265861a20453b1e58357147eab59bdc90ce7f2f68f1f35104d3bb" +checksum = "c620d105aa208fcee45c588765483314eb415f5571cfd6c1bae3a59c5b4d15bb" dependencies = [ "arrow", "arrow-buffer", @@ -2287,9 +2304,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f07e49733d847be0a05235e17b884d326a2fd402c97a89fe8bcf0bfba310005" +checksum = "35f61d5198a35ed368bf3aacac74f0d0fa33de7a7cb0c57e9f68ab1346d2f952" dependencies = [ "ahash 0.8.12", "arrow", @@ -2308,9 +2325,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4512607e10d72b0b0a1dc08f42cb5bd5284cb8348b7fea49dc83409493e32b1b" +checksum = "13efdb17362be39b5024f6da0d977ffe49c0212929ec36eec550e07e2bc7812f" dependencies = [ "ahash 0.8.12", "arrow", @@ -2321,9 +2338,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab331806e34f5545e5f03396e4d5068077395b1665795d8f88c14ec4f1e0b7a" +checksum = "9187678af567d7c9e004b72a0b6dc5b0a00ebf4901cb3511ed2db4effe092e66" dependencies = [ "arrow", "arrow-ord", @@ -2333,6 +2350,7 @@ dependencies = [ "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", "itertools 0.14.0", @@ -2342,9 +2360,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4ac2c0be983a06950ef077e34e0174aa0cb9e346f3aeae459823158037ade37" +checksum = "ecf156589cc21ef59fe39c7a9a841b4a97394549643bbfa88cc44e8588cf8fe5" dependencies = [ "arrow", "async-trait", @@ -2358,9 +2376,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f3d92731de384c90906941d36dcadf6a86d4128409a9c5cd916662baed5f53" +checksum = "edcb25e3e369f1366ec9a261456e45b5aad6ea1c0c8b4ce546587207c501ed9e" dependencies = [ "arrow", "datafusion-common", @@ -2376,9 +2394,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c679f8bf0971704ec8fd4249fcbb2eb49d6a12cc3e7a840ac047b4928d3541b5" +checksum = "8996a8e11174d0bd7c62dc2f316485affc6ae5ffd5b8a68b508137ace2310294" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2386,9 +2404,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2821de7cb0362d12e75a5196b636a59ea3584ec1e1cc7dc6f5e34b9e8389d251" +checksum = "95ee8d1be549eb7316f437035f2cec7ec42aba8374096d807c4de006a3b5d78a" dependencies = [ "datafusion-expr", "quote", @@ -2397,14 +2415,15 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1594c7a97219ede334f25347ad8d57056621e7f4f35a0693c8da876e10dd6a53" +checksum = "c9fa98671458254928af854e5f6c915e66b860a8bde505baea0ff2892deab74d" dependencies = [ "arrow", "chrono", "datafusion-common", "datafusion-expr", + "datafusion-expr-common", "datafusion-physical-expr", "indexmap 2.11.0", "itertools 0.14.0", @@ -2416,9 +2435,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6da0f2412088d23f6b01929dedd687b5aee63b19b674eb73d00c3eb3c883b7" +checksum = "3515d51531cca5f7b5a6f3ea22742b71bb36fc378b465df124ff9a2fa349b002" dependencies = [ "ahash 0.8.12", "arrow", @@ -2438,9 +2457,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb0dbd9213078a593c3fe28783beaa625a4e6c6a6c797856ee2ba234311fb96" +checksum = "24485475d9c618a1d33b2a3dad003d946dc7a7bbf0354d125301abc0a5a79e3e" dependencies = [ "ahash 0.8.12", "arrow", @@ -2452,9 +2471,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d140854b2db3ef8ac611caad12bfb2e1e1de827077429322a6188f18fc0026a" +checksum = "b9da411a0a64702f941a12af2b979434d14ec5d36c6f49296966b2c7639cbb3a" dependencies = [ "arrow", "datafusion-common", @@ -2464,6 +2483,7 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-pruning", "itertools 0.14.0", "log", "recursive", @@ -2471,9 +2491,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b46cbdf21a01206be76d467f325273b22c559c744a012ead5018dfe79597de08" +checksum = "a6d168282bb7b54880bb3159f89b51c047db4287f5014d60c3ef4c6e1468212b" dependencies = [ "ahash 0.8.12", "arrow", @@ -2499,11 +2519,29 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-pruning" +version = "49.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "391a457b9d23744c53eeb89edd1027424cba100581488d89800ed841182df905" +dependencies = [ + "arrow", + "arrow-schema", + "datafusion-common", + "datafusion-datasource", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "itertools 0.14.0", + "log", +] + [[package]] name = "datafusion-session" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a72733766ddb5b41534910926e8da5836622316f6283307fd9fb7e19811a59c" +checksum = "053201c2bb729c7938f85879034df2b5a52cfaba16f1b3b66ab8505c81b2aad3" dependencies = [ "arrow", "async-trait", @@ -2525,9 +2563,9 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f469b1301a5a2dec8b944eacb7b1194ed8c0f8203ce460b7451bb6fa2e89060" +checksum = "126ceb2436b498ddd66c0a9f58304bce6b903f622c681c0ca9e52cf229c55ea0" dependencies = [ "arrow", "datafusion-catalog", @@ -2541,9 +2579,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5162338cdec9cc7ea13a0e6015c361acad5ec1d88d83f7c86301f789473971f" +checksum = "9082779be8ce4882189b229c0cff4393bd0808282a7194130c9f32159f185e25" dependencies = [ "arrow", "bigdecimal", @@ -2558,9 +2596,9 @@ dependencies = [ [[package]] name = "datafusion-sqllogictest" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3a05ab096d3dd4c234f4ecf3c9c08b71396ac1810d4d57d3a13049bbf8a1b30" +checksum = "2c2358f74d54fb9895b3b962f14877e414327eaaad4a95154abacc3e6bdcd38b" dependencies = [ "arrow", "async-trait", @@ -2585,9 +2623,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af6029c08002772fb2c38a191d21a8b03af78750f42d9b896222f6b0e16c84cf" +checksum = "4189e480ae42f06ade81a6d12853c5f3f84fbe237d73fb8795c712b6f1923afb" dependencies = [ "async-recursion", "async-trait", @@ -4251,6 +4289,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "libbz2-rs-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" + [[package]] name = "libc" version = "0.2.175" @@ -5040,6 +5084,7 @@ dependencies = [ "num-bigint", "object_store", "paste", + "ring", "seq-macro", "simdutf8", "snap", @@ -7169,9 +7214,9 @@ dependencies = [ [[package]] name = "substrait" -version = "0.56.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13de2e20128f2a018dab1cfa30be83ae069219a65968c6f89df66ad124de2397" +checksum = "de6d24c270c6c672a86c183c3a8439ba46c1936f93cf7296aa692de3b0ff0228" dependencies = [ "heck", "pbjson", diff --git a/Cargo.toml b/Cargo.toml index 999b91175..5b5fcca05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,14 +42,14 @@ rust-version = "1.87" anyhow = "1.0.72" apache-avro = { version = "0.20", features = ["zstandard"] } array-init = "2" -arrow-arith = { version = "55.1" } -arrow-array = { version = "55.1" } -arrow-buffer = { version = "55.1" } -arrow-cast = { version = "55.1" } -arrow-ord = { version = "55.1" } -arrow-schema = { version = "55.1" } -arrow-select = { version = "55.1" } -arrow-string = { version = "55.1" } +arrow-arith = { version = "55.2" } +arrow-array = { version = "55.2" } +arrow-buffer = { version = "55.2" } +arrow-cast = { version = "55.2" } +arrow-ord = { version = "55.2" } +arrow-schema = { version = "55.2" } +arrow-select = { version = "55.2" } +arrow-string = { version = "55.2" } as-any = "0.3.2" async-trait = "0.1.88" aws-config = "1.8.1" @@ -62,9 +62,9 @@ bytes = "1.10" chrono = "0.4.41" clap = { version = "4.5.41", features = ["derive", "cargo"] } ctor = "0.2.8" -datafusion = "48.0.1" -datafusion-cli = "48.0.1" -datafusion-sqllogictest = "48.0.1" +datafusion = "49" +datafusion-cli = "49" +datafusion-sqllogictest = "49" derive_builder = "0.20" dirs = "6" enum-ordinalize = "4.3.0" @@ -94,7 +94,7 @@ num-bigint = "0.4.6" once_cell = "1.20" opendal = "0.54.0" ordered-float = "4" -parquet = "55.1" +parquet = "55.2" pilota = "0.11.10" port_scanner = "0.1.5" pretty_assertions = "1.4" @@ -102,7 +102,7 @@ rand = "0.8.5" regex = "1.10.5" reqwest = { version = "0.12.12", default-features = false, features = ["json"] } roaring = { version = "0.11" } -rust_decimal = "1.37.1" +rust_decimal = "1.37.2" serde = { version = "1.0.219", features = ["rc"] } serde_bytes = "0.11.17" serde_derive = "1.0.219" diff --git a/bindings/python/Cargo.lock b/bindings/python/Cargo.lock index 1a619dca0..70e95a37a 100644 --- a/bindings/python/Cargo.lock +++ b/bindings/python/Cargo.lock @@ -157,6 +157,7 @@ dependencies = [ "strum_macros", "thiserror 2.0.16", "uuid", + "zstd", ] [[package]] @@ -301,6 +302,7 @@ dependencies = [ "arrow-schema", "flatbuffers", "lz4_flex", + "zstd", ] [[package]] @@ -429,7 +431,7 @@ version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" dependencies = [ - "bzip2", + "bzip2 0.5.2", "flate2", "futures-core", "memchr", @@ -717,6 +719,15 @@ dependencies = [ "bzip2-sys", ] +[[package]] +name = "bzip2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea8dcd42434048e4f7a304411d9273a411f647446c1234a65ce0554923f4cff" +dependencies = [ + "libbz2-rs-sys", +] + [[package]] name = "bzip2-sys" version = "0.1.13+1.0.8" @@ -1030,16 +1041,16 @@ dependencies = [ [[package]] name = "datafusion" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a11e19a7ccc5bb979c95c1dceef663eab39c9061b3bbf8d1937faf0f03bf41f" +checksum = "69dfeda1633bf8ec75b068d9f6c27cdc392ffcf5ff83128d5dbab65b73c1fd02" dependencies = [ "arrow", "arrow-ipc", "arrow-schema", "async-trait", "bytes", - "bzip2", + "bzip2 0.6.0", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -1066,6 +1077,7 @@ dependencies = [ "datafusion-sql", "flate2", "futures", + "hex", "itertools 0.14.0", "log", "object_store", @@ -1084,9 +1096,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94985e67cab97b1099db2a7af11f31a45008b282aba921c1e1d35327c212ec18" +checksum = "2848fd1e85e2953116dab9cc2eb109214b0888d7bbd2230e30c07f1794f642c0" dependencies = [ "arrow", "async-trait", @@ -1110,9 +1122,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e002df133bdb7b0b9b429d89a69aa77b35caeadee4498b2ce1c7c23a99516988" +checksum = "051a1634628c2d1296d4e326823e7536640d87a118966cdaff069b68821ad53b" dependencies = [ "arrow", "async-trait", @@ -1133,16 +1145,18 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13242fc58fd753787b0a538e5ae77d356cb9d0656fa85a591a33c5f106267f6" +checksum = "765e4ad4ef7a4500e389a3f1e738791b71ff4c29fd00912c2f541d62b25da096" dependencies = [ "ahash 0.8.12", "arrow", "arrow-ipc", "base64", + "chrono", "half", "hashbrown 0.14.5", + "hex", "indexmap 2.11.0", "libc", "log", @@ -1157,9 +1171,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2239f964e95c3a5d6b4a8cde07e646de8995c1396a7fd62c6e784f5341db499" +checksum = "40a2ae8393051ce25d232a6065c4558ab5a535c9637d5373bacfd464ac88ea12" dependencies = [ "futures", "log", @@ -1168,15 +1182,15 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cf792579bc8bf07d1b2f68c2d5382f8a63679cce8fbebfd4ba95742b6e08864" +checksum = "90cd841a77f378bc1a5c4a1c37345e1885a9203b008203f9f4b3a769729bf330" dependencies = [ "arrow", "async-compression", "async-trait", "bytes", - "bzip2", + "bzip2 0.6.0", "chrono", "datafusion-common", "datafusion-common-runtime", @@ -1204,9 +1218,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc114f9a1415174f3e8d2719c371fc72092ef2195a7955404cfe6b2ba29a706" +checksum = "77f4a2c64939c6f0dd15b246723a699fa30d59d0133eb36a86e8ff8c6e2a8dc6" dependencies = [ "arrow", "async-trait", @@ -1229,9 +1243,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d88dd5e215c420a52362b9988ecd4cefd71081b730663d4f7d886f706111fc75" +checksum = "11387aaf931b2993ad9273c63ddca33f05aef7d02df9b70fb757429b4b71cdae" dependencies = [ "arrow", "async-trait", @@ -1254,9 +1268,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33692acdd1fbe75280d14f4676fe43f39e9cb36296df56575aa2cac9a819e4cf" +checksum = "028f430c5185120bf806347848b8d8acd9823f4038875b3820eeefa35f2bb4a2" dependencies = [ "arrow", "async-trait", @@ -1272,8 +1286,10 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-optimizer", "datafusion-physical-plan", + "datafusion-pruning", "datafusion-session", "futures", + "hex", "itertools 0.14.0", "log", "object_store", @@ -1285,15 +1301,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0e7b648387b0c1937b83cb328533c06c923799e73a9e3750b762667f32662c0" +checksum = "8ff336d1d755399753a9e4fbab001180e346fc8bfa063a97f1214b82274c00f8" [[package]] name = "datafusion-execution" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9609d83d52ff8315283c6dad3b97566e877d8f366fab4c3297742f33dcd636c7" +checksum = "042ea192757d1b2d7dcf71643e7ff33f6542c7704f00228d8b85b40003fd8e0f" dependencies = [ "arrow", "dashmap", @@ -1310,11 +1326,12 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e75230cd67f650ef0399eb00f54d4a073698f2c0262948298e5299fc7324da63" +checksum = "025222545d6d7fab71e2ae2b356526a1df67a2872222cbae7535e557a42abd2e" dependencies = [ "arrow", + "async-trait", "chrono", "datafusion-common", "datafusion-doc", @@ -1331,9 +1348,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70fafb3a045ed6c49cfca0cd090f62cf871ca6326cc3355cb0aaf1260fa760b6" +checksum = "9d5c267104849d5fa6d81cf5ba88f35ecd58727729c5eb84066c25227b644ae2" dependencies = [ "arrow", "datafusion-common", @@ -1344,9 +1361,9 @@ dependencies = [ [[package]] name = "datafusion-ffi" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "257de77f836e9bad3e0274d4ffc88555ebf559ae30d4e9c674b9809104c1cc3b" +checksum = "ec21805d9df2d834e4c6ddfbf8a1bed2bd460b89b01686fe0dcd1cee06d0b60f" dependencies = [ "abi_stable", "arrow", @@ -1366,9 +1383,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf9a9cf655265861a20453b1e58357147eab59bdc90ce7f2f68f1f35104d3bb" +checksum = "c620d105aa208fcee45c588765483314eb415f5571cfd6c1bae3a59c5b4d15bb" dependencies = [ "arrow", "arrow-buffer", @@ -1395,9 +1412,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f07e49733d847be0a05235e17b884d326a2fd402c97a89fe8bcf0bfba310005" +checksum = "35f61d5198a35ed368bf3aacac74f0d0fa33de7a7cb0c57e9f68ab1346d2f952" dependencies = [ "ahash 0.8.12", "arrow", @@ -1416,9 +1433,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4512607e10d72b0b0a1dc08f42cb5bd5284cb8348b7fea49dc83409493e32b1b" +checksum = "13efdb17362be39b5024f6da0d977ffe49c0212929ec36eec550e07e2bc7812f" dependencies = [ "ahash 0.8.12", "arrow", @@ -1429,9 +1446,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab331806e34f5545e5f03396e4d5068077395b1665795d8f88c14ec4f1e0b7a" +checksum = "9187678af567d7c9e004b72a0b6dc5b0a00ebf4901cb3511ed2db4effe092e66" dependencies = [ "arrow", "arrow-ord", @@ -1441,6 +1458,7 @@ dependencies = [ "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", "itertools 0.14.0", @@ -1450,9 +1468,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4ac2c0be983a06950ef077e34e0174aa0cb9e346f3aeae459823158037ade37" +checksum = "ecf156589cc21ef59fe39c7a9a841b4a97394549643bbfa88cc44e8588cf8fe5" dependencies = [ "arrow", "async-trait", @@ -1466,9 +1484,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f3d92731de384c90906941d36dcadf6a86d4128409a9c5cd916662baed5f53" +checksum = "edcb25e3e369f1366ec9a261456e45b5aad6ea1c0c8b4ce546587207c501ed9e" dependencies = [ "arrow", "datafusion-common", @@ -1484,9 +1502,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c679f8bf0971704ec8fd4249fcbb2eb49d6a12cc3e7a840ac047b4928d3541b5" +checksum = "8996a8e11174d0bd7c62dc2f316485affc6ae5ffd5b8a68b508137ace2310294" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1494,9 +1512,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2821de7cb0362d12e75a5196b636a59ea3584ec1e1cc7dc6f5e34b9e8389d251" +checksum = "95ee8d1be549eb7316f437035f2cec7ec42aba8374096d807c4de006a3b5d78a" dependencies = [ "datafusion-expr", "quote", @@ -1505,14 +1523,15 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1594c7a97219ede334f25347ad8d57056621e7f4f35a0693c8da876e10dd6a53" +checksum = "c9fa98671458254928af854e5f6c915e66b860a8bde505baea0ff2892deab74d" dependencies = [ "arrow", "chrono", "datafusion-common", "datafusion-expr", + "datafusion-expr-common", "datafusion-physical-expr", "indexmap 2.11.0", "itertools 0.14.0", @@ -1524,9 +1543,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6da0f2412088d23f6b01929dedd687b5aee63b19b674eb73d00c3eb3c883b7" +checksum = "3515d51531cca5f7b5a6f3ea22742b71bb36fc378b465df124ff9a2fa349b002" dependencies = [ "ahash 0.8.12", "arrow", @@ -1546,9 +1565,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb0dbd9213078a593c3fe28783beaa625a4e6c6a6c797856ee2ba234311fb96" +checksum = "24485475d9c618a1d33b2a3dad003d946dc7a7bbf0354d125301abc0a5a79e3e" dependencies = [ "ahash 0.8.12", "arrow", @@ -1560,9 +1579,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d140854b2db3ef8ac611caad12bfb2e1e1de827077429322a6188f18fc0026a" +checksum = "b9da411a0a64702f941a12af2b979434d14ec5d36c6f49296966b2c7639cbb3a" dependencies = [ "arrow", "datafusion-common", @@ -1572,6 +1591,7 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-pruning", "itertools 0.14.0", "log", "recursive", @@ -1579,9 +1599,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b46cbdf21a01206be76d467f325273b22c559c744a012ead5018dfe79597de08" +checksum = "a6d168282bb7b54880bb3159f89b51c047db4287f5014d60c3ef4c6e1468212b" dependencies = [ "ahash 0.8.12", "arrow", @@ -1609,9 +1629,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3fc7a2744332c2ef8804274c21f9fa664b4ca5889169250a6fd6b649ee5d16c" +checksum = "1b36a0c84f4500efd90487a004b533bd81de1f2bb3f143f71b7526f33b85d2e2" dependencies = [ "arrow", "chrono", @@ -1625,20 +1645,38 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "800add86852f12e3d249867425de2224c1e9fb7adc2930460548868781fbeded" +checksum = "2ec788be522806740ad6372c0a2f7e45fb37cb37f786d9b77933add49cdd058f" dependencies = [ "arrow", "datafusion-common", "prost", ] +[[package]] +name = "datafusion-pruning" +version = "49.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "391a457b9d23744c53eeb89edd1027424cba100581488d89800ed841182df905" +dependencies = [ + "arrow", + "arrow-schema", + "datafusion-common", + "datafusion-datasource", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "itertools 0.14.0", + "log", +] + [[package]] name = "datafusion-session" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a72733766ddb5b41534910926e8da5836622316f6283307fd9fb7e19811a59c" +checksum = "053201c2bb729c7938f85879034df2b5a52cfaba16f1b3b66ab8505c81b2aad3" dependencies = [ "arrow", "async-trait", @@ -1660,9 +1698,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "48.0.1" +version = "49.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5162338cdec9cc7ea13a0e6015c361acad5ec1d88d83f7c86301f789473971f" +checksum = "9082779be8ce4882189b229c0cff4393bd0808282a7194130c9f32159f185e25" dependencies = [ "arrow", "bigdecimal", @@ -2617,6 +2655,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "libbz2-rs-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" + [[package]] name = "libc" version = "0.2.175" @@ -3048,6 +3092,7 @@ dependencies = [ "num-bigint", "object_store", "paste", + "ring", "seq-macro", "simdutf8", "snap", diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 879a1450b..f7f33d072 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -31,9 +31,9 @@ license = "Apache-2.0" crate-type = ["cdylib"] [dependencies] -arrow = { version = "55.1", features = ["pyarrow", "chrono-tz"] } +arrow = { version = "55.2", features = ["pyarrow", "chrono-tz"] } iceberg = { path = "../../crates/iceberg" } pyo3 = { version = "0.24.1", features = ["extension-module", "abi3-py39"] } iceberg-datafusion = { path = "../../crates/integrations/datafusion" } -datafusion-ffi = { version = "48.0.1" } +datafusion-ffi = { version = "49" } tokio = { version = "1.46.1", default-features = false } diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 36b583422..a9c3c290c 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -49,7 +49,7 @@ include = [ ignore = ["F403", "F405"] [tool.hatch.envs.dev] -dependencies = ["maturin>=1.0,<2.0", "pytest>=8.3.2", "datafusion==45.*", "pyiceberg[sql-sqlite,pyarrow]>=0.10.0", "fastavro>=1.11.1"] +dependencies = ["maturin>=1.0,<2.0", "pytest>=8.3.2", "datafusion==49.*", "pyiceberg[sql-sqlite,pyarrow]>=0.10.0", "fastavro>=1.11.1"] [tool.hatch.envs.dev.scripts] build = "maturin build --out dist --sdist" diff --git a/crates/integrations/datafusion/src/physical_plan/commit.rs b/crates/integrations/datafusion/src/physical_plan/commit.rs index 0ee0504fa..65e66a91f 100644 --- a/crates/integrations/datafusion/src/physical_plan/commit.rs +++ b/crates/integrations/datafusion/src/physical_plan/commit.rs @@ -84,7 +84,10 @@ impl IcebergCommitExec { let count_array = Arc::new(UInt64Array::from(vec![count])) as ArrayRef; RecordBatch::try_from_iter_with_nullable(vec![("count", count_array, false)]).map_err(|e| { - DataFusionError::ArrowError(e, Some("Failed to make count batch!".to_string())) + DataFusionError::ArrowError( + Box::new(e), + Some("Failed to make count batch!".to_string()), + ) }) } diff --git a/crates/integrations/datafusion/src/physical_plan/write.rs b/crates/integrations/datafusion/src/physical_plan/write.rs index 625405c95..8f3613235 100644 --- a/crates/integrations/datafusion/src/physical_plan/write.rs +++ b/crates/integrations/datafusion/src/physical_plan/write.rs @@ -100,7 +100,10 @@ impl IcebergWriteExec { let files_array = Arc::new(StringArray::from(data_files)) as ArrayRef; RecordBatch::try_new(Self::make_result_schema(), vec![files_array]).map_err(|e| { - DataFusionError::ArrowError(e, Some("Failed to make result batch".to_string())) + DataFusionError::ArrowError( + Box::new(e), + Some("Failed to make result batch".to_string()), + ) }) } diff --git a/crates/integrations/datafusion/src/table/table_provider_factory.rs b/crates/integrations/datafusion/src/table/table_provider_factory.rs index a6d3146e5..e8e87dd31 100644 --- a/crates/integrations/datafusion/src/table/table_provider_factory.rs +++ b/crates/integrations/datafusion/src/table/table_provider_factory.rs @@ -241,7 +241,7 @@ mod tests { options: Default::default(), table_partition_cols: Default::default(), order_exprs: Default::default(), - constraints: Constraints::empty(), + constraints: Constraints::default(), column_defaults: Default::default(), if_not_exists: Default::default(), temporary: false,