diff --git a/Cargo.lock b/Cargo.lock index c54a4c92..3fad89ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,15 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "addr2line" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" -dependencies = [ - "gimli", -] - [[package]] name = "adler2" version = "2.0.1" @@ -135,51 +126,39 @@ version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" -[[package]] -name = "arrayref" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" - -[[package]] -name = "arrayvec" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" - [[package]] name = "arrow" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc" +checksum = "4df8bb5b0bd64c0b9bc61317fcc480bad0f00e56d3bc32c69a4c8dada4786bae" dependencies = [ "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-cast 57.0.0", "arrow-csv", - "arrow-data", - "arrow-ipc", + "arrow-data 57.0.0", + "arrow-ipc 57.0.0", "arrow-json", "arrow-ord", "arrow-row", - "arrow-schema", - "arrow-select", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", "arrow-string", ] [[package]] name = "arrow-arith" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" +checksum = "a1a640186d3bd30a24cb42264c2dafb30e236a6f50d510e56d40b708c9582491" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", "chrono", - "num", + "num-traits", ] [[package]] @@ -189,16 +168,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" dependencies = [ "ahash", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "chrono", - "chrono-tz", "half", - "hashbrown 0.16.0", + "hashbrown 0.16.1", "num", ] +[[package]] +name = "arrow-array" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219fe420e6800979744c8393b687afb0252b3f8a89b91027d27887b72aa36d31" +dependencies = [ + "ahash", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.16.1", + "num-complex", + "num-integer", + "num-traits", +] + [[package]] name = "arrow-buffer" version = "56.2.0" @@ -210,36 +207,68 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-buffer" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76885a2697a7edf6b59577f568b456afc94ce0e2edc15b784ce3685b6c3c5c27" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + [[package]] name = "arrow-cast" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "atoi", "base64 0.22.1", "chrono", - "comfy-table", "half", "lexical-core", "num", "ryu", ] +[[package]] +name = "arrow-cast" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9ebb4c987e6b3b236fb4a14b20b34835abfdd80acead3ccf1f9bf399e1f168" +dependencies = [ + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", + "atoi", + "base64 0.22.1", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + [[package]] name = "arrow-csv" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa9bf02705b5cf762b6f764c65f04ae9082c7cfc4e96e0c33548ee3f67012eb" +checksum = "92386159c8d4bce96f8bd396b0642a0d544d471bdc2ef34d631aec80db40a09c" dependencies = [ - "arrow-array", - "arrow-cast", - "arrow-schema", + "arrow-array 57.0.0", + "arrow-cast 57.0.0", + "arrow-schema 57.0.0", "chrono", "csv", "csv-core", @@ -252,29 +281,62 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 56.2.0", + "arrow-schema 56.2.0", "half", "num", ] +[[package]] +name = "arrow-data" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727681b95de313b600eddc2a37e736dcb21980a40f640314dcf360e2f36bc89b" +dependencies = [ + "arrow-buffer 57.0.0", + "arrow-schema 57.0.0", + "half", + "num-integer", + "num-traits", +] + [[package]] name = "arrow-flight" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c8b0ba0784d56bc6266b79f5de7a24b47024e7b3a0045d2ad4df3d9b686099f" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-ipc", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-ipc 56.2.0", + "arrow-schema 56.2.0", + "base64 0.22.1", + "bytes", + "futures", + "prost 0.13.5", + "prost-types 0.13.5", + "tonic 0.13.1", +] + +[[package]] +name = "arrow-flight" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f70bb56412a007b0cfc116d15f24dda6adeed9611a213852a004cda20085a3b9" +dependencies = [ + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-cast 57.0.0", + "arrow-ipc 57.0.0", + "arrow-schema 57.0.0", "base64 0.22.1", "bytes", "futures", - "prost", - "prost-types", - "tonic", + "prost 0.14.1", + "prost-types 0.14.1", + "tonic 0.14.2", + "tonic-prost", ] [[package]] @@ -283,61 +345,76 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d3594dcddccc7f20fd069bc8e9828ce37220372680ff638c5e00dea427d88f5" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "flatbuffers", +] + +[[package]] +name = "arrow-ipc" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9ba92e3de170295c98a84e5af22e2b037f0c7b32449445e6c493b5fca27f27" +dependencies = [ + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", "flatbuffers", "lz4_flex", - "zstd", ] [[package]] name = "arrow-json" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88cf36502b64a127dc659e3b305f1d993a544eab0d48cce704424e62074dc04b" +checksum = "b969b4a421ae83828591c6bf5450bd52e6d489584142845ad6a861f42fe35df8" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-cast 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", "chrono", "half", "indexmap", + "itoa", "lexical-core", "memchr", - "num", - "serde", + "num-traits", + "ryu", + "serde_core", "serde_json", "simdutf8", ] [[package]] name = "arrow-ord" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" +checksum = "141c05298b21d03e88062317a1f1a73f5ba7b6eb041b350015b1cd6aabc0519b" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", ] [[package]] name = "arrow-row" -version = "56.2.0" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753" +checksum = "c5f3c06a6abad6164508ed283c7a02151515cef3de4b4ff2cebbcaeb85533db2" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", "half", ] @@ -346,8 +423,14 @@ name = "arrow-schema" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" + +[[package]] +name = "arrow-schema" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cfa7a03d1eee2a4d061476e1840ad5c9867a544ca6c4c59256496af5d0a8be5" dependencies = [ - "serde", + "serde_core", "serde_json", ] @@ -358,45 +441,42 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "num", ] [[package]] -name = "arrow-string" -version = "56.2.0" +name = "arrow-select" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" +checksum = "bafa595babaad59f2455f4957d0f26448fb472722c186739f4fac0823a1bdb47" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", - "memchr", - "num", - "regex", - "regex-syntax", + "ahash", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "num-traits", ] [[package]] -name = "async-compression" -version = "0.4.19" +name = "arrow-string" +version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" +checksum = "32f46457dbbb99f2650ff3ac23e46a929e0ab81db809b02aa5511c258348bef2" dependencies = [ - "bzip2 0.5.2", - "flate2", - "futures-core", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", "memchr", - "pin-project-lite", - "tokio", - "xz2", - "zstd", - "zstd-safe", + "num-traits", + "regex", + "regex-syntax", ] [[package]] @@ -407,7 +487,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -922,21 +1002,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "backtrace" -version = "0.3.75" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-targets 0.52.6", -] - [[package]] name = "base64" version = "0.21.7" @@ -961,9 +1026,9 @@ dependencies = [ [[package]] name = "bigdecimal" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a22f228ab7a1b23027ccc6c350b72868017af7ea8356fbdf19f8d991c690013" +checksum = "560f42649de9fa436b73517378a147ec21f6c997a546581df4b4b31677828934" dependencies = [ "autocfg", "libm", @@ -984,28 +1049,6 @@ version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" -[[package]] -name = "blake2" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" -dependencies = [ - "digest", -] - -[[package]] -name = "blake3" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" -dependencies = [ - "arrayref", - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -1064,34 +1107,6 @@ dependencies = [ "either", ] -[[package]] -name = "bzip2" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" -dependencies = [ - "bzip2-sys", -] - -[[package]] -name = "bzip2" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bea8dcd42434048e4f7a304411d9273a411f647446c1234a65ce0554923f4cff" -dependencies = [ - "libbz2-rs-sys", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "cc" version = "1.2.30" @@ -1203,12 +1218,6 @@ dependencies = [ "tiny-keccak", ] -[[package]] -name = "constant_time_eq" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" - [[package]] name = "core-foundation" version = "0.9.4" @@ -1312,22 +1321,21 @@ dependencies = [ [[package]] name = "datafusion" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "481d0c1cad7606cee11233abcdff8eec46e43dd25abda007db6d5d26ae8483c4" +checksum = "8ba7cb113e9c0bedf9e9765926031e132fa05a1b09ba6e93a6d1a4d7044457b8" dependencies = [ "arrow", - "arrow-ipc", - "arrow-schema", + "arrow-schema 57.0.0", "async-trait", "bytes", - "bzip2 0.6.0", "chrono", "datafusion-catalog", "datafusion-catalog-listing", "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", + "datafusion-datasource-arrow", "datafusion-datasource-csv", "datafusion-datasource-json", "datafusion-datasource-parquet", @@ -1347,29 +1355,27 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "datafusion-sql", - "flate2", "futures", "itertools", "log", "object_store", "parking_lot", - "parquet", + "parquet 57.0.0", "rand 0.9.2", "regex", + "rstest", "sqlparser", "tempfile", "tokio", "url", "uuid", - "xz2", - "zstd", ] [[package]] name = "datafusion-catalog" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d70327e81ab3a1f5832d8b372d55fa607851d7cea6d1f8e65ff0c98fcc32d222" +checksum = "66a3a799f914a59b1ea343906a0486f17061f39509af74e874a866428951130d" dependencies = [ "arrow", "async-trait", @@ -1382,7 +1388,6 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-plan", "datafusion-session", - "datafusion-sql", "futures", "itertools", "log", @@ -1393,9 +1398,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "268819e6bb20ba70a664abddc20deac604f30d3267f8c91847064542a8c0720c" +checksum = "6db1b113c80d7a0febcd901476a57aef378e717c54517a163ed51417d87621b0" dependencies = [ "arrow", "async-trait", @@ -1405,10 +1410,11 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-physical-expr", + "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", - "datafusion-session", "futures", + "itertools", "log", "object_store", "tokio", @@ -1416,14 +1422,13 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "054873d5563f115f83ef4270b560ac2ce4de713905e825a40cac49d6ff348254" +checksum = "7c10f7659e96127d25e8366be7c8be4109595d6a2c3eac70421f380a7006a1b0" dependencies = [ "ahash", "arrow", - "arrow-ipc", - "base64 0.22.1", + "arrow-ipc 57.0.0", "chrono", "half", "hashbrown 0.14.5", @@ -1431,9 +1436,8 @@ dependencies = [ "libc", "log", "object_store", - "parquet", + "parquet 57.0.0", "paste", - "recursive", "sqlparser", "tokio", "web-time", @@ -1441,9 +1445,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a1d1bc69aaaadb8008b65329ed890b33e845dc063225c190f77b20328fbe1d" +checksum = "b92065bbc6532c6651e2f7dd30b55cba0c7a14f860c7e1d15f165c41a1868d95" dependencies = [ "futures", "log", @@ -1452,15 +1456,13 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d855160469020982880fd9bd0962e033d2f4728f56f85a83d8c90785638b6519" +checksum = "fde13794244bc7581cd82f6fff217068ed79cdc344cafe4ab2c3a1c3510b38d6" dependencies = [ "arrow", - "async-compression", "async-trait", "bytes", - "bzip2 0.6.0", "chrono", "datafusion-common", "datafusion-common-runtime", @@ -1471,38 +1473,54 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", - "flate2", "futures", "glob", "itertools", "log", "object_store", - "parquet", "rand 0.9.2", - "tempfile", "tokio", - "tokio-util", "url", - "xz2", - "zstd", +] + +[[package]] +name = "datafusion-datasource-arrow" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804fa9b4ecf3157982021770617200ef7c1b2979d57bec9044748314775a9aea" +dependencies = [ + "arrow", + "arrow-ipc 57.0.0", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools", + "object_store", + "tokio", ] [[package]] name = "datafusion-datasource-csv" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ec3aa7575378d23aae96b955b5233bea6f9d461648174f6ccc8f3c160f2b7a7" +checksum = "61a1641a40b259bab38131c5e6f48fac0717bedb7dc93690e604142a849e0568" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-catalog", "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", "datafusion-execution", "datafusion-expr", - "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", @@ -1514,49 +1532,44 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00cfb8f33e2864eeb3188b6818acf5546d56a5a487d423cce9b684a554caabfa" +checksum = "adeacdb00c1d37271176f8fb6a1d8ce096baba16ea7a4b2671840c5c9c64fe85" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-catalog", "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", "datafusion-execution", "datafusion-expr", - "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", "futures", "object_store", - "serde_json", "tokio", ] [[package]] name = "datafusion-datasource-parquet" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3bfb48fb4ff42ac1485a12ea56434eaab53f7da8f00b2443b1a3d35a0b6d10" +checksum = "43d0b60ffd66f28bfb026565d62b0a6cbc416da09814766a3797bba7d85a3cd9" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-catalog", "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", "datafusion-execution", "datafusion-expr", - "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", "datafusion-physical-expr", "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", - "datafusion-physical-optimizer", "datafusion-physical-plan", "datafusion-pruning", "datafusion-session", @@ -1565,8 +1578,7 @@ dependencies = [ "log", "object_store", "parking_lot", - "parquet", - "rand 0.9.2", + "parquet 57.0.0", "tokio", ] @@ -1575,8 +1587,8 @@ name = "datafusion-distributed" version = "0.1.0" dependencies = [ "arrow", - "arrow-flight", - "arrow-select", + "arrow-flight 57.0.0", + "arrow-select 57.0.0", "async-trait", "bytes", "chrono", @@ -1590,15 +1602,15 @@ dependencies = [ "insta", "itertools", "object_store", - "parquet", + "parquet 57.0.0", "pin-project", "pretty_assertions", - "prost", + "prost 0.14.1", "rand 0.8.5", "structopt", "tokio", "tokio-stream", - "tonic", + "tonic 0.14.2", "tower", "tpchgen", "tpchgen-arrow", @@ -1610,7 +1622,7 @@ dependencies = [ name = "datafusion-distributed-benchmarks" version = "0.1.0" dependencies = [ - "arrow-flight", + "arrow-flight 56.2.0", "async-trait", "aws-config", "aws-sdk-ec2", @@ -1624,27 +1636,27 @@ dependencies = [ "futures", "log", "object_store", - "parquet", - "prost", + "parquet 56.2.0", + "prost 0.13.5", "serde", "serde_json", "structopt", "tokio", - "tonic", + "tonic 0.13.1", "url", ] [[package]] name = "datafusion-doc" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fbf41013cf55c2369b5229594898e8108c8a1beeb49d97feb5e0cce9933eb8f" +checksum = "2b99e13947667b36ad713549237362afb054b2d8f8cc447751e23ec61202db07" [[package]] name = "datafusion-execution" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fd0c1ffe3885687758f985ed548184bf63b17b2a7a5ae695de422ad6432118" +checksum = "63695643190679037bc946ad46a263b62016931547bf119859c511f7ff2f5178" dependencies = [ "arrow", "async-trait", @@ -1662,9 +1674,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c4fe6411218a9dab656437b1e69b00a470a7a2d7db087867a366c145eb164a7" +checksum = "f9a4787cbf5feb1ab351f789063398f67654a6df75c4d37d7f637dc96f951a91" dependencies = [ "arrow", "async-trait", @@ -1676,17 +1688,17 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr-common", "indexmap", + "itertools", "paste", - "recursive", "serde_json", "sqlparser", ] [[package]] name = "datafusion-expr-common" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a45bee7d2606bfb41ceb1d904ba7cecf69bd5a6f8f3e6c57c3f5a83d84bdd97" +checksum = "5ce2fb1b8c15c9ac45b0863c30b268c69dc9ee7a1ee13ecf5d067738338173dc" dependencies = [ "arrow", "datafusion-common", @@ -1697,15 +1709,13 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7e1c532ff9d14f291160bca23e55ffd4899800301dd2389786c2f02d76904a" +checksum = "794a9db7f7b96b3346fc007ff25e994f09b8f0511b4cf7dff651fadfe3ebb28f" dependencies = [ "arrow", - "arrow-buffer", + "arrow-buffer 57.0.0", "base64 0.22.1", - "blake2", - "blake3", "chrono", "datafusion-common", "datafusion-doc", @@ -1716,19 +1726,18 @@ dependencies = [ "hex", "itertools", "log", - "md-5", + "num-traits", "rand 0.9.2", "regex", - "sha2", "unicode-segmentation", "uuid", ] [[package]] name = "datafusion-functions-aggregate" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05d47426645aef1e73b1a034c75ab2401bc504175feb191accbe211ec24a342" +checksum = "1c25210520a9dcf9c2b2cbbce31ebd4131ef5af7fc60ee92b266dc7d159cb305" dependencies = [ "ahash", "arrow", @@ -1747,9 +1756,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c99f648b2b1743de0c1c19eef07e8cc5a085237f172b2e20bf6934e0a804e4" +checksum = "62f4a66f3b87300bb70f4124b55434d2ae3fe80455f3574701d0348da040b55d" dependencies = [ "ahash", "arrow", @@ -1760,9 +1769,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4227782023f4fb68d3d5c5eb190665212f43c9a0b437553e4b938b379aff6cf6" +checksum = "ae5c06eed03918dc7fe7a9f082a284050f0e9ecf95d72f57712d1496da03b8c4" dependencies = [ "arrow", "arrow-ord", @@ -1770,6 +1779,7 @@ dependencies = [ "datafusion-doc", "datafusion-execution", "datafusion-expr", + "datafusion-expr-common", "datafusion-functions", "datafusion-functions-aggregate", "datafusion-functions-aggregate-common", @@ -1782,9 +1792,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d902b1769f69058236e89f04f3bff2cf62f24311adb7bf3c6c3e945c9451076" +checksum = "db4fed1d71738fbe22e2712d71396db04c25de4111f1ec252b8f4c6d3b25d7f5" dependencies = [ "arrow", "async-trait", @@ -1798,9 +1808,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8ee43974c92eb9920fe8e97e0fab48675e93b062abcb48bef4c1d4305b6ee4" +checksum = "1d92206aa5ae21892f1552b4d61758a862a70956e6fd7a95cb85db1de74bc6d1" dependencies = [ "arrow", "datafusion-common", @@ -1816,9 +1826,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e149d36cdd44fb425dc815c5fac55025aa9a592dd65cb3c421881096292c02" +checksum = "53ae9bcc39800820d53a22d758b3b8726ff84a5a3e24cecef04ef4e5fdf1c7cc" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1826,20 +1836,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07c9faa0cdefb6e6e756482b846397b5c2d84d369e30b009472b9ab9b1430fbd" +checksum = "1063ad4c9e094b3f798acee16d9a47bd7372d9699be2de21b05c3bd3f34ab848" dependencies = [ - "datafusion-expr", + "datafusion-doc", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] name = "datafusion-optimizer" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f16a4f7059302ad1de6e97ab0eebb5c34405917b1f80806a30a66e38ad118251" +checksum = "9f35f9ec5d08b87fd1893a30c2929f2559c2f9806ca072d8fefca5009dc0f06a" dependencies = [ "arrow", "chrono", @@ -1850,16 +1860,15 @@ dependencies = [ "indexmap", "itertools", "log", - "recursive", "regex", "regex-syntax", ] [[package]] name = "datafusion-physical-expr" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10bb87a605d8ce9672d5347c0293c12211b0c03923fc12fbdc665fe76e6f9e01" +checksum = "c30cc8012e9eedcb48bbe112c6eff4ae5ed19cf3003cb0f505662e88b7014c5d" dependencies = [ "ahash", "arrow", @@ -1872,7 +1881,6 @@ dependencies = [ "hashbrown 0.14.5", "indexmap", "itertools", - "log", "parking_lot", "paste", "petgraph", @@ -1880,9 +1888,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da3a7429a555dd5ff0bec4d24bd5532ec43876764088da635cad55b2f178dc2" +checksum = "7f9ff2dbd476221b1f67337699eff432781c4e6e1713d2aefdaa517dfbf79768" dependencies = [ "arrow", "datafusion-common", @@ -1895,9 +1903,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "845eb44ef1e04d2a15c6d955cb146b40a41814a7be4377f0a541857d3e257d6f" +checksum = "90da43e1ec550b172f34c87ec68161986ced70fd05c8d2a2add66eef9c276f03" dependencies = [ "ahash", "arrow", @@ -1909,9 +1917,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b9b648ee2785722c79eae366528e52e93ece6808aef9297cf8e5521de381da" +checksum = "ce9804f799acd7daef3be7aaffe77c0033768ed8fdbf5fb82fc4c5f2e6bc14e6" dependencies = [ "arrow", "datafusion-common", @@ -1923,20 +1931,18 @@ dependencies = [ "datafusion-physical-plan", "datafusion-pruning", "itertools", - "log", - "recursive", ] [[package]] name = "datafusion-physical-plan" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e6688d17b78104e169d7069749832c20ff50f112be853d2c058afe46c889064" +checksum = "0acf0ad6b6924c6b1aa7d213b181e012e2d3ec0a64ff5b10ee6282ab0f8532ac" dependencies = [ "ahash", "arrow", "arrow-ord", - "arrow-schema", + "arrow-schema 57.0.0", "async-trait", "chrono", "datafusion-common", @@ -1960,39 +1966,49 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521cd45740788e751bf59b25d2879d162b157a45fb9b5fa2ec03034923f3658a" +checksum = "d368093a98a17d1449b1083ac22ed16b7128e4c67789991869480d8c4a40ecb9" dependencies = [ "arrow", "chrono", - "datafusion", + "datafusion-catalog", + "datafusion-catalog-listing", "datafusion-common", + "datafusion-datasource", + "datafusion-datasource-arrow", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", + "datafusion-execution", "datafusion-expr", + "datafusion-functions-table", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", "datafusion-proto-common", "object_store", - "prost", + "prost 0.14.1", ] [[package]] name = "datafusion-proto-common" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c15ede0e0f1e51d5b5bea1a196db35e00aa3cae9e58cc12df3cc900e36328437" +checksum = "3b6aef3d5e5c1d2bc3114c4876730cb76a9bdc5a8df31ef1b6db48f0c1671895" dependencies = [ "arrow", "datafusion-common", - "prost", + "prost 0.14.1", ] [[package]] name = "datafusion-pruning" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a893a46c56f5f190085e13949eb8ec163672c7ec2ac33bdb82c84572e71ca73" +checksum = "ac2c2498a1f134a9e11a9f5ed202a2a7d7e9774bd9249295593053ea3be999db" dependencies = [ "arrow", - "arrow-schema", "datafusion-common", "datafusion-datasource", "datafusion-expr-common", @@ -2005,41 +2021,31 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8b62684c7a1db6121a8c83100209cffa1e664a8d9ced87e1a32f8cdc2fff3c2" +checksum = "8f96eebd17555386f459037c65ab73aae8df09f464524c709d6a3134ad4f4776" dependencies = [ - "arrow", "async-trait", - "dashmap", "datafusion-common", - "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", - "datafusion-physical-expr", "datafusion-physical-plan", - "datafusion-sql", - "futures", - "itertools", - "log", - "object_store", "parking_lot", - "tokio", ] [[package]] name = "datafusion-sql" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09cff94b8242843e1da5d069e9d2cfc53807f1f00b1c0da78c297f47c21456e" +checksum = "3fc195fe60634b2c6ccfd131b487de46dc30eccae8a3c35a13f136e7f440414f" dependencies = [ "arrow", "bigdecimal", + "chrono", "datafusion-common", "datafusion-expr", "indexmap", "log", - "recursive", "regex", "sqlparser", ] @@ -2052,7 +2058,7 @@ checksum = "6178a82cf56c836a3ba61a7935cdb1c49bfaa6fa4327cd5bf554a503087de26b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -2089,7 +2095,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -2253,7 +2259,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -2268,6 +2274,12 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + [[package]] name = "futures-util" version = "0.3.31" @@ -2323,12 +2335,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "gimli" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" - [[package]] name = "glob" version = "0.3.2" @@ -2375,13 +2381,14 @@ dependencies = [ [[package]] name = "half" -version = "2.6.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "cfg-if", "crunchy", "num-traits", + "zerocopy", ] [[package]] @@ -2400,16 +2407,14 @@ version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" dependencies = [ - "allocator-api2", - "equivalent", "foldhash", ] [[package]] name = "hashbrown" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" [[package]] name = "heck" @@ -2772,12 +2777,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.11.4" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" +checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" dependencies = [ "equivalent", - "hashbrown 0.16.0", + "hashbrown 0.16.1", ] [[package]] @@ -2798,17 +2803,6 @@ version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" -[[package]] -name = "io-uring" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" -dependencies = [ - "bitflags 2.9.1", - "cfg-if", - "libc", -] - [[package]] name = "ipnet" version = "2.11.0" @@ -2867,7 +2861,7 @@ checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -2960,17 +2954,11 @@ dependencies = [ "static_assertions", ] -[[package]] -name = "libbz2-rs-sys" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" - [[package]] name = "libc" -version = "0.2.176" +version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f929b4d672ea937a23a1ab494143d968337a5f47e56d0815df1e0890ddf174" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" [[package]] name = "libm" @@ -3030,17 +3018,6 @@ dependencies = [ "twox-hash", ] -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "matchit" version = "0.7.3" @@ -3175,15 +3152,6 @@ dependencies = [ "libm", ] -[[package]] -name = "object" -version = "0.36.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" -dependencies = [ - "memchr", -] - [[package]] name = "object_store" version = "0.12.4" @@ -3283,27 +3251,60 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0dbd48ad52d7dccf8ea1b90a3ddbfaea4f69878dd7683e51c507d4bc52b5b27" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ipc", - "arrow-schema", - "arrow-select", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-data 56.2.0", + "arrow-ipc 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "base64 0.22.1", "brotli", "bytes", "chrono", "flate2", - "futures", "half", - "hashbrown 0.16.0", + "hashbrown 0.16.1", "lz4_flex", "num", "num-bigint", + "paste", + "seq-macro", + "simdutf8", + "snap", + "thrift", + "twox-hash", + "zstd", +] + +[[package]] +name = "parquet" +version = "57.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a0f31027ef1af7549f7cec603a9a21dce706d3f8d7c2060a68f43c1773be95a" +dependencies = [ + "ahash", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-cast 57.0.0", + "arrow-data 57.0.0", + "arrow-ipc 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", + "base64 0.22.1", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.16.1", + "lz4_flex", + "num-bigint", + "num-integer", + "num-traits", "object_store", "paste", - "ring", "seq-macro", "simdutf8", "snap", @@ -3327,9 +3328,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "petgraph" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.4", @@ -3372,7 +3373,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -3442,6 +3443,15 @@ dependencies = [ "yansi", ] +[[package]] +name = "proc-macro-crate" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -3482,7 +3492,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.13.5", +] + +[[package]] +name = "prost" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" +dependencies = [ + "bytes", + "prost-derive 0.14.1", ] [[package]] @@ -3495,7 +3515,20 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", +] + +[[package]] +name = "prost-derive" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.110", ] [[package]] @@ -3504,16 +3537,16 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ - "prost", + "prost 0.13.5", ] [[package]] -name = "psm" -version = "0.1.26" +name = "prost-types" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e944464ec8536cd1beb0bbfd96987eb5e3b72f2ecdafdc5c769a37f1fa2ae1f" +checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" dependencies = [ - "cc", + "prost 0.14.1", ] [[package]] @@ -3583,9 +3616,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.40" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" dependencies = [ "proc-macro2", ] @@ -3655,26 +3688,6 @@ dependencies = [ "getrandom 0.3.3", ] -[[package]] -name = "recursive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" -dependencies = [ - "recursive-proc-macro-impl", - "stacker", -] - -[[package]] -name = "recursive-proc-macro-impl" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" -dependencies = [ - "quote", - "syn 2.0.106", -] - [[package]] name = "redox_syscall" version = "0.5.15" @@ -3686,9 +3699,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.1" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", @@ -3698,9 +3711,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.9" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", @@ -3719,6 +3732,12 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "reqwest" version = "0.12.24" @@ -3776,10 +3795,33 @@ dependencies = [ ] [[package]] -name = "rustc-demangle" -version = "0.1.25" +name = "rstest" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros", +] + +[[package]] +name = "rstest_macros" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.110", + "unicode-ident", +] [[package]] name = "rustc-hash" @@ -4020,7 +4062,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -4143,12 +4185,11 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.58.0" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec4b661c54b1e4b603b37873a18c59920e4c51ea8ea2cf527d925424dbd4437c" +checksum = "4591acadbcf52f0af60eafbb2c003232b2b4cd8de5f0e9437cb8b1b59046cc0f" dependencies = [ "log", - "recursive", "sqlparser_derive", ] @@ -4160,7 +4201,7 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -4169,19 +4210,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" -[[package]] -name = "stacker" -version = "0.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cddb07e32ddb770749da91081d8d0ac3a16f1a569a18b20348cd371f5dead06b" -dependencies = [ - "cc", - "cfg-if", - "libc", - "psm", - "windows-sys 0.59.0", -] - [[package]] name = "static_assertions" version = "1.1.0" @@ -4234,7 +4262,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -4256,9 +4284,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.106" +version = "2.0.110" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea" dependencies = [ "proc-macro2", "quote", @@ -4282,7 +4310,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -4324,7 +4352,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -4404,33 +4432,30 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.47.1" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" dependencies = [ - "backtrace", "bytes", - "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", - "slab", "socket2 0.6.0", "tokio-macros", - "windows-sys 0.59.0", + "windows-sys 0.61.1", ] [[package]] name = "tokio-macros" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -4477,6 +4502,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml_datetime" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.23.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e" +dependencies = [ + "winnow", +] + [[package]] name = "tonic" version = "0.13.1" @@ -4496,7 +4551,7 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost", + "prost 0.13.5", "socket2 0.5.10", "tokio", "tokio-stream", @@ -4506,6 +4561,46 @@ dependencies = [ "tracing", ] +[[package]] +name = "tonic" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" +dependencies = [ + "async-trait", + "axum 0.8.4", + "base64 0.22.1", + "bytes", + "h2 0.4.11", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "hyper 1.6.0", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2 0.6.0", + "sync_wrapper", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-prost" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" +dependencies = [ + "bytes", + "prost 0.14.1", + "tonic 0.14.2", +] + [[package]] name = "tower" version = "0.5.2" @@ -4558,12 +4653,12 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tpchgen" version = "2.0.1" -source = "git+https://github.com/clflushopt/tpchgen-rs?rev=482ee68#482ee68404d61a5308ca1fd0095b347b3831a5b4" +source = "git+https://github.com/clflushopt/tpchgen-rs?rev=e83365a5a9101906eb9f78c5607b83bc59849acf#e83365a5a9101906eb9f78c5607b83bc59849acf" [[package]] name = "tpchgen-arrow" version = "2.0.1" -source = "git+https://github.com/clflushopt/tpchgen-rs?rev=482ee68#482ee68404d61a5308ca1fd0095b347b3831a5b4" +source = "git+https://github.com/clflushopt/tpchgen-rs?rev=e83365a5a9101906eb9f78c5607b83bc59849acf#e83365a5a9101906eb9f78c5607b83bc59849acf" dependencies = [ "arrow", "tpchgen", @@ -4589,7 +4684,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -4764,7 +4859,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", "wasm-bindgen-shared", ] @@ -4799,7 +4894,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4898,7 +4993,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -4909,7 +5004,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -5106,6 +5201,15 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +[[package]] +name = "winnow" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen-rt" version = "0.39.0" @@ -5127,15 +5231,6 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - [[package]] name = "yansi" version = "1.0.1" @@ -5162,7 +5257,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", "synstructure", ] @@ -5183,7 +5278,7 @@ checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] @@ -5203,7 +5298,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", "synstructure", ] @@ -5243,7 +5338,7 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.110", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 6740029c..34b4d0e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,8 +2,8 @@ members = ["benchmarks"] [workspace.dependencies] -datafusion = { version = "50.0.0", default-features = false } -datafusion-proto = { version = "50.0.0" } +datafusion = { version = "51.0.0", default-features = false } +datafusion-proto = { version = "51.0.0" } [package] name = "datafusion-distributed" @@ -12,14 +12,18 @@ edition = "2024" [dependencies] chrono = { version = "0.4.42" } -datafusion = { workspace = true } +datafusion = { workspace = true, features = [ + "parquet", + "sql", + "unicode_expressions", + "datetime_expressions", +] } datafusion-proto = { workspace = true } -arrow-flight = "56.1.0" -arrow-select = "56.1.0" +arrow-flight = "57.0.0" +arrow-select = "57.0.0" async-trait = "0.1.88" tokio = { version = "1.46.1", features = ["full"] } -# Updated to 0.13.1 to match arrow-flight 56.1.0 -tonic = { version = "0.13.1", features = ["transport"] } +tonic = { version = "0.14.2", features = ["transport"] } tower = "0.5.2" http = "1.3.1" itertools = "0.14.0" @@ -28,7 +32,7 @@ url = "2.5.4" uuid = "1.17.0" delegate = "0.13.4" dashmap = "6.1.0" -prost = "0.13.5" +prost = "0.14.0" rand = "0.8.5" object_store = "0.12.3" bytes = "1.10.1" @@ -36,10 +40,10 @@ pin-project = "1.1.10" # integration_tests deps insta = { version = "1.43.1", features = ["filters"], optional = true } -tpchgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "482ee68", optional = true } -tpchgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "482ee68", optional = true } -parquet = { version = "56.1.0", optional = true } -arrow = { version = "56.1.0", optional = true } +tpchgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "e83365a5a9101906eb9f78c5607b83bc59849acf", optional = true } +tpchgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "e83365a5a9101906eb9f78c5607b83bc59849acf", optional = true } +parquet = { version = "57.0.0", optional = true } +arrow = { version = "57.0.0", optional = true } tokio-stream = { version = "0.1.17", optional = true } hyper-util = { version = "0.1.16", optional = true } pretty_assertions = { version = "1.4", optional = true } @@ -53,7 +57,7 @@ integration = [ "arrow", "tokio-stream", "hyper-util", - "pretty_assertions" + "pretty_assertions", ] tpch = ["integration"] @@ -61,10 +65,10 @@ tpch = ["integration"] [dev-dependencies] structopt = "0.3" insta = { version = "1.43.1", features = ["filters"] } -tpchgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "482ee68" } -tpchgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "482ee68" } -parquet = "56.1.0" -arrow = "56.1.0" +tpchgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "e83365a5a9101906eb9f78c5607b83bc59849acf" } +tpchgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "e83365a5a9101906eb9f78c5607b83bc59849acf" } +parquet = "57.0.0" +arrow = "57.0.0" tokio-stream = "0.1.17" hyper-util = "0.1.16" pretty_assertions = "1.4" diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 4bd1ff45..ab40f4f4 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.86.0" +channel = "1.88.0" profile = "default" diff --git a/src/common/ttl_map.rs b/src/common/ttl_map.rs index 4b808513..7edbf725 100644 --- a/src/common/ttl_map.rs +++ b/src/common/ttl_map.rs @@ -409,7 +409,7 @@ mod tests { } tokio::time::sleep(Duration::from_millis(10)).await; } - panic!("Assertion failed within {:?}", timeout); + panic!("Assertion failed within {timeout:?}"); } #[tokio::test(flavor = "multi_thread", worker_threads = 8)] @@ -543,9 +543,9 @@ mod tests { let elapsed = start_time.elapsed(); println!("\n=== TTLMap Lock Contention Benchmark ==="); - println!("Tasks: {}", task_count); - println!("Total time: {:.2?}", elapsed); - println!("Average latency: {:.2} ns per operation", avg_time); + println!("Tasks: {task_count}"); + println!("Total time: {elapsed:.2?}"); + println!("Average latency: {avg_time:.2} ns per operation"); println!("Entries remaining: {}", ttl_map.data.len()); println!( "DashMap Lock contention time: {}ms", diff --git a/src/distributed_ext.rs b/src/distributed_ext.rs index 89e98b2c..6efdda02 100644 --- a/src/distributed_ext.rs +++ b/src/distributed_ext.rs @@ -137,7 +137,7 @@ pub trait DistributedExt: Sized { /// ``` /// # use std::sync::Arc; /// # use datafusion::common::DataFusionError; - /// # use datafusion::execution::{SessionState, FunctionRegistry, SessionStateBuilder}; + /// # use datafusion::execution::{SessionState, FunctionRegistry, SessionStateBuilder, TaskContext}; /// # use datafusion::physical_plan::ExecutionPlan; /// # use datafusion::prelude::SessionConfig; /// # use datafusion_proto::physical_plan::PhysicalExtensionCodec; @@ -147,7 +147,7 @@ pub trait DistributedExt: Sized { /// struct CustomExecCodec; /// /// impl PhysicalExtensionCodec for CustomExecCodec { - /// fn try_decode(&self, buf: &[u8], inputs: &[Arc], registry: &dyn FunctionRegistry) -> datafusion::common::Result> { + /// fn try_decode(&self, buf: &[u8], inputs: &[Arc], ctx: &TaskContext) -> datafusion::common::Result> { /// todo!() /// } /// diff --git a/src/execution_plans/network_shuffle.rs b/src/execution_plans/network_shuffle.rs index 93376dde..1e2f3138 100644 --- a/src/execution_plans/network_shuffle.rs +++ b/src/execution_plans/network_shuffle.rs @@ -112,6 +112,7 @@ use tonic::metadata::MetadataMap; /// 2. Ready: runs within a distributed stage and queries the next input stage over the network /// using Arrow Flight. #[derive(Debug, Clone)] +#[allow(clippy::large_enum_variant)] pub enum NetworkShuffleExec { Pending(NetworkShufflePendingExec), Ready(NetworkShuffleReadyExec), diff --git a/src/execution_plans/partition_isolator.rs b/src/execution_plans/partition_isolator.rs index dfe64c93..5fb8d088 100644 --- a/src/execution_plans/partition_isolator.rs +++ b/src/execution_plans/partition_isolator.rs @@ -50,6 +50,7 @@ use std::{fmt::Formatter, sync::Arc}; /// └───────────────────────────┘ └───────────────────────────┘ └───────────────────────────┘ ■ /// ``` #[derive(Debug)] +#[allow(clippy::large_enum_variant)] pub enum PartitionIsolatorExec { Pending(PartitionIsolatorPendingExec), Ready(PartitionIsolatorReadyExec), @@ -223,7 +224,7 @@ impl ExecutionPlan for PartitionIsolatorExec { // then look up that index in our group and execute that partition, in this // example partition 8 - let output_stream = match partition_group.get(partition) { + match partition_group.get(partition) { Some(actual_partition_number) => { if *actual_partition_number >= input_partitions { //trace!("{} returning empty stream", ctx_name); @@ -239,8 +240,7 @@ impl ExecutionPlan for PartitionIsolatorExec { Box::pin(EmptyRecordBatchStream::new(self_ready.input.schema())) as SendableRecordBatchStream, ), - }; - output_stream + } } } diff --git a/src/flight_service/do_get.rs b/src/flight_service/do_get.rs index 396e9c71..6e03324a 100644 --- a/src/flight_service/do_get.rs +++ b/src/flight_service/do_get.rs @@ -96,7 +96,7 @@ impl ArrowFlightEndpoint { let stage_data = once .get_or_try_init(|| async { let proto_node = PhysicalPlanNode::try_decode(doget.plan_proto.as_ref())?; - let mut plan = proto_node.try_into_physical_plan(&ctx, &self.runtime, &codec)?; + let mut plan = proto_node.try_into_physical_plan(&ctx.task_ctx(), &codec)?; for hook in self.hooks.on_plan.iter() { plan = hook(plan) } diff --git a/src/flight_service/session_builder.rs b/src/flight_service/session_builder.rs index 3cf617bc..1a387bc5 100644 --- a/src/flight_service/session_builder.rs +++ b/src/flight_service/session_builder.rs @@ -25,7 +25,7 @@ pub trait DistributedSessionBuilder { /// # use std::sync::Arc; /// # use async_trait::async_trait; /// # use datafusion::error::DataFusionError; - /// # use datafusion::execution::{FunctionRegistry, SessionState, SessionStateBuilder}; + /// # use datafusion::execution::{FunctionRegistry, SessionState, SessionStateBuilder, TaskContext}; /// # use datafusion::physical_plan::ExecutionPlan; /// # use datafusion_proto::physical_plan::PhysicalExtensionCodec; /// # use datafusion_distributed::{DistributedExt, DistributedSessionBuilder, DistributedSessionBuilderContext}; @@ -34,7 +34,7 @@ pub trait DistributedSessionBuilder { /// struct CustomExecCodec; /// /// impl PhysicalExtensionCodec for CustomExecCodec { - /// fn try_decode(&self, buf: &[u8], inputs: &[Arc], registry: &dyn FunctionRegistry) -> datafusion::common::Result> { + /// fn try_decode(&self, buf: &[u8], inputs: &[Arc], ctx: &TaskContext) -> datafusion::common::Result> { /// todo!() /// } /// diff --git a/src/metrics/metrics_collecting_stream.rs b/src/metrics/metrics_collecting_stream.rs index 744a944d..593e09b2 100644 --- a/src/metrics/metrics_collecting_stream.rs +++ b/src/metrics/metrics_collecting_stream.rs @@ -47,7 +47,7 @@ where let metadata = FlightAppMetadata::decode(flight_data.app_metadata.as_ref()).map_err(|e| { - FlightError::ProtocolError(format!("failed to decode app_metadata: {}", e)) + FlightError::ProtocolError(format!("failed to decode app_metadata: {e}")) })?; let Some(content) = metadata.content else { diff --git a/src/metrics/proto.rs b/src/metrics/proto.rs index 90b3b5f7..bbfcf66b 100644 --- a/src/metrics/proto.rs +++ b/src/metrics/proto.rs @@ -160,8 +160,8 @@ pub fn df_metrics_set_to_proto( Err(err) => { // Check if this is the specific custom metrics error we want to filter out if let DataFusionError::Internal(msg) = &err { - if msg == CUSTOM_METRICS_NOT_SUPPORTED { - // Filter out custom metrics error - continue processing other metrics + if msg == CUSTOM_METRICS_NOT_SUPPORTED || msg == UNSUPPORTED_METRICS { + // Filter out custom/unsupported metrics error - continue processing other metrics continue; } } @@ -191,6 +191,9 @@ pub fn metrics_set_proto_to_df( const CUSTOM_METRICS_NOT_SUPPORTED: &str = "custom metrics are not supported in metrics proto conversion"; +/// New DataFusion metrics that are not yet supported in proto conversion. +const UNSUPPORTED_METRICS: &str = "metric type not supported in proto conversion"; + /// df_metric_to_proto converts a `datafusion::physical_plan::metrics::Metric` to a `MetricProto`. It does not consume the Arc. pub fn df_metric_to_proto(metric: Arc) -> Result { let partition = metric.partition().map(|p| p as u64); @@ -285,6 +288,10 @@ pub fn df_metric_to_proto(metric: Arc) -> Result internal_err!("{}", CUSTOM_METRICS_NOT_SUPPORTED), + MetricValue::OutputBytes(_) | MetricValue::PruningMetrics { .. } | MetricValue::Ratio { .. } => { + // TODO: Support these metrics + internal_err!("{}", UNSUPPORTED_METRICS) + } } } @@ -433,8 +440,7 @@ mod tests { let roundtrip_count = roundtrip_metrics_set.iter().count(); assert_eq!( original_count, roundtrip_count, - "roundtrip should preserve metrics count for {}", - test_name + "roundtrip should preserve metrics count for {test_name}" ); // Verify equivalence of each metric. @@ -442,29 +448,25 @@ mod tests { assert_eq!( original.partition(), roundtrip.partition(), - "partition mismatch in {}", - test_name + "partition mismatch in {test_name}" ); assert_eq!( original.labels().len(), roundtrip.labels().len(), - "label count mismatch in {}", - test_name + "label count mismatch in {test_name}" ); for (orig_label, rt_label) in original.labels().iter().zip(roundtrip.labels().iter()) { assert_eq!( orig_label.name(), rt_label.name(), - "label name mismatch in {}", - test_name + "label name mismatch in {test_name}" ); assert_eq!( orig_label.value(), rt_label.value(), - "label value mismatch in {}", - test_name + "label value mismatch in {test_name}" ); } diff --git a/src/protobuf/distributed_codec.rs b/src/protobuf/distributed_codec.rs index 59025e7f..ee4fb04d 100644 --- a/src/protobuf/distributed_codec.rs +++ b/src/protobuf/distributed_codec.rs @@ -8,11 +8,11 @@ use datafusion::arrow::datatypes::Schema; use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::internal_datafusion_err; use datafusion::error::DataFusionError; -use datafusion::execution::{FunctionRegistry, SessionStateBuilder}; +use datafusion::execution::TaskContext; use datafusion::physical_expr::EquivalenceProperties; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::{ExecutionPlan, Partitioning, PlanProperties}; -use datafusion::prelude::{SessionConfig, SessionContext}; +use datafusion::prelude::SessionConfig; use datafusion_proto::physical_plan::from_proto::parse_protobuf_partitioning; use datafusion_proto::physical_plan::to_proto::serialize_partitioning; use datafusion_proto::physical_plan::{ComposedPhysicalExtensionCodec, PhysicalExtensionCodec}; @@ -40,7 +40,7 @@ impl PhysicalExtensionCodec for DistributedCodec { &self, buf: &[u8], inputs: &[Arc], - registry: &dyn FunctionRegistry, + ctx: &TaskContext, ) -> datafusion::common::Result> { let DistributedExecProto { node: Some(distributed_exec_node), @@ -51,20 +51,6 @@ impl PhysicalExtensionCodec for DistributedCodec { )); }; - // TODO: The PhysicalExtensionCodec trait doesn't provide access to session state, - // so we create a new SessionContext which loses any custom UDFs, UDAFs, and other - // user configurations. This is a limitation of the current trait design. - let state = SessionStateBuilder::new() - .with_scalar_functions( - registry - .udfs() - .iter() - .map(|f| registry.udf(f)) - .collect::, _>>()?, - ) - .build(); - let ctx = SessionContext::from(state); - fn parse_stage_proto( proto: Option, inputs: &[Arc], @@ -114,7 +100,7 @@ impl PhysicalExtensionCodec for DistributedCodec { let partitioning = parse_protobuf_partitioning( partitioning.as_ref(), - &ctx, + ctx, &schema, &DistributedCodec {}, )? @@ -138,7 +124,7 @@ impl PhysicalExtensionCodec for DistributedCodec { let partitioning = parse_protobuf_partitioning( partitioning.as_ref(), - &ctx, + ctx, &schema, &DistributedCodec {}, )? @@ -403,11 +389,12 @@ mod tests { use datafusion::physical_expr::LexOrdering; use datafusion::physical_plan::empty::EmptyExec; use datafusion::{ - execution::registry::MemoryFunctionRegistry, physical_expr::{Partitioning, PhysicalSortExpr, expressions::Column, expressions::col}, physical_plan::{ExecutionPlan, displayable, sorts::sort::SortExec, union::UnionExec}, }; + use datafusion::prelude::SessionContext; + fn empty_exec() -> Arc { Arc::new(EmptyExec::new(SchemaRef::new(Schema::empty()))) } @@ -429,10 +416,14 @@ mod tests { displayable(plan.as_ref()).indent(true).to_string() } + fn create_context() -> Arc { + SessionContext::new().task_ctx() + } + #[test] fn test_roundtrip_single_flight() -> datafusion::common::Result<()> { let codec = DistributedCodec; - let registry = MemoryFunctionRegistry::new(); + let ctx = create_context(); let schema = schema_i32("a"); let part = Partitioning::Hash(vec![Arc::new(Column::new("a", 0))], 4); @@ -442,7 +433,7 @@ mod tests { let mut buf = Vec::new(); codec.try_encode(plan.clone(), &mut buf)?; - let decoded = codec.try_decode(&buf, &[empty_exec()], ®istry)?; + let decoded = codec.try_decode(&buf, &[empty_exec()], &ctx)?; assert_eq!(repr(&plan), repr(&decoded)); Ok(()) @@ -451,7 +442,7 @@ mod tests { #[test] fn test_roundtrip_isolator_flight() -> datafusion::common::Result<()> { let codec = DistributedCodec; - let registry = MemoryFunctionRegistry::new(); + let ctx = create_context(); let schema = schema_i32("b"); let flight = Arc::new(new_network_hash_shuffle_exec( @@ -466,7 +457,7 @@ mod tests { let mut buf = Vec::new(); codec.try_encode(plan.clone(), &mut buf)?; - let decoded = codec.try_decode(&buf, &[flight], ®istry)?; + let decoded = codec.try_decode(&buf, &[flight], &ctx)?; assert_eq!(repr(&plan), repr(&decoded)); Ok(()) @@ -475,7 +466,7 @@ mod tests { #[test] fn test_roundtrip_isolator_union() -> datafusion::common::Result<()> { let codec = DistributedCodec; - let registry = MemoryFunctionRegistry::new(); + let ctx = create_context(); let schema = schema_i32("c"); let left = Arc::new(new_network_hash_shuffle_exec( @@ -489,14 +480,14 @@ mod tests { dummy_stage(), )); - let union = Arc::new(UnionExec::new(vec![left.clone(), right.clone()])); + let union = UnionExec::try_new(vec![left.clone(), right.clone()])?; let plan: Arc = Arc::new(PartitionIsolatorExec::new_ready(union.clone(), 1)?); let mut buf = Vec::new(); codec.try_encode(plan.clone(), &mut buf)?; - let decoded = codec.try_decode(&buf, &[union], ®istry)?; + let decoded = codec.try_decode(&buf, &[union], &ctx)?; assert_eq!(repr(&plan), repr(&decoded)); Ok(()) @@ -505,7 +496,7 @@ mod tests { #[test] fn test_roundtrip_isolator_sort_flight() -> datafusion::common::Result<()> { let codec = DistributedCodec; - let registry = MemoryFunctionRegistry::new(); + let ctx = create_context(); let schema = schema_i32("d"); let flight = Arc::new(new_network_hash_shuffle_exec( @@ -529,7 +520,7 @@ mod tests { let mut buf = Vec::new(); codec.try_encode(plan.clone(), &mut buf)?; - let decoded = codec.try_decode(&buf, &[sort], ®istry)?; + let decoded = codec.try_decode(&buf, &[sort], &ctx)?; assert_eq!(repr(&plan), repr(&decoded)); Ok(()) @@ -538,7 +529,7 @@ mod tests { #[test] fn test_roundtrip_single_flight_coalesce() -> datafusion::common::Result<()> { let codec = DistributedCodec; - let registry = MemoryFunctionRegistry::new(); + let ctx = create_context(); let schema = schema_i32("e"); let plan: Arc = Arc::new(new_network_coalesce_tasks_exec( @@ -550,7 +541,7 @@ mod tests { let mut buf = Vec::new(); codec.try_encode(plan.clone(), &mut buf)?; - let decoded = codec.try_decode(&buf, &[empty_exec()], ®istry)?; + let decoded = codec.try_decode(&buf, &[empty_exec()], &ctx)?; assert_eq!(repr(&plan), repr(&decoded)); Ok(()) @@ -559,7 +550,7 @@ mod tests { #[test] fn test_roundtrip_isolator_flight_coalesce() -> datafusion::common::Result<()> { let codec = DistributedCodec; - let registry = MemoryFunctionRegistry::new(); + let ctx = create_context(); let schema = schema_i32("f"); let flight = Arc::new(new_network_coalesce_tasks_exec( @@ -574,7 +565,7 @@ mod tests { let mut buf = Vec::new(); codec.try_encode(plan.clone(), &mut buf)?; - let decoded = codec.try_decode(&buf, &[flight], ®istry)?; + let decoded = codec.try_decode(&buf, &[flight], &ctx)?; assert_eq!(repr(&plan), repr(&decoded)); Ok(()) @@ -583,7 +574,7 @@ mod tests { #[test] fn test_roundtrip_isolator_union_coalesce() -> datafusion::common::Result<()> { let codec = DistributedCodec; - let registry = MemoryFunctionRegistry::new(); + let ctx = create_context(); let schema = schema_i32("g"); let left = Arc::new(new_network_coalesce_tasks_exec( @@ -597,14 +588,14 @@ mod tests { dummy_stage(), )); - let union = Arc::new(UnionExec::new(vec![left.clone(), right.clone()])); + let union = UnionExec::try_new(vec![left.clone(), right.clone()])?; let plan: Arc = Arc::new(PartitionIsolatorExec::new_ready(union.clone(), 3)?); let mut buf = Vec::new(); codec.try_encode(plan.clone(), &mut buf)?; - let decoded = codec.try_decode(&buf, &[union], ®istry)?; + let decoded = codec.try_decode(&buf, &[union], &ctx)?; assert_eq!(repr(&plan), repr(&decoded)); Ok(()) diff --git a/src/protobuf/errors/arrow_error.rs b/src/protobuf/errors/arrow_error.rs index 01f2a31e..a8630bba 100644 --- a/src/protobuf/errors/arrow_error.rs +++ b/src/protobuf/errors/arrow_error.rs @@ -53,6 +53,8 @@ pub enum ArrowErrorInnerProto { RunEndIndexOverflowError(bool), #[prost(uint64, tag = "20")] OffsetOverflowError(u64), + #[prost(string, tag = "21")] + AvroError(String), } impl ArrowErrorProto { @@ -136,6 +138,10 @@ impl ArrowErrorProto { inner: Some(ArrowErrorInnerProto::OffsetOverflowError(*offset as u64)), ctx: ctx.cloned(), }, + ArrowError::AvroError(msg) => ArrowErrorProto { + inner: Some(ArrowErrorInnerProto::AvroError(msg.to_string())), + ctx: ctx.cloned(), + }, } } @@ -185,6 +191,7 @@ impl ArrowErrorProto { ArrowErrorInnerProto::OffsetOverflowError(offset) => { ArrowError::OffsetOverflowError(*offset as usize) } + ArrowErrorInnerProto::AvroError(msg) => ArrowError::AvroError(msg.to_string()), }; (err, self.ctx.clone()) } @@ -200,10 +207,7 @@ mod tests { fn test_arrow_error_roundtrip() { let test_cases = vec![ ArrowError::NotYetImplemented("test not implemented".to_string()), - ArrowError::ExternalError(Box::new(std::io::Error::new( - ErrorKind::Other, - "external error", - ))), + ArrowError::ExternalError(Box::new(std::io::Error::other("external error"))), ArrowError::CastError("cast error".to_string()), ArrowError::MemoryError("memory error".to_string()), ArrowError::ParseError("parse error".to_string()), @@ -235,8 +239,8 @@ mod tests { let (recovered_error, recovered_ctx) = proto.to_arrow_error(); if original_error.to_string() != recovered_error.to_string() { - println!("original error: {}", original_error); - println!("recovered error: {}", recovered_error); + println!("original error: {original_error}"); + println!("recovered error: {recovered_error}"); } assert_eq!(original_error.to_string(), recovered_error.to_string()); diff --git a/src/protobuf/errors/datafusion_error.rs b/src/protobuf/errors/datafusion_error.rs index d1312b72..36095718 100644 --- a/src/protobuf/errors/datafusion_error.rs +++ b/src/protobuf/errors/datafusion_error.rs @@ -313,7 +313,7 @@ mod tests { ), DataFusionError::Execution("execution error".to_string()), DataFusionError::ResourcesExhausted("resources exhausted".to_string()), - DataFusionError::External(Box::new(std::io::Error::new(ErrorKind::Other, "external"))), + DataFusionError::External(Box::new(std::io::Error::other("external"))), DataFusionError::Context( "context message".to_string(), Box::new(DataFusionError::Internal("nested".to_string())), diff --git a/src/protobuf/errors/objectstore_error.rs b/src/protobuf/errors/objectstore_error.rs index 45d0ff7e..ea0bb3ff 100644 --- a/src/protobuf/errors/objectstore_error.rs +++ b/src/protobuf/errors/objectstore_error.rs @@ -267,7 +267,7 @@ mod tests { // Use known store names that will be preserved ObjectStoreError::Generic { store: "S3", - source: Box::new(std::io::Error::new(ErrorKind::Other, "generic error")), + source: Box::new(std::io::Error::other("generic error")), }, ObjectStoreError::NotFound { path: "test/path".to_string(), @@ -279,14 +279,14 @@ mod tests { }, ObjectStoreError::Precondition { path: "precondition/path".to_string(), - source: Box::new(std::io::Error::new(ErrorKind::Other, "precondition failed")), + source: Box::new(std::io::Error::other("precondition failed")), }, ObjectStoreError::NotSupported { source: Box::new(std::io::Error::new(ErrorKind::Unsupported, "not supported")), }, ObjectStoreError::NotModified { path: "not/modified".to_string(), - source: Box::new(std::io::Error::new(ErrorKind::Other, "not modified")), + source: Box::new(std::io::Error::other("not modified")), }, ObjectStoreError::NotImplemented, ObjectStoreError::PermissionDenied { @@ -298,7 +298,7 @@ mod tests { }, ObjectStoreError::Unauthenticated { path: "auth/path".to_string(), - source: Box::new(std::io::Error::new(ErrorKind::Other, "unauthenticated")), + source: Box::new(std::io::Error::other("unauthenticated")), }, ObjectStoreError::UnknownConfigurationKey { key: "unknown_key".to_string(), diff --git a/src/protobuf/errors/parquet_error.rs b/src/protobuf/errors/parquet_error.rs index 991e68e5..3ddd6530 100644 --- a/src/protobuf/errors/parquet_error.rs +++ b/src/protobuf/errors/parquet_error.rs @@ -104,10 +104,7 @@ mod tests { ParquetError::EOF("end of file".to_string()), ParquetError::ArrowError("arrow error".to_string()), ParquetError::IndexOutOfBound(42, 100), - ParquetError::External(Box::new(std::io::Error::new( - std::io::ErrorKind::Other, - "external error", - ))), + ParquetError::External(Box::new(std::io::Error::other("external error"))), ParquetError::NeedMoreData(1024), ]; diff --git a/src/stage.rs b/src/stage.rs index fa4a37e8..6b10db02 100644 --- a/src/stage.rs +++ b/src/stage.rs @@ -343,10 +343,9 @@ pub fn display_plan_graphviz(plan: Arc) -> Result { f, "digraph G {{ rankdir=BT - edge[colorscheme={}, penwidth=2.0] + edge[colorscheme={COLOR_SCHEME}, penwidth=2.0] splines=false -", - COLOR_SCHEME +" )?; if plan.as_any().is::() { @@ -367,7 +366,7 @@ pub fn display_plan_graphviz(plan: Arc) -> Result { for stage in &all_stages { for i in 0..stage.tasks.iter().len() { let p = display_single_task(stage, i)?; - writeln!(f, "{}", p)?; + writeln!(f, "{p}")?; } } // now draw edges between the tasks @@ -390,7 +389,7 @@ pub fn display_plan_graphviz(plan: Arc) -> Result { // single plan, not a stage tree writeln!(f, "node[shape=none]")?; let p = display_plan(&plan, 0, 1, 0)?; - writeln!(f, "{}", p)?; + writeln!(f, "{p}")?; } writeln!(f, "}}")?; @@ -457,7 +456,7 @@ fn display_plan( while let Some(plan) = queue.pop_front() { node_index += 1; let p = display_single_plan(plan.as_ref(), stage_num, task_i, node_index)?; - writeln!(f, "{}", p)?; + writeln!(f, "{p}")?; if plan.is_network_boundary() { continue; @@ -605,7 +604,7 @@ pub fn display_single_plan( )?; for i in 0..output_partitions { - writeln!(f, " ", i)?; + writeln!(f, " ")?; } writeln!( @@ -631,7 +630,7 @@ pub fn display_single_plan( )?; for i in 0..input_partitions { - writeln!(f, " ", i)?; + writeln!(f, " ")?; } writeln!( diff --git a/src/test_utils/insta.rs b/src/test_utils/insta.rs index 5fca0bfc..110cd016 100644 --- a/src/test_utils/insta.rs +++ b/src/test_utils/insta.rs @@ -62,6 +62,22 @@ pub fn settings() -> insta::Settings { settings.add_filter(r"bytes_scanned=\d+", "bytes_scanned="); settings.add_filter(r"file_open_errors=\d+", "file_open_errors="); settings.add_filter(r"file_scan_errors=\d+", "file_scan_errors="); + settings.add_filter( + r"aggregate_arguments_time=[\d.]+[a-zA-Zµnms]+", + "aggregate_arguments_time=", + ); + settings.add_filter( + r"aggregation_time=[\d.]+[a-zA-Zµnms]+", + "aggregation_time=", + ); + settings.add_filter( + r"emitting_time=[\d.]+[a-zA-Zµnms]+", + "emitting_time=", + ); + settings.add_filter( + r"time_calculating_group_ids=[\d.]+[a-zA-Zµnms]+", + "time_calculating_group_ids=", + ); settings.add_filter( r"files_ranges_pruned_statistics=\d+", "files_ranges_pruned_statistics=", @@ -151,6 +167,7 @@ pub fn settings() -> insta::Settings { settings.add_filter(r"build_mem_used=\d+", "build_mem_used="); settings.add_filter(r"build_time=[\d.]+[a-zA-Zµnms]+", "build_time="); settings.add_filter(r"join_time=[\d.]+[a-zA-Zµnms]+", "join_time="); + settings.add_filter(r"total → \d+ matched", "total → X matched"); settings } diff --git a/src/test_utils/session_context.rs b/src/test_utils/session_context.rs index 276010d2..804d30bb 100644 --- a/src/test_utils/session_context.rs +++ b/src/test_utils/session_context.rs @@ -32,7 +32,7 @@ pub async fn register_temp_parquet_table( let temp_dir = std::env::temp_dir(); let file_id = Uuid::new_v4(); - let temp_file_path = temp_dir.join(format!("{}_{}.parquet", table_name, file_id,)); + let temp_file_path = temp_dir.join(format!("{table_name}_{file_id}.parquet",)); let file = std::fs::File::create(&temp_file_path)?; let schema = batches[0].schema(); diff --git a/src/test_utils/tpch.rs b/src/test_utils/tpch.rs index bef375ff..0d63f0e4 100644 --- a/src/test_utils/tpch.rs +++ b/src/test_utils/tpch.rs @@ -21,7 +21,7 @@ use tpchgen_arrow::{ pub fn tpch_query_from_dir(queries_dir: &std::path::Path, num: u8) -> String { let query_path = queries_dir.join(format!("q{num}.sql")); fs::read_to_string(query_path) - .unwrap_or_else(|_| panic!("Failed to read TPCH query file: q{}.sql", num)) + .unwrap_or_else(|_| panic!("Failed to read TPCH query file: q{num}.sql")) .trim() .to_string() } @@ -137,7 +137,7 @@ fn generate_table( where A: Iterator, { - let output_path = data_dir.join(format!("{}.parquet", table_name)); + let output_path = data_dir.join(format!("{table_name}.parquet")); if let Some(first_batch) = data_source.next() { let file = fs::File::create(&output_path)?; diff --git a/tests/custom_config_extension.rs b/tests/custom_config_extension.rs index ecd98fd5..2ffa0c9d 100644 --- a/tests/custom_config_extension.rs +++ b/tests/custom_config_extension.rs @@ -5,7 +5,7 @@ mod tests { use datafusion::config::ConfigExtension; use datafusion::error::DataFusionError; use datafusion::execution::{ - FunctionRegistry, SendableRecordBatchStream, SessionState, SessionStateBuilder, TaskContext, + SendableRecordBatchStream, SessionState, SessionStateBuilder, TaskContext, }; use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; @@ -162,7 +162,7 @@ mod tests { &self, _buf: &[u8], _: &[Arc], - _registry: &dyn FunctionRegistry, + _ctx: &TaskContext, ) -> datafusion::common::Result> { Ok(Arc::new(CustomConfigExtensionRequiredExec::new())) } diff --git a/tests/custom_extension_codec.rs b/tests/custom_extension_codec.rs index 546bed4c..547613c1 100644 --- a/tests/custom_extension_codec.rs +++ b/tests/custom_extension_codec.rs @@ -7,7 +7,7 @@ mod tests { use datafusion::arrow::util::pretty::pretty_format_batches; use datafusion::error::DataFusionError; use datafusion::execution::{ - FunctionRegistry, SendableRecordBatchStream, SessionState, SessionStateBuilder, TaskContext, + SendableRecordBatchStream, SessionState, SessionStateBuilder, TaskContext, }; use datafusion::logical_expr::Operator; use datafusion::physical_expr::expressions::{BinaryExpr, col, lit}; @@ -253,7 +253,7 @@ mod tests { &self, buf: &[u8], _: &[Arc], - _registry: &dyn FunctionRegistry, + _ctx: &TaskContext, ) -> datafusion::common::Result> { let node = Int64ListExecProto::decode(buf).map_err(|err| proto_error(format!("{err}")))?; diff --git a/tests/distributed_aggregation.rs b/tests/distributed_aggregation.rs index eea4f3a0..4869bc51 100644 --- a/tests/distributed_aggregation.rs +++ b/tests/distributed_aggregation.rs @@ -236,8 +236,8 @@ mod tests { +----------+--------+-----+"; // Print them out, the error message from `assert_eq` is otherwise hard to read. - println!("{}", expected_result); - println!("{}", actual_result); + println!("{expected_result}"); + println!("{actual_result}"); // Compare against result. The regression this is testing for would have NULL values in // the second and third column. diff --git a/tests/error_propagation.rs b/tests/error_propagation.rs index 04b44c41..241b5c6e 100644 --- a/tests/error_propagation.rs +++ b/tests/error_propagation.rs @@ -3,7 +3,7 @@ mod tests { use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::error::DataFusionError; use datafusion::execution::{ - FunctionRegistry, SendableRecordBatchStream, SessionState, SessionStateBuilder, TaskContext, + SendableRecordBatchStream, SessionState, SessionStateBuilder, TaskContext, }; use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; @@ -141,7 +141,7 @@ mod tests { &self, buf: &[u8], _: &[Arc], - _registry: &dyn FunctionRegistry, + _ctx: &TaskContext, ) -> datafusion::common::Result> { let node = ErrorExecProto::decode(buf).map_err(|err| proto_error(format!("{err}")))?; Ok(Arc::new(ErrorExec::new(&node.msg))) diff --git a/tests/stateful_execution_plan.rs b/tests/stateful_execution_plan.rs index 40d3fea2..13284030 100644 --- a/tests/stateful_execution_plan.rs +++ b/tests/stateful_execution_plan.rs @@ -8,7 +8,7 @@ mod tests { use datafusion::common::runtime::SpawnedTask; use datafusion::error::DataFusionError; use datafusion::execution::{ - FunctionRegistry, SendableRecordBatchStream, SessionState, SessionStateBuilder, TaskContext, + SendableRecordBatchStream, SessionState, SessionStateBuilder, TaskContext, }; use datafusion::logical_expr::Operator; use datafusion::physical_expr::expressions::{BinaryExpr, col, lit}; @@ -261,7 +261,7 @@ mod tests { &self, buf: &[u8], _: &[Arc], - _registry: &dyn FunctionRegistry, + _ctx: &TaskContext, ) -> datafusion::common::Result> { let node = Int64ListExecProto::decode(buf).map_err(|err| proto_error(format!("{err}")))?; diff --git a/tests/tpch_explain_analyze.rs b/tests/tpch_explain_analyze.rs index eb47b57d..6395eb37 100644 --- a/tests/tpch_explain_analyze.rs +++ b/tests/tpch_explain_analyze.rs @@ -21,24 +21,24 @@ mod tests { let plan = test_tpch_query(1).await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST], metrics=[output_rows=4, elapsed_compute=] + │ SortPreservingMergeExec: [l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST], metrics=[output_rows=4, elapsed_compute=, output_bytes=608.0 B] │ [Stage 2] => NetworkCoalesceExec: output_partitions=12, input_tasks=2, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] │ SortExec: expr=[l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=4, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] │ ProjectionExec: expr=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus, sum(lineitem.l_quantity)@2 as sum_qty, sum(lineitem.l_extendedprice)@3 as sum_base_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@4 as sum_disc_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax)@5 as sum_charge, avg(lineitem.l_quantity)@6 as avg_qty, avg(lineitem.l_extendedprice)@7 as avg_price, avg(lineitem.l_discount)@8 as avg_disc, count(Int64(1))@9 as count_order], metrics=[output_rows=4, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))], metrics=[output_rows=4, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] + │ AggregateExec: mode=FinalPartitioned, gby=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))], metrics=[output_rows=4, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=24, elapsed_compute=] │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p11] t1:[p0..p11] t2:[p0..p11] t3:[p0..p11] - │ RepartitionExec: partitioning=Hash([l_returnflag@0, l_linestatus@1], 12), input_partitions=2, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[l_returnflag@5 as l_returnflag, l_linestatus@6 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))], metrics=[output_rows=24, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([l_returnflag@0, l_linestatus@1], 12), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[l_returnflag@5 as l_returnflag, l_linestatus@6 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))], metrics=[output_rows=24, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ ProjectionExec: expr=[l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as __common_expr_1, l_quantity@0 as l_quantity, l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, l_tax@3 as l_tax, l_returnflag@4 as l_returnflag, l_linestatus@5 as l_linestatus], metrics=[output_rows=591856, elapsed_compute=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=591856, elapsed_compute=] │ FilterExec: l_shipdate@6 <= 1998-09-02, projection=[l_quantity@0, l_extendedprice@1, l_discount@2, l_tax@3, l_returnflag@4, l_linestatus@5], metrics=[output_rows=591856, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate], file_type=parquet, predicate=l_shipdate@6 <= 1998-09-02, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@0 <= 1998-09-02, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate], file_type=parquet, predicate=l_shipdate@6 <= 1998-09-02, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@0 <= 1998-09-02, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -49,7 +49,7 @@ mod tests { let plan = test_tpch_query(2).await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], metrics=[output_rows=11264, elapsed_compute=] + │ SortPreservingMergeExec: [s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], metrics=[output_rows=11264, elapsed_compute=, output_bytes=5.3 MB] │ [Stage 7] => NetworkCoalesceExec: output_partitions=24, input_tasks=4, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] @@ -63,7 +63,7 @@ mod tests { │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=3, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p23] - │ RepartitionExec: partitioning=Hash([p_partkey@0, ps_supplycost@7], 24), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([p_partkey@0, ps_supplycost@7], 24), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16128, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@9)], projection=[p_partkey@1, p_mfgr@2, s_name@3, s_address@4, s_phone@5, s_acctbal@6, s_comment@7, ps_supplycost@8, n_name@9], metrics=[output_rows=16128, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=] @@ -72,40 +72,40 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=4672, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@4)], projection=[n_name@1, n_regionkey@2, p_partkey@3, p_mfgr@4, s_name@5, s_address@6, s_phone@8, s_acctbal@9, s_comment@10, ps_supplycost@11], metrics=[output_rows=4672, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] │ ProjectionExec: expr=[p_partkey@6 as p_partkey, p_mfgr@7 as p_mfgr, s_name@0 as s_name, s_address@1 as s_address, s_nationkey@2 as s_nationkey, s_phone@3 as s_phone, s_acctbal@4 as s_acctbal, s_comment@5 as s_comment, ps_supplycost@8 as ps_supplycost], metrics=[output_rows=292, elapsed_compute=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=292, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@2)], projection=[s_name@1, s_address@2, s_nationkey@3, s_phone@4, s_acctbal@5, s_comment@6, p_partkey@7, p_mfgr@8, ps_supplycost@10], metrics=[output_rows=292, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment], file_type=parquet, metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=292, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, ps_partkey@0)], projection=[p_partkey@0, p_mfgr@1, ps_suppkey@3, ps_supplycost@4], metrics=[output_rows=292, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=73, elapsed_compute=] │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ], metrics=[output_rows=80000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], metrics=[output_rows=80000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] │ FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0], metrics=[output_rows=16, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/region/1.parquet, /testdata/tpch/explain_analyze_sf0.1/region/10.parquet, /testdata/tpch/explain_analyze_sf0.1/region/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/12.parquet, /testdata/tpch/explain_analyze_sf0.1/region/13.parquet, /testdata/tpch/explain_analyze_sf0.1/region/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/15.parquet, /testdata/tpch/explain_analyze_sf0.1/region/16.parquet, /testdata/tpch/explain_analyze_sf0.1/region/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/3.parquet, /testdata/tpch/explain_analyze_sf0.1/region/4.parquet, /testdata/tpch/explain_analyze_sf0.1/region/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/6.parquet, /testdata/tpch/explain_analyze_sf0.1/region/7.parquet, /testdata/tpch/explain_analyze_sf0.1/region/8.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = EUROPE, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= EUROPE AND EUROPE <= r_name_max@1, required_guarantees=[r_name in (EUROPE)], metrics=[output_rows=80, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/region/1.parquet, /testdata/tpch/explain_analyze_sf0.1/region/10.parquet, /testdata/tpch/explain_analyze_sf0.1/region/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/12.parquet, /testdata/tpch/explain_analyze_sf0.1/region/13.parquet, /testdata/tpch/explain_analyze_sf0.1/region/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/15.parquet, /testdata/tpch/explain_analyze_sf0.1/region/16.parquet, /testdata/tpch/explain_analyze_sf0.1/region/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/3.parquet, /testdata/tpch/explain_analyze_sf0.1/region/4.parquet, /testdata/tpch/explain_analyze_sf0.1/region/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/6.parquet, /testdata/tpch/explain_analyze_sf0.1/region/7.parquet, /testdata/tpch/explain_analyze_sf0.1/region/8.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = EUROPE, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= EUROPE AND EUROPE <= r_name_max@1, required_guarantees=[r_name in (EUROPE)], metrics=[output_rows=80, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=73, elapsed_compute=] │ FilterExec: p_size@3 = 15 AND p_type@2 LIKE %BRASS, projection=[p_partkey@0, p_mfgr@1], metrics=[output_rows=73, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_mfgr, p_type, p_size], file_type=parquet, predicate=p_size@3 = 15 AND p_type@2 LIKE %BRASS, pruning_predicate=p_size_null_count@2 != row_count@3 AND p_size_min@0 <= 15 AND 15 <= p_size_max@1, required_guarantees=[p_size in (15)], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_mfgr, p_type, p_size], file_type=parquet, predicate=p_size@3 = 15 AND p_type@2 LIKE %BRASS, pruning_predicate=p_size_null_count@2 != row_count@3 AND p_size_min@0 <= 15 AND 15 <= p_size_max@1, required_guarantees=[p_size in (15)], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 6 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] - │ RepartitionExec: partitioning=Hash([ps_partkey@1, min(partsupp.ps_supplycost)@0], 24), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([ps_partkey@1, min(partsupp.ps_supplycost)@0], 24), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ ProjectionExec: expr=[min(partsupp.ps_supplycost)@1 as min(partsupp.ps_supplycost), ps_partkey@0 as ps_partkey], metrics=[output_rows=11985, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)], metrics=[output_rows=11985, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] + │ AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)], metrics=[output_rows=11985, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=11985, elapsed_compute=] │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 5 ── Tasks: t0:[p0..p17] - │ RepartitionExec: partitioning=Hash([ps_partkey@0], 18), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)], metrics=[output_rows=11985, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([ps_partkey@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)], metrics=[output_rows=11985, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=4177920, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@2)], projection=[ps_partkey@1, ps_supplycost@2], metrics=[output_rows=4177920, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=] @@ -114,19 +114,19 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1280000, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_regionkey@1, ps_partkey@2, ps_supplycost@3], metrics=[output_rows=1280000, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_regionkey], file_type=parquet, metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_supplycost@2 as ps_supplycost, s_nationkey@0 as s_nationkey], metrics=[output_rows=80000, elapsed_compute=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=80000, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@1)], projection=[s_nationkey@1, ps_partkey@2, ps_supplycost@4], metrics=[output_rows=80000, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ], metrics=[output_rows=80000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=80000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 4 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] │ FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0], metrics=[output_rows=16, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/region/1.parquet, /testdata/tpch/explain_analyze_sf0.1/region/10.parquet, /testdata/tpch/explain_analyze_sf0.1/region/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/12.parquet, /testdata/tpch/explain_analyze_sf0.1/region/13.parquet, /testdata/tpch/explain_analyze_sf0.1/region/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/15.parquet, /testdata/tpch/explain_analyze_sf0.1/region/16.parquet, /testdata/tpch/explain_analyze_sf0.1/region/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/3.parquet, /testdata/tpch/explain_analyze_sf0.1/region/4.parquet, /testdata/tpch/explain_analyze_sf0.1/region/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/6.parquet, /testdata/tpch/explain_analyze_sf0.1/region/7.parquet, /testdata/tpch/explain_analyze_sf0.1/region/8.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = EUROPE, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= EUROPE AND EUROPE <= r_name_max@1, required_guarantees=[r_name in (EUROPE)], metrics=[output_rows=80, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/region/1.parquet, /testdata/tpch/explain_analyze_sf0.1/region/10.parquet, /testdata/tpch/explain_analyze_sf0.1/region/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/12.parquet, /testdata/tpch/explain_analyze_sf0.1/region/13.parquet, /testdata/tpch/explain_analyze_sf0.1/region/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/15.parquet, /testdata/tpch/explain_analyze_sf0.1/region/16.parquet, /testdata/tpch/explain_analyze_sf0.1/region/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/3.parquet, /testdata/tpch/explain_analyze_sf0.1/region/4.parquet, /testdata/tpch/explain_analyze_sf0.1/region/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/6.parquet, /testdata/tpch/explain_analyze_sf0.1/region/7.parquet, /testdata/tpch/explain_analyze_sf0.1/region/8.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = EUROPE, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= EUROPE AND EUROPE <= r_name_max@1, required_guarantees=[r_name in (EUROPE)], metrics=[output_rows=80, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -137,19 +137,19 @@ mod tests { let plan = test_tpch_query(3).await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [revenue@1 DESC, o_orderdate@2 ASC NULLS LAST], metrics=[output_rows=1216, elapsed_compute=] + │ SortPreservingMergeExec: [revenue@1 DESC, o_orderdate@2 ASC NULLS LAST], metrics=[output_rows=1216, elapsed_compute=, output_bytes=38.0 KB] │ [Stage 3] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ SortExec: expr=[revenue@1 DESC, o_orderdate@2 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=1216, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] │ ProjectionExec: expr=[l_orderkey@0 as l_orderkey, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 as revenue, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority], metrics=[output_rows=1216, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=1216, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] + │ AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=1216, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1216, elapsed_compute=] │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p17] - │ RepartitionExec: partitioning=Hash([l_orderkey@0, o_orderdate@1, o_shippriority@2], 18), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[l_orderkey@2 as l_orderkey, o_orderdate@0 as o_orderdate, o_shippriority@1 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=1216, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([l_orderkey@0, o_orderdate@1, o_shippriority@2], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[l_orderkey@2 as l_orderkey, o_orderdate@0 as o_orderdate, o_shippriority@1 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=1216, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=3321, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, o_shippriority@2, l_orderkey@3, l_extendedprice@4, l_discount@5], metrics=[output_rows=3321, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=15224, elapsed_compute=] @@ -159,16 +159,16 @@ mod tests { │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=72678, elapsed_compute=] │ FilterExec: o_orderdate@2 < 1995-03-15, metrics=[output_rows=72678, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_orderdate, o_shippriority], file_type=parquet, predicate=o_orderdate@2 < 1995-03-15 AND DynamicFilterPhysicalExpr [ true ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@0 < 1995-03-15, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_orderdate, o_shippriority], file_type=parquet, predicate=o_orderdate@2 < 1995-03-15 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@0 < 1995-03-15, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=324322, elapsed_compute=] │ FilterExec: l_shipdate@3 > 1995-03-15, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2], metrics=[output_rows=324322, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 > 1995-03-15 AND DynamicFilterPhysicalExpr [ true ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 > 1995-03-15, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 > 1995-03-15 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 > 1995-03-15, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=3111, elapsed_compute=] │ FilterExec: c_mktsegment@1 = BUILDING, projection=[c_custkey@0], metrics=[output_rows=3111, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_mktsegment], file_type=parquet, predicate=c_mktsegment@1 = BUILDING, pruning_predicate=c_mktsegment_null_count@2 != row_count@3 AND c_mktsegment_min@0 <= BUILDING AND BUILDING <= c_mktsegment_max@1, required_guarantees=[c_mktsegment in (BUILDING)], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_mktsegment], file_type=parquet, predicate=c_mktsegment@1 = BUILDING, pruning_predicate=c_mktsegment_null_count@2 != row_count@3 AND c_mktsegment_min@0 <= BUILDING AND BUILDING <= c_mktsegment_max@1, required_guarantees=[c_mktsegment in (BUILDING)], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -179,32 +179,32 @@ mod tests { let plan = test_tpch_query(4).await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [o_orderpriority@0 ASC NULLS LAST], metrics=[output_rows=5, elapsed_compute=] + │ SortPreservingMergeExec: [o_orderpriority@0 ASC NULLS LAST], metrics=[output_rows=5, elapsed_compute=, output_bytes=440.0 B] │ [Stage 3] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ SortExec: expr=[o_orderpriority@0 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=5, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] │ ProjectionExec: expr=[o_orderpriority@0 as o_orderpriority, count(Int64(1))@1 as order_count], metrics=[output_rows=5, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))], metrics=[output_rows=5, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] + │ AggregateExec: mode=FinalPartitioned, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))], metrics=[output_rows=5, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=30, elapsed_compute=] │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p17] - │ RepartitionExec: partitioning=Hash([o_orderpriority@0], 18), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))], metrics=[output_rows=30, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([o_orderpriority@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))], metrics=[output_rows=30, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5093, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(l_orderkey@0, o_orderkey@0)], projection=[o_orderpriority@1], metrics=[output_rows=5093, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=379809, elapsed_compute=] │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5552, elapsed_compute=] │ FilterExec: o_orderdate@1 >= 1993-07-01 AND o_orderdate@1 < 1993-10-01, projection=[o_orderkey@0, o_orderpriority@2], metrics=[output_rows=5552, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_orderdate, o_orderpriority], file_type=parquet, predicate=o_orderdate@1 >= 1993-07-01 AND o_orderdate@1 < 1993-10-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-07-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1993-10-01, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_orderdate, o_orderpriority], file_type=parquet, predicate=o_orderdate@1 >= 1993-07-01 AND o_orderdate@1 < 1993-10-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-07-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1993-10-01, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=379809, elapsed_compute=] │ FilterExec: l_receiptdate@2 > l_commitdate@1, projection=[l_orderkey@0], metrics=[output_rows=379809, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@2 > l_commitdate@1, metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@2 > l_commitdate@1, metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -215,19 +215,19 @@ mod tests { let plan = test_tpch_query(5).await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [revenue@1 DESC], metrics=[output_rows=5, elapsed_compute=] + │ SortPreservingMergeExec: [revenue@1 DESC], metrics=[output_rows=5, elapsed_compute=, output_bytes=160.0 B] │ [Stage 4] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ SortExec: expr=[revenue@1 DESC], preserve_partitioning=[true], metrics=[output_rows=5, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] │ ProjectionExec: expr=[n_name@0 as n_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as revenue], metrics=[output_rows=5, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[n_name@0 as n_name], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=5, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] + │ AggregateExec: mode=FinalPartitioned, gby=[n_name@0 as n_name], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=5, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=30, elapsed_compute=] │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p17] - │ RepartitionExec: partitioning=Hash([n_name@0], 18), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[n_name@2 as n_name], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=30, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([n_name@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[n_name@2 as n_name], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=30, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=221440, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@3)], projection=[l_extendedprice@1, l_discount@2, n_name@3], metrics=[output_rows=221440, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=] @@ -236,12 +236,12 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=59040, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_name@1, n_regionkey@2, l_extendedprice@3, l_discount@4], metrics=[output_rows=59040, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, s_nationkey@0 as s_nationkey], metrics=[output_rows=3690, elapsed_compute=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=3690, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1), (s_nationkey@1, c_nationkey@0)], projection=[s_nationkey@1, l_extendedprice@4, l_discount@5], metrics=[output_rows=3690, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=92293, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@1, l_orderkey@0)], projection=[c_nationkey@0, l_suppkey@3, l_extendedprice@4, l_discount@5], metrics=[output_rows=92293, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=22958, elapsed_compute=] @@ -250,20 +250,20 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_custkey@1, c_custkey@0)], projection=[o_orderkey@0, c_nationkey@3], metrics=[output_rows=22958, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=22958, elapsed_compute=] │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] │ FilterExec: r_name@1 = ASIA, projection=[r_regionkey@0], metrics=[output_rows=16, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/region/1.parquet, /testdata/tpch/explain_analyze_sf0.1/region/10.parquet, /testdata/tpch/explain_analyze_sf0.1/region/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/12.parquet, /testdata/tpch/explain_analyze_sf0.1/region/13.parquet, /testdata/tpch/explain_analyze_sf0.1/region/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/15.parquet, /testdata/tpch/explain_analyze_sf0.1/region/16.parquet, /testdata/tpch/explain_analyze_sf0.1/region/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/3.parquet, /testdata/tpch/explain_analyze_sf0.1/region/4.parquet, /testdata/tpch/explain_analyze_sf0.1/region/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/6.parquet, /testdata/tpch/explain_analyze_sf0.1/region/7.parquet, /testdata/tpch/explain_analyze_sf0.1/region/8.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = ASIA, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= ASIA AND ASIA <= r_name_max@1, required_guarantees=[r_name in (ASIA)], metrics=[output_rows=80, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/region/1.parquet, /testdata/tpch/explain_analyze_sf0.1/region/10.parquet, /testdata/tpch/explain_analyze_sf0.1/region/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/12.parquet, /testdata/tpch/explain_analyze_sf0.1/region/13.parquet, /testdata/tpch/explain_analyze_sf0.1/region/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/15.parquet, /testdata/tpch/explain_analyze_sf0.1/region/16.parquet, /testdata/tpch/explain_analyze_sf0.1/region/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/3.parquet, /testdata/tpch/explain_analyze_sf0.1/region/4.parquet, /testdata/tpch/explain_analyze_sf0.1/region/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/6.parquet, /testdata/tpch/explain_analyze_sf0.1/region/7.parquet, /testdata/tpch/explain_analyze_sf0.1/region/8.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = ASIA, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= ASIA AND ASIA <= r_name_max@1, required_guarantees=[r_name in (ASIA)], metrics=[output_rows=80, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=22958, elapsed_compute=] │ FilterExec: o_orderdate@2 >= 1994-01-01 AND o_orderdate@2 < 1995-01-01, projection=[o_orderkey@0, o_custkey@1], metrics=[output_rows=22958, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1994-01-01 AND o_orderdate@2 < 1995-01-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1994-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1995-01-01, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1994-01-01 AND o_orderdate@2 < 1995-01-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1994-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1995-01-01, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -274,9 +274,9 @@ mod tests { let plan = test_tpch_query(6).await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ ProjectionExec: expr=[sum(lineitem.l_extendedprice * lineitem.l_discount)@0 as revenue], metrics=[output_rows=1, elapsed_compute=] - │ AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice * lineitem.l_discount)], metrics=[output_rows=1, elapsed_compute=] - │ CoalescePartitionsExec, metrics=[output_rows=8, elapsed_compute=] + │ ProjectionExec: expr=[sum(lineitem.l_extendedprice * lineitem.l_discount)@0 as revenue], metrics=[output_rows=1, elapsed_compute=, output_bytes=16.0 B] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice * lineitem.l_discount)], metrics=[output_rows=1, elapsed_compute=, output_bytes=16.0 B] + │ CoalescePartitionsExec, metrics=[output_rows=8, elapsed_compute=, output_bytes=1024.0 B] │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] @@ -284,7 +284,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=11618, elapsed_compute=] │ FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND l_discount@2 >= Some(5),15,2 AND l_discount@2 <= Some(7),15,2 AND l_quantity@0 < Some(2400),15,2, projection=[l_extendedprice@1, l_discount@2], metrics=[output_rows=11618, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_quantity, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND l_discount@2 >= Some(5),15,2 AND l_discount@2 <= Some(7),15,2 AND l_quantity@0 < Some(2400),15,2, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01 AND l_discount_null_count@5 != row_count@2 AND l_discount_max@4 >= Some(5),15,2 AND l_discount_null_count@5 != row_count@2 AND l_discount_min@6 <= Some(7),15,2 AND l_quantity_null_count@8 != row_count@2 AND l_quantity_min@7 < Some(2400),15,2, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_quantity, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND l_discount@2 >= Some(5),15,2 AND l_discount@2 <= Some(7),15,2 AND l_quantity@0 < Some(2400),15,2, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01 AND l_discount_null_count@5 != row_count@2 AND l_discount_max@4 >= Some(5),15,2 AND l_discount_null_count@5 != row_count@2 AND l_discount_min@6 <= Some(7),15,2 AND l_quantity_null_count@8 != row_count@2 AND l_quantity_min@7 < Some(2400),15,2, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -295,19 +295,19 @@ mod tests { let plan = test_tpch_query(7).await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST], metrics=[output_rows=4, elapsed_compute=] + │ SortPreservingMergeExec: [supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST], metrics=[output_rows=4, elapsed_compute=, output_bytes=208.0 B] │ [Stage 8] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ SortExec: expr=[supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=4, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] │ ProjectionExec: expr=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year, sum(shipping.volume)@3 as revenue], metrics=[output_rows=4, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)], metrics=[output_rows=4, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] + │ AggregateExec: mode=FinalPartitioned, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)], metrics=[output_rows=4, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=24, elapsed_compute=] │ [Stage 7] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 7 ── Tasks: t0:[p0..p17] - │ RepartitionExec: partitioning=Hash([supp_nation@0, cust_nation@1, l_year@2], 18), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)], metrics=[output_rows=24, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([supp_nation@0, cust_nation@1, l_year@2], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)], metrics=[output_rows=24, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ ProjectionExec: expr=[n_name@4 as supp_nation, n_name@0 as cust_nation, date_part(YEAR, l_shipdate@3) as l_year, l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as volume], metrics=[output_rows=164608, elapsed_compute=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=164608, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@3)], filter=n_name@0 = FRANCE AND n_name@1 = GERMANY OR n_name@0 = GERMANY AND n_name@1 = FRANCE, projection=[n_name@1, l_extendedprice@2, l_discount@3, l_shipdate@4, n_name@6], metrics=[output_rows=164608, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] @@ -330,21 +330,21 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32, elapsed_compute=] │ FilterExec: n_name@1 = GERMANY OR n_name@1 = FRANCE, metrics=[output_rows=32, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY OR n_name@1 = FRANCE, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1 OR n_name_null_count@2 != row_count@3 AND n_name_min@0 <= FRANCE AND FRANCE <= n_name_max@1, required_guarantees=[n_name in (FRANCE, GERMANY)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY OR n_name@1 = FRANCE, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1 OR n_name_null_count@2 != row_count@3 AND n_name_min@0 <= FRANCE AND FRANCE <= n_name_max@1, required_guarantees=[n_name in (FRANCE, GERMANY)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32, elapsed_compute=] │ FilterExec: n_name@1 = FRANCE OR n_name@1 = GERMANY, metrics=[output_rows=32, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = FRANCE OR n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= FRANCE AND FRANCE <= n_name_max@1 OR n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (FRANCE, GERMANY)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = FRANCE OR n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= FRANCE AND FRANCE <= n_name_max@1 OR n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (FRANCE, GERMANY)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ RepartitionExec: partitioning=Hash([c_custkey@0], 6), input_partitions=2, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([c_custkey@0], 6), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ RepartitionExec: partitioning=Hash([o_custkey@4], 6), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([o_custkey@4], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=182762, elapsed_compute=] │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_orderkey@1, o_orderkey@0)], projection=[s_nationkey@0, l_extendedprice@2, l_discount@3, l_shipdate@4, o_custkey@6], metrics=[output_rows=182762, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=182762, elapsed_compute=] @@ -353,19 +353,19 @@ mod tests { │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 4 ── Tasks: t0:[p0..p23] - │ RepartitionExec: partitioning=Hash([l_orderkey@1], 24), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([l_orderkey@1], 24), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=182762, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_nationkey@1, l_orderkey@2, l_extendedprice@4, l_discount@5, l_shipdate@6], metrics=[output_rows=182762, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=182762, elapsed_compute=] │ FilterExec: l_shipdate@4 >= 1995-01-01 AND l_shipdate@4 <= 1996-12-31, metrics=[output_rows=182762, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@4 >= 1995-01-01 AND l_shipdate@4 <= 1996-12-31 AND DynamicFilterPhysicalExpr [ true ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 <= 1996-12-31, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@4 >= 1995-01-01 AND l_shipdate@4 <= 1996-12-31 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 <= 1996-12-31, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 5 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] - │ RepartitionExec: partitioning=Hash([o_orderkey@0], 24), input_partitions=2, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([o_orderkey@0], 24), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -376,19 +376,19 @@ mod tests { let plan = test_tpch_query(8).await?; assert_snapshot!(plan, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [o_year@0 ASC NULLS LAST], metrics=[output_rows=2, elapsed_compute=] + │ SortPreservingMergeExec: [o_year@0 ASC NULLS LAST], metrics=[output_rows=2, elapsed_compute=, output_bytes=40.0 B] │ [Stage 8] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ SortExec: expr=[o_year@0 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=2, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] │ ProjectionExec: expr=[o_year@0 as o_year, sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END)@1 / sum(all_nations.volume)@2 as mkt_share], metrics=[output_rows=2, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)], metrics=[output_rows=2, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] + │ AggregateExec: mode=FinalPartitioned, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)], metrics=[output_rows=2, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=12, elapsed_compute=] │ [Stage 7] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 7 ── Tasks: t0:[p0..p17] - │ RepartitionExec: partitioning=Hash([o_year@0], 18), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)], metrics=[output_rows=12, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([o_year@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)], metrics=[output_rows=12, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ ProjectionExec: expr=[date_part(YEAR, o_orderdate@2) as o_year, l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as volume, n_name@3 as nation], metrics=[output_rows=1155072, elapsed_compute=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1155072, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@3)], projection=[l_extendedprice@1, l_discount@2, o_orderdate@3, n_name@5], metrics=[output_rows=1155072, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] @@ -398,12 +398,12 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=365824, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_name@1, l_extendedprice@2, l_discount@3, o_orderdate@5, n_regionkey@6], metrics=[output_rows=365824, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, s_nationkey@3 as s_nationkey, o_orderdate@4 as o_orderdate, n_regionkey@0 as n_regionkey], metrics=[output_rows=22864, elapsed_compute=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=22864, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@4)], projection=[n_regionkey@1, l_extendedprice@2, l_discount@3, s_nationkey@4, o_orderdate@5], metrics=[output_rows=22864, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_regionkey], file_type=parquet, metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1429, elapsed_compute=] │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_custkey@3, c_custkey@0)], projection=[l_extendedprice@0, l_discount@1, s_nationkey@2, o_orderdate@4, c_nationkey@6], metrics=[output_rows=1429, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1429, elapsed_compute=] @@ -415,10 +415,10 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] │ FilterExec: r_name@1 = AMERICA, projection=[r_regionkey@0], metrics=[output_rows=16, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/region/1.parquet, /testdata/tpch/explain_analyze_sf0.1/region/10.parquet, /testdata/tpch/explain_analyze_sf0.1/region/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/12.parquet, /testdata/tpch/explain_analyze_sf0.1/region/13.parquet, /testdata/tpch/explain_analyze_sf0.1/region/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/15.parquet, /testdata/tpch/explain_analyze_sf0.1/region/16.parquet, /testdata/tpch/explain_analyze_sf0.1/region/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/3.parquet, /testdata/tpch/explain_analyze_sf0.1/region/4.parquet, /testdata/tpch/explain_analyze_sf0.1/region/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/6.parquet, /testdata/tpch/explain_analyze_sf0.1/region/7.parquet, /testdata/tpch/explain_analyze_sf0.1/region/8.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = AMERICA, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= AMERICA AND AMERICA <= r_name_max@1, required_guarantees=[r_name in (AMERICA)], metrics=[output_rows=80, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/region/1.parquet, /testdata/tpch/explain_analyze_sf0.1/region/10.parquet, /testdata/tpch/explain_analyze_sf0.1/region/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/12.parquet, /testdata/tpch/explain_analyze_sf0.1/region/13.parquet, /testdata/tpch/explain_analyze_sf0.1/region/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/15.parquet, /testdata/tpch/explain_analyze_sf0.1/region/16.parquet, /testdata/tpch/explain_analyze_sf0.1/region/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/3.parquet, /testdata/tpch/explain_analyze_sf0.1/region/4.parquet, /testdata/tpch/explain_analyze_sf0.1/region/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/6.parquet, /testdata/tpch/explain_analyze_sf0.1/region/7.parquet, /testdata/tpch/explain_analyze_sf0.1/region/8.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = AMERICA, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= AMERICA AND AMERICA <= r_name_max@1, required_guarantees=[r_name in (AMERICA)], metrics=[output_rows=80, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ RepartitionExec: partitioning=Hash([o_custkey@3], 6), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([o_custkey@3], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@4 as s_nationkey, o_custkey@0 as o_custkey, o_orderdate@1 as o_orderdate], metrics=[output_rows=1429, elapsed_compute=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1429, elapsed_compute=] │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_custkey@1, o_orderdate@2, l_extendedprice@4, l_discount@5, s_nationkey@6], metrics=[output_rows=1429, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] @@ -428,35 +428,35 @@ mod tests { │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] - │ RepartitionExec: partitioning=Hash([o_orderkey@0], 18), input_partitions=2, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([o_orderkey@0], 18), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=45624, elapsed_compute=] │ FilterExec: o_orderdate@2 >= 1995-01-01 AND o_orderdate@2 <= 1996-12-31, metrics=[output_rows=45624, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1995-01-01 AND o_orderdate@2 <= 1996-12-31, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1995-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 <= 1996-12-31, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1995-01-01 AND o_orderdate@2 <= 1996-12-31, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1995-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 <= 1996-12-31, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 4 ── Tasks: t0:[p0..p17] - │ RepartitionExec: partitioning=Hash([l_orderkey@0], 18), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([l_orderkey@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@0 as s_nationkey], metrics=[output_rows=4485, elapsed_compute=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=4485, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_nationkey@1, l_orderkey@2, l_extendedprice@4, l_discount@5], metrics=[output_rows=4485, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=4485, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@1, l_suppkey@3, l_extendedprice@4, l_discount@5], metrics=[output_rows=4485, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=147, elapsed_compute=] │ [Stage 3] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_partkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_partkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=147, elapsed_compute=] │ FilterExec: p_type@1 = ECONOMY ANODIZED STEEL, projection=[p_partkey@0], metrics=[output_rows=147, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_type], file_type=parquet, predicate=p_type@1 = ECONOMY ANODIZED STEEL, pruning_predicate=p_type_null_count@2 != row_count@3 AND p_type_min@0 <= ECONOMY ANODIZED STEEL AND ECONOMY ANODIZED STEEL <= p_type_max@1, required_guarantees=[p_type in (ECONOMY ANODIZED STEEL)], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_type], file_type=parquet, predicate=p_type@1 = ECONOMY ANODIZED STEEL, pruning_predicate=p_type_null_count@2 != row_count@3 AND p_type_min@0 <= ECONOMY ANODIZED STEEL AND ECONOMY ANODIZED STEEL <= p_type_max@1, required_guarantees=[p_type in (ECONOMY ANODIZED STEEL)], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ RepartitionExec: partitioning=Hash([c_custkey@0], 6), input_partitions=2, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([c_custkey@0], 6), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "#); Ok(()) @@ -467,24 +467,24 @@ mod tests { let plan = test_tpch_query(9).await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [nation@0 ASC NULLS LAST, o_year@1 DESC], metrics=[output_rows=175, elapsed_compute=] + │ SortPreservingMergeExec: [nation@0 ASC NULLS LAST, o_year@1 DESC], metrics=[output_rows=175, elapsed_compute=, output_bytes=13.9 KB] │ [Stage 7] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ SortExec: expr=[nation@0 ASC NULLS LAST, o_year@1 DESC], preserve_partitioning=[true], metrics=[output_rows=175, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] │ ProjectionExec: expr=[nation@0 as nation, o_year@1 as o_year, sum(profit.amount)@2 as sum_profit], metrics=[output_rows=175, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)], metrics=[output_rows=175, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] + │ AggregateExec: mode=FinalPartitioned, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)], metrics=[output_rows=175, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1050, elapsed_compute=] │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 6 ── Tasks: t0:[p0..p17] - │ RepartitionExec: partitioning=Hash([nation@0, o_year@1], 18), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)], metrics=[output_rows=1050, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([nation@0, o_year@1], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)], metrics=[output_rows=1050, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ ProjectionExec: expr=[n_name@0 as nation, date_part(YEAR, o_orderdate@5) as o_year, l_extendedprice@2 * (Some(1),20,0 - l_discount@3) - ps_supplycost@4 * l_quantity@1 as amount], metrics=[output_rows=514560, elapsed_compute=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=514560, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[n_name@1, l_quantity@2, l_extendedprice@3, l_discount@4, ps_supplycost@6, o_orderdate@7], metrics=[output_rows=514560, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] │ ProjectionExec: expr=[l_quantity@1 as l_quantity, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@4 as s_nationkey, ps_supplycost@5 as ps_supplycost, o_orderdate@0 as o_orderdate], metrics=[output_rows=32160, elapsed_compute=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32160, elapsed_compute=] │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, l_quantity@3, l_extendedprice@4, l_discount@5, s_nationkey@6, ps_supplycost@7], metrics=[output_rows=32160, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] @@ -494,12 +494,12 @@ mod tests { │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ RepartitionExec: partitioning=Hash([o_orderkey@0], 6), input_partitions=2, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([o_orderkey@0], 6), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_orderdate], file_type=parquet, metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_orderdate], file_type=parquet, metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ RepartitionExec: partitioning=Hash([l_orderkey@0], 6), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([l_orderkey@0], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32160, elapsed_compute=] │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_suppkey@2, ps_suppkey@1), (l_partkey@1, ps_partkey@0)], projection=[l_orderkey@0, l_quantity@3, l_extendedprice@4, l_discount@5, s_nationkey@6, ps_supplycost@9], metrics=[output_rows=32160, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32160, elapsed_compute=] @@ -508,28 +508,28 @@ mod tests { │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p23] - │ RepartitionExec: partitioning=Hash([l_suppkey@2, l_partkey@1], 24), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([l_suppkey@2, l_partkey@1], 24), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_partkey@2 as l_partkey, l_suppkey@3 as l_suppkey, l_quantity@4 as l_quantity, l_extendedprice@5 as l_extendedprice, l_discount@6 as l_discount, s_nationkey@0 as s_nationkey], metrics=[output_rows=32160, elapsed_compute=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32160, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@2)], projection=[s_nationkey@1, l_orderkey@2, l_partkey@3, l_suppkey@4, l_quantity@5, l_extendedprice@6, l_discount@7], metrics=[output_rows=32160, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32160, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@1, l_partkey@2, l_suppkey@3, l_quantity@4, l_extendedprice@5, l_discount@6], metrics=[output_rows=32160, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=1075, elapsed_compute=] │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1075, elapsed_compute=] │ FilterExec: p_name@1 LIKE %green%, projection=[p_partkey@0], metrics=[output_rows=1075, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_name], file_type=parquet, predicate=p_name@1 LIKE %green%, metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_name], file_type=parquet, predicate=p_name@1 LIKE %green%, metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 4 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] - │ RepartitionExec: partitioning=Hash([ps_suppkey@1, ps_partkey@0], 24), input_partitions=2, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([ps_suppkey@1, ps_partkey@0], 24), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ], metrics=[output_rows=80000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=80000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -540,24 +540,24 @@ mod tests { let plan = test_tpch_query(10).await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [revenue@2 DESC], metrics=[output_rows=3767, elapsed_compute=] + │ SortPreservingMergeExec: [revenue@2 DESC], metrics=[output_rows=3767, elapsed_compute=, output_bytes=1382.4 KB] │ [Stage 3] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ SortExec: expr=[revenue@2 DESC], preserve_partitioning=[true], metrics=[output_rows=3767, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] │ ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@7 as revenue, c_acctbal@2 as c_acctbal, n_name@4 as n_name, c_address@5 as c_address, c_phone@3 as c_phone, c_comment@6 as c_comment], metrics=[output_rows=3767, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@2 as c_acctbal, c_phone@3 as c_phone, n_name@4 as n_name, c_address@5 as c_address, c_comment@6 as c_comment], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=3767, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] + │ AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@2 as c_acctbal, c_phone@3 as c_phone, n_name@4 as n_name, c_address@5 as c_address, c_comment@6 as c_comment], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=3767, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=4648, elapsed_compute=] │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p17] - │ RepartitionExec: partitioning=Hash([c_custkey@0, c_name@1, c_acctbal@2, c_phone@3, n_name@4, c_address@5, c_comment@6], 18), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@4 as c_acctbal, c_phone@3 as c_phone, n_name@8 as n_name, c_address@2 as c_address, c_comment@5 as c_comment], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=4648, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([c_custkey@0, c_name@1, c_acctbal@2, c_phone@3, n_name@4, c_address@5, c_comment@6], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@4 as c_acctbal, c_phone@3 as c_phone, n_name@8 as n_name, c_address@2 as c_address, c_comment@5 as c_comment], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=4648, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ ProjectionExec: expr=[c_custkey@1 as c_custkey, c_name@2 as c_name, c_address@3 as c_address, c_phone@4 as c_phone, c_acctbal@5 as c_acctbal, c_comment@6 as c_comment, l_extendedprice@7 as l_extendedprice, l_discount@8 as l_discount, n_name@0 as n_name], metrics=[output_rows=183024, elapsed_compute=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=183024, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@3)], projection=[n_name@1, c_custkey@2, c_name@3, c_address@4, c_phone@6, c_acctbal@7, c_comment@8, l_extendedprice@9, l_discount@10], metrics=[output_rows=183024, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=11439, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@7, l_orderkey@0)], projection=[c_custkey@0, c_name@1, c_address@2, c_nationkey@3, c_phone@4, c_acctbal@5, c_comment@6, l_extendedprice@9, l_discount@10], metrics=[output_rows=11439, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=5677, elapsed_compute=] @@ -566,16 +566,16 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_custkey@1, c_custkey@0)], projection=[o_orderkey@0, c_custkey@2, c_name@3, c_address@4, c_nationkey@5, c_phone@6, c_acctbal@7, c_comment@8], metrics=[output_rows=5677, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=5677, elapsed_compute=] │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_comment], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_comment], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=148301, elapsed_compute=] │ FilterExec: l_returnflag@3 = R, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2], metrics=[output_rows=148301, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_extendedprice, l_discount, l_returnflag], file_type=parquet, predicate=l_returnflag@3 = R AND DynamicFilterPhysicalExpr [ true ], pruning_predicate=l_returnflag_null_count@2 != row_count@3 AND l_returnflag_min@0 <= R AND R <= l_returnflag_max@1, required_guarantees=[l_returnflag in (R)], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_extendedprice, l_discount, l_returnflag], file_type=parquet, predicate=l_returnflag@3 = R AND DynamicFilter [ empty ], pruning_predicate=l_returnflag_null_count@2 != row_count@3 AND l_returnflag_min@0 <= R AND R <= l_returnflag_max@1, required_guarantees=[l_returnflag in (R)], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5677, elapsed_compute=] │ FilterExec: o_orderdate@2 >= 1993-10-01 AND o_orderdate@2 < 1994-01-01, projection=[o_orderkey@0, o_custkey@1], metrics=[output_rows=5677, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1993-10-01 AND o_orderdate@2 < 1994-01-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-10-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1994-01-01, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1993-10-01 AND o_orderdate@2 < 1994-01-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-10-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1994-01-01, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -586,50 +586,51 @@ mod tests { let plan = test_tpch_query(11).await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [value@1 DESC], metrics=[output_rows=2541, elapsed_compute=] - │ SortExec: expr=[value@1 DESC], preserve_partitioning=[true], metrics=[output_rows=2541, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] - │ ProjectionExec: expr=[ps_partkey@0 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 as value], metrics=[output_rows=2541, elapsed_compute=] - │ NestedLoopJoinExec: join_type=Inner, filter=CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@0 AS Decimal128(38, 15)) > sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)@1, projection=[ps_partkey@1, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@2], metrics=[output_rows=2541, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ ProjectionExec: expr=[CAST(CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@0 AS Float64) * 0.0001 AS Decimal128(38, 15)) as sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)], metrics=[output_rows=1, elapsed_compute=] - │ AggregateExec: mode=Final, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)], metrics=[output_rows=1, elapsed_compute=] - │ CoalescePartitionsExec, metrics=[output_rows=6, elapsed_compute=] - │ AggregateExec: mode=Partial, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)], metrics=[output_rows=6, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=64000, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[ps_availqty@1, ps_supplycost@2], metrics=[output_rows=64000, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=] + │ SortPreservingMergeExec: [value@1 DESC], metrics=[output_rows=2541, elapsed_compute=, output_bytes=59.6 KB] + │ SortExec: expr=[value@1 DESC], preserve_partitioning=[true], metrics=[output_rows=2541, elapsed_compute=, output_bytes=0.0 B, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] + │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@2 as value], metrics=[output_rows=2541, elapsed_compute=, output_bytes=1152.0 KB] + │ NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 > sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)@0, projection=[sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)@0, ps_partkey@1, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@2], metrics=[output_rows=2541, elapsed_compute=, output_bytes=1920.0 KB, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=, selectivity=68% (2541/3716)] + │ ProjectionExec: expr=[CAST(CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@0 AS Float64) * 0.0001 AS Decimal128(38, 15)) as sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)], metrics=[output_rows=1, elapsed_compute=, output_bytes=64.0 B] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)], metrics=[output_rows=1, elapsed_compute=, output_bytes=16.0 B] + │ CoalescePartitionsExec, metrics=[output_rows=6, elapsed_compute=, output_bytes=96.0 B] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)], metrics=[output_rows=6, elapsed_compute=, output_bytes=96.0 B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=64000, elapsed_compute=, output_bytes=2.5 MB] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[ps_availqty@1, ps_supplycost@2], metrics=[output_rows=64000, elapsed_compute=, output_bytes=1250.0 KB, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=, output_bytes=768.0 B] │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ ProjectionExec: expr=[ps_availqty@1 as ps_availqty, ps_supplycost@2 as ps_supplycost, s_nationkey@0 as s_nationkey], metrics=[output_rows=80000, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=80000, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@0)], projection=[s_nationkey@1, ps_availqty@3, ps_supplycost@4], metrics=[output_rows=80000, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ ps_suppkey@0 >= 1 AND ps_suppkey@0 <= 1000 OR ps_suppkey@0 >= 1 AND ps_suppkey@0 <= 1000 OR ps_suppkey@0 >= 1 AND ps_suppkey@0 <= 1000 OR ps_suppkey@0 >= 1 AND ps_suppkey@0 <= 1000 OR ps_suppkey@0 >= 1 AND ps_suppkey@0 <= 1000 OR ps_suppkey@0 >= 1 AND ps_suppkey@0 <= 1000 ], pruning_predicate=ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000 OR ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000 OR ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000 OR ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000 OR ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000 OR ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000, required_guarantees=[], metrics=[output_rows=80000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)], metrics=[output_rows=3716, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=3716, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([ps_partkey@0], 6), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)], metrics=[output_rows=3716, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=64000, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[ps_partkey@1, ps_availqty@2, ps_supplycost@3], metrics=[output_rows=64000, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_availqty@2 as ps_availqty, ps_supplycost@3 as ps_supplycost, s_nationkey@0 as s_nationkey], metrics=[output_rows=80000, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=80000, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@1)], projection=[s_nationkey@1, ps_partkey@2, ps_availqty@4, ps_supplycost@5], metrics=[output_rows=80000, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ ps_suppkey@1 >= 1 AND ps_suppkey@1 <= 1000 OR ps_suppkey@1 >= 1 AND ps_suppkey@1 <= 1000 OR ps_suppkey@1 >= 1 AND ps_suppkey@1 <= 1000 OR ps_suppkey@1 >= 1 AND ps_suppkey@1 <= 1000 OR ps_suppkey@1 >= 1 AND ps_suppkey@1 <= 1000 OR ps_suppkey@1 >= 1 AND ps_suppkey@1 <= 1000 ], pruning_predicate=ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000 OR ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000 OR ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000 OR ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000 OR ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000 OR ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000, required_guarantees=[], metrics=[output_rows=80000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ ProjectionExec: expr=[ps_availqty@1 as ps_availqty, ps_supplycost@2 as ps_supplycost, s_nationkey@0 as s_nationkey], metrics=[output_rows=80000, elapsed_compute=, output_bytes=2.1 MB] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=80000, elapsed_compute=, output_bytes=2.1 MB] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@0)], projection=[s_nationkey@1, ps_availqty@3, ps_supplycost@4], metrics=[output_rows=80000, elapsed_compute=, output_bytes=2.1 MB, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=, output_bytes=15.6 KB] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ s_nationkey@1 >= 7 AND s_nationkey@1 <= 7 ], pruning_predicate=s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 7 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 7, required_guarantees=[], metrics=[output_rows=1000, elapsed_compute=, output_bytes=15.6 KB, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ ps_suppkey@0 >= 1 AND ps_suppkey@0 <= 1000 ], pruning_predicate=ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000, required_guarantees=[], metrics=[output_rows=80000, elapsed_compute=, output_bytes=2.1 MB, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ ProjectionExec: expr=[ps_partkey@0 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 as sum(partsupp.ps_supplycost * partsupp.ps_availqty), CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 AS Decimal128(38, 15)) as join_proj_push_down_1], metrics=[output_rows=3716, elapsed_compute=, output_bytes=164.2 KB] + │ AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)], metrics=[output_rows=3716, elapsed_compute=, output_bytes=106.1 KB, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=3716, elapsed_compute=, output_bytes=2.2 MB] + │ RepartitionExec: partitioning=Hash([ps_partkey@0], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)], metrics=[output_rows=3716, elapsed_compute=, output_bytes=113.8 KB, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=, reduction_factor=5.8% (3716/64000)] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=64000, elapsed_compute=, output_bytes=3.5 MB] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[ps_partkey@1, ps_availqty@2, ps_supplycost@3], metrics=[output_rows=64000, elapsed_compute=, output_bytes=1750.0 KB, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=, output_bytes=768.0 B] + │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] + │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_availqty@2 as ps_availqty, ps_supplycost@3 as ps_supplycost, s_nationkey@0 as s_nationkey], metrics=[output_rows=80000, elapsed_compute=, output_bytes=2.7 MB] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=80000, elapsed_compute=, output_bytes=2.7 MB] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@1)], projection=[s_nationkey@1, ps_partkey@2, ps_availqty@4, ps_supplycost@5], metrics=[output_rows=80000, elapsed_compute=, output_bytes=2.7 MB, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=, output_bytes=15.6 KB] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ s_nationkey@1 >= 7 AND s_nationkey@1 <= 7 ], pruning_predicate=s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 7 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 7, required_guarantees=[], metrics=[output_rows=1000, elapsed_compute=, output_bytes=15.6 KB, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ ps_suppkey@1 >= 1 AND ps_suppkey@1 <= 1000 ], pruning_predicate=ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000, required_guarantees=[], metrics=[output_rows=80000, elapsed_compute=, output_bytes=2.7 MB, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] │ FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0], metrics=[output_rows=16, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (GERMANY)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (GERMANY)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] │ FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0], metrics=[output_rows=16, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (GERMANY)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (GERMANY)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -640,19 +641,19 @@ mod tests { let plan = test_tpch_query(12).await?; assert_snapshot!(plan, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [l_shipmode@0 ASC NULLS LAST], metrics=[output_rows=2, elapsed_compute=] + │ SortPreservingMergeExec: [l_shipmode@0 ASC NULLS LAST], metrics=[output_rows=2, elapsed_compute=, output_bytes=64.0 B] │ [Stage 4] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ SortExec: expr=[l_shipmode@0 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=2, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] │ ProjectionExec: expr=[l_shipmode@0 as l_shipmode, sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@1 as high_line_count, sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@2 as low_line_count], metrics=[output_rows=2, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)], metrics=[output_rows=2, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] + │ AggregateExec: mode=FinalPartitioned, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)], metrics=[output_rows=2, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=36, elapsed_compute=] │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=3, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] - │ RepartitionExec: partitioning=Hash([l_shipmode@0], 18), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)], metrics=[output_rows=36, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([l_shipmode@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)], metrics=[output_rows=36, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=3155, elapsed_compute=] │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_orderkey@0, o_orderkey@0)], projection=[l_shipmode@1, o_orderpriority@3], metrics=[output_rows=3155, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=3155, elapsed_compute=] @@ -661,16 +662,16 @@ mod tests { │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] - │ RepartitionExec: partitioning=Hash([l_orderkey@0], 18), input_partitions=2, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([l_orderkey@0], 18), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=3155, elapsed_compute=] │ FilterExec: (l_shipmode@4 = MAIL OR l_shipmode@4 = SHIP) AND l_receiptdate@3 > l_commitdate@2 AND l_shipdate@1 < l_commitdate@2 AND l_receiptdate@3 >= 1994-01-01 AND l_receiptdate@3 < 1995-01-01, projection=[l_orderkey@0, l_shipmode@4], metrics=[output_rows=3155, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_shipdate, l_commitdate, l_receiptdate, l_shipmode], file_type=parquet, predicate=(l_shipmode@4 = MAIL OR l_shipmode@4 = SHIP) AND l_receiptdate@3 > l_commitdate@2 AND l_shipdate@1 < l_commitdate@2 AND l_receiptdate@3 >= 1994-01-01 AND l_receiptdate@3 < 1995-01-01, pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= MAIL AND MAIL <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1) AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_max@4 >= 1994-01-01 AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_min@6 < 1995-01-01, required_guarantees=[l_shipmode in (MAIL, SHIP)], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_shipdate, l_commitdate, l_receiptdate, l_shipmode], file_type=parquet, predicate=(l_shipmode@4 = MAIL OR l_shipmode@4 = SHIP) AND l_receiptdate@3 > l_commitdate@2 AND l_shipdate@1 < l_commitdate@2 AND l_receiptdate@3 >= 1994-01-01 AND l_receiptdate@3 < 1995-01-01, pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= MAIL AND MAIL <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1) AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_max@4 >= 1994-01-01 AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_min@6 < 1995-01-01, required_guarantees=[l_shipmode in (MAIL, SHIP)], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] - │ RepartitionExec: partitioning=Hash([o_orderkey@0], 18), input_partitions=2, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([o_orderkey@0], 18), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_orderpriority], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_orderpriority], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "#); Ok(()) @@ -681,39 +682,39 @@ mod tests { let plan = test_tpch_query(13).await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [custdist@1 DESC, c_count@0 DESC], metrics=[output_rows=37, elapsed_compute=] + │ SortPreservingMergeExec: [custdist@1 DESC, c_count@0 DESC], metrics=[output_rows=37, elapsed_compute=, output_bytes=592.0 B] │ [Stage 4] => NetworkCoalesceExec: output_partitions=12, input_tasks=2, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] │ SortExec: expr=[custdist@1 DESC, c_count@0 DESC], preserve_partitioning=[true], metrics=[output_rows=37, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] │ ProjectionExec: expr=[c_count@0 as c_count, count(Int64(1))@1 as custdist], metrics=[output_rows=37, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[c_count@0 as c_count], aggr=[count(Int64(1))], metrics=[output_rows=37, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] + │ AggregateExec: mode=FinalPartitioned, gby=[c_count@0 as c_count], aggr=[count(Int64(1))], metrics=[output_rows=37, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=568, elapsed_compute=] │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=3, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p11] t1:[p0..p11] t2:[p0..p11] - │ RepartitionExec: partitioning=Hash([c_count@0], 12), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[c_count@0 as c_count], aggr=[count(Int64(1))], metrics=[output_rows=568, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([c_count@0], 12), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[c_count@0 as c_count], aggr=[count(Int64(1))], metrics=[output_rows=568, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ ProjectionExec: expr=[count(orders.o_orderkey)@1 as c_count], metrics=[output_rows=15000, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey], aggr=[count(orders.o_orderkey)], metrics=[output_rows=15000, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] + │ AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey], aggr=[count(orders.o_orderkey)], metrics=[output_rows=15000, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=15000, elapsed_compute=] │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p17] - │ RepartitionExec: partitioning=Hash([c_custkey@0], 18), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey], aggr=[count(orders.o_orderkey)], metrics=[output_rows=15000, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([c_custkey@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey], aggr=[count(orders.o_orderkey)], metrics=[output_rows=15000, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ ProjectionExec: expr=[c_custkey@1 as c_custkey, o_orderkey@0 as o_orderkey], metrics=[output_rows=153318, elapsed_compute=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=153318, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(o_custkey@1, c_custkey@0)], projection=[o_orderkey@0, c_custkey@2], metrics=[output_rows=153318, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=148318, elapsed_compute=] │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey], file_type=parquet, metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey], file_type=parquet, metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=148318, elapsed_compute=] │ FilterExec: o_comment@2 NOT LIKE %special%requests%, projection=[o_orderkey@0, o_custkey@1], metrics=[output_rows=148318, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_comment], file_type=parquet, predicate=o_comment@2 NOT LIKE %special%requests%, metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_comment], file_type=parquet, predicate=o_comment@2 NOT LIKE %special%requests%, metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -724,9 +725,9 @@ mod tests { let plan = test_tpch_query(14).await?; assert_snapshot!(plan, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] - │ ProjectionExec: expr=[100 * CAST(sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END)@0 AS Float64) / CAST(sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 AS Float64) as promo_revenue], metrics=[output_rows=1, elapsed_compute=] - │ AggregateExec: mode=Final, gby=[], aggr=[sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=1, elapsed_compute=] - │ CoalescePartitionsExec, metrics=[output_rows=24, elapsed_compute=] + │ ProjectionExec: expr=[100 * CAST(sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END)@0 AS Float64) / CAST(sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 AS Float64) as promo_revenue], metrics=[output_rows=1, elapsed_compute=, output_bytes=8.0 B] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=1, elapsed_compute=, output_bytes=32.0 B] + │ CoalescePartitionsExec, metrics=[output_rows=24, elapsed_compute=, output_bytes=6.0 KB] │ [Stage 3] => NetworkCoalesceExec: output_partitions=24, input_tasks=4, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] @@ -740,16 +741,16 @@ mod tests { │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] - │ RepartitionExec: partitioning=Hash([p_partkey@0], 24), input_partitions=2, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([p_partkey@0], 24), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_type], file_type=parquet, metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_type], file_type=parquet, metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] - │ RepartitionExec: partitioning=Hash([l_partkey@0], 24), input_partitions=2, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([l_partkey@0], 24), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=7630, elapsed_compute=] │ FilterExec: l_shipdate@3 >= 1995-09-01 AND l_shipdate@3 < 1995-10-01, projection=[l_partkey@0, l_extendedprice@1, l_discount@2], metrics=[output_rows=7630, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_partkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1995-09-01 AND l_shipdate@3 < 1995-10-01 AND DynamicFilterPhysicalExpr [ true ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-09-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-10-01, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_partkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1995-09-01 AND l_shipdate@3 < 1995-10-01 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-09-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-10-01, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "#); Ok(()) @@ -760,44 +761,44 @@ mod tests { let plan = test_tpch_query(15).await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [s_suppkey@0 ASC NULLS LAST], metrics=[output_rows=1, elapsed_compute=] - │ SortExec: expr=[s_suppkey@0 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=1, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(max(revenue0.total_revenue)@0, total_revenue@4)], projection=[s_suppkey@1, s_name@2, s_address@3, s_phone@4, total_revenue@5], metrics=[output_rows=1, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ AggregateExec: mode=Final, gby=[], aggr=[max(revenue0.total_revenue)], metrics=[output_rows=1, elapsed_compute=] - │ CoalescePartitionsExec, metrics=[output_rows=12, elapsed_compute=] + │ SortPreservingMergeExec: [s_suppkey@0 ASC NULLS LAST], metrics=[output_rows=1, elapsed_compute=, output_bytes=24.1 KB] + │ SortExec: expr=[s_suppkey@0 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=1, elapsed_compute=, output_bytes=0.0 B, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1, elapsed_compute=, output_bytes=600.0 KB] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(max(revenue0.total_revenue)@0, total_revenue@4)], projection=[s_suppkey@1, s_name@2, s_address@3, s_phone@4, total_revenue@5], metrics=[output_rows=1, elapsed_compute=, output_bytes=24.1 KB, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ AggregateExec: mode=Final, gby=[], aggr=[max(revenue0.total_revenue)], metrics=[output_rows=1, elapsed_compute=, output_bytes=16.0 B] + │ CoalescePartitionsExec, metrics=[output_rows=12, elapsed_compute=, output_bytes=1536.0 B] │ [Stage 2] => NetworkCoalesceExec: output_partitions=12, input_tasks=2, metrics=[] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1000, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, supplier_no@0)], projection=[s_suppkey@0, s_name@1, s_address@2, s_phone@3, total_revenue@5], metrics=[output_rows=1000, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_phone], file_type=parquet, metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ ProjectionExec: expr=[l_suppkey@0 as supplier_no, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue], metrics=[output_rows=1000, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=1000, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5825, elapsed_compute=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1000, elapsed_compute=, output_bytes=3.5 MB] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, supplier_no@0)], projection=[s_suppkey@0, s_name@1, s_address@2, s_phone@3, total_revenue@5], metrics=[output_rows=1000, elapsed_compute=, output_bytes=481.7 KB, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=, output_bytes=127.4 KB] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_phone], file_type=parquet, metrics=[output_rows=1000, elapsed_compute=, output_bytes=127.4 KB, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ ProjectionExec: expr=[l_suppkey@0 as supplier_no, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue], metrics=[output_rows=1000, elapsed_compute=, output_bytes=27.6 KB] + │ AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=1000, elapsed_compute=, output_bytes=27.6 KB, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5825, elapsed_compute=, output_bytes=2.2 MB] │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[max(revenue0.total_revenue)], metrics=[output_rows=12, elapsed_compute=] │ ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue], metrics=[output_rows=1000, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=1000, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] + │ AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=1000, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5825, elapsed_compute=] │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p11] t1:[p0..p11] t2:[p0..p11] t3:[p0..p11] - │ RepartitionExec: partitioning=Hash([l_suppkey@0], 12), input_partitions=2, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=5825, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([l_suppkey@0], 12), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=5825, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=22830, elapsed_compute=] │ FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2], metrics=[output_rows=22830, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ RepartitionExec: partitioning=Hash([l_suppkey@0], 6), input_partitions=2, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=5825, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([l_suppkey@0], 6), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=5825, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=22830, elapsed_compute=] │ FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2], metrics=[output_rows=22830, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -806,28 +807,28 @@ mod tests { #[tokio::test] async fn test_tpch_16() -> Result<(), Box> { let plan = test_tpch_query(16).await?; - assert_snapshot!(plan, @r#" + assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], metrics=[output_rows=2762, elapsed_compute=] + │ SortPreservingMergeExec: [supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], metrics=[output_rows=2762, elapsed_compute=, output_bytes=297.3 KB] │ [Stage 5] => NetworkCoalesceExec: output_partitions=12, input_tasks=2, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] │ SortExec: expr=[supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=2762, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] │ ProjectionExec: expr=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, count(alias1)@3 as supplier_cnt], metrics=[output_rows=2762, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)], metrics=[output_rows=2762, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] + │ AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)], metrics=[output_rows=2762, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=10538, elapsed_compute=] │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=3, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 4 ── Tasks: t0:[p0..p11] t1:[p0..p11] t2:[p0..p11] - │ RepartitionExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2], 12), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)], metrics=[output_rows=10538, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] - │ AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, alias1@3 as alias1], aggr=[], metrics=[output_rows=11632, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2], 12), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)], metrics=[output_rows=10538, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, alias1@3 as alias1], aggr=[], metrics=[output_rows=11632, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=11634, elapsed_compute=] │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p17] - │ RepartitionExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2, alias1@3], 18), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[p_brand@1 as p_brand, p_type@2 as p_type, p_size@3 as p_size, ps_suppkey@0 as alias1], aggr=[], metrics=[output_rows=11634, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2, alias1@3], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[p_brand@1 as p_brand, p_type@2 as p_type, p_size@3 as p_size, ps_suppkey@0 as alias1], aggr=[], metrics=[output_rows=11634, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=11635, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(s_suppkey@0, ps_suppkey@0)], metrics=[output_rows=11635, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=1, elapsed_compute=] @@ -837,21 +838,21 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, ps_partkey@0)], projection=[p_brand@1, p_type@2, p_size@3, ps_suppkey@5], metrics=[output_rows=11644, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=2911, elapsed_compute=] │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ], metrics=[output_rows=80000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=80000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1, elapsed_compute=] │ FilterExec: s_comment@1 LIKE %Customer%Complaints%, projection=[s_suppkey@0], metrics=[output_rows=1, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_comment], file_type=parquet, predicate=s_comment@1 LIKE %Customer%Complaints%, metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_comment], file_type=parquet, predicate=s_comment@1 LIKE %Customer%Complaints%, metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=2911, elapsed_compute=] - │ FilterExec: p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND Use p_size@3 IN (SET) ([Literal { value: Int32(49), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(14), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(23), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(45), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(19), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(3), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(36), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(9), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]), metrics=[output_rows=2911, elapsed_compute=] + │ FilterExec: p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND p_size@3 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]), metrics=[output_rows=2911, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_brand, p_type, p_size], file_type=parquet, predicate=p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND Use p_size@3 IN (SET) ([Literal { value: Int32(49), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(14), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(23), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(45), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(19), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(3), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(36), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(9), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]), pruning_predicate=p_brand_null_count@2 != row_count@3 AND (p_brand_min@0 != Brand#45 OR Brand#45 != p_brand_max@1) AND p_type_null_count@6 != row_count@3 AND (p_type_min@4 NOT LIKE MEDIUM POLISHED% OR p_type_max@5 NOT LIKE MEDIUM POLISHED%) AND (p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 49 AND 49 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 14 AND 14 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 23 AND 23 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 45 AND 45 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 19 AND 19 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 3 AND 3 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 36 AND 36 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 9 AND 9 <= p_size_max@8), required_guarantees=[p_brand not in (Brand#45), p_size in (14, 19, 23, 3, 36, 45, 49, 9)], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_brand, p_type, p_size], file_type=parquet, predicate=p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND p_size@3 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]), pruning_predicate=p_brand_null_count@2 != row_count@3 AND (p_brand_min@0 != Brand#45 OR Brand#45 != p_brand_max@1) AND p_type_null_count@6 != row_count@3 AND (p_type_min@4 NOT LIKE MEDIUM POLISHED% OR p_type_max@5 NOT LIKE MEDIUM POLISHED%) AND (p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 49 AND 49 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 14 AND 14 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 23 AND 23 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 45 AND 45 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 19 AND 19 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 3 AND 3 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 36 AND 36 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 9 AND 9 <= p_size_max@8), required_guarantees=[p_brand not in (Brand#45), p_size in (14, 19, 23, 3, 36, 45, 49, 9)], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - "#); + "); Ok(()) } @@ -860,9 +861,9 @@ mod tests { let plan = test_tpch_query(17).await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ ProjectionExec: expr=[CAST(sum(lineitem.l_extendedprice)@0 AS Float64) / 7 as avg_yearly], metrics=[output_rows=1, elapsed_compute=] - │ AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice)], metrics=[output_rows=1, elapsed_compute=] - │ CoalescePartitionsExec, metrics=[output_rows=24, elapsed_compute=] + │ ProjectionExec: expr=[CAST(sum(lineitem.l_extendedprice)@0 AS Float64) / 7 as avg_yearly], metrics=[output_rows=1, elapsed_compute=, output_bytes=8.0 B] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice)], metrics=[output_rows=1, elapsed_compute=, output_bytes=16.0 B] + │ CoalescePartitionsExec, metrics=[output_rows=24, elapsed_compute=, output_bytes=3.0 KB] │ [Stage 4] => NetworkCoalesceExec: output_partitions=24, input_tasks=4, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] @@ -872,30 +873,30 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=555, elapsed_compute=] │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] │ ProjectionExec: expr=[CAST(0.2 * CAST(avg(lineitem.l_quantity)@1 AS Float64) AS Decimal128(30, 15)) as Float64(0.2) * avg(lineitem.l_quantity), l_partkey@0 as l_partkey], metrics=[output_rows=20000, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)], metrics=[output_rows=20000, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] + │ AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)], metrics=[output_rows=20000, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=118823, elapsed_compute=] │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p23] - │ RepartitionExec: partitioning=Hash([p_partkey@2], 24), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([p_partkey@2], 24), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ ProjectionExec: expr=[l_quantity@1 as l_quantity, l_extendedprice@2 as l_extendedprice, p_partkey@0 as p_partkey], metrics=[output_rows=555, elapsed_compute=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=555, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@0)], projection=[p_partkey@0, l_quantity@2, l_extendedprice@3], metrics=[output_rows=555, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=18, elapsed_compute=] │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_partkey, l_quantity, l_extendedprice], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_partkey, l_quantity, l_extendedprice], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=18, elapsed_compute=] │ FilterExec: p_brand@1 = Brand#23 AND p_container@2 = MED BOX, projection=[p_partkey@0], metrics=[output_rows=18, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_brand, p_container], file_type=parquet, predicate=p_brand@1 = Brand#23 AND p_container@2 = MED BOX, pruning_predicate=p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5, required_guarantees=[p_brand in (Brand#23), p_container in (MED BOX)], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_brand, p_container], file_type=parquet, predicate=p_brand@1 = Brand#23 AND p_container@2 = MED BOX, pruning_predicate=p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5, required_guarantees=[p_brand in (Brand#23), p_container in (MED BOX)], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] - │ RepartitionExec: partitioning=Hash([l_partkey@0], 24), input_partitions=2, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)], metrics=[output_rows=118823, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([l_partkey@0], 24), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)], metrics=[output_rows=118823, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_partkey, l_quantity], file_type=parquet, metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_partkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -906,18 +907,18 @@ mod tests { let plan = test_tpch_query(18).await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST], metrics=[output_rows=5, elapsed_compute=] + │ SortPreservingMergeExec: [o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST], metrics=[output_rows=5, elapsed_compute=, output_bytes=4.4 KB] │ [Stage 8] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ SortExec: expr=[o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=5, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] - │ AggregateExec: mode=FinalPartitioned, gby=[c_name@0 as c_name, c_custkey@1 as c_custkey, o_orderkey@2 as o_orderkey, o_orderdate@3 as o_orderdate, o_totalprice@4 as o_totalprice], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=5, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] + │ AggregateExec: mode=FinalPartitioned, gby=[c_name@0 as c_name, c_custkey@1 as c_custkey, o_orderkey@2 as o_orderkey, o_orderdate@3 as o_orderdate, o_totalprice@4 as o_totalprice], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=5, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5, elapsed_compute=] │ [Stage 7] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 7 ── Tasks: t0:[p0..p17] - │ RepartitionExec: partitioning=Hash([c_name@0, c_custkey@1, o_orderkey@2, o_orderdate@3, o_totalprice@4], 18), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[c_name@1 as c_name, c_custkey@0 as c_custkey, o_orderkey@2 as o_orderkey, o_orderdate@4 as o_orderdate, o_totalprice@3 as o_totalprice], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=5, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([c_name@0, c_custkey@1, o_orderkey@2, o_orderdate@3, o_totalprice@4], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[c_name@1 as c_name, c_custkey@0 as c_custkey, o_orderkey@2 as o_orderkey, o_orderdate@4 as o_orderdate, o_totalprice@3 as o_totalprice], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=5, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=35, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(l_orderkey@0, o_orderkey@2)], metrics=[output_rows=35, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=5, elapsed_compute=] @@ -932,18 +933,18 @@ mod tests { ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5, elapsed_compute=] │ FilterExec: sum(lineitem.l_quantity)@1 > Some(30000),25,2, projection=[l_orderkey@0], metrics=[output_rows=5, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=150000, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] + │ AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=150000, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=150000, elapsed_compute=] │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] - │ RepartitionExec: partitioning=Hash([l_orderkey@0], 18), input_partitions=2, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=150000, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([l_orderkey@0], 18), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=150000, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_quantity], file_type=parquet, metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_quantity], file_type=parquet, metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ RepartitionExec: partitioning=Hash([o_orderkey@2], 6), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([o_orderkey@2], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=150000, elapsed_compute=] │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c_custkey@0, o_custkey@1)], projection=[c_custkey@0, c_name@1, o_orderkey@2, o_totalprice@4, o_orderdate@5], metrics=[output_rows=150000, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=15000, elapsed_compute=] @@ -952,19 +953,19 @@ mod tests { │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] - │ RepartitionExec: partitioning=Hash([c_custkey@0], 24), input_partitions=2, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([c_custkey@0], 24), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_name], file_type=parquet, metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_name], file_type=parquet, metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 4 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] - │ RepartitionExec: partitioning=Hash([o_custkey@1], 24), input_partitions=2, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([o_custkey@1], 24), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_totalprice, o_orderdate], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_totalprice, o_orderdate], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ RepartitionExec: partitioning=Hash([l_orderkey@0], 6), input_partitions=2, metrics=[fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([l_orderkey@0], 6), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_quantity], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -973,27 +974,27 @@ mod tests { #[tokio::test] async fn test_tpch_19() -> Result<(), Box> { let plan = test_tpch_query(19).await?; - assert_snapshot!(plan, @r#" + assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@0 as revenue], metrics=[output_rows=1, elapsed_compute=] - │ AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=1, elapsed_compute=] - │ CoalescePartitionsExec, metrics=[output_rows=6, elapsed_compute=] - │ AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=6, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=10, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN ([Literal { value: Utf8View("SM CASE"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND l_quantity@0 >= Some(100),15,2 AND l_quantity@0 <= Some(1100),15,2 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([Literal { value: Utf8View("MED BAG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND l_quantity@0 >= Some(1000),15,2 AND l_quantity@0 <= Some(2000),15,2 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([Literal { value: Utf8View("LG CASE"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND l_quantity@0 >= Some(2000),15,2 AND l_quantity@0 <= Some(3000),15,2 AND p_size@2 <= 15, projection=[l_extendedprice@6, l_discount@7], metrics=[output_rows=10, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=42, elapsed_compute=] + │ ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@0 as revenue], metrics=[output_rows=1, elapsed_compute=, output_bytes=16.0 B] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=1, elapsed_compute=, output_bytes=16.0 B] + │ CoalescePartitionsExec, metrics=[output_rows=6, elapsed_compute=, output_bytes=160.0 B] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=6, elapsed_compute=, output_bytes=160.0 B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=10, elapsed_compute=, output_bytes=1280.0 KB] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN ([SM CASE, SM BOX, SM PACK, SM PKG]) AND l_quantity@0 >= Some(100),15,2 AND l_quantity@0 <= Some(1100),15,2 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([MED BAG, MED BOX, MED PKG, MED PACK]) AND l_quantity@0 >= Some(1000),15,2 AND l_quantity@0 <= Some(2000),15,2 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([LG CASE, LG BOX, LG PACK, LG PKG]) AND l_quantity@0 >= Some(2000),15,2 AND l_quantity@0 <= Some(3000),15,2 AND p_size@2 <= 15, projection=[l_extendedprice@6, l_discount@7], metrics=[output_rows=10, elapsed_compute=, output_bytes=320.0 B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=42, elapsed_compute=, output_bytes=4.0 KB] │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=12635, elapsed_compute=] - │ FilterExec: (l_quantity@1 >= Some(100),15,2 AND l_quantity@1 <= Some(1100),15,2 OR l_quantity@1 >= Some(1000),15,2 AND l_quantity@1 <= Some(2000),15,2 OR l_quantity@1 >= Some(2000),15,2 AND l_quantity@1 <= Some(3000),15,2) AND (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON, projection=[l_partkey@0, l_quantity@1, l_extendedprice@2, l_discount@3], metrics=[output_rows=12635, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], file_type=parquet, predicate=(l_quantity@1 >= Some(100),15,2 AND l_quantity@1 <= Some(1100),15,2 OR l_quantity@1 >= Some(1000),15,2 AND l_quantity@1 <= Some(2000),15,2 OR l_quantity@1 >= Some(2000),15,2 AND l_quantity@1 <= Some(3000),15,2) AND (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON AND DynamicFilterPhysicalExpr [ l_partkey@0 >= 55 AND l_partkey@0 <= 19916 OR l_partkey@0 >= 55 AND l_partkey@0 <= 19916 OR l_partkey@0 >= 55 AND l_partkey@0 <= 19916 OR l_partkey@0 >= 55 AND l_partkey@0 <= 19916 OR l_partkey@0 >= 55 AND l_partkey@0 <= 19916 OR l_partkey@0 >= 55 AND l_partkey@0 <= 19916 ], pruning_predicate=(l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(100),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(1100),15,2 OR l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(1000),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(2000),15,2 OR l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(2000),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(3000),15,2) AND (l_shipmode_null_count@6 != row_count@2 AND l_shipmode_min@4 <= AIR AND AIR <= l_shipmode_max@5 OR l_shipmode_null_count@6 != row_count@2 AND l_shipmode_min@4 <= AIR REG AND AIR REG <= l_shipmode_max@5) AND l_shipinstruct_null_count@9 != row_count@2 AND l_shipinstruct_min@7 <= DELIVER IN PERSON AND DELIVER IN PERSON <= l_shipinstruct_max@8 AND (l_partkey_null_count@11 != row_count@2 AND l_partkey_max@10 >= 55 AND l_partkey_null_count@11 != row_count@2 AND l_partkey_min@12 <= 19916 OR l_partkey_null_count@11 != row_count@2 AND l_partkey_max@10 >= 55 AND l_partkey_null_count@11 != row_count@2 AND l_partkey_min@12 <= 19916 OR l_partkey_null_count@11 != row_count@2 AND l_partkey_max@10 >= 55 AND l_partkey_null_count@11 != row_count@2 AND l_partkey_min@12 <= 19916 OR l_partkey_null_count@11 != row_count@2 AND l_partkey_max@10 >= 55 AND l_partkey_null_count@11 != row_count@2 AND l_partkey_min@12 <= 19916 OR l_partkey_null_count@11 != row_count@2 AND l_partkey_max@10 >= 55 AND l_partkey_null_count@11 != row_count@2 AND l_partkey_min@12 <= 19916 OR l_partkey_null_count@11 != row_count@2 AND l_partkey_max@10 >= 55 AND l_partkey_null_count@11 != row_count@2 AND l_partkey_min@12 <= 19916), required_guarantees=[l_shipinstruct in (DELIVER IN PERSON), l_shipmode in (AIR, AIR REG)], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=12635, elapsed_compute=, output_bytes=5.2 MB] + │ FilterExec: (l_quantity@1 >= Some(100),15,2 AND l_quantity@1 <= Some(1100),15,2 OR l_quantity@1 >= Some(1000),15,2 AND l_quantity@1 <= Some(2000),15,2 OR l_quantity@1 >= Some(2000),15,2 AND l_quantity@1 <= Some(3000),15,2) AND (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON, projection=[l_partkey@0, l_quantity@1, l_extendedprice@2, l_discount@3], metrics=[output_rows=12635, elapsed_compute=, output_bytes=691.0 KB, selectivity=2.1% (12635/600572)] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], file_type=parquet, predicate=(l_quantity@1 >= Some(100),15,2 AND l_quantity@1 <= Some(1100),15,2 OR l_quantity@1 >= Some(1000),15,2 AND l_quantity@1 <= Some(2000),15,2 OR l_quantity@1 >= Some(2000),15,2 AND l_quantity@1 <= Some(3000),15,2) AND (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON AND DynamicFilter [ l_partkey@0 >= 55 AND l_partkey@0 <= 19916 ], pruning_predicate=(l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(100),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(1100),15,2 OR l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(1000),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(2000),15,2 OR l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(2000),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(3000),15,2) AND (l_shipmode_null_count@6 != row_count@2 AND l_shipmode_min@4 <= AIR AND AIR <= l_shipmode_max@5 OR l_shipmode_null_count@6 != row_count@2 AND l_shipmode_min@4 <= AIR REG AND AIR REG <= l_shipmode_max@5) AND l_shipinstruct_null_count@9 != row_count@2 AND l_shipinstruct_min@7 <= DELIVER IN PERSON AND DELIVER IN PERSON <= l_shipinstruct_max@8 AND l_partkey_null_count@11 != row_count@2 AND l_partkey_max@10 >= 55 AND l_partkey_null_count@11 != row_count@2 AND l_partkey_min@12 <= 19916, required_guarantees=[l_shipinstruct in (DELIVER IN PERSON), l_shipmode in (AIR, AIR REG)], metrics=[output_rows=600572, elapsed_compute=, output_bytes=52.1 MB, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=42, elapsed_compute=] - │ FilterExec: (p_brand@1 = Brand#12 AND p_container@3 IN ([Literal { value: Utf8View("SM CASE"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([Literal { value: Utf8View("MED BAG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([Literal { value: Utf8View("LG CASE"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 15) AND p_size@2 >= 1, metrics=[output_rows=42, elapsed_compute=] + │ FilterExec: (p_brand@1 = Brand#12 AND p_container@3 IN ([SM CASE, SM BOX, SM PACK, SM PKG]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([MED BAG, MED BOX, MED PKG, MED PACK]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([LG CASE, LG BOX, LG PACK, LG PKG]) AND p_size@2 <= 15) AND p_size@2 >= 1, metrics=[output_rows=42, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_brand, p_size, p_container], file_type=parquet, predicate=(p_brand@1 = Brand#12 AND p_container@3 IN ([Literal { value: Utf8View("SM CASE"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([Literal { value: Utf8View("MED BAG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([Literal { value: Utf8View("LG CASE"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 15) AND p_size@2 >= 1, pruning_predicate=(p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#12 AND Brand#12 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM CASE AND SM CASE <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM BOX AND SM BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM PACK AND SM PACK <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM PKG AND SM PKG <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 5 OR p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BAG AND MED BAG <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED PKG AND MED PKG <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED PACK AND MED PACK <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 10 OR p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#34 AND Brand#34 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG CASE AND LG CASE <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG BOX AND LG BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG PACK AND LG PACK <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG PKG AND LG PKG <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 15) AND p_size_null_count@8 != row_count@3 AND p_size_max@9 >= 1, required_guarantees=[p_brand in (Brand#12, Brand#23, Brand#34), p_container in (LG BOX, LG CASE, LG PACK, LG PKG, MED BAG, MED BOX, MED PACK, MED PKG, SM BOX, SM CASE, SM PACK, SM PKG)], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_brand, p_size, p_container], file_type=parquet, predicate=(p_brand@1 = Brand#12 AND p_container@3 IN ([SM CASE, SM BOX, SM PACK, SM PKG]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([MED BAG, MED BOX, MED PKG, MED PACK]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([LG CASE, LG BOX, LG PACK, LG PKG]) AND p_size@2 <= 15) AND p_size@2 >= 1, pruning_predicate=(p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#12 AND Brand#12 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM CASE AND SM CASE <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM BOX AND SM BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM PACK AND SM PACK <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM PKG AND SM PKG <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 5 OR p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BAG AND MED BAG <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED PKG AND MED PKG <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED PACK AND MED PACK <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 10 OR p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#34 AND Brand#34 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG CASE AND LG CASE <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG BOX AND LG BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG PACK AND LG PACK <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG PKG AND LG PKG <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 15) AND p_size_null_count@8 != row_count@3 AND p_size_max@9 >= 1, required_guarantees=[p_brand in (Brand#12, Brand#23, Brand#34), p_container in (LG BOX, LG CASE, LG PACK, LG PKG, MED BAG, MED BOX, MED PACK, MED PKG, SM BOX, SM CASE, SM PACK, SM PKG)], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - "#); + "); Ok(()) } @@ -1002,48 +1003,48 @@ mod tests { let plan = test_tpch_query(20).await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [s_name@0 ASC NULLS LAST], metrics=[output_rows=144, elapsed_compute=] - │ SortExec: expr=[s_name@0 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=144, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=144, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(s_suppkey@0, ps_suppkey@0)], projection=[s_name@1, s_address@2], metrics=[output_rows=144, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=592, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=592, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[s_suppkey@1, s_name@2, s_address@3], metrics=[output_rows=592, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=] + │ SortPreservingMergeExec: [s_name@0 ASC NULLS LAST], metrics=[output_rows=144, elapsed_compute=, output_bytes=20.5 KB] + │ SortExec: expr=[s_name@0 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=144, elapsed_compute=, output_bytes=0.0 B, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=144, elapsed_compute=, output_bytes=272.0 KB] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(s_suppkey@0, ps_suppkey@0)], projection=[s_name@1, s_address@2], metrics=[output_rows=144, elapsed_compute=, output_bytes=154.5 KB, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=592, elapsed_compute=, output_bytes=3.6 MB] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=592, elapsed_compute=, output_bytes=3.6 MB] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[s_suppkey@1, s_name@2, s_address@3], metrics=[output_rows=592, elapsed_compute=, output_bytes=70.0 KB, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=, output_bytes=768.0 B] │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_nationkey], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ s_nationkey@3 >= 3 AND s_nationkey@3 <= 3 OR s_nationkey@3 >= 3 AND s_nationkey@3 <= 3 OR s_nationkey@3 >= 3 AND s_nationkey@3 <= 3 OR s_nationkey@3 >= 3 AND s_nationkey@3 <= 3 OR s_nationkey@3 >= 3 AND s_nationkey@3 <= 3 OR s_nationkey@3 >= 3 AND s_nationkey@3 <= 3 ], pruning_predicate=s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 3 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 3 OR s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 3 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 3 OR s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 3 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 3 OR s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 3 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 3 OR s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 3 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 3 OR s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 3 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 3, required_guarantees=[], metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=521, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ps_partkey@0, l_partkey@1), (ps_suppkey@1, l_suppkey@2)], filter=CAST(ps_availqty@0 AS Float64) > Float64(0.5) * sum(lineitem.l_quantity)@1, projection=[ps_suppkey@1], metrics=[output_rows=521, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=760, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=760, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(p_partkey@0, ps_partkey@0)], metrics=[output_rows=760, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=190, elapsed_compute=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_nationkey], file_type=parquet, predicate=DynamicFilter [ s_nationkey@3 >= 3 AND s_nationkey@3 <= 3 ], pruning_predicate=s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 3 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 3, required_guarantees=[], metrics=[output_rows=1000, elapsed_compute=, output_bytes=99.6 KB, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=521, elapsed_compute=, output_bytes=768.0 KB] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ps_partkey@0, l_partkey@1), (ps_suppkey@1, l_suppkey@2)], filter=CAST(ps_availqty@0 AS Float64) > Float64(0.5) * sum(lineitem.l_quantity)@1, projection=[ps_suppkey@1], metrics=[output_rows=521, elapsed_compute=, output_bytes=4.1 KB, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=760, elapsed_compute=, output_bytes=1920.0 KB] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=760, elapsed_compute=, output_bytes=1920.0 KB] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(p_partkey@0, ps_partkey@0)], metrics=[output_rows=760, elapsed_compute=, output_bytes=14.8 KB, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=190, elapsed_compute=, output_bytes=2.1 KB] │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_availqty], file_type=parquet, metrics=[output_rows=80000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ ProjectionExec: expr=[0.5 * CAST(sum(lineitem.l_quantity)@2 AS Float64) as Float64(0.5) * sum(lineitem.l_quantity), l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], metrics=[output_rows=54539, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=54539, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=83515, elapsed_compute=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_availqty], file_type=parquet, metrics=[output_rows=80000, elapsed_compute=, output_bytes=1562.5 KB, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ ProjectionExec: expr=[0.5 * CAST(sum(lineitem.l_quantity)@2 AS Float64) as Float64(0.5) * sum(lineitem.l_quantity), l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], metrics=[output_rows=54539, elapsed_compute=, output_bytes=3.4 MB] + │ AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=54539, elapsed_compute=, output_bytes=5.4 MB, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=83515, elapsed_compute=, output_bytes=6.0 MB] │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] │ FilterExec: n_name@1 = CANADA, projection=[n_nationkey@0], metrics=[output_rows=16, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = CANADA, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= CANADA AND CANADA <= n_name_max@1, required_guarantees=[n_name in (CANADA)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = CANADA, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= CANADA AND CANADA <= n_name_max@1, required_guarantees=[n_name in (CANADA)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=190, elapsed_compute=] │ FilterExec: p_name@1 LIKE forest%, projection=[p_partkey@0], metrics=[output_rows=190, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_name], file_type=parquet, predicate=p_name@1 LIKE forest%, pruning_predicate=p_name_null_count@2 != row_count@3 AND p_name_min@0 <= foresu AND forest <= p_name_max@1, required_guarantees=[], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_name], file_type=parquet, predicate=p_name@1 LIKE forest%, pruning_predicate=p_name_null_count@2 != row_count@3 AND p_name_min@0 <= foresu AND forest <= p_name_max@1, required_guarantees=[], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ RepartitionExec: partitioning=Hash([l_partkey@0, l_suppkey@1], 6), input_partitions=2, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=83515, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([l_partkey@0, l_suppkey@1], 6), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=83515, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=92040, elapsed_compute=] │ FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01, projection=[l_partkey@0, l_suppkey@1, l_quantity@2], metrics=[output_rows=92040, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND DynamicFilter [ l_partkey@0 >= 5 AND l_partkey@0 <= 19852 AND l_suppkey@1 >= 4 AND l_suppkey@1 <= 1000 ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01 AND l_partkey_null_count@5 != row_count@2 AND l_partkey_max@4 >= 5 AND l_partkey_null_count@5 != row_count@2 AND l_partkey_min@6 <= 19852 AND l_suppkey_null_count@8 != row_count@2 AND l_suppkey_max@7 >= 4 AND l_suppkey_null_count@8 != row_count@2 AND l_suppkey_min@9 <= 1000, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -1054,19 +1055,19 @@ mod tests { let plan = test_tpch_query(21).await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [numwait@1 DESC, s_name@0 ASC NULLS LAST], metrics=[output_rows=47, elapsed_compute=] + │ SortPreservingMergeExec: [numwait@1 DESC, s_name@0 ASC NULLS LAST], metrics=[output_rows=47, elapsed_compute=, output_bytes=6.9 KB] │ [Stage 4] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ SortExec: expr=[numwait@1 DESC, s_name@0 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=47, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] │ ProjectionExec: expr=[s_name@0 as s_name, count(Int64(1))@1 as numwait], metrics=[output_rows=47, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[s_name@0 as s_name], aggr=[count(Int64(1))], metrics=[output_rows=47, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] + │ AggregateExec: mode=FinalPartitioned, gby=[s_name@0 as s_name], aggr=[count(Int64(1))], metrics=[output_rows=47, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=47, elapsed_compute=] │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p17] - │ RepartitionExec: partitioning=Hash([s_name@0], 18), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[s_name@0 as s_name], aggr=[count(Int64(1))], metrics=[output_rows=47, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] + │ RepartitionExec: partitioning=Hash([s_name@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[s_name@0 as s_name], aggr=[count(Int64(1))], metrics=[output_rows=47, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=7440, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0, projection=[s_name@0], metrics=[output_rows=7440, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=132720, elapsed_compute=] @@ -1084,26 +1085,26 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=379809, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_name@1, s_nationkey@2, l_orderkey@3, l_suppkey@4], metrics=[output_rows=379809, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_nationkey], file_type=parquet, metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=379809, elapsed_compute=] │ FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1], metrics=[output_rows=379809, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@3 > l_commitdate@2 AND DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey], file_type=parquet, metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@3 > l_commitdate@2 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey], file_type=parquet, metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=379809, elapsed_compute=] │ FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1], metrics=[output_rows=379809, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@3 > l_commitdate@2, metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@3 > l_commitdate@2, metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] │ FilterExec: n_name@1 = SAUDI ARABIA, projection=[n_nationkey@0], metrics=[output_rows=16, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = SAUDI ARABIA, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= SAUDI ARABIA AND SAUDI ARABIA <= n_name_max@1, required_guarantees=[n_name in (SAUDI ARABIA)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = SAUDI ARABIA, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= SAUDI ARABIA AND SAUDI ARABIA <= n_name_max@1, required_guarantees=[n_name in (SAUDI ARABIA)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=72884, elapsed_compute=] │ FilterExec: o_orderstatus@1 = F, projection=[o_orderkey@0], metrics=[output_rows=72884, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_orderstatus], file_type=parquet, predicate=o_orderstatus@1 = F, pruning_predicate=o_orderstatus_null_count@2 != row_count@3 AND o_orderstatus_min@0 <= F AND F <= o_orderstatus_max@1, required_guarantees=[o_orderstatus in (F)], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_orderstatus], file_type=parquet, predicate=o_orderstatus@1 = F, pruning_predicate=o_orderstatus_null_count@2 != row_count@3 AND o_orderstatus_min@0 <= F AND F <= o_orderstatus_max@1, required_guarantees=[o_orderstatus in (F)], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -1112,46 +1113,47 @@ mod tests { #[tokio::test] async fn test_tpch_22() -> Result<(), Box> { let plan = test_tpch_query(22).await?; - assert_snapshot!(plan, @r#" + assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [cntrycode@0 ASC NULLS LAST], metrics=[output_rows=7, elapsed_compute=] + │ SortPreservingMergeExec: [cntrycode@0 ASC NULLS LAST], metrics=[output_rows=7, elapsed_compute=, output_bytes=280.0 B] │ [Stage 4] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ SortExec: expr=[cntrycode@0 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=7, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] │ ProjectionExec: expr=[cntrycode@0 as cntrycode, count(Int64(1))@1 as numcust, sum(custsale.c_acctbal)@2 as totacctbal], metrics=[output_rows=7, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)], metrics=[output_rows=7, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=] + │ AggregateExec: mode=FinalPartitioned, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)], metrics=[output_rows=7, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=7, elapsed_compute=] │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p17] - │ RepartitionExec: partitioning=Hash([cntrycode@0], 18), input_partitions=6, metrics=[fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)], metrics=[output_rows=7, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=] - │ ProjectionExec: expr=[substr(c_phone@0, 1, 2) as cntrycode, c_acctbal@1 as c_acctbal], metrics=[output_rows=641, elapsed_compute=] - │ NestedLoopJoinExec: join_type=Inner, filter=CAST(c_acctbal@0 AS Decimal128(19, 6)) > avg(customer.c_acctbal)@1, projection=[c_phone@1, c_acctbal@2], metrics=[output_rows=641, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ RepartitionExec: partitioning=Hash([cntrycode@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)], metrics=[output_rows=7, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ ProjectionExec: expr=[substr(c_phone@1, 1, 2) as cntrycode, c_acctbal@2 as c_acctbal], metrics=[output_rows=641, elapsed_compute=] + │ NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 > avg(customer.c_acctbal)@0, projection=[avg(customer.c_acctbal)@0, c_phone@1, c_acctbal@2], metrics=[output_rows=641, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ AggregateExec: mode=Final, gby=[], aggr=[avg(customer.c_acctbal)], metrics=[output_rows=1, elapsed_compute=] │ CoalescePartitionsExec, metrics=[output_rows=8, elapsed_compute=] │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1360, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(c_custkey@0, o_custkey@0)], projection=[c_phone@1, c_acctbal@2], metrics=[output_rows=1360, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=4115, elapsed_compute=] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_custkey], file_type=parquet, metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ ProjectionExec: expr=[c_phone@0 as c_phone, c_acctbal@1 as c_acctbal, CAST(c_acctbal@1 AS Decimal128(19, 6)) as join_proj_push_down_1], metrics=[output_rows=1360, elapsed_compute=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1360, elapsed_compute=] + │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(c_custkey@0, o_custkey@0)], projection=[c_phone@1, c_acctbal@2], metrics=[output_rows=1360, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=4115, elapsed_compute=] + │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_custkey], file_type=parquet, metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ AggregateExec: mode=Partial, gby=[], aggr=[avg(customer.c_acctbal)], metrics=[output_rows=8, elapsed_compute=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=3741, elapsed_compute=] - │ FilterExec: c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN ([Literal { value: Utf8View("13"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("31"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("23"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("29"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("30"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("18"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("17"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]), projection=[c_acctbal@1], metrics=[output_rows=3741, elapsed_compute=] + │ FilterExec: c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]), projection=[c_acctbal@1], metrics=[output_rows=3741, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_phone, c_acctbal], file_type=parquet, predicate=c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN ([Literal { value: Utf8View("13"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("31"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("23"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("29"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("30"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("18"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("17"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]), pruning_predicate=c_acctbal_null_count@1 != row_count@2 AND c_acctbal_max@0 > Some(0),15,2, required_guarantees=[], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_phone, c_acctbal], file_type=parquet, predicate=c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]), pruning_predicate=c_acctbal_null_count@1 != row_count@2 AND c_acctbal_max@0 > Some(0),15,2, required_guarantees=[], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=4115, elapsed_compute=] - │ FilterExec: substr(c_phone@1, 1, 2) IN ([Literal { value: Utf8View("13"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("31"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("23"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("29"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("30"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("18"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("17"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]), metrics=[output_rows=4115, elapsed_compute=] + │ FilterExec: substr(c_phone@1, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]), metrics=[output_rows=4115, elapsed_compute=] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_phone, c_acctbal], file_type=parquet, predicate=substr(c_phone@1, 1, 2) IN ([Literal { value: Utf8View("13"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("31"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("23"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("29"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("30"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("18"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("17"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]), metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, files_ranges_pruned_statistics=, num_predicate_creation_errors=, page_index_rows_matched=, page_index_rows_pruned=, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, row_groups_matched_bloom_filter=, row_groups_matched_statistics=, row_groups_pruned_bloom_filter=, row_groups_pruned_statistics=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_phone, c_acctbal], file_type=parquet, predicate=substr(c_phone@1, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]), metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - "#); + "); Ok(()) } diff --git a/tests/tpch_plans_test.rs b/tests/tpch_plans_test.rs index 369111fb..e440fdf6 100644 --- a/tests/tpch_plans_test.rs +++ b/tests/tpch_plans_test.rs @@ -69,17 +69,17 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@4)], projection=[n_name@1, n_regionkey@2, p_partkey@3, p_mfgr@4, s_name@5, s_address@6, s_phone@8, s_acctbal@9, s_comment@10, ps_supplycost@11] │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] │ ProjectionExec: expr=[p_partkey@6 as p_partkey, p_mfgr@7 as p_mfgr, s_name@0 as s_name, s_address@1 as s_address, s_nationkey@2 as s_nationkey, s_phone@3 as s_phone, s_acctbal@4 as s_acctbal, s_comment@5 as s_comment, ps_supplycost@8 as ps_supplycost] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@2)], projection=[s_name@1, s_address@2, s_nationkey@3, s_phone@4, s_acctbal@5, s_comment@6, p_partkey@7, p_mfgr@8, ps_supplycost@10] │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment], file_type=parquet + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment], file_type=parquet, predicate=DynamicFilter [ empty ] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, ps_partkey@0)], projection=[p_partkey@0, p_mfgr@1, ps_suppkey@3, ps_supplycost@4] │ CoalescePartitionsExec │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192 @@ -111,13 +111,13 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_regionkey@1, ps_partkey@2, ps_supplycost@3] │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_regionkey], file_type=parquet + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_supplycost@2 as ps_supplycost, s_nationkey@0 as s_nationkey] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@1)], projection=[s_nationkey@1, ps_partkey@2, ps_supplycost@4] │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 4 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192 @@ -156,10 +156,10 @@ mod tests { │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: o_orderdate@2 < 1995-03-15 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_custkey, o_orderdate, o_shippriority], file_type=parquet, predicate=o_orderdate@2 < 1995-03-15 AND DynamicFilterPhysicalExpr [ true ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@0 < 1995-03-15, required_guarantees=[] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_custkey, o_orderdate, o_shippriority], file_type=parquet, predicate=o_orderdate@2 < 1995-03-15 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@0 < 1995-03-15, required_guarantees=[] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: l_shipdate@3 > 1995-03-15, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 > 1995-03-15 AND DynamicFilterPhysicalExpr [ true ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 > 1995-03-15, required_guarantees=[] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 > 1995-03-15 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 > 1995-03-15, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192 @@ -233,12 +233,12 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_name@1, n_regionkey@2, l_extendedprice@3, l_discount@4] │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, s_nationkey@0 as s_nationkey] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1), (s_nationkey@1, c_nationkey@0)], projection=[s_nationkey@1, l_extendedprice@4, l_discount@5] │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@1, l_orderkey@0)], projection=[c_nationkey@0, l_suppkey@3, l_extendedprice@4, l_discount@5] │ CoalescePartitionsExec @@ -247,8 +247,8 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_custkey@1, c_custkey@0)], projection=[o_orderkey@0, c_nationkey@3] │ CoalescePartitionsExec │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192 @@ -319,7 +319,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_custkey@0, o_custkey@4)], projection=[c_nationkey@1, s_nationkey@2, l_extendedprice@3, l_discount@4, l_shipdate@5] │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_orderkey@1, o_orderkey@0)], projection=[s_nationkey@0, l_extendedprice@2, l_discount@3, l_shipdate@4, o_custkey@6] │ CoalesceBatchesExec: target_batch_size=8192 @@ -327,10 +327,10 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_nationkey@1, l_orderkey@2, l_extendedprice@4, l_discount@5, l_shipdate@6] │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: l_shipdate@4 >= 1995-01-01 AND l_shipdate@4 <= 1996-12-31 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@4 >= 1995-01-01 AND l_shipdate@4 <= 1996-12-31 AND DynamicFilterPhysicalExpr [ true ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 <= 1996-12-31, required_guarantees=[] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@4 >= 1995-01-01 AND l_shipdate@4 <= 1996-12-31 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 <= 1996-12-31, required_guarantees=[] │ CoalesceBatchesExec: target_batch_size=8192 │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=4 └────────────────────────────────────────────────── @@ -349,7 +349,7 @@ mod tests { ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] │ RepartitionExec: partitioning=Hash([o_orderkey@0], 6), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_custkey], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); Ok(()) @@ -387,7 +387,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@4)], projection=[n_regionkey@1, l_extendedprice@2, l_discount@3, s_nationkey@4, o_orderdate@5] │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_regionkey], file_type=parquet + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_custkey@3, c_custkey@0)], projection=[l_extendedprice@0, l_discount@1, s_nationkey@2, o_orderdate@4, c_nationkey@6] │ CoalescePartitionsExec @@ -400,13 +400,13 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_nationkey@1, l_orderkey@2, l_extendedprice@4, l_discount@5] │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@1, l_suppkey@3, l_extendedprice@4, l_discount@5] │ CoalescePartitionsExec │ [Stage 3] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_partkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_partkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192 @@ -481,12 +481,12 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@2)], projection=[s_nationkey@1, l_orderkey@2, l_partkey@3, l_suppkey@4, l_quantity@5, l_extendedprice@6, l_discount@7] │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@1, l_partkey@2, l_suppkey@3, l_quantity@4, l_extendedprice@5, l_discount@6] │ CoalescePartitionsExec │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192 @@ -497,7 +497,7 @@ mod tests { ┌───── Stage 4 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] │ RepartitionExec: partitioning=Hash([ps_suppkey@1, ps_partkey@0], 24), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── "); Ok(()) @@ -534,10 +534,10 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_custkey@1, c_custkey@0)], projection=[o_orderkey@0, c_custkey@2, c_name@3, c_address@4, c_nationkey@5, c_phone@6, c_acctbal@7, c_comment@8] │ CoalescePartitionsExec │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_comment], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_comment], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: l_returnflag@3 = R, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_extendedprice, l_discount, l_returnflag], file_type=parquet, predicate=l_returnflag@3 = R AND DynamicFilterPhysicalExpr [ true ], pruning_predicate=l_returnflag_null_count@2 != row_count@3 AND l_returnflag_min@0 <= R AND R <= l_returnflag_max@1, required_guarantees=[l_returnflag in (R)] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_extendedprice, l_discount, l_returnflag], file_type=parquet, predicate=l_returnflag@3 = R AND DynamicFilter [ empty ], pruning_predicate=l_returnflag_null_count@2 != row_count@3 AND l_returnflag_min@0 <= R AND R <= l_returnflag_max@1, required_guarantees=[l_returnflag in (R)] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192 @@ -556,8 +556,8 @@ mod tests { ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [value@1 DESC] │ SortExec: expr=[value@1 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[ps_partkey@0 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 as value] - │ NestedLoopJoinExec: join_type=Inner, filter=CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@0 AS Decimal128(38, 15)) > sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)@1, projection=[ps_partkey@1, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@2] + │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@2 as value] + │ NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 > sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)@0, projection=[sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)@0, ps_partkey@1, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@2] │ ProjectionExec: expr=[CAST(CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@0 AS Float64) * 0.0001 AS Decimal128(38, 15)) as sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)] │ AggregateExec: mode=Final, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)] │ CoalescePartitionsExec @@ -570,22 +570,23 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@0)], projection=[s_nationkey@1, ps_availqty@3, ps_supplycost@4] │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] - │ AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ps_partkey@0], 6), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[ps_partkey@1, ps_availqty@2, ps_supplycost@3] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_availqty@2 as ps_availqty, ps_supplycost@3 as ps_supplycost, s_nationkey@0 as s_nationkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@1)], projection=[s_nationkey@1, ps_partkey@2, ps_availqty@4, ps_supplycost@5] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ps_partkey@0 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 as sum(partsupp.ps_supplycost * partsupp.ps_availqty), CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 AS Decimal128(38, 15)) as join_proj_push_down_1] + │ AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ps_partkey@0], 6), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[ps_partkey@1, ps_availqty@2, ps_supplycost@3] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 + │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_availqty@2 as ps_availqty, ps_supplycost@3 as ps_supplycost, s_nationkey@0 as s_nationkey] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@1)], projection=[s_nationkey@1, ps_partkey@2, ps_availqty@4, ps_supplycost@5] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192 @@ -638,7 +639,7 @@ mod tests { ┌───── Stage 2 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] │ RepartitionExec: partitioning=Hash([o_orderkey@0], 18), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_orderpriority], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_orderpriority], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── "#); Ok(()) @@ -702,7 +703,7 @@ mod tests { DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet, /testdata/tpch/plan_sf0.02/part/10.parquet, /testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet, /testdata/tpch/plan_sf0.02/part/13.parquet, /testdata/tpch/plan_sf0.02/part/14.parquet], [/testdata/tpch/plan_sf0.02/part/15.parquet, /testdata/tpch/plan_sf0.02/part/16.parquet, /testdata/tpch/plan_sf0.02/part/2.parquet], [/testdata/tpch/plan_sf0.02/part/3.parquet, /testdata/tpch/plan_sf0.02/part/4.parquet, /testdata/tpch/plan_sf0.02/part/5.parquet], [/testdata/tpch/plan_sf0.02/part/6.parquet, /testdata/tpch/plan_sf0.02/part/7.parquet, /testdata/tpch/plan_sf0.02/part/8.parquet], [/testdata/tpch/plan_sf0.02/part/9.parquet]]}, projection=[p_partkey, p_type], file_type=parquet CoalesceBatchesExec: target_batch_size=8192 FilterExec: l_shipdate@3 >= 1995-09-01 AND l_shipdate@3 < 1995-10-01, projection=[l_partkey@0, l_extendedprice@1, l_discount@2] - DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], [/testdata/tpch/plan_sf0.02/lineitem/9.parquet]]}, projection=[l_partkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1995-09-01 AND l_shipdate@3 < 1995-10-01 AND DynamicFilterPhysicalExpr [ true ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-09-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-10-01, required_guarantees=[] + DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], [/testdata/tpch/plan_sf0.02/lineitem/9.parquet]]}, projection=[l_partkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1995-09-01 AND l_shipdate@3 < 1995-10-01 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-09-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-10-01, required_guarantees=[] "#); Ok(()) } @@ -758,7 +759,7 @@ mod tests { #[tokio::test] async fn test_tpch_16() -> Result<(), Box> { let plan = test_tpch_query(16).await?; - assert_snapshot!(plan, @r#" + assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST] │ [Stage 5] => NetworkCoalesceExec: output_partitions=12, input_tasks=2 @@ -789,7 +790,7 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, ps_partkey@0)], projection=[p_brand@1, p_type@2, p_size@3, ps_suppkey@5] │ CoalescePartitionsExec │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_partkey, ps_suppkey], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_partkey, ps_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192 @@ -799,11 +800,11 @@ mod tests { └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND Use p_size@3 IN (SET) ([Literal { value: Int32(49), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(14), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(23), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(45), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(19), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(3), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(36), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(9), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) + │ FilterExec: p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND p_size@3 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]) │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet, /testdata/tpch/plan_sf0.02/part/10.parquet, /testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet, /testdata/tpch/plan_sf0.02/part/13.parquet, /testdata/tpch/plan_sf0.02/part/14.parquet], [/testdata/tpch/plan_sf0.02/part/15.parquet, /testdata/tpch/plan_sf0.02/part/16.parquet, /testdata/tpch/plan_sf0.02/part/2.parquet], [/testdata/tpch/plan_sf0.02/part/3.parquet, /testdata/tpch/plan_sf0.02/part/4.parquet, /testdata/tpch/plan_sf0.02/part/5.parquet], [/testdata/tpch/plan_sf0.02/part/6.parquet, /testdata/tpch/plan_sf0.02/part/7.parquet, /testdata/tpch/plan_sf0.02/part/8.parquet], ...]}, projection=[p_partkey, p_brand, p_type, p_size], file_type=parquet, predicate=p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND Use p_size@3 IN (SET) ([Literal { value: Int32(49), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(14), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(23), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(45), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(19), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(3), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(36), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(9), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]), pruning_predicate=p_brand_null_count@2 != row_count@3 AND (p_brand_min@0 != Brand#45 OR Brand#45 != p_brand_max@1) AND p_type_null_count@6 != row_count@3 AND (p_type_min@4 NOT LIKE MEDIUM POLISHED% OR p_type_max@5 NOT LIKE MEDIUM POLISHED%) AND (p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 49 AND 49 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 14 AND 14 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 23 AND 23 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 45 AND 45 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 19 AND 19 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 3 AND 3 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 36 AND 36 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 9 AND 9 <= p_size_max@8), required_guarantees=[p_brand not in (Brand#45), p_size in (14, 19, 23, 3, 36, 45, 49, 9)] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet, /testdata/tpch/plan_sf0.02/part/10.parquet, /testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet, /testdata/tpch/plan_sf0.02/part/13.parquet, /testdata/tpch/plan_sf0.02/part/14.parquet], [/testdata/tpch/plan_sf0.02/part/15.parquet, /testdata/tpch/plan_sf0.02/part/16.parquet, /testdata/tpch/plan_sf0.02/part/2.parquet], [/testdata/tpch/plan_sf0.02/part/3.parquet, /testdata/tpch/plan_sf0.02/part/4.parquet, /testdata/tpch/plan_sf0.02/part/5.parquet], [/testdata/tpch/plan_sf0.02/part/6.parquet, /testdata/tpch/plan_sf0.02/part/7.parquet, /testdata/tpch/plan_sf0.02/part/8.parquet], ...]}, projection=[p_partkey, p_brand, p_type, p_size], file_type=parquet, predicate=p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND p_size@3 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]), pruning_predicate=p_brand_null_count@2 != row_count@3 AND (p_brand_min@0 != Brand#45 OR Brand#45 != p_brand_max@1) AND p_type_null_count@6 != row_count@3 AND (p_type_min@4 NOT LIKE MEDIUM POLISHED% OR p_type_max@5 NOT LIKE MEDIUM POLISHED%) AND (p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 49 AND 49 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 14 AND 14 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 23 AND 23 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 45 AND 45 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 19 AND 19 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 3 AND 3 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 36 AND 36 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 9 AND 9 <= p_size_max@8), required_guarantees=[p_brand not in (Brand#45), p_size in (14, 19, 23, 3, 36, 45, 49, 9)] └────────────────────────────────────────────────── - "#); + "); Ok(()) } @@ -824,7 +825,7 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@0)], projection=[p_partkey@0, l_quantity@2, l_extendedprice@3] │ CoalescePartitionsExec │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_partkey, l_quantity, l_extendedprice], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_partkey, l_quantity, l_extendedprice], file_type=parquet, predicate=DynamicFilter [ empty ] │ ProjectionExec: expr=[CAST(0.2 * CAST(avg(lineitem.l_quantity)@1 AS Float64) AS Decimal128(30, 15)) as Float64(0.2) * avg(lineitem.l_quantity), l_partkey@0 as l_partkey] │ AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)] │ CoalesceBatchesExec: target_batch_size=8192 @@ -840,7 +841,7 @@ mod tests { │ RepartitionExec: partitioning=Hash([l_partkey@0], 6), input_partitions=2 │ AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_partkey, l_quantity], file_type=parquet + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_partkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── "); Ok(()) @@ -874,8 +875,8 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_custkey@0, o_custkey@1)], projection=[c_custkey@0, c_name@1, o_orderkey@2, o_totalprice@4, o_orderdate@5] │ CoalescePartitionsExec │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_name], file_type=parquet - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_custkey, o_totalprice, o_orderdate], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_quantity], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_custkey, o_totalprice, o_orderdate], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 @@ -897,27 +898,27 @@ mod tests { #[tokio::test] async fn test_tpch_19() -> Result<(), Box> { let plan = test_tpch_query(19).await?; - assert_snapshot!(plan, @r#" + assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@0 as revenue] │ AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] │ CoalescePartitionsExec │ AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN ([Literal { value: Utf8View("SM CASE"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND l_quantity@0 >= Some(100),15,2 AND l_quantity@0 <= Some(1100),15,2 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([Literal { value: Utf8View("MED BAG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND l_quantity@0 >= Some(1000),15,2 AND l_quantity@0 <= Some(2000),15,2 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([Literal { value: Utf8View("LG CASE"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND l_quantity@0 >= Some(2000),15,2 AND l_quantity@0 <= Some(3000),15,2 AND p_size@2 <= 15, projection=[l_extendedprice@6, l_discount@7] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN ([SM CASE, SM BOX, SM PACK, SM PKG]) AND l_quantity@0 >= Some(100),15,2 AND l_quantity@0 <= Some(1100),15,2 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([MED BAG, MED BOX, MED PKG, MED PACK]) AND l_quantity@0 >= Some(1000),15,2 AND l_quantity@0 <= Some(2000),15,2 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([LG CASE, LG BOX, LG PACK, LG PKG]) AND l_quantity@0 >= Some(2000),15,2 AND l_quantity@0 <= Some(3000),15,2 AND p_size@2 <= 15, projection=[l_extendedprice@6, l_discount@7] │ CoalescePartitionsExec │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: (l_quantity@1 >= Some(100),15,2 AND l_quantity@1 <= Some(1100),15,2 OR l_quantity@1 >= Some(1000),15,2 AND l_quantity@1 <= Some(2000),15,2 OR l_quantity@1 >= Some(2000),15,2 AND l_quantity@1 <= Some(3000),15,2) AND (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON, projection=[l_partkey@0, l_quantity@1, l_extendedprice@2, l_discount@3] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], file_type=parquet, predicate=(l_quantity@1 >= Some(100),15,2 AND l_quantity@1 <= Some(1100),15,2 OR l_quantity@1 >= Some(1000),15,2 AND l_quantity@1 <= Some(2000),15,2 OR l_quantity@1 >= Some(2000),15,2 AND l_quantity@1 <= Some(3000),15,2) AND (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON AND DynamicFilterPhysicalExpr [ true ], pruning_predicate=(l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(100),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(1100),15,2 OR l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(1000),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(2000),15,2 OR l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(2000),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(3000),15,2) AND (l_shipmode_null_count@6 != row_count@2 AND l_shipmode_min@4 <= AIR AND AIR <= l_shipmode_max@5 OR l_shipmode_null_count@6 != row_count@2 AND l_shipmode_min@4 <= AIR REG AND AIR REG <= l_shipmode_max@5) AND l_shipinstruct_null_count@9 != row_count@2 AND l_shipinstruct_min@7 <= DELIVER IN PERSON AND DELIVER IN PERSON <= l_shipinstruct_max@8, required_guarantees=[l_shipinstruct in (DELIVER IN PERSON), l_shipmode in (AIR, AIR REG)] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], file_type=parquet, predicate=(l_quantity@1 >= Some(100),15,2 AND l_quantity@1 <= Some(1100),15,2 OR l_quantity@1 >= Some(1000),15,2 AND l_quantity@1 <= Some(2000),15,2 OR l_quantity@1 >= Some(2000),15,2 AND l_quantity@1 <= Some(3000),15,2) AND (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON AND DynamicFilter [ empty ], pruning_predicate=(l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(100),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(1100),15,2 OR l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(1000),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(2000),15,2 OR l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(2000),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(3000),15,2) AND (l_shipmode_null_count@6 != row_count@2 AND l_shipmode_min@4 <= AIR AND AIR <= l_shipmode_max@5 OR l_shipmode_null_count@6 != row_count@2 AND l_shipmode_min@4 <= AIR REG AND AIR REG <= l_shipmode_max@5) AND l_shipinstruct_null_count@9 != row_count@2 AND l_shipinstruct_min@7 <= DELIVER IN PERSON AND DELIVER IN PERSON <= l_shipinstruct_max@8, required_guarantees=[l_shipinstruct in (DELIVER IN PERSON), l_shipmode in (AIR, AIR REG)] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: (p_brand@1 = Brand#12 AND p_container@3 IN ([Literal { value: Utf8View("SM CASE"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([Literal { value: Utf8View("MED BAG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([Literal { value: Utf8View("LG CASE"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 15) AND p_size@2 >= 1 + │ FilterExec: (p_brand@1 = Brand#12 AND p_container@3 IN ([SM CASE, SM BOX, SM PACK, SM PKG]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([MED BAG, MED BOX, MED PKG, MED PACK]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([LG CASE, LG BOX, LG PACK, LG PKG]) AND p_size@2 <= 15) AND p_size@2 >= 1 │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet, /testdata/tpch/plan_sf0.02/part/10.parquet, /testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet, /testdata/tpch/plan_sf0.02/part/13.parquet, /testdata/tpch/plan_sf0.02/part/14.parquet], [/testdata/tpch/plan_sf0.02/part/15.parquet, /testdata/tpch/plan_sf0.02/part/16.parquet, /testdata/tpch/plan_sf0.02/part/2.parquet], [/testdata/tpch/plan_sf0.02/part/3.parquet, /testdata/tpch/plan_sf0.02/part/4.parquet, /testdata/tpch/plan_sf0.02/part/5.parquet], [/testdata/tpch/plan_sf0.02/part/6.parquet, /testdata/tpch/plan_sf0.02/part/7.parquet, /testdata/tpch/plan_sf0.02/part/8.parquet], ...]}, projection=[p_partkey, p_brand, p_size, p_container], file_type=parquet, predicate=(p_brand@1 = Brand#12 AND p_container@3 IN ([Literal { value: Utf8View("SM CASE"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([Literal { value: Utf8View("MED BAG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([Literal { value: Utf8View("LG CASE"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 15) AND p_size@2 >= 1, pruning_predicate=(p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#12 AND Brand#12 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM CASE AND SM CASE <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM BOX AND SM BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM PACK AND SM PACK <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM PKG AND SM PKG <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 5 OR p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BAG AND MED BAG <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED PKG AND MED PKG <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED PACK AND MED PACK <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 10 OR p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#34 AND Brand#34 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG CASE AND LG CASE <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG BOX AND LG BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG PACK AND LG PACK <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG PKG AND LG PKG <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 15) AND p_size_null_count@8 != row_count@3 AND p_size_max@9 >= 1, required_guarantees=[p_brand in (Brand#12, Brand#23, Brand#34), p_container in (LG BOX, LG CASE, LG PACK, LG PKG, MED BAG, MED BOX, MED PACK, MED PKG, SM BOX, SM CASE, SM PACK, SM PKG)] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet, /testdata/tpch/plan_sf0.02/part/10.parquet, /testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet, /testdata/tpch/plan_sf0.02/part/13.parquet, /testdata/tpch/plan_sf0.02/part/14.parquet], [/testdata/tpch/plan_sf0.02/part/15.parquet, /testdata/tpch/plan_sf0.02/part/16.parquet, /testdata/tpch/plan_sf0.02/part/2.parquet], [/testdata/tpch/plan_sf0.02/part/3.parquet, /testdata/tpch/plan_sf0.02/part/4.parquet, /testdata/tpch/plan_sf0.02/part/5.parquet], [/testdata/tpch/plan_sf0.02/part/6.parquet, /testdata/tpch/plan_sf0.02/part/7.parquet, /testdata/tpch/plan_sf0.02/part/8.parquet], ...]}, projection=[p_partkey, p_brand, p_size, p_container], file_type=parquet, predicate=(p_brand@1 = Brand#12 AND p_container@3 IN ([SM CASE, SM BOX, SM PACK, SM PKG]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([MED BAG, MED BOX, MED PKG, MED PACK]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([LG CASE, LG BOX, LG PACK, LG PKG]) AND p_size@2 <= 15) AND p_size@2 >= 1, pruning_predicate=(p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#12 AND Brand#12 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM CASE AND SM CASE <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM BOX AND SM BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM PACK AND SM PACK <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM PKG AND SM PKG <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 5 OR p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BAG AND MED BAG <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED PKG AND MED PKG <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED PACK AND MED PACK <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 10 OR p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#34 AND Brand#34 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG CASE AND LG CASE <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG BOX AND LG BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG PACK AND LG PACK <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG PKG AND LG PKG <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 15) AND p_size_null_count@8 != row_count@3 AND p_size_max@9 >= 1, required_guarantees=[p_brand in (Brand#12, Brand#23, Brand#34), p_container in (LG BOX, LG CASE, LG PACK, LG PKG, MED BAG, MED BOX, MED PACK, MED PKG, SM BOX, SM CASE, SM PACK, SM PKG)] └────────────────────────────────────────────────── - "#); + "); Ok(()) } @@ -935,7 +936,7 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[s_suppkey@1, s_name@2, s_address@3] │ CoalescePartitionsExec │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_nationkey], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ps_partkey@0, l_partkey@1), (ps_suppkey@1, l_suppkey@2)], filter=CAST(ps_availqty@0 AS Float64) > Float64(0.5) * sum(lineitem.l_quantity)@1, projection=[ps_suppkey@1] │ CoalescePartitionsExec @@ -967,7 +968,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01, projection=[l_partkey@0, l_suppkey@1, l_quantity@2] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01, required_guarantees=[] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01, required_guarantees=[] └────────────────────────────────────────────────── "); Ok(()) @@ -1008,10 +1009,10 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_name@1, s_nationkey@2, l_orderkey@3, l_suppkey@4] │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_nationkey], file_type=parquet + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@3 > l_commitdate@2 AND DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@3 > l_commitdate@2 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_suppkey], file_type=parquet │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1] @@ -1036,7 +1037,7 @@ mod tests { #[tokio::test] async fn test_tpch_22() -> Result<(), Box> { let plan = test_tpch_query(22).await?; - assert_snapshot!(plan, @r#" + assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [cntrycode@0 ASC NULLS LAST] │ [Stage 4] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 @@ -1051,31 +1052,32 @@ mod tests { ┌───── Stage 3 ── Tasks: t0:[p0..p17] │ RepartitionExec: partitioning=Hash([cntrycode@0], 18), input_partitions=6 │ AggregateExec: mode=Partial, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)] - │ ProjectionExec: expr=[substr(c_phone@0, 1, 2) as cntrycode, c_acctbal@1 as c_acctbal] - │ NestedLoopJoinExec: join_type=Inner, filter=CAST(c_acctbal@0 AS Decimal128(19, 6)) > avg(customer.c_acctbal)@1, projection=[c_phone@1, c_acctbal@2] + │ ProjectionExec: expr=[substr(c_phone@1, 1, 2) as cntrycode, c_acctbal@2 as c_acctbal] + │ NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 > avg(customer.c_acctbal)@0, projection=[avg(customer.c_acctbal)@0, c_phone@1, c_acctbal@2] │ AggregateExec: mode=Final, gby=[], aggr=[avg(customer.c_acctbal)] │ CoalescePartitionsExec │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(c_custkey@0, o_custkey@0)], projection=[c_phone@1, c_acctbal@2] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_custkey], file_type=parquet + │ ProjectionExec: expr=[c_phone@0 as c_phone, c_acctbal@1 as c_acctbal, CAST(c_acctbal@1 AS Decimal128(19, 6)) as join_proj_push_down_1] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(c_custkey@0, o_custkey@0)], projection=[c_phone@1, c_acctbal@2] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_custkey], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ AggregateExec: mode=Partial, gby=[], aggr=[avg(customer.c_acctbal)] │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN ([Literal { value: Utf8View("13"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("31"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("23"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("29"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("30"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("18"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("17"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]), projection=[c_acctbal@1] + │ FilterExec: c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]), projection=[c_acctbal@1] │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_phone, c_acctbal], file_type=parquet, predicate=c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN ([Literal { value: Utf8View("13"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("31"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("23"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("29"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("30"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("18"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("17"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]), pruning_predicate=c_acctbal_null_count@1 != row_count@2 AND c_acctbal_max@0 > Some(0),15,2, required_guarantees=[] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_phone, c_acctbal], file_type=parquet, predicate=c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]), pruning_predicate=c_acctbal_null_count@1 != row_count@2 AND c_acctbal_max@0 > Some(0),15,2, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: substr(c_phone@1, 1, 2) IN ([Literal { value: Utf8View("13"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("31"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("23"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("29"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("30"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("18"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("17"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) + │ FilterExec: substr(c_phone@1, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]) │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_phone, c_acctbal], file_type=parquet, predicate=substr(c_phone@1, 1, 2) IN ([Literal { value: Utf8View("13"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("31"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("23"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("29"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("30"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("18"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("17"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_phone, c_acctbal], file_type=parquet, predicate=substr(c_phone@1, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]) └────────────────────────────────────────────────── - "#); + "); Ok(()) }