diff --git a/.gitignore b/.gitignore index c5731cc..d883831 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,4 @@ target/ .idea # Generated TPCH data (not tracked) -integration-utils/data/ \ No newline at end of file +integration-utils/data/*.parquet diff --git a/Cargo.lock b/Cargo.lock index e88a04f..9d41f79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,12 +1,12 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" dependencies = [ "gimli", ] @@ -70,11 +70,32 @@ dependencies = [ "libc", ] +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + [[package]] name = "anyhow" -version = "1.0.99" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" @@ -187,6 +208,25 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-flight" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c8b0ba0784d56bc6266b79f5de7a24b47024e7b3a0045d2ad4df3d9b686099f" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ipc", + "arrow-schema", + "base64", + "bytes", + "futures", + "prost", + "prost-types", + "tonic", +] + [[package]] name = "arrow-ipc" version = "56.2.0" @@ -200,6 +240,7 @@ dependencies = [ "arrow-select", "flatbuffers", "lz4_flex", + "zstd", ] [[package]] @@ -291,6 +332,23 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "async-compression" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" +dependencies = [ + "bzip2 0.5.2", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "xz2", + "zstd", + "zstd-safe", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -299,7 +357,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] @@ -317,17 +375,71 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "axum" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18ed336352031311f4e0b4dd2ff392d4fbb370777c9d18d7fc9d7359f73871" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59446ce19cd142f8833f856eb31f3eb097812d1479ab224f54d72428ca21ea22" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" -version = "0.3.75" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" dependencies = [ "addr2line", "cfg-if", @@ -335,7 +447,7 @@ dependencies = [ "miniz_oxide", "object", "rustc-demangle", - "windows-targets", + "windows-link", ] [[package]] @@ -357,12 +469,49 @@ dependencies = [ "num-traits", ] +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + +[[package]] +name = "blake3" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "brotli" version = "8.0.2" @@ -402,11 +551,39 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "bzip2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +dependencies = [ + "bzip2-sys", +] + +[[package]] +name = "bzip2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea8dcd42434048e4f7a304411d9273a411f647446c1234a65ce0554923f4cff" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "cc" -version = "1.2.37" +version = "1.2.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65193589c6404eb80b450d618eaf9a2cafaaafd57ecce47370519ef674a7bd44" +checksum = "e1d05d92f4b1fd76aad469d46cdd858ca761576082cd37df81416691e50199fb" dependencies = [ "find-msvc-tools", "jobserver", @@ -427,7 +604,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" dependencies = [ "iana-time-zone", + "js-sys", "num-traits", + "wasm-bindgen", "windows-link", ] @@ -441,6 +620,21 @@ dependencies = [ "phf", ] +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags 1.3.2", + "strsim", + "textwrap", + "unicode-width 0.1.14", + "vec_map", +] + [[package]] name = "comfy-table" version = "7.1.2" @@ -451,7 +645,7 @@ dependencies = [ "crossterm 0.28.1", "strum", "strum_macros", - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -486,12 +680,27 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "core-foundation-sys" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -513,7 +722,7 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" dependencies = [ - "bitflags", + "bitflags 2.9.4", "crossterm_winapi", "libc", "parking_lot", @@ -526,7 +735,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags", + "bitflags 2.9.4", "parking_lot", "rustix 0.38.44", ] @@ -546,6 +755,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "csv" version = "1.3.1" @@ -583,15 +802,16 @@ dependencies = [ [[package]] name = "datafusion" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "481d0c1cad7606cee11233abcdff8eec46e43dd25abda007db6d5d26ae8483c4" +checksum = "4016a135c11820d9c9884a1f7924d5456c563bd3657b7d691a6e7b937a452df7" dependencies = [ "arrow", "arrow-ipc", "arrow-schema", "async-trait", "bytes", + "bzip2 0.6.0", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -617,26 +837,29 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "datafusion-sql", + "flate2", "futures", "itertools", "log", "object_store", "parking_lot", "parquet", - "rand", + "rand 0.9.2", "regex", "sqlparser", "tempfile", "tokio", "url", "uuid", + "xz2", + "zstd", ] [[package]] name = "datafusion-catalog" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d70327e81ab3a1f5832d8b372d55fa607851d7cea6d1f8e65ff0c98fcc32d222" +checksum = "1721d3973afeb8a0c3f235a79101cc61e4a558dd3f02fdc9ae6c61e882e544d9" dependencies = [ "arrow", "async-trait", @@ -660,9 +883,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "268819e6bb20ba70a664abddc20deac604f30d3267f8c91847064542a8c0720c" +checksum = "44841d3efb0c89c6a5ac6fde5ac61d4f2474a2767f170db6d97300a8b4df8904" dependencies = [ "arrow", "async-trait", @@ -683,9 +906,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "054873d5563f115f83ef4270b560ac2ce4de713905e825a40cac49d6ff348254" +checksum = "eabb89b9d1ea8198d174b0838b91b40293b780261d694d6ac59bd20c38005115" dependencies = [ "ahash", "arrow", @@ -700,6 +923,7 @@ dependencies = [ "object_store", "parquet", "paste", + "recursive", "sqlparser", "tokio", "web-time", @@ -707,9 +931,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a1d1bc69aaaadb8008b65329ed890b33e845dc063225c190f77b20328fbe1d" +checksum = "f03fe3936f978fe8e76776d14ad8722e33843b01d81d11707ca72d54d2867787" dependencies = [ "futures", "log", @@ -718,13 +942,15 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d855160469020982880fd9bd0962e033d2f4728f56f85a83d8c90785638b6519" +checksum = "4543216d2f4fc255780a46ae9e062e50c86ac23ecab6718cc1ba3fe4a8d5a8f2" dependencies = [ "arrow", + "async-compression", "async-trait", "bytes", + "bzip2 0.6.0", "chrono", "datafusion-common", "datafusion-common-runtime", @@ -735,23 +961,27 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", + "flate2", "futures", "glob", "itertools", "log", "object_store", "parquet", - "rand", + "rand 0.9.2", "tempfile", "tokio", + "tokio-util", "url", + "xz2", + "zstd", ] [[package]] name = "datafusion-datasource-csv" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ec3aa7575378d23aae96b955b5233bea6f9d461648174f6ccc8f3c160f2b7a7" +checksum = "8ab662d4692ca5929ce32eb609c6c8a741772537d98363b3efb3bc68148cd530" dependencies = [ "arrow", "async-trait", @@ -774,9 +1004,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00cfb8f33e2864eeb3188b6818acf5546d56a5a487d423cce9b684a554caabfa" +checksum = "7dad4492ba9a2fca417cb211f8f05ffeb7f12a1f0f8e5bdcf548c353ff923779" dependencies = [ "arrow", "async-trait", @@ -799,9 +1029,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3bfb48fb4ff42ac1485a12ea56434eaab53f7da8f00b2443b1a3d35a0b6d10" +checksum = "2925432ce04847cc09b4789a53fc22b0fdf5f2e73289ad7432759d76c6026e9e" dependencies = [ "arrow", "async-trait", @@ -826,21 +1056,48 @@ dependencies = [ "object_store", "parking_lot", "parquet", - "rand", + "rand 0.9.2", "tokio", ] +[[package]] +name = "datafusion-distributed" +version = "0.1.0" +source = "git+https://github.com/geoffreyclaude/datafusion-distributed?branch=feat%2Ftracing#78ad4895fc8b6f3679ad1b87df407c6f1d62f2b2" +dependencies = [ + "arrow-flight", + "async-trait", + "bytes", + "chrono", + "dashmap", + "datafusion", + "datafusion-proto", + "delegate", + "futures", + "http", + "itertools", + "object_store", + "pin-project", + "prost", + "rand 0.8.5", + "tokio", + "tonic", + "tower", + "url", + "uuid", +] + [[package]] name = "datafusion-doc" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fbf41013cf55c2369b5229594898e8108c8a1beeb49d97feb5e0cce9933eb8f" +checksum = "b71f8c2c0d5c57620003c3bf1ee577b738404a7fd9642f6cf73d10e44ffaa70f" [[package]] name = "datafusion-execution" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fd0c1ffe3885687758f985ed548184bf63b17b2a7a5ae695de422ad6432118" +checksum = "aa51cf4d253927cb65690c05a18e7720cdda4c47c923b0dd7d641f7fcfe21b14" dependencies = [ "arrow", "async-trait", @@ -851,16 +1108,16 @@ dependencies = [ "log", "object_store", "parking_lot", - "rand", + "rand 0.9.2", "tempfile", "url", ] [[package]] name = "datafusion-expr" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c4fe6411218a9dab656437b1e69b00a470a7a2d7db087867a366c145eb164a7" +checksum = "4a347435cfcd1de0498c8410d32e0b1fc3920e198ce0378f8e259da717af9e0f" dependencies = [ "arrow", "async-trait", @@ -873,15 +1130,16 @@ dependencies = [ "datafusion-physical-expr-common", "indexmap", "paste", + "recursive", "serde_json", "sqlparser", ] [[package]] name = "datafusion-expr-common" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a45bee7d2606bfb41ceb1d904ba7cecf69bd5a6f8f3e6c57c3f5a83d84bdd97" +checksum = "4e73951bdf1047d7af212bb11310407230b4067921df648781ae7f7f1241e87e" dependencies = [ "arrow", "datafusion-common", @@ -892,13 +1150,15 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7e1c532ff9d14f291160bca23e55ffd4899800301dd2389786c2f02d76904a" +checksum = "a3b181e79552d764a2589910d1e0420ef41b07ab97c3e3efdbce612b692141e7" dependencies = [ "arrow", "arrow-buffer", "base64", + "blake2", + "blake3", "chrono", "datafusion-common", "datafusion-doc", @@ -909,17 +1169,19 @@ dependencies = [ "hex", "itertools", "log", - "rand", + "md-5", + "rand 0.9.2", "regex", + "sha2", "unicode-segmentation", "uuid", ] [[package]] name = "datafusion-functions-aggregate" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05d47426645aef1e73b1a034c75ab2401bc504175feb191accbe211ec24a342" +checksum = "b7e8cfb3b3f9e48e756939c85816b388264bed378d166a993fb265d800e1c83c" dependencies = [ "ahash", "arrow", @@ -938,9 +1200,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c99f648b2b1743de0c1c19eef07e8cc5a085237f172b2e20bf6934e0a804e4" +checksum = "9501537e235e4e86828bc8bf4e22968c1514c2cb4c860b7c7cf7dc99e172d43c" dependencies = [ "ahash", "arrow", @@ -951,9 +1213,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4227782023f4fb68d3d5c5eb190665212f43c9a0b437553e4b938b379aff6cf6" +checksum = "6cbc3ecce122389530af091444e923f2f19153c38731893f5b798e19a46fbf86" dependencies = [ "arrow", "arrow-ord", @@ -973,9 +1235,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d902b1769f69058236e89f04f3bff2cf62f24311adb7bf3c6c3e945c9451076" +checksum = "a8ad370763644d6626b15900fe2268e7d55c618fadf5cff3a7f717bb6fb50ec1" dependencies = [ "arrow", "async-trait", @@ -989,9 +1251,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8ee43974c92eb9920fe8e97e0fab48675e93b062abcb48bef4c1d4305b6ee4" +checksum = "44b14fc52c77461f359d1697826a4373c7887a6adfca94eedc81c35decd0df9f" dependencies = [ "arrow", "datafusion-common", @@ -1007,9 +1269,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e149d36cdd44fb425dc815c5fac55025aa9a592dd65cb3c421881096292c02" +checksum = "851c80de71ff8bc9be7f8478f26e8060e25cab868a36190c4ebdaacc72ceade1" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1017,20 +1279,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07c9faa0cdefb6e6e756482b846397b5c2d84d369e30b009472b9ab9b1430fbd" +checksum = "386208ac4f475a099920cdbe9599188062276a09cb4c3f02efdc54e0c015ab14" dependencies = [ "datafusion-expr", "quote", - "syn", + "syn 2.0.106", ] [[package]] name = "datafusion-optimizer" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f16a4f7059302ad1de6e97ab0eebb5c34405917b1f80806a30a66e38ad118251" +checksum = "b20ff1cec8c23fbab8523e2937790fb374b92d3b273306a64b7d8889ff3b8614" dependencies = [ "arrow", "chrono", @@ -1041,15 +1303,16 @@ dependencies = [ "indexmap", "itertools", "log", + "recursive", "regex", "regex-syntax", ] [[package]] name = "datafusion-physical-expr" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10bb87a605d8ce9672d5347c0293c12211b0c03923fc12fbdc665fe76e6f9e01" +checksum = "945659046d27372e38e8a37927f0b887f50846202792063ad6b197c6eaf9fb5b" dependencies = [ "ahash", "arrow", @@ -1085,9 +1348,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "845eb44ef1e04d2a15c6d955cb146b40a41814a7be4377f0a541857d3e257d6f" +checksum = "218d60e94d829d8a52bf50e694f2f567313508f0c684af4954def9f774ce3518" dependencies = [ "ahash", "arrow", @@ -1099,9 +1362,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b9b648ee2785722c79eae366528e52e93ece6808aef9297cf8e5521de381da" +checksum = "f96a93ebfd35cc52595e85c3100730a5baa6def39ff5390d6f90d2f3f89ce53f" dependencies = [ "arrow", "datafusion-common", @@ -1114,13 +1377,14 @@ dependencies = [ "datafusion-pruning", "itertools", "log", + "recursive", ] [[package]] name = "datafusion-physical-plan" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e6688d17b78104e169d7069749832c20ff50f112be853d2c058afe46c889064" +checksum = "3f6516a95911f763f05ec29bddd6fe987a0aa987409c213eac12faa5db7f3c9c" dependencies = [ "ahash", "arrow", @@ -1147,11 +1411,38 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-proto" +version = "50.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca714dff69fe3de2901ec64ec3dba8d0623ae583f6fae3c6fa57355d7882017" +dependencies = [ + "arrow", + "chrono", + "datafusion", + "datafusion-common", + "datafusion-expr", + "datafusion-proto-common", + "object_store", + "prost", +] + +[[package]] +name = "datafusion-proto-common" +version = "50.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7b628ba0f7bd1fa9565f80b19a162bcb3cbc082bbc42b29c4619760621f4e32" +dependencies = [ + "arrow", + "datafusion-common", + "prost", +] + [[package]] name = "datafusion-pruning" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a893a46c56f5f190085e13949eb8ec163672c7ec2ac33bdb82c84572e71ca73" +checksum = "40befe63ab3bd9f3b05d02d13466055aa81876ad580247b10bdde1ba3782cebb" dependencies = [ "arrow", "arrow-schema", @@ -1167,9 +1458,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8b62684c7a1db6121a8c83100209cffa1e664a8d9ced87e1a32f8cdc2fff3c2" +checksum = "26aa059f478e6fa31158e80e4685226490b39f67c2e357401e26da84914be8b2" dependencies = [ "arrow", "async-trait", @@ -1191,9 +1482,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "50.0.0" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09cff94b8242843e1da5d069e9d2cfc53807f1f00b1c0da78c297f47c21456e" +checksum = "ea3ce7cb3c31bfc6162026f6f4b11eb5a3a83c8a6b88d8b9c529ddbe97d53525" dependencies = [ "arrow", "bigdecimal", @@ -1201,6 +1492,7 @@ dependencies = [ "datafusion-expr", "indexmap", "log", + "recursive", "regex", "sqlparser", ] @@ -1218,7 +1510,7 @@ dependencies = [ "tokio", "tracing", "tracing-futures", - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -1243,6 +1535,8 @@ dependencies = [ "datafusion", "insta", "integration-utils", + "opentelemetry", + "opentelemetry_sdk", "serde_json", "tokio", "tracing", @@ -1257,7 +1551,18 @@ checksum = "6178a82cf56c836a3ba61a7935cdb1c49bfaa6fa4327cd5bf554a503087de26b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", ] [[package]] @@ -1268,7 +1573,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] @@ -1296,7 +1601,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -1307,9 +1612,9 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "find-msvc-tools" -version = "0.1.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" +checksum = "0399f9d26e5191ce32c498bebd31e7a3ceabc2745f0ac54af3f335126c3f24b3" [[package]] name = "fixedbitset" @@ -1319,19 +1624,19 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flatbuffers" -version = "25.2.10" +version = "25.9.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1" +checksum = "09b6620799e7340ebd9968d2e0708eb82cf1971e9a16821e2091b6d6e475eed5" dependencies = [ - "bitflags", + "bitflags 2.9.4", "rustc_version", ] [[package]] name = "flate2" -version = "1.1.2" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" +checksum = "dc5a4e564e38c699f2880d3fda590bedc2e69f3f84cd48b457bd892ce61d0aa9" dependencies = [ "crc32fast", "libz-rs-sys", @@ -1415,7 +1720,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] @@ -1448,6 +1753,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.16" @@ -1470,14 +1785,14 @@ dependencies = [ "cfg-if", "libc", "r-efi", - "wasi 0.14.5+wasi-0.2.4", + "wasi 0.14.7+wasi-0.2.4", ] [[package]] name = "gimli" -version = "0.31.1" +version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" [[package]] name = "glob" @@ -1506,13 +1821,14 @@ dependencies = [ [[package]] name = "half" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +checksum = "e54c115d4f30f52c67202f079c5f9d8b49db4691f460fdb0b4c2e838261b2ba5" dependencies = [ "cfg-if", "crunchy", "num-traits", + "zerocopy", ] [[package]] @@ -1531,8 +1847,6 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "allocator-api2", - "equivalent", "foldhash", ] @@ -1542,12 +1856,30 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hex" version = "0.4.3" @@ -1594,6 +1926,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "humantime" version = "2.3.0" @@ -1614,6 +1952,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "pin-utils", @@ -1637,9 +1976,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" +checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" dependencies = [ "base64", "bytes", @@ -1653,7 +1992,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.0", "tokio", "tower-service", "tracing", @@ -1792,12 +2131,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.11.1" +version = "2.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206a8042aec68fa4a62e8d3f7aa4ceb508177d9324faf261e1959e495b7a1921" +checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" dependencies = [ "equivalent", - "hashbrown 0.15.5", + "hashbrown 0.16.0", ] [[package]] @@ -1837,11 +2176,27 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" name = "integration-utils" version = "50.0.2" dependencies = [ + "arrow", + "arrow-flight", + "async-trait", + "dashmap", "datafusion", + "datafusion-distributed", "datafusion-tracing", + "futures", + "http", + "hyper-util", "instrumented-object-store", "object_store", + "opentelemetry", + "pin-project-lite", + "structopt", + "tokio", + "tokio-stream", + "tonic", + "tower", "tracing", + "tracing-opentelemetry", "url", ] @@ -1851,7 +2206,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" dependencies = [ - "bitflags", + "bitflags 2.9.4", "cfg-if", "libc", ] @@ -1899,9 +2254,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.78" +version = "0.3.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0b063578492ceec17683ef2f8c5e89121fbd0b172cbc280635ab7567db2738" +checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" dependencies = [ "once_cell", "wasm-bindgen", @@ -1915,9 +2270,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lexical-core" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b765c31809609075565a70b4b71402281283aeda7ecaf4818ac14a7b2ade8958" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" dependencies = [ "lexical-parse-float", "lexical-parse-integer", @@ -1928,60 +2283,59 @@ dependencies = [ [[package]] name = "lexical-parse-float" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" dependencies = [ "lexical-parse-integer", "lexical-util", - "static_assertions", ] [[package]] name = "lexical-parse-integer" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" dependencies = [ "lexical-util", - "static_assertions", ] [[package]] name = "lexical-util" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3" -dependencies = [ - "static_assertions", -] +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" [[package]] name = "lexical-write-float" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5afc668a27f460fb45a81a757b6bf2f43c2d7e30cb5a2dcd3abf294c78d62bd" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" dependencies = [ "lexical-util", "lexical-write-integer", - "static_assertions", ] [[package]] name = "lexical-write-integer" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629ddff1a914a836fb245616a7888b62903aae58fa771e1d83943035efa0f978" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" dependencies = [ "lexical-util", - "static_assertions", ] +[[package]] +name = "libbz2-rs-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" + [[package]] name = "libc" -version = "0.2.175" +version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" [[package]] name = "libm" @@ -2018,11 +2372,10 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "lock_api" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" dependencies = [ - "autocfg", "scopeguard", ] @@ -2041,6 +2394,17 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "matchers" version = "0.2.0" @@ -2050,11 +2414,33 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" -version = "2.7.5" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" @@ -2063,6 +2449,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -2161,18 +2548,18 @@ dependencies = [ [[package]] name = "object" -version = "0.36.7" +version = "0.37.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" dependencies = [ "memchr", ] [[package]] name = "object_store" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efc4f07659e11cd45a341cd24d71e683e3be65d9ff1f8150061678fe60437496" +checksum = "4c1be0c6c22ec0817cdc77d3842f721a17fd30ab6965001415b5402a74e6b740" dependencies = [ "async-trait", "bytes", @@ -2267,7 +2654,7 @@ dependencies = [ "futures-util", "opentelemetry", "percent-encoding", - "rand", + "rand 0.9.2", "serde_json", "thiserror", "tokio", @@ -2285,9 +2672,9 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ "lock_api", "parking_lot_core", @@ -2295,22 +2682,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.11" +version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets", + "windows-link", ] [[package]] name = "parquet" -version = "56.1.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b56b41d1bd36aae415e42f91cae70ee75cf6cba74416b14dce3e958d5990ec" +checksum = "f0dbd48ad52d7dccf8ea1b90a3ddbfaea4f69878dd7683e51c507d4bc52b5b27" dependencies = [ "ahash", "arrow-array", @@ -2327,7 +2714,7 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.15.5", + "hashbrown 0.16.0", "lz4_flex", "num", "num-bigint", @@ -2357,9 +2744,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "petgraph" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.5", @@ -2402,7 +2789,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] @@ -2441,6 +2828,30 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" version = "1.0.101" @@ -2470,14 +2881,32 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn", + "syn 2.0.106", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "psm" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e66fcd288453b748497d8fb18bccc83a16b0518e3906d4b8df0a8d42d93dbb1c" +dependencies = [ + "cc", ] [[package]] name = "quote" -version = "1.0.40" +version = "1.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" dependencies = [ "proc-macro2", ] @@ -2488,14 +2917,35 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ - "rand_chacha", - "rand_core", + "rand_chacha 0.9.0", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", ] [[package]] @@ -2505,7 +2955,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.16", ] [[package]] @@ -2517,20 +2976,40 @@ dependencies = [ "getrandom 0.3.3", ] +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn 2.0.106", +] + [[package]] name = "redox_syscall" -version = "0.5.17" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.9.4", ] [[package]] name = "regex" -version = "1.11.2" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +checksum = "8b5288124840bee7b386bc413c487869b360b2b4ec421ea56425128692f2a82c" dependencies = [ "aho-corasick", "memchr", @@ -2540,9 +3019,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +checksum = "833eb9ce86d40ef33cb1306d8accf7bc8ec2bfea4355cbdebb3df68b40925cad" dependencies = [ "aho-corasick", "memchr", @@ -2624,7 +3103,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags", + "bitflags 2.9.4", "errno", "libc", "linux-raw-sys 0.4.15", @@ -2637,11 +3116,11 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" dependencies = [ - "bitflags", + "bitflags 2.9.4", "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2685,9 +3164,9 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "serde" -version = "1.0.223" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a505d71960adde88e293da5cb5eda57093379f64e61cf77bf0e6a63af07a7bac" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", "serde_derive", @@ -2695,22 +3174,22 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.223" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20f57cbd357666aa7b3ac84a90b4ea328f1d4ddb6772b430caa5d9e1309bb9e9" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.223" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d428d07faf17e306e699ec1e91996e5a165ba5d6bce5b5155173e91a8a01a56" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] @@ -2738,6 +3217,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2753,6 +3243,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook-registry" +version = "1.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" +dependencies = [ + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "simdutf8" version = "0.1.5" @@ -2789,6 +3294,16 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socket2" version = "0.6.0" @@ -2806,6 +3321,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec4b661c54b1e4b603b37873a18c59920e4c51ea8ea2cf527d925424dbd4437c" dependencies = [ "log", + "recursive", "sqlparser_derive", ] @@ -2817,20 +3333,57 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] name = "stable_deref_trait" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] -name = "static_assertions" -version = "1.1.0" +name = "stacker" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +checksum = "e1f8b29fb42aafcea4edeeb6b2f2d7ecd0d969c48b4cf0d2e64aafc471dd6e59" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.59.0", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "structopt" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" +dependencies = [ + "clap", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" +dependencies = [ + "heck 0.3.3", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] [[package]] name = "strum" @@ -2844,11 +3397,28 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "rustversion", - "syn", + "syn 2.0.106", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", ] [[package]] @@ -2879,40 +3449,49 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] name = "tempfile" -version = "3.22.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84fa4d11fadde498443cca10fd3ac23c951f0dc59e080e9f4b93d4df4e4eea53" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" dependencies = [ "fastrand", "getrandom 0.3.3", "once_cell", "rustix 1.1.2", - "windows-sys 0.59.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width 0.1.14", ] [[package]] name = "thiserror" -version = "2.0.16" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.16" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] @@ -2965,9 +3544,11 @@ dependencies = [ "io-uring", "libc", "mio", + "parking_lot", "pin-project-lite", + "signal-hook-registry", "slab", - "socket2", + "socket2 0.6.0", "tokio-macros", "windows-sys 0.59.0", ] @@ -2980,7 +3561,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] @@ -3014,8 +3595,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" dependencies = [ "async-trait", + "axum", "base64", "bytes", + "h2", "http", "http-body", "http-body-util", @@ -3025,6 +3608,7 @@ dependencies = [ "percent-encoding", "pin-project", "prost", + "socket2 0.5.10", "tokio", "tokio-stream", "tower", @@ -3058,7 +3642,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ - "bitflags", + "bitflags 2.9.4", "bytes", "futures-util", "http", @@ -3101,7 +3685,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] @@ -3198,6 +3782,12 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + [[package]] name = "unicode-ident" version = "1.0.19" @@ -3212,9 +3802,15 @@ checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" -version = "0.2.1" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" [[package]] name = "untrusted" @@ -3257,6 +3853,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + [[package]] name = "version_check" version = "0.9.5" @@ -3290,27 +3892,27 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasi" -version = "0.14.5+wasi-0.2.4" +version = "0.14.7+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4494f6290a82f5fe584817a676a34b9d6763e8d9d18204009fb31dceca98fd4" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" dependencies = [ "wasip2", ] [[package]] name = "wasip2" -version = "1.0.0+wasi-0.2.4" +version = "1.0.1+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03fa2761397e5bd52002cd7e73110c71af2109aca4e521a9f40473fe685b0a24" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" dependencies = [ "wit-bindgen", ] [[package]] name = "wasm-bindgen" -version = "0.2.101" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e14915cadd45b529bb8d1f343c4ed0ac1de926144b746e2710f9cd05df6603b" +checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" dependencies = [ "cfg-if", "once_cell", @@ -3321,23 +3923,23 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.101" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28d1ba982ca7923fd01448d5c30c6864d0a14109560296a162f80f305fb93bb" +checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" dependencies = [ "bumpalo", "log", "proc-macro2", "quote", - "syn", + "syn 2.0.106", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.51" +version = "0.4.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca85039a9b469b38336411d6d6ced91f3fc87109a2a27b0c197663f5144dffe" +checksum = "7e038d41e478cc73bae0ff9b36c60cff1c98b8f38f8d7e8061e79ee63608ac5c" dependencies = [ "cfg-if", "js-sys", @@ -3348,9 +3950,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.101" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c3d463ae3eff775b0c45df9da45d68837702ac35af998361e2c84e7c5ec1b0d" +checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3358,31 +3960,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.101" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bb4ce89b08211f923caf51d527662b75bdc9c9c7aab40f86dcb9fb85ac552aa" +checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.101" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f143854a3b13752c6950862c906306adb27c7e839f7414cec8fea35beab624c1" +checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.78" +version = "0.3.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e4b637749ff0d92b8fad63aa1f7cff3cbe125fd49c175cd6345e7272638b12" +checksum = "9367c417a924a74cae129e6a2ae3b47fabb1f8995595ab474029da749a8be120" dependencies = [ "js-sys", "wasm-bindgen", @@ -3420,7 +4022,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3431,9 +4033,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.62.0" +version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57fe7168f7de578d2d8a05b07fd61870d2e73b4020e9f49aa00da8471723497c" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ "windows-implement", "windows-interface", @@ -3444,46 +4046,46 @@ dependencies = [ [[package]] name = "windows-implement" -version = "0.60.0" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] name = "windows-interface" -version = "0.59.1" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] name = "windows-link" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] name = "windows-result" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7084dcc306f89883455a206237404d3eaf961e5bd7e0f312f7c91f57eb44167f" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ "windows-link", ] [[package]] name = "windows-strings" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7218c655a553b0bed4426cf54b20d7ba363ef543b52d515b3e48d7fd55318dda" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ "windows-link", ] @@ -3506,6 +4108,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -3572,9 +4183,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "wit-bindgen" -version = "0.45.1" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c573471f125075647d03df72e026074b7203790d41351cd6edc96f46bcccd36" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" [[package]] name = "writeable" @@ -3582,6 +4193,15 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "yoke" version = "0.8.0" @@ -3602,7 +4222,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", "synstructure", ] @@ -3623,7 +4243,7 @@ checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] @@ -3643,7 +4263,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", "synstructure", ] @@ -3677,7 +4297,7 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.106", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 7dfede2..21eabc1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,11 +30,11 @@ resolver = "2" [workspace.package] description = "DataFusion tracing of execution plans" version = "50.0.2" -edition = "2021" +edition = "2024" license = "Apache-2.0" repository = "https://github.com/datafusion-contrib/datafusion-tracing" authors = ["DataDog "] -rust-version = "1.82.0" +rust-version = "1.86.0" [workspace.dependencies] datafusion = { version = "50.0.0", default-features = false } diff --git a/datafusion-tracing/src/instrument_rule.rs b/datafusion-tracing/src/instrument_rule.rs index c43700e..5602b3f 100644 --- a/datafusion-tracing/src/instrument_rule.rs +++ b/datafusion-tracing/src/instrument_rule.rs @@ -20,14 +20,14 @@ use crate::instrumented::InstrumentedExec; use crate::instrumented::SpanCreateFn; use crate::options::InstrumentationOptions; -use datafusion::common::runtime::{set_join_set_tracer, JoinSetTracer}; +use datafusion::common::runtime::{JoinSetTracer, set_join_set_tracer}; use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode}; use datafusion::{ config::ConfigOptions, physical_optimizer::PhysicalOptimizerRule, physical_plan::ExecutionPlan, }; -use futures::future::BoxFuture; use futures::FutureExt; +use futures::future::BoxFuture; use std::any::Any; use std::fmt::Debug; use std::sync::Arc; @@ -68,19 +68,19 @@ impl PhysicalOptimizerRule for InstrumentRule { plan: Arc, _config: &ConfigOptions, ) -> datafusion::error::Result> { - // Iterate over the plan and wrap each node with InstrumentedExec + // Wrap each node in the plan tree with InstrumentedExec. + // Since InstrumentedExec delegates as_any() to its inner node, it's transparent + // for downcasting - other code can still identify wrapped nodes by their actual type. + // + // Note: This will wrap every node, even if the rule is called multiple times on + // the same plan (unusual but possible). The resulting double-wrapping creates + // duplicate telemetry but remains functionally correct. plan.transform(|plan| { - if plan.as_any().downcast_ref::().is_none() { - // Node is not InstrumentedExec; wrap it - Ok(Transformed::yes(Arc::new(InstrumentedExec::new( - plan, - self.span_create_fn.clone(), - &self.options, - )))) - } else { - // Node is already InstrumentedExec; do not wrap again - Ok(Transformed::no(plan)) - } + Ok(Transformed::yes(Arc::new(InstrumentedExec::new( + plan, + self.span_create_fn.clone(), + &self.options, + )))) }) .data() } diff --git a/datafusion-tracing/src/instrumented.rs b/datafusion-tracing/src/instrumented.rs index eb4552d..e4f512a 100644 --- a/datafusion-tracing/src/instrumented.rs +++ b/datafusion-tracing/src/instrumented.rs @@ -31,6 +31,8 @@ use datafusion::{ execution::{SendableRecordBatchStream, TaskContext}, physical_expr::{Distribution, OrderingRequirements}, physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, + PhysicalExpr, PlanProperties, execution_plan::{CardinalityEffect, InvariantLevel}, filter_pushdown::{ ChildPushdownResult, FilterDescription, FilterPushdownPhase, @@ -39,8 +41,6 @@ use datafusion::{ metrics::MetricsSet, projection::ProjectionExec, stream::RecordBatchStreamAdapter, - DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, - PhysicalExpr, PlanProperties, }, }; use delegate::delegate; @@ -50,7 +50,7 @@ use std::{ fmt::{self, Debug}, sync::{Arc, OnceLock}, }; -use tracing::{field, Span}; +use tracing::{Span, field}; use tracing_futures::Instrument; /// Type alias for a function that creates a tracing span. @@ -347,8 +347,9 @@ impl ExecutionPlan for InstrumentedExec { Some(self.with_new_inner(new_inner)) } + /// Delegate to the inner plan for downcasting. fn as_any(&self) -> &dyn Any { - self + self.inner.as_any() } /// Executes the plan for a given partition and context, instrumented with tracing and metrics recording. diff --git a/datafusion-tracing/src/metrics.rs b/datafusion-tracing/src/metrics.rs index b51c3d8..8e0ba0f 100644 --- a/datafusion-tracing/src/metrics.rs +++ b/datafusion-tracing/src/metrics.rs @@ -26,7 +26,7 @@ use pin_project::pin_project; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; -use tracing::{field, Span}; +use tracing::{Span, field}; /// Records execution metrics and automatically logs them via tracing upon completion. /// diff --git a/datafusion-tracing/src/node.rs b/datafusion-tracing/src/node.rs index 753c39f..f454f43 100644 --- a/datafusion-tracing/src/node.rs +++ b/datafusion-tracing/src/node.rs @@ -27,7 +27,7 @@ use pin_project::pin_project; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; -use tracing::{field, Span}; +use tracing::{Span, field}; /// Records the `datafusion.node` span field once execution completes across all /// partitions, when the value is fully qualified. diff --git a/datafusion-tracing/src/preview.rs b/datafusion-tracing/src/preview.rs index 4e76717..ae051c5 100644 --- a/datafusion-tracing/src/preview.rs +++ b/datafusion-tracing/src/preview.rs @@ -165,7 +165,7 @@ impl RecordBatchStream for PreviewRecordingStream { } fn default_preview_fn(batch: &RecordBatch) -> Result { - pretty_format_batches(&[batch.clone()]).map(|b| b.to_string()) + pretty_format_batches(std::slice::from_ref(batch)).map(|b| b.to_string()) } impl PreviewRecorder { diff --git a/datafusion-tracing/src/preview_utils.rs b/datafusion-tracing/src/preview_utils.rs index e896006..606a1d0 100644 --- a/datafusion-tracing/src/preview_utils.rs +++ b/datafusion-tracing/src/preview_utils.rs @@ -141,7 +141,7 @@ mod tests { use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::arrow::error::ArrowError; use datafusion::arrow::record_batch::RecordBatch; - use insta::{assert_snapshot, Settings}; + use insta::{Settings, assert_snapshot}; use std::sync::Arc; fn insta_settings() -> Settings { @@ -206,7 +206,7 @@ mod tests { let description = Arc::new(StringArray::from(vec![ "Paris is renowned as the City of Light, celebrated for its rich history, magnificent architecture, and vibrant arts scene. The city boasts iconic landmarks such as the Eiffel Tower and the Louvre, along with charming streets, quaint cafés, and a deep cultural heritage that continues to inspire artists, writers, and travelers from around the world.", "London is a dynamic and cosmopolitan metropolis that seamlessly blends its storied past with modern innovation. The city offers an array of historical sites, diverse neighborhoods, and world-class museums and theaters. Its bustling markets, green parks, and ever-evolving cultural scene make London a hub of creativity, commerce, and community life.", - "Barcelona is a lively coastal city known for its striking modernist architecture, Mediterranean beaches, and eclectic cultural offerings. From the whimsical creations of Antoni Gaudí to the vibrant street life and renowned culinary delights, Barcelona captivates visitors with its unique blend of historic charm and contemporary energy." + "Barcelona is a lively coastal city known for its striking modernist architecture, Mediterranean beaches, and eclectic cultural offerings. From the whimsical creations of Antoni Gaudí to the vibrant street life and renowned culinary delights, Barcelona captivates visitors with its unique blend of historic charm and contemporary energy.", ])); let city_name = Arc::new(StringArray::from(vec!["Paris", "London", "Barcelona"])); let emojis = Arc::new(StringArray::from(vec![ diff --git a/dev/generate_tpch_parquet.sh b/dev/generate_tpch_parquet.sh index 840c421..d1c2290 100755 --- a/dev/generate_tpch_parquet.sh +++ b/dev/generate_tpch_parquet.sh @@ -33,10 +33,10 @@ if ! command -v tpchgen-cli >/dev/null 2>&1; then exit 1 fi -echo "Generating TPCH Parquet data (SF=0.1) into $OUT_DIR" +echo "Generating TPCH Parquet data (SF=1) into $OUT_DIR" # Generate all tables, single file per table, Parquet format -tpchgen-cli --scale-factor 0.1 --format parquet --output-dir "$OUT_DIR" +tpchgen-cli --scale-factor 1 --format parquet --output-dir "$OUT_DIR" echo "TPCH Parquet generation complete. Files in $OUT_DIR:" ls -1 "$OUT_DIR" | sed 's/^/ - /' diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 555507a..e49c49a 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -35,6 +35,10 @@ workspace = true name = "otlp" path = "otlp.rs" +[[example]] +name = "distributed_otlp" +path = "distributed_otlp.rs" + [dependencies] datafusion = { workspace = true, features = ["parquet", "nested_expressions"] } integration-utils = { path = "../integration-utils" } diff --git a/examples/distributed_otlp.rs b/examples/distributed_otlp.rs new file mode 100644 index 0000000..a96637c --- /dev/null +++ b/examples/distributed_otlp.rs @@ -0,0 +1,239 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025 Datadog, Inc. + +//! # Distributed OTLP Example +//! +//! This example demonstrates OpenTelemetry integration with DataFusion for distributed tracing. +//! It shows how to: +//! +//! - Configure OpenTelemetry with an OTLP exporter +//! - Set up service metadata and tracing layers +//! - Instrument DataFusion query execution for distributed plans +//! - Execute multiple queries with separate contexts +//! +//! ## Prerequisites +//! +//! Before running this example, you'll need an OpenTelemetry collector. For local development, +//! Jaeger is recommended: +//! +//! ```bash +//! docker run --rm --name jaeger \ +//! -p 16686:16686 \ +//! -p 4317:4317 \ +//! jaegertracing/jaeger:2.7.0 +//! ``` +//! +//! After starting Jaeger, you can view traces at http://localhost:16686 +//! +//! ## Running the Example +//! +//! ```bash +//! cargo run --example distributed_otlp +//! ``` +//! +//! This example executes all 22 TPCH benchmark queries with distributed query execution enabled. + +use std::time::Duration; + +use datafusion::{common::internal_datafusion_err, error::Result}; +use integration_utils::{DistributedMode, SessionBuilder, run_traced_query}; +use opentelemetry::{KeyValue, trace::TracerProvider}; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_sdk::{Resource, trace::Sampler}; +use tracing::{Instrument, Level}; +use tracing_subscriber::{Registry, fmt, prelude::*}; + +/// Internal helper macro for generating match arms using repetition. +/// This uses Rust's macro repetition syntax ($(...)*). +macro_rules! generate_query_span_match { + ($i:expr, $mode_name:expr, $query_name:expr, $($num:tt),*) => { + match ($i, $mode_name) { + $( + ($num, "Memory") => tracing::info_span!( + concat!("tpch_query_", stringify!($num), "_memory"), + query = %$query_name, + query_num = $i, + distributed_mode = $mode_name + ), + ($num, "Localhost") => tracing::info_span!( + concat!("tpch_query_", stringify!($num), "_localhost"), + query = %$query_name, + query_num = $i, + distributed_mode = $mode_name + ), + )* + _ => unreachable!("Invalid query number or mode"), + } + }; +} + +/// Macro to generate span creation for all query/mode combinations. +/// This is necessary because tracing span names must be compile-time constants. +/// +/// Usage: create_query_span!(query_num, mode_name, query_name) +macro_rules! create_query_span { + ($i:expr, $mode_name:expr, $query_name:expr) => { + generate_query_span_match!( + $i, + $mode_name, + $query_name, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22 + ) + }; +} + +#[tokio::main] +async fn main() -> Result<()> { + // Initialize tracing infrastructure and obtain a tracer provider. + let tracer_provider = init_tracing()?; + + // Run the example under the root span. + run_distributed_otlp_example().await?; + + // Properly shutdown tracing to ensure all data is flushed. + tracer_provider + .shutdown() + .map_err(|e| internal_datafusion_err!("Tracer shutdown error: {}", e)) +} + +async fn run_distributed_otlp_example() -> Result<()> { + // Test both distributed execution modes + let modes = [ + (DistributedMode::Localhost, "Localhost"), + (DistributedMode::Memory, "Memory"), + ]; + + for (mode, mode_name) in modes { + tracing::info!("Starting TPCH queries with {} distributed mode", mode_name); + + // Loop over all 22 TPCH queries + for i in 1..=22 { + let query_name = format!("tpch/q{}", i); + + // Create a new root span for each query to ensure independent traces. + // This span will be the root of a new trace tree. + // Note: Span names must be compile-time constants, so we use a macro to generate them. + let span = create_query_span!(i, mode_name, query_name); + + // Execute the query within the new root span context. + async { + tracing::info!( + "Running TPCH query: {} in {} mode", + query_name, + mode_name + ); + + // Initialize a distinct DataFusion session context for each query. + let ctx = SessionBuilder::new() + .with_metrics() + .with_preview(5) + .with_compact_preview() + .with_distributed_mode(mode) + .build() + .await?; + + // Run the SQL query with tracing enabled. + run_traced_query(&ctx, &query_name).await + } + .instrument(span) + .await?; + } + + tracing::info!( + "Completed all TPCH queries with {} distributed mode", + mode_name + ); + } + + Ok(()) +} + +/// Initializes OpenTelemetry and tracing infrastructure to enable tracing of query execution. +fn init_tracing() -> Result { + // Set up the global text map propagator for trace context propagation across gRPC boundaries. + // This is essential for distributed tracing to work - without it, worker spans won't be + // linked to parent traces. + opentelemetry::global::set_text_map_propagator( + opentelemetry_sdk::propagation::TraceContextPropagator::new(), + ); + + // Set service metadata for tracing. + let resource = Resource::builder() + .with_attribute(KeyValue::new("service.name", "datafusion-tracing")) + .build(); + + // Configure an OTLP exporter to send tracing data. + let exporter = opentelemetry_otlp::SpanExporter::builder() + .with_tonic() + .with_endpoint("http://localhost:4317") // Endpoint for OTLP collector. + .with_timeout(Duration::from_secs(10)) + .build() + .map_err(|e| internal_datafusion_err!("OTLP exporter error: {}", e))?; + + // Create a tracer provider configured with the exporter and sampling strategy. + let tracer_provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_batch_exporter(exporter) + .with_resource(resource) + .with_sampler(Sampler::AlwaysOn) + .build(); + + // Obtain a tracer instance for recording tracing information. + let tracer = tracer_provider.tracer("datafusion-tracing-query"); + + // Create a telemetry layer using the tracer to collect and filter tracing data at INFO level. + let telemetry_layer = tracing_opentelemetry::layer() + .with_tracer(tracer) + .with_filter(tracing::level_filters::LevelFilter::INFO); + + // Create a formatting layer to output logs to stdout, including thread IDs and names. + let fmt_layer = fmt::layer() + .with_thread_ids(true) + .with_thread_names(true) + .with_writer(std::io::stdout.with_max_level(Level::INFO)); + + // Combine the telemetry and formatting layers into a tracing subscriber and initialize it. + Registry::default() + .with(telemetry_layer) + .with(fmt_layer) + .init(); + + // Return the configured tracer provider + Ok(tracer_provider) +} diff --git a/examples/otlp.rs b/examples/otlp.rs index 01d2c4a..b2aba94 100644 --- a/examples/otlp.rs +++ b/examples/otlp.rs @@ -53,12 +53,12 @@ use std::time::Duration; use datafusion::{common::internal_datafusion_err, error::Result}; -use integration_utils::{init_session, run_traced_query}; -use opentelemetry::{trace::TracerProvider, KeyValue}; +use integration_utils::{SessionBuilder, run_traced_query}; +use opentelemetry::{KeyValue, trace::TracerProvider}; use opentelemetry_otlp::WithExportConfig; -use opentelemetry_sdk::{trace::Sampler, Resource}; -use tracing::{instrument, Level}; -use tracing_subscriber::{fmt, prelude::*, Registry}; +use opentelemetry_sdk::{Resource, trace::Sampler}; +use tracing::{Level, instrument}; +use tracing_subscriber::{Registry, fmt, prelude::*}; // Query to be executed for demonstration purposes. const QUERY_NAME: &str = "tpch_scrabble"; @@ -80,7 +80,13 @@ async fn main() -> Result<()> { #[instrument(level = "info")] async fn run_otlp_example() -> Result<()> { // Initialize the DataFusion session context. - let ctx = init_session(true, true, 5, true).await?; + let ctx = SessionBuilder::new() + .with_object_store_tracing() + .with_metrics() + .with_preview(5) + .with_compact_preview() + .build() + .await?; // Run the SQL query with tracing enabled. run_traced_query(&ctx, QUERY_NAME).await?; diff --git a/instrumented-object-store/src/instrumented_object_store.rs b/instrumented-object-store/src/instrumented_object_store.rs index 47d40ce..6695771 100644 --- a/instrumented-object-store/src/instrumented_object_store.rs +++ b/instrumented-object-store/src/instrumented_object_store.rs @@ -19,17 +19,17 @@ use async_trait::async_trait; use bytes::Bytes; -use futures::stream::BoxStream; use futures::StreamExt; +use futures::stream::BoxStream; use object_store::{ - path::Path, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, - ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, - UploadPart, + GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, + PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, UploadPart, + path::Path, }; use std::fmt::{Display, Formatter}; use std::ops::Range; use std::sync::Arc; -use tracing::{instrument, Span}; +use tracing::{Span, instrument}; use tracing_futures::Instrument; /// Instruments the provided `ObjectStore` with tracing. diff --git a/integration-utils/Cargo.toml b/integration-utils/Cargo.toml index 61a49fd..1a19785 100644 --- a/integration-utils/Cargo.toml +++ b/integration-utils/Cargo.toml @@ -31,9 +31,26 @@ rust-version = { workspace = true } workspace = true [dependencies] +arrow = "56.1" +arrow-flight = "56.1" +async-trait = "0.1" +dashmap = "6.1" datafusion = { workspace = true, features = ["parquet", "nested_expressions"] } +datafusion-distributed = { git = "https://github.com/geoffreyclaude/datafusion-distributed", branch = "feat/tracing" } +# datafusion-distributed = { path = "../../datafusion-distributed" } datafusion-tracing = { workspace = true } +futures = { workspace = true } +http = "1.0" +hyper-util = "0.1" instrumented-object-store = { workspace = true } object_store = { version = "0.12.1", default-features = false } +opentelemetry = { version = "0.30", features = ["trace"] } +pin-project-lite = "0.2" +structopt = "0.3" +tokio = { workspace = true, features = ["full"] } +tokio-stream = "0.1" +tonic = { version = "0.13", features = ["transport"] } +tower = "0.5" tracing = { workspace = true } +tracing-opentelemetry = { version = "0.31" } url = { version = "2.5" } diff --git a/integration-utils/data/weather/result-000000.parquet b/integration-utils/data/weather/result-000000.parquet new file mode 100644 index 0000000..873db32 Binary files /dev/null and b/integration-utils/data/weather/result-000000.parquet differ diff --git a/integration-utils/data/weather/result-000001.parquet b/integration-utils/data/weather/result-000001.parquet new file mode 100644 index 0000000..f870416 Binary files /dev/null and b/integration-utils/data/weather/result-000001.parquet differ diff --git a/integration-utils/data/weather/result-000002.parquet b/integration-utils/data/weather/result-000002.parquet new file mode 100644 index 0000000..1ccd8e0 Binary files /dev/null and b/integration-utils/data/weather/result-000002.parquet differ diff --git a/integration-utils/queries/tpch/q1.sql b/integration-utils/queries/tpch/q1.sql new file mode 100644 index 0000000..a0fcf15 --- /dev/null +++ b/integration-utils/queries/tpch/q1.sql @@ -0,0 +1,21 @@ +select + l_returnflag, + l_linestatus, + sum(l_quantity) as sum_qty, + sum(l_extendedprice) as sum_base_price, + sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, + sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, + avg(l_quantity) as avg_qty, + avg(l_extendedprice) as avg_price, + avg(l_discount) as avg_disc, + count(*) as count_order +from + lineitem +where + l_shipdate <= date '1998-09-02' +group by + l_returnflag, + l_linestatus +order by + l_returnflag, + l_linestatus; \ No newline at end of file diff --git a/integration-utils/queries/tpch/q10.sql b/integration-utils/queries/tpch/q10.sql new file mode 100644 index 0000000..cf45e43 --- /dev/null +++ b/integration-utils/queries/tpch/q10.sql @@ -0,0 +1,31 @@ +select + c_custkey, + c_name, + sum(l_extendedprice * (1 - l_discount)) as revenue, + c_acctbal, + n_name, + c_address, + c_phone, + c_comment +from + customer, + orders, + lineitem, + nation +where + c_custkey = o_custkey + and l_orderkey = o_orderkey + and o_orderdate >= date '1993-10-01' + and o_orderdate < date '1994-01-01' + and l_returnflag = 'R' + and c_nationkey = n_nationkey +group by + c_custkey, + c_name, + c_acctbal, + c_phone, + n_name, + c_address, + c_comment +order by + revenue desc; \ No newline at end of file diff --git a/integration-utils/queries/tpch/q11.sql b/integration-utils/queries/tpch/q11.sql new file mode 100644 index 0000000..c23ed1c --- /dev/null +++ b/integration-utils/queries/tpch/q11.sql @@ -0,0 +1,27 @@ +select + ps_partkey, + sum(ps_supplycost * ps_availqty) as value +from + partsupp, + supplier, + nation +where + ps_suppkey = s_suppkey + and s_nationkey = n_nationkey + and n_name = 'GERMANY' +group by + ps_partkey having + sum(ps_supplycost * ps_availqty) > ( + select + sum(ps_supplycost * ps_availqty) * 0.0001 + from + partsupp, + supplier, + nation + where + ps_suppkey = s_suppkey + and s_nationkey = n_nationkey + and n_name = 'GERMANY' + ) +order by + value desc; \ No newline at end of file diff --git a/integration-utils/queries/tpch/q12.sql b/integration-utils/queries/tpch/q12.sql new file mode 100644 index 0000000..f8e6d96 --- /dev/null +++ b/integration-utils/queries/tpch/q12.sql @@ -0,0 +1,30 @@ +select + l_shipmode, + sum(case + when o_orderpriority = '1-URGENT' + or o_orderpriority = '2-HIGH' + then 1 + else 0 + end) as high_line_count, + sum(case + when o_orderpriority <> '1-URGENT' + and o_orderpriority <> '2-HIGH' + then 1 + else 0 + end) as low_line_count +from + lineitem + join + orders + on + l_orderkey = o_orderkey +where + l_shipmode in ('MAIL', 'SHIP') + and l_commitdate < l_receiptdate + and l_shipdate < l_commitdate + and l_receiptdate >= date '1994-01-01' + and l_receiptdate < date '1995-01-01' +group by + l_shipmode +order by + l_shipmode; \ No newline at end of file diff --git a/integration-utils/queries/tpch/q13.sql b/integration-utils/queries/tpch/q13.sql new file mode 100644 index 0000000..4bfe8c3 --- /dev/null +++ b/integration-utils/queries/tpch/q13.sql @@ -0,0 +1,20 @@ +select + c_count, + count(*) as custdist +from + ( + select + c_custkey, + count(o_orderkey) + from + customer left outer join orders on + c_custkey = o_custkey + and o_comment not like '%special%requests%' + group by + c_custkey + ) as c_orders (c_custkey, c_count) +group by + c_count +order by + custdist desc, + c_count desc; \ No newline at end of file diff --git a/integration-utils/queries/tpch/q14.sql b/integration-utils/queries/tpch/q14.sql new file mode 100644 index 0000000..d8ef6af --- /dev/null +++ b/integration-utils/queries/tpch/q14.sql @@ -0,0 +1,13 @@ +select + 100.00 * sum(case + when p_type like 'PROMO%' + then l_extendedprice * (1 - l_discount) + else 0 + end) / sum(l_extendedprice * (1 - l_discount)) as promo_revenue +from + lineitem, + part +where + l_partkey = p_partkey + and l_shipdate >= date '1995-09-01' + and l_shipdate < date '1995-10-01'; \ No newline at end of file diff --git a/integration-utils/queries/tpch/q15.sql b/integration-utils/queries/tpch/q15.sql new file mode 100644 index 0000000..e586c91 --- /dev/null +++ b/integration-utils/queries/tpch/q15.sql @@ -0,0 +1,31 @@ +with revenue0 (supplier_no, total_revenue) as ( + select + l_suppkey, + sum(l_extendedprice * (1 - l_discount)) + from + lineitem + where + l_shipdate >= date '1996-01-01' + and l_shipdate < date '1996-01-01' + interval '3' month + group by + l_suppkey +) +select + s_suppkey, + s_name, + s_address, + s_phone, + total_revenue +from + supplier, + revenue0 +where + s_suppkey = supplier_no + and total_revenue = ( + select + max(total_revenue) + from + revenue0 + ) +order by + s_suppkey; \ No newline at end of file diff --git a/integration-utils/queries/tpch/q16.sql b/integration-utils/queries/tpch/q16.sql new file mode 100644 index 0000000..36b7c07 --- /dev/null +++ b/integration-utils/queries/tpch/q16.sql @@ -0,0 +1,30 @@ +select + p_brand, + p_type, + p_size, + count(distinct ps_suppkey) as supplier_cnt +from + partsupp, + part +where + p_partkey = ps_partkey + and p_brand <> 'Brand#45' + and p_type not like 'MEDIUM POLISHED%' + and p_size in (49, 14, 23, 45, 19, 3, 36, 9) + and ps_suppkey not in ( + select + s_suppkey + from + supplier + where + s_comment like '%Customer%Complaints%' +) +group by + p_brand, + p_type, + p_size +order by + supplier_cnt desc, + p_brand, + p_type, + p_size; \ No newline at end of file diff --git a/integration-utils/queries/tpch/q17.sql b/integration-utils/queries/tpch/q17.sql new file mode 100644 index 0000000..1e65550 --- /dev/null +++ b/integration-utils/queries/tpch/q17.sql @@ -0,0 +1,17 @@ +select + sum(l_extendedprice) / 7.0 as avg_yearly +from + lineitem, + part +where + p_partkey = l_partkey + and p_brand = 'Brand#23' + and p_container = 'MED BOX' + and l_quantity < ( + select + 0.2 * avg(l_quantity) + from + lineitem + where + l_partkey = p_partkey +); \ No newline at end of file diff --git a/integration-utils/queries/tpch/q18.sql b/integration-utils/queries/tpch/q18.sql new file mode 100644 index 0000000..835de28 --- /dev/null +++ b/integration-utils/queries/tpch/q18.sql @@ -0,0 +1,32 @@ +select + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice, + sum(l_quantity) +from + customer, + orders, + lineitem +where + o_orderkey in ( + select + l_orderkey + from + lineitem + group by + l_orderkey having + sum(l_quantity) > 300 + ) + and c_custkey = o_custkey + and o_orderkey = l_orderkey +group by + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice +order by + o_totalprice desc, + o_orderdate; \ No newline at end of file diff --git a/integration-utils/queries/tpch/q19.sql b/integration-utils/queries/tpch/q19.sql new file mode 100644 index 0000000..56668e7 --- /dev/null +++ b/integration-utils/queries/tpch/q19.sql @@ -0,0 +1,35 @@ +select + sum(l_extendedprice* (1 - l_discount)) as revenue +from + lineitem, + part +where + ( + p_partkey = l_partkey + and p_brand = 'Brand#12' + and p_container in ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG') + and l_quantity >= 1 and l_quantity <= 1 + 10 + and p_size between 1 and 5 + and l_shipmode in ('AIR', 'AIR REG') + and l_shipinstruct = 'DELIVER IN PERSON' + ) + or + ( + p_partkey = l_partkey + and p_brand = 'Brand#23' + and p_container in ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK') + and l_quantity >= 10 and l_quantity <= 10 + 10 + and p_size between 1 and 10 + and l_shipmode in ('AIR', 'AIR REG') + and l_shipinstruct = 'DELIVER IN PERSON' + ) + or + ( + p_partkey = l_partkey + and p_brand = 'Brand#34' + and p_container in ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG') + and l_quantity >= 20 and l_quantity <= 20 + 10 + and p_size between 1 and 15 + and l_shipmode in ('AIR', 'AIR REG') + and l_shipinstruct = 'DELIVER IN PERSON' + ); \ No newline at end of file diff --git a/integration-utils/queries/tpch/q2.sql b/integration-utils/queries/tpch/q2.sql new file mode 100644 index 0000000..f66af21 --- /dev/null +++ b/integration-utils/queries/tpch/q2.sql @@ -0,0 +1,43 @@ +select + s_acctbal, + s_name, + n_name, + p_partkey, + p_mfgr, + s_address, + s_phone, + s_comment +from + part, + supplier, + partsupp, + nation, + region +where + p_partkey = ps_partkey + and s_suppkey = ps_suppkey + and p_size = 15 + and p_type like '%BRASS' + and s_nationkey = n_nationkey + and n_regionkey = r_regionkey + and r_name = 'EUROPE' + and ps_supplycost = ( + select + min(ps_supplycost) + from + partsupp, + supplier, + nation, + region + where + p_partkey = ps_partkey + and s_suppkey = ps_suppkey + and s_nationkey = n_nationkey + and n_regionkey = r_regionkey + and r_name = 'EUROPE' +) +order by + s_acctbal desc, + n_name, + s_name, + p_partkey; \ No newline at end of file diff --git a/integration-utils/queries/tpch/q20.sql b/integration-utils/queries/tpch/q20.sql new file mode 100644 index 0000000..dd61a7d --- /dev/null +++ b/integration-utils/queries/tpch/q20.sql @@ -0,0 +1,37 @@ +select + s_name, + s_address +from + supplier, + nation +where + s_suppkey in ( + select + ps_suppkey + from + partsupp + where + ps_partkey in ( + select + p_partkey + from + part + where + p_name like 'forest%' + ) + and ps_availqty > ( + select + 0.5 * sum(l_quantity) + from + lineitem + where + l_partkey = ps_partkey + and l_suppkey = ps_suppkey + and l_shipdate >= date '1994-01-01' + and l_shipdate < date '1994-01-01' + interval '1' year + ) + ) + and s_nationkey = n_nationkey + and n_name = 'CANADA' +order by + s_name; \ No newline at end of file diff --git a/integration-utils/queries/tpch/q21.sql b/integration-utils/queries/tpch/q21.sql new file mode 100644 index 0000000..9d2fe32 --- /dev/null +++ b/integration-utils/queries/tpch/q21.sql @@ -0,0 +1,39 @@ +select + s_name, + count(*) as numwait +from + supplier, + lineitem l1, + orders, + nation +where + s_suppkey = l1.l_suppkey + and o_orderkey = l1.l_orderkey + and o_orderstatus = 'F' + and l1.l_receiptdate > l1.l_commitdate + and exists ( + select + * + from + lineitem l2 + where + l2.l_orderkey = l1.l_orderkey + and l2.l_suppkey <> l1.l_suppkey + ) + and not exists ( + select + * + from + lineitem l3 + where + l3.l_orderkey = l1.l_orderkey + and l3.l_suppkey <> l1.l_suppkey + and l3.l_receiptdate > l3.l_commitdate + ) + and s_nationkey = n_nationkey + and n_name = 'SAUDI ARABIA' +group by + s_name +order by + numwait desc, + s_name; \ No newline at end of file diff --git a/integration-utils/queries/tpch/q22.sql b/integration-utils/queries/tpch/q22.sql new file mode 100644 index 0000000..90aea6f --- /dev/null +++ b/integration-utils/queries/tpch/q22.sql @@ -0,0 +1,37 @@ +select + cntrycode, + count(*) as numcust, + sum(c_acctbal) as totacctbal +from + ( + select + substring(c_phone from 1 for 2) as cntrycode, + c_acctbal + from + customer + where + substring(c_phone from 1 for 2) in + ('13', '31', '23', '29', '30', '18', '17') + and c_acctbal > ( + select + avg(c_acctbal) + from + customer + where + c_acctbal > 0.00 + and substring(c_phone from 1 for 2) in + ('13', '31', '23', '29', '30', '18', '17') + ) + and not exists ( + select + * + from + orders + where + o_custkey = c_custkey + ) + ) as custsale +group by + cntrycode +order by + cntrycode; \ No newline at end of file diff --git a/integration-utils/queries/tpch/q3.sql b/integration-utils/queries/tpch/q3.sql new file mode 100644 index 0000000..7dbc6d9 --- /dev/null +++ b/integration-utils/queries/tpch/q3.sql @@ -0,0 +1,22 @@ +select + l_orderkey, + sum(l_extendedprice * (1 - l_discount)) as revenue, + o_orderdate, + o_shippriority +from + customer, + orders, + lineitem +where + c_mktsegment = 'BUILDING' + and c_custkey = o_custkey + and l_orderkey = o_orderkey + and o_orderdate < date '1995-03-15' + and l_shipdate > date '1995-03-15' +group by + l_orderkey, + o_orderdate, + o_shippriority +order by + revenue desc, + o_orderdate; \ No newline at end of file diff --git a/integration-utils/queries/tpch/q4.sql b/integration-utils/queries/tpch/q4.sql new file mode 100644 index 0000000..74a620d --- /dev/null +++ b/integration-utils/queries/tpch/q4.sql @@ -0,0 +1,21 @@ +select + o_orderpriority, + count(*) as order_count +from + orders +where + o_orderdate >= '1993-07-01' + and o_orderdate < date '1993-07-01' + interval '3' month + and exists ( + select + * + from + lineitem + where + l_orderkey = o_orderkey + and l_commitdate < l_receiptdate + ) +group by + o_orderpriority +order by + o_orderpriority; \ No newline at end of file diff --git a/integration-utils/queries/tpch/q5.sql b/integration-utils/queries/tpch/q5.sql new file mode 100644 index 0000000..5a336b2 --- /dev/null +++ b/integration-utils/queries/tpch/q5.sql @@ -0,0 +1,24 @@ +select + n_name, + sum(l_extendedprice * (1 - l_discount)) as revenue +from + customer, + orders, + lineitem, + supplier, + nation, + region +where + c_custkey = o_custkey + and l_orderkey = o_orderkey + and l_suppkey = s_suppkey + and c_nationkey = s_nationkey + and s_nationkey = n_nationkey + and n_regionkey = r_regionkey + and r_name = 'ASIA' + and o_orderdate >= date '1994-01-01' + and o_orderdate < date '1995-01-01' +group by + n_name +order by + revenue desc; \ No newline at end of file diff --git a/integration-utils/queries/tpch/q6.sql b/integration-utils/queries/tpch/q6.sql new file mode 100644 index 0000000..5806f98 --- /dev/null +++ b/integration-utils/queries/tpch/q6.sql @@ -0,0 +1,9 @@ +select + sum(l_extendedprice * l_discount) as revenue +from + lineitem +where + l_shipdate >= date '1994-01-01' + and l_shipdate < date '1995-01-01' + and l_discount between 0.06 - 0.01 and 0.06 + 0.01 + and l_quantity < 24; \ No newline at end of file diff --git a/integration-utils/queries/tpch/q7.sql b/integration-utils/queries/tpch/q7.sql new file mode 100644 index 0000000..512e5be --- /dev/null +++ b/integration-utils/queries/tpch/q7.sql @@ -0,0 +1,39 @@ +select + supp_nation, + cust_nation, + l_year, + sum(volume) as revenue +from + ( + select + n1.n_name as supp_nation, + n2.n_name as cust_nation, + extract(year from l_shipdate) as l_year, + l_extendedprice * (1 - l_discount) as volume + from + supplier, + lineitem, + orders, + customer, + nation n1, + nation n2 + where + s_suppkey = l_suppkey + and o_orderkey = l_orderkey + and c_custkey = o_custkey + and s_nationkey = n1.n_nationkey + and c_nationkey = n2.n_nationkey + and ( + (n1.n_name = 'FRANCE' and n2.n_name = 'GERMANY') + or (n1.n_name = 'GERMANY' and n2.n_name = 'FRANCE') + ) + and l_shipdate between date '1995-01-01' and date '1996-12-31' + ) as shipping +group by + supp_nation, + cust_nation, + l_year +order by + supp_nation, + cust_nation, + l_year; diff --git a/integration-utils/queries/tpch/q8.sql b/integration-utils/queries/tpch/q8.sql new file mode 100644 index 0000000..6ddb2a6 --- /dev/null +++ b/integration-utils/queries/tpch/q8.sql @@ -0,0 +1,37 @@ +select + o_year, + sum(case + when nation = 'BRAZIL' then volume + else 0 + end) / sum(volume) as mkt_share +from + ( + select + extract(year from o_orderdate) as o_year, + l_extendedprice * (1 - l_discount) as volume, + n2.n_name as nation + from + part, + supplier, + lineitem, + orders, + customer, + nation n1, + nation n2, + region + where + p_partkey = l_partkey + and s_suppkey = l_suppkey + and l_orderkey = o_orderkey + and o_custkey = c_custkey + and c_nationkey = n1.n_nationkey + and n1.n_regionkey = r_regionkey + and r_name = 'AMERICA' + and s_nationkey = n2.n_nationkey + and o_orderdate between date '1995-01-01' and date '1996-12-31' + and p_type = 'ECONOMY ANODIZED STEEL' + ) as all_nations +group by + o_year +order by + o_year; \ No newline at end of file diff --git a/integration-utils/queries/tpch/q9.sql b/integration-utils/queries/tpch/q9.sql new file mode 100644 index 0000000..587bbc8 --- /dev/null +++ b/integration-utils/queries/tpch/q9.sql @@ -0,0 +1,32 @@ +select + nation, + o_year, + sum(amount) as sum_profit +from + ( + select + n_name as nation, + extract(year from o_orderdate) as o_year, + l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as amount + from + part, + supplier, + lineitem, + partsupp, + orders, + nation + where + s_suppkey = l_suppkey + and ps_suppkey = l_suppkey + and ps_partkey = l_partkey + and p_partkey = l_partkey + and o_orderkey = l_orderkey + and s_nationkey = n_nationkey + and p_name like '%green%' + ) as profit +group by + nation, + o_year +order by + nation, + o_year desc; \ No newline at end of file diff --git a/integration-utils/queries/weather.sql b/integration-utils/queries/weather.sql new file mode 100644 index 0000000..2d18f68 --- /dev/null +++ b/integration-utils/queries/weather.sql @@ -0,0 +1,5 @@ +SELECT + count(*), + "MinTemp" +FROM weather +GROUP BY "MinTemp" diff --git a/integration-utils/src/data.rs b/integration-utils/src/data.rs new file mode 100644 index 0000000..93cc042 --- /dev/null +++ b/integration-utils/src/data.rs @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025 Datadog, Inc. + +//! Data and table utilities + +use std::path::PathBuf; +use std::sync::Arc; + +use datafusion::common::internal_datafusion_err; +use datafusion::datasource::file_format::parquet::ParquetFormat; +use datafusion::datasource::listing::ListingOptions; +use datafusion::{error::Result, prelude::*}; +use tracing::{info, instrument}; + +/// Returns the path to the directory containing the Parquet tables. +pub fn data_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("data") +} + +/// Registers all TPCH Parquet tables required for executing the queries. +#[instrument(level = "info", skip(ctx))] +pub(crate) async fn register_tpch_tables(ctx: &SessionContext) -> Result<()> { + // Construct the path to the directory containing Parquet data. + let data_dir = data_dir(); + + // Register the weather table. + ctx.register_parquet( + "weather", + data_dir.join("weather").to_string_lossy(), + ParquetReadOptions::default(), + ) + .await?; + + // Generate and register each table from Parquet files. + // This includes all standard TPCH tables so examples/tests can rely on them. + for table in [ + "nation", "region", "part", "supplier", "partsupp", "customer", "orders", + "lineitem", + ] { + let listing_options = ListingOptions::new(Arc::new(ParquetFormat::default())); + + let parquet_path = data_dir.join(table).with_extension("parquet"); + if !parquet_path.exists() { + return Err(internal_datafusion_err!( + "Missing TPCH Parquet file: {}.\nGenerate TPCH data first by running: ./dev/generate_tpch_parquet.sh\nThis script requires 'tpchgen-cli' (install with: cargo install tpchgen-cli)", + parquet_path.display() + )); + } + + // Generate the file path URL for the Parquet data. + let table_path = format!("file://{}", parquet_path.to_string_lossy()); + + info!("Registering table '{}' from {}", table, table_path); + + // Register the table with DataFusion's session context. + ctx.register_listing_table(table, &table_path, listing_options, None, None) + .await?; + } + + Ok(()) +} diff --git a/integration-utils/src/distributed/in_memory.rs b/integration-utils/src/distributed/in_memory.rs new file mode 100644 index 0000000..b872cc2 --- /dev/null +++ b/integration-utils/src/distributed/in_memory.rs @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025 Datadog, Inc. + +use arrow_flight::flight_service_client::FlightServiceClient; +use arrow_flight::flight_service_server::FlightServiceServer; +use async_trait::async_trait; +use datafusion::common::DataFusionError; +use datafusion::execution::SessionStateBuilder; +use datafusion_distributed::{ + ArrowFlightEndpoint, BoxCloneSyncChannel, ChannelResolver, DistributedExt, + DistributedSessionBuilderContext, +}; +use hyper_util::rt::TokioIo; +use tonic::transport::{Endpoint, Server}; +use tower::ServiceBuilder; + +use super::trace_middleware::{TracingClientLayer, TracingServerLayer}; + +const DUMMY_URL: &str = "http://localhost:50051"; +const MAX_MESSAGE_SIZE: usize = 2 * 1024 * 1024 * 1024; // 2GB + +/// [ChannelResolver] implementation that returns gRPC clients backed by an in-memory +/// tokio duplex rather than a TCP connection. +#[derive(Clone)] +pub(crate) struct InMemoryChannelResolver { + channel: FlightServiceClient, +} + +impl InMemoryChannelResolver { + pub fn new() -> Self { + let (client, server) = tokio::io::duplex(1024 * 1024); // 1MB buffer + + let mut client = Some(client); + let channel = Endpoint::try_from(DUMMY_URL) + .expect( + "Invalid dummy URL for building an endpoint. This should never happen", + ) + .connect_with_connector_lazy(tower::service_fn(move |_| { + let client = client + .take() + .expect("Client taken twice. This should never happen"); + async move { Ok::<_, std::io::Error>(TokioIo::new(client)) } + })); + + // Wrap the channel with tracing middleware to inject trace context + let channel = ServiceBuilder::new() + .layer(TracingClientLayer) + .service(channel); + + let this = Self { + channel: FlightServiceClient::new(BoxCloneSyncChannel::new(channel)) + .max_decoding_message_size(MAX_MESSAGE_SIZE) + .max_encoding_message_size(MAX_MESSAGE_SIZE), + }; + let this_clone = this.clone(); + + let endpoint = + ArrowFlightEndpoint::try_new(move |ctx: DistributedSessionBuilderContext| { + let this = this.clone(); + async move { + let builder = SessionStateBuilder::new() + .with_default_features() + .with_distributed_channel_resolver(this) + .with_runtime_env(ctx.runtime_env.clone()); + Ok(builder.build()) + } + }) + .unwrap(); + + tokio::spawn(async move { + // Wrap the FlightService with tracing middleware to extract trace context + let service = ServiceBuilder::new().layer(TracingServerLayer).service( + FlightServiceServer::new(endpoint) + .max_decoding_message_size(MAX_MESSAGE_SIZE) + .max_encoding_message_size(MAX_MESSAGE_SIZE), + ); + + Server::builder() + .add_service(service) + .serve_with_incoming(tokio_stream::once(Ok::<_, std::io::Error>(server))) + .await + }); + + this_clone + } +} + +#[async_trait] +impl ChannelResolver for InMemoryChannelResolver { + fn get_urls(&self) -> Result, DataFusionError> { + Ok(vec![url::Url::parse(DUMMY_URL).unwrap()]) + } + + async fn get_flight_client_for_url( + &self, + _: &url::Url, + ) -> Result, DataFusionError> { + Ok(self.channel.clone()) + } +} diff --git a/integration-utils/src/distributed/localhost.rs b/integration-utils/src/distributed/localhost.rs new file mode 100644 index 0000000..4006133 --- /dev/null +++ b/integration-utils/src/distributed/localhost.rs @@ -0,0 +1,363 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025 Datadog, Inc. + +use std::future::Future; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::pin::Pin; + +use arrow_flight::flight_service_client::FlightServiceClient; +use arrow_flight::flight_service_server::FlightServiceServer; +use async_trait::async_trait; +use dashmap::{DashMap, Entry}; +use datafusion::common::DataFusionError; +use datafusion::error::Result; +use datafusion::execution::SessionStateBuilder; +use datafusion_distributed::{ + ArrowFlightEndpoint, BoxCloneSyncChannel, ChannelResolver, DistributedExt, + DistributedSessionBuilderContext, +}; +use tokio::sync::oneshot; +use tonic::transport::{Channel, Server}; +use tower::ServiceBuilder; +use url::Url; + +use super::trace_middleware::{TracingClientLayer, TracingServerLayer}; + +/// Channel resolver for localhost workers that connects via TCP +#[derive(Clone)] +pub(crate) struct LocalhostChannelResolver { + ports: Vec, + cached: DashMap>, +} + +impl LocalhostChannelResolver { + pub(crate) fn new(ports: Vec) -> Self { + Self { + ports, + cached: DashMap::new(), + } + } +} + +#[async_trait] +impl ChannelResolver for LocalhostChannelResolver { + fn get_urls(&self) -> Result, DataFusionError> { + Ok(self + .ports + .iter() + .map(|port| Url::parse(&format!("http://localhost:{port}")).unwrap()) + .collect()) + } + + async fn get_flight_client_for_url( + &self, + url: &Url, + ) -> Result, DataFusionError> { + match self.cached.entry(url.clone()) { + Entry::Occupied(v) => Ok(v.get().clone()), + Entry::Vacant(v) => { + let channel = Channel::from_shared(url.to_string()) + .unwrap() + .connect_lazy(); + + // Wrap the channel with tracing middleware to inject trace context + let channel = ServiceBuilder::new() + .layer(TracingClientLayer) + .service(channel); + + let channel = FlightServiceClient::new(BoxCloneSyncChannel::new(channel)); + v.insert(channel.clone()); + Ok(channel) + } + } + } +} + +/// Configuration for worker instrumentation +#[derive(Debug, Clone, Copy)] +pub(crate) struct WorkerInstrumentationConfig { + pub record_metrics: bool, + pub preview_limit: usize, + pub compact_preview: bool, +} + +/// Spawns localhost workers on the specified ports +pub(crate) async fn spawn_localhost_workers( + ports: &[u16], + instrumentation_config: Option, +) -> Result<()> { + let mut ready_receivers = Vec::new(); + + for &port in ports { + let all_ports = ports.to_vec(); + let (ready_tx, ready_rx) = oneshot::channel(); + ready_receivers.push(ready_rx); + + // Don't create a worker span here - workers should only create spans when handling + // requests, using the trace context extracted from the gRPC metadata. This ensures + // worker spans are properly linked to the parent coordinator trace. + tracing::info!("Spawning localhost worker on port {}", port); + + tokio::spawn(async move { + let localhost_resolver = LocalhostChannelResolver::new(all_ports); + + let endpoint = ArrowFlightEndpoint::try_new( + move |ctx: DistributedSessionBuilderContext| { + let local_host_resolver = localhost_resolver.clone(); + async move { + let mut builder = SessionStateBuilder::new() + .with_runtime_env(ctx.runtime_env) + .with_distributed_channel_resolver(local_host_resolver) + .with_default_features(); + + // Build and add instrumentation rule if config is provided + if let Some(config) = instrumentation_config { + let instrumentation_rule = crate::session::create_instrumentation_rule_with_worker_id( + config.record_metrics, + config.preview_limit, + config.compact_preview, + Some(format!("localhost:{}", port)), + ); + builder = builder.with_physical_optimizer_rule(instrumentation_rule); + } + + Ok(builder.build()) + } + }, + ) + .expect("Failed to create ArrowFlightEndpoint"); + + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port); + + // Wrap the FlightService with tracing middleware to extract trace context + let service = ServiceBuilder::new() + .layer(TracingServerLayer) + .service(FlightServiceServer::new(endpoint)); + + let server = Server::builder().add_service(service).serve(addr); + + // Notify that we're about to start serving (the server will bind immediately) + let _ = ready_tx.send(()); + + server + .await + .unwrap_or_else(|_| panic!("Failed to start worker on port {}", port)); + }); + } + + // Wait for all workers to signal they're ready + for (i, ready_rx) in ready_receivers.into_iter().enumerate() { + ready_rx.await.map_err(|_| { + DataFusionError::Execution(format!( + "Worker on port {} failed to start", + ports[i] + )) + })?; + } + + Ok(()) +} + +/// Spawns localhost workers with task-local buffer support for separate log buffers per worker +pub(crate) async fn spawn_localhost_workers_with_buffers( + ports: &[u16], + instrumentation_config: Option, + task_local_wrapper: F, +) -> Result<()> +where + F: Fn(u16, Pin + Send>>) -> Fut + Send + Sync + 'static, + Fut: Future + Send + 'static, +{ + let mut ready_receivers = Vec::new(); + + for &port in ports { + let all_ports = ports.to_vec(); + let (ready_tx, ready_rx) = oneshot::channel(); + ready_receivers.push(ready_rx); + + // Don't create a worker span here - workers should only create spans when handling + // requests, using the trace context extracted from the gRPC metadata. This ensures + // worker spans are properly linked to the parent coordinator trace. + tracing::info!("Spawning localhost worker on port {}", port); + + // Clone the wrapper function for this task + let wrapper = &task_local_wrapper; + + let worker_future = async move { + let localhost_resolver = LocalhostChannelResolver::new(all_ports); + + let endpoint = ArrowFlightEndpoint::try_new( + move |ctx: DistributedSessionBuilderContext| { + let local_host_resolver = localhost_resolver.clone(); + async move { + let mut builder = SessionStateBuilder::new() + .with_runtime_env(ctx.runtime_env) + .with_distributed_channel_resolver(local_host_resolver) + .with_default_features(); + + // Build and add instrumentation rule if config is provided + if let Some(config) = instrumentation_config { + let instrumentation_rule = crate::session::create_instrumentation_rule_with_worker_id( + config.record_metrics, + config.preview_limit, + config.compact_preview, + Some(format!("localhost:{}", port)), + ); + builder = builder.with_physical_optimizer_rule(instrumentation_rule); + } + + Ok(builder.build()) + } + }, + ) + .expect("Failed to create ArrowFlightEndpoint"); + + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port); + + // Wrap the FlightService with tracing middleware to extract trace context + let service = ServiceBuilder::new() + .layer(TracingServerLayer) + .service(FlightServiceServer::new(endpoint)); + + let server = Server::builder().add_service(service).serve(addr); + + // Notify that we're about to start serving (the server will bind immediately) + let _ = ready_tx.send(()); + + server + .await + .unwrap_or_else(|_| panic!("Failed to start worker on port {}", port)); + }; + + tokio::spawn(wrapper(port, Box::pin(worker_future))); + } + + // Wait for all workers to signal they're ready + for (i, ready_rx) in ready_receivers.into_iter().enumerate() { + ready_rx.await.map_err(|_| { + DataFusionError::Execution(format!( + "Worker on port {} failed to start", + ports[i] + )) + })?; + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::net::TcpStream; + + #[tokio::test] + async fn test_spawn_and_connect_to_workers() { + // Use high-numbered ports to avoid conflicts + let ports = vec![50051, 50052, 50053]; + + // Spawn workers (without instrumentation for this test) + spawn_localhost_workers(&ports, None) + .await + .expect("Failed to spawn workers"); + + // Verify each worker is accepting connections + for &port in &ports { + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port); + let result = TcpStream::connect(addr).await; + assert!( + result.is_ok(), + "Failed to connect to worker on port {}", + port + ); + } + + // Create a resolver and verify we can get URLs + let resolver = LocalhostChannelResolver::new(ports.clone()); + let urls = resolver.get_urls().expect("Failed to get URLs"); + assert_eq!(urls.len(), ports.len()); + + // Verify URLs are correctly formatted + for (i, url) in urls.iter().enumerate() { + assert_eq!( + url.as_str(), + format!("http://localhost:{}/", ports[i]), + "URL format mismatch for port {}", + ports[i] + ); + } + + // Create flight clients for each URL + for url in &urls { + let _client = resolver + .get_flight_client_for_url(url) + .await + .expect("Failed to create flight client"); + // Verify client was cached + assert!(resolver.cached.contains_key(url)); + + // Get the same client again and verify it's cached + let _client2 = resolver + .get_flight_client_for_url(url) + .await + .expect("Failed to get cached flight client"); + // Both clients should be the same (from cache) + assert_eq!( + resolver.cached.len(), + urls.iter().position(|u| u == url).unwrap() + 1 + ); + } + } + + #[tokio::test] + async fn test_worker_readiness_signal() { + // Use a different set of ports to avoid conflicts with other tests + let ports = vec![50061, 50062]; + + // Measure time to spawn and become ready + let start = std::time::Instant::now(); + spawn_localhost_workers(&ports, None) + .await + .expect("Failed to spawn workers"); + let elapsed = start.elapsed(); + + // Workers should be ready very quickly (well under 1 second) + // This verifies we're not sleeping unnecessarily + assert!( + elapsed.as_millis() < 1000, + "Workers took too long to become ready: {:?}", + elapsed + ); + + // Verify workers are actually running + for &port in &ports { + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port); + TcpStream::connect(addr).await.expect(&format!( + "Worker on port {} is not accepting connections", + port + )); + } + } + + #[tokio::test] + async fn test_empty_ports_list() { + // Spawning with no ports should succeed (no-op) + let result = spawn_localhost_workers(&[], None).await; + assert!(result.is_ok()); + } +} diff --git a/integration-utils/src/distributed/mod.rs b/integration-utils/src/distributed/mod.rs new file mode 100644 index 0000000..7ea7ee7 --- /dev/null +++ b/integration-utils/src/distributed/mod.rs @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025 Datadog, Inc. + +//! Distributed execution utilities +//! +//! This module provides channel resolvers and utilities for distributed query execution: +//! - `in_memory`: In-memory channel resolver for fast testing +//! - `localhost`: TCP-based localhost worker resolver for more realistic testing +//! - `trace_middleware`: Tower middleware for trace context propagation + +mod in_memory; +mod localhost; +pub mod trace_middleware; + +pub(crate) use in_memory::InMemoryChannelResolver; +pub(crate) use localhost::{ + LocalhostChannelResolver, WorkerInstrumentationConfig, spawn_localhost_workers, + spawn_localhost_workers_with_buffers, +}; + +/// Default ports to use for localhost workers in distributed mode +pub(crate) const DEFAULT_WORKER_PORTS: &[u16] = &[50051, 50052, 50053]; + +/// Distributed execution mode configuration +#[derive(Debug, Clone, Copy)] +pub enum DistributedMode { + /// Use in-memory channels (faster, for testing) + Memory, + /// Use localhost TCP workers (more realistic) + Localhost, +} diff --git a/integration-utils/src/distributed/trace_middleware.rs b/integration-utils/src/distributed/trace_middleware.rs new file mode 100644 index 0000000..9017117 --- /dev/null +++ b/integration-utils/src/distributed/trace_middleware.rs @@ -0,0 +1,292 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025 Datadog, Inc. + +//! Tower middleware layers for OpenTelemetry trace context propagation. +//! +//! This module provides middleware that automatically handles trace context propagation +//! across gRPC boundaries using the W3C Trace Context format. +//! +//! - `TracingClientLayer`: Injects trace context into outgoing gRPC requests +//! - `TracingServerLayer`: Extracts trace context from incoming gRPC requests and creates spans + +use opentelemetry::propagation::{Extractor, Injector}; +use pin_project_lite::pin_project; +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tower::{Layer, Service}; +use tracing::Instrument; +use tracing_opentelemetry::OpenTelemetrySpanExt; + +// ============================================================================ +// Client-side middleware (injects trace context into outgoing requests) +// ============================================================================ + +/// Tower layer that injects OpenTelemetry trace context into outgoing gRPC requests. +/// +/// This middleware reads the current tracing span's OpenTelemetry context and injects +/// it into the HTTP headers of outgoing requests using the configured global propagator +/// (typically W3C Trace Context format). +/// +/// # Example +/// +/// ```ignore +/// use tower::ServiceBuilder; +/// use tonic::transport::Channel; +/// +/// let channel = Channel::from_static("http://localhost:50051") +/// .connect_lazy(); +/// +/// let channel = ServiceBuilder::new() +/// .layer(TracingClientLayer) +/// .service(channel); +/// ``` +#[derive(Debug, Clone, Copy)] +pub struct TracingClientLayer; + +impl Layer for TracingClientLayer { + type Service = TracingClient; + + fn layer(&self, inner: S) -> Self::Service { + TracingClient { inner } + } +} + +/// Service wrapper that injects trace context into requests. +#[derive(Debug, Clone)] +pub struct TracingClient { + inner: S, +} + +impl Service> for TracingClient +where + S: Service, Response = http::Response>, +{ + type Response = S::Response; + type Error = S::Error; + type Future = S::Future; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, mut request: http::Request) -> Self::Future { + // Get the current OpenTelemetry context from the tracing span + let context = tracing::Span::current().context(); + + // Inject the context into HTTP headers + let headers = request.headers_mut(); + let mut injector = HttpHeaderInjector(headers); + opentelemetry::global::get_text_map_propagator(|propagator| { + propagator.inject_context(&context, &mut injector); + }); + + self.inner.call(request) + } +} + +/// Helper struct for injecting trace context into HTTP headers. +struct HttpHeaderInjector<'a>(&'a mut http::HeaderMap); + +impl Injector for HttpHeaderInjector<'_> { + fn set(&mut self, key: &str, value: String) { + if let Ok(name) = http::HeaderName::try_from(key) { + if let Ok(val) = http::HeaderValue::try_from(value) { + self.0.insert(name, val); + } + } + } +} + +// ============================================================================ +// Server-side middleware (extracts trace context and creates spans) +// ============================================================================ + +/// Tower layer that extracts OpenTelemetry trace context from incoming gRPC requests +/// and creates instrumented spans. +/// +/// This middleware reads W3C Trace Context headers from incoming requests, extracts +/// the parent context, and creates a new span that is properly linked to the parent trace. +/// The span remains active for the duration of the request processing. +/// +/// # Example +/// +/// ```ignore +/// use tower::ServiceBuilder; +/// use arrow_flight::flight_service_server::FlightServiceServer; +/// +/// let service = FlightServiceServer::new(endpoint); +/// +/// let service = ServiceBuilder::new() +/// .layer(TracingServerLayer) +/// .service(service); +/// +/// Server::builder() +/// .add_service(service) +/// .serve(addr) +/// .await?; +/// ``` +#[derive(Debug, Clone, Copy)] +pub struct TracingServerLayer; + +impl Layer for TracingServerLayer { + type Service = TracingServer; + + fn layer(&self, inner: S) -> Self::Service { + TracingServer { inner } + } +} + +/// Service wrapper that extracts trace context and creates spans. +#[derive(Debug, Clone)] +pub struct TracingServer { + inner: S, +} + +impl Service> for TracingServer +where + S: Service, Response = http::Response>, + S::Future: Send + 'static, +{ + type Response = S::Response; + type Error = S::Error; + type Future = TracingServerFuture; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, request: http::Request) -> Self::Future { + // Extract trace context from HTTP headers + let extractor = HttpHeaderExtractor(request.headers()); + let parent_context = + opentelemetry::global::get_text_map_propagator(|propagator| { + propagator.extract(&extractor) + }); + + // Parse the gRPC path to extract useful span information + let path = request.uri().path(); + let method_name = path.rsplit('/').next().unwrap_or("unknown"); + + // Attach the parent context and create a span for this request. + // This must be INFO level or above to be captured by the OpenTelemetry layer. + let span = { + let _guard = parent_context.attach(); + tracing::info_span!( + "grpc_request", + otel.kind = "server", + rpc.system = "grpc", + rpc.service = "arrow.flight.protocol.FlightService", + rpc.method = method_name, + ) + }; + + let inner_future = self.inner.call(request); + TracingServerFuture { + inner: inner_future.instrument(span), + } + } +} + +pin_project! { + /// Future wrapper that instruments the request processing with a tracing span. + pub struct TracingServerFuture { + #[pin] + inner: tracing::instrument::Instrumented, + } +} + +impl Future for TracingServerFuture +where + F: Future>, +{ + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.project(); + this.inner.poll(cx) + } +} + +/// Helper struct for extracting trace context from HTTP headers. +struct HttpHeaderExtractor<'a>(&'a http::HeaderMap); + +impl Extractor for HttpHeaderExtractor<'_> { + fn get(&self, key: &str) -> Option<&str> { + self.0.get(key).and_then(|v| v.to_str().ok()) + } + + fn keys(&self) -> Vec<&str> { + self.0.keys().map(|k| k.as_str()).collect() + } +} + +// Implement tonic's NamedService for TracingServer to allow it to be used with add_service +impl tonic::server::NamedService for TracingServer +where + S: tonic::server::NamedService, +{ + const NAME: &'static str = S::NAME; +} + +#[cfg(test)] +mod tests { + use super::*; + use tower::ServiceExt; + + #[tokio::test] + async fn test_client_middleware_doesnt_panic() { + // Create a simple mock service + let service = tower::service_fn(|req: http::Request| async move { + Ok::<_, std::convert::Infallible>(http::Response::new(req.into_body())) + }); + + // Wrap with tracing middleware + let mut service = TracingClientLayer.layer(service); + + // Make a request + let request = http::Request::builder() + .uri("http://test.com") + .body("test".to_string()) + .unwrap(); + + let response = service.ready().await.unwrap().call(request).await; + assert!(response.is_ok()); + } + + #[tokio::test] + async fn test_server_middleware_doesnt_panic() { + // Create a simple mock service + let service = tower::service_fn(|req: http::Request| async move { + Ok::<_, std::convert::Infallible>(http::Response::new(req.into_body())) + }); + + // Wrap with tracing middleware + let mut service = TracingServerLayer.layer(service); + + // Make a request + let request = http::Request::builder() + .uri("http://test.com/arrow.flight.protocol.FlightService/DoGet") + .body("test".to_string()) + .unwrap(); + + let response = service.ready().await.unwrap().call(request).await; + assert!(response.is_ok()); + } +} diff --git a/integration-utils/src/lib.rs b/integration-utils/src/lib.rs index 6638774..5ece988 100644 --- a/integration-utils/src/lib.rs +++ b/integration-utils/src/lib.rs @@ -36,11 +36,32 @@ //! //! ```rust //! use datafusion::prelude::*; -//! use integration_utils::{init_session, run_traced_query}; +//! use integration_utils::{SessionBuilder, run_traced_query, DistributedMode}; //! //! # async fn example() -> datafusion::error::Result<()> { -//! // Initialize a session with all tracing options enabled -//! let ctx = init_session(true, true, 5, true).await?; +//! // Initialize a session with default options +//! let ctx = SessionBuilder::new().build().await?; +//! +//! // Initialize with all tracing options enabled +//! let ctx = SessionBuilder::new() +//! .with_object_store_tracing() +//! .with_metrics() +//! .with_preview(5) +//! .with_compact_preview() +//! .build() +//! .await?; +//! +//! // Or with in-memory distributed execution +//! let ctx = SessionBuilder::new() +//! .with_distributed_mode(DistributedMode::Memory) +//! .build() +//! .await?; +//! +//! // Or with localhost worker distributed execution +//! let ctx = SessionBuilder::new() +//! .with_distributed_mode(DistributedMode::Localhost) +//! .build() +//! .await?; //! //! // Run a traced query - results and execution details will be logged //! run_traced_query(&ctx, "simple_query").await?; @@ -48,188 +69,13 @@ //! # } //! ``` -use std::path::PathBuf; -use std::sync::Arc; - -use datafusion::arrow::array::RecordBatch; -use datafusion::arrow::util::pretty::pretty_format_batches; -use datafusion::common::internal_datafusion_err; -use datafusion::datasource::file_format::parquet::ParquetFormat; -use datafusion::datasource::listing::ListingOptions; -use datafusion::physical_plan::{collect, displayable, ExecutionPlan}; -use datafusion::{ - error::Result, execution::SessionStateBuilder, - physical_optimizer::PhysicalOptimizerRule, prelude::*, -}; -use datafusion_tracing::{ - instrument_with_info_spans, pretty_format_compact_batch, InstrumentationOptions, -}; -use instrumented_object_store::instrument_object_store; -use tracing::{field, info, instrument}; -use url::Url; - -/// Executes the SQL query with instrumentation enabled, providing detailed tracing output. -#[instrument(level = "info", skip(ctx))] -pub async fn run_traced_query(ctx: &SessionContext, query_name: &str) -> Result<()> { - info!("Beginning traced query execution"); - - let query = read_query(query_name)?; - - // Parse the SQL query into a DataFusion dataframe. - let df = parse_sql(ctx, query.as_str()).await?; - - // Generate a physical execution plan from the logical plan. - let physical_plan = create_physical_plan(df).await?; - - // Execute the physical plan and collect results. - let results = collect(physical_plan.clone(), ctx.task_ctx()).await?; - - info!("Query Results:\n{}", pretty_format_batches(&results)?); - - Ok(()) -} - -#[instrument(level = "info")] -pub async fn init_session( - record_object_store: bool, - record_metrics: bool, - preview_limit: usize, - compact_preview: bool, -) -> Result { - // Configure the session state with instrumentation for query execution. - let session_state = SessionStateBuilder::new() - .with_default_features() - .with_config(SessionConfig::default().with_target_partitions(8)) // Enforce target partitions to ensure consistent test results regardless of the number of CPU cores. - .with_physical_optimizer_rule(create_instrumentation_rule( - record_metrics, - preview_limit, - compact_preview, - )) - .build(); - - let ctx = SessionContext::new_with_state(session_state); - - // Instrument the local filesystem object store for tracing file access. - let local_store = Arc::new(object_store::local::LocalFileSystem::new()); - let object_store = if record_object_store { - instrument_object_store(local_store, "local_fs") - } else { - local_store - }; - - // Register the instrumented object store for handling file:// URLs. - ctx.register_object_store(&Url::parse("file://").unwrap(), object_store); - - // Register the tpch tables. - register_tpch_tables(&ctx).await?; - - Ok(ctx) -} - -/// Creates an instrumentation rule that captures metrics and provides previews of data during execution. -pub fn create_instrumentation_rule( - record_metrics: bool, - preview_limit: usize, - compact_preview: bool, -) -> Arc { - let options_builder = InstrumentationOptions::builder() - .add_custom_field("env", "production") // Custom fields - .add_custom_field("region", "us-west") - .record_metrics(record_metrics) - .preview_limit(preview_limit); - let options_builder = if compact_preview { - options_builder.preview_fn(Arc::new(|batch: &RecordBatch| { - // Format data batches for compact preview in span fields. - pretty_format_compact_batch(batch, 64, 3, 10).map(|fmt| fmt.to_string()) - })) - } else { - options_builder - }; - - instrument_with_info_spans!( - options: options_builder.build(), - env = field::Empty, // custom fields keys must be defined at compile time - region = field::Empty, - ) -} - -/// Returns the path to the directory containing the TPCH Parquet tables. -pub fn tpch_tables_dir() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("data") -} - -/// Registers all TPCH Parquet tables required for executing the queries. -#[instrument(level = "info", skip(ctx))] -async fn register_tpch_tables(ctx: &SessionContext) -> Result<()> { - // Construct the path to the directory containing Parquet data. - let data_dir = tpch_tables_dir(); - - // Generate and register each table from Parquet files. - // This includes all standard TPCH tables so examples/tests can rely on them. - for table in [ - "nation", "region", "part", "supplier", "partsupp", "customer", "orders", - "lineitem", - ] { - let listing_options = ListingOptions::new(Arc::new(ParquetFormat::default())); - - let parquet_path = data_dir.join(table).with_extension("parquet"); - if !parquet_path.exists() { - return Err(internal_datafusion_err!( - "Missing TPCH Parquet file: {}.\nGenerate TPCH data first by running: ./dev/generate_tpch_parquet.sh\nThis script requires 'tpchgen-cli' (install with: cargo install tpchgen-cli)", - parquet_path.display() - )); - } - - // Generate the file path URL for the Parquet data. - let table_path = format!("file://{}", parquet_path.to_string_lossy()); - - info!("Registering table '{}' from {}", table, table_path); - - // Register the table with DataFusion's session context. - ctx.register_listing_table(table, &table_path, listing_options, None, None) - .await?; - } - - Ok(()) -} - -#[instrument(level = "info", fields(query))] -pub fn read_query(name: &str) -> Result { - // Construct the path to the directory containing the SQL queries. - let query_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("queries"); - let query_path = query_dir.join(name).with_extension("sql"); - let query = std::fs::read_to_string(query_path) - .map_err(|e| internal_datafusion_err!("Failed to read query file: {}", e))?; - - // Record the query as part of the current span. - tracing::Span::current().record("query", query.as_str()); - info!("SQL Query:\n{}", query); - - Ok(query) -} - -#[instrument(level = "info", skip(ctx), fields(logical_plan))] -pub async fn parse_sql(ctx: &SessionContext, sql: &str) -> Result { - // Parse the SQL query into a DataFrame. - let df = ctx.sql(sql).await?; - - // Record the logical plan as part of the current span. - let logical_plan = df.logical_plan().display_indent_schema().to_string(); - tracing::Span::current().record("logical_plan", logical_plan.as_str()); - info!("Logical Plan:\n{}", logical_plan); - - Ok(df) -} - -#[instrument(level = "info", skip(df), fields(physical_plan))] -pub async fn create_physical_plan(df: DataFrame) -> Result> { - // Create a physical plan from the DataFrame. - let physical_plan = df.create_physical_plan().await?; - - // Record the physical plan as part of the current span. - let physical_plan_str = displayable(physical_plan.as_ref()).indent(true).to_string(); - tracing::Span::current().record("physical_plan", physical_plan_str.as_str()); - info!("Physical Plan:\n{}", physical_plan_str); +mod data; +mod distributed; +mod query; +mod session; - Ok(physical_plan) -} +// Re-export public API +pub use data::data_dir; +pub use distributed::DistributedMode; +pub use query::{create_physical_plan, parse_sql, read_query, run_traced_query}; +pub use session::{SessionBuilder, WorkerTaskWrapper, create_instrumentation_rule}; diff --git a/integration-utils/src/query.rs b/integration-utils/src/query.rs new file mode 100644 index 0000000..275c3e3 --- /dev/null +++ b/integration-utils/src/query.rs @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025 Datadog, Inc. + +//! Query execution utilities + +use std::path::PathBuf; +use std::sync::Arc; + +use datafusion::arrow::util::pretty::pretty_format_batches; +use datafusion::common::internal_datafusion_err; +use datafusion::physical_plan::{ExecutionPlan, collect, displayable}; +use datafusion::{error::Result, prelude::*}; +use tracing::{info, instrument}; + +/// Executes the SQL query with instrumentation enabled, providing detailed tracing output. +#[instrument(level = "info", skip(ctx))] +pub async fn run_traced_query(ctx: &SessionContext, query_name: &str) -> Result<()> { + info!("Beginning traced query execution"); + + let query = read_query(query_name)?; + + // Parse the SQL query into a DataFusion dataframe. + let df = parse_sql(ctx, query.as_str()).await?; + + // Generate a physical execution plan from the logical plan. + let physical_plan = create_physical_plan(df).await?; + + // Execute the physical plan and collect results. + let results = collect(physical_plan.clone(), ctx.task_ctx()).await?; + + info!("Query Results:\n{}", pretty_format_batches(&results)?); + + Ok(()) +} + +/// Reads a SQL query from a file. +#[instrument(level = "info", fields(query))] +pub fn read_query(name: &str) -> Result { + // Construct the path to the directory containing the SQL queries. + let query_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("queries"); + let query_path = query_dir.join(name).with_extension("sql"); + let query = std::fs::read_to_string(query_path) + .map_err(|e| internal_datafusion_err!("Failed to read query file: {}", e))?; + + // Record the query as part of the current span. + tracing::Span::current().record("query", query.as_str()); + info!("SQL Query:\n{}", query); + + Ok(query) +} + +/// Parses SQL into a DataFusion DataFrame. +#[instrument(level = "info", skip(ctx), fields(logical_plan))] +pub async fn parse_sql(ctx: &SessionContext, sql: &str) -> Result { + // Parse the SQL query into a DataFrame. + let df = ctx.sql(sql).await?; + + // Record the logical plan as part of the current span. + let logical_plan = df.logical_plan().display_indent_schema().to_string(); + tracing::Span::current().record("logical_plan", logical_plan.as_str()); + info!("Logical Plan:\n{}", logical_plan); + + Ok(df) +} + +/// Creates a physical execution plan from a DataFrame. +#[instrument(level = "info", skip(df), fields(physical_plan))] +pub async fn create_physical_plan(df: DataFrame) -> Result> { + // Create a physical plan from the DataFrame. + let physical_plan = df.create_physical_plan().await?; + + // Record the physical plan as part of the current span. + let physical_plan_str = displayable(physical_plan.as_ref()).indent(true).to_string(); + tracing::Span::current().record("physical_plan", physical_plan_str.as_str()); + info!("Physical Plan:\n{}", physical_plan_str); + + Ok(physical_plan) +} diff --git a/integration-utils/src/session.rs b/integration-utils/src/session.rs new file mode 100644 index 0000000..d56d913 --- /dev/null +++ b/integration-utils/src/session.rs @@ -0,0 +1,333 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025 Datadog, Inc. + +//! Session configuration and initialization + +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use datafusion::arrow::array::RecordBatch; +use datafusion::{ + error::Result, execution::SessionStateBuilder, + physical_optimizer::PhysicalOptimizerRule, prelude::*, +}; +use datafusion_distributed::{DistributedExt, DistributedPhysicalOptimizerRule}; +use datafusion_tracing::{ + InstrumentationOptions, instrument_with_info_spans, pretty_format_compact_batch, +}; +use instrumented_object_store::instrument_object_store; +use tracing::{field, info, instrument}; +use url::Url; + +use crate::data::register_tpch_tables; +use crate::distributed::{ + DEFAULT_WORKER_PORTS, DistributedMode, InMemoryChannelResolver, + LocalhostChannelResolver, WorkerInstrumentationConfig, spawn_localhost_workers, + spawn_localhost_workers_with_buffers, +}; + +/// Type alias for the worker task wrapper function used to set up task-local storage +pub type WorkerTaskWrapper = Arc< + dyn Fn( + u16, + Pin + Send>>, + ) -> Pin + Send>> + + Send + + Sync, +>; + +/// Builder for configuring and creating a DataFusion SessionContext with tracing instrumentation. +/// +/// # Example +/// +/// ```rust +/// use integration_utils::{SessionBuilder, DistributedMode}; +/// +/// # async fn example() -> datafusion::error::Result<()> { +/// // Create a session with custom configuration +/// let ctx = SessionBuilder::new() +/// .with_object_store_tracing() +/// .with_metrics() +/// .with_preview(10) +/// .with_compact_preview() +/// .with_distributed_mode(DistributedMode::Memory) +/// .build() +/// .await?; +/// # Ok(()) +/// # } +/// ``` +#[derive(Clone)] +pub struct SessionBuilder { + record_object_store: bool, + record_metrics: bool, + preview_limit: usize, + compact_preview: bool, + distributed_mode: Option, + worker_task_wrapper: Option, +} + +impl std::fmt::Debug for SessionBuilder { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SessionBuilder") + .field("record_object_store", &self.record_object_store) + .field("record_metrics", &self.record_metrics) + .field("preview_limit", &self.preview_limit) + .field("compact_preview", &self.compact_preview) + .field("distributed_mode", &self.distributed_mode) + .field("worker_task_wrapper", &self.worker_task_wrapper.is_some()) + .finish() + } +} + +impl Default for SessionBuilder { + fn default() -> Self { + Self::new() + } +} + +impl SessionBuilder { + /// Creates a new SessionBuilder with sensible default values. + /// + /// Default configuration: + /// - Object store tracing: disabled + /// - Metrics recording: disabled + /// - Preview limit: 0 (no preview) + /// - Compact preview: false + /// - Distributed mode: None (non-distributed) + pub fn new() -> Self { + Self { + record_object_store: false, + record_metrics: false, + preview_limit: 0, + compact_preview: false, + distributed_mode: None, + worker_task_wrapper: None, + } + } + + /// Enable object store tracing instrumentation. + /// + /// When enabled, file system access operations will be traced. + pub fn with_object_store_tracing(mut self) -> Self { + self.record_object_store = true; + self + } + + /// Enable metrics recording during query execution. + /// + /// When enabled, execution metrics like row counts and elapsed time will be recorded. + pub fn with_metrics(mut self) -> Self { + self.record_metrics = true; + self + } + + /// Set the number of rows to preview in trace spans. + /// + /// Set to 0 to disable previews. Non-zero values enable data previews in spans. + pub fn with_preview(mut self, limit: usize) -> Self { + self.preview_limit = limit; + self + } + + /// Enable compact preview formatting. + /// + /// When enabled (along with a non-zero preview limit), data previews will use + /// a more compact formatting style suitable for narrow displays. + pub fn with_compact_preview(mut self) -> Self { + self.compact_preview = true; + self + } + + /// Set the distributed execution mode. + /// + /// Use `Some(DistributedMode::Memory)` for in-memory distributed execution (testing), + /// or `Some(DistributedMode::Localhost)` for localhost TCP workers (more realistic). + /// Use `None` for non-distributed execution (default). + pub fn with_distributed_mode(mut self, mode: DistributedMode) -> Self { + self.distributed_mode = Some(mode); + self + } + + /// Set a custom worker task wrapper for localhost distributed mode. + /// + /// This wrapper function is called for each worker task and can be used to set up + /// task-local storage or other per-worker state (e.g., for routing logs to separate buffers). + pub fn with_worker_task_wrapper(mut self, wrapper: WorkerTaskWrapper) -> Self { + self.worker_task_wrapper = Some(wrapper); + self + } + + /// Returns the configured distributed mode, if any. + pub fn distributed_mode(&self) -> Option { + self.distributed_mode + } + + /// Returns the configured preview limit. + pub fn preview_limit(&self) -> usize { + self.preview_limit + } + + /// Builds and returns the configured SessionContext. + /// + /// This method initializes the DataFusion session with all the configured options, + /// sets up tracing instrumentation, registers the object store, and loads TPCH tables. + #[instrument(level = "info")] + pub async fn build(self) -> Result { + // Configure the session state with instrumentation for query execution. + let mut session_state_builder = SessionStateBuilder::new() + .with_default_features() + .with_config(SessionConfig::default().with_target_partitions(8)); // Enforce target partitions to ensure consistent test results regardless of the number of CPU cores. + + // Configure distributed execution if requested + if let Some(mode) = self.distributed_mode { + match mode { + DistributedMode::Memory => { + info!("Using in-memory channel resolver for distributed execution"); + session_state_builder = session_state_builder + .with_distributed_channel_resolver(InMemoryChannelResolver::new()) + .with_physical_optimizer_rule(Arc::new( + DistributedPhysicalOptimizerRule::new(), + )); + } + DistributedMode::Localhost => { + info!( + "Spawning localhost workers on ports {:?}", + DEFAULT_WORKER_PORTS + ); + // Create instrumentation config to pass to workers + let worker_config = if self.preview_limit > 0 || self.record_metrics { + Some(WorkerInstrumentationConfig { + record_metrics: self.record_metrics, + preview_limit: self.preview_limit, + compact_preview: self.compact_preview, + }) + } else { + None + }; + + // Spawn localhost workers - use the wrapper if provided + if let Some(wrapper) = self.worker_task_wrapper { + let wrapper_clone = wrapper.clone(); + spawn_localhost_workers_with_buffers( + DEFAULT_WORKER_PORTS, + worker_config, + move |port, fut| { + let w = wrapper_clone.clone(); + Box::pin(w(port, fut)) + }, + ) + .await?; + } else { + spawn_localhost_workers(DEFAULT_WORKER_PORTS, worker_config) + .await?; + } + + let localhost_resolver = + LocalhostChannelResolver::new(DEFAULT_WORKER_PORTS.to_vec()); + + session_state_builder = session_state_builder + .with_distributed_channel_resolver(localhost_resolver) + .with_physical_optimizer_rule(Arc::new( + DistributedPhysicalOptimizerRule::new() + .with_network_coalesce_tasks(2) + .with_network_shuffle_tasks(2), + )); + } + } + } + + // Create the instrumentation rule for the coordinator + let instrumentation_rule = create_instrumentation_rule( + self.record_metrics, + self.preview_limit, + self.compact_preview, + ); + session_state_builder = + session_state_builder.with_physical_optimizer_rule(instrumentation_rule); + let session_state = session_state_builder.build(); + + let ctx = SessionContext::new_with_state(session_state); + + // Instrument the local filesystem object store for tracing file access. + let local_store = Arc::new(object_store::local::LocalFileSystem::new()); + let object_store = if self.record_object_store { + instrument_object_store(local_store, "local_fs") + } else { + local_store + }; + + // Register the instrumented object store for handling file:// URLs. + ctx.register_object_store(&Url::parse("file://").unwrap(), object_store); + + // Register the tpch tables. + register_tpch_tables(&ctx).await?; + + Ok(ctx) + } +} + +/// Creates an instrumentation rule that captures metrics and provides previews of data during execution. +pub fn create_instrumentation_rule( + record_metrics: bool, + preview_limit: usize, + compact_preview: bool, +) -> Arc { + create_instrumentation_rule_with_worker_id( + record_metrics, + preview_limit, + compact_preview, + None, + ) +} + +/// Creates an instrumentation rule with an optional worker identifier. +pub fn create_instrumentation_rule_with_worker_id( + record_metrics: bool, + preview_limit: usize, + compact_preview: bool, + worker_id: Option, +) -> Arc { + let mut options_builder = InstrumentationOptions::builder() + .add_custom_field("env", "production") // Custom fields + .add_custom_field("region", "us-west") + .record_metrics(record_metrics) + .preview_limit(preview_limit); + + // Add worker identifier if provided + if let Some(id) = worker_id { + options_builder = options_builder.add_custom_field("worker.id", id); + } + + let options_builder = if compact_preview { + options_builder.preview_fn(Arc::new(|batch: &RecordBatch| { + // Format data batches for compact preview in span fields. + pretty_format_compact_batch(batch, 64, 3, 10).map(|fmt| fmt.to_string()) + })) + } else { + options_builder + }; + + instrument_with_info_spans!( + options: options_builder.build(), + env = field::Empty, // custom fields keys must be defined at compile time + region = field::Empty, + worker.id = field::Empty, + ) +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml index ca2c3e1..6f7d2a7 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -18,5 +18,5 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025 Datadog, Inc. [toolchain] -channel = "1.86.0" +channel = "1.89.0" components = ["rustfmt", "clippy"] diff --git a/rustfmt.toml b/rustfmt.toml index 6168c44..9b7c827 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -17,5 +17,5 @@ # # This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025 Datadog, Inc. -edition = "2021" +edition = "2024" max_width = 90 diff --git a/tests/Cargo.toml b/tests/Cargo.toml index abdd4e2..ecb0b1e 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -45,6 +45,8 @@ path = "integration_tests.rs" datafusion = { workspace = true, features = ["parquet", "nested_expressions"] } insta = { version = "1.43", features = ["filters", "json"] } integration-utils = { path = "../integration-utils" } +opentelemetry = { version = "0.30", features = ["trace"] } +opentelemetry_sdk = { version = "0.30", features = ["trace"] } serde_json = "1.0" tokio = { workspace = true, features = ["rt-multi-thread"] } tracing = { workspace = true } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 283ab41..197cf67 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -19,39 +19,37 @@ mod test_utils; +use std::collections::HashMap; use std::sync::{Arc, Mutex, Once, OnceLock}; use datafusion::error::Result; use insta::{assert_json_snapshot, assert_snapshot}; -use integration_utils::{init_session, run_traced_query}; +use integration_utils::{DistributedMode, SessionBuilder, run_traced_query}; use serde_json::Value; use tracing::{Instrument, Level}; use tracing_subscriber::fmt::format::FmtSpan; -use tracing_subscriber::{fmt, prelude::*, Registry}; +use tracing_subscriber::{Registry, fmt, prelude::*}; -use crate::test_utils::in_memory_writer::InMemoryMakeWriter; use crate::test_utils::insta_settings; use crate::test_utils::insta_settings::preview_redacted_settings; +use crate::test_utils::multi_buffer_writer::MultiBufferMakeWriter; // A global buffer to store logs across tests, initialized only once. static LOG_BUFFER: OnceLock>>> = OnceLock::new(); +// A global multi-buffer writer for distributed localhost mode +static MULTI_BUFFER_WRITER: OnceLock = OnceLock::new(); + // A global "once" to ensure the subscriber is initialized only once. static SUBSCRIBER_INIT: Once = Once::new(); /// A struct describing how a particular query test should be run. -#[derive(Debug, Default)] +#[derive(Debug)] struct QueryTestCase<'a> { /// The SQL query to run. sql_query: &'a str, - /// Whether to instrument the object store for this test. - record_object_store: bool, - /// Whether to collect (record) metrics for this test. - should_record_metrics: bool, - /// Maximum number of rows to preview in logs. - row_limit: usize, - /// Use compact formatting for the row preview. - use_compact_preview: bool, + /// Session builder for configuring the DataFusion session. + session_builder: SessionBuilder, /// Indices of spans for which preview assertions should be skipped. ignored_preview_spans: &'a [usize], /// Whether to ignore the full trace in assertions. @@ -62,27 +60,32 @@ impl<'a> QueryTestCase<'a> { fn new(sql_query: &'a str) -> Self { Self { sql_query, - ..Default::default() + session_builder: SessionBuilder::new(), + ignored_preview_spans: &[], + ignore_full_trace: false, } } - fn with_object_store_collection(mut self) -> Self { - self.record_object_store = true; + fn with_object_store_tracing(mut self) -> Self { + self.session_builder = self.session_builder.with_object_store_tracing(); self } - fn with_metrics_collection(mut self) -> Self { - self.should_record_metrics = true; + fn with_metrics(mut self) -> Self { + self.session_builder = self.session_builder.with_metrics(); self } - fn with_row_limit(mut self, limit: usize) -> Self { - self.row_limit = limit; + fn with_preview(mut self, limit: usize) -> Self { + self.session_builder = self.session_builder.with_preview(limit); self } - fn with_compact_preview(mut self) -> Self { - self.use_compact_preview = true; + fn with_compact_preview(mut self, limit: usize) -> Self { + self.session_builder = self + .session_builder + .with_preview(limit) + .with_compact_preview(); self } @@ -95,18 +98,45 @@ impl<'a> QueryTestCase<'a> { self.ignore_full_trace = true; self } + + fn distributed_memory(mut self) -> Self { + self.session_builder = self + .session_builder + .with_distributed_mode(DistributedMode::Memory); + self + } + + fn distributed_localhost(mut self) -> Self { + use crate::test_utils::multi_buffer_writer::WORKER_PORT; + use integration_utils::WorkerTaskWrapper; + + // Create a worker task wrapper that sets the WORKER_PORT task-local + let wrapper: WorkerTaskWrapper = + Arc::new(move |port, fut| Box::pin(WORKER_PORT.scope(port, fut))); + + self.session_builder = self + .session_builder + .with_distributed_mode(DistributedMode::Localhost) + .with_worker_task_wrapper(wrapper); + self + } + + /// Returns the session builder for this test case. + fn session_builder(self) -> SessionBuilder { + self.session_builder + } } #[tokio::test(flavor = "multi_thread", worker_threads = 8)] async fn test_basic() -> Result<()> { - execute_test_case("01_basic", &QueryTestCase::new("select_one")).await + execute_test_case("01_basic", QueryTestCase::new("select_one")).await } #[tokio::test(flavor = "multi_thread", worker_threads = 8)] async fn test_basic_metrics() -> Result<()> { execute_test_case( "02_basic_metrics", - &QueryTestCase::new("select_one").with_metrics_collection(), + QueryTestCase::new("select_one").with_metrics(), ) .await } @@ -115,7 +145,7 @@ async fn test_basic_metrics() -> Result<()> { async fn test_basic_preview() -> Result<()> { execute_test_case( "03_basic_preview", - &QueryTestCase::new("select_one").with_row_limit(5), + QueryTestCase::new("select_one").with_preview(5), ) .await } @@ -124,9 +154,7 @@ async fn test_basic_preview() -> Result<()> { async fn test_basic_compact_preview() -> Result<()> { execute_test_case( "04_basic_compact_preview", - &QueryTestCase::new("select_one") - .with_row_limit(5) - .with_compact_preview(), + QueryTestCase::new("select_one").with_compact_preview(5), ) .await } @@ -135,10 +163,9 @@ async fn test_basic_compact_preview() -> Result<()> { async fn test_basic_all_options() -> Result<()> { execute_test_case( "05_basic_all_options", - &QueryTestCase::new("select_one") - .with_metrics_collection() - .with_row_limit(5) - .with_compact_preview(), + QueryTestCase::new("select_one") + .with_metrics() + .with_compact_preview(5), ) .await } @@ -147,11 +174,10 @@ async fn test_basic_all_options() -> Result<()> { async fn test_object_store_all_options() -> Result<()> { execute_test_case( "06_object_store_all_options", - &QueryTestCase::new("order_nations") - .with_object_store_collection() - .with_metrics_collection() - .with_row_limit(5) - .with_compact_preview(), + QueryTestCase::new("order_nations") + .with_object_store_tracing() + .with_metrics() + .with_compact_preview(5), ) .await } @@ -160,10 +186,9 @@ async fn test_object_store_all_options() -> Result<()> { async fn test_scrabble_all_options() -> Result<()> { execute_test_case( "07_scrabble_all_options", - &QueryTestCase::new("tpch_scrabble") - .with_metrics_collection() - .with_row_limit(5) - .with_compact_preview() + QueryTestCase::new("tpch_scrabble") + .with_metrics() + .with_compact_preview(5) // skip preview assertions for these spans as they depend on the partitioning // and are not deterministic .ignore_preview_spans(&[15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]) @@ -175,17 +200,16 @@ async fn test_scrabble_all_options() -> Result<()> { #[tokio::test(flavor = "multi_thread", worker_threads = 8)] async fn test_recursive() -> Result<()> { - execute_test_case("08_recursive", &QueryTestCase::new("recursive")).await + execute_test_case("08_recursive", QueryTestCase::new("recursive")).await } #[tokio::test(flavor = "multi_thread", worker_threads = 8)] async fn test_recursive_all_options() -> Result<()> { execute_test_case( "09_recursive_all_options", - &QueryTestCase::new("recursive") - .with_metrics_collection() - .with_row_limit(5) - .with_compact_preview(), + QueryTestCase::new("recursive") + .with_metrics() + .with_compact_preview(5), ) .await } @@ -193,66 +217,153 @@ async fn test_recursive_all_options() -> Result<()> { #[tokio::test(flavor = "multi_thread", worker_threads = 8)] async fn test_topk_lineitem() -> Result<()> { // Expect a filled `DynamicFilterPhysicalExpr` on the `DataSourceExec` "datafusion.node" field - execute_test_case("10_topk_lineitem", &QueryTestCase::new("topk_lineitem")).await + execute_test_case("10_topk_lineitem", QueryTestCase::new("topk_lineitem")).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 8)] +async fn test_weather() -> Result<()> { + execute_test_case( + "11_weather", + QueryTestCase::new("weather") + .with_metrics() + .with_compact_preview(5), + ) + .await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 8)] +async fn test_distributed_weather_memory() -> Result<()> { + execute_test_case( + "12_distributed_weather_memory", + QueryTestCase::new("weather") + .distributed_memory() + .with_metrics() + .with_compact_preview(5), + ) + .await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 8)] +async fn test_distributed_weather_localhost() -> Result<()> { + execute_test_case( + "13_distributed_weather_localhost", + QueryTestCase::new("weather") + .distributed_localhost() + .with_metrics() + .with_compact_preview(5), + ) + .await } /// Executes the provided [`QueryTestCase`], setting up tracing and verifying /// log output according to its parameters. -async fn execute_test_case(test_name: &str, test_case: &QueryTestCase<'_>) -> Result<()> { - // Initialize tracing infrastructure and collect a log buffer. - let log_buffer = init_tracing(); +async fn execute_test_case(test_name: &str, test_case: QueryTestCase<'_>) -> Result<()> { + // Store test case parameters before consuming it + let sql_query = test_case.sql_query; + let preview_limit = test_case.session_builder.preview_limit(); + let ignored_preview_spans = test_case.ignored_preview_spans; + let ignore_full_trace = test_case.ignore_full_trace; + let is_localhost_distributed = matches!( + test_case.session_builder.distributed_mode(), + Some(DistributedMode::Localhost) + ); + + // Initialize tracing infrastructure (always uses multi-buffer mode) + let _log_buffer = init_tracing(); // Initialize the DataFusion session with the requested options. - let ctx = init_session( - test_case.record_object_store, - test_case.should_record_metrics, - test_case.row_limit, - test_case.use_compact_preview, - ) - .await?; + let ctx = test_case.session_builder().build().await?; // Run the SQL query with tracing enabled. - run_traced_query(&ctx, test_case.sql_query) + run_traced_query(&ctx, sql_query) .instrument(tracing::info_span!("test", test_name = %test_name)) .await?; - // Clone the log buffer to avoid holding the lock while processing. - let logs = log_buffer.lock().unwrap().clone(); + // Collect logs - use multi-buffer writer for localhost distributed tests + let logs = if is_localhost_distributed { + if let Some(multi_buffer_writer) = MULTI_BUFFER_WRITER.get() { + multi_buffer_writer.collect_all_buffers() + } else { + // Fallback to main buffer if multi-buffer wasn't initialized + _log_buffer.lock().unwrap().clone() + } + } else { + // Non-distributed or memory-distributed tests use the main buffer + _log_buffer.lock().unwrap().clone() + }; + let log_str = String::from_utf8_lossy(&logs); // Convert each log line to JSON, filtering by this test case name. + // For localhost distributed tests, also include spans without a test name + // (these are worker execution spans that run in separate gRPC handlers) let json_lines = log_str .lines() .map(|line| serde_json::from_str(line).expect("Failed to parse JSON line")) - .filter(|json_line| extract_test_name(json_line) == *test_name) + .filter(|json_line| { + let test_name_in_span = extract_test_name(json_line); + if is_localhost_distributed { + // For localhost tests, include spans that match the test name OR have no test name + // (worker spans don't have test context due to gRPC boundary) + test_name_in_span.is_empty() || test_name_in_span == *test_name + } else { + // For non-distributed tests, only include spans that match the test name + test_name_in_span == *test_name + } + }) .collect::>(); // Bind insta settings for snapshot testing. let _insta_guard = insta_settings::settings().bind_to_scope(); - // If we have a preview row_limit, do dedicated assertions on the previews. - if test_case.row_limit > 0 { - let mut preview_id = 0; + // If we have a preview limit, do dedicated assertions on the previews. + if preview_limit > 0 { + // Collect all preview spans with their worker identifier and node information + let mut preview_spans: Vec<(String, String, Option, String)> = vec![]; + for json_line in &json_lines { if let Some(span_name) = extract_json_field_value(json_line, "otel.name") { if let Some(preview) = extract_json_field_value(json_line, "datafusion.preview") { - // Only assert if we have not been asked to ignore this preview span. - if !test_case.ignored_preview_spans.contains(&preview_id) { - assert_snapshot!( - format!("{test_name}_{:02}_{span_name}", preview_id), - preview - ); - } - preview_id += 1; + // Extract a unique identifier for the span (use datafusion.node as it's unique per operator) + let span_id = extract_json_field_value(json_line, "datafusion.node") + .unwrap_or_else(|| span_name.clone()); + // Extract worker identifier if available + let worker_id = extract_json_field_value(json_line, "worker.id"); + preview_spans.push((span_name, span_id, worker_id, preview)); } } } + + // Sort spans deterministically by node description and worker ID + // This ensures consistent ordering across test runs + preview_spans.sort_by(|a, b| { + // First sort by node description, then by worker ID + match a.1.cmp(&b.1) { + std::cmp::Ordering::Equal => a.2.cmp(&b.2), + other => other, + } + }); + + for (preview_id, (span_name, _, worker_id, preview)) in + preview_spans.iter().enumerate() + { + // Only assert if we have not been asked to ignore this preview span. + if !ignored_preview_spans.contains(&preview_id) { + // Include worker ID in snapshot name for distributed tests if available + let snapshot_name = if let Some(worker) = worker_id { + format!("{test_name}_{:02}_{span_name}_{worker}", preview_id) + } else { + format!("{test_name}_{:02}_{span_name}", preview_id) + }; + assert_snapshot!(snapshot_name, preview); + } + } } // General assertion on the full trace. - if !test_case.ignore_full_trace { + if !ignore_full_trace { // Redact `datafusion.preview` values as they are often non-deterministic. preview_redacted_settings().bind(|| { assert_json_snapshot!(format!("{test_name}_trace"), json_lines); @@ -287,21 +398,47 @@ fn extract_test_name(json_line: &Value) -> String { /// Initializes an in-memory logging/tracing pipeline exactly once, /// returning a reference to the shared log buffer for all tests. +/// +/// Always uses multi-buffer routing so localhost worker tests can route logs +/// to separate buffers. Non-localhost tests will simply use the main buffer. fn init_tracing() -> &'static Arc>> { let log_buffer = LOG_BUFFER.get_or_init(Arc::default); - // Ensure the global subscriber is initialized only once. + // Ensure the global subscriber is initialized only once with multi-buffer support SUBSCRIBER_INIT.call_once(|| { - init_subscriber(log_buffer.clone()); + init_subscriber_multi_buffer(log_buffer.clone()); }); log_buffer } -/// Sets up a tracing subscriber pipeline that writes to both the shared in-memory buffer -/// and stdout (for info-level logs). All subsequent tests share this subscriber setup. -fn init_subscriber(log_buffer: Arc>>) { - // Create a layer that outputs only spans to the log buffer as JSON. +/// Sets up a tracing subscriber with multi-buffer support for localhost workers. +/// Each worker gets its own buffer based on task-local storage. +fn init_subscriber_multi_buffer(main_buffer: Arc>>) { + // Set up the global text map propagator for trace context propagation. + // Even though these tests don't use OpenTelemetry exporters, setting this up + // ensures that if OpenTelemetry is added in the future, context will propagate correctly. + // The propagator is a noop when no OpenTelemetry tracer is active. + use opentelemetry_sdk::propagation::TraceContextPropagator; + opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new()); + + // Create worker buffers for the default worker ports + let worker_buffers = Arc::new(Mutex::new(HashMap::new())); + + // Pre-create buffers for all expected worker ports + { + let mut buffers = worker_buffers.lock().unwrap(); + for &port in &[50051u16, 50052, 50053, 50054] { + buffers.insert(port, Arc::new(Mutex::new(Vec::new()))); + } + } + + let multi_buffer_writer = MultiBufferMakeWriter::new(main_buffer, worker_buffers); + + // Store the multi-buffer writer globally so we can access it later for collecting logs + let _ = MULTI_BUFFER_WRITER.set(multi_buffer_writer.clone()); + + // Create a layer that outputs only spans to the appropriate buffer as JSON let assertion_layer = fmt::layer() .with_span_events(FmtSpan::CLOSE) .event_format(fmt::format().json().flatten_event(true).without_time()) @@ -309,17 +446,17 @@ fn init_subscriber(log_buffer: Arc>>) { .flatten_event(true) .with_ansi(false) .with_writer( - InMemoryMakeWriter::new(log_buffer) + multi_buffer_writer .with_max_level(Level::INFO) .with_filter(|meta| meta.is_span()), ); - // Create a layer for INFO-level logs/spans to stdout. + // Create a layer for INFO-level logs/spans to stdout let stdout_layer = fmt::layer() .with_span_events(FmtSpan::CLOSE) .with_writer(std::io::stdout.with_max_level(Level::INFO)); - // Register the layers. + // Register the layers Registry::default() .with(assertion_layer) .with(stdout_layer) diff --git a/tests/snapshots/01_basic_trace.snap b/tests/snapshots/01_basic_trace.snap index 1b3a573..87f5714 100644 --- a/tests/snapshots/01_basic_trace.snap +++ b/tests/snapshots/01_basic_trace.snap @@ -20,7 +20,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -42,7 +42,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -63,7 +63,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -90,7 +90,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -117,7 +117,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -134,7 +134,7 @@ expression: json_lines "test_name": "01_basic" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" } diff --git a/tests/snapshots/02_basic_metrics_trace.snap b/tests/snapshots/02_basic_metrics_trace.snap index a809c95..89e021f 100644 --- a/tests/snapshots/02_basic_metrics_trace.snap +++ b/tests/snapshots/02_basic_metrics_trace.snap @@ -20,7 +20,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -42,7 +42,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -63,7 +63,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -90,7 +90,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -121,7 +121,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -138,7 +138,7 @@ expression: json_lines "test_name": "02_basic_metrics" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" } diff --git a/tests/snapshots/03_basic_preview_trace.snap b/tests/snapshots/03_basic_preview_trace.snap index 311caef..f128c23 100644 --- a/tests/snapshots/03_basic_preview_trace.snap +++ b/tests/snapshots/03_basic_preview_trace.snap @@ -20,7 +20,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -42,7 +42,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -63,7 +63,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -91,7 +91,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -119,7 +119,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -136,7 +136,7 @@ expression: json_lines "test_name": "03_basic_preview" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" } diff --git a/tests/snapshots/04_basic_compact_preview_trace.snap b/tests/snapshots/04_basic_compact_preview_trace.snap index f1501cc..42554d9 100644 --- a/tests/snapshots/04_basic_compact_preview_trace.snap +++ b/tests/snapshots/04_basic_compact_preview_trace.snap @@ -20,7 +20,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -42,7 +42,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -63,7 +63,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -91,7 +91,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -119,7 +119,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -136,7 +136,7 @@ expression: json_lines "test_name": "04_basic_compact_preview" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" } diff --git a/tests/snapshots/05_basic_all_options_trace.snap b/tests/snapshots/05_basic_all_options_trace.snap index 133397d..8be8b75 100644 --- a/tests/snapshots/05_basic_all_options_trace.snap +++ b/tests/snapshots/05_basic_all_options_trace.snap @@ -20,7 +20,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -42,7 +42,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -63,7 +63,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -91,7 +91,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -123,7 +123,7 @@ expression: json_lines "query_name": "select_one" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -140,7 +140,7 @@ expression: json_lines "test_name": "05_basic_all_options" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" } diff --git a/tests/snapshots/06_object_store_all_options_trace.snap b/tests/snapshots/06_object_store_all_options_trace.snap index 1f52372..5af2ddf 100644 --- a/tests/snapshots/06_object_store_all_options_trace.snap +++ b/tests/snapshots/06_object_store_all_options_trace.snap @@ -20,7 +20,7 @@ expression: json_lines "query_name": "order_nations" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -42,7 +42,7 @@ expression: json_lines "query_name": "order_nations" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -89,7 +89,7 @@ expression: json_lines "query_name": "order_nations" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -185,7 +185,7 @@ expression: json_lines "query_name": "order_nations" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -220,7 +220,7 @@ expression: json_lines "query_name": "order_nations" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -237,7 +237,7 @@ expression: json_lines "test_name": "06_object_store_all_options" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" } diff --git a/tests/snapshots/07_scrabble_all_options_00_AggregateExec.snap b/tests/snapshots/07_scrabble_all_options_00_AggregateExec.snap new file mode 100644 index 0000000..9fc2dc2 --- /dev/null +++ b/tests/snapshots/07_scrabble_all_options_00_AggregateExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++-------------+-------------+-------------+---------------+ +| n_nationkey | nation_name | region_name | sum(s.points) | +|=============+=============+=============+===============| +| 6 | FRANCE | EUROPE | 11 | +|-------------+-------------+-------------+---------------| +| 19 | ROMANIA | EUROPE | 9 | +|-------------+-------------+-------------+---------------| +| 5 | ETHIOPIA | AFRICA | 13 | +|-------------+-------------+-------------+---------------| +| 9 | INDONESIA | ASIA | 10 | +|-------------+-------------+-------------+---------------| +| 12 | JAPAN | ASIA | 14 | ++-------------+-------------+-------------+---------------+ diff --git a/tests/snapshots/07_scrabble_all_options_01_AggregateExec.snap b/tests/snapshots/07_scrabble_all_options_01_AggregateExec.snap new file mode 100644 index 0000000..7a17d46 --- /dev/null +++ b/tests/snapshots/07_scrabble_all_options_01_AggregateExec.snap @@ -0,0 +1,18 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++-------------+---------------+-------------+------------------+ +| n_nationkey | nation_name | region_name | sum(s.points)[su | +| | | | m] | +|=============+===============+=============+==================| +| 1 | ARGENTINA | AMERICA | 10 | +|-------------+---------------+-------------+------------------| +| 2 | BRAZIL | AMERICA | 17 | +|-------------+---------------+-------------+------------------| +| 3 | CANADA | AMERICA | 9 | +|-------------+---------------+-------------+------------------| +| 17 | PERU | AMERICA | 6 | +|-------------+---------------+-------------+------------------| +| 24 | UNITED STATES | AMERICA | 13 | ++-------------+---------------+-------------+------------------+ diff --git a/tests/snapshots/07_scrabble_all_options_01_ProjectionExec.snap b/tests/snapshots/07_scrabble_all_options_01_ProjectionExec.snap deleted file mode 100644 index 8ba4f7a..0000000 --- a/tests/snapshots/07_scrabble_all_options_01_ProjectionExec.snap +++ /dev/null @@ -1,17 +0,0 @@ ---- -source: tests/integration_tests.rs -expression: preview ---- -+--------+--------+ -| letter | points | -|========+========| -| A | 1 | -|--------+--------| -| B | 3 | -|--------+--------| -| C | 3 | -|--------+--------| -| D | 2 | -|--------+--------| -| E | 1 | -+--------+--------+ diff --git a/tests/snapshots/07_scrabble_all_options_26_BoundedWindowAggExec.snap b/tests/snapshots/07_scrabble_all_options_02_BoundedWindowAggExec.snap similarity index 100% rename from tests/snapshots/07_scrabble_all_options_26_BoundedWindowAggExec.snap rename to tests/snapshots/07_scrabble_all_options_02_BoundedWindowAggExec.snap diff --git a/tests/snapshots/07_scrabble_all_options_04_RepartitionExec.snap b/tests/snapshots/07_scrabble_all_options_03_CoalesceBatchesExec.snap similarity index 100% rename from tests/snapshots/07_scrabble_all_options_04_RepartitionExec.snap rename to tests/snapshots/07_scrabble_all_options_03_CoalesceBatchesExec.snap diff --git a/tests/snapshots/07_scrabble_all_options_03_RepartitionExec.snap b/tests/snapshots/07_scrabble_all_options_03_RepartitionExec.snap deleted file mode 100644 index 382fce2..0000000 --- a/tests/snapshots/07_scrabble_all_options_03_RepartitionExec.snap +++ /dev/null @@ -1,17 +0,0 @@ ---- -source: tests/integration_tests.rs -expression: preview ---- -+-------------+-----------+-------------+ -| n_nationkey | n_name | n_regionkey | -|=============+===========+=============| -| 0 | ALGERIA | 0 | -|-------------+-----------+-------------| -| 1 | ARGENTINA | 1 | -|-------------+-----------+-------------| -| 2 | BRAZIL | 1 | -|-------------+-----------+-------------| -| 3 | CANADA | 1 | -|-------------+-----------+-------------| -| 4 | EGYPT | 4 | -+-------------+-----------+-------------+ diff --git a/tests/snapshots/07_scrabble_all_options_08_RepartitionExec.snap b/tests/snapshots/07_scrabble_all_options_04_CoalesceBatchesExec.snap similarity index 100% rename from tests/snapshots/07_scrabble_all_options_08_RepartitionExec.snap rename to tests/snapshots/07_scrabble_all_options_04_CoalesceBatchesExec.snap diff --git a/tests/snapshots/07_scrabble_all_options_05_CoalesceBatchesExec.snap b/tests/snapshots/07_scrabble_all_options_05_CoalesceBatchesExec.snap index 50aaeda..6bad06c 100644 --- a/tests/snapshots/07_scrabble_all_options_05_CoalesceBatchesExec.snap +++ b/tests/snapshots/07_scrabble_all_options_05_CoalesceBatchesExec.snap @@ -2,16 +2,16 @@ source: tests/integration_tests.rs expression: preview --- -+-------------+---------------+-------------+ -| n_nationkey | n_name | n_regionkey | -|=============+===============+=============| -| 1 | ARGENTINA | 1 | -|-------------+---------------+-------------| -| 2 | BRAZIL | 1 | -|-------------+---------------+-------------| -| 3 | CANADA | 1 | -|-------------+---------------+-------------| -| 17 | PERU | 1 | -|-------------+---------------+-------------| -| 24 | UNITED STATES | 1 | -+-------------+---------------+-------------+ ++-------------+---------------+---------+ +| n_nationkey | n_name | r_name | +|=============+===============+=========| +| 1 | ARGENTINA | AMERICA | +|-------------+---------------+---------| +| 2 | BRAZIL | AMERICA | +|-------------+---------------+---------| +| 3 | CANADA | AMERICA | +|-------------+---------------+---------| +| 17 | PERU | AMERICA | +|-------------+---------------+---------| +| 24 | UNITED STATES | AMERICA | ++-------------+---------------+---------+ diff --git a/tests/snapshots/07_scrabble_all_options_06_CoalesceBatchesExec.snap b/tests/snapshots/07_scrabble_all_options_06_CoalesceBatchesExec.snap new file mode 100644 index 0000000..ce6b631 --- /dev/null +++ b/tests/snapshots/07_scrabble_all_options_06_CoalesceBatchesExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++--------+-------------+-------------+-------------+ +| points | n_nationkey | nation_name | region_name | +|========+=============+=============+=============| +| 1 | 1 | ARGENTINA | AMERICA | +|--------+-------------+-------------+-------------| +| 1 | 1 | ARGENTINA | AMERICA | +|--------+-------------+-------------+-------------| +| 2 | 1 | ARGENTINA | AMERICA | +|--------+-------------+-------------+-------------| +| 1 | 1 | ARGENTINA | AMERICA | +|--------+-------------+-------------+-------------| +| 1 | 1 | ARGENTINA | AMERICA | ++--------+-------------+-------------+-------------+ diff --git a/tests/snapshots/07_scrabble_all_options_07_CoalesceBatchesExec.snap b/tests/snapshots/07_scrabble_all_options_07_CoalesceBatchesExec.snap new file mode 100644 index 0000000..d7a835c --- /dev/null +++ b/tests/snapshots/07_scrabble_all_options_07_CoalesceBatchesExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++-------------+-------------+-------------+--------------------+ +| n_nationkey | nation_name | region_name | sum(s.points)[sum] | +|=============+=============+=============+====================| +| 6 | FRANCE | EUROPE | 11 | +|-------------+-------------+-------------+--------------------| +| 19 | ROMANIA | EUROPE | 9 | +|-------------+-------------+-------------+--------------------| +| 5 | ETHIOPIA | AFRICA | 13 | +|-------------+-------------+-------------+--------------------| +| 9 | INDONESIA | ASIA | 10 | +|-------------+-------------+-------------+--------------------| +| 12 | JAPAN | ASIA | 14 | ++-------------+-------------+-------------+--------------------+ diff --git a/tests/snapshots/07_scrabble_all_options_07_RepartitionExec.snap b/tests/snapshots/07_scrabble_all_options_07_RepartitionExec.snap deleted file mode 100644 index 13a3a65..0000000 --- a/tests/snapshots/07_scrabble_all_options_07_RepartitionExec.snap +++ /dev/null @@ -1,17 +0,0 @@ ---- -source: tests/integration_tests.rs -expression: preview ---- -+-------------+-------------+ -| r_regionkey | r_name | -|=============+=============| -| 0 | AFRICA | -|-------------+-------------| -| 1 | AMERICA | -|-------------+-------------| -| 2 | ASIA | -|-------------+-------------| -| 3 | EUROPE | -|-------------+-------------| -| 4 | MIDDLE EAST | -+-------------+-------------+ diff --git a/tests/snapshots/07_scrabble_all_options_08_CoalesceBatchesExec.snap b/tests/snapshots/07_scrabble_all_options_08_CoalesceBatchesExec.snap new file mode 100644 index 0000000..7b1bf48 --- /dev/null +++ b/tests/snapshots/07_scrabble_all_options_08_CoalesceBatchesExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++-------------+---------------+----------------+ +| region_name | nation_name | scrabble_score | +|=============+===============+================| +| AMERICA | ARGENTINA | 10 | +|-------------+---------------+----------------| +| AMERICA | UNITED STATES | 13 | +|-------------+---------------+----------------| +| AMERICA | PERU | 6 | +|-------------+---------------+----------------| +| AMERICA | BRAZIL | 17 | +|-------------+---------------+----------------| +| AMERICA | CANADA | 9 | ++-------------+---------------+----------------+ diff --git a/tests/snapshots/07_scrabble_all_options_09_CoalesceBatchesExec.snap b/tests/snapshots/07_scrabble_all_options_09_CoalesceBatchesExec.snap deleted file mode 100644 index f0cae99..0000000 --- a/tests/snapshots/07_scrabble_all_options_09_CoalesceBatchesExec.snap +++ /dev/null @@ -1,17 +0,0 @@ ---- -source: tests/integration_tests.rs -expression: preview ---- -+-------------+-------------+ -| r_regionkey | r_name | -|=============+=============| -| 1 | AMERICA | -|-------------+-------------| -| 3 | EUROPE | -|-------------+-------------| -| 0 | AFRICA | -|-------------+-------------| -| 2 | ASIA | -|-------------+-------------| -| 4 | MIDDLE EAST | -+-------------+-------------+ diff --git a/tests/snapshots/07_scrabble_all_options_02_DataSourceExec.snap b/tests/snapshots/07_scrabble_all_options_09_DataSourceExec.snap similarity index 100% rename from tests/snapshots/07_scrabble_all_options_02_DataSourceExec.snap rename to tests/snapshots/07_scrabble_all_options_09_DataSourceExec.snap diff --git a/tests/snapshots/07_scrabble_all_options_06_DataSourceExec.snap b/tests/snapshots/07_scrabble_all_options_10_DataSourceExec.snap similarity index 100% rename from tests/snapshots/07_scrabble_all_options_06_DataSourceExec.snap rename to tests/snapshots/07_scrabble_all_options_10_DataSourceExec.snap diff --git a/tests/snapshots/07_scrabble_all_options_11_CoalesceBatchesExec.snap b/tests/snapshots/07_scrabble_all_options_11_CoalesceBatchesExec.snap deleted file mode 100644 index 6bad06c..0000000 --- a/tests/snapshots/07_scrabble_all_options_11_CoalesceBatchesExec.snap +++ /dev/null @@ -1,17 +0,0 @@ ---- -source: tests/integration_tests.rs -expression: preview ---- -+-------------+---------------+---------+ -| n_nationkey | n_name | r_name | -|=============+===============+=========| -| 1 | ARGENTINA | AMERICA | -|-------------+---------------+---------| -| 2 | BRAZIL | AMERICA | -|-------------+---------------+---------| -| 3 | CANADA | AMERICA | -|-------------+---------------+---------| -| 17 | PERU | AMERICA | -|-------------+---------------+---------| -| 24 | UNITED STATES | AMERICA | -+-------------+---------------+---------+ diff --git a/tests/snapshots/07_scrabble_all_options_00_DataSourceExec.snap b/tests/snapshots/07_scrabble_all_options_11_DataSourceExec.snap similarity index 100% rename from tests/snapshots/07_scrabble_all_options_00_DataSourceExec.snap rename to tests/snapshots/07_scrabble_all_options_11_DataSourceExec.snap diff --git a/tests/snapshots/07_scrabble_all_options_12_HashJoinExec.snap b/tests/snapshots/07_scrabble_all_options_12_HashJoinExec.snap new file mode 100644 index 0000000..ce6b631 --- /dev/null +++ b/tests/snapshots/07_scrabble_all_options_12_HashJoinExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++--------+-------------+-------------+-------------+ +| points | n_nationkey | nation_name | region_name | +|========+=============+=============+=============| +| 1 | 1 | ARGENTINA | AMERICA | +|--------+-------------+-------------+-------------| +| 1 | 1 | ARGENTINA | AMERICA | +|--------+-------------+-------------+-------------| +| 2 | 1 | ARGENTINA | AMERICA | +|--------+-------------+-------------+-------------| +| 1 | 1 | ARGENTINA | AMERICA | +|--------+-------------+-------------+-------------| +| 1 | 1 | ARGENTINA | AMERICA | ++--------+-------------+-------------+-------------+ diff --git a/tests/snapshots/07_scrabble_all_options_12_ProjectionExec.snap b/tests/snapshots/07_scrabble_all_options_12_ProjectionExec.snap deleted file mode 100644 index dcf116f..0000000 --- a/tests/snapshots/07_scrabble_all_options_12_ProjectionExec.snap +++ /dev/null @@ -1,23 +0,0 @@ ---- -source: tests/integration_tests.rs -expression: preview ---- -+-------------+---------------+-------------+------------------+ -| n_nationkey | nation_name | region_name | __unnest_placeho | -| | | | lder(n.letters) | -|=============+===============+=============+==================| -| 1 | ARGENTINA | AMERICA | [A, R, G, E, N, | -| | | | T, I, N, A] | -|-------------+---------------+-------------+------------------| -| 2 | BRAZIL | AMERICA | [B, R, A, Z, I, | -| | | | L] | -|-------------+---------------+-------------+------------------| -| 3 | CANADA | AMERICA | [C, A, N, A, D, | -| | | | A] | -|-------------+---------------+-------------+------------------| -| 17 | PERU | AMERICA | [P, E, R, U] | -|-------------+---------------+-------------+------------------| -| 24 | UNITED STATES | AMERICA | [U, N, I, T, E, | -| | | | D, , S, T, A, | -| | | | T, E, S] | -+-------------+---------------+-------------+------------------+ diff --git a/tests/snapshots/07_scrabble_all_options_10_HashJoinExec.snap b/tests/snapshots/07_scrabble_all_options_13_HashJoinExec.snap similarity index 100% rename from tests/snapshots/07_scrabble_all_options_10_HashJoinExec.snap rename to tests/snapshots/07_scrabble_all_options_13_HashJoinExec.snap diff --git a/tests/snapshots/07_scrabble_all_options_14_ProjectionExec.snap b/tests/snapshots/07_scrabble_all_options_14_ProjectionExec.snap index 95a38e6..8ba4f7a 100644 --- a/tests/snapshots/07_scrabble_all_options_14_ProjectionExec.snap +++ b/tests/snapshots/07_scrabble_all_options_14_ProjectionExec.snap @@ -2,16 +2,16 @@ source: tests/integration_tests.rs expression: preview --- -+-------------+-------------+-------------+--------+ -| n_nationkey | nation_name | region_name | letter | -|=============+=============+=============+========| -| 1 | ARGENTINA | AMERICA | A | -|-------------+-------------+-------------+--------| -| 1 | ARGENTINA | AMERICA | R | -|-------------+-------------+-------------+--------| -| 1 | ARGENTINA | AMERICA | G | -|-------------+-------------+-------------+--------| -| 1 | ARGENTINA | AMERICA | E | -|-------------+-------------+-------------+--------| -| 1 | ARGENTINA | AMERICA | N | -+-------------+-------------+-------------+--------+ ++--------+--------+ +| letter | points | +|========+========| +| A | 1 | +|--------+--------| +| B | 3 | +|--------+--------| +| C | 3 | +|--------+--------| +| D | 2 | +|--------+--------| +| E | 1 | ++--------+--------+ diff --git a/tests/snapshots/07_scrabble_all_options_28_ProjectionExec.snap b/tests/snapshots/07_scrabble_all_options_26_SortExec.snap similarity index 100% rename from tests/snapshots/07_scrabble_all_options_28_ProjectionExec.snap rename to tests/snapshots/07_scrabble_all_options_26_SortExec.snap diff --git a/tests/snapshots/07_scrabble_all_options_27_SortExec.snap b/tests/snapshots/07_scrabble_all_options_27_SortExec.snap new file mode 100644 index 0000000..9b3d6f2 --- /dev/null +++ b/tests/snapshots/07_scrabble_all_options_27_SortExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++-------------+---------------+----------------+ +| region_name | nation_name | scrabble_score | +|=============+===============+================| +| AMERICA | BRAZIL | 17 | +|-------------+---------------+----------------| +| AMERICA | UNITED STATES | 13 | +|-------------+---------------+----------------| +| AMERICA | ARGENTINA | 10 | +|-------------+---------------+----------------| +| AMERICA | CANADA | 9 | +|-------------+---------------+----------------| +| AMERICA | PERU | 6 | ++-------------+---------------+----------------+ diff --git a/tests/snapshots/07_scrabble_all_options_30_SortPreservingMergeExec.snap b/tests/snapshots/07_scrabble_all_options_28_SortPreservingMergeExec.snap similarity index 100% rename from tests/snapshots/07_scrabble_all_options_30_SortPreservingMergeExec.snap rename to tests/snapshots/07_scrabble_all_options_28_SortPreservingMergeExec.snap diff --git a/tests/snapshots/07_scrabble_all_options_29_SortExec.snap b/tests/snapshots/07_scrabble_all_options_29_SortExec.snap deleted file mode 100644 index c9aaa6e..0000000 --- a/tests/snapshots/07_scrabble_all_options_29_SortExec.snap +++ /dev/null @@ -1,20 +0,0 @@ ---- -source: tests/integration_tests.rs -expression: preview ---- -+------------+------------+------------+-----------+-----------+ -| region_nam | nation_nam | scrabble_s | region_av | nation_ra | -| e | e | core | g_scrabbl | nk_in_reg | -| | | | e_score | ion | -|============+============+============+===========+===========| -| AMERICA | BRAZIL | 17 | 11.0 | 1 | -|------------+------------+------------+-----------+-----------| -| AMERICA | UNITED | 13 | 11.0 | 2 | -| | STATES | | | | -|------------+------------+------------+-----------+-----------| -| AMERICA | ARGENTINA | 10 | 11.0 | 3 | -|------------+------------+------------+-----------+-----------| -| AMERICA | CANADA | 9 | 11.0 | 4 | -|------------+------------+------------+-----------+-----------| -| AMERICA | PERU | 6 | 11.0 | 5 | -+------------+------------+------------+-----------+-----------+ diff --git a/tests/snapshots/07_scrabble_all_options_13_UnnestExec.snap b/tests/snapshots/07_scrabble_all_options_29_UnnestExec.snap similarity index 100% rename from tests/snapshots/07_scrabble_all_options_13_UnnestExec.snap rename to tests/snapshots/07_scrabble_all_options_29_UnnestExec.snap diff --git a/tests/snapshots/07_scrabble_all_options_27_WindowAggExec.snap b/tests/snapshots/07_scrabble_all_options_30_WindowAggExec.snap similarity index 100% rename from tests/snapshots/07_scrabble_all_options_27_WindowAggExec.snap rename to tests/snapshots/07_scrabble_all_options_30_WindowAggExec.snap diff --git a/tests/snapshots/08_recursive_trace.snap b/tests/snapshots/08_recursive_trace.snap index 8f578bb..5d7aa88 100644 --- a/tests/snapshots/08_recursive_trace.snap +++ b/tests/snapshots/08_recursive_trace.snap @@ -20,7 +20,7 @@ expression: json_lines "query_name": "recursive" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -42,7 +42,7 @@ expression: json_lines "query_name": "recursive" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -63,43 +63,7 @@ expression: json_lines "query_name": "recursive" } ], - "target": "integration_utils", - "time.busy": "0.00ms", - "time.idle": "0.00ms" - }, - { - "level": "INFO", - "message": "close", - "span": { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.node": "WorkTableExec: name=numbers", - "datafusion.partitioning": "UnknownPartitioning(1)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "WorkTableExec", - "region": "us-west" - }, - "spans": [ - { - "name": "test", - "test_name": "08_recursive" - }, - { - "name": "run_traced_query", - "query_name": "recursive" - }, - { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.partitioning": "UnknownPartitioning(1)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "RecursiveQueryExec", - "region": "us-west" - } - ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -135,7 +99,7 @@ expression: json_lines "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -171,7 +135,7 @@ expression: json_lines "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -207,7 +171,7 @@ expression: json_lines "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -243,7 +207,7 @@ expression: json_lines "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -279,43 +243,7 @@ expression: json_lines "region": "us-west" } ], - "target": "integration_utils", - "time.busy": "0.00ms", - "time.idle": "0.00ms" - }, - { - "level": "INFO", - "message": "close", - "span": { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.node": "WorkTableExec: name=numbers", - "datafusion.partitioning": "UnknownPartitioning(1)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "WorkTableExec", - "region": "us-west" - }, - "spans": [ - { - "name": "test", - "test_name": "08_recursive" - }, - { - "name": "run_traced_query", - "query_name": "recursive" - }, - { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.partitioning": "UnknownPartitioning(1)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "RecursiveQueryExec", - "region": "us-west" - } - ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -351,7 +279,7 @@ expression: json_lines "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -387,7 +315,7 @@ expression: json_lines "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -423,7 +351,7 @@ expression: json_lines "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -459,7 +387,7 @@ expression: json_lines "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -495,7 +423,7 @@ expression: json_lines "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -505,11 +433,11 @@ expression: json_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.node": "WorkTableExec: name=numbers", + "datafusion.node": "PlaceholderRowExec", "datafusion.partitioning": "UnknownPartitioning(1)", "env": "production", "name": "InstrumentedExec", - "otel.name": "WorkTableExec", + "otel.name": "PlaceholderRowExec", "region": "us-west" }, "spans": [ @@ -520,18 +448,9 @@ expression: json_lines { "name": "run_traced_query", "query_name": "recursive" - }, - { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.partitioning": "UnknownPartitioning(1)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "RecursiveQueryExec", - "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -541,11 +460,11 @@ expression: json_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.node": "RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", - "datafusion.partitioning": "RoundRobinBatch(8)", + "datafusion.node": "ProjectionExec: expr=[1 as n]", + "datafusion.partitioning": "UnknownPartitioning(1)", "env": "production", "name": "InstrumentedExec", - "otel.name": "RepartitionExec", + "otel.name": "ProjectionExec", "region": "us-west" }, "spans": [ @@ -556,18 +475,9 @@ expression: json_lines { "name": "run_traced_query", "query_name": "recursive" - }, - { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.partitioning": "UnknownPartitioning(1)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "RecursiveQueryExec", - "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -577,11 +487,11 @@ expression: json_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.node": "FilterExec: n@0 < 3", - "datafusion.partitioning": "RoundRobinBatch(8)", + "datafusion.node": "WorkTableExec: name=numbers", + "datafusion.partitioning": "UnknownPartitioning(1)", "env": "production", "name": "InstrumentedExec", - "otel.name": "FilterExec", + "otel.name": "WorkTableExec", "region": "us-west" }, "spans": [ @@ -592,18 +502,9 @@ expression: json_lines { "name": "run_traced_query", "query_name": "recursive" - }, - { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.partitioning": "UnknownPartitioning(1)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "RecursiveQueryExec", - "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -613,11 +514,11 @@ expression: json_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.node": "CoalesceBatchesExec: target_batch_size=8192", + "datafusion.node": "RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", "datafusion.partitioning": "RoundRobinBatch(8)", "env": "production", "name": "InstrumentedExec", - "otel.name": "CoalesceBatchesExec", + "otel.name": "RepartitionExec", "region": "us-west" }, "spans": [ @@ -628,18 +529,9 @@ expression: json_lines { "name": "run_traced_query", "query_name": "recursive" - }, - { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.partitioning": "UnknownPartitioning(1)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "RecursiveQueryExec", - "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -649,11 +541,11 @@ expression: json_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.node": "ProjectionExec: expr=[n@0 + 1 as numbers.n + Int64(1)]", + "datafusion.node": "FilterExec: n@0 < 3", "datafusion.partitioning": "RoundRobinBatch(8)", "env": "production", "name": "InstrumentedExec", - "otel.name": "ProjectionExec", + "otel.name": "FilterExec", "region": "us-west" }, "spans": [ @@ -664,18 +556,9 @@ expression: json_lines { "name": "run_traced_query", "query_name": "recursive" - }, - { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.partitioning": "UnknownPartitioning(1)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "RecursiveQueryExec", - "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -685,11 +568,11 @@ expression: json_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.node": "CoalescePartitionsExec", - "datafusion.partitioning": "UnknownPartitioning(1)", + "datafusion.node": "CoalesceBatchesExec: target_batch_size=8192", + "datafusion.partitioning": "RoundRobinBatch(8)", "env": "production", "name": "InstrumentedExec", - "otel.name": "CoalescePartitionsExec", + "otel.name": "CoalesceBatchesExec", "region": "us-west" }, "spans": [ @@ -700,18 +583,9 @@ expression: json_lines { "name": "run_traced_query", "query_name": "recursive" - }, - { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.partitioning": "UnknownPartitioning(1)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "RecursiveQueryExec", - "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -721,11 +595,11 @@ expression: json_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.node": "PlaceholderRowExec", - "datafusion.partitioning": "UnknownPartitioning(1)", + "datafusion.node": "ProjectionExec: expr=[n@0 + 1 as numbers.n + Int64(1)]", + "datafusion.partitioning": "RoundRobinBatch(8)", "env": "production", "name": "InstrumentedExec", - "otel.name": "PlaceholderRowExec", + "otel.name": "ProjectionExec", "region": "us-west" }, "spans": [ @@ -738,7 +612,7 @@ expression: json_lines "query_name": "recursive" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -748,11 +622,11 @@ expression: json_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.node": "ProjectionExec: expr=[1 as n]", + "datafusion.node": "CoalescePartitionsExec", "datafusion.partitioning": "UnknownPartitioning(1)", "env": "production", "name": "InstrumentedExec", - "otel.name": "ProjectionExec", + "otel.name": "CoalescePartitionsExec", "region": "us-west" }, "spans": [ @@ -765,7 +639,7 @@ expression: json_lines "query_name": "recursive" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -792,7 +666,7 @@ expression: json_lines "query_name": "recursive" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -809,7 +683,7 @@ expression: json_lines "test_name": "08_recursive" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" } diff --git a/tests/snapshots/09_recursive_all_options_09_CoalesceBatchesExec.snap b/tests/snapshots/09_recursive_all_options_00_CoalesceBatchesExec.snap similarity index 100% rename from tests/snapshots/09_recursive_all_options_09_CoalesceBatchesExec.snap rename to tests/snapshots/09_recursive_all_options_00_CoalesceBatchesExec.snap diff --git a/tests/snapshots/09_recursive_all_options_00_WorkTableExec.snap b/tests/snapshots/09_recursive_all_options_01_CoalesceBatchesExec.snap similarity index 100% rename from tests/snapshots/09_recursive_all_options_00_WorkTableExec.snap rename to tests/snapshots/09_recursive_all_options_01_CoalesceBatchesExec.snap diff --git a/tests/snapshots/09_recursive_all_options_10_ProjectionExec.snap b/tests/snapshots/09_recursive_all_options_02_CoalescePartitionsExec.snap similarity index 100% rename from tests/snapshots/09_recursive_all_options_10_ProjectionExec.snap rename to tests/snapshots/09_recursive_all_options_02_CoalescePartitionsExec.snap diff --git a/tests/snapshots/09_recursive_all_options_04_ProjectionExec.snap b/tests/snapshots/09_recursive_all_options_03_CoalescePartitionsExec.snap similarity index 100% rename from tests/snapshots/09_recursive_all_options_04_ProjectionExec.snap rename to tests/snapshots/09_recursive_all_options_03_CoalescePartitionsExec.snap diff --git a/tests/snapshots/09_recursive_all_options_05_CoalescePartitionsExec.snap b/tests/snapshots/09_recursive_all_options_04_FilterExec.snap similarity index 100% rename from tests/snapshots/09_recursive_all_options_05_CoalescePartitionsExec.snap rename to tests/snapshots/09_recursive_all_options_04_FilterExec.snap diff --git a/tests/snapshots/09_recursive_all_options_01_RepartitionExec.snap b/tests/snapshots/09_recursive_all_options_05_FilterExec.snap similarity index 100% rename from tests/snapshots/09_recursive_all_options_01_RepartitionExec.snap rename to tests/snapshots/09_recursive_all_options_05_FilterExec.snap diff --git a/tests/snapshots/09_recursive_all_options_14_PlaceholderRowExec.snap b/tests/snapshots/09_recursive_all_options_06_PlaceholderRowExec.snap similarity index 100% rename from tests/snapshots/09_recursive_all_options_14_PlaceholderRowExec.snap rename to tests/snapshots/09_recursive_all_options_06_PlaceholderRowExec.snap diff --git a/tests/snapshots/09_recursive_all_options_02_FilterExec.snap b/tests/snapshots/09_recursive_all_options_07_ProjectionExec.snap similarity index 100% rename from tests/snapshots/09_recursive_all_options_02_FilterExec.snap rename to tests/snapshots/09_recursive_all_options_07_ProjectionExec.snap diff --git a/tests/snapshots/09_recursive_all_options_08_FilterExec.snap b/tests/snapshots/09_recursive_all_options_08_FilterExec.snap deleted file mode 100644 index c3cfc8b..0000000 --- a/tests/snapshots/09_recursive_all_options_08_FilterExec.snap +++ /dev/null @@ -1,9 +0,0 @@ ---- -source: tests/integration_tests.rs -expression: preview ---- -+----------------------+ -| numbers.n + Int64(1) | -|======================| -| 2 | -+----------------------+ diff --git a/tests/snapshots/09_recursive_all_options_11_CoalescePartitionsExec.snap b/tests/snapshots/09_recursive_all_options_08_ProjectionExec.snap similarity index 100% rename from tests/snapshots/09_recursive_all_options_11_CoalescePartitionsExec.snap rename to tests/snapshots/09_recursive_all_options_08_ProjectionExec.snap diff --git a/tests/snapshots/09_recursive_all_options_06_WorkTableExec.snap b/tests/snapshots/09_recursive_all_options_09_ProjectionExec.snap similarity index 100% rename from tests/snapshots/09_recursive_all_options_06_WorkTableExec.snap rename to tests/snapshots/09_recursive_all_options_09_ProjectionExec.snap diff --git a/tests/snapshots/09_recursive_all_options_16_RecursiveQueryExec.snap b/tests/snapshots/09_recursive_all_options_10_RecursiveQueryExec.snap similarity index 100% rename from tests/snapshots/09_recursive_all_options_16_RecursiveQueryExec.snap rename to tests/snapshots/09_recursive_all_options_10_RecursiveQueryExec.snap diff --git a/tests/snapshots/09_recursive_all_options_07_RepartitionExec.snap b/tests/snapshots/09_recursive_all_options_11_RepartitionExec.snap similarity index 100% rename from tests/snapshots/09_recursive_all_options_07_RepartitionExec.snap rename to tests/snapshots/09_recursive_all_options_11_RepartitionExec.snap diff --git a/tests/snapshots/09_recursive_all_options_12_WorkTableExec.snap b/tests/snapshots/09_recursive_all_options_12_RepartitionExec.snap similarity index 100% rename from tests/snapshots/09_recursive_all_options_12_WorkTableExec.snap rename to tests/snapshots/09_recursive_all_options_12_RepartitionExec.snap diff --git a/tests/snapshots/09_recursive_all_options_13_RepartitionExec.snap b/tests/snapshots/09_recursive_all_options_13_RepartitionExec.snap index 7f8e94d..82b032d 100644 --- a/tests/snapshots/09_recursive_all_options_13_RepartitionExec.snap +++ b/tests/snapshots/09_recursive_all_options_13_RepartitionExec.snap @@ -2,8 +2,8 @@ source: tests/integration_tests.rs expression: preview --- -+----------------------+ -| numbers.n + Int64(1) | -|======================| -| 3 | -+----------------------+ ++---+ +| n | +|===| +| 1 | ++---+ diff --git a/tests/snapshots/09_recursive_all_options_03_CoalesceBatchesExec.snap b/tests/snapshots/09_recursive_all_options_14_WorkTableExec.snap similarity index 100% rename from tests/snapshots/09_recursive_all_options_03_CoalesceBatchesExec.snap rename to tests/snapshots/09_recursive_all_options_14_WorkTableExec.snap diff --git a/tests/snapshots/09_recursive_all_options_15_ProjectionExec.snap b/tests/snapshots/09_recursive_all_options_15_ProjectionExec.snap deleted file mode 100644 index 82b032d..0000000 --- a/tests/snapshots/09_recursive_all_options_15_ProjectionExec.snap +++ /dev/null @@ -1,9 +0,0 @@ ---- -source: tests/integration_tests.rs -expression: preview ---- -+---+ -| n | -|===| -| 1 | -+---+ diff --git a/tests/snapshots/09_recursive_all_options_trace.snap b/tests/snapshots/09_recursive_all_options_trace.snap index 86a39cd..56e734d 100644 --- a/tests/snapshots/09_recursive_all_options_trace.snap +++ b/tests/snapshots/09_recursive_all_options_trace.snap @@ -20,7 +20,7 @@ expression: json_lines "query_name": "recursive" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -42,7 +42,7 @@ expression: json_lines "query_name": "recursive" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -63,44 +63,7 @@ expression: json_lines "query_name": "recursive" } ], - "target": "integration_utils", - "time.busy": "0.00ms", - "time.idle": "0.00ms" - }, - { - "level": "INFO", - "message": "close", - "span": { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.node": "WorkTableExec: name=numbers", - "datafusion.partitioning": "UnknownPartitioning(1)", - "datafusion.preview": "", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "WorkTableExec", - "region": "us-west" - }, - "spans": [ - { - "name": "test", - "test_name": "09_recursive_all_options" - }, - { - "name": "run_traced_query", - "query_name": "recursive" - }, - { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.partitioning": "UnknownPartitioning(1)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "RecursiveQueryExec", - "region": "us-west" - } - ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -140,7 +103,7 @@ expression: json_lines "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -181,7 +144,7 @@ expression: json_lines "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -222,7 +185,7 @@ expression: json_lines "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -263,7 +226,7 @@ expression: json_lines "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -304,44 +267,7 @@ expression: json_lines "region": "us-west" } ], - "target": "integration_utils", - "time.busy": "0.00ms", - "time.idle": "0.00ms" - }, - { - "level": "INFO", - "message": "close", - "span": { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.node": "WorkTableExec: name=numbers", - "datafusion.partitioning": "UnknownPartitioning(1)", - "datafusion.preview": "", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "WorkTableExec", - "region": "us-west" - }, - "spans": [ - { - "name": "test", - "test_name": "09_recursive_all_options" - }, - { - "name": "run_traced_query", - "query_name": "recursive" - }, - { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.partitioning": "UnknownPartitioning(1)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "RecursiveQueryExec", - "region": "us-west" - } - ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -381,7 +307,7 @@ expression: json_lines "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -393,11 +319,10 @@ expression: json_lines "datafusion.emission_type": "Incremental", "datafusion.metrics.elapsed_compute": "0.00ms", "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.metrics.output_rows": "1", + "datafusion.metrics.output_rows": "0", "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", "datafusion.node": "FilterExec: n@0 < 3", "datafusion.partitioning": "RoundRobinBatch(8)", - "datafusion.preview": "", "env": "production", "name": "InstrumentedExec", "otel.name": "FilterExec", @@ -422,7 +347,7 @@ expression: json_lines "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -434,11 +359,10 @@ expression: json_lines "datafusion.emission_type": "Incremental", "datafusion.metrics.elapsed_compute": "0.00ms", "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.metrics.output_rows": "1", + "datafusion.metrics.output_rows": "0", "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", "datafusion.node": "CoalesceBatchesExec: target_batch_size=8192", "datafusion.partitioning": "RoundRobinBatch(8)", - "datafusion.preview": "", "env": "production", "name": "InstrumentedExec", "otel.name": "CoalesceBatchesExec", @@ -463,7 +387,7 @@ expression: json_lines "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -475,11 +399,10 @@ expression: json_lines "datafusion.emission_type": "Incremental", "datafusion.metrics.elapsed_compute": "0.00ms", "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.metrics.output_rows": "1", + "datafusion.metrics.output_rows": "0", "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", "datafusion.node": "ProjectionExec: expr=[n@0 + 1 as numbers.n + Int64(1)]", "datafusion.partitioning": "RoundRobinBatch(8)", - "datafusion.preview": "", "env": "production", "name": "InstrumentedExec", "otel.name": "ProjectionExec", @@ -504,7 +427,7 @@ expression: json_lines "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -516,11 +439,10 @@ expression: json_lines "datafusion.emission_type": "Incremental", "datafusion.metrics.elapsed_compute": "0.00ms", "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.metrics.output_rows": "1", + "datafusion.metrics.output_rows": "0", "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", "datafusion.node": "CoalescePartitionsExec", "datafusion.partitioning": "UnknownPartitioning(1)", - "datafusion.preview": "", "env": "production", "name": "InstrumentedExec", "otel.name": "CoalescePartitionsExec", @@ -545,7 +467,7 @@ expression: json_lines "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -555,12 +477,12 @@ expression: json_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.node": "WorkTableExec: name=numbers", + "datafusion.node": "PlaceholderRowExec", "datafusion.partitioning": "UnknownPartitioning(1)", "datafusion.preview": "", "env": "production", "name": "InstrumentedExec", - "otel.name": "WorkTableExec", + "otel.name": "PlaceholderRowExec", "region": "us-west" }, "spans": [ @@ -571,18 +493,9 @@ expression: json_lines { "name": "run_traced_query", "query_name": "recursive" - }, - { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.partitioning": "UnknownPartitioning(1)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "RecursiveQueryExec", - "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -592,15 +505,16 @@ expression: json_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.metrics.fetch_time": "0.00ms", - "datafusion.metrics.repartition_time": "0.00ms", - "datafusion.metrics.send_time": "0.00ms", - "datafusion.node": "RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", - "datafusion.partitioning": "RoundRobinBatch(8)", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.output_rows": "1", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.node": "ProjectionExec: expr=[1 as n]", + "datafusion.partitioning": "UnknownPartitioning(1)", "datafusion.preview": "", "env": "production", "name": "InstrumentedExec", - "otel.name": "RepartitionExec", + "otel.name": "ProjectionExec", "region": "us-west" }, "spans": [ @@ -611,18 +525,9 @@ expression: json_lines { "name": "run_traced_query", "query_name": "recursive" - }, - { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.partitioning": "UnknownPartitioning(1)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "RecursiveQueryExec", - "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -632,15 +537,12 @@ expression: json_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.metrics.elapsed_compute": "0.00ms", - "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.metrics.output_rows": "0", - "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.node": "FilterExec: n@0 < 3", - "datafusion.partitioning": "RoundRobinBatch(8)", + "datafusion.node": "WorkTableExec: name=numbers", + "datafusion.partitioning": "UnknownPartitioning(1)", + "datafusion.preview": "", "env": "production", "name": "InstrumentedExec", - "otel.name": "FilterExec", + "otel.name": "WorkTableExec", "region": "us-west" }, "spans": [ @@ -651,18 +553,9 @@ expression: json_lines { "name": "run_traced_query", "query_name": "recursive" - }, - { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.partitioning": "UnknownPartitioning(1)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "RecursiveQueryExec", - "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -672,15 +565,15 @@ expression: json_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.metrics.elapsed_compute": "0.00ms", - "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.metrics.output_rows": "0", - "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.node": "CoalesceBatchesExec: target_batch_size=8192", + "datafusion.metrics.fetch_time": "0.00ms", + "datafusion.metrics.repartition_time": "0.00ms", + "datafusion.metrics.send_time": "0.00ms", + "datafusion.node": "RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", "datafusion.partitioning": "RoundRobinBatch(8)", + "datafusion.preview": "", "env": "production", "name": "InstrumentedExec", - "otel.name": "CoalesceBatchesExec", + "otel.name": "RepartitionExec", "region": "us-west" }, "spans": [ @@ -691,18 +584,9 @@ expression: json_lines { "name": "run_traced_query", "query_name": "recursive" - }, - { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.partitioning": "UnknownPartitioning(1)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "RecursiveQueryExec", - "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -714,13 +598,14 @@ expression: json_lines "datafusion.emission_type": "Incremental", "datafusion.metrics.elapsed_compute": "0.00ms", "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.metrics.output_rows": "0", + "datafusion.metrics.output_rows": "1", "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.node": "ProjectionExec: expr=[n@0 + 1 as numbers.n + Int64(1)]", + "datafusion.node": "FilterExec: n@0 < 3", "datafusion.partitioning": "RoundRobinBatch(8)", + "datafusion.preview": "", "env": "production", "name": "InstrumentedExec", - "otel.name": "ProjectionExec", + "otel.name": "FilterExec", "region": "us-west" }, "spans": [ @@ -731,18 +616,9 @@ expression: json_lines { "name": "run_traced_query", "query_name": "recursive" - }, - { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.partitioning": "UnknownPartitioning(1)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "RecursiveQueryExec", - "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -754,13 +630,14 @@ expression: json_lines "datafusion.emission_type": "Incremental", "datafusion.metrics.elapsed_compute": "0.00ms", "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.metrics.output_rows": "0", + "datafusion.metrics.output_rows": "1", "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.node": "CoalescePartitionsExec", - "datafusion.partitioning": "UnknownPartitioning(1)", + "datafusion.node": "CoalesceBatchesExec: target_batch_size=8192", + "datafusion.partitioning": "RoundRobinBatch(8)", + "datafusion.preview": "", "env": "production", "name": "InstrumentedExec", - "otel.name": "CoalescePartitionsExec", + "otel.name": "CoalesceBatchesExec", "region": "us-west" }, "spans": [ @@ -771,18 +648,9 @@ expression: json_lines { "name": "run_traced_query", "query_name": "recursive" - }, - { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.partitioning": "UnknownPartitioning(1)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "RecursiveQueryExec", - "region": "us-west" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -792,12 +660,16 @@ expression: json_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.node": "PlaceholderRowExec", - "datafusion.partitioning": "UnknownPartitioning(1)", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.output_rows": "1", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.node": "ProjectionExec: expr=[n@0 + 1 as numbers.n + Int64(1)]", + "datafusion.partitioning": "RoundRobinBatch(8)", "datafusion.preview": "", "env": "production", "name": "InstrumentedExec", - "otel.name": "PlaceholderRowExec", + "otel.name": "ProjectionExec", "region": "us-west" }, "spans": [ @@ -810,7 +682,7 @@ expression: json_lines "query_name": "recursive" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -824,12 +696,12 @@ expression: json_lines "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", "datafusion.metrics.output_rows": "1", "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.node": "ProjectionExec: expr=[1 as n]", + "datafusion.node": "CoalescePartitionsExec", "datafusion.partitioning": "UnknownPartitioning(1)", "datafusion.preview": "", "env": "production", "name": "InstrumentedExec", - "otel.name": "ProjectionExec", + "otel.name": "CoalescePartitionsExec", "region": "us-west" }, "spans": [ @@ -842,7 +714,7 @@ expression: json_lines "query_name": "recursive" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -874,7 +746,7 @@ expression: json_lines "query_name": "recursive" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -891,7 +763,7 @@ expression: json_lines "test_name": "09_recursive_all_options" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" } diff --git a/tests/snapshots/10_topk_lineitem_trace.snap b/tests/snapshots/10_topk_lineitem_trace.snap index c0c3bb7..36a7533 100644 --- a/tests/snapshots/10_topk_lineitem_trace.snap +++ b/tests/snapshots/10_topk_lineitem_trace.snap @@ -20,7 +20,7 @@ expression: json_lines "query_name": "topk_lineitem" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -42,7 +42,7 @@ expression: json_lines "query_name": "topk_lineitem" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -51,7 +51,7 @@ expression: json_lines "message": "close", "span": { "name": "create_physical_plan", - "physical_plan": "SortPreservingMergeExec: [l_orderkey@0 ASC NULLS LAST, l_linenumber@1 ASC NULLS LAST], fetch=3\n SortExec: TopK(fetch=3), expr=[l_orderkey@0 ASC NULLS LAST, l_linenumber@1 ASC NULLS LAST], preserve_partitioning=[true]\n ProjectionExec: expr=[l_orderkey@0 as l_orderkey, l_linenumber@1 as l_linenumber, l_shipmode@3 as l_shipmode, l_quantity@2 as l_quantity]\n CoalesceBatchesExec: target_batch_size=8192\n FilterExec: l_shipmode@3 = SHIP\n DataSourceExec: file_groups={8 groups: [[/lineitem.parquet:0..2516294], [/lineitem.parquet:2516294..5032588], [/lineitem.parquet:5032588..7548882], [/lineitem.parquet:7548882..10065176], [/lineitem.parquet:10065176..12581470], [/lineitem.parquet:12581470..15097764], [/lineitem.parquet:15097764..17614058], [/lineitem.parquet:17614058..20130345]]}, projection=[l_orderkey, l_linenumber, l_quantity, l_shipmode], file_type=parquet, predicate=l_shipmode@3 = SHIP AND DynamicFilterPhysicalExpr [ true ], pruning_predicate=l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1, required_guarantees=[l_shipmode in (SHIP)]\n" + "physical_plan": "SortPreservingMergeExec: [l_orderkey@0 ASC NULLS LAST, l_linenumber@1 ASC NULLS LAST], fetch=3\n SortExec: TopK(fetch=3), expr=[l_orderkey@0 ASC NULLS LAST, l_linenumber@1 ASC NULLS LAST], preserve_partitioning=[true]\n ProjectionExec: expr=[l_orderkey@0 as l_orderkey, l_linenumber@1 as l_linenumber, l_shipmode@3 as l_shipmode, l_quantity@2 as l_quantity]\n CoalesceBatchesExec: target_batch_size=8192\n FilterExec: l_shipmode@3 = SHIP\n DataSourceExec: file_groups={8 groups: [[/lineitem.parquet:0..28958694], [/lineitem.parquet:28958694..57917388], [/lineitem.parquet:57917388..86876082], [/lineitem.parquet:86876082..115834776], [/lineitem.parquet:115834776..144793470], [/lineitem.parquet:144793470..173752164], [/lineitem.parquet:173752164..202710858], [/lineitem.parquet:202710858..231669547]]}, projection=[l_orderkey, l_linenumber, l_quantity, l_shipmode], file_type=parquet, predicate=l_shipmode@3 = SHIP AND DynamicFilterPhysicalExpr [ true ], pruning_predicate=l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1, required_guarantees=[l_shipmode in (SHIP)]\n" }, "spans": [ { @@ -63,7 +63,7 @@ expression: json_lines "query_name": "topk_lineitem" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -73,7 +73,7 @@ expression: json_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.node": "DataSourceExec: file_groups={8 groups: [[/lineitem.parquet:0..2516294], [/lineitem.parquet:2516294..5032588], [/lineitem.parquet:5032588..7548882], [/lineitem.parquet:7548882..10065176], [/lineitem.parquet:10065176..12581470], ...]}, projection=[l_orderkey, l_linenumber, l_quantity, l_shipmode], file_type=parquet, predicate=l_shipmode@3 = SHIP AND DynamicFilterPhysicalExpr [ l_orderkey@0 < 35 OR l_orderkey@0 = 35 AND l_linenumber@1 < 4 ], pruning_predicate=l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1 AND (l_orderkey_null_count@5 != row_count@3 AND l_orderkey_min@4 < 35 OR l_orderkey_null_count@5 != row_count@3 AND l_orderkey_min@4 <= 35 AND 35 <= l_orderkey_max@6 AND l_linenumber_null_count@8 != row_count@3 AND l_linenumber_min@7 < 4), required_guarantees=[l_shipmode in (SHIP)]", + "datafusion.node": "DataSourceExec: file_groups={8 groups: [[/lineitem.parquet:0..28958694], [/lineitem.parquet:28958694..57917388], [/lineitem.parquet:57917388..86876082], [/lineitem.parquet:86876082..115834776], [/lineitem.parquet:115834776..144793470], ...]}, projection=[l_orderkey, l_linenumber, l_quantity, l_shipmode], file_type=parquet, predicate=l_shipmode@3 = SHIP AND DynamicFilterPhysicalExpr [ l_orderkey@0 < 35 OR l_orderkey@0 = 35 AND l_linenumber@1 < 4 ], pruning_predicate=l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1 AND (l_orderkey_null_count@5 != row_count@3 AND l_orderkey_min@4 < 35 OR l_orderkey_null_count@5 != row_count@3 AND l_orderkey_min@4 <= 35 AND 35 <= l_orderkey_max@6 AND l_linenumber_null_count@8 != row_count@3 AND l_linenumber_min@7 < 4), required_guarantees=[l_shipmode in (SHIP)]", "datafusion.partitioning": "UnknownPartitioning(8)", "env": "production", "name": "InstrumentedExec", @@ -90,7 +90,7 @@ expression: json_lines "query_name": "topk_lineitem" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -117,7 +117,7 @@ expression: json_lines "query_name": "topk_lineitem" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -144,7 +144,7 @@ expression: json_lines "query_name": "topk_lineitem" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -171,7 +171,7 @@ expression: json_lines "query_name": "topk_lineitem" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -198,7 +198,7 @@ expression: json_lines "query_name": "topk_lineitem" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -225,7 +225,7 @@ expression: json_lines "query_name": "topk_lineitem" } ], - "target": "integration_utils", + "target": "integration_utils::session", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -242,7 +242,7 @@ expression: json_lines "test_name": "10_topk_lineitem" } ], - "target": "integration_utils", + "target": "integration_utils::query", "time.busy": "0.00ms", "time.idle": "0.00ms" } diff --git a/tests/snapshots/11_weather_00_AggregateExec.snap b/tests/snapshots/11_weather_00_AggregateExec.snap new file mode 100644 index 0000000..ed92268 --- /dev/null +++ b/tests/snapshots/11_weather_00_AggregateExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+-----------------+ +| MinTemp | count(Int64(1)) | +|=========+=================| +| 10.1 | 4 | +|---------+-----------------| +| 13.6 | 3 | +|---------+-----------------| +| 17.2 | 2 | +|---------+-----------------| +| 11.8 | 1 | +|---------+-----------------| +| 14.8 | 2 | ++---------+-----------------+ diff --git a/tests/snapshots/11_weather_01_AggregateExec.snap b/tests/snapshots/11_weather_01_AggregateExec.snap new file mode 100644 index 0000000..7a0ff84 --- /dev/null +++ b/tests/snapshots/11_weather_01_AggregateExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+------------------------+ +| MinTemp | count(Int64(1))[count] | +|=========+========================| +| 8.0 | 1 | +|---------+------------------------| +| 14.0 | 1 | +|---------+------------------------| +| 13.7 | 1 | +|---------+------------------------| +| 13.3 | 3 | +|---------+------------------------| +| 7.6 | 1 | ++---------+------------------------+ diff --git a/tests/snapshots/11_weather_02_CoalesceBatchesExec.snap b/tests/snapshots/11_weather_02_CoalesceBatchesExec.snap new file mode 100644 index 0000000..2778f14 --- /dev/null +++ b/tests/snapshots/11_weather_02_CoalesceBatchesExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+------------------------+ +| MinTemp | count(Int64(1))[count] | +|=========+========================| +| 10.1 | 3 | +|---------+------------------------| +| 13.6 | 3 | +|---------+------------------------| +| 17.2 | 2 | +|---------+------------------------| +| 11.8 | 1 | +|---------+------------------------| +| 14.8 | 2 | ++---------+------------------------+ diff --git a/tests/snapshots/11_weather_03_DataSourceExec.snap b/tests/snapshots/11_weather_03_DataSourceExec.snap new file mode 100644 index 0000000..fb6149f --- /dev/null +++ b/tests/snapshots/11_weather_03_DataSourceExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+ +| MinTemp | +|=========| +| 8.0 | +|---------| +| 14.0 | +|---------| +| 13.7 | +|---------| +| 13.3 | +|---------| +| 7.6 | ++---------+ diff --git a/tests/snapshots/11_weather_04_ProjectionExec.snap b/tests/snapshots/11_weather_04_ProjectionExec.snap new file mode 100644 index 0000000..ce3c068 --- /dev/null +++ b/tests/snapshots/11_weather_04_ProjectionExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++----------+---------+ +| count(*) | MinTemp | +|==========+=========| +| 4 | 10.1 | +|----------+---------| +| 3 | 13.6 | +|----------+---------| +| 2 | 17.2 | +|----------+---------| +| 1 | 11.8 | +|----------+---------| +| 2 | 14.8 | ++----------+---------+ diff --git a/tests/snapshots/11_weather_05_RepartitionExec.snap b/tests/snapshots/11_weather_05_RepartitionExec.snap new file mode 100644 index 0000000..2778f14 --- /dev/null +++ b/tests/snapshots/11_weather_05_RepartitionExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+------------------------+ +| MinTemp | count(Int64(1))[count] | +|=========+========================| +| 10.1 | 3 | +|---------+------------------------| +| 13.6 | 3 | +|---------+------------------------| +| 17.2 | 2 | +|---------+------------------------| +| 11.8 | 1 | +|---------+------------------------| +| 14.8 | 2 | ++---------+------------------------+ diff --git a/tests/snapshots/11_weather_06_RepartitionExec.snap b/tests/snapshots/11_weather_06_RepartitionExec.snap new file mode 100644 index 0000000..7a0ff84 --- /dev/null +++ b/tests/snapshots/11_weather_06_RepartitionExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+------------------------+ +| MinTemp | count(Int64(1))[count] | +|=========+========================| +| 8.0 | 1 | +|---------+------------------------| +| 14.0 | 1 | +|---------+------------------------| +| 13.7 | 1 | +|---------+------------------------| +| 13.3 | 3 | +|---------+------------------------| +| 7.6 | 1 | ++---------+------------------------+ diff --git a/tests/snapshots/11_weather_trace.snap b/tests/snapshots/11_weather_trace.snap new file mode 100644 index 0000000..eb81e6c --- /dev/null +++ b/tests/snapshots/11_weather_trace.snap @@ -0,0 +1,340 @@ +--- +source: tests/integration_tests.rs +expression: json_lines +--- +[ + { + "level": "INFO", + "message": "close", + "span": { + "name": "read_query", + "query": "SELECT\n count(*),\n \"MinTemp\"\nFROM weather\nGROUP BY \"MinTemp\"\n" + }, + "spans": [ + { + "name": "test", + "test_name": "11_weather" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::query", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "logical_plan": "Projection: count(Int64(1)) AS count(*), weather.MinTemp [count(*):Int64, MinTemp:Float64;N]\n Aggregate: groupBy=[[weather.MinTemp]], aggr=[[count(Int64(1))]] [MinTemp:Float64;N, count(Int64(1)):Int64]\n TableScan: weather [MinTemp:Float64;N, MaxTemp:Float64;N, Rainfall:Float64;N, Evaporation:Float64;N, Sunshine:Utf8View;N, WindGustDir:Utf8View;N, WindGustSpeed:Utf8View;N, WindDir9am:Utf8View;N, WindDir3pm:Utf8View;N, WindSpeed9am:Utf8View;N, WindSpeed3pm:Int64;N, Humidity9am:Int64;N, Humidity3pm:Int64;N, Pressure9am:Float64;N, Pressure3pm:Float64;N, Cloud9am:Int64;N, Cloud3pm:Int64;N, Temp9am:Float64;N, Temp3pm:Float64;N, RainToday:Utf8View;N, RISK_MM:Float64;N, RainTomorrow:Utf8View;N]", + "name": "parse_sql", + "sql": "SELECT\n count(*),\n \"MinTemp\"\nFROM weather\nGROUP BY \"MinTemp\"\n" + }, + "spans": [ + { + "name": "test", + "test_name": "11_weather" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::query", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "create_physical_plan", + "physical_plan": "ProjectionExec: expr=[count(Int64(1))@1 as count(*), MinTemp@0 as MinTemp]\n AggregateExec: mode=FinalPartitioned, gby=[MinTemp@0 as MinTemp], aggr=[count(Int64(1))]\n CoalesceBatchesExec: target_batch_size=8192\n RepartitionExec: partitioning=Hash([MinTemp@0], 8), input_partitions=8\n RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=3\n AggregateExec: mode=Partial, gby=[MinTemp@0 as MinTemp], aggr=[count(Int64(1))]\n DataSourceExec: file_groups={3 groups: [[/weather/result-000000.parquet], [/weather/result-000001.parquet], [/weather/result-000002.parquet]]}, projection=[MinTemp], file_type=parquet\n" + }, + "spans": [ + { + "name": "test", + "test_name": "11_weather" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::query", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.metrics.bloom_filter_eval_time": "0.00ms", + "datafusion.metrics.bytes_scanned": "5404", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.file_open_errors": "0", + "datafusion.metrics.file_scan_errors": "0", + "datafusion.metrics.metadata_load_time": "0.00ms", + "datafusion.metrics.num_predicate_creation_errors": "0", + "datafusion.metrics.output_rows": "366", + "datafusion.metrics.page_index_eval_time": "0.00ms", + "datafusion.metrics.page_index_rows_matched": "0", + "datafusion.metrics.page_index_rows_pruned": "0", + "datafusion.metrics.predicate_evaluation_errors": "0", + "datafusion.metrics.pushdown_rows_matched": "0", + "datafusion.metrics.pushdown_rows_pruned": "0", + "datafusion.metrics.row_groups_matched_bloom_filter": "0", + "datafusion.metrics.row_groups_matched_statistics": "0", + "datafusion.metrics.row_groups_pruned_bloom_filter": "0", + "datafusion.metrics.row_groups_pruned_statistics": "0", + "datafusion.metrics.row_pushdown_eval_time": "0.00ms", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.statistics_eval_time": "0.00ms", + "datafusion.metrics.time_elapsed_opening": "0.00ms", + "datafusion.metrics.time_elapsed_processing": "0.00ms", + "datafusion.metrics.time_elapsed_scanning_total": "0.00ms", + "datafusion.metrics.time_elapsed_scanning_until_data": "0.00ms", + "datafusion.node": "DataSourceExec: file_groups={3 groups: [[/weather/result-000000.parquet], [/weather/result-000001.parquet], [/weather/result-000002.parquet]]}, projection=[MinTemp], file_type=parquet", + "datafusion.partitioning": "UnknownPartitioning(3)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "DataSourceExec", + "region": "us-west" + }, + "spans": [ + { + "name": "test", + "test_name": "11_weather" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.output_rows": "249", + "datafusion.metrics.peak_mem_used": "19080", + "datafusion.metrics.skipped_aggregation_rows": "0", + "datafusion.metrics.spill_count": "0", + "datafusion.metrics.spilled_bytes": "0.0 B", + "datafusion.metrics.spilled_rows": "0", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.node": "AggregateExec: mode=Partial, gby=[MinTemp@0 as MinTemp], aggr=[count(Int64(1))]", + "datafusion.partitioning": "UnknownPartitioning(3)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "AggregateExec", + "region": "us-west" + }, + "spans": [ + { + "name": "test", + "test_name": "11_weather" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.fetch_time": "0.00ms", + "datafusion.metrics.repartition_time": "0.00ms", + "datafusion.metrics.send_time": "0.00ms", + "datafusion.node": "RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=3", + "datafusion.partitioning": "RoundRobinBatch(8)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "RepartitionExec", + "region": "us-west" + }, + "spans": [ + { + "name": "test", + "test_name": "11_weather" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.fetch_time": "0.00ms", + "datafusion.metrics.repartition_time": "0.00ms", + "datafusion.metrics.send_time": "0.00ms", + "datafusion.node": "RepartitionExec: partitioning=Hash([MinTemp@0], 8), input_partitions=8", + "datafusion.partitioning": "Hash([MinTemp@0], 8)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "RepartitionExec", + "region": "us-west" + }, + "spans": [ + { + "name": "test", + "test_name": "11_weather" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.output_rows": "249", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.node": "CoalesceBatchesExec: target_batch_size=8192", + "datafusion.partitioning": "Hash([MinTemp@0], 8)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "CoalesceBatchesExec", + "region": "us-west" + }, + "spans": [ + { + "name": "test", + "test_name": "11_weather" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.output_rows": "180", + "datafusion.metrics.peak_mem_used": "41632", + "datafusion.metrics.spill_count": "0", + "datafusion.metrics.spilled_bytes": "0.0 B", + "datafusion.metrics.spilled_rows": "0", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.node": "AggregateExec: mode=FinalPartitioned, gby=[MinTemp@0 as MinTemp], aggr=[count(Int64(1))]", + "datafusion.partitioning": "Hash([MinTemp@0], 8)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "AggregateExec", + "region": "us-west" + }, + "spans": [ + { + "name": "test", + "test_name": "11_weather" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.output_rows": "180", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.node": "ProjectionExec: expr=[count(Int64(1))@1 as count(*), MinTemp@0 as MinTemp]", + "datafusion.partitioning": "Hash([MinTemp@1], 8)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "ProjectionExec", + "region": "us-west" + }, + "spans": [ + { + "name": "test", + "test_name": "11_weather" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "run_traced_query", + "query_name": "weather" + }, + "spans": [ + { + "name": "test", + "test_name": "11_weather" + } + ], + "target": "integration_utils::query", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + } +] diff --git a/tests/snapshots/12_distributed_weather_memory_00_AggregateExec.snap b/tests/snapshots/12_distributed_weather_memory_00_AggregateExec.snap new file mode 100644 index 0000000..7dadd2e --- /dev/null +++ b/tests/snapshots/12_distributed_weather_memory_00_AggregateExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+-----------------+ +| MinTemp | count(Int64(1)) | +|=========+=================| +| 10.1 | 4 | +|---------+-----------------| +| 12.3 | 2 | +|---------+-----------------| +| 8.1 | 2 | +|---------+-----------------| +| 6.9 | 1 | +|---------+-----------------| +| 6.3 | 3 | ++---------+-----------------+ diff --git a/tests/snapshots/12_distributed_weather_memory_01_AggregateExec.snap b/tests/snapshots/12_distributed_weather_memory_01_AggregateExec.snap new file mode 100644 index 0000000..7a0ff84 --- /dev/null +++ b/tests/snapshots/12_distributed_weather_memory_01_AggregateExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+------------------------+ +| MinTemp | count(Int64(1))[count] | +|=========+========================| +| 8.0 | 1 | +|---------+------------------------| +| 14.0 | 1 | +|---------+------------------------| +| 13.7 | 1 | +|---------+------------------------| +| 13.3 | 3 | +|---------+------------------------| +| 7.6 | 1 | ++---------+------------------------+ diff --git a/tests/snapshots/12_distributed_weather_memory_02_CoalesceBatchesExec.snap b/tests/snapshots/12_distributed_weather_memory_02_CoalesceBatchesExec.snap new file mode 100644 index 0000000..d24dc04 --- /dev/null +++ b/tests/snapshots/12_distributed_weather_memory_02_CoalesceBatchesExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+------------------------+ +| MinTemp | count(Int64(1))[count] | +|=========+========================| +| 10.1 | 1 | +|---------+------------------------| +| 12.3 | 1 | +|---------+------------------------| +| 8.1 | 2 | +|---------+------------------------| +| 6.9 | 1 | +|---------+------------------------| +| 6.3 | 2 | ++---------+------------------------+ diff --git a/tests/snapshots/12_distributed_weather_memory_03_CoalescePartitionsExec.snap b/tests/snapshots/12_distributed_weather_memory_03_CoalescePartitionsExec.snap new file mode 100644 index 0000000..6141518 --- /dev/null +++ b/tests/snapshots/12_distributed_weather_memory_03_CoalescePartitionsExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++----------+---------+ +| count(*) | MinTemp | +|==========+=========| +| 3 | 11.3 | +|----------+---------| +| 2 | 4.2 | +|----------+---------| +| 3 | 5.3 | +|----------+---------| +| 3 | 8.4 | +|----------+---------| +| 1 | 1.5 | ++----------+---------+ diff --git a/tests/snapshots/12_distributed_weather_memory_04_DataSourceExec.snap b/tests/snapshots/12_distributed_weather_memory_04_DataSourceExec.snap new file mode 100644 index 0000000..fb6149f --- /dev/null +++ b/tests/snapshots/12_distributed_weather_memory_04_DataSourceExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+ +| MinTemp | +|=========| +| 8.0 | +|---------| +| 14.0 | +|---------| +| 13.7 | +|---------| +| 13.3 | +|---------| +| 7.6 | ++---------+ diff --git a/tests/snapshots/12_distributed_weather_memory_05_DistributedExec.snap b/tests/snapshots/12_distributed_weather_memory_05_DistributedExec.snap new file mode 100644 index 0000000..6141518 --- /dev/null +++ b/tests/snapshots/12_distributed_weather_memory_05_DistributedExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++----------+---------+ +| count(*) | MinTemp | +|==========+=========| +| 3 | 11.3 | +|----------+---------| +| 2 | 4.2 | +|----------+---------| +| 3 | 5.3 | +|----------+---------| +| 3 | 8.4 | +|----------+---------| +| 1 | 1.5 | ++----------+---------+ diff --git a/tests/snapshots/12_distributed_weather_memory_06_ProjectionExec.snap b/tests/snapshots/12_distributed_weather_memory_06_ProjectionExec.snap new file mode 100644 index 0000000..dcbaa67 --- /dev/null +++ b/tests/snapshots/12_distributed_weather_memory_06_ProjectionExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++----------+---------+ +| count(*) | MinTemp | +|==========+=========| +| 4 | 10.1 | +|----------+---------| +| 2 | 12.3 | +|----------+---------| +| 2 | 8.1 | +|----------+---------| +| 1 | 6.9 | +|----------+---------| +| 3 | 6.3 | ++----------+---------+ diff --git a/tests/snapshots/12_distributed_weather_memory_07_RepartitionExec.snap b/tests/snapshots/12_distributed_weather_memory_07_RepartitionExec.snap new file mode 100644 index 0000000..d24dc04 --- /dev/null +++ b/tests/snapshots/12_distributed_weather_memory_07_RepartitionExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+------------------------+ +| MinTemp | count(Int64(1))[count] | +|=========+========================| +| 10.1 | 1 | +|---------+------------------------| +| 12.3 | 1 | +|---------+------------------------| +| 8.1 | 2 | +|---------+------------------------| +| 6.9 | 1 | +|---------+------------------------| +| 6.3 | 2 | ++---------+------------------------+ diff --git a/tests/snapshots/12_distributed_weather_memory_08_RepartitionExec.snap b/tests/snapshots/12_distributed_weather_memory_08_RepartitionExec.snap new file mode 100644 index 0000000..44b4754 --- /dev/null +++ b/tests/snapshots/12_distributed_weather_memory_08_RepartitionExec.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+------------------------+ +| MinTemp | count(Int64(1))[count] | +|=========+========================| +| 7.4 | 2 | +|---------+------------------------| +| 8.3 | 3 | +|---------+------------------------| +| 10.1 | 1 | +|---------+------------------------| +| 12.0 | 1 | +|---------+------------------------| +| 12.9 | 1 | ++---------+------------------------+ diff --git a/tests/snapshots/12_distributed_weather_memory_trace.snap b/tests/snapshots/12_distributed_weather_memory_trace.snap new file mode 100644 index 0000000..4e734be --- /dev/null +++ b/tests/snapshots/12_distributed_weather_memory_trace.snap @@ -0,0 +1,400 @@ +--- +source: tests/integration_tests.rs +expression: json_lines +--- +[ + { + "level": "INFO", + "message": "close", + "span": { + "name": "read_query", + "query": "SELECT\n count(*),\n \"MinTemp\"\nFROM weather\nGROUP BY \"MinTemp\"\n" + }, + "spans": [ + { + "name": "test", + "test_name": "12_distributed_weather_memory" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::query", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "logical_plan": "Projection: count(Int64(1)) AS count(*), weather.MinTemp [count(*):Int64, MinTemp:Float64;N]\n Aggregate: groupBy=[[weather.MinTemp]], aggr=[[count(Int64(1))]] [MinTemp:Float64;N, count(Int64(1)):Int64]\n TableScan: weather [MinTemp:Float64;N, MaxTemp:Float64;N, Rainfall:Float64;N, Evaporation:Float64;N, Sunshine:Utf8View;N, WindGustDir:Utf8View;N, WindGustSpeed:Utf8View;N, WindDir9am:Utf8View;N, WindDir3pm:Utf8View;N, WindSpeed9am:Utf8View;N, WindSpeed3pm:Int64;N, Humidity9am:Int64;N, Humidity3pm:Int64;N, Pressure9am:Float64;N, Pressure3pm:Float64;N, Cloud9am:Int64;N, Cloud3pm:Int64;N, Temp9am:Float64;N, Temp3pm:Float64;N, RainToday:Utf8View;N, RISK_MM:Float64;N, RainTomorrow:Utf8View;N]", + "name": "parse_sql", + "sql": "SELECT\n count(*),\n \"MinTemp\"\nFROM weather\nGROUP BY \"MinTemp\"\n" + }, + "spans": [ + { + "name": "test", + "test_name": "12_distributed_weather_memory" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::query", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "create_physical_plan", + "physical_plan": "DistributedExec\n CoalescePartitionsExec\n ProjectionExec: expr=[count(Int64(1))@1 as count(*), MinTemp@0 as MinTemp]\n AggregateExec: mode=FinalPartitioned, gby=[MinTemp@0 as MinTemp], aggr=[count(Int64(1))]\n CoalesceBatchesExec: target_batch_size=8192\n RepartitionExec: partitioning=Hash([MinTemp@0], 8), input_partitions=8\n RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=3\n AggregateExec: mode=Partial, gby=[MinTemp@0 as MinTemp], aggr=[count(Int64(1))]\n DataSourceExec: file_groups={3 groups: [[/weather/result-000000.parquet], [/weather/result-000001.parquet], [/weather/result-000002.parquet]]}, projection=[MinTemp], file_type=parquet\n" + }, + "spans": [ + { + "name": "test", + "test_name": "12_distributed_weather_memory" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::query", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.metrics.bloom_filter_eval_time": "0.00ms", + "datafusion.metrics.bytes_scanned": "5404", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.file_open_errors": "0", + "datafusion.metrics.file_scan_errors": "0", + "datafusion.metrics.metadata_load_time": "0.00ms", + "datafusion.metrics.num_predicate_creation_errors": "0", + "datafusion.metrics.output_rows": "366", + "datafusion.metrics.page_index_eval_time": "0.00ms", + "datafusion.metrics.page_index_rows_matched": "0", + "datafusion.metrics.page_index_rows_pruned": "0", + "datafusion.metrics.predicate_evaluation_errors": "0", + "datafusion.metrics.pushdown_rows_matched": "0", + "datafusion.metrics.pushdown_rows_pruned": "0", + "datafusion.metrics.row_groups_matched_bloom_filter": "0", + "datafusion.metrics.row_groups_matched_statistics": "0", + "datafusion.metrics.row_groups_pruned_bloom_filter": "0", + "datafusion.metrics.row_groups_pruned_statistics": "0", + "datafusion.metrics.row_pushdown_eval_time": "0.00ms", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.statistics_eval_time": "0.00ms", + "datafusion.metrics.time_elapsed_opening": "0.00ms", + "datafusion.metrics.time_elapsed_processing": "0.00ms", + "datafusion.metrics.time_elapsed_scanning_total": "0.00ms", + "datafusion.metrics.time_elapsed_scanning_until_data": "0.00ms", + "datafusion.node": "DataSourceExec: file_groups={3 groups: [[/weather/result-000000.parquet], [/weather/result-000001.parquet], [/weather/result-000002.parquet]]}, projection=[MinTemp], file_type=parquet", + "datafusion.partitioning": "UnknownPartitioning(3)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "DataSourceExec", + "region": "us-west" + }, + "spans": [ + { + "name": "test", + "test_name": "12_distributed_weather_memory" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.output_rows": "249", + "datafusion.metrics.peak_mem_used": "19080", + "datafusion.metrics.skipped_aggregation_rows": "0", + "datafusion.metrics.spill_count": "0", + "datafusion.metrics.spilled_bytes": "0.0 B", + "datafusion.metrics.spilled_rows": "0", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.node": "AggregateExec: mode=Partial, gby=[MinTemp@0 as MinTemp], aggr=[count(Int64(1))]", + "datafusion.partitioning": "UnknownPartitioning(3)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "AggregateExec", + "region": "us-west" + }, + "spans": [ + { + "name": "test", + "test_name": "12_distributed_weather_memory" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.fetch_time": "0.00ms", + "datafusion.metrics.repartition_time": "0.00ms", + "datafusion.metrics.send_time": "0.00ms", + "datafusion.node": "RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=3", + "datafusion.partitioning": "RoundRobinBatch(8)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "RepartitionExec", + "region": "us-west" + }, + "spans": [ + { + "name": "test", + "test_name": "12_distributed_weather_memory" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.fetch_time": "0.00ms", + "datafusion.metrics.repartition_time": "0.00ms", + "datafusion.metrics.send_time": "0.00ms", + "datafusion.node": "RepartitionExec: partitioning=Hash([MinTemp@0], 8), input_partitions=8", + "datafusion.partitioning": "Hash([MinTemp@0], 8)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "RepartitionExec", + "region": "us-west" + }, + "spans": [ + { + "name": "test", + "test_name": "12_distributed_weather_memory" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.output_rows": "249", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.node": "CoalesceBatchesExec: target_batch_size=8192", + "datafusion.partitioning": "Hash([MinTemp@0], 8)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "CoalesceBatchesExec", + "region": "us-west" + }, + "spans": [ + { + "name": "test", + "test_name": "12_distributed_weather_memory" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.output_rows": "180", + "datafusion.metrics.peak_mem_used": "41632", + "datafusion.metrics.spill_count": "0", + "datafusion.metrics.spilled_bytes": "0.0 B", + "datafusion.metrics.spilled_rows": "0", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.node": "AggregateExec: mode=FinalPartitioned, gby=[MinTemp@0 as MinTemp], aggr=[count(Int64(1))]", + "datafusion.partitioning": "Hash([MinTemp@0], 8)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "AggregateExec", + "region": "us-west" + }, + "spans": [ + { + "name": "test", + "test_name": "12_distributed_weather_memory" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.output_rows": "180", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.node": "ProjectionExec: expr=[count(Int64(1))@1 as count(*), MinTemp@0 as MinTemp]", + "datafusion.partitioning": "Hash([MinTemp@1], 8)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "ProjectionExec", + "region": "us-west" + }, + "spans": [ + { + "name": "test", + "test_name": "12_distributed_weather_memory" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.output_rows": "180", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.node": "CoalescePartitionsExec", + "datafusion.partitioning": "UnknownPartitioning(1)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "CoalescePartitionsExec", + "region": "us-west" + }, + "spans": [ + { + "name": "test", + "test_name": "12_distributed_weather_memory" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.node": "DistributedExec", + "datafusion.partitioning": "UnknownPartitioning(1)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "DistributedExec", + "region": "us-west" + }, + "spans": [ + { + "name": "test", + "test_name": "12_distributed_weather_memory" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "run_traced_query", + "query_name": "weather" + }, + "spans": [ + { + "name": "test", + "test_name": "12_distributed_weather_memory" + } + ], + "target": "integration_utils::query", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + } +] diff --git a/tests/snapshots/13_distributed_weather_localhost_00_AggregateExec_localhost:50052.snap b/tests/snapshots/13_distributed_weather_localhost_00_AggregateExec_localhost:50052.snap new file mode 100644 index 0000000..8827167 --- /dev/null +++ b/tests/snapshots/13_distributed_weather_localhost_00_AggregateExec_localhost:50052.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+-----------------+ +| MinTemp | count(Int64(1)) | +|=========+=================| +| 10.1 | 4 | +|---------+-----------------| +| 17.2 | 2 | +|---------+-----------------| +| 11.8 | 1 | +|---------+-----------------| +| 16.7 | 1 | +|---------+-----------------| +| 12.3 | 2 | ++---------+-----------------+ diff --git a/tests/snapshots/13_distributed_weather_localhost_01_AggregateExec_localhost:50053.snap b/tests/snapshots/13_distributed_weather_localhost_01_AggregateExec_localhost:50053.snap new file mode 100644 index 0000000..58aa55b --- /dev/null +++ b/tests/snapshots/13_distributed_weather_localhost_01_AggregateExec_localhost:50053.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+-----------------+ +| MinTemp | count(Int64(1)) | +|=========+=================| +| -2.6 | 1 | +|---------+-----------------| +| 0.5 | 6 | +|---------+-----------------| +| -2.9 | 1 | +|---------+-----------------| +| 6.3 | 3 | +|---------+-----------------| +| 1.1 | 1 | ++---------+-----------------+ diff --git a/tests/snapshots/13_distributed_weather_localhost_02_AggregateExec_localhost:50051.snap b/tests/snapshots/13_distributed_weather_localhost_02_AggregateExec_localhost:50051.snap new file mode 100644 index 0000000..7a0ff84 --- /dev/null +++ b/tests/snapshots/13_distributed_weather_localhost_02_AggregateExec_localhost:50051.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+------------------------+ +| MinTemp | count(Int64(1))[count] | +|=========+========================| +| 8.0 | 1 | +|---------+------------------------| +| 14.0 | 1 | +|---------+------------------------| +| 13.7 | 1 | +|---------+------------------------| +| 13.3 | 3 | +|---------+------------------------| +| 7.6 | 1 | ++---------+------------------------+ diff --git a/tests/snapshots/13_distributed_weather_localhost_03_AggregateExec_localhost:50052.snap b/tests/snapshots/13_distributed_weather_localhost_03_AggregateExec_localhost:50052.snap new file mode 100644 index 0000000..487c85a --- /dev/null +++ b/tests/snapshots/13_distributed_weather_localhost_03_AggregateExec_localhost:50052.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+------------------------+ +| MinTemp | count(Int64(1))[count] | +|=========+========================| +| 6.6 | 1 | +|---------+------------------------| +| -1.6 | 2 | +|---------+------------------------| +| -3.1 | 1 | +|---------+------------------------| +| -0.1 | 2 | +|---------+------------------------| +| -0.6 | 3 | ++---------+------------------------+ diff --git a/tests/snapshots/13_distributed_weather_localhost_04_CoalesceBatchesExec_localhost:50052.snap b/tests/snapshots/13_distributed_weather_localhost_04_CoalesceBatchesExec_localhost:50052.snap new file mode 100644 index 0000000..7a87f13 --- /dev/null +++ b/tests/snapshots/13_distributed_weather_localhost_04_CoalesceBatchesExec_localhost:50052.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+------------------------+ +| MinTemp | count(Int64(1))[count] | +|=========+========================| +| 10.1 | 3 | +|---------+------------------------| +| 17.2 | 2 | +|---------+------------------------| +| 11.8 | 1 | +|---------+------------------------| +| 16.7 | 1 | +|---------+------------------------| +| 10.1 | 1 | ++---------+------------------------+ diff --git a/tests/snapshots/13_distributed_weather_localhost_05_CoalesceBatchesExec_localhost:50053.snap b/tests/snapshots/13_distributed_weather_localhost_05_CoalesceBatchesExec_localhost:50053.snap new file mode 100644 index 0000000..e700655 --- /dev/null +++ b/tests/snapshots/13_distributed_weather_localhost_05_CoalesceBatchesExec_localhost:50053.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+------------------------+ +| MinTemp | count(Int64(1))[count] | +|=========+========================| +| -2.6 | 1 | +|---------+------------------------| +| 0.5 | 4 | +|---------+------------------------| +| -2.9 | 1 | +|---------+------------------------| +| 6.3 | 1 | +|---------+------------------------| +| 1.1 | 1 | ++---------+------------------------+ diff --git a/tests/snapshots/13_distributed_weather_localhost_06_DataSourceExec_localhost:50051.snap b/tests/snapshots/13_distributed_weather_localhost_06_DataSourceExec_localhost:50051.snap new file mode 100644 index 0000000..fb6149f --- /dev/null +++ b/tests/snapshots/13_distributed_weather_localhost_06_DataSourceExec_localhost:50051.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+ +| MinTemp | +|=========| +| 8.0 | +|---------| +| 14.0 | +|---------| +| 13.7 | +|---------| +| 13.3 | +|---------| +| 7.6 | ++---------+ diff --git a/tests/snapshots/13_distributed_weather_localhost_07_DataSourceExec_localhost:50052.snap b/tests/snapshots/13_distributed_weather_localhost_07_DataSourceExec_localhost:50052.snap new file mode 100644 index 0000000..0ea64f3 --- /dev/null +++ b/tests/snapshots/13_distributed_weather_localhost_07_DataSourceExec_localhost:50052.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+ +| MinTemp | +|=========| +| 6.6 | +|---------| +| -1.6 | +|---------| +| -3.1 | +|---------| +| -0.1 | +|---------| +| -0.6 | ++---------+ diff --git a/tests/snapshots/13_distributed_weather_localhost_08_PartitionIsolatorExec_localhost:50051.snap b/tests/snapshots/13_distributed_weather_localhost_08_PartitionIsolatorExec_localhost:50051.snap new file mode 100644 index 0000000..fb6149f --- /dev/null +++ b/tests/snapshots/13_distributed_weather_localhost_08_PartitionIsolatorExec_localhost:50051.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+ +| MinTemp | +|=========| +| 8.0 | +|---------| +| 14.0 | +|---------| +| 13.7 | +|---------| +| 13.3 | +|---------| +| 7.6 | ++---------+ diff --git a/tests/snapshots/13_distributed_weather_localhost_09_PartitionIsolatorExec_localhost:50052.snap b/tests/snapshots/13_distributed_weather_localhost_09_PartitionIsolatorExec_localhost:50052.snap new file mode 100644 index 0000000..0ea64f3 --- /dev/null +++ b/tests/snapshots/13_distributed_weather_localhost_09_PartitionIsolatorExec_localhost:50052.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+ +| MinTemp | +|=========| +| 6.6 | +|---------| +| -1.6 | +|---------| +| -3.1 | +|---------| +| -0.1 | +|---------| +| -0.6 | ++---------+ diff --git a/tests/snapshots/13_distributed_weather_localhost_10_ProjectionExec_localhost:50052.snap b/tests/snapshots/13_distributed_weather_localhost_10_ProjectionExec_localhost:50052.snap new file mode 100644 index 0000000..3e502d8 --- /dev/null +++ b/tests/snapshots/13_distributed_weather_localhost_10_ProjectionExec_localhost:50052.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++----------+---------+ +| count(*) | MinTemp | +|==========+=========| +| 4 | 10.1 | +|----------+---------| +| 2 | 17.2 | +|----------+---------| +| 1 | 11.8 | +|----------+---------| +| 1 | 16.7 | +|----------+---------| +| 2 | 12.3 | ++----------+---------+ diff --git a/tests/snapshots/13_distributed_weather_localhost_11_ProjectionExec_localhost:50053.snap b/tests/snapshots/13_distributed_weather_localhost_11_ProjectionExec_localhost:50053.snap new file mode 100644 index 0000000..f7f1d85 --- /dev/null +++ b/tests/snapshots/13_distributed_weather_localhost_11_ProjectionExec_localhost:50053.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++----------+---------+ +| count(*) | MinTemp | +|==========+=========| +| 1 | -2.6 | +|----------+---------| +| 6 | 0.5 | +|----------+---------| +| 1 | -2.9 | +|----------+---------| +| 3 | 6.3 | +|----------+---------| +| 1 | 1.1 | ++----------+---------+ diff --git a/tests/snapshots/13_distributed_weather_localhost_12_RepartitionExec_localhost:50051.snap b/tests/snapshots/13_distributed_weather_localhost_12_RepartitionExec_localhost:50051.snap new file mode 100644 index 0000000..7a87f13 --- /dev/null +++ b/tests/snapshots/13_distributed_weather_localhost_12_RepartitionExec_localhost:50051.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+------------------------+ +| MinTemp | count(Int64(1))[count] | +|=========+========================| +| 10.1 | 3 | +|---------+------------------------| +| 17.2 | 2 | +|---------+------------------------| +| 11.8 | 1 | +|---------+------------------------| +| 16.7 | 1 | +|---------+------------------------| +| 10.1 | 1 | ++---------+------------------------+ diff --git a/tests/snapshots/13_distributed_weather_localhost_13_RepartitionExec_localhost:50052.snap b/tests/snapshots/13_distributed_weather_localhost_13_RepartitionExec_localhost:50052.snap new file mode 100644 index 0000000..145c69e --- /dev/null +++ b/tests/snapshots/13_distributed_weather_localhost_13_RepartitionExec_localhost:50052.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+------------------------+ +| MinTemp | count(Int64(1))[count] | +|=========+========================| +| 1.8 | 1 | +|---------+------------------------| +| 2.7 | 1 | +|---------+------------------------| +| -0.9 | 3 | +|---------+------------------------| +| 1.3 | 1 | +|---------+------------------------| +| -2.2 | 1 | ++---------+------------------------+ diff --git a/tests/snapshots/13_distributed_weather_localhost_14_RepartitionExec_localhost:50051.snap b/tests/snapshots/13_distributed_weather_localhost_14_RepartitionExec_localhost:50051.snap new file mode 100644 index 0000000..7a0ff84 --- /dev/null +++ b/tests/snapshots/13_distributed_weather_localhost_14_RepartitionExec_localhost:50051.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+------------------------+ +| MinTemp | count(Int64(1))[count] | +|=========+========================| +| 8.0 | 1 | +|---------+------------------------| +| 14.0 | 1 | +|---------+------------------------| +| 13.7 | 1 | +|---------+------------------------| +| 13.3 | 3 | +|---------+------------------------| +| 7.6 | 1 | ++---------+------------------------+ diff --git a/tests/snapshots/13_distributed_weather_localhost_15_RepartitionExec_localhost:50052.snap b/tests/snapshots/13_distributed_weather_localhost_15_RepartitionExec_localhost:50052.snap new file mode 100644 index 0000000..487c85a --- /dev/null +++ b/tests/snapshots/13_distributed_weather_localhost_15_RepartitionExec_localhost:50052.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+------------------------+ +| MinTemp | count(Int64(1))[count] | +|=========+========================| +| 6.6 | 1 | +|---------+------------------------| +| -1.6 | 2 | +|---------+------------------------| +| -3.1 | 1 | +|---------+------------------------| +| -0.1 | 2 | +|---------+------------------------| +| -0.6 | 3 | ++---------+------------------------+ diff --git a/tests/snapshots/13_distributed_weather_localhost_16_NetworkShuffleExec_localhost:50052.snap b/tests/snapshots/13_distributed_weather_localhost_16_NetworkShuffleExec_localhost:50052.snap new file mode 100644 index 0000000..7a87f13 --- /dev/null +++ b/tests/snapshots/13_distributed_weather_localhost_16_NetworkShuffleExec_localhost:50052.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+------------------------+ +| MinTemp | count(Int64(1))[count] | +|=========+========================| +| 10.1 | 3 | +|---------+------------------------| +| 17.2 | 2 | +|---------+------------------------| +| 11.8 | 1 | +|---------+------------------------| +| 16.7 | 1 | +|---------+------------------------| +| 10.1 | 1 | ++---------+------------------------+ diff --git a/tests/snapshots/13_distributed_weather_localhost_17_NetworkShuffleExec_localhost:50053.snap b/tests/snapshots/13_distributed_weather_localhost_17_NetworkShuffleExec_localhost:50053.snap new file mode 100644 index 0000000..e700655 --- /dev/null +++ b/tests/snapshots/13_distributed_weather_localhost_17_NetworkShuffleExec_localhost:50053.snap @@ -0,0 +1,17 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---------+------------------------+ +| MinTemp | count(Int64(1))[count] | +|=========+========================| +| -2.6 | 1 | +|---------+------------------------| +| 0.5 | 4 | +|---------+------------------------| +| -2.9 | 1 | +|---------+------------------------| +| 6.3 | 1 | +|---------+------------------------| +| 1.1 | 1 | ++---------+------------------------+ diff --git a/tests/snapshots/13_distributed_weather_localhost_trace.snap b/tests/snapshots/13_distributed_weather_localhost_trace.snap new file mode 100644 index 0000000..62bb824 --- /dev/null +++ b/tests/snapshots/13_distributed_weather_localhost_trace.snap @@ -0,0 +1,2704 @@ +--- +source: tests/integration_tests.rs +expression: json_lines +--- +[ + { + "level": "INFO", + "message": "close", + "span": { + "name": "list", + "object_store.prefix": "/weather", + "otel.name": "local_fs.list" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/weather/result-000001.parquet", + "object_store.range": "45453..45461", + "object_store.result.content_length": 8, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/weather/result-000000.parquet", + "object_store.range": "46076..46084", + "object_store.result.content_length": 8, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/weather/result-000002.parquet", + "object_store.range": "45816..45824", + "object_store.result.content_length": 8, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/weather/result-000001.parquet", + "object_store.range": "42765..45453", + "object_store.result.content_length": 2688, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/weather/result-000000.parquet", + "object_store.range": "43382..46076", + "object_store.result.content_length": 2694, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/weather/result-000002.parquet", + "object_store.range": "43126..45816", + "object_store.result.content_length": 2690, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/weather/result-000001.parquet", + "object_store.range": "33199..42765", + "object_store.result.content_length": 9566, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/weather/result-000000.parquet", + "object_store.range": "33807..43382", + "object_store.result.content_length": 9575, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/weather/result-000002.parquet", + "object_store.range": "33542..43126", + "object_store.result.content_length": 9584, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "head", + "object_store.location": "/nation.parquet", + "object_store.result.meta": "ObjectMeta { location: Path { raw: \"/nation.parquet\" }, last_modified: 1970-01-01T00:00:00Z, size: 2670, e_tag: Some\("ffffffff-fffffffffffff-fff"\), version: None }", + "otel.name": "local_fs.head" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/nation.parquet", + "object_store.range": "2662..2670", + "object_store.result.content_length": 8, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/nation.parquet", + "object_store.range": "1987..2662", + "object_store.result.content_length": 675, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/nation.parquet", + "object_store.range": "1714..1987", + "object_store.result.content_length": 273, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "head", + "object_store.location": "/region.parquet", + "object_store.result.meta": "ObjectMeta { location: Path { raw: \"/region.parquet\" }, last_modified: 1970-01-01T00:00:00Z, size: 1227, e_tag: Some\("ffffffff-fffffffffffff-fff"\), version: None }", + "otel.name": "local_fs.head" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/region.parquet", + "object_store.range": "1219..1227", + "object_store.result.content_length": 8, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/region.parquet", + "object_store.range": "698..1219", + "object_store.result.content_length": 521, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/region.parquet", + "object_store.range": "486..698", + "object_store.result.content_length": 212, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "head", + "object_store.location": "/part.parquet", + "object_store.result.meta": "ObjectMeta { location: Path { raw: \"/part.parquet\" }, last_modified: 1970-01-01T00:00:00Z, size: 6902976, e_tag: Some\("ffffffff-fffffffffffff-fff"\), version: None }", + "otel.name": "local_fs.head" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/part.parquet", + "object_store.range": "6902968..6902976", + "object_store.result.content_length": 8, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/part.parquet", + "object_store.range": "6900356..6902968", + "object_store.result.content_length": 2612, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/part.parquet", + "object_store.range": "6895687..6900356", + "object_store.result.content_length": 4669, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "head", + "object_store.location": "/supplier.parquet", + "object_store.result.meta": "ObjectMeta { location: Path { raw: \"/supplier.parquet\" }, last_modified: 1970-01-01T00:00:00Z, size: 901201, e_tag: Some\("ffffffff-fffffffffffff-fff"\), version: None }", + "otel.name": "local_fs.head" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/supplier.parquet", + "object_store.range": "901193..901201", + "object_store.result.content_length": 8, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/supplier.parquet", + "object_store.range": "900000..901193", + "object_store.result.content_length": 1193, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/supplier.parquet", + "object_store.range": "899497..900000", + "object_store.result.content_length": 503, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "head", + "object_store.location": "/partsupp.parquet", + "object_store.result.meta": "ObjectMeta { location: Path { raw: \"/partsupp.parquet\" }, last_modified: 1970-01-01T00:00:00Z, size: 44858110, e_tag: Some\("ffffffff-fffffffffffff-fff"\), version: None }", + "otel.name": "local_fs.head" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/partsupp.parquet", + "object_store.range": "44858102..44858110", + "object_store.result.content_length": 8, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/partsupp.parquet", + "object_store.range": "44845947..44858102", + "object_store.result.content_length": 12155, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/partsupp.parquet", + "object_store.range": "44824571..44845947", + "object_store.result.content_length": 21376, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "head", + "object_store.location": "/customer.parquet", + "object_store.result.meta": "ObjectMeta { location: Path { raw: \"/customer.parquet\" }, last_modified: 1970-01-01T00:00:00Z, size: 13933574, e_tag: Some\("ffffffff-fffffffffffff-fff"\), version: None }", + "otel.name": "local_fs.head" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/customer.parquet", + "object_store.range": "13933566..13933574", + "object_store.result.content_length": 8, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/customer.parquet", + "object_store.range": "13928771..13933566", + "object_store.result.content_length": 4795, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/customer.parquet", + "object_store.range": "13923894..13928771", + "object_store.result.content_length": 4877, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "head", + "object_store.location": "/orders.parquet", + "object_store.result.meta": "ObjectMeta { location: Path { raw: \"/orders.parquet\" }, last_modified: 1970-01-01T00:00:00Z, size: 63497280, e_tag: Some\("ffffffff-fffffffffffff-fff"\), version: None }", + "otel.name": "local_fs.head" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/orders.parquet", + "object_store.range": "63497272..63497280", + "object_store.result.content_length": 8, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/orders.parquet", + "object_store.range": "63478285..63497272", + "object_store.result.content_length": 18987, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "register_tpch_tables" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: Some(Memory), worker_task_wrapper: false }" + } + ], + "target": "integration_utils::data", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/orders.parquet", + "object_store.range": "63446814..63478285", + "object_store.result.content_length": 31471, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: Some(Memory), worker_task_wrapper: false }" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "register_tpch_tables" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: true, preview_limit: 0, compact_preview: false, distributed_mode: None, worker_task_wrapper: false }" + } + ], + "target": "integration_utils::data", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: true, preview_limit: 0, compact_preview: false, distributed_mode: None, worker_task_wrapper: false }" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "head", + "object_store.location": "/lineitem.parquet", + "object_store.result.meta": "ObjectMeta { location: Path { raw: \"/lineitem.parquet\" }, last_modified: 1970-01-01T00:00:00Z, size: 231669547, e_tag: Some\("ffffffff-fffffffffffff-fff"\), version: None }", + "otel.name": "local_fs.head" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "register_tpch_tables" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + } + ], + "target": "integration_utils::data", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "register_tpch_tables" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: false, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + } + ], + "target": "integration_utils::data", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: false, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/lineitem.parquet", + "object_store.range": "231669539..231669547", + "object_store.result.content_length": 8, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/lineitem.parquet", + "object_store.range": "231563065..231669539", + "object_store.result.content_length": 106474, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "register_tpch_tables" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: Some(Localhost), worker_task_wrapper: true }" + } + ], + "target": "integration_utils::data", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: Some(Localhost), worker_task_wrapper: true }" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "get_range", + "object_store.location": "/lineitem.parquet", + "object_store.range": "231380664..231563065", + "object_store.result.content_length": 182401, + "otel.name": "local_fs.get_range" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + { + "name": "register_tpch_tables" + } + ], + "target": "instrumented_object_store::instrumented_object_store", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "register_tpch_tables" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: false, preview_limit: 5, compact_preview: false, distributed_mode: None, worker_task_wrapper: false }" + } + ], + "target": "integration_utils::data", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "read_query", + "query": "SELECT\n count(*),\n \"MinTemp\"\nFROM weather\nGROUP BY \"MinTemp\"\n" + }, + "spans": [ + { + "name": "test", + "test_name": "13_distributed_weather_localhost" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::query", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "register_tpch_tables" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: false, preview_limit: 0, compact_preview: false, distributed_mode: None, worker_task_wrapper: false }" + } + ], + "target": "integration_utils::data", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: false, preview_limit: 0, compact_preview: false, distributed_mode: None, worker_task_wrapper: false }" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: false, preview_limit: 5, compact_preview: false, distributed_mode: None, worker_task_wrapper: false }" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "logical_plan": "Projection: count(Int64(1)) AS count(*), weather.MinTemp [count(*):Int64, MinTemp:Float64;N]\n Aggregate: groupBy=[[weather.MinTemp]], aggr=[[count(Int64(1))]] [MinTemp:Float64;N, count(Int64(1)):Int64]\n TableScan: weather [MinTemp:Float64;N, MaxTemp:Float64;N, Rainfall:Float64;N, Evaporation:Float64;N, Sunshine:Utf8View;N, WindGustDir:Utf8View;N, WindGustSpeed:Utf8View;N, WindDir9am:Utf8View;N, WindDir3pm:Utf8View;N, WindSpeed9am:Utf8View;N, WindSpeed3pm:Int64;N, Humidity9am:Int64;N, Humidity3pm:Int64;N, Pressure9am:Float64;N, Pressure3pm:Float64;N, Cloud9am:Int64;N, Cloud3pm:Int64;N, Temp9am:Float64;N, Temp3pm:Float64;N, RainToday:Utf8View;N, RISK_MM:Float64;N, RainTomorrow:Utf8View;N]", + "name": "parse_sql", + "sql": "SELECT\n count(*),\n \"MinTemp\"\nFROM weather\nGROUP BY \"MinTemp\"\n" + }, + "spans": [ + { + "name": "test", + "test_name": "13_distributed_weather_localhost" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::query", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "register_tpch_tables" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + } + ], + "target": "integration_utils::data", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "register_tpch_tables" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: false, preview_limit: 0, compact_preview: false, distributed_mode: None, worker_task_wrapper: false }" + } + ], + "target": "integration_utils::data", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: false, preview_limit: 0, compact_preview: false, distributed_mode: None, worker_task_wrapper: false }" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "test", + "test_name": "02_basic_metrics" + }, + "spans": [], + "target": "integration_tests", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "register_tpch_tables" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + } + ], + "target": "integration_utils::data", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "test", + "test_name": "01_basic" + }, + "spans": [], + "target": "integration_tests", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "create_physical_plan", + "physical_plan": "DistributedExec\n CoalescePartitionsExec\n [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=2\n ProjectionExec: expr=[count(Int64(1))@1 as count(*), MinTemp@0 as MinTemp]\n AggregateExec: mode=FinalPartitioned, gby=[MinTemp@0 as MinTemp], aggr=[count(Int64(1))]\n CoalesceBatchesExec: target_batch_size=8192\n [Stage 1] => NetworkShuffleExec: output_partitions=8, input_tasks=2\n RepartitionExec: partitioning=Hash([MinTemp@0], 16), input_partitions=8\n RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=2\n AggregateExec: mode=Partial, gby=[MinTemp@0 as MinTemp], aggr=[count(Int64(1))]\n PartitionIsolatorExec: t0:[p0,p1,__] t1:[__,__,p0] \n DataSourceExec: file_groups={3 groups: [[/weather/result-000000.parquet], [/weather/result-000001.parquet], [/weather/result-000002.parquet]]}, projection=[MinTemp], file_type=parquet\n" + }, + "spans": [ + { + "name": "test", + "test_name": "13_distributed_weather_localhost" + }, + { + "name": "run_traced_query", + "query_name": "weather" + } + ], + "target": "integration_utils::query", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "test", + "test_name": "04_basic_compact_preview" + }, + "spans": [], + "target": "integration_tests", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "build", + "self": "SessionBuilder { record_object_store: true, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "test", + "test_name": "05_basic_all_options" + }, + "spans": [], + "target": "integration_tests", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "test", + "test_name": "12_distributed_weather_memory" + }, + "spans": [], + "target": "integration_tests", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "test", + "test_name": "03_basic_preview" + }, + "spans": [], + "target": "integration_tests", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "test", + "test_name": "08_recursive" + }, + "spans": [], + "target": "integration_tests", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "test", + "test_name": "09_recursive_all_options" + }, + "spans": [], + "target": "integration_tests", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "test", + "test_name": "06_object_store_all_options" + }, + "spans": [], + "target": "integration_tests", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "register_tpch_tables" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + } + ], + "target": "integration_utils::data", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "register_tpch_tables" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: false, preview_limit: 0, compact_preview: false, distributed_mode: None, worker_task_wrapper: false }" + } + ], + "target": "integration_utils::data", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "register_tpch_tables" + }, + "spans": [ + { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + } + ], + "target": "integration_utils::data", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: true, preview_limit: 5, compact_preview: true, distributed_mode: None, worker_task_wrapper: false }" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "build", + "self": "SessionBuilder { record_object_store: false, record_metrics: false, preview_limit: 0, compact_preview: false, distributed_mode: None, worker_task_wrapper: false }" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "test", + "test_name": "11_weather" + }, + "spans": [], + "target": "integration_tests", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.metrics.bloom_filter_eval_time": "0.00ms", + "datafusion.metrics.bytes_scanned": "3569", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.file_open_errors": "0", + "datafusion.metrics.file_scan_errors": "0", + "datafusion.metrics.metadata_load_time": "0.00ms", + "datafusion.metrics.num_predicate_creation_errors": "0", + "datafusion.metrics.output_rows": "244", + "datafusion.metrics.page_index_eval_time": "0.00ms", + "datafusion.metrics.page_index_rows_matched": "0", + "datafusion.metrics.page_index_rows_pruned": "0", + "datafusion.metrics.predicate_evaluation_errors": "0", + "datafusion.metrics.pushdown_rows_matched": "0", + "datafusion.metrics.pushdown_rows_pruned": "0", + "datafusion.metrics.row_groups_matched_bloom_filter": "0", + "datafusion.metrics.row_groups_matched_statistics": "0", + "datafusion.metrics.row_groups_pruned_bloom_filter": "0", + "datafusion.metrics.row_groups_pruned_statistics": "0", + "datafusion.metrics.row_pushdown_eval_time": "0.00ms", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.statistics_eval_time": "0.00ms", + "datafusion.metrics.time_elapsed_opening": "0.00ms", + "datafusion.metrics.time_elapsed_processing": "0.00ms", + "datafusion.metrics.time_elapsed_scanning_total": "0.00ms", + "datafusion.metrics.time_elapsed_scanning_until_data": "0.00ms", + "datafusion.node": "DataSourceExec: file_groups={3 groups: [[/weather/result-000000.parquet], [/weather/result-000001.parquet], [/weather/result-000002.parquet]]}, projection=[MinTemp], file_type=parquet", + "datafusion.partitioning": "UnknownPartitioning(3)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "DataSourceExec", + "region": "us-west", + "worker.id": "localhost:50051" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.node": "PartitionIsolatorExec: t0:[p0,p1,__] t1:[__,__,p0] ", + "datafusion.partitioning": "UnknownPartitioning(2)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "PartitionIsolatorExec", + "region": "us-west", + "worker.id": "localhost:50051" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.output_rows": "160", + "datafusion.metrics.peak_mem_used": "12672", + "datafusion.metrics.skipped_aggregation_rows": "0", + "datafusion.metrics.spill_count": "0", + "datafusion.metrics.spilled_bytes": "0.0 B", + "datafusion.metrics.spilled_rows": "0", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.node": "AggregateExec: mode=Partial, gby=[MinTemp@0 as MinTemp], aggr=[count(Int64(1))]", + "datafusion.partitioning": "UnknownPartitioning(2)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "AggregateExec", + "region": "us-west", + "worker.id": "localhost:50051" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.fetch_time": "0.00ms", + "datafusion.metrics.repartition_time": "0.00ms", + "datafusion.metrics.send_time": "0.00ms", + "datafusion.node": "RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=2", + "datafusion.partitioning": "RoundRobinBatch(8)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "RepartitionExec", + "region": "us-west", + "worker.id": "localhost:50051" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.fetch_time": "0.00ms", + "datafusion.metrics.repartition_time": "0.00ms", + "datafusion.metrics.send_time": "0.00ms", + "datafusion.node": "RepartitionExec: partitioning=Hash([MinTemp@0], 16), input_partitions=8", + "datafusion.partitioning": "Hash([MinTemp@0], 16)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "RepartitionExec", + "region": "us-west", + "worker.id": "localhost:50051" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.metrics.bloom_filter_eval_time": "0.00ms", + "datafusion.metrics.bytes_scanned": "1835", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.file_open_errors": "0", + "datafusion.metrics.file_scan_errors": "0", + "datafusion.metrics.metadata_load_time": "0.00ms", + "datafusion.metrics.num_predicate_creation_errors": "0", + "datafusion.metrics.output_rows": "122", + "datafusion.metrics.page_index_eval_time": "0.00ms", + "datafusion.metrics.page_index_rows_matched": "0", + "datafusion.metrics.page_index_rows_pruned": "0", + "datafusion.metrics.predicate_evaluation_errors": "0", + "datafusion.metrics.pushdown_rows_matched": "0", + "datafusion.metrics.pushdown_rows_pruned": "0", + "datafusion.metrics.row_groups_matched_bloom_filter": "0", + "datafusion.metrics.row_groups_matched_statistics": "0", + "datafusion.metrics.row_groups_pruned_bloom_filter": "0", + "datafusion.metrics.row_groups_pruned_statistics": "0", + "datafusion.metrics.row_pushdown_eval_time": "0.00ms", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.statistics_eval_time": "0.00ms", + "datafusion.metrics.time_elapsed_opening": "0.00ms", + "datafusion.metrics.time_elapsed_processing": "0.00ms", + "datafusion.metrics.time_elapsed_scanning_total": "0.00ms", + "datafusion.metrics.time_elapsed_scanning_until_data": "0.00ms", + "datafusion.node": "DataSourceExec: file_groups={3 groups: [[/weather/result-000000.parquet], [/weather/result-000001.parquet], [/weather/result-000002.parquet]]}, projection=[MinTemp], file_type=parquet", + "datafusion.partitioning": "UnknownPartitioning(3)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "DataSourceExec", + "region": "us-west", + "worker.id": "localhost:50052" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.node": "PartitionIsolatorExec: t0:[p0,p1,__] t1:[__,__,p0] ", + "datafusion.partitioning": "UnknownPartitioning(2)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "PartitionIsolatorExec", + "region": "us-west", + "worker.id": "localhost:50052" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.output_rows": "89", + "datafusion.metrics.peak_mem_used": "6472", + "datafusion.metrics.skipped_aggregation_rows": "0", + "datafusion.metrics.spill_count": "0", + "datafusion.metrics.spilled_bytes": "0.0 B", + "datafusion.metrics.spilled_rows": "0", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.node": "AggregateExec: mode=Partial, gby=[MinTemp@0 as MinTemp], aggr=[count(Int64(1))]", + "datafusion.partitioning": "UnknownPartitioning(2)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "AggregateExec", + "region": "us-west", + "worker.id": "localhost:50052" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.fetch_time": "0.00ms", + "datafusion.metrics.repartition_time": "0.00ms", + "datafusion.metrics.send_time": "0.00ms", + "datafusion.node": "RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=2", + "datafusion.partitioning": "RoundRobinBatch(8)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "RepartitionExec", + "region": "us-west", + "worker.id": "localhost:50052" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.fetch_time": "0.00ms", + "datafusion.metrics.repartition_time": "0.00ms", + "datafusion.metrics.send_time": "0.00ms", + "datafusion.node": "RepartitionExec: partitioning=Hash([MinTemp@0], 16), input_partitions=8", + "datafusion.partitioning": "Hash([MinTemp@0], 16)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "RepartitionExec", + "region": "us-west", + "worker.id": "localhost:50052" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.node": "[Stage 1] => NetworkShuffleExec: output_partitions=8, input_tasks=2", + "datafusion.partitioning": "Hash([MinTemp@0], 8)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "NetworkShuffleExec", + "region": "us-west", + "worker.id": "localhost:50052" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.output_rows": "138", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.node": "CoalesceBatchesExec: target_batch_size=8192", + "datafusion.partitioning": "Hash([MinTemp@0], 8)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "CoalesceBatchesExec", + "region": "us-west", + "worker.id": "localhost:50052" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.output_rows": "102", + "datafusion.metrics.peak_mem_used": "39856", + "datafusion.metrics.spill_count": "0", + "datafusion.metrics.spilled_bytes": "0.0 B", + "datafusion.metrics.spilled_rows": "0", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.node": "AggregateExec: mode=FinalPartitioned, gby=[MinTemp@0 as MinTemp], aggr=[count(Int64(1))]", + "datafusion.partitioning": "Hash([MinTemp@0], 8)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "AggregateExec", + "region": "us-west", + "worker.id": "localhost:50052" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.output_rows": "102", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.node": "ProjectionExec: expr=[count(Int64(1))@1 as count(*), MinTemp@0 as MinTemp]", + "datafusion.partitioning": "Hash([MinTemp@1], 8)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "ProjectionExec", + "region": "us-west", + "worker.id": "localhost:50052" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.node": "[Stage 1] => NetworkShuffleExec: output_partitions=8, input_tasks=2", + "datafusion.partitioning": "Hash([MinTemp@0], 8)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "NetworkShuffleExec", + "region": "us-west", + "worker.id": "localhost:50053" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.output_rows": "111", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.node": "CoalesceBatchesExec: target_batch_size=8192", + "datafusion.partitioning": "Hash([MinTemp@0], 8)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "CoalesceBatchesExec", + "region": "us-west", + "worker.id": "localhost:50053" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.output_rows": "78", + "datafusion.metrics.peak_mem_used": "39472", + "datafusion.metrics.spill_count": "0", + "datafusion.metrics.spilled_bytes": "0.0 B", + "datafusion.metrics.spilled_rows": "0", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.node": "AggregateExec: mode=FinalPartitioned, gby=[MinTemp@0 as MinTemp], aggr=[count(Int64(1))]", + "datafusion.partitioning": "Hash([MinTemp@0], 8)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "AggregateExec", + "region": "us-west", + "worker.id": "localhost:50053" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Final", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.output_rows": "78", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.node": "ProjectionExec: expr=[count(Int64(1))@1 as count(*), MinTemp@0 as MinTemp]", + "datafusion.partitioning": "Hash([MinTemp@1], 8)", + "datafusion.preview": "", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "ProjectionExec", + "region": "us-west", + "worker.id": "localhost:50053" + }, + "spans": [], + "target": "integration_utils::session", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "grpc_request", + "otel.kind": "server", + "rpc.method": "DoGet", + "rpc.service": "arrow.flight.protocol.FlightService", + "rpc.system": "grpc" + }, + "spans": [], + "target": "integration_utils::distributed::trace_middleware", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + } +] diff --git a/tests/test_utils/in_memory_writer.rs b/tests/test_utils/in_memory_writer.rs deleted file mode 100644 index 956b6c1..0000000 --- a/tests/test_utils/in_memory_writer.rs +++ /dev/null @@ -1,64 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025 Datadog, Inc. - -use std::{ - io::{Result as IoResult, Write}, - sync::{Arc, Mutex, MutexGuard}, -}; -use tracing_subscriber::fmt::MakeWriter; - -/// A type implementing `MakeWriter` for an `Arc>>`. -/// -/// When asked to produce a writer, it locks the `Arc>>` and -/// returns a guard that implements `io::Write`. -#[derive(Clone)] -pub struct InMemoryMakeWriter(Arc>>); - -impl InMemoryMakeWriter { - pub fn new(inner: Arc>>) -> Self { - Self(inner) - } -} - -impl<'a> MakeWriter<'a> for InMemoryMakeWriter { - // We'll return our own guard type that implements `Write`. - type Writer = InMemoryWriter<'a>; - - fn make_writer(&'a self) -> Self::Writer { - // Lock the mutex and wrap the guard in `InMemoryWriter`. - InMemoryWriter { - guard: self.0.lock().unwrap(), - } - } -} - -/// Our guard type that implements `io::Write` by forwarding to the locked buffer. -pub struct InMemoryWriter<'a> { - guard: MutexGuard<'a, Vec>, -} - -impl Write for InMemoryWriter<'_> { - fn write(&mut self, buf: &[u8]) -> IoResult { - self.guard.write(buf) - } - - fn flush(&mut self) -> IoResult<()> { - self.guard.flush() - } -} diff --git a/tests/test_utils/insta_settings.rs b/tests/test_utils/insta_settings.rs index 90eba73..8ee8511 100644 --- a/tests/test_utils/insta_settings.rs +++ b/tests/test_utils/insta_settings.rs @@ -49,7 +49,7 @@ pub fn settings() -> Settings { // Redact environment dependent paths to parquet files // from: "object_store.location": "usr/home/user.name/workspace/datafusion-tracing/integration/data/nation.parquet" // to : "object_store.location": "/nation.parquet" - let tpch_tables_dir = integration_utils::tpch_tables_dir() + let tpch_tables_dir = integration_utils::data_dir() .to_string_lossy() .trim_start_matches(std::path::is_separator) .to_string(); diff --git a/tests/test_utils/mod.rs b/tests/test_utils/mod.rs index a14fed4..08057a4 100644 --- a/tests/test_utils/mod.rs +++ b/tests/test_utils/mod.rs @@ -17,5 +17,5 @@ // // This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025 Datadog, Inc. -pub mod in_memory_writer; pub mod insta_settings; +pub mod multi_buffer_writer; diff --git a/tests/test_utils/multi_buffer_writer.rs b/tests/test_utils/multi_buffer_writer.rs new file mode 100644 index 0000000..44aea90 --- /dev/null +++ b/tests/test_utils/multi_buffer_writer.rs @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025 Datadog, Inc. + +use std::{ + collections::HashMap, + io::{Result as IoResult, Write}, + sync::{Arc, Mutex}, +}; +use tracing_subscriber::fmt::MakeWriter; + +tokio::task_local! { + /// Task-local storage for the worker port, used to route logs to the correct buffer + pub static WORKER_PORT: u16; +} + +/// A type implementing `MakeWriter` that routes logs to different buffers based on worker port. +/// +/// The main buffer receives logs from the coordinator/main thread. +/// Worker buffers receive logs from worker tasks based on task-local storage. +#[derive(Clone)] +pub struct MultiBufferMakeWriter { + /// Main buffer for coordinator logs + main_buffer: Arc>>, + /// Map of worker port -> buffer for worker logs + worker_buffers: Arc>>>>>, +} + +impl MultiBufferMakeWriter { + pub fn new( + main_buffer: Arc>>, + worker_buffers: Arc>>>>>, + ) -> Self { + Self { + main_buffer, + worker_buffers, + } + } + + /// Get all buffers (main + workers) for concatenation + pub fn collect_all_buffers(&self) -> Vec { + let mut result = Vec::new(); + + // First, collect main buffer + { + let main = self.main_buffer.lock().unwrap(); + result.extend_from_slice(&main); + } + + // Then, collect worker buffers in port order + { + let worker_buffers = self.worker_buffers.lock().unwrap(); + let mut ports: Vec<_> = worker_buffers.keys().copied().collect(); + ports.sort(); + + for port in ports { + if let Some(buffer) = worker_buffers.get(&port) { + let buffer = buffer.lock().unwrap(); + result.extend_from_slice(&buffer); + } + } + } + + result + } +} + +impl<'a> MakeWriter<'a> for MultiBufferMakeWriter { + type Writer = MultiBufferWriter; + + fn make_writer(&'a self) -> Self::Writer { + // Try to get the worker port from task-local storage + let worker_port = WORKER_PORT.try_with(|&port| port).ok(); + + // Select the appropriate buffer + let buffer = if let Some(port) = worker_port { + let worker_buffers = self.worker_buffers.lock().unwrap(); + if let Some(worker_buffer) = worker_buffers.get(&port) { + Arc::clone(worker_buffer) + } else { + Arc::clone(&self.main_buffer) + } + } else { + Arc::clone(&self.main_buffer) + }; + + MultiBufferWriter { buffer } + } +} + +/// Guard type that implements `io::Write` by forwarding to the selected buffer. +pub struct MultiBufferWriter { + buffer: Arc>>, +} + +impl Write for MultiBufferWriter { + fn write(&mut self, buf: &[u8]) -> IoResult { + let mut guard = self.buffer.lock().unwrap(); + guard.write(buf) + } + + fn flush(&mut self) -> IoResult<()> { + let mut guard = self.buffer.lock().unwrap(); + guard.flush() + } +}