[Feature/Datafusion] Changes for tokio io/cpu runtime + search flow async changes (#20071)

* Changes for tokio io/cpu runtime + search flow async changes

Signed-off-by: bharath-techie <[email protected]>

* minor fixes and todos

Signed-off-by: bharath-techie <[email protected]>

* Addressing comment to move to ActionListener

Signed-off-by: bharath-techie <[email protected]>

* addressing comments

Signed-off-by: bharath-techie <[email protected]>

* fix to sort numeric aggs

Signed-off-by: bharath-techie <[email protected]>

---------

Signed-off-by: bharath-techie <[email protected]>

Author: bharath-techie

plugins/engine-datafusion/.cargo/config.toml

@@ -0,0 +1,2 @@
+[build]
+rustflags = ["--cfg", "tokio_unstable", "-C", "force-frame-pointers=yes", "-C", "symbol-mangling-version=v0"]

plugins/engine-datafusion/Cargo.toml

@@ -30,6 +30,7 @@ prost = "0.13"
 tokio = { version = "1.0", features = ["full"] }
 futures = "0.3"
 #tokio = { version = "1.0", features = ["rt", "rt-multi-thread", "macros"] }
+tokio-metrics = "0.4"
 
 # Serialization
 serde = { version = "1.0", features = ["derive"] }
@@ -63,14 +64,23 @@ regex = "1.11.2"
 #[build-dependencies]
 #cbindgen = "0.27"
 
+once_cell = "1.21.3"
+tokio-stream = "0.1.17"
+parking_lot = "0.12.5"
+tracing = "0.1.41"
 
 [profile.release]
 lto = true
 codegen-units = 1
 panic = "abort"
+incremental = true    # Enable incremental compilation
+debug = "line-tables-only"
+strip = false
 
 [profile.dev]
 opt-level = 1          # Some optimization for reasonable performance
 lto = false           # Disable LTO for faster builds
-codegen-units = 16    # More parallel compilation
+codegen-units = 1    # More parallel compilation
 incremental = true    # Enable incremental compilation
+debug = "full"
+strip = false

plugins/engine-datafusion/jni/.cargo/config.toml

@@ -0,0 +1,2 @@
+[build]
+rustflags = ["--cfg", "tokio_unstable", "-C", "force-frame-pointers=yes", "-C", "symbol-mangling-version=v0"]

plugins/engine-datafusion/jni/Cargo.toml

@@ -34,6 +34,7 @@ prost = { workspace = true }
 tokio = { workspace = true }
 futures = { workspace = true }
 #tokio = { version = "1.0", features = ["rt", "rt-multi-thread", "macros"] }
+tokio-metrics = { workspace = true }
 
 # Serialization
 serde = { workspace = true }
@@ -48,6 +49,9 @@ log ={ workspace = true }
 # Parquet support
 parquet = { workspace = true }
 
+# System info
+num_cpus = "1.16"
+
 
 # Object store for file access
 object_store = { workspace = true }
@@ -64,19 +68,21 @@ async-trait = { workspace = true }
 itertools = { workspace = true }
 rstest = { workspace = true }
 regex = { workspace = true }
-parking_lot = "0.12.5"
+
+once_cell = { workspace = true }
+tokio-stream = { workspace = true }
+parking_lot = { workspace = true }
+tracing = { workspace = true }
 
 [build-dependencies]
 cbindgen = "0.27"
-[build]
-rustflags = ["-C", "force-frame-pointers=yes","symbol-mangling-version=v0"]
 
 [profile.release]
 lto = true
 codegen-units = 1
 panic = "abort"
 incremental = true
-debug = "full"
+debug = "line-tables-only"        #TODO : remove this
 strip = false
 
 [profile.dev]