add controller package and agent CLI

This change relates to #214. It adds a new package to this crate called `datafusion-distributed-controller` which will contain control-plane-related tooling for running a distributed datafusion clusters. This utility will be used for large-scale integration tests and benchmarking.

The proposed architecture is:
- have a local CLI "controller" which can interface with agents running on workers
- the agents can run code on the machine (ex. download data from blob store, start arrow flight server, execute sql on the cluster etc.)

This change adds an agent. The controller will be added in a future commit. Usually an agent runs on a port and is interfaced with via RPC, but since this utility is just for testing, this change proposes that the agent can be interfaced with via a CLI.

Author: jayshrivastava

.github/workflows/ci.yml

@@ -39,7 +39,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup
       - run: cargo test --features tpch --test tpch_validation_test
-
+  
   format-check:
     runs-on: ubuntu-latest
     steps:
@@ -48,3 +48,4 @@ jobs:
         with:
           components: rustfmt
       - run: cargo fmt --all -- --check
+

Cargo.lock

@@ -1,30 +1,36 @@
 [workspace]
-members = ["benchmarks"]
-
+members = ["benchmarks", "datafusion_distributed_controller"]
 [workspace.dependencies]
+arrow-flight = "56.1.0"
+async-trait = "0.1.88"
 datafusion = { version = "50.0.0", default-features = false }
 datafusion-proto = { version = "50.0.0" }
+tokio = { version = "1.46.1", features = ["full"] }
+tonic = { version = "0.13.1", features = ["transport"] }
+# Updated to 0.13.1 to match arrow-flight 56.1.0
+tower = "0.5.2"
+url = "2.5.4"
+futures = "0.3.31"
 
 [package]
 name = "datafusion-distributed"
 version = "0.1.0"
 edition = "2024"
 
 [dependencies]
+arrow-flight = { workspace = true }
+async-trait = { workspace = true }
 chrono = { version = "0.4.42" }
 datafusion = { workspace = true }
 datafusion-proto = { workspace = true }
-arrow-flight = "56.1.0"
 arrow-select = "56.1.0"
-async-trait = "0.1.88"
-tokio = { version = "1.46.1", features = ["full"] }
-# Updated to 0.13.1 to match arrow-flight 56.1.0
-tonic = { version = "0.13.1", features = ["transport"] }
-tower = "0.5.2"
+tokio = { workspace = true }
+tonic = { workspace = true }
+tower = { workspace = true }
+url = { workspace = true }
 http = "1.3.1"
 itertools = "0.14.0"
 futures = "0.3.31"
-url = "2.5.4"
 uuid = "1.17.0"
 delegate = "0.13.4"
 dashmap = "6.1.0"

Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "distributed-datafusion-controller"
+version = "0.1.0"
+edition = "2024"
+
+[[bin]]
+name = "ddf_test"
+path = "src/main.rs"
+
+[[bin]]
+name = "agent"
+path = "src/bin/agent.rs"
+
+[dependencies]
+clap = { version = "4.5.51", features = ["derive"] }
+datafusion-distributed = { path = "..", features = ["integration"] }
+moka = {version = "0.12.11",  features = ["sync"]}
+tokio = { workspace = true }
+tonic = { workspace = true }
+tower = { workspace = true }
+url = { workspace = true }
+futures = { workspace = true }
+datafusion = { workspace = true }
+arrow-flight = "56.1.0"
+async-trait = "0.1.88"
+serde = { version = "1.0", features = ["derive"] }
+serde_yaml = "0.9"
+tempfile = "3.8"
+log = "0.4.28"
+
+