add controller package and agent CLI

This change relates to #214. It adds a new package to this crate called
`datafusion-distributed-controller` which will contain control-plane-related tooling for running a distributed datafusion clusters. This
utility will be used for large-scale integration tests and benchmarking.

The `agent` is a binary introduced in this package which can be installed on remote workers. This allows a future "controller" (ex. a CLI
on my laptop) to run code on remote workers. Some agent operations include
- starting an arrow flight endpoint for distribued workers
- downloading test fixtures from blobstorage
- running sql on the cluster

Author: jayshrivastava

.github/workflows/ci.yml

@@ -39,7 +39,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup
       - run: cargo test --features tpch --test tpch_validation_test
-
+  
   format-check:
     runs-on: ubuntu-latest
     steps:
@@ -48,3 +48,4 @@ jobs:
         with:
           components: rustfmt
       - run: cargo fmt --all -- --check
+

Cargo.lock

@@ -1,30 +1,36 @@
 [workspace]
-members = ["benchmarks"]
-
+members = ["benchmarks", "datafusion_distributed_controller"]
 [workspace.dependencies]
+arrow-flight = "56.1.0"
+async-trait = "0.1.88"
 datafusion = { version = "50.0.0", default-features = false }
 datafusion-proto = { version = "50.0.0" }
+tokio = { version = "1.46.1", features = ["full"] }
+tonic = { version = "0.13.1", features = ["transport"] }
+# Updated to 0.13.1 to match arrow-flight 56.1.0
+tower = "0.5.2"
+url = "2.5.4"
+futures = "0.3.31"
 
 [package]
 name = "datafusion-distributed"
 version = "0.1.0"
 edition = "2024"
 
 [dependencies]
+arrow-flight = { workspace = true }
+async-trait = { workspace = true }
 chrono = { version = "0.4.42" }
 datafusion = { workspace = true }
 datafusion-proto = { workspace = true }
-arrow-flight = "56.1.0"
 arrow-select = "56.1.0"
-async-trait = "0.1.88"
-tokio = { version = "1.46.1", features = ["full"] }
-# Updated to 0.13.1 to match arrow-flight 56.1.0
-tonic = { version = "0.13.1", features = ["transport"] }
-tower = "0.5.2"
+tokio = { workspace = true }
+tonic = { workspace = true }
+tower = { workspace = true }
+url = { workspace = true }
 http = "1.3.1"
 itertools = "0.14.0"
 futures = "0.3.31"
-url = "2.5.4"
 uuid = "1.17.0"
 delegate = "0.13.4"
 dashmap = "6.1.0"

Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "distributed-datafusion-controller"
+version = "0.1.0"
+edition = "2024"
+
+[[bin]]
+name = "ddf_test"
+path = "src/main.rs"
+
+[[bin]]
+name = "agent"
+path = "src/bin/agent.rs"
+
+[dependencies]
+clap = { version = "4.5.51", features = ["derive"] }
+datafusion-distributed = { path = "..", features = ["integration"] }
+moka = {version = "0.12.11",  features = ["sync"]}
+tokio = { workspace = true }
+tonic = { workspace = true }
+tower = { workspace = true }
+url = { workspace = true }
+futures = { workspace = true }
+datafusion = { workspace = true }
+arrow-flight = "56.1.0"
+async-trait = "0.1.88"
+serde = { version = "1.0", features = ["derive"] }
+serde_yaml = "0.9"
+tempfile = "3.8"
+log = "0.4.28"
+
+