Merge pull request #4 from robtandy/rob.tandy/ray_shuffle

save point before arrow flight refactor

Author: robtandy

Cargo.lock

@@ -37,6 +37,12 @@ datafusion-proto = "43.0"
 futures = "0.3"
 glob = "0.3.1"
 log = "0.4"
+object_store = { version = "0.11.0", features = [
+  "aws",
+  "gcp",
+  "azure",
+  "http",
+] }
 prost = "0.13"
 pyo3 = { version = "0.22.6", features = [
   "extension-module",
@@ -51,6 +57,7 @@ tokio = { version = "1.40", features = [
   "sync",
 ] }
 uuid = "1.11.0"
+url = "2"
 
 [build-dependencies]
 prost-types = "0.13"
@@ -59,6 +66,7 @@ tonic-build = { version = "0.8", default-features = false, features = [
   "transport",
   "prost",
 ] }
+url = "2"
 
 [dev-dependencies]
 anyhow = "1.0.89"

Cargo.toml

@@ -19,38 +19,38 @@ use std::path::Path;
 
 fn main() -> Result<(), String> {
     use std::io::Write;
-
-    let out = std::path::PathBuf::from(std::env::var("OUT_DIR").unwrap());
-
-    // for use in docker build where file changes can be wonky
-    println!("cargo:rerun-if-env-changed=FORCE_REBUILD");
-
-    let version = rustc_version::version().unwrap();
-    println!("cargo:rustc-env=RUSTC_VERSION={version}");
-
-    let path = "src/proto/generated/protobuf.rs";
-
-    // We don't include the proto files in releases so that downstreams
-    // do not need to have PROTOC included
-    if Path::new("src/proto/datafusion_ray.proto").exists() {
-        println!("cargo:rerun-if-changed=src/proto/datafusion_common.proto");
-        println!("cargo:rerun-if-changed=src/proto/datafusion.proto");
-        println!("cargo:rerun-if-changed=src/proto/datafusion_ray.proto");
-        tonic_build::configure()
-            .extern_path(".datafusion", "::datafusion_proto::protobuf")
-            .extern_path(".datafusion_common", "::datafusion_proto::protobuf")
-            .compile(&["src/proto/datafusion_ray.proto"], &["src/proto"])
-            .map_err(|e| format!("protobuf compilation failed: {e}"))?;
-        let generated_source_path = out.join("datafusion_ray.protobuf.rs");
-        let code = std::fs::read_to_string(generated_source_path).unwrap();
-        let mut file = std::fs::OpenOptions::new()
-            .write(true)
-            .truncate(true)
-            .create(true)
-            .open(path)
-            .unwrap();
-        file.write_all(code.as_str().as_ref()).unwrap();
-    }
-
+    /*
+        let out = std::path::PathBuf::from(std::env::var("OUT_DIR").unwrap());
+
+        // for use in docker build where file changes can be wonky
+        println!("cargo:rerun-if-env-changed=FORCE_REBUILD");
+
+        let version = rustc_version::version().unwrap();
+        println!("cargo:rustc-env=RUSTC_VERSION={version}");
+
+        let path = "src/proto/generated/protobuf.rs";
+
+        // We don't include the proto files in releases so that downstreams
+        // do not need to have PROTOC included
+        if Path::new("src/proto/datafusion_ray.proto").exists() {
+            println!("cargo:rerun-if-changed=src/proto/datafusion_common.proto");
+            println!("cargo:rerun-if-changed=src/proto/datafusion.proto");
+            println!("cargo:rerun-if-changed=src/proto/datafusion_ray.proto");
+            tonic_build::configure()
+                .extern_path(".datafusion", "::datafusion_proto::protobuf")
+                .extern_path(".datafusion_common", "::datafusion_proto::protobuf")
+                .compile(&["src/proto/datafusion_ray.proto"], &["src/proto"])
+                .map_err(|e| format!("protobuf compilation failed: {e}"))?;
+            let generated_source_path = out.join("datafusion_ray.protobuf.rs");
+            let code = std::fs::read_to_string(generated_source_path).unwrap();
+            let mut file = std::fs::OpenOptions::new()
+                .write(true)
+                .truncate(true)
+                .create(true)
+                .open(path)
+                .unwrap();
+            file.write_all(code.as_str().as_ref()).unwrap();
+        }
+    */
     Ok(())
 }