chore: Adding an optional hdfs crate (#1377)

* Adding an optional `hdfs` crate

* Update NOTICE.txt

Co-authored-by: Andy Grove <[email protected]>

* Update NOTICE.txt

Co-authored-by: Andy Grove <[email protected]>

---------

Co-authored-by: Andy Grove <[email protected]>

Author: comphead

.github/actions/setup-builder/action.yaml

@@ -34,6 +34,7 @@ runs:
       run: |
         apt-get update
         apt-get install -y protobuf-compiler
+        apt-get install -y clang
 
     - name: Install JDK ${{inputs.jdk-version}}
       uses: actions/setup-java@v4

NOTICE.txt

@@ -8,3 +8,6 @@ This product includes software developed at
 Apache Gluten (https://github.com/apache/incubator-gluten/)
 Specifically:
 - Optimizer rule to replace SortMergeJoin with ShuffleHashJoin
+
+This product includes software developed at
+DataFusion HDFS ObjectStore Contrib Package(https://github.com/datafusion-contrib/datafusion-objectstore-hdfs)

native/Cargo.lock

@@ -16,7 +16,8 @@
 # under the License.
 
 [workspace]
-members = ["core", "spark-expr", "proto"]
+default-members = ["core", "spark-expr", "proto"]
+members = ["core", "spark-expr", "proto", "hdfs"]
 resolver = "2"
 
 [workspace.package]
@@ -38,6 +39,8 @@ arrow-array = { version = "54.1.0" }
 arrow-buffer = { version = "54.1.0" }
 arrow-data = { version = "54.1.0" }
 arrow-schema = { version = "54.1.0" }
+async-trait = { version = "0.1" }
+bytes = { version = "1.10.0" }
 parquet = { version = "54.1.0", default-features = false, features = ["experimental"] }
 datafusion = { version = "45.0.0", default-features = false, features = ["unicode_expressions", "crypto_expressions"] }
 datafusion-common = { version = "45.0.0", default-features = false }

native/Cargo.toml

@@ -44,7 +44,7 @@ parquet = { workspace = true, default-features = false, features = ["experimenta
 futures = { workspace = true }
 mimalloc = { version = "*", default-features = false, optional = true }
 tokio = { version = "1", features = ["rt-multi-thread"] }
-async-trait = "0.1"
+async-trait = { workspace = true }
 log = "0.4"
 log4rs = "1.2.0"
 thiserror = { workspace = true }
@@ -58,9 +58,9 @@ lz4_flex = { version = "0.11.3", default-features = false }
 zstd = "0.11"
 rand = { workspace = true}
 num = { workspace = true }
-bytes = "1.5.0"
+bytes = { workspace = true }
 tempfile = "3.8.0"
-itertools = "0.11.0"
+itertools = "0.14.0"
 paste = "1.0.14"
 datafusion-common = { workspace = true, features= ["object_store"] }
 datafusion = {  workspace = true }
@@ -78,6 +78,7 @@ object_store = { workspace = true }
 url = { workspace = true }
 chrono = { workspace = true }
 parking_lot = "0.12.3"
+datafusion-comet-objectstore-hdfs = { path = "../hdfs", optional = true}
 
 [dev-dependencies]
 pprof = { version = "0.14.0", features = ["flamegraph"] }
@@ -89,6 +90,11 @@ hex = "0.4.3"
 
 [features]
 default = []
+hdfs=["datafusion-comet-objectstore-hdfs"]
+
+# exclude optional packages from cargo machete verifications
+[package.metadata.cargo-machete]
+ignored = ["datafusion-comet-objectstore-hdfs"]
 
 [lib]
 name = "comet"

native/core/Cargo.toml

@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This is an optional HDFS crate
+# To build it from root is required to provide a valid JAVA_HOME
+# and enable `hdfs` feature
+# Example: JAVA_HOME="/opt/homebrew/opt/openjdk@11" cargo build --features=hdfs
+
+[package]
+name = "datafusion-comet-objectstore-hdfs"
+description = "Comet HDFS integration"
+version = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+authors = { workspace = true }
+readme = { workspace = true }
+license = { workspace = true }
+edition = { workspace = true }
+
+[features]
+default = ["hdfs", "try_spawn_blocking"]
+hdfs = ["fs-hdfs"]
+hdfs3 = ["fs-hdfs3"]
+# Used for trying to spawn a blocking thread for implementing each object store interface when running in a tokio runtime
+try_spawn_blocking = []
+
+[dependencies]
+async-trait = { workspace = true }
+bytes = { workspace = true }
+chrono = { workspace = true }
+fs-hdfs = { version = "^0.1.12", optional = true }
+fs-hdfs3 = { version = "^0.1.12", optional = true }
+futures = { workspace = true }
+object_store = { workspace = true }
+tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot"] }
+
+[package.metadata.cargo-machete]
+ignored = ["fs-hdfs", "fs-hdfs3"]