refactor: Bump OpenDAL to 0.53 (#1182)

## Which issue does this PR close?

None

## What changes are included in this PR?

Bump OpenDAL to 0.53, remove not needed hacks.

## Are these changes tested?

None.

---------

Signed-off-by: Xuanwo <[email protected]>

Author: Xuanwo

Cargo.lock

@@ -74,7 +74,7 @@ mockito = "1"
 murmur3 = "0.5.2"
 num-bigint = "0.4.6"
 once_cell = "1.20"
-opendal = "0.51.2"
+opendal = "0.53.0"
 ordered-float = "4"
 parquet = "54.2.0"
 pilota = "0.11.2"

Cargo.toml

@@ -309,7 +309,8 @@ impl FileWrite for opendal::Writer {
     }
 
     async fn close(&mut self) -> crate::Result<()> {
-        Ok(opendal::Writer::close(self).await?)
+        let _ = opendal::Writer::close(self).await?;
+        Ok(())
     }
 }
 

crates/iceberg/src/io/file_io.rs

@@ -39,10 +39,6 @@ pub(crate) enum Storage {
         /// s3 storage could have `s3://` and `s3a://`.
         /// Storing the scheme string here to return the correct path.
         scheme_str: String,
-        /// uses the same client for one FileIO Storage.
-        ///
-        /// TODO: allow users to configure this client.
-        client: reqwest::Client,
         config: Arc<S3Config>,
     },
     #[cfg(feature = "storage-gcs")]
@@ -63,7 +59,6 @@ impl Storage {
             #[cfg(feature = "storage-s3")]
             Scheme::S3 => Ok(Self::S3 {
                 scheme_str,
-                client: reqwest::Client::new(),
                 config: super::s3_config_parse(props)?.into(),
             }),
             #[cfg(feature = "storage-gcs")]
@@ -115,12 +110,8 @@ impl Storage {
                 }
             }
             #[cfg(feature = "storage-s3")]
-            Storage::S3 {
-                scheme_str,
-                client,
-                config,
-            } => {
-                let op = super::s3_config_build(client, config, path)?;
+            Storage::S3 { scheme_str, config } => {
+                let op = super::s3_config_build(config, path)?;
                 let op_info = op.info();
 
                 // Check prefix of s3 path.

crates/iceberg/src/io/storage.rs

@@ -17,7 +17,6 @@
 
 use std::collections::HashMap;
 
-use opendal::raw::HttpClient;
 use opendal::services::S3Config;
 use opendal::{Configurator, Operator};
 use url::Url;
@@ -152,11 +151,7 @@ pub(crate) fn s3_config_parse(mut m: HashMap<String, String>) -> Result<S3Config
 }
 
 /// Build new opendal operator from give path.
-pub(crate) fn s3_config_build(
-    client: &reqwest::Client,
-    cfg: &S3Config,
-    path: &str,
-) -> Result<Operator> {
+pub(crate) fn s3_config_build(cfg: &S3Config, path: &str) -> Result<Operator> {
     let url = Url::parse(path)?;
     let bucket = url.host_str().ok_or_else(|| {
         Error::new(
@@ -169,9 +164,7 @@ pub(crate) fn s3_config_build(
         .clone()
         .into_builder()
         // Set bucket name.
-        .bucket(bucket)
-        // Set http client we want to use.
-        .http_client(HttpClient::with(client.clone()));
+        .bucket(bucket);
 
     Ok(Operator::new(builder)?.finish())
 }