feat: remove redundant Handle from parse_url_opts

It looks like `StorageConfig` has everything that implementors of the
trait should need, so just using that to pass information through!

Signed-off-by: R. Tyler Croy <[email protected]>

Author: rtyler

crates/aws/src/storage.rs

@@ -21,7 +21,6 @@ use futures::stream::BoxStream;
 use futures::Future;
 use object_store::aws::AmazonS3;
 use object_store::client::SpawnedReqwestConnector;
-use tokio::runtime::Handle;
 use tracing::log::*;
 use url::Url;
 
@@ -41,7 +40,6 @@ impl ObjectStoreFactory for S3ObjectStoreFactory {
         &self,
         url: &Url,
         config: &StorageConfig,
-        handle: Option<Handle>,
     ) -> DeltaResult<(ObjectStoreRef, Path)> {
         let options = self.with_env_s3(&config.raw);
 
@@ -50,8 +48,9 @@ impl ObjectStoreFactory for S3ObjectStoreFactory {
             .with_url(url.to_string())
             .with_retry(config.retry.clone());
 
-        if let Some(handle) = handle {
-            builder = builder.with_http_connector(SpawnedReqwestConnector::new(handle));
+        if let Some(runtime) = &config.runtime {
+            builder =
+                builder.with_http_connector(SpawnedReqwestConnector::new(runtime.get_handle()));
         }
 
         for (key, value) in options.iter() {

crates/azure/src/lib.rs

@@ -10,7 +10,6 @@ use deltalake_core::{DeltaResult, DeltaTableError, Path};
 use object_store::azure::{AzureConfigKey, MicrosoftAzureBuilder};
 use object_store::client::SpawnedReqwestConnector;
 use object_store::ObjectStoreScheme;
-use tokio::runtime::Handle;
 use url::Url;
 
 mod config;
@@ -41,17 +40,17 @@ impl ObjectStoreFactory for AzureFactory {
         &self,
         url: &Url,
         config: &StorageConfig,
-        handle: Option<Handle>,
     ) -> DeltaResult<(ObjectStoreRef, Path)> {
         let mut builder = MicrosoftAzureBuilder::new()
             .with_url(url.to_string())
             .with_retry(config.retry.clone());
-        let config = config::AzureConfigHelper::try_new(config.raw.as_azure_options())?.build()?;
-
-        if let Some(handle) = handle {
-            builder = builder.with_http_connector(SpawnedReqwestConnector::new(handle));
+        if let Some(runtime) = &config.runtime {
+            builder =
+                builder.with_http_connector(SpawnedReqwestConnector::new(runtime.get_handle()));
         }
 
+        let config = config::AzureConfigHelper::try_new(config.raw.as_azure_options())?.build()?;
+
         for (key, value) in config.iter() {
             builder = builder.with_config(*key, value.clone());
         }

crates/catalog-unity/src/lib.rs

@@ -17,7 +17,6 @@ use std::collections::HashMap;
 use std::future::Future;
 use std::str::FromStr;
 use std::sync::Arc;
-use tokio::runtime::Handle;
 
 use crate::credential::{
     AzureCliCredential, ClientSecretOAuthProvider, CredentialProvider, WorkspaceOAuthProvider,
@@ -35,7 +34,7 @@ use deltalake_core::{
 
 use crate::client::retry::*;
 use deltalake_core::logstore::{
-    config::str_is_truthy, object_store_factories, IORuntime, ObjectStoreFactory, ObjectStoreRef,
+    config::str_is_truthy, object_store_factories, ObjectStoreFactory, ObjectStoreRef,
 };
 pub mod client;
 pub mod credential;
@@ -840,7 +839,6 @@ impl ObjectStoreFactory for UnityCatalogFactory {
         &self,
         table_uri: &Url,
         config: &StorageConfig,
-        handle: Option<Handle>,
     ) -> DeltaResult<(ObjectStoreRef, Path)> {
         let (table_path, temp_creds) = UnityCatalogBuilder::execute_uc_future(
             UnityCatalogBuilder::get_uc_location_and_token(table_uri.as_str()),

crates/core/src/logstore/config.rs

@@ -345,6 +345,7 @@ mod tests {
     }
 
     // Test StorageConfig parsing
+    #[cfg(feature = "cloud")]
     #[test]
     fn test_storage_config_parsing() {
         let options = hashmap! {

crates/core/src/logstore/factories.rs

@@ -28,7 +28,6 @@ pub trait ObjectStoreFactory: Send + Sync {
         &self,
         url: &Url,
         config: &StorageConfig,
-        handle: Option<Handle>,
     ) -> DeltaResult<(ObjectStoreRef, Path)>;
 }
 
@@ -40,12 +39,12 @@ impl ObjectStoreFactory for DefaultObjectStoreFactory {
         &self,
         url: &Url,
         config: &StorageConfig,
-        handle: Option<Handle>,
     ) -> DeltaResult<(ObjectStoreRef, Path)> {
         let (mut store, path) = default_parse_url_opts(url, &config.raw)?;
 
-        if let Some(handle) = handle {
-            store = Arc::new(DeltaIOStorageBackend::new(store, handle)) as Arc<DynObjectStore>;
+        if let Some(runtime) = &config.runtime {
+            store =
+                Arc::new(DeltaIOStorageBackend::new(store, runtime.clone())) as Arc<DynObjectStore>;
         }
         Ok((store, path))
     }
@@ -89,7 +88,7 @@ where
     let scheme = Url::parse(&format!("{}://", url.scheme())).unwrap();
     let storage_config = StorageConfig::parse_options(options)?;
     if let Some(factory) = object_store_factories().get(&scheme) {
-        let (store, _prefix) = factory.parse_url_opts(url, &storage_config, None)?;
+        let (store, _prefix) = factory.parse_url_opts(url, &storage_config)?;
         let store = storage_config.decorate_store(store, url)?;
         Ok(Arc::new(store))
     } else {

crates/core/src/logstore/mod.rs

@@ -180,11 +180,7 @@ pub fn logstore_for(location: Url, storage_config: StorageConfig) -> DeltaResult
 
     if let Some(entry) = object_store_factories().get(&scheme) {
         debug!("Found a storage provider for {scheme} ({location})");
-        let (root_store, _prefix) = entry.value().parse_url_opts(
-            &location,
-            &storage_config,
-            storage_config.runtime.clone().map(|rt| rt.get_handle()),
-        )?;
+        let (root_store, _prefix) = entry.value().parse_url_opts(&location, &storage_config)?;
         return logstore_with(root_store, location, storage_config);
     }
 

crates/core/src/logstore/storage/runtime.rs

@@ -108,18 +108,15 @@ impl IORuntime {
 #[derive(Clone)]
 pub struct DeltaIOStorageBackend<T: ObjectStore + Clone> {
     pub inner: T,
-    pub rt_handle: Handle,
+    rt: IORuntime,
 }
 
 impl<T> DeltaIOStorageBackend<T>
 where
     T: ObjectStore + Clone,
 {
-    pub fn new(store: T, handle: Handle) -> Self {
-        Self {
-            inner: store,
-            rt_handle: handle,
-        }
+    pub fn new(store: T, rt: IORuntime) -> Self {
+        Self { inner: store, rt }
     }
 }
 
@@ -136,7 +133,10 @@ impl<T: ObjectStore + Clone> DeltaIOStorageBackend<T> {
         O: Send + 'static,
     {
         let store = store.clone();
-        let fut = self.rt_handle.spawn(async move { f(&store, &path).await });
+        let fut = self
+            .rt
+            .get_handle()
+            .spawn(async move { f(&store, &path).await });
         fut.unwrap_or_else(|e| match e.try_into_panic() {
             Ok(p) => std::panic::resume_unwind(p),
             Err(e) => Err(ObjectStoreError::JoinError { source: e }),
@@ -160,7 +160,8 @@ impl<T: ObjectStore + Clone> DeltaIOStorageBackend<T> {
     {
         let store = store.clone();
         let fut = self
-            .rt_handle
+            .rt
+            .get_handle()
             .spawn(async move { f(&store, &from, &to).await });
         fut.unwrap_or_else(|e| match e.try_into_panic() {
             Ok(p) => std::panic::resume_unwind(p),
@@ -316,3 +317,13 @@ impl<T: ObjectStore + Clone> ObjectStore for DeltaIOStorageBackend<T> {
         .await
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_ioruntime_default() {
+        let _ = IORuntime::default();
+    }
+}

crates/gcp/src/lib.rs

@@ -10,7 +10,6 @@ use deltalake_core::logstore::{
 };
 use deltalake_core::{DeltaResult, DeltaTableError, Path};
 use object_store::client::SpawnedReqwestConnector;
-use tokio::runtime::Handle;
 use url::Url;
 
 mod config;
@@ -42,11 +41,14 @@ impl ObjectStoreFactory for GcpFactory {
         &self,
         url: &Url,
         config: &StorageConfig,
-        handle: Option<Handle>,
     ) -> DeltaResult<(ObjectStoreRef, Path)> {
         let mut builder = GoogleCloudStorageBuilder::new().with_url(url.to_string());
         builder = builder.with_retry(config.retry.clone());
 
+        if let Some(runtime) = &config.runtime {
+            builder =
+                builder.with_http_connector(SpawnedReqwestConnector::new(runtime.get_handle()));
+        }
         let config = config::GcpConfigHelper::try_new(config.raw.as_gcp_options())?.build()?;
 
         let (_, path) =
@@ -55,10 +57,6 @@ impl ObjectStoreFactory for GcpFactory {
             })?;
         let prefix = Path::parse(path)?;
 
-        if let Some(handle) = handle {
-            builder = builder.with_http_connector(SpawnedReqwestConnector::new(handle));
-        }
-
         for (key, value) in config.iter() {
             builder = builder.with_config(*key, value.clone());
         }

crates/hdfs/src/lib.rs

@@ -7,7 +7,6 @@ use deltalake_core::logstore::{
 use deltalake_core::logstore::{object_store_factories, ObjectStoreFactory, ObjectStoreRef};
 use deltalake_core::{DeltaResult, Path};
 use hdfs_native_object_store::HdfsObjectStore;
-use tokio::runtime::Handle;
 use url::Url;
 
 #[derive(Clone, Default, Debug)]
@@ -18,16 +17,15 @@ impl ObjectStoreFactory for HdfsFactory {
         &self,
         url: &Url,
         config: &StorageConfig,
-        handle: Option<Handle>,
     ) -> DeltaResult<(ObjectStoreRef, Path)> {
         let mut store: ObjectStoreRef = Arc::new(HdfsObjectStore::with_config(
             url.as_str(),
             config.raw.clone(),
         )?);
 
         // HDFS doesn't have the spawnService, so we still wrap it in the old io storage backend (not as optimal though)
-        if let Some(handle) = handle {
-            store = Arc::new(DeltaIOStorageBackend::new(store, handle));
+        if let Some(runtime) = &config.runtime {
+            store = Arc::new(DeltaIOStorageBackend::new(store, runtime.clone()));
         };
         let prefix = Path::parse(url.path())?;
         Ok((store, prefix))
@@ -71,7 +69,6 @@ mod tests {
         let _ = factory.parse_url_opts(
             &Url::parse("hdfs://localhost:9000").expect("Failed to parse hdfs://"),
             &StorageConfig::default(),
-            None,
         )?;
         Ok(())
     }

crates/lakefs/src/logstore.rs

@@ -92,22 +92,14 @@ impl LakeFSLogStore {
 
     /// Build a new object store for an URL using the existing storage options. After
     /// branch creation a new object store needs to be created for the branch uri
-    fn build_new_store(
-        &self,
-        url: &Url,
-        io_runtime: Option<IORuntime>,
-    ) -> DeltaResult<ObjectStoreRef> {
+    fn build_new_store(&self, url: &Url) -> DeltaResult<ObjectStoreRef> {
         // turn location into scheme
         let scheme = Url::parse(&format!("{}://", url.scheme()))
             .map_err(|_| DeltaTableError::InvalidTableLocation(url.clone().into()))?;
 
         if let Some(entry) = self.config().object_store_factory().get(&scheme) {
             debug!("Creating new storage with storage provider for {scheme} ({url})");
-            let (store, _prefix) = entry.value().parse_url_opts(
-                url,
-                &self.config().options,
-                io_runtime.map(|rt| rt.get_handle()),
-            )?;
+            let (store, _prefix) = entry.value().parse_url_opts(url, &self.config().options)?;
             return Ok(store);
         }
         Err(DeltaTableError::InvalidTableLocation(url.to_string()))
@@ -151,8 +143,7 @@ impl LakeFSLogStore {
             .await?;
 
         // Build new object store store using the new lakefs url
-        let txn_root_store =
-            self.build_new_store(&lakefs_url, self.config().options.runtime.clone())?;
+        let txn_root_store = self.build_new_store(&lakefs_url)?;
         let txn_store = Arc::new(
             self.config
                 .decorate_store(txn_root_store.clone(), Some(&lakefs_url))?,