Skip to content

Commit e47bba5

Browse files
rtylerrtyler
committed
chore: reduce feature/dependency footprint for subcrates
The object_store crate does not require its cloud feature in order to use RetryConfig, so most of the subcrates can shed a cloud and datafusion feature. Cleaning this up allows for avoiding the ObjectStoreFactory trait's ambiguous implementation which can cause problem if a subcrate is implemented using the "non-cloud" arm but then is included in a dependency tree where the "cloud" feature is enabled by another dependency. This was sort of only theoretically possible but did manifest during `cargo publish` operations. Additionally the removal of a datafusion feature when it iis not necessary results in ~100 fewer crates at compile and link time for those (hi!) working within subcrates Signed-off-by: R. Tyler Croy <[email protected]>
1 parent b30c2ed commit e47bba5

File tree

12 files changed

+18
-55
lines changed

12 files changed

+18
-55
lines changed

‎crates/aws/Cargo.toml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "deltalake-aws"
3-
version = "0.9.0"
3+
version = "0.9.1"
44
authors.workspace = true
55
keywords.workspace = true
66
readme.workspace = true
@@ -13,7 +13,7 @@ rust-version.workspace = true
1313

1414
[dependencies]
1515
# path dependencies
16-
deltalake-core = { version = "0.26.0", path = "../core", features = ["cloud"] }
16+
deltalake-core = { version = "0.26.0", path = "../core" }
1717

1818
# workspace dependencies
1919
async-trait = { workspace = true }
@@ -51,7 +51,7 @@ hyper-tls = { version = "0.5", optional = true }
5151
maplit = "1"
5252

5353
[dev-dependencies]
54-
deltalake-core = { path = "../core", features = ["datafusion"] }
54+
deltalake-core = { path = "../core" }
5555
chrono = { workspace = true }
5656
serial_test = "3"
5757
deltalake-test = { path = "../test" }

‎crates/azure/Cargo.toml‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "deltalake-azure"
3-
version = "0.9.0"
3+
version = "0.9.1"
44
authors.workspace = true
55
keywords.workspace = true
66
readme.workspace = true
@@ -12,9 +12,7 @@ repository.workspace = true
1212
rust-version.workspace = true
1313

1414
[dependencies]
15-
deltalake-core = { version = "0.26.0", path = "../core", features = [
16-
"datafusion", "cloud",
17-
]}
15+
deltalake-core = { version = "0.26.0", path = "../core" }
1816

1917
# workspace depenndecies
2018
async-trait = { workspace = true }

‎crates/core/src/logstore/config.rs‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
//! defines how to update internal fields based on key-value pairs.
99
use std::collections::HashMap;
1010

11-
#[cfg(feature = "cloud")]
1211
use ::object_store::RetryConfig;
1312
use object_store::{path::Path, prefix::PrefixStore, ObjectStore, ObjectStoreScheme};
1413
use tokio::runtime::Handle;
@@ -95,7 +94,6 @@ pub struct StorageConfig {
9594
/// Configuration to set up a dedicated IO runtime to execute IO related operations.
9695
pub runtime: Option<RuntimeConfig>,
9796

98-
#[cfg(feature = "cloud")]
9997
pub retry: ::object_store::RetryConfig,
10098

10199
/// Limit configuration.
@@ -178,7 +176,6 @@ where
178176

179177
let remainder = result.unparsed;
180178

181-
#[cfg(feature = "cloud")]
182179
let remainder = {
183180
let result = ParseResult::<RetryConfig>::from_iter(remainder);
184181
config.retry = result.config;
@@ -229,7 +226,6 @@ impl StorageConfig {
229226
props.limit = (!result.is_default).then_some(result.config);
230227
let remainder = result.unparsed;
231228

232-
#[cfg(feature = "cloud")]
233229
let remainder = {
234230
let (retry, remainder): (RetryConfig, _) = try_parse_impl(remainder)?;
235231
props.retry = retry;
@@ -299,7 +295,7 @@ pub fn str_is_truthy(val: &str) -> bool {
299295
| val.eq_ignore_ascii_case("y")
300296
}
301297

302-
#[cfg(all(test, feature = "cloud"))]
298+
#[cfg(test)]
303299
mod tests {
304300
use maplit::hashmap;
305301
use object_store::RetryConfig;

‎crates/core/src/logstore/factories.rs‎

Lines changed: 1 addition & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ use std::{
55

66
use dashmap::DashMap;
77
use object_store::path::Path;
8-
#[cfg(feature = "cloud")]
98
use object_store::RetryConfig;
109
use url::Url;
1110

@@ -24,37 +23,19 @@ pub trait ObjectStoreFactory: Send + Sync {
2423
/// The path segment is returned as second element of the tuple. It must point at the path
2524
/// corresponding to the path segment of the URL.
2625
///
27-
/// The store should __NOT__ apply the decorations via the passed `StorageConfig`
28-
#[cfg(feature = "cloud")]
26+
/// The store should __NOT__ apply the decorations via the passed `options`
2927
fn parse_url_opts(
3028
&self,
3129
url: &Url,
3230
options: &HashMap<String, String>,
3331
retry: &RetryConfig,
3432
) -> DeltaResult<(ObjectStoreRef, Path)>;
35-
36-
#[cfg(not(feature = "cloud"))]
37-
fn parse_url_opts(
38-
&self,
39-
url: &Url,
40-
options: &HashMap<String, String>,
41-
) -> DeltaResult<(ObjectStoreRef, Path)>;
4233
}
4334

4435
#[derive(Clone, Debug, Default)]
4536
pub(crate) struct DefaultObjectStoreFactory {}
4637

4738
impl ObjectStoreFactory for DefaultObjectStoreFactory {
48-
#[cfg(not(feature = "cloud"))]
49-
fn parse_url_opts(
50-
&self,
51-
url: &Url,
52-
options: &HashMap<String, String>,
53-
) -> DeltaResult<(ObjectStoreRef, Path)> {
54-
default_parse_url_opts(url, options)
55-
}
56-
57-
#[cfg(feature = "cloud")]
5839
fn parse_url_opts(
5940
&self,
6041
url: &Url,
@@ -103,11 +84,8 @@ where
10384
let scheme = Url::parse(&format!("{}://", url.scheme())).unwrap();
10485
let storage_config = StorageConfig::parse_options(options)?;
10586
if let Some(factory) = object_store_factories().get(&scheme) {
106-
#[cfg(feature = "cloud")]
10787
let (store, _prefix) =
10888
factory.parse_url_opts(url, &storage_config.raw, &storage_config.retry)?;
109-
#[cfg(not(feature = "cloud"))]
110-
let (store, _prefix) = factory.parse_url_opts(url, &storage_config.raw)?;
11189
let store = storage_config.decorate_store(store, url, None)?;
11290
Ok(Arc::new(store))
11391
} else {

‎crates/core/src/logstore/mod.rs‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -176,16 +176,11 @@ where
176176

177177
if let Some(entry) = object_store_factories().get(&scheme) {
178178
debug!("Found a storage provider for {scheme} ({location})");
179-
#[cfg(feature = "cloud")]
180179
let (store, _prefix) = entry.value().parse_url_opts(
181180
&location,
182181
&storage_options.raw,
183182
&storage_options.retry,
184183
)?;
185-
#[cfg(not(feature = "cloud"))]
186-
let (store, _prefix) = entry
187-
.value()
188-
.parse_url_opts(&location, &storage_options.raw)?;
189184
return logstore_with(store, location, storage_options.raw, io_runtime);
190185
}
191186

‎crates/core/src/logstore/storage/retry_ext.rs‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ use ::object_store::path::Path;
44
use ::object_store::{Error, ObjectStore, PutPayload, PutResult, Result};
55
use tracing::log::*;
66

7-
#[cfg(feature = "cloud")]
87
use crate::logstore::config;
98

109
impl<T: ObjectStore + ?Sized> ObjectStoreRetryExt for T {}
@@ -78,7 +77,6 @@ pub trait ObjectStoreRetryExt: ObjectStore {
7877
}
7978
}
8079

81-
#[cfg(feature = "cloud")]
8280
impl config::TryUpdateKey for object_store::RetryConfig {
8381
fn try_update_key(&mut self, key: &str, v: &str) -> crate::DeltaResult<Option<()>> {
8482
match key {

‎crates/deltalake/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "deltalake"
3-
version = "0.26.0"
3+
version = "0.26.1"
44
authors.workspace = true
55
keywords.workspace = true
66
readme.workspace = true

‎crates/derive/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[package]
22
name = "deltalake-derive"
33
version = "0.26.0"
4-
description = "Dervice macoros for use in delta ecosystem crates"
4+
description = "Dervice macros for use in delta ecosystem crates"
55
authors.workspace = true
66
rust-version.workspace = true
77
keywords.workspace = true

‎crates/gcp/Cargo.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "deltalake-gcp"
3-
version = "0.10.0"
3+
version = "0.10.1"
44
authors.workspace = true
55
keywords.workspace = true
66
readme.workspace = true
@@ -12,7 +12,7 @@ repository.workspace = true
1212
rust-version.workspace = true
1313

1414
[dependencies]
15-
deltalake-core = { version = "0.26.0", path = "../core", features = ["cloud"] }
15+
deltalake-core = { version = "0.26.0", path = "../core" }
1616

1717
# workspace depenndecies
1818
async-trait = { workspace = true }

‎crates/hdfs/Cargo.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "deltalake-hdfs"
3-
version = "0.10.0"
3+
version = "0.10.1"
44
authors.workspace = true
55
keywords.workspace = true
66
readme.workspace = true
@@ -12,7 +12,7 @@ repository.workspace = true
1212
rust-version.workspace = true
1313

1414
[dependencies]
15-
deltalake-core = { version = "0.26.0", path = "../core", features = ["cloud"] }
15+
deltalake-core = { version = "0.26.0", path = "../core"}
1616
hdfs-native-object-store = "0.14"
1717

1818
# workspace dependecies

0 commit comments

Comments
 (0)