feat: make s3 support optional (#1950)

Author: baszalmstra

Cargo.lock

@@ -51,7 +51,7 @@ default-run = "rattler-build"
 rust-version = "1.86.0"
 
 [features]
-default = ['rustls-tls', 'recipe-generation']
+default = ['rustls-tls', 'recipe-generation', 's3']
 native-tls = [
   'reqwest/native-tls',
   'rattler/native-tls',
@@ -75,6 +75,10 @@ tui = [
   'throbber-widgets-tui',
   'tui-input',
 ]
+s3 = [
+  'rattler_networking/s3',
+  'rattler_upload/s3',
+]
 recipe-generation = ["rattler_build_recipe_generator"]
 # This feature needs to add a dependency on
 # clap-markdown = { git = "https://github.com/ruben-arts/clap-markdown", branch = "main" }
@@ -171,9 +175,6 @@ reflink-copy = "0.1.26"
 rayon = "1.11.0"
 regex = { workspace = true }
 async-recursion = { workspace = true }
-opendal = { version = "0.54.0", default-features = false, features = [
-  "services-s3",
-] }
 
 # Rattler crates
 rattler_config = { version = "0.2.11" }
@@ -186,12 +187,9 @@ rattler_conda_types = { workspace = true, features = ["rayon"] }
 rattler_digest = { workspace = true }
 rattler_index = { version = "0.25.5", default-features = false }
 rattler_networking = { version = "0.25.16", default-features = false, features = [
-  "s3",
   "rattler_config",
 ] }
-rattler_upload = { version = "0.3.4", default-features = false, features = [
-  "s3",
-] }
+rattler_upload = { version = "0.3.4", default-features = false }
 rattler_redaction = { version = "0.1.12" }
 rattler_repodata_gateway = { version = "0.24.7", default-features = false, features = [
   "gateway",

Cargo.toml

@@ -135,6 +135,7 @@ pub fn get_tool_config(
 ) -> miette::Result<Configuration> {
     let client = tool_configuration::reqwest_client_from_auth_storage(
         build_data.common.auth_file.clone(),
+        #[cfg(feature = "s3")]
         build_data.common.s3_config.clone(),
         build_data.common.mirror_config.clone(),
         build_data.common.allow_insecure_host.clone(),
@@ -694,6 +695,7 @@ pub async fn run_test(
         .with_reqwest_client(
             tool_configuration::reqwest_client_from_auth_storage(
                 test_data.common.auth_file,
+                #[cfg(feature = "s3")]
                 test_data.common.s3_config,
                 test_data.common.mirror_config,
                 test_data.common.allow_insecure_host.clone(),
@@ -752,6 +754,7 @@ pub async fn rebuild(
 ) -> miette::Result<()> {
     let reqwest_client = tool_configuration::reqwest_client_from_auth_storage(
         rebuild_data.common.auth_file,
+        #[cfg(feature = "s3")]
         rebuild_data.common.s3_config.clone(),
         rebuild_data.common.mirror_config.clone(),
         rebuild_data.common.allow_insecure_host.clone(),

py-rattler-build/pixi.lock

@@ -11,7 +11,9 @@ use rattler_conda_types::{
     NamedChannelOrUrl, Platform, compression_level::CompressionLevel, package::ArchiveType,
 };
 use rattler_config::config::build::PackageFormatAndCompression;
-use rattler_networking::{mirror_middleware, s3_middleware};
+use rattler_networking::mirror_middleware;
+#[cfg(feature = "s3")]
+use rattler_networking::s3_middleware;
 use rattler_solve::ChannelPriority;
 use rattler_upload::upload::opt::{Config, UploadOpts};
 use serde_json::{Value, json};
@@ -231,6 +233,7 @@ pub struct CommonData {
     pub experimental: bool,
     pub auth_file: Option<PathBuf>,
     pub channel_priority: ChannelPriority,
+    #[cfg(feature = "s3")]
     pub s3_config: HashMap<String, s3_middleware::S3Config>,
     pub mirror_config: HashMap<Url, Vec<mirror_middleware::Mirror>>,
     pub allow_insecure_host: Option<Vec<String>>,
@@ -285,11 +288,14 @@ impl CommonData {
             mirror_config.insert(ensure_trailing_slash(key), mirrors);
         }
 
+        #[cfg(feature = "s3")]
         let s3_config = rattler_networking::s3_middleware::compute_s3_config(&config.s3_options.0);
+
         Self {
             output_dir: output_dir.unwrap_or_else(|| PathBuf::from("./output")),
             experimental,
             auth_file,
+            #[cfg(feature = "s3")]
             s3_config,
             mirror_config,
             channel_priority: channel_priority.unwrap_or(ChannelPriority::Strict),

src/lib.rs

@@ -6,10 +6,12 @@ use std::{collections::HashMap, path::PathBuf, sync::Arc};
 use clap::ValueEnum;
 use rattler::package_cache::PackageCache;
 use rattler_conda_types::{ChannelConfig, Platform};
+#[cfg(feature = "s3")]
+use rattler_networking::s3_middleware;
 use rattler_networking::{
     AuthenticationMiddleware, AuthenticationStorage,
     authentication_storage::{self, AuthenticationStorageError},
-    mirror_middleware, s3_middleware,
+    mirror_middleware,
 };
 use rattler_repodata_gateway::Gateway;
 use rattler_solve::ChannelPriority;
@@ -85,12 +87,13 @@ impl BaseClient {
     pub fn new(
         auth_file: Option<PathBuf>,
         allow_insecure_host: Option<Vec<String>>,
-        s3_middleware_config: HashMap<String, s3_middleware::S3Config>,
+        #[cfg(feature = "s3")] s3_middleware_config: HashMap<String, s3_middleware::S3Config>,
         mirror_middleware_config: HashMap<Url, Vec<mirror_middleware::Mirror>>,
     ) -> Result<Self, AuthenticationStorageError> {
         let auth_storage = get_auth_store(auth_file)?;
         let timeout = 5 * 60;
 
+        #[cfg(feature = "s3")]
         let s3_middleware =
             s3_middleware::S3Middleware::new(s3_middleware_config, auth_storage.clone());
         let mirror_middleware =
@@ -104,20 +107,24 @@ impl BaseClient {
                 .read_timeout(std::time::Duration::from_secs(timeout))
         };
 
-        let client = reqwest_middleware::ClientBuilder::new(
+        let client_builder = reqwest_middleware::ClientBuilder::new(
             common_settings(reqwest::Client::builder())
                 .build()
                 .expect("failed to create client"),
         )
-        .with(mirror_middleware)
-        .with(s3_middleware)
-        .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage(
-            auth_storage.clone(),
-        )))
-        .with(RetryTransientMiddleware::new_with_policy(
-            ExponentialBackoff::builder().build_with_max_retries(3),
-        ))
-        .build();
+        .with(mirror_middleware);
+
+        #[cfg(feature = "s3")]
+        let client_builder = client_builder.with(s3_middleware);
+
+        let client = client_builder
+            .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage(
+                auth_storage.clone(),
+            )))
+            .with(RetryTransientMiddleware::new_with_policy(
+                ExponentialBackoff::builder().build_with_max_retries(3),
+            ))
+            .build();
 
         let dangerous_client = reqwest_middleware::ClientBuilder::new(
             common_settings(reqwest::Client::builder())
@@ -259,13 +266,14 @@ pub fn get_auth_store(
 /// * `allow_insecure_host` - Optional list of hosts for which to disable SSL certificate verification
 pub fn reqwest_client_from_auth_storage(
     auth_file: Option<PathBuf>,
-    s3_middleware_config: HashMap<String, s3_middleware::S3Config>,
+    #[cfg(feature = "s3")] s3_middleware_config: HashMap<String, s3_middleware::S3Config>,
     mirror_middleware_config: HashMap<Url, Vec<mirror_middleware::Mirror>>,
     allow_insecure_host: Option<Vec<String>>,
 ) -> Result<BaseClient, AuthenticationStorageError> {
     BaseClient::new(
         auth_file,
         allow_insecure_host,
+        #[cfg(feature = "s3")]
         s3_middleware_config,
         mirror_middleware_config,
     )