feat: read channel_sources from variant file (#1597)

Co-authored-by: Michał Górny <[email protected]>

Author: pavelzw

docs/variants.md

@@ -186,6 +186,14 @@ requirements:
   - libcurl >=8,<9
 ```
 
+### Channel sources
+
+You can specify the channels when building by adjusting `channel_sources` in your variant file:
+
+```yaml
+channel_sources: conda-forge/label/rust_dev,conda-forge
+```
+
 ## Prioritizing variants
 
 You might produce multiple variants for a package, but want to define a _priority_ for a given variant.

py-rattler-build/src/lib.rs

@@ -55,10 +55,13 @@ fn build_recipes_py(
         .map(|c| ChannelPriorityWrapper::from_str(&c).map(|c| c.value))
         .transpose()
         .map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
+    // todo: allow custom config here
+    let config = pixi_config::Config::default();
     let common = CommonData::new(
         output_dir,
         false,
         auth_file.map(|a| a.into()),
+        config,
         channel_priority,
         allow_insecure_host,
     );
@@ -146,13 +149,28 @@ fn test_package_py(
         .map(|c| ChannelPriorityWrapper::from_str(&c).map(|c| c.value))
         .transpose()
         .map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
+    // todo: allow custom config here
+    let config = pixi_config::Config::default();
     let common = CommonData::new(
         None,
         false,
         auth_file,
+        config,
         channel_priority,
         allow_insecure_host,
     );
+    let channel = match channel {
+        None => None,
+        Some(channel) => Some(
+            channel
+                .iter()
+                .map(|c| {
+                    NamedChannelOrUrl::from_str(c)
+                        .map_err(|e| PyRuntimeError::new_err(e.to_string()))
+                })
+                .collect::<PyResult<_>>()?,
+        ),
+    };
     let test_data = TestData::new(package_file, channel, compression_threads, common);
 
     let rt = tokio::runtime::Runtime::new().unwrap();

src/lib.rs

@@ -46,6 +46,7 @@ pub mod source_code;
 use std::{
     collections::{BTreeMap, HashMap},
     path::{Path, PathBuf},
+    str::FromStr,
     sync::{Arc, Mutex},
 };
 
@@ -65,7 +66,8 @@ use package_test::TestConfiguration;
 use petgraph::{algo::toposort, graph::DiGraph, visit::DfsPostOrder};
 use pixi_config::PackageFormatAndCompression;
 use rattler_conda_types::{
-    Channel, GenericVirtualPackage, MatchSpec, PackageName, Platform, package::ArchiveType,
+    GenericVirtualPackage, MatchSpec, NamedChannelOrUrl, PackageName, Platform,
+    package::ArchiveType,
 };
 use rattler_package_streaming::write::CompressionLevel;
 use rattler_solve::SolveStrategy;
@@ -310,10 +312,40 @@ pub async fn get_build_output(
             recipe.package().name().as_normalized().to_string()
         };
 
-        // Add the channels from the args and by default always conda-forge
-        let channels = build_data
-            .channels
-            .clone()
+        let variant_channels = if let Some(channel_sources) = discovered_output
+            .used_vars
+            .get(&NormalizedKey("channel_sources".to_string()))
+        {
+            Some(
+                channel_sources
+                    .to_string()
+                    .split(',')
+                    .map(str::trim)
+                    .map(|s| NamedChannelOrUrl::from_str(s).into_diagnostic())
+                    .collect::<miette::Result<Vec<_>>>()?,
+            )
+        } else {
+            None
+        };
+
+        // priorities
+        // 1. channel_sources from variant file
+        // 2. channels from args
+        // 3. channels from pixi_config
+        // 4. conda-forge as fallback
+        if variant_channels.is_some() && build_data.channels.is_some() {
+            return Err(miette::miette!(
+                "channel_sources and channels cannot both be set at the same time"
+            ));
+        }
+        let channels = variant_channels.unwrap_or_else(|| {
+            build_data
+                .channels
+                .clone()
+                .unwrap_or(vec![NamedChannelOrUrl::Name("conda-forge".to_string())])
+        });
+
+        let channels = channels
             .into_iter()
             .map(|c| c.into_base_url(&tool_config.channel_config))
             .collect::<Result<Vec<_>, _>>()
@@ -632,8 +664,10 @@ pub async fn run_test(
 
     let channels = test_data
         .channels
+        .unwrap_or(vec![NamedChannelOrUrl::Name("conda-forge".to_string())]);
+    let channels = channels
         .into_iter()
-        .map(|name| Channel::from_str(name, &tool_config.channel_config).map(|c| c.base_url))
+        .map(|c| c.into_base_url(&tool_config.channel_config))
         .collect::<Result<Vec<_>, _>>()
         .into_diagnostic()?;
 

src/opt.rs

@@ -237,7 +237,7 @@ impl CommonData {
         // mirror config
         // todo: this is a duplicate in pixi and pixi-pack: do it like in `compute_s3_config`
         let mut mirror_config = HashMap::new();
-        tracing::info!("Using mirrors: {:?}", config.mirror_map());
+        tracing::debug!("Using mirrors: {:?}", config.mirror_map());
 
         fn ensure_trailing_slash(url: &url::Url) -> url::Url {
             if url.path().ends_with('/') {
@@ -449,7 +449,7 @@ pub struct BuildData {
     pub build_platform: Platform,
     pub target_platform: Platform,
     pub host_platform: Platform,
-    pub channels: Vec<NamedChannelOrUrl>,
+    pub channels: Option<Vec<NamedChannelOrUrl>>,
     pub variant_config: Vec<PathBuf>,
     pub ignore_recipe_variants: bool,
     pub render_only: bool,
@@ -508,7 +508,7 @@ impl BuildData {
             host_platform: host_platform
                 .or(target_platform)
                 .unwrap_or(Platform::current()),
-            channels: channels.unwrap_or(vec![NamedChannelOrUrl::Name("conda-forge".to_string())]),
+            channels,
             variant_config: variant_config.unwrap_or_default(),
             ignore_recipe_variants,
             render_only,
@@ -603,7 +603,7 @@ fn parse_key_val(s: &str) -> Result<(String, Value), Box<dyn Error + Send + Sync
 pub struct TestOpts {
     /// Channels to use when testing
     #[arg(short = 'c', long = "channel")]
-    pub channels: Option<Vec<String>>,
+    pub channels: Option<Vec<NamedChannelOrUrl>>,
 
     /// The package file to test
     #[arg(short, long)]
@@ -621,7 +621,7 @@ pub struct TestOpts {
 #[derive(Debug, Clone)]
 #[allow(missing_docs)]
 pub struct TestData {
-    pub channels: Vec<String>,
+    pub channels: Option<Vec<NamedChannelOrUrl>>,
     pub package_file: PathBuf,
     pub compression_threads: Option<u32>,
     pub common: CommonData,
@@ -642,13 +642,13 @@ impl TestData {
     /// Create a new instance of `TestData`
     pub fn new(
         package_file: PathBuf,
-        channels: Option<Vec<String>>,
+        channels: Option<Vec<NamedChannelOrUrl>>,
         compression_threads: Option<u32>,
         common: CommonData,
     ) -> Self {
         Self {
             package_file,
-            channels: channels.unwrap_or(vec!["conda-forge".to_string()]),
+            channels,
             compression_threads,
             common,
         }
@@ -1202,7 +1202,7 @@ pub struct DebugData {
     /// Host platform for runtime dependencies
     pub host_platform: Platform,
     /// List of channels to search for dependencies
-    pub channels: Vec<NamedChannelOrUrl>,
+    pub channels: Option<Vec<NamedChannelOrUrl>>,
     /// Common configuration options
     pub common: CommonData,
     /// Name of the specific output to debug (if recipe has multiple outputs)
@@ -1221,9 +1221,7 @@ impl DebugData {
             host_platform: opts
                 .host_platform
                 .unwrap_or_else(|| opts.target_platform.unwrap_or(Platform::current())),
-            channels: opts.channels.unwrap_or(vec![
-                NamedChannelOrUrl::from_str("conda-forge").expect("conda-forge is parseable"),
-            ]),
+            channels: opts.channels,
             common: CommonData::from_opts_and_config(opts.common, config.unwrap_or_default()),
             output_name: opts.output_name,
         }

src/variant_render.rs

@@ -372,8 +372,9 @@ pub(crate) fn stage_1_render<S: SourceCode>(
                 additional_variables.insert("CONDA_BUILD_SYSROOT".into());
             }
 
-            // also always add `target_platform` and `channel_targets`
+            // also always add `target_platform`, `channel_sources` and `channel_targets`
             additional_variables.insert("target_platform".into());
+            additional_variables.insert("channel_sources".into());
             additional_variables.insert("channel_targets".into());
 
             // Environment variables can be overwritten by the variant configuration

test-data/recipes/channel_sources/conda_build_config.yaml

@@ -0,0 +1,2 @@
+channel_sources:
+  - conda-forge/label/rust_dev,conda-forge

test-data/recipes/channel_sources/recipe.yaml

@@ -0,0 +1,7 @@
+package:
+  name: channel-sources
+  version: 0.1.0
+
+requirements:
+  build:
+    - rust ==1.88.0.dev20250428

test/end-to-end/test_simple.py

@@ -1496,3 +1496,27 @@ def test_line_breaks(rattler_build: RattlerBuild, recipes: Path, tmp_path: Path)
         assert found_lines[i], f"Expected to find 'line {i}' in the output"
 
     assert any("done" in line for line in output_lines)
+
+
+def test_channel_sources(
+    rattler_build: RattlerBuild, recipes: Path, tmp_path: Path, monkeypatch
+):
+    with pytest.raises(CalledProcessError):
+        # channel_sources and channels cannot both be set at the same time
+        rattler_build.build(
+            recipes / "channel_sources",
+            tmp_path,
+            custom_channels=["conda-forge"],
+        )
+
+    output = rattler_build.build(
+        recipes / "channel_sources",
+        tmp_path,
+        extra_args=["--render-only"],
+    )
+
+    output_json = json.loads(output)
+    assert output_json[0]["build_configuration"]["channels"] == [
+        "https://conda.anaconda.org/conda-forge/label/rust_dev",
+        "https://conda.anaconda.org/conda-forge",
+    ]