wip: expand dev sources

Author: baszalmstra

crates/pixi_command_dispatcher/src/command_dispatcher/mod.rs

@@ -510,6 +510,41 @@ impl CommandDispatcher {
         spec.request(self.clone()).await
     }
 
+    /// Expands a list of dev sources into their dependencies.
+    ///
+    /// Dev sources are source packages whose dependencies should be installed
+    /// without building the packages themselves. This is particularly useful
+    /// for development environments where you want to work on a package while
+    /// having all its dependencies available.
+    ///
+    /// For each dev source, this method:
+    /// 1. Checks out the source code
+    /// 2. Extracts all dependencies (build, host, and run) and constraints
+    /// 3. Filters out dependencies that are themselves dev sources (to avoid
+    ///    duplication)
+    ///
+    /// The result can then be passed to `solve_pixi_environment` to create
+    /// an environment with all the dependencies but without the dev source
+    /// packages themselves.
+    ///
+    /// # Example use case
+    ///
+    /// You're developing two packages (`pkgA` and `pkgB`) where `pkgA` depends
+    /// on `pkgB`. Instead of building both packages, you can use
+    /// `expand_dev_sources` to:
+    /// 1. Get the dependencies of both `pkgA` and `pkgB`
+    /// 2. Remove `pkgB` from `pkgA`'s dependencies (since it's also a dev source)
+    /// 3. Create an environment with just the external dependencies
+    ///
+    /// This allows you to work on the source code of both packages directly.
+    pub async fn expand_dev_sources(
+        &self,
+        spec: crate::ExpandDevSourcesSpec,
+    ) -> Result<crate::ExpandedDevSources, CommandDispatcherError<crate::ExpandDevSourcesError>>
+    {
+        spec.request(self.clone()).await
+    }
+
     /// Calls into a pixi build backend to perform a source build.
     pub(crate) async fn backend_source_build(
         &self,

crates/pixi_command_dispatcher/src/expand_dev_sources/mod.rs

@@ -0,0 +1,184 @@
+use std::collections::{BTreeMap, HashSet};
+
+use miette::Diagnostic;
+use pixi_build_discovery::EnabledProtocols;
+use pixi_spec::{BinarySpec, PixiSpec, SourceSpec};
+use pixi_spec_containers::DependencyMap;
+use rattler_conda_types::{ChannelConfig, ChannelUrl, PackageName};
+use thiserror::Error;
+use tracing::instrument;
+
+use crate::{
+    BuildEnvironment, CommandDispatcher, CommandDispatcherError, CommandDispatcherErrorResultExt,
+    GetOutputDependenciesError, GetOutputDependenciesSpec, SourceCheckoutError,
+};
+
+/// A source package whose dependencies should be installed without building
+/// the package itself. This is useful for development environments where you
+/// want the dependencies of a package but not the package itself.
+#[derive(Debug, Clone, Eq, PartialEq, Hash, serde::Serialize)]
+pub struct DependencyOnlySource {
+    /// The source specification (path/git/url)
+    pub source: SourceSpec,
+
+    /// The name of the output to extract dependencies from
+    pub output_name: PackageName,
+}
+
+/// Result of expanding dev_sources into their dependencies.
+#[derive(Debug, Clone, Default)]
+pub struct ExpandedDevSources {
+    /// All dependencies (build, host, and run) extracted from dev_sources
+    pub dependencies: DependencyMap<PackageName, PixiSpec>,
+
+    /// All constraints (build, host, and run) extracted from dev_sources
+    pub constraints: DependencyMap<PackageName, BinarySpec>,
+}
+
+/// A specification for expanding dev sources into their dependencies.
+///
+/// Dev sources are source packages whose dependencies should be installed
+/// without building the packages themselves. This is useful for development
+/// environments where you want to work on a package while having its
+/// dependencies available.
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct ExpandDevSourcesSpec {
+    /// The list of dev sources to expand
+    pub dev_sources: Vec<DependencyOnlySource>,
+
+    /// The channel configuration to use
+    pub channel_config: ChannelConfig,
+
+    /// The channels to use for solving
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub channels: Vec<ChannelUrl>,
+
+    /// Information about the build environment
+    pub build_environment: BuildEnvironment,
+
+    /// Variant configuration
+    pub variants: Option<BTreeMap<String, Vec<String>>>,
+
+    /// The protocols that are enabled for source packages
+    #[serde(skip_serializing_if = "crate::is_default")]
+    pub enabled_protocols: EnabledProtocols,
+}
+
+/// An error that can occur while expanding dev sources.
+#[derive(Debug, Error, Diagnostic)]
+pub enum ExpandDevSourcesError {
+    #[error("failed to checkout source for package '{name}'")]
+    SourceCheckout {
+        name: String,
+        #[source]
+        #[diagnostic_source]
+        error: SourceCheckoutError,
+    },
+
+    #[error("failed to get output dependencies for package '{}'", .name.as_source())]
+    GetOutputDependencies {
+        name: PackageName,
+        #[source]
+        #[diagnostic_source]
+        error: GetOutputDependenciesError,
+    },
+}
+
+impl ExpandDevSourcesSpec {
+    /// Expands dev_sources into their dependencies.
+    ///
+    /// When a dev_source has a dependency on another package that is also
+    /// in dev_sources, that dependency is not added to the result (since it
+    /// will be processed separately).
+    #[instrument(
+        skip_all,
+        name = "expand-dev-sources",
+        fields(
+            count = self.dev_sources.len(),
+            platform = %self.build_environment.host_platform,
+        )
+    )]
+    pub(crate) async fn request(
+        self,
+        command_dispatcher: CommandDispatcher,
+    ) -> Result<ExpandedDevSources, CommandDispatcherError<ExpandDevSourcesError>> {
+        let mut result = ExpandedDevSources::default();
+
+        // Create a lookup set for dev_sources by output_name
+        // We'll use this to skip dependencies that are also dev_sources
+        // TODO: In the future, we might want to also match by source location for more
+        // precision
+        let dev_source_names: HashSet<_> = self
+            .dev_sources
+            .iter()
+            .map(|ds| ds.output_name.clone())
+            .collect();
+
+        // Process each dev_source
+        for dev_source in self.dev_sources {
+            // Pin and checkout the source
+            let pinned_source = command_dispatcher
+                .pin_and_checkout(dev_source.source.clone())
+                .await
+                .map_err_with(|error| ExpandDevSourcesError::SourceCheckout {
+                    name: dev_source.output_name.as_source().to_string(),
+                    error,
+                })?;
+
+            // Get the output dependencies
+            let spec = GetOutputDependenciesSpec {
+                source: pinned_source.pinned,
+                output_name: dev_source.output_name.clone(),
+                channel_config: self.channel_config.clone(),
+                channels: self.channels.clone(),
+                build_environment: self.build_environment.clone(),
+                variants: self.variants.clone(),
+                enabled_protocols: self.enabled_protocols.clone(),
+            };
+
+            let output_deps = command_dispatcher
+                .get_output_dependencies(spec)
+                .await
+                .map_err_with(|error| ExpandDevSourcesError::GetOutputDependencies {
+                    name: dev_source.output_name.clone(),
+                    error,
+                })?;
+
+            // Helper to process dependencies
+            let mut process_deps = |deps: Option<DependencyMap<PackageName, PixiSpec>>| {
+                if let Some(deps) = deps {
+                    for (name, spec) in deps.into_specs() {
+                        // Skip dependencies that are also dev_sources
+                        // TODO: Currently matching by name only. In the future, we might want to
+                        // also check if the source location matches for more precise matching.
+                        if !dev_source_names.contains(&name) {
+                            result.dependencies.insert(name, spec);
+                        }
+                    }
+                }
+            };
+
+            // Process all dependency types
+            process_deps(output_deps.build_dependencies);
+            process_deps(output_deps.host_dependencies);
+            process_deps(Some(output_deps.run_dependencies));
+
+            // Merge constraints
+            if let Some(build_constraints) = output_deps.build_constraints {
+                for (name, spec) in build_constraints.into_specs() {
+                    result.constraints.insert(name, spec);
+                }
+            }
+            if let Some(host_constraints) = output_deps.host_constraints {
+                for (name, spec) in host_constraints.into_specs() {
+                    result.constraints.insert(name, spec);
+                }
+            }
+            for (name, spec) in output_deps.run_constraints.into_specs() {
+                result.constraints.insert(name, spec);
+            }
+        }
+
+        Ok(result)
+    }
+}

crates/pixi_command_dispatcher/src/lib.rs

@@ -40,6 +40,7 @@ mod command_dispatcher;
 mod command_dispatcher_processor;
 mod discover_backend_cache;
 mod executor;
+mod expand_dev_sources;
 mod install_pixi;
 mod instantiate_tool_env;
 mod limits;
@@ -68,6 +69,9 @@ pub use command_dispatcher::{
     CommandDispatcherErrorResultExt, InstantiateBackendError, InstantiateBackendSpec,
 };
 pub use executor::Executor;
+pub use expand_dev_sources::{
+    DependencyOnlySource, ExpandDevSourcesError, ExpandDevSourcesSpec, ExpandedDevSources,
+};
 pub use install_pixi::{
     InstallPixiEnvironmentError, InstallPixiEnvironmentResult, InstallPixiEnvironmentSpec,
 };