fix: better out-of source caching and source code propagation

crates/pixi_cli/src/build.rs

@@ -5,6 +5,7 @@ use indicatif::ProgressBar;
 use miette::{Context, IntoDiagnostic};
 use pixi_command_dispatcher::{
     BuildBackendMetadataSpec, BuildEnvironment, BuildProfile, CacheDirs, SourceBuildSpec,
+    build::SourceCodeLocation,
 };
 use pixi_config::ConfigCli;
 use pixi_core::WorkspaceLocator;
@@ -145,7 +146,7 @@ pub async fn execute(args: Args) -> miette::Result<()> {
 
     // Create the build backend metadata specification.
     let backend_metadata_spec = BuildBackendMetadataSpec {
-        manifest_source: manifest_source.clone(),
+        source: SourceCodeLocation::new(manifest_source.clone(), None),
         channels: channels.clone(),
         channel_config: channel_config.clone(),
         build_environment: build_environment.clone(),
@@ -183,8 +184,7 @@ pub async fn execute(args: Args) -> miette::Result<()> {
                 package,
                 // Build into a temporary directory first
                 output_directory: Some(temp_output_dir.path().to_path_buf()),
-                manifest_source: manifest_source.clone(),
-                build_source: None,
+                source: SourceCodeLocation::new(manifest_source.clone(), None),
                 channels: channels.clone(),
                 channel_config: channel_config.clone(),
                 build_environment: build_environment.clone(),

crates/pixi_command_dispatcher/src/build/build_cache.rs

@@ -19,7 +19,7 @@ use thiserror::Error;
 use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
 use xxhash_rust::xxh3::Xxh3;
 
-use crate::build::source_checkout_cache_key;
+use crate::build::{SourceCodeLocation, source_checkout_cache_key};
 
 /// A cache for caching build artifacts of a source checkout.
 #[derive(Clone)]
@@ -249,7 +249,7 @@ pub struct BuildHostPackage {
     pub repodata_record: RepoDataRecord,
 
     /// The source location from which the package was built.
-    pub source: Option<PinnedSourceSpec>,
+    pub source: Option<SourceCodeLocation>,
 }
 
 /// A cache entry returned by [`BuildCache::entry`] which enables

crates/pixi_command_dispatcher/src/build/mod.rs

@@ -22,12 +22,88 @@ pub use dependencies::{
 };
 pub(crate) use move_file::{MoveError, move_file};
 use pixi_record::PinnedSourceSpec;
+use serde::{Deserialize, Serialize};
 use url::Url;
 pub use work_dir_key::{SourceRecordOrCheckout, WorkDirKey};
 use xxhash_rust::xxh3::Xxh3;
 
 const KNOWN_SUFFIXES: [&str; 3] = [".git", ".tar.gz", ".zip"];
 
+/// Stores the two possible locations for the source code,
+/// in the case of an out-of-tree source build.
+///
+/// Something which looks like:
+/// ```toml
+/// [package.build]
+/// source = { path = "some-path" }
+/// ```
+///
+/// We want to prefer that location for our cache checks
+#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)]
+pub struct SourceCodeLocation {
+    /// The location of the manifest and the possible source code
+    manifest_source: PinnedSourceSpec,
+    /// The location of the source code that should be queried and build
+    build_source: Option<PinnedSourceSpec>,
+}
+
+impl SourceCodeLocation {
+    pub fn new(manifest_source: PinnedSourceSpec, build_source: Option<PinnedSourceSpec>) -> Self {
+        Self {
+            manifest_source,
+            build_source,
+        }
+    }
+
+    /// Get the reference to the manifest source
+    pub fn manifest_source(&self) -> &PinnedSourceSpec {
+        &self.manifest_source
+    }
+
+    /// Get the pinned source spec to the actual source code
+    /// This is the normally the path to the manifest_source
+    /// but when set is the path to the build_source
+    pub fn source_code(&self) -> &PinnedSourceSpec {
+        self.build_source.as_ref().unwrap_or(&self.manifest_source)
+    }
+
+    /// Get the optional explicit build source override.
+    pub fn build_source(&self) -> Option<&PinnedSourceSpec> {
+        self.build_source.as_ref()
+    }
+
+    pub fn as_source_and_alternative_root(&self) -> (&PinnedSourceSpec, Option<&PinnedSourceSpec>) {
+        if let Some(build_source) = &self.build_source {
+            (build_source, Some(&self.manifest_source))
+        } else {
+            (&self.manifest_source, None)
+        }
+    }
+}
+
+impl std::fmt::Display for SourceCodeLocation {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "(manifest-src: {}, build-src: {})",
+            self.manifest_source(),
+            self.build_source
+                .as_ref()
+                .map(|build| format!("{build}"))
+                .unwrap_or("undefined".to_string())
+        )
+    }
+}
+
+impl From<PinnedSourceSpec> for SourceCodeLocation {
+    fn from(manifest_source: PinnedSourceSpec) -> Self {
+        Self {
+            manifest_source,
+            build_source: None,
+        }
+    }
+}
+
 /// Try to deduce a name from a url.
 fn pretty_url_name(url: &Url) -> String {
     if let Some(last_segment) = url

crates/pixi_command_dispatcher/src/build_backend_metadata/mod.rs

@@ -25,7 +25,7 @@ use crate::{
     BuildEnvironment, CommandDispatcher, CommandDispatcherError, CommandDispatcherErrorResultExt,
     InstantiateBackendError, InstantiateBackendSpec, SourceCheckout, SourceCheckoutError,
     build::{
-        SourceRecordOrCheckout, WorkDirKey,
+        SourceCodeLocation, SourceRecordOrCheckout, WorkDirKey,
         source_metadata_cache::{self, CachedCondaMetadata, MetadataKind, SourceMetadataKey},
     },
 };
@@ -49,8 +49,8 @@ fn warn_once_per_backend(backend_name: &str) {
 /// particular source.
 #[derive(Debug, Clone, Eq, PartialEq, Hash, serde::Serialize)]
 pub struct BuildBackendMetadataSpec {
-    /// The source specification where manifest is located at.
-    pub manifest_source: PinnedSourceSpec,
+    /// The manifest and optional build source location.
+    pub source: SourceCodeLocation,
 
     /// The channel configuration to use for the build backend.
     pub channel_config: ChannelConfig,
@@ -81,11 +81,8 @@ pub struct BuildBackendMetadataSpec {
 /// The metadata of a source checkout.
 #[derive(Debug)]
 pub struct BuildBackendMetadata {
-    /// The source checkout that the manifest was extracted from.
-    pub manifest_source: PinnedSourceSpec,
-
-    /// The source checkout from which we want to build package.
-    pub build_source: Option<PinnedSourceSpec>,
+    /// The manifest and optional build source location for this metadata.
+    pub source: SourceCodeLocation,
 
     /// The cache entry that contains the metadata acquired from the build
     /// backend.
@@ -103,7 +100,8 @@ impl BuildBackendMetadataSpec {
         skip_all,
         name="backend-metadata",
         fields(
-            source = %self.manifest_source,
+            manifest_source = %self.source.manifest_source(),
+            build_source = ?self.source.build_source(),
             platform = %self.build_environment.host_platform,
         )
     )]
@@ -112,9 +110,12 @@ impl BuildBackendMetadataSpec {
         command_dispatcher: CommandDispatcher,
         log_sink: UnboundedSender<String>,
     ) -> Result<BuildBackendMetadata, CommandDispatcherError<BuildBackendMetadataError>> {
+        let manifest_source = self.source.manifest_source().clone();
+
         // Ensure that the source is checked out before proceeding.
         let manifest_source_checkout = command_dispatcher
-            .checkout_pinned_source(self.manifest_source.clone())
+            // Never has an alternative root because we want to get the manifest
+            .checkout_pinned_source(&manifest_source)
             .await
             .map_err_with(BuildBackendMetadataError::SourceCheckout)?;
 
@@ -131,7 +132,8 @@ impl BuildBackendMetadataSpec {
         let build_source_checkout = if let Some(pin_override) = &self.pin_override {
             Some(
                 command_dispatcher
-                    .checkout_pinned_source(pin_override.clone())
+                    // We use the pinned override directly
+                    .checkout_pinned_source(pin_override)
                     .await
                     .map_err_with(BuildBackendMetadataError::SourceCheckout)?,
             )
@@ -167,6 +169,10 @@ impl BuildBackendMetadataSpec {
         } else {
             (manifest_source_checkout.clone(), None)
         };
+        let source_location = SourceCodeLocation::new(
+            manifest_source_checkout.pinned.clone(),
+            build_source.clone(),
+        );
 
         // Calculate the hash of the project model
         let additional_glob_hash = calculate_additional_glob_hash(
@@ -200,10 +206,9 @@ impl BuildBackendMetadataSpec {
             .await?
             {
                 return Ok(BuildBackendMetadata {
+                    source: source_location.clone(),
                     metadata,
                     cache_entry,
-                    manifest_source: manifest_source_checkout.pinned,
-                    build_source,
                 });
             }
         } else {
@@ -219,7 +224,7 @@ impl BuildBackendMetadataSpec {
             command_dispatcher
                 .instantiate_backend(InstantiateBackendSpec {
                     backend_spec: discovered_backend.backend_spec.clone().resolve(
-                        SourceAnchor::from(SourceSpec::from(self.manifest_source.clone())),
+                        SourceAnchor::from(SourceSpec::from(manifest_source.clone())),
                     ),
                     init_params: discovered_backend.init_params.clone(),
                     build_source_dir,
@@ -230,7 +235,6 @@ impl BuildBackendMetadataSpec {
                 .map_err_with(BuildBackendMetadataError::Initialize)?;
 
         // Call the conda_outputs method to get metadata.
-        let manifest_source = manifest_source_checkout.pinned.clone();
         if !backend.capabilities().provides_conda_outputs() {
             return Err(CommandDispatcherError::Failed(
                 BuildBackendMetadataError::BackendMissingCapabilities(
@@ -261,8 +265,7 @@ impl BuildBackendMetadataSpec {
             .map_err(CommandDispatcherError::Failed)?;
 
         Ok(BuildBackendMetadata {
-            manifest_source,
-            build_source,
+            source: source_location,
             metadata,
             cache_entry,
         })
@@ -480,7 +483,7 @@ impl BuildBackendMetadataSpec {
             build_environment: self.build_environment.clone(),
             build_variants: self.variants.clone().unwrap_or_default(),
             enabled_protocols: self.enabled_protocols.clone(),
-            pinned_source: self.manifest_source.clone(),
+            pinned_source: self.source.manifest_source().clone(),
         }
     }
 }

crates/pixi_command_dispatcher/src/command_dispatcher/mod.rs

@@ -35,7 +35,7 @@ use crate::{
     SourceBuildCacheEntry, SourceBuildCacheStatusError, SourceBuildCacheStatusSpec, SourceCheckout,
     SourceCheckoutError, SourceMetadata, SourceMetadataError, SourceMetadataSpec,
     backend_source_build::{BackendBuiltSource, BackendSourceBuildError, BackendSourceBuildSpec},
-    build::{BuildCache, source_metadata_cache::SourceMetadataCache},
+    build::{BuildCache, SourceCodeLocation, source_metadata_cache::SourceMetadataCache},
     cache_dirs::CacheDirs,
     discover_backend_cache::DiscoveryCache,
     install_pixi::{
@@ -625,19 +625,50 @@ impl CommandDispatcher {
     /// - For URL sources: Extracts the archive with the exact checksum
     ///   (unimplemented)
     pub async fn checkout_pinned_source(
+        &self,
+        pinned_spec: &PinnedSourceSpec,
+    ) -> Result<SourceCheckout, CommandDispatcherError<SourceCheckoutError>> {
+        self.checkout_pinned_spec(pinned_spec.clone(), None).await
+    }
+
+    /// Checkout a source described by a [`SourceCodeLocation`], automatically
+    /// falling back to the manifest path when no dedicated build source is set.
+    ///
+    /// This is useful for the case where you want to checkout a source that is at
+    /// different location than the manifest, most obviously in an out-of-tree build.
+    pub async fn checkout_source_location(
+        &self,
+        source: &SourceCodeLocation,
+    ) -> Result<SourceCheckout, CommandDispatcherError<SourceCheckoutError>> {
+        let (primary, alternative_root) = source.as_source_and_alternative_root();
+        self.checkout_pinned_spec(primary.clone(), alternative_root.cloned())
+            .await
+    }
+
+    async fn checkout_pinned_spec(
         &self,
         pinned_spec: PinnedSourceSpec,
+        alternative_root: Option<PinnedSourceSpec>,
     ) -> Result<SourceCheckout, CommandDispatcherError<SourceCheckoutError>> {
         match pinned_spec {
-            PinnedSourceSpec::Path(ref path) => {
+            PinnedSourceSpec::Path(path_spec) => {
+                let alternative_root_path = match alternative_root.as_ref() {
+                    Some(PinnedSourceSpec::Path(alt_path)) => Some(
+                        self.data
+                            .resolve_typed_path(alt_path.path.to_path(), None)
+                            .map_err(|e| CommandDispatcherError::Failed(e.into()))?,
+                    ),
+                    _ => None,
+                };
+
                 let source_path = self
                     .data
-                    .resolve_typed_path(path.path.to_path(), None)
+                    .resolve_typed_path(path_spec.path.to_path(), alternative_root_path.as_deref())
                     .map_err(SourceCheckoutError::from)
                     .map_err(CommandDispatcherError::Failed)?;
                 Ok(SourceCheckout {
                     path: source_path,
-                    pinned: pinned_spec,
+                    pinned: PinnedSourceSpec::Path(path_spec),
                 })
             }
             PinnedSourceSpec::Git(git_spec) => self.checkout_pinned_git(git_spec).await,

crates/pixi_command_dispatcher/src/install_pixi/mod.rs

@@ -23,8 +23,8 @@ use thiserror::Error;
 
 use crate::{
     BuildEnvironment, BuildProfile, CommandDispatcher, CommandDispatcherError,
-    CommandDispatcherErrorResultExt, SourceBuildError, SourceBuildSpec, executor::ExecutorFutures,
-    install_pixi::reporter::WrappingInstallReporter,
+    CommandDispatcherErrorResultExt, SourceBuildError, SourceBuildSpec, build::SourceCodeLocation,
+    executor::ExecutorFutures, install_pixi::reporter::WrappingInstallReporter,
 };
 
 #[derive(Debug, Clone, serde::Serialize)]
@@ -215,7 +215,10 @@ impl InstallPixiEnvironmentSpec {
             .contains(&source_record.package_record.name);
         let built_source = command_dispatcher
             .source_build(SourceBuildSpec {
-                manifest_source: source_record.manifest_source.clone(),
+                source: SourceCodeLocation::new(
+                    source_record.manifest_source.clone(),
+                    source_record.build_source.clone(),
+                ),
                 package: source_record.into(),
                 channel_config: self.channel_config.clone(),
                 channels: self.channels.clone(),
@@ -230,7 +233,6 @@ impl InstallPixiEnvironmentSpec {
                 force,
                 // When we install a pixi environment we always build in development mode.
                 build_profile: BuildProfile::Development,
-                build_source: None,
             })
             .await?;
 

crates/pixi_command_dispatcher/src/solve_conda/mod.rs

@@ -156,7 +156,7 @@ impl SolveCondaEnvironmentSpec {
                         channel: None,
                     };
                     let mut record = record.clone();
-                    record.build_source = source_metadata.build_source.clone();
+                    record.build_source = source_metadata.source.build_source().cloned();
                     url_to_source_package.insert(url, (record, repodata_record));
                 }
             }

crates/pixi_command_dispatcher/src/solve_pixi/source_metadata_collector.rs

@@ -15,6 +15,7 @@ use thiserror::Error;
 use crate::{
     BuildBackendMetadataSpec, BuildEnvironment, CommandDispatcher, CommandDispatcherError,
     SourceCheckoutError, SourceMetadataSpec,
+    build::SourceCodeLocation,
     executor::ExecutorFutures,
     source_metadata::{CycleEnvironment, SourceMetadata, SourceMetadataError},
 };
@@ -185,7 +186,7 @@ impl SourceMetadataCollector {
             .source_metadata(SourceMetadataSpec {
                 package: name.clone(),
                 backend_metadata: BuildBackendMetadataSpec {
-                    manifest_source: source.pinned,
+                    source: SourceCodeLocation::new(source.pinned, None),
                     channel_config: self.channel_config.clone(),
                     channels: self.channels.clone(),
                     build_environment: self.build_environment.clone(),
@@ -230,7 +231,7 @@ impl SourceMetadataCollector {
                         source_metadata.skipped_packages.clone(),
                     ),
                     name,
-                    pinned_source: Box::new(source_metadata.manifest_source.clone()),
+                    pinned_source: Box::new(source_metadata.source.manifest_source().clone()),
                 },
             ));
         }