fix: better out-of source caching and source code propagation (#4875)

Author: tdejager

Cargo.lock

@@ -1,4 +1,4 @@
-use pixi_spec::{BinarySpec, PixiSpec, SourceAnchor};
+use pixi_spec::{BinarySpec, PixiSpec, SourceAnchor, SourceSpec};
 use pixi_spec_containers::DependencyMap;
 use rattler_conda_types::ChannelUrl;
 /// Describes how a backend should be instantiated.
@@ -45,8 +45,10 @@ impl JsonRpcBackendSpec {
                         let maybe_source_spec = env_spec.requirement.1.try_into_source_spec();
                         let pixi_spec = match maybe_source_spec {
                             Ok(source_spec) => {
-                                let resolved_spec = source_anchor.resolve(source_spec);
-                                PixiSpec::from(resolved_spec)
+                                let resolved_spec = source_anchor.resolve(source_spec.location);
+                                PixiSpec::from(SourceSpec {
+                                    location: resolved_spec,
+                                })
                             }
                             Err(pixi_spec) => pixi_spec,
                         };
@@ -133,8 +135,10 @@ impl EnvironmentSpec {
         let maybe_source_spec = self.requirement.1.try_into_source_spec();
         let pixi_spec = match maybe_source_spec {
             Ok(source_spec) => {
-                let resolved_spec = source_anchor.resolve(source_spec);
-                PixiSpec::from(resolved_spec)
+                let resolved_spec = source_anchor.resolve(source_spec.location);
+                PixiSpec::from(SourceSpec {
+                    location: resolved_spec,
+                })
             }
             Err(pixi_spec) => pixi_spec,
         };

crates/pixi_build_discovery/src/backend_spec.rs

@@ -27,14 +27,14 @@ pub const METHOD_NAME: &str = "initialize";
 pub struct InitializeParams {
     /// The manifest that the build backend should use.
     ///
-    /// This is an absolute path.
+    /// This is an absolute path to a manifest file.
     pub manifest_path: PathBuf,
 
     /// The root directory of the source code that the build backend should use.
     /// If this is `None`, the backend should use the directory of the
     /// `manifest_path` as the source directory.
     ///
-    /// This is an absolute path.
+    /// This is an absolute path. This is always a directory.
     pub source_dir: Option<PathBuf>,
 
     /// The root directory of the workspace.

crates/pixi_build_types/src/procedures/initialize.rs

@@ -7,6 +7,7 @@ use miette::{Context, IntoDiagnostic};
 use pixi_build_frontend::BackendOverride;
 use pixi_command_dispatcher::{
     BuildBackendMetadataSpec, BuildEnvironment, BuildProfile, CacheDirs, SourceBuildSpec,
+    build::SourceCodeLocation,
 };
 use pixi_config::ConfigCli;
 use pixi_consts::consts::{
@@ -270,13 +271,13 @@ pub async fn execute(args: Args) -> miette::Result<()> {
     // Create the build backend metadata specification.
     let backend_metadata_spec = BuildBackendMetadataSpec {
         manifest_source: manifest_source.clone(),
+        preferred_build_source: None,
         channels: channels.clone(),
         channel_config: channel_config.clone(),
         build_environment: build_environment.clone(),
         variants: Some(variants.clone()),
         variant_files: Some(variant_files.clone()),
         enabled_protocols: Default::default(),
-        pin_override: None,
     };
     let backend_metadata = command_dispatcher
         .build_backend_metadata(backend_metadata_spec.clone())
@@ -307,8 +308,7 @@ pub async fn execute(args: Args) -> miette::Result<()> {
                 package,
                 // Build into a temporary directory first
                 output_directory: Some(temp_output_dir.path().to_path_buf()),
-                manifest_source: manifest_source.clone(),
-                build_source: None,
+                source: SourceCodeLocation::new(manifest_source.clone(), None),
                 channels: channels.clone(),
                 channel_config: channel_config.clone(),
                 build_environment: build_environment.clone(),

crates/pixi_cli/src/build.rs

@@ -19,7 +19,7 @@ use thiserror::Error;
 use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
 use xxhash_rust::xxh3::Xxh3;
 
-use crate::build::source_checkout_cache_key;
+use crate::build::{SourceCodeLocation, source_checkout_cache_key};
 
 /// A cache for caching build artifacts of a source checkout.
 #[derive(Clone)]
@@ -249,7 +249,7 @@ pub struct BuildHostPackage {
     pub repodata_record: RepoDataRecord,
 
     /// The source location from which the package was built.
-    pub source: Option<PinnedSourceSpec>,
+    pub source: Option<SourceCodeLocation>,
 }
 
 /// A cache entry returned by [`BuildCache::entry`] which enables

crates/pixi_command_dispatcher/src/build/build_cache.rs

@@ -8,7 +8,7 @@ use pixi_build_types::{
     },
 };
 use pixi_record::PixiRecord;
-use pixi_spec::{BinarySpec, DetailedSpec, PixiSpec, SourceAnchor, UrlBinarySpec};
+use pixi_spec::{BinarySpec, DetailedSpec, PixiSpec, SourceAnchor, SourceSpec, UrlBinarySpec};
 use pixi_spec_containers::DependencyMap;
 use rattler_conda_types::{
     InvalidPackageNameError, MatchSpec, NamedChannelOrUrl, NamelessMatchSpec, PackageName,
@@ -95,12 +95,12 @@ impl Dependencies {
             })?;
             match conversion::from_package_spec_v1(depend.spec.clone()).into_source_or_binary() {
                 Either::Left(source) => {
-                    let source = if let Some(anchor) = &source_anchor {
-                        anchor.resolve(source)
+                    let location = if let Some(anchor) = &source_anchor {
+                        anchor.resolve(source.location)
                     } else {
-                        source
+                        source.location
                     };
-                    dependencies.insert(name, PixiSpec::from(source).into());
+                    dependencies.insert(name, PixiSpec::from(SourceSpec { location }).into());
                 }
                 Either::Right(binary) => {
                     dependencies.insert(name, PixiSpec::from(binary).into());

crates/pixi_command_dispatcher/src/build/dependencies.rs

@@ -22,12 +22,79 @@ pub use dependencies::{
 };
 pub(crate) use move_file::{MoveError, move_file};
 use pixi_record::PinnedSourceSpec;
+use serde::{Deserialize, Serialize};
 use url::Url;
 pub use work_dir_key::{SourceRecordOrCheckout, WorkDirKey};
 use xxhash_rust::xxh3::Xxh3;
 
 const KNOWN_SUFFIXES: [&str; 3] = [".git", ".tar.gz", ".zip"];
 
+/// Stores the two possible locations for the source code,
+/// in the case of an out-of-tree source build.
+///
+/// Something which looks like:
+/// ```toml
+/// [package.build]
+/// source = { path = "some-path" }
+/// ```
+///
+/// We want to prefer that location for our cache checks
+#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)]
+pub struct SourceCodeLocation {
+    /// The location of the manifest and the possible source code
+    manifest_source: PinnedSourceSpec,
+    /// The location of the source code that should be queried and build
+    build_source: Option<PinnedSourceSpec>,
+}
+
+impl SourceCodeLocation {
+    pub fn new(manifest_source: PinnedSourceSpec, build_source: Option<PinnedSourceSpec>) -> Self {
+        Self {
+            manifest_source,
+            build_source,
+        }
+    }
+
+    /// Get the reference to the manifest source
+    pub fn manifest_source(&self) -> &PinnedSourceSpec {
+        &self.manifest_source
+    }
+
+    /// Get the pinned source spec to the actual source code
+    /// This is the normally the path to the manifest_source
+    /// but when set is the path to the build_source
+    pub fn source_code(&self) -> &PinnedSourceSpec {
+        self.build_source.as_ref().unwrap_or(&self.manifest_source)
+    }
+
+    /// Get the optional explicit build source override.
+    pub fn build_source(&self) -> Option<&PinnedSourceSpec> {
+        self.build_source.as_ref()
+    }
+
+    pub fn as_source_and_alternative_root(&self) -> (&PinnedSourceSpec, Option<&PinnedSourceSpec>) {
+        if let Some(build_source) = &self.build_source {
+            (build_source, Some(&self.manifest_source))
+        } else {
+            (&self.manifest_source, None)
+        }
+    }
+}
+
+impl std::fmt::Display for SourceCodeLocation {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "(manifest-src: {}, build-src: {})",
+            self.manifest_source(),
+            self.build_source
+                .as_ref()
+                .map(|build| format!("{build}"))
+                .unwrap_or("undefined".to_string())
+        )
+    }
+}
+
 /// Try to deduce a name from a url.
 fn pretty_url_name(url: &Url) -> String {
     if let Some(last_segment) = url