Skip to content

Commit b494101

Browse files
committed
wip: add dev-source-metadata
1 parent c6efec5 commit b494101

File tree

21 files changed

+415
-6
lines changed

21 files changed

+415
-6
lines changed

Cargo.lock

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

crates/pixi_command_dispatcher/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ dirs = { workspace = true }
1818
dunce = { workspace = true }
1919
fs-err = { workspace = true }
2020
futures = { workspace = true }
21+
indexmap = { workspace = true }
2122
itertools = { workspace = true }
2223
miette = { workspace = true }
2324
ordermap = { workspace = true }

crates/pixi_command_dispatcher/src/command_dispatcher/mod.rs

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -468,6 +468,26 @@ impl CommandDispatcher {
468468
self.execute_task(spec).await
469469
}
470470

471+
/// Returns the metadata for development sources.
472+
///
473+
/// This method queries the build backend for all outputs from a dev source
474+
/// and creates DevSourceRecords for each one. These records contain the
475+
/// combined dependencies (build, host, run) for each output.
476+
///
477+
/// Unlike `source_metadata`, this is specifically for development sources
478+
/// where the dependencies are installed but the package itself is not built.
479+
///
480+
/// # Requirements
481+
///
482+
/// - The build backend must support the `conda/outputs` procedure (API v1+)
483+
pub async fn dev_source_metadata(
484+
&self,
485+
spec: crate::DevSourceMetadataSpec,
486+
) -> Result<crate::DevSourceMetadata, CommandDispatcherError<crate::DevSourceMetadataError>>
487+
{
488+
spec.request(self.clone()).await
489+
}
490+
471491
/// Query the source build cache for a particular source package.
472492
pub async fn source_build_cache_status(
473493
&self,
Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
use std::collections::BTreeMap;
2+
3+
use miette::Diagnostic;
4+
use pixi_record::{DevSourceRecord, PinnedSourceSpec};
5+
use pixi_spec::{BinarySpec, PixiSpec, SourceAnchor};
6+
use pixi_spec_containers::DependencyMap;
7+
use rattler_conda_types::PackageName;
8+
use thiserror::Error;
9+
use tracing::instrument;
10+
11+
use crate::{
12+
BuildBackendMetadataError, BuildBackendMetadataSpec, CommandDispatcher, CommandDispatcherError,
13+
CommandDispatcherErrorResultExt, build::source_metadata_cache::MetadataKind,
14+
};
15+
16+
/// A specification for retrieving development source metadata.
17+
///
18+
/// This queries the build backend for all outputs from a source and creates
19+
/// DevSourceRecords for each one.
20+
#[derive(Debug, Clone, Eq, PartialEq, Hash, serde::Serialize)]
21+
pub struct DevSourceMetadataSpec {
22+
/// The development source specification
23+
pub package_name: PackageName,
24+
25+
/// Information about the build backend to request the information from
26+
pub backend_metadata: BuildBackendMetadataSpec,
27+
}
28+
29+
/// The result of querying development source metadata.
30+
#[derive(Debug, Clone)]
31+
pub struct DevSourceMetadata {
32+
/// Information about the source checkout that was used
33+
pub source: PinnedSourceSpec,
34+
35+
/// All the dev source records for outputs from this source
36+
pub records: Vec<DevSourceRecord>,
37+
}
38+
39+
/// An error that can occur while retrieving dev source metadata.
40+
#[derive(Debug, Error, Diagnostic)]
41+
pub enum DevSourceMetadataError {
42+
#[error(transparent)]
43+
#[diagnostic(transparent)]
44+
BuildBackendMetadata(#[from] BuildBackendMetadataError),
45+
46+
#[error(
47+
"the build backend does not support the `conda/outputs` procedure, which is required for development sources"
48+
)]
49+
UnsupportedProtocol,
50+
}
51+
52+
impl DevSourceMetadataSpec {
53+
/// Retrieves development source metadata by querying the build backend.
54+
///
55+
/// This method:
56+
/// 1. Gets metadata from the build backend
57+
/// 2. Creates a DevSourceRecord for each output
58+
/// 3. Combines build/host/run dependencies for each output
59+
#[instrument(
60+
skip_all,
61+
name = "dev-source-metadata",
62+
fields(
63+
source = %self.backend_metadata.source,
64+
platform = %self.backend_metadata.build_environment.host_platform,
65+
)
66+
)]
67+
pub(crate) async fn request(
68+
self,
69+
command_dispatcher: CommandDispatcher,
70+
) -> Result<DevSourceMetadata, CommandDispatcherError<DevSourceMetadataError>> {
71+
// Get the metadata from the build backend
72+
let build_backend_metadata = command_dispatcher
73+
.build_backend_metadata(self.backend_metadata.clone())
74+
.await
75+
.map_err_with(DevSourceMetadataError::BuildBackendMetadata)?;
76+
77+
// We only support the Outputs protocol for dev sources
78+
let outputs = match &build_backend_metadata.metadata.metadata {
79+
MetadataKind::Outputs { outputs } => outputs,
80+
MetadataKind::GetMetadata { .. } => {
81+
return Err(CommandDispatcherError::Failed(
82+
DevSourceMetadataError::UnsupportedProtocol,
83+
));
84+
}
85+
};
86+
87+
// Create a SourceAnchor for resolving relative paths in dependencies
88+
let source_anchor = SourceAnchor::from(pixi_spec::SourceSpec::from(
89+
build_backend_metadata.source.clone(),
90+
));
91+
92+
// Create a DevSourceRecord for each output
93+
let mut records = Vec::new();
94+
for output in outputs {
95+
if output.metadata.name != self.package_name {
96+
continue;
97+
}
98+
let record = Self::create_dev_source_record(
99+
output,
100+
&build_backend_metadata.source,
101+
&build_backend_metadata.metadata.input_hash,
102+
&self.backend_metadata.variants,
103+
&source_anchor,
104+
)?;
105+
records.push(record);
106+
}
107+
108+
Ok(DevSourceMetadata {
109+
source: build_backend_metadata.source.clone(),
110+
records,
111+
})
112+
}
113+
114+
/// Creates a DevSourceRecord from a CondaOutput.
115+
///
116+
/// This combines all dependencies (build, host, run) into a single map
117+
/// and resolves relative source paths.
118+
fn create_dev_source_record(
119+
output: &pixi_build_types::procedures::conda_outputs::CondaOutput,
120+
source: &PinnedSourceSpec,
121+
input_hash: &Option<pixi_record::InputHash>,
122+
variants: &Option<BTreeMap<String, Vec<String>>>,
123+
source_anchor: &SourceAnchor,
124+
) -> Result<DevSourceRecord, CommandDispatcherError<DevSourceMetadataError>> {
125+
// Combine all dependencies into a single map
126+
let mut all_dependencies = DependencyMap::default();
127+
let mut all_constraints = DependencyMap::default();
128+
129+
// Helper to process dependencies and resolve paths
130+
let process_deps =
131+
|deps: Option<
132+
&pixi_build_types::procedures::conda_outputs::CondaOutputDependencies,
133+
>,
134+
dependencies: &mut DependencyMap<PackageName, PixiSpec>,
135+
constraints: &mut DependencyMap<PackageName, BinarySpec>| {
136+
if let Some(deps) = deps {
137+
// Process depends
138+
for depend in &deps.depends {
139+
let name = PackageName::new_unchecked(&depend.name);
140+
let spec =
141+
crate::build::conversion::from_package_spec_v1(depend.spec.clone());
142+
143+
// Resolve relative paths for source dependencies
144+
let resolved_spec = match spec.into_source_or_binary() {
145+
itertools::Either::Left(source_spec) => {
146+
PixiSpec::from(source_anchor.resolve(source_spec))
147+
}
148+
itertools::Either::Right(binary_spec) => PixiSpec::from(binary_spec),
149+
};
150+
dependencies.insert(name, resolved_spec);
151+
}
152+
153+
// Process constraints
154+
for constraint in &deps.constraints {
155+
let name = PackageName::new_unchecked(&constraint.name);
156+
let spec =
157+
crate::build::conversion::from_binary_spec_v1(constraint.spec.clone());
158+
constraints.insert(name, spec);
159+
}
160+
}
161+
};
162+
163+
// Process all dependency types
164+
process_deps(
165+
output.build_dependencies.as_ref(),
166+
&mut all_dependencies,
167+
&mut all_constraints,
168+
);
169+
process_deps(
170+
output.host_dependencies.as_ref(),
171+
&mut all_dependencies,
172+
&mut all_constraints,
173+
);
174+
process_deps(
175+
Some(&output.run_dependencies),
176+
&mut all_dependencies,
177+
&mut all_constraints,
178+
);
179+
180+
// Extract variant values (not the lists of possible values)
181+
// For now, we'll take the first value of each variant
182+
// TODO: This needs to be properly handled based on the actual variant selection
183+
let variant_values = variants
184+
.as_ref()
185+
.map(|v| {
186+
v.iter()
187+
.filter_map(|(k, values)| {
188+
values.first().map(|first| (k.clone(), first.clone()))
189+
})
190+
.collect()
191+
})
192+
.unwrap_or_default();
193+
194+
Ok(DevSourceRecord {
195+
name: output.metadata.name.clone(),
196+
source: source.clone(),
197+
input_hash: input_hash.clone(),
198+
variants: variant_values,
199+
dependencies: all_dependencies,
200+
constraints: all_constraints,
201+
})
202+
}
203+
}

crates/pixi_command_dispatcher/src/instantiate_tool_env/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,7 @@ impl InstantiateToolEnvironmentSpec {
200200
.chain([self.requirement.clone()])
201201
.collect(),
202202
constraints,
203+
dev_sources: Default::default(),
203204
build_environment: self.build_environment.clone(),
204205
exclude_newer: self.exclude_newer,
205206
channel_config: self.channel_config.clone(),

crates/pixi_command_dispatcher/src/lib.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ mod build_backend_metadata;
3838
mod cache_dirs;
3939
mod command_dispatcher;
4040
mod command_dispatcher_processor;
41+
mod dev_source_metadata;
4142
mod discover_backend_cache;
4243
mod executor;
4344
mod expand_dev_sources;
@@ -68,6 +69,7 @@ pub use command_dispatcher::{
6869
CommandDispatcher, CommandDispatcherBuilder, CommandDispatcherError,
6970
CommandDispatcherErrorResultExt, InstantiateBackendError, InstantiateBackendSpec,
7071
};
72+
pub use dev_source_metadata::{DevSourceMetadata, DevSourceMetadataError, DevSourceMetadataSpec};
7173
pub use executor::Executor;
7274
pub use expand_dev_sources::{
7375
DependencyOnlySource, ExpandDevSourcesError, ExpandDevSourcesSpec, ExpandedDevSources,

crates/pixi_command_dispatcher/src/output_dependencies/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ pub struct OutputDependencies {
7373
impl GetOutputDependenciesSpec {
7474
#[instrument(
7575
skip_all,
76-
name = "output-dependencies",
76+
name = "dev-sources",
7777
fields(
7878
source = %self.source,
7979
output = %self.output_name.as_source(),

crates/pixi_command_dispatcher/src/solve_pixi/mod.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ mod source_metadata_collector;
44
use std::{borrow::Borrow, collections::BTreeMap, path::PathBuf, time::Instant};
55

66
use chrono::{DateTime, Utc};
7+
use indexmap::IndexMap;
78
use itertools::{Either, Itertools};
89
use miette::Diagnostic;
910
use pixi_build_discovery::EnabledProtocols;
@@ -51,6 +52,11 @@ pub struct PixiEnvironmentSpec {
5152
#[serde(skip_serializing_if = "DependencyMap::is_empty")]
5253
pub constraints: DependencyMap<rattler_conda_types::PackageName, BinarySpec>,
5354

55+
/// Development sources whose dependencies should be installed without
56+
/// building the packages themselves.
57+
#[serde(skip_serializing_if = "IndexMap::is_empty")]
58+
pub dev_sources: IndexMap<rattler_conda_types::PackageName, pixi_spec::DevSourceSpec>,
59+
5460
/// The records of the packages that are currently already installed. These
5561
/// are used as hints to reduce the difference between individual solves.
5662
#[serde(skip)]
@@ -91,6 +97,7 @@ impl Default for PixiEnvironmentSpec {
9197
name: None,
9298
dependencies: DependencyMap::default(),
9399
constraints: DependencyMap::default(),
100+
dev_sources: IndexMap::new(),
94101
installed: Vec::new(),
95102
build_environment: BuildEnvironment::default(),
96103
channels: vec![],

crates/pixi_command_dispatcher/src/source_build/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -663,6 +663,7 @@ impl SourceBuildSpec {
663663
.into_specs()
664664
.map(|(name, spec)| (name, spec.value))
665665
.collect(),
666+
dev_sources: Default::default(),
666667
installed: vec![], // TODO: To lock build environments, fill this.
667668
build_environment,
668669
channels: self.channels.clone(),

crates/pixi_command_dispatcher/src/source_metadata/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,7 @@ impl SourceMetadataSpec {
373373
.into_specs()
374374
.map(|(name, spec)| (name, spec.value))
375375
.collect(),
376+
dev_sources: Default::default(),
376377
installed: vec![], // TODO: To lock build environments, fill this.
377378
build_environment,
378379
channels: self.backend_metadata.channels.clone(),

0 commit comments

Comments
 (0)