From 4c69029a4f2a76cec4426c17922bb153ea653de7 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Thu, 30 Jan 2025 13:24:12 -0800 Subject: [PATCH 01/64] Preliminary work to adopt Resolvo as the solver Resolvo is a CDCL sat solver for package managers and will hopefully prove to be much better at solving quickly than our mostly brute-force approach. Making a checkpoint here after achieving a simple "solve" to get a feel for how spk type map onto Resolvo's concepts. Although currently the smallest unit the solver works with is a package, and that likely needs to change to a component. Signed-off-by: J Robert Ray --- Cargo.lock | 220 +++++++++++++----- Cargo.toml | 1 + .../crates/ident/src/ident_located.rs | 6 + crates/spk-solve/Cargo.toml | 1 + crates/spk-solve/src/lib.rs | 2 + crates/spk-solve/src/solvers/mod.rs | 1 + crates/spk-solve/src/solvers/resolvo/mod.rs | 100 ++++++++ .../resolvo/pkg_request_version_set.rs | 14 ++ .../src/solvers/resolvo/resolvo_tests.rs | 24 ++ .../src/solvers/resolvo/spk_provider.rs | 210 +++++++++++++++++ cspell.json | 4 + 11 files changed, 531 insertions(+), 52 deletions(-) create mode 100644 crates/spk-solve/src/solvers/resolvo/mod.rs create mode 100644 crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs create mode 100644 crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs create mode 100644 crates/spk-solve/src/solvers/resolvo/spk_provider.rs diff --git a/Cargo.lock b/Cargo.lock index 46bb44f784..e2f74ad456 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,14 +29,15 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.8" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42cd52102d3df161c77a887b608d7a4897d7cc112886a9537b738a887a03aaff" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "getrandom 0.3.3", "once_cell", "version_check", - "zerocopy 0.7.32", + "zerocopy", ] [[package]] @@ -364,6 +365,18 @@ dependencies = [ "serde", ] +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -652,6 +665,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "config" version = "0.14.0" @@ -700,7 +722,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom", + "getrandom 0.2.10", "once_cell", "tiny-keccak", ] @@ -1127,6 +1149,15 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +[[package]] +name = "elsa" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9abf33c656a7256451ebb7d0082c5a471820c31269e49d807c538c252352186e" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "encode_unicode" version = "0.3.6" @@ -1161,6 +1192,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "event-listener" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + [[package]] name = "faccess" version = "0.2.4" @@ -1196,6 +1238,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flatbuffers" version = "25.2.10" @@ -1266,6 +1314,12 @@ dependencies = [ "serde_yaml 0.8.26", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "fuser" version = "0.15.1" @@ -1279,7 +1333,7 @@ dependencies = [ "page_size", "pkg-config", "smallvec", - "zerocopy 0.8.24", + "zerocopy", ] [[package]] @@ -1395,7 +1449,19 @@ checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", ] [[package]] @@ -1826,15 +1892,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.14.0" @@ -2038,7 +2095,7 @@ dependencies = [ "hermit-abi 0.3.9", "libc", "log", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -2306,6 +2363,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.1" @@ -2420,7 +2483,17 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ - "fixedbitset", + "fixedbitset 0.4.2", + "indexmap 2.9.0", +] + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset 0.5.7", "indexmap 2.9.0", ] @@ -2673,11 +2746,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.12.1", + "itertools 0.14.0", "log", "multimap", "once_cell", - "petgraph", + "petgraph 0.6.4", "prettyplease", "prost", "prost-types", @@ -2693,7 +2766,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.100", @@ -2738,6 +2811,18 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.8.5" @@ -2765,7 +2850,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.10", ] [[package]] @@ -2830,7 +2915,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom", + "getrandom 0.2.10", "redox_syscall 0.2.16", "thiserror", ] @@ -2947,6 +3032,23 @@ dependencies = [ "winreg", ] +[[package]] +name = "resolvo" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dba027c8e5dd4b5e5a690cfcfb3900d5ffe6985adb048cbd111d5aa596a6c0c8" +dependencies = [ + "ahash", + "bitvec", + "elsa", + "event-listener", + "futures", + "indexmap 2.9.0", + "itertools 0.14.0", + "petgraph 0.7.1", + "tracing", +] + [[package]] name = "ring" version = "0.17.14" @@ -2955,7 +3057,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.10", "libc", "untrusted", "windows-sys 0.52.0", @@ -4418,6 +4520,7 @@ dependencies = [ "num-format", "once_cell", "priority-queue", + "resolvo", "rstest", "sentry", "serde_json", @@ -4602,6 +4705,12 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" @@ -5110,9 +5219,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "pin-project-lite", "tracing-attributes", @@ -5121,9 +5230,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", @@ -5145,9 +5254,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", @@ -5384,7 +5493,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ - "getrandom", + "getrandom 0.2.10", ] [[package]] @@ -5393,7 +5502,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" dependencies = [ - "getrandom", + "getrandom 0.2.10", "serde", ] @@ -5494,6 +5603,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasite" version = "0.1.0" @@ -5862,6 +5980,24 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags 2.9.0", +] + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "xattr" version = "1.0.1" @@ -5880,33 +6016,13 @@ dependencies = [ "linked-hash-map", ] -[[package]] -name = "zerocopy" -version = "0.7.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" -dependencies = [ - "zerocopy-derive 0.7.32", -] - [[package]] name = "zerocopy" version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" dependencies = [ - "zerocopy-derive 0.8.24", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", + "zerocopy-derive", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f48fba4fa0..96bbe8f52d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,6 +87,7 @@ prost = "0.13" rand = "0.8.5" regex = "1.6" relative-path = "1.3" +resolvo = "0.9.1" ring = "0.17.14" rstest = "0.25" sentry = { version = "0.34.0", default-features = false, features = [ diff --git a/crates/spk-schema/crates/ident/src/ident_located.rs b/crates/spk-schema/crates/ident/src/ident_located.rs index 4f924c4adc..98121e809d 100644 --- a/crates/spk-schema/crates/ident/src/ident_located.rs +++ b/crates/spk-schema/crates/ident/src/ident_located.rs @@ -86,3 +86,9 @@ impl TagPath for LocatedBuildIdent { .join(self.build().verbatim_tag_path()) } } + +impl std::fmt::Display for LocatedBuildIdent { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}/{}", self.base(), self.target()) + } +} diff --git a/crates/spk-solve/Cargo.toml b/crates/spk-solve/Cargo.toml index b6feaa292a..e6d3da330c 100644 --- a/crates/spk-solve/Cargo.toml +++ b/crates/spk-solve/Cargo.toml @@ -41,6 +41,7 @@ once_cell = { workspace = true } priority-queue = "1.2" num-bigint = "0.4.3" num-format = { version = "0.4.4", features = ["with-num-bigint"] } +resolvo = { workspace = true } serde_json = { workspace = true } sentry = { workspace = true, optional = true } signal-hook = "0.3" diff --git a/crates/spk-solve/src/lib.rs b/crates/spk-solve/src/lib.rs index aadd40d278..d65a6e5d45 100644 --- a/crates/spk-solve/src/lib.rs +++ b/crates/spk-solve/src/lib.rs @@ -34,6 +34,8 @@ pub use metrics::{ get_metrics_client, }; pub(crate) use search_space::show_search_space_stats; +// Publicly exported CdclSolver to stop dead code warnings +pub use solvers::resolvo::Solver as CdclSolver; pub use solvers::{StepSolver, StepSolverRuntime}; pub use spk_schema::foundation::ident_build::Build; pub use spk_schema::foundation::ident_component::Component; diff --git a/crates/spk-solve/src/solvers/mod.rs b/crates/spk-solve/src/solvers/mod.rs index 3cd8193fa7..864cf1c32c 100644 --- a/crates/spk-solve/src/solvers/mod.rs +++ b/crates/spk-solve/src/solvers/mod.rs @@ -4,6 +4,7 @@ //! Spk package solver implementations. +pub(crate) mod resolvo; pub(crate) mod step; pub use step::{ErrorFreq, Solver as StepSolver, SolverRuntime as StepSolverRuntime}; diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs new file mode 100644 index 0000000000..d60ec5bb3b --- /dev/null +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -0,0 +1,100 @@ +// Copyright (c) Contributors to the SPK project. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/spkenv/spk + +//! A CDCL SAT solver for Spk. +//! +//! This solver uses [Resolvo](https://github.com/prefix-dev/resolvo) and is +//! able to handle more complex problems than the original Spk solver. However +//! the tradeoff is that it requires reading all the package metadata up front +//! so it can be slower than the original solver for small cases. +//! +//! When there is no solution, Resolvo provides a useful error message to help +//! explain the problem whereas the original solver requires reading the solver +//! log to deduce the real cause of the failure. + +mod pkg_request_version_set; +mod spk_provider; + +use std::borrow::Cow; +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; + +use spk_provider::SpkProvider; +use spk_schema::Request; +use spk_schema::ident::{InclusionPolicy, PinPolicy, PkgRequest, RangeIdent}; +use spk_schema::version_range::VersionFilter; +use spk_solve_solution::{PackageSource, Solution}; +use spk_solve_validation::Validators; +use spk_storage::RepositoryHandle; + +use crate::{Error, Result}; + +#[cfg(test)] +#[path = "resolvo_tests.rs"] +mod resolvo_tests; + +#[derive(Clone)] +pub struct Solver { + repos: Vec>, + _validators: Cow<'static, [Validators]>, +} + +impl Solver { + pub fn new(repos: Vec>, validators: Cow<'static, [Validators]>) -> Self { + Self { + repos, + _validators: validators, + } + } + + pub async fn solve(&mut self, requests: &[Request]) -> Result { + let provider = SpkProvider::new(self.repos.clone()); + let pkg_requirements = provider.pkg_requirements(requests); + let var_requirements = provider.var_requirements(requests); + let mut solver = resolvo::Solver::new(provider); + let problem = resolvo::Problem::new() + .requirements(pkg_requirements) + .constraints(var_requirements); + let solved = solver + .solve(problem) + .map_err(|err| Error::String(format!("{err:?}")))?; + + let pool = &solver.provider().pool; + let mut solution = Solution::default(); + for solvable_id in solved { + let solvable = pool.resolve_solvable(solvable_id); + let located_build_ident = &solvable.record; + let pkg_request = PkgRequest { + pkg: RangeIdent { + repository_name: None, + name: located_build_ident.name().to_owned(), + components: BTreeSet::new(), + version: VersionFilter::default(), + build: None, + }, + prerelease_policy: None, + inclusion_policy: InclusionPolicy::default(), + pin: None, + pin_policy: PinPolicy::default(), + required_compat: None, + requested_by: BTreeMap::new(), + }; + let repo = self + .repos + .iter() + .find(|repo| repo.name() == located_build_ident.repository_name()) + .expect("Expected solved package's repository to be in the list of repositories"); + solution.add( + pkg_request, + repo.read_package(located_build_ident.target()).await?, + PackageSource::Repository { + repo: Arc::clone(repo), + // XXX: Why is this needed? + components: repo.read_components(located_build_ident.target()).await?, + }, + ); + } + Ok(solution) + } +} diff --git a/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs b/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs new file mode 100644 index 0000000000..938ccf8000 --- /dev/null +++ b/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs @@ -0,0 +1,14 @@ +// Copyright (c) Contributors to the SPK project. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/spkenv/spk + +use resolvo::utils::VersionSet; +use spk_schema::ident::{LocatedBuildIdent, PkgRequest}; + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +#[repr(transparent)] +pub(crate) struct PkgRequestVS(pub(crate) PkgRequest); + +impl VersionSet for PkgRequestVS { + type V = LocatedBuildIdent; +} diff --git a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs new file mode 100644 index 0000000000..0f71882dff --- /dev/null +++ b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs @@ -0,0 +1,24 @@ +// Copyright (c) Contributors to the SPK project. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/spkenv/spk + +use std::borrow::Cow; + +use rstest::rstest; +use spk_solve_macros::{make_repo, request}; + +use super::Solver; + +#[rstest] +#[tokio::test] +async fn basic() { + let repo = make_repo!( + [ + {"pkg": "basic/1.0.0"}, + ] + ); + + let mut solver = Solver::new(vec![repo.into()], Cow::Borrowed(&[])); + let solution = solver.solve(&[request!("basic")]).await.unwrap(); + assert_eq!(solution.len(), 1); +} diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs new file mode 100644 index 0000000000..2fee7bcef0 --- /dev/null +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -0,0 +1,210 @@ +// Copyright (c) Contributors to the SPK project. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/spkenv/spk + +use std::cell::RefCell; +use std::collections::HashMap; +use std::sync::Arc; + +use resolvo::utils::Pool; +use resolvo::{ + Candidates, + Dependencies, + DependencyProvider, + Interner, + KnownDependencies, + NameId, + Requirement, + SolvableId, + SolverCache, + StringId, + VersionSetId, + VersionSetUnionId, +}; +use spk_schema::ident::LocatedBuildIdent; +use spk_schema::name::PkgNameBuf; +use spk_schema::{Package, Request, VersionIdent}; +use spk_storage::RepositoryHandle; + +use super::pkg_request_version_set::PkgRequestVS; + +pub(crate) struct SpkProvider { + pub(crate) pool: Pool, + repos: Vec>, + interned_solvables: RefCell>, +} + +impl SpkProvider { + pub fn new(repos: Vec>) -> Self { + Self { + pool: Pool::new(), + repos, + interned_solvables: Default::default(), + } + } + + pub fn pkg_requirements(&self, requests: &[Request]) -> Vec { + requests + .iter() + .filter_map(|req| match req { + Request::Pkg(pkg) => Some(pkg), + _ => None, + }) + .map(|req| { + let dep_name = self.pool.intern_package_name(req.pkg.name().to_owned()); + self.pool + .intern_version_set(dep_name, PkgRequestVS(req.clone())) + .into() + }) + .collect() + } + + pub fn var_requirements(&self, _requests: &[Request]) -> Vec { + // TODO + Vec::new() + } +} + +impl DependencyProvider for SpkProvider { + async fn filter_candidates( + &self, + candidates: &[SolvableId], + version_set: VersionSetId, + inverse: bool, + ) -> Vec { + let _pkg_request_vs = self.pool.resolve_version_set(version_set); + dbg!(_pkg_request_vs); + candidates + .iter() + .filter_map(|candidate| { + let _located_build_ident = self.pool.resolve_solvable(*candidate); + // TODO: filter! + if inverse { None } else { Some(*candidate) } + }) + .collect() + } + + async fn get_candidates(&self, name: NameId) -> Option { + let pkg_name = self.pool.resolve_package_name(name); + + let mut located_builds = Vec::new(); + + for repo in &self.repos { + let versions = repo + .list_package_versions(pkg_name) + .await + .unwrap_or_default(); + for version in versions.iter() { + // TODO: We need a borrowing version of this to avoid cloning. + let pkg_version = VersionIdent::new(pkg_name.clone(), (**version).clone()); + + let builds = repo + .list_package_builds(&pkg_version) + .await + .unwrap_or_default(); + + located_builds.extend( + builds + .into_iter() + .map(|build| LocatedBuildIdent::new(repo.name().to_owned(), build)), + ); + } + } + + if located_builds.is_empty() { + return None; + } + + let mut candidates = Candidates { + candidates: Vec::with_capacity(located_builds.len()), + ..Default::default() + }; + + for build in located_builds { + let solvable_id = *self + .interned_solvables + .borrow_mut() + .entry(build.clone()) + .or_insert_with(|| self.pool.intern_solvable(name, build)); + candidates.candidates.push(solvable_id); + } + + Some(candidates) + } + + async fn sort_candidates(&self, _solver: &SolverCache, _solvables: &mut [SolvableId]) { + // TODO: sort! + } + + async fn get_dependencies(&self, solvable: SolvableId) -> Dependencies { + // TODO: get dependencies! + let solvable = self.pool.resolve_solvable(solvable); + let located_build_ident = &solvable.record; + // XXX: This find runtime will add up. + let repo = self + .repos + .iter() + .find(|repo| repo.name() == located_build_ident.repository_name()) + .expect("Expected solved package's repository to be in the list of repositories"); + match repo.read_package(located_build_ident.target()).await { + Ok(package) => { + let mut known_deps = KnownDependencies { + requirements: Vec::with_capacity(package.runtime_requirements().len()), + // This is where IfAlreadyPresent constraints would go. + constrains: Vec::new(), + }; + for requirement in package.runtime_requirements().iter() { + // TODO: var requests? + let Request::Pkg(req) = requirement else { + continue; + }; + let dep_name = self.pool.intern_package_name(req.pkg.name().to_owned()); + known_deps.requirements.push( + self.pool + .intern_version_set(dep_name, PkgRequestVS(req.clone())) + .into(), + ); + } + Dependencies::Known(known_deps) + } + Err(err) => { + let msg = self.pool.intern_string(err.to_string()); + Dependencies::Unknown(msg) + } + } + } +} + +impl Interner for SpkProvider { + fn display_solvable(&self, _solvable: SolvableId) -> impl std::fmt::Display + '_ { + "todo: display_solvable" + } + + fn display_name(&self, _name: NameId) -> impl std::fmt::Display + '_ { + "todo: display_name" + } + + fn display_version_set(&self, _version_set: VersionSetId) -> impl std::fmt::Display + '_ { + "todo: display_version_set" + } + + fn display_string(&self, _string_id: StringId) -> impl std::fmt::Display + '_ { + "todo: display_string" + } + + fn version_set_name(&self, version_set: VersionSetId) -> NameId { + self.pool.resolve_version_set_package_name(version_set) + } + + fn solvable_name(&self, solvable: SolvableId) -> NameId { + self.pool.resolve_solvable(solvable).name + } + + fn version_sets_in_union( + &self, + _version_set_union: VersionSetUnionId, + ) -> impl Iterator { + // TODO + Vec::new().into_iter() + } +} diff --git a/cspell.json b/cspell.json index af53298b08..ca5b59870f 100644 --- a/cspell.json +++ b/cspell.json @@ -73,6 +73,7 @@ "canonicalize", "canonicalized", "CCAA", + "cdcl", "CFLAGS", "chdir", "CHDIR", @@ -303,6 +304,7 @@ "INLINES", "inodes", "insertcopying", + "interner", "intree", "inutes", "invalidapi", @@ -591,6 +593,7 @@ "repr", "reqby", "reqs", + "resolvo", "respecifying", "retpoline", "retryable", @@ -643,6 +646,7 @@ "SIGWINCH", "SIMD", "SNAPPROCESS", + "solvables", "somedata", "SOMEDIGEST", "somedir", From e04c7e97dc44aa7018de81d074b8c9c5c00459cb Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Fri, 31 Jan 2025 14:56:07 -0800 Subject: [PATCH 02/64] Implement candidate sorting Signed-off-by: J Robert Ray --- .../src/solvers/resolvo/resolvo_tests.rs | 21 +++++++++++++++++++ .../src/solvers/resolvo/spk_provider.rs | 9 ++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs index 0f71882dff..083d7e2074 100644 --- a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs +++ b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs @@ -5,6 +5,7 @@ use std::borrow::Cow; use rstest::rstest; +use spk_schema::prelude::HasVersion; use spk_solve_macros::{make_repo, request}; use super::Solver; @@ -22,3 +23,23 @@ async fn basic() { let solution = solver.solve(&[request!("basic")]).await.unwrap(); assert_eq!(solution.len(), 1); } + +#[rstest] +#[tokio::test] +async fn two_choices() { + let repo = make_repo!( + [ + {"pkg": "basic/2.0.0"}, + {"pkg": "basic/1.0.0"}, + ] + ); + + let mut solver = Solver::new(vec![repo.into()], Cow::Borrowed(&[])); + let solution = solver.solve(&[request!("basic")]).await.unwrap(); + assert_eq!(solution.len(), 1); + // All things being equal it should pick the higher version + assert_eq!( + solution.items().next().unwrap().spec.version().to_string(), + "2.0.0" + ); +} diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index 2fee7bcef0..36b8925fa1 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -132,8 +132,13 @@ impl DependencyProvider for SpkProvider { Some(candidates) } - async fn sort_candidates(&self, _solver: &SolverCache, _solvables: &mut [SolvableId]) { - // TODO: sort! + async fn sort_candidates(&self, _solver: &SolverCache, solvables: &mut [SolvableId]) { + // This implementation just picks the highest version. + solvables.sort_by(|a, b| { + let a = self.pool.resolve_solvable(*a); + let b = self.pool.resolve_solvable(*b); + b.record.version().cmp(a.record.version()) + }); } async fn get_dependencies(&self, solvable: SolvableId) -> Dependencies { From 032b5498334ade0c738137996ccaa113d120f57e Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Fri, 31 Jan 2025 14:56:07 -0800 Subject: [PATCH 03/64] Implement candidate filtering by version number Signed-off-by: J Robert Ray --- .../src/solvers/resolvo/resolvo_tests.rs | 19 +++++++++ .../src/solvers/resolvo/spk_provider.rs | 41 ++++++++++++++----- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs index 083d7e2074..099f92aea9 100644 --- a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs +++ b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs @@ -43,3 +43,22 @@ async fn two_choices() { "2.0.0" ); } + +#[rstest] +#[tokio::test] +async fn two_choices_request_lower() { + let repo = make_repo!( + [ + {"pkg": "basic/2.0.0"}, + {"pkg": "basic/1.0.0"}, + ] + ); + + let mut solver = Solver::new(vec![repo.into()], Cow::Borrowed(&[])); + let solution = solver.solve(&[request!("basic/1.0.0")]).await.unwrap(); + assert_eq!(solution.len(), 1); + assert_eq!( + solution.items().next().unwrap().spec.version().to_string(), + "1.0.0" + ); +} diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index 36b8925fa1..7e59948954 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -72,16 +72,37 @@ impl DependencyProvider for SpkProvider { version_set: VersionSetId, inverse: bool, ) -> Vec { - let _pkg_request_vs = self.pool.resolve_version_set(version_set); - dbg!(_pkg_request_vs); - candidates - .iter() - .filter_map(|candidate| { - let _located_build_ident = self.pool.resolve_solvable(*candidate); - // TODO: filter! - if inverse { None } else { Some(*candidate) } - }) - .collect() + let mut selected = Vec::with_capacity(candidates.len()); + let pkg_request_vs = self.pool.resolve_version_set(version_set); + for candidate in candidates { + let solvable = self.pool.resolve_solvable(*candidate); + let located_build_ident = &solvable.record; + let compatible = pkg_request_vs + .0 + .is_version_applicable(located_build_ident.version()); + if compatible.is_ok() { + // XXX: This find runtime will add up. + let repo = self + .repos + .iter() + .find(|repo| repo.name() == located_build_ident.repository_name()) + .expect( + "Expected solved package's repository to be in the list of repositories", + ); + if let Ok(package) = repo.read_package(located_build_ident.target()).await { + if pkg_request_vs.0.is_satisfied_by(&package).is_ok() ^ inverse { + selected.push(*candidate); + } + } else if inverse { + // If reading the package failed but inverse is true, should + // we include the package as a candidate? Unclear. + selected.push(*candidate); + } + } else if inverse { + selected.push(*candidate); + } + } + selected } async fn get_candidates(&self, name: NameId) -> Option { From d64c8b5ce3b7215e81343c4e8181eda352d83fda Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Fri, 31 Jan 2025 16:35:17 -0800 Subject: [PATCH 04/64] Add basic unsat test Signed-off-by: J Robert Ray --- .../src/solvers/resolvo/resolvo_tests.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs index 099f92aea9..dcebe97f65 100644 --- a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs +++ b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs @@ -62,3 +62,20 @@ async fn two_choices_request_lower() { "1.0.0" ); } + +#[rstest] +#[tokio::test] +async fn two_choices_request_missing() { + let repo = make_repo!( + [ + {"pkg": "basic/3.0.0"}, + {"pkg": "basic/2.0.0"}, + ] + ); + + let mut solver = Solver::new(vec![repo.into()], Cow::Borrowed(&[])); + let _solution = solver + .solve(&[request!("basic/1.0.0")]) + .await + .expect_err("Nothing satisfies 1.0.0"); +} From e4466b7467d5c2dc1c59109f73030b44225b9378 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Fri, 31 Jan 2025 17:03:10 -0800 Subject: [PATCH 05/64] Implement simple dependency handling Signed-off-by: J Robert Ray --- .../src/solvers/resolvo/resolvo_tests.rs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs index dcebe97f65..dce1a0a193 100644 --- a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs +++ b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs @@ -79,3 +79,24 @@ async fn two_choices_request_missing() { .await .expect_err("Nothing satisfies 1.0.0"); } + +#[rstest] +#[tokio::test] +async fn package_with_dependency() { + let repo = make_repo!( + [ + {"pkg": "dep/1.0.0"}, + {"pkg": "needs-dep/1.0.0", + "install": { + "requirements": [ + {"pkg": "dep"} + ] + } + }, + ] + ); + + let mut solver = Solver::new(vec![repo.into()], Cow::Borrowed(&[])); + let solution = solver.solve(&[request!("needs-dep/1.0.0")]).await.unwrap(); + assert_eq!(solution.len(), 2); +} From 57578b0724cf8f77a0e66d231f10c4736c7b5bf3 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Fri, 31 Jan 2025 17:03:10 -0800 Subject: [PATCH 06/64] Handle var dependencies This supports package-specific vars but not non-namespaced vars. Signed-off-by: J Robert Ray --- .../resolvo/pkg_request_version_set.rs | 7 +- .../src/solvers/resolvo/resolvo_tests.rs | 49 ++++++++ .../src/solvers/resolvo/spk_provider.rs | 111 ++++++++++++------ 3 files changed, 130 insertions(+), 37 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs b/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs index 938ccf8000..6d916b85ab 100644 --- a/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs +++ b/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs @@ -3,12 +3,13 @@ // https://github.com/spkenv/spk use resolvo::utils::VersionSet; -use spk_schema::ident::{LocatedBuildIdent, PkgRequest}; +use spk_schema::Request; +use spk_schema::ident::LocatedBuildIdent; #[derive(Clone, Debug, Eq, Hash, PartialEq)] #[repr(transparent)] -pub(crate) struct PkgRequestVS(pub(crate) PkgRequest); +pub(crate) struct RequestVS(pub(crate) Request); -impl VersionSet for PkgRequestVS { +impl VersionSet for RequestVS { type V = LocatedBuildIdent; } diff --git a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs index dce1a0a193..df5bb6373d 100644 --- a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs +++ b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs @@ -100,3 +100,52 @@ async fn package_with_dependency() { let solution = solver.solve(&[request!("needs-dep/1.0.0")]).await.unwrap(); assert_eq!(solution.len(), 2); } + +#[rstest] +#[case::expect_blue("dep.color/blue", "blue")] +#[case::expect_red("dep.color/red", "red")] +#[should_panic] +#[case::expect_green("dep.color/green", "green")] +#[tokio::test] +async fn package_with_dependency_on_variant( + #[case] color_spec: &str, + #[case] expected_color: &str, +) { + use spk_schema::{Package, opt_name}; + + let repo = make_repo!( + [ + {"pkg": "dep/1.0.0", + "build": { + "options": [ + {"var": "color/blue"} + ] + } + }, + {"pkg": "dep/1.0.0", + "build": { + "options": [ + {"var": "color/red"} + ] + } + }, + {"pkg": "needs-dep/1.0.0", + "install": { + "requirements": [ + {"pkg": "dep"}, + {"var": color_spec}, + ] + } + }, + ] + ); + + let mut solver = Solver::new(vec![repo.into()], Cow::Borrowed(&[])); + let solution = solver.solve(&[request!("needs-dep/1.0.0")]).await.unwrap(); + assert_eq!(solution.len(), 2); + let dep = solution.get("dep").unwrap(); + assert_eq!( + dep.spec.option_values().get(opt_name!("color")).unwrap(), + expected_color + ); +} diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index 7e59948954..c7d17d4ab7 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -26,10 +26,10 @@ use spk_schema::name::PkgNameBuf; use spk_schema::{Package, Request, VersionIdent}; use spk_storage::RepositoryHandle; -use super::pkg_request_version_set::PkgRequestVS; +use super::pkg_request_version_set::RequestVS; pub(crate) struct SpkProvider { - pub(crate) pool: Pool, + pub(crate) pool: Pool, repos: Vec>, interned_solvables: RefCell>, } @@ -53,7 +53,7 @@ impl SpkProvider { .map(|req| { let dep_name = self.pool.intern_package_name(req.pkg.name().to_owned()); self.pool - .intern_version_set(dep_name, PkgRequestVS(req.clone())) + .intern_version_set(dep_name, RequestVS(Request::Pkg(req.clone()))) .into() }) .collect() @@ -73,33 +73,60 @@ impl DependencyProvider for SpkProvider { inverse: bool, ) -> Vec { let mut selected = Vec::with_capacity(candidates.len()); - let pkg_request_vs = self.pool.resolve_version_set(version_set); + let request_vs = self.pool.resolve_version_set(version_set); for candidate in candidates { let solvable = self.pool.resolve_solvable(*candidate); let located_build_ident = &solvable.record; - let compatible = pkg_request_vs - .0 - .is_version_applicable(located_build_ident.version()); - if compatible.is_ok() { - // XXX: This find runtime will add up. - let repo = self - .repos - .iter() - .find(|repo| repo.name() == located_build_ident.repository_name()) - .expect( - "Expected solved package's repository to be in the list of repositories", - ); - if let Ok(package) = repo.read_package(located_build_ident.target()).await { - if pkg_request_vs.0.is_satisfied_by(&package).is_ok() ^ inverse { + match &request_vs.0 { + Request::Pkg(pkg_request) => { + let compatible = + pkg_request.is_version_applicable(located_build_ident.version()); + if compatible.is_ok() { + // XXX: This find runtime will add up. + let repo = self + .repos + .iter() + .find(|repo| repo.name() == located_build_ident.repository_name()) + .expect( + "Expected solved package's repository to be in the list of repositories", + ); + if let Ok(package) = repo.read_package(located_build_ident.target()).await { + if pkg_request.is_satisfied_by(&package).is_ok() ^ inverse { + selected.push(*candidate); + } + } else if inverse { + // If reading the package failed but inverse is true, should + // we include the package as a candidate? Unclear. + selected.push(*candidate); + } + } else if inverse { selected.push(*candidate); } - } else if inverse { - // If reading the package failed but inverse is true, should - // we include the package as a candidate? Unclear. - selected.push(*candidate); } - } else if inverse { - selected.push(*candidate); + Request::Var(var_request) => match var_request.var.namespace() { + Some(pkg_name) => { + // Will this ever not match? + debug_assert_eq!(pkg_name, located_build_ident.name()); + // XXX: This find runtime will add up. + let repo = self + .repos + .iter() + .find(|repo| repo.name() == located_build_ident.repository_name()) + .expect( + "Expected solved package's repository to be in the list of repositories", + ); + if let Ok(package) = repo.read_package(located_build_ident.target()).await { + if var_request.is_satisfied_by(&package).is_ok() ^ inverse { + selected.push(*candidate); + } + } else if inverse { + // If reading the package failed but inverse is true, should + // we include the package as a candidate? Unclear. + selected.push(*candidate); + } + } + None => todo!("how do we handle 'global' vars?"), + }, } } selected @@ -180,16 +207,32 @@ impl DependencyProvider for SpkProvider { constrains: Vec::new(), }; for requirement in package.runtime_requirements().iter() { - // TODO: var requests? - let Request::Pkg(req) = requirement else { - continue; - }; - let dep_name = self.pool.intern_package_name(req.pkg.name().to_owned()); - known_deps.requirements.push( - self.pool - .intern_version_set(dep_name, PkgRequestVS(req.clone())) - .into(), - ); + match requirement { + Request::Pkg(pkg_request) => { + let dep_name = self + .pool + .intern_package_name(pkg_request.pkg.name().to_owned()); + known_deps.requirements.push( + self.pool + .intern_version_set(dep_name, RequestVS(requirement.clone())) + .into(), + ); + } + Request::Var(var_request) => match var_request.var.namespace() { + Some(pkg_name) => { + // If we end up adding pkg_name to the solve, + // it needs to satisfy this var request. + let dep_name = self.pool.intern_package_name(pkg_name.to_owned()); + known_deps.constrains.push( + self.pool.intern_version_set( + dep_name, + RequestVS(requirement.clone()), + ), + ); + } + None => todo!("how do we handle 'global' vars?"), + }, + } } Dependencies::Known(known_deps) } From baeeee4579c8f8c30757f542c028036764683a04 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Fri, 31 Jan 2025 19:33:58 -0800 Subject: [PATCH 07/64] Fill in missing interner functions Signed-off-by: J Robert Ray --- .../src/solvers/resolvo/spk_provider.rs | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index c7d17d4ab7..c21b0be8ad 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -245,20 +245,21 @@ impl DependencyProvider for SpkProvider { } impl Interner for SpkProvider { - fn display_solvable(&self, _solvable: SolvableId) -> impl std::fmt::Display + '_ { - "todo: display_solvable" + fn display_solvable(&self, solvable: SolvableId) -> impl std::fmt::Display + '_ { + let solvable = self.pool.resolve_solvable(solvable); + format!("{}={}", solvable.record.name(), solvable.record) } - fn display_name(&self, _name: NameId) -> impl std::fmt::Display + '_ { - "todo: display_name" + fn display_name(&self, name: NameId) -> impl std::fmt::Display + '_ { + self.pool.resolve_package_name(name) } - fn display_version_set(&self, _version_set: VersionSetId) -> impl std::fmt::Display + '_ { - "todo: display_version_set" + fn display_version_set(&self, version_set: VersionSetId) -> impl std::fmt::Display + '_ { + self.pool.resolve_version_set(version_set).0.clone() } - fn display_string(&self, _string_id: StringId) -> impl std::fmt::Display + '_ { - "todo: display_string" + fn display_string(&self, string_id: StringId) -> impl std::fmt::Display + '_ { + self.pool.resolve_string(string_id) } fn version_set_name(&self, version_set: VersionSetId) -> NameId { @@ -271,9 +272,8 @@ impl Interner for SpkProvider { fn version_sets_in_union( &self, - _version_set_union: VersionSetUnionId, + version_set_union: VersionSetUnionId, ) -> impl Iterator { - // TODO - Vec::new().into_iter() + self.pool.resolve_version_set_union(version_set_union) } } From ba7dc3ddb62fcc14352a5819e36973be652bad59 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Fri, 31 Jan 2025 19:33:58 -0800 Subject: [PATCH 08/64] Better solver errors Signed-off-by: J Robert Ray --- crates/spk-solve/src/solvers/resolvo/mod.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index d60ec5bb3b..35da9123a7 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -56,9 +56,14 @@ impl Solver { let problem = resolvo::Problem::new() .requirements(pkg_requirements) .constraints(var_requirements); - let solved = solver - .solve(problem) - .map_err(|err| Error::String(format!("{err:?}")))?; + let solved = solver.solve(problem).map_err(|err| match err { + resolvo::UnsolvableOrCancelled::Unsolvable(conflict) => { + Error::String(format!("{}", conflict.display_user_friendly(&solver))) + } + resolvo::UnsolvableOrCancelled::Cancelled(any) => { + Error::String(format!("Solve cancelled: {any:?}")) + } + })?; let pool = &solver.provider().pool; let mut solution = Solution::default(); From 1c4e02eb1335b2b62ba130c138b2ef183189b0bc Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Sat, 1 Feb 2025 20:22:39 -0800 Subject: [PATCH 09/64] Change the solvable to the component level A necessary step towards supporting solving for specific components and handling their dependencies. Signed-off-by: J Robert Ray --- crates/spk-solve/src/solvers/resolvo/mod.rs | 20 +++++-- .../resolvo/pkg_request_version_set.rs | 60 ++++++++++++++++++- .../src/solvers/resolvo/spk_provider.rs | 56 ++++++++++------- 3 files changed, 109 insertions(+), 27 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index 35da9123a7..30dfb6840d 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -69,12 +69,15 @@ impl Solver { let mut solution = Solution::default(); for solvable_id in solved { let solvable = pool.resolve_solvable(solvable_id); - let located_build_ident = &solvable.record; + let located_build_ident_with_component = &solvable.record; let pkg_request = PkgRequest { pkg: RangeIdent { repository_name: None, - name: located_build_ident.name().to_owned(), - components: BTreeSet::new(), + name: located_build_ident_with_component.ident.name().to_owned(), + components: BTreeSet::from_iter([located_build_ident_with_component + .component + .clone() + .into()]), version: VersionFilter::default(), build: None, }, @@ -88,15 +91,20 @@ impl Solver { let repo = self .repos .iter() - .find(|repo| repo.name() == located_build_ident.repository_name()) + .find(|repo| { + repo.name() == located_build_ident_with_component.ident.repository_name() + }) .expect("Expected solved package's repository to be in the list of repositories"); solution.add( pkg_request, - repo.read_package(located_build_ident.target()).await?, + repo.read_package(located_build_ident_with_component.ident.target()) + .await?, PackageSource::Repository { repo: Arc::clone(repo), // XXX: Why is this needed? - components: repo.read_components(located_build_ident.target()).await?, + components: repo + .read_components(located_build_ident_with_component.ident.target()) + .await?, }, ); } diff --git a/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs b/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs index 6d916b85ab..f327f12b88 100644 --- a/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs +++ b/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs @@ -5,11 +5,69 @@ use resolvo::utils::VersionSet; use spk_schema::Request; use spk_schema::ident::LocatedBuildIdent; +use spk_schema::ident_component::Component; #[derive(Clone, Debug, Eq, Hash, PartialEq)] #[repr(transparent)] pub(crate) struct RequestVS(pub(crate) Request); +/// Like `Component` but without the `All` variant. +#[derive(Clone, Eq, Hash, PartialEq)] +pub(crate) enum ComponentWithoutAll { + Build, + Run, + Source, + Named(String), +} + +impl From for Component { + fn from(c: ComponentWithoutAll) -> Self { + match c { + ComponentWithoutAll::Build => Component::Build, + ComponentWithoutAll::Run => Component::Run, + ComponentWithoutAll::Source => Component::Source, + ComponentWithoutAll::Named(name) => Component::Named(name), + } + } +} + +impl TryFrom for ComponentWithoutAll { + type Error = Component; + + fn try_from(c: Component) -> Result { + match c { + Component::All => Err(Component::All), + Component::Build => Ok(ComponentWithoutAll::Build), + Component::Run => Ok(ComponentWithoutAll::Run), + Component::Source => Ok(ComponentWithoutAll::Source), + Component::Named(name) => Ok(ComponentWithoutAll::Named(name)), + } + } +} + +impl std::fmt::Display for ComponentWithoutAll { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + ComponentWithoutAll::Build => write!(f, "build"), + ComponentWithoutAll::Run => write!(f, "run"), + ComponentWithoutAll::Source => write!(f, "source"), + ComponentWithoutAll::Named(name) => write!(f, "{name}"), + } + } +} + +#[derive(Clone, Eq, Hash, PartialEq)] +pub(crate) struct LocatedBuildIdentWithComponent { + pub(crate) ident: LocatedBuildIdent, + pub(crate) component: ComponentWithoutAll, +} + impl VersionSet for RequestVS { - type V = LocatedBuildIdent; + type V = LocatedBuildIdentWithComponent; +} + +impl std::fmt::Display for LocatedBuildIdentWithComponent { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}:{}", self.ident, self.component) + } } diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index c21b0be8ad..256f467156 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -26,12 +26,12 @@ use spk_schema::name::PkgNameBuf; use spk_schema::{Package, Request, VersionIdent}; use spk_storage::RepositoryHandle; -use super::pkg_request_version_set::RequestVS; +use super::pkg_request_version_set::{LocatedBuildIdentWithComponent, RequestVS}; pub(crate) struct SpkProvider { pub(crate) pool: Pool, repos: Vec>, - interned_solvables: RefCell>, + interned_solvables: RefCell>, } impl SpkProvider { @@ -76,21 +76,24 @@ impl DependencyProvider for SpkProvider { let request_vs = self.pool.resolve_version_set(version_set); for candidate in candidates { let solvable = self.pool.resolve_solvable(*candidate); - let located_build_ident = &solvable.record; + let located_build_ident_with_component = &solvable.record; match &request_vs.0 { Request::Pkg(pkg_request) => { - let compatible = - pkg_request.is_version_applicable(located_build_ident.version()); + let compatible = pkg_request + .is_version_applicable(located_build_ident_with_component.ident.version()); if compatible.is_ok() { // XXX: This find runtime will add up. let repo = self .repos .iter() - .find(|repo| repo.name() == located_build_ident.repository_name()) + .find(|repo| repo.name() == located_build_ident_with_component.ident.repository_name()) .expect( "Expected solved package's repository to be in the list of repositories", ); - if let Ok(package) = repo.read_package(located_build_ident.target()).await { + if let Ok(package) = repo + .read_package(located_build_ident_with_component.ident.target()) + .await + { if pkg_request.is_satisfied_by(&package).is_ok() ^ inverse { selected.push(*candidate); } @@ -106,16 +109,19 @@ impl DependencyProvider for SpkProvider { Request::Var(var_request) => match var_request.var.namespace() { Some(pkg_name) => { // Will this ever not match? - debug_assert_eq!(pkg_name, located_build_ident.name()); + debug_assert_eq!(pkg_name, located_build_ident_with_component.ident.name()); // XXX: This find runtime will add up. let repo = self .repos .iter() - .find(|repo| repo.name() == located_build_ident.repository_name()) + .find(|repo| repo.name() == located_build_ident_with_component.ident.repository_name()) .expect( "Expected solved package's repository to be in the list of repositories", ); - if let Ok(package) = repo.read_package(located_build_ident.target()).await { + if let Ok(package) = repo + .read_package(located_build_ident_with_component.ident.target()) + .await + { if var_request.is_satisfied_by(&package).is_ok() ^ inverse { selected.push(*candidate); } @@ -151,11 +157,18 @@ impl DependencyProvider for SpkProvider { .await .unwrap_or_default(); - located_builds.extend( - builds - .into_iter() - .map(|build| LocatedBuildIdent::new(repo.name().to_owned(), build)), - ); + for build in builds { + let components = repo.list_build_components(&build).await.unwrap_or_default(); + let located_build_ident = LocatedBuildIdent::new(repo.name().to_owned(), build); + for component in components { + located_builds.push(LocatedBuildIdentWithComponent { + ident: located_build_ident.clone(), + component: component + .try_into() + .expect("list_build_components will never return an All component"), + }); + } + } } } @@ -185,21 +198,24 @@ impl DependencyProvider for SpkProvider { solvables.sort_by(|a, b| { let a = self.pool.resolve_solvable(*a); let b = self.pool.resolve_solvable(*b); - b.record.version().cmp(a.record.version()) + b.record.ident.version().cmp(a.record.ident.version()) }); } async fn get_dependencies(&self, solvable: SolvableId) -> Dependencies { // TODO: get dependencies! let solvable = self.pool.resolve_solvable(solvable); - let located_build_ident = &solvable.record; + let located_build_ident_with_component = &solvable.record; // XXX: This find runtime will add up. let repo = self .repos .iter() - .find(|repo| repo.name() == located_build_ident.repository_name()) + .find(|repo| repo.name() == located_build_ident_with_component.ident.repository_name()) .expect("Expected solved package's repository to be in the list of repositories"); - match repo.read_package(located_build_ident.target()).await { + match repo + .read_package(located_build_ident_with_component.ident.target()) + .await + { Ok(package) => { let mut known_deps = KnownDependencies { requirements: Vec::with_capacity(package.runtime_requirements().len()), @@ -247,7 +263,7 @@ impl DependencyProvider for SpkProvider { impl Interner for SpkProvider { fn display_solvable(&self, solvable: SolvableId) -> impl std::fmt::Display + '_ { let solvable = self.pool.resolve_solvable(solvable); - format!("{}={}", solvable.record.name(), solvable.record) + format!("{}={}", solvable.record.ident.name(), solvable.record) } fn display_name(&self, name: NameId) -> impl std::fmt::Display + '_ { From 8f2da396d64c7e394b8ae1ed8926d61898a59243 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Fri, 31 Jan 2025 19:33:58 -0800 Subject: [PATCH 10/64] Crudely support global vars Resolvo doesn't offer a way to declare constraints that aren't associated with a package. The strategy employed here is to create synthetic packages to represent global variables. Whenever a new possible value for a global variable is discovered, the solver needs to be restarted so when "candidates" for a global variable are requested, it returns a "package" (solvable) for each possible value that variable might take. Then, the requirement for packages that have that specific value can be models as a dependency on the synthetic package. It is not too common for packages to have install requirements on non-namespaced vars, so in practice this solver restart may not be a frequent occurrence. Also, top level options (not yet implemented) will allow packages that expect vars with other values to be excluded instead of having to ingest those different values. Of course it would be preferable for the solver to support this concept without the restart mechanic, but maybe it is worth considering making it a requirement for packages to only have namespaced var requirements. Another option is to maintain a cache of known var values somehow either in the repo or in an external service. This could cut down on the number of instances where the solver needs to be restarted. Signed-off-by: J Robert Ray --- Cargo.lock | 1 + .../crates/foundation/src/name/mod.rs | 2 +- crates/spk-solve/Cargo.toml | 2 +- crates/spk-solve/src/solvers/resolvo/mod.rs | 76 +++- .../resolvo/pkg_request_version_set.rs | 102 ++++- .../src/solvers/resolvo/resolvo_tests.rs | 47 ++- .../src/solvers/resolvo/spk_provider.rs | 361 +++++++++++++++--- 7 files changed, 518 insertions(+), 73 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e2f74ad456..ee740a0f9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3046,6 +3046,7 @@ dependencies = [ "indexmap 2.9.0", "itertools 0.14.0", "petgraph 0.7.1", + "tokio", "tracing", ] diff --git a/crates/spk-schema/crates/foundation/src/name/mod.rs b/crates/spk-schema/crates/foundation/src/name/mod.rs index 901fbb4c8f..1a962ee9e3 100644 --- a/crates/spk-schema/crates/foundation/src/name/mod.rs +++ b/crates/spk-schema/crates/foundation/src/name/mod.rs @@ -31,7 +31,7 @@ mod name_test; /// ``` #[macro_export] macro_rules! pkg_name { - ($name:literal) => { + ($name:expr) => { $crate::name::PkgName::new($name).unwrap() }; } diff --git a/crates/spk-solve/Cargo.toml b/crates/spk-solve/Cargo.toml index e6d3da330c..e4cc9218d5 100644 --- a/crates/spk-solve/Cargo.toml +++ b/crates/spk-solve/Cargo.toml @@ -41,7 +41,7 @@ once_cell = { workspace = true } priority-queue = "1.2" num-bigint = "0.4.3" num-format = { version = "0.4.4", features = ["with-num-bigint"] } -resolvo = { workspace = true } +resolvo = { workspace = true, features = ["tokio"] } serde_json = { workspace = true } sentry = { workspace = true, optional = true } signal-hook = "0.3" diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index 30dfb6840d..2e4cf8e7bd 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -20,6 +20,7 @@ use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; +use pkg_request_version_set::SpkSolvable; use spk_provider::SpkProvider; use spk_schema::Request; use spk_schema::ident::{InclusionPolicy, PinPolicy, PkgRequest, RangeIdent}; @@ -49,27 +50,64 @@ impl Solver { } pub async fn solve(&mut self, requests: &[Request]) -> Result { - let provider = SpkProvider::new(self.repos.clone()); - let pkg_requirements = provider.pkg_requirements(requests); - let var_requirements = provider.var_requirements(requests); - let mut solver = resolvo::Solver::new(provider); - let problem = resolvo::Problem::new() - .requirements(pkg_requirements) - .constraints(var_requirements); - let solved = solver.solve(problem).map_err(|err| match err { - resolvo::UnsolvableOrCancelled::Unsolvable(conflict) => { - Error::String(format!("{}", conflict.display_user_friendly(&solver))) - } - resolvo::UnsolvableOrCancelled::Cancelled(any) => { - Error::String(format!("Solve cancelled: {any:?}")) - } - })?; + let repos = self.repos.clone(); + // XXX: Taking a slice reference doesn't make sense anymore. + let requests = requests.to_vec(); + // Use a blocking thread so resolvo can call `block_on` on the runtime. + let solvables = tokio::task::spawn_blocking(move || { + let mut provider = Some(SpkProvider::new(repos.clone())); + let (solver, solved) = loop { + let this_iter_provider = provider.take().expect("provider is always Some"); + let pkg_requirements = this_iter_provider.pkg_requirements(&requests); + let var_requirements = this_iter_provider.var_requirements(&requests); + let mut solver = resolvo::Solver::new(this_iter_provider) + .with_runtime(tokio::runtime::Handle::current()); + let problem = resolvo::Problem::new() + .requirements(pkg_requirements) + .constraints(var_requirements); + match solver.solve(problem) { + Ok(solved) => break (solver, solved), + Err(resolvo::UnsolvableOrCancelled::Cancelled(_)) => { + provider = Some(solver.provider().reset()); + continue; + } + Err(resolvo::UnsolvableOrCancelled::Unsolvable(conflict)) => { + // Edge case: a need to retry was detected but the + // solver arrived at a decision before it noticed it + // needs to cancel (unknown if this ever happens). + if solver.provider().is_canceled() { + provider = Some(solver.provider().reset()); + continue; + } + return Err(Error::String(format!( + "{}", + conflict.display_user_friendly(&solver) + ))); + } + } + }; + + let pool = &solver.provider().pool; + Ok(solved + .into_iter() + .filter_map(|solvable_id| { + let solvable = pool.resolve_solvable(solvable_id); + if let SpkSolvable::LocatedBuildIdentWithComponent( + located_build_ident_with_component, + ) = &solvable.record + { + Some(located_build_ident_with_component.clone()) + } else { + None + } + }) + .collect::>()) + }) + .await + .map_err(|err| Error::String(format!("Tokio panicked? {err}")))??; - let pool = &solver.provider().pool; let mut solution = Solution::default(); - for solvable_id in solved { - let solvable = pool.resolve_solvable(solvable_id); - let located_build_ident_with_component = &solvable.record; + for located_build_ident_with_component in solvables { let pkg_request = PkgRequest { pkg: RangeIdent { repository_name: None, diff --git a/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs b/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs index f327f12b88..64e0ecb6dd 100644 --- a/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs +++ b/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs @@ -2,17 +2,92 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/spkenv/spk +use std::sync::Arc; + use resolvo::utils::VersionSet; use spk_schema::Request; use spk_schema::ident::LocatedBuildIdent; use spk_schema::ident_component::Component; +/// This allows for storing strings of different types but hash and compare by +/// the underlying strings. +#[derive(Clone, Debug)] +pub(crate) enum VarValue { + ArcStr(Arc), + Owned(String), +} + +impl VarValue { + #[inline] + fn as_str(&self) -> &str { + match self { + VarValue::ArcStr(a) => a, + VarValue::Owned(a) => a.as_str(), + } + } +} + +impl std::hash::Hash for VarValue { + fn hash(&self, state: &mut H) { + self.as_str().hash(state) + } +} + +impl Eq for VarValue {} + +impl Ord for VarValue { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.as_str().cmp(other.as_str()) + } +} + +impl PartialOrd for VarValue { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl PartialEq for VarValue { + fn eq(&self, other: &Self) -> bool { + self.as_str() == other.as_str() + } +} + +impl PartialEq> for VarValue { + fn eq(&self, other: &Arc) -> bool { + self.as_str() == &**other + } +} + +impl PartialEq for Arc { + fn eq(&self, other: &VarValue) -> bool { + other.as_str() == &**self + } +} + +impl std::fmt::Display for VarValue { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.as_str().fmt(f) + } +} + #[derive(Clone, Debug, Eq, Hash, PartialEq)] -#[repr(transparent)] -pub(crate) struct RequestVS(pub(crate) Request); +pub(crate) enum RequestVS { + SpkRequest(Request), + GlobalVar { key: String, value: VarValue }, +} + +impl std::fmt::Display for RequestVS { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + RequestVS::SpkRequest(req) => write!(f, "{req}"), + RequestVS::GlobalVar { key, value } => write!(f, "GlobalVar({key}={value})"), + } + } +} /// Like `Component` but without the `All` variant. -#[derive(Clone, Eq, Hash, PartialEq)] +#[derive(Clone, Debug, Eq, Hash, PartialEq)] pub(crate) enum ComponentWithoutAll { Build, Run, @@ -56,14 +131,31 @@ impl std::fmt::Display for ComponentWithoutAll { } } -#[derive(Clone, Eq, Hash, PartialEq)] +#[derive(Clone, Debug, Eq, Hash, PartialEq)] pub(crate) struct LocatedBuildIdentWithComponent { pub(crate) ident: LocatedBuildIdent, pub(crate) component: ComponentWithoutAll, } +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub(crate) enum SpkSolvable { + LocatedBuildIdentWithComponent(LocatedBuildIdentWithComponent), + GlobalVar { key: String, value: VarValue }, +} + +impl std::fmt::Display for SpkSolvable { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + SpkSolvable::LocatedBuildIdentWithComponent(located_build_ident_with_component) => { + write!(f, "{located_build_ident_with_component}") + } + SpkSolvable::GlobalVar { key, value } => write!(f, "GlobalVar({key}={value})"), + } + } +} + impl VersionSet for RequestVS { - type V = LocatedBuildIdentWithComponent; + type V = SpkSolvable; } impl std::fmt::Display for LocatedBuildIdentWithComponent { diff --git a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs index df5bb6373d..2351a9d7bc 100644 --- a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs +++ b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs @@ -6,6 +6,7 @@ use std::borrow::Cow; use rstest::rstest; use spk_schema::prelude::HasVersion; +use spk_schema::{Package, opt_name}; use spk_solve_macros::{make_repo, request}; use super::Solver; @@ -111,8 +112,6 @@ async fn package_with_dependency_on_variant( #[case] color_spec: &str, #[case] expected_color: &str, ) { - use spk_schema::{Package, opt_name}; - let repo = make_repo!( [ {"pkg": "dep/1.0.0", @@ -149,3 +148,47 @@ async fn package_with_dependency_on_variant( expected_color ); } + +#[rstest] +#[case::expect_blue("color/blue", "blue")] +#[case::expect_red("color/red", "red")] +#[should_panic] +#[case::expect_green("color/green", "green")] +#[tokio::test] +async fn global_vars(#[case] global_spec: &str, #[case] expected_color: &str) { + let repo = make_repo!( + [ + {"pkg": "dep/1.0.0", + "build": { + "options": [ + {"var": "color/blue"} + ] + } + }, + {"pkg": "dep/1.0.0", + "build": { + "options": [ + {"var": "color/red"} + ] + } + }, + {"pkg": "needs-dep/1.0.0", + "install": { + "requirements": [ + {"pkg": "dep"}, + {"var": global_spec}, + ] + } + }, + ] + ); + + let mut solver = Solver::new(vec![repo.into()], Cow::Borrowed(&[])); + let solution = solver.solve(&[request!("needs-dep/1.0.0")]).await.unwrap(); + assert_eq!(solution.len(), 2); + let dep = solution.get("dep").unwrap(); + assert_eq!( + dep.spec.option_values().get(opt_name!("color")).unwrap(), + expected_color + ); +} diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index 256f467156..69b97b2592 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/spkenv/spk -use std::cell::RefCell; -use std::collections::HashMap; +use std::cell::{Cell, RefCell}; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use resolvo::utils::Pool; @@ -21,17 +21,36 @@ use resolvo::{ VersionSetId, VersionSetUnionId, }; +use spk_schema::foundation::pkg_name; use spk_schema::ident::LocatedBuildIdent; use spk_schema::name::PkgNameBuf; -use spk_schema::{Package, Request, VersionIdent}; +use spk_schema::{Opt, Package, Request, VersionIdent}; use spk_storage::RepositoryHandle; -use super::pkg_request_version_set::{LocatedBuildIdentWithComponent, RequestVS}; +use super::pkg_request_version_set::{ + LocatedBuildIdentWithComponent, + RequestVS, + SpkSolvable, + VarValue, +}; + +// "global-vars--" represents a pseudo- package that accumulates global var +// constraints. The name is intended to never conflict with a real package name, +// however since it is a legal package name that can't be guaranteed. +const PSEUDO_PKG_NAME_PREFIX: &str = "global-vars--"; pub(crate) struct SpkProvider { pub(crate) pool: Pool, repos: Vec>, - interned_solvables: RefCell>, + interned_solvables: RefCell>, + /// Track all the global var keys and values that have been witnessed while + /// solving. + known_global_var_values: RefCell>>, + /// Track which global var candidates have been queried by the solver. Once + /// queried, it is no longer possible to add more possible values without + /// restarting the solve. + queried_global_var_values: RefCell>, + cancel_solving: Cell, } impl SpkProvider { @@ -40,6 +59,9 @@ impl SpkProvider { pool: Pool::new(), repos, interned_solvables: Default::default(), + known_global_var_values: Default::default(), + queried_global_var_values: Default::default(), + cancel_solving: Default::default(), } } @@ -53,12 +75,29 @@ impl SpkProvider { .map(|req| { let dep_name = self.pool.intern_package_name(req.pkg.name().to_owned()); self.pool - .intern_version_set(dep_name, RequestVS(Request::Pkg(req.clone()))) + .intern_version_set(dep_name, RequestVS::SpkRequest(Request::Pkg(req.clone()))) .into() }) .collect() } + pub fn is_canceled(&self) -> bool { + self.cancel_solving.get() + } + + /// Return a new provider to restart the solve, preserving what was learned + /// about global variables. + pub fn reset(&self) -> Self { + Self { + pool: Pool::new(), + repos: self.repos.clone(), + interned_solvables: Default::default(), + known_global_var_values: RefCell::new(self.known_global_var_values.take()), + queried_global_var_values: Default::default(), + cancel_solving: Default::default(), + } + } + pub fn var_requirements(&self, _requests: &[Request]) -> Vec { // TODO Vec::new() @@ -76,9 +115,17 @@ impl DependencyProvider for SpkProvider { let request_vs = self.pool.resolve_version_set(version_set); for candidate in candidates { let solvable = self.pool.resolve_solvable(*candidate); - let located_build_ident_with_component = &solvable.record; - match &request_vs.0 { - Request::Pkg(pkg_request) => { + match &request_vs { + RequestVS::SpkRequest(Request::Pkg(pkg_request)) => { + let SpkSolvable::LocatedBuildIdentWithComponent( + located_build_ident_with_component, + ) = &solvable.record + else { + if inverse { + selected.push(*candidate); + } + continue; + }; let compatible = pkg_request .is_version_applicable(located_build_ident_with_component.ident.version()); if compatible.is_ok() { @@ -106,33 +153,83 @@ impl DependencyProvider for SpkProvider { selected.push(*candidate); } } - Request::Var(var_request) => match var_request.var.namespace() { - Some(pkg_name) => { - // Will this ever not match? - debug_assert_eq!(pkg_name, located_build_ident_with_component.ident.name()); - // XXX: This find runtime will add up. - let repo = self + RequestVS::SpkRequest(Request::Var(var_request)) => { + match var_request.var.namespace() { + Some(pkg_name) => { + let SpkSolvable::LocatedBuildIdentWithComponent( + located_build_ident_with_component, + ) = &solvable.record + else { + if inverse { + selected.push(*candidate); + } + continue; + }; + // Will this ever not match? + debug_assert_eq!( + pkg_name, + located_build_ident_with_component.ident.name() + ); + // XXX: This find runtime will add up. + let repo = self .repos .iter() .find(|repo| repo.name() == located_build_ident_with_component.ident.repository_name()) .expect( "Expected solved package's repository to be in the list of repositories", ); - if let Ok(package) = repo - .read_package(located_build_ident_with_component.ident.target()) - .await - { - if var_request.is_satisfied_by(&package).is_ok() ^ inverse { + if let Ok(package) = repo + .read_package(located_build_ident_with_component.ident.target()) + .await + { + if var_request.is_satisfied_by(&package).is_ok() ^ inverse { + selected.push(*candidate); + } + } else if inverse { + // If reading the package failed but inverse is true, should + // we include the package as a candidate? Unclear. selected.push(*candidate); } - } else if inverse { - // If reading the package failed but inverse is true, should - // we include the package as a candidate? Unclear. + } + None => match &var_request.value { + spk_schema::ident::PinnableValue::FromBuildEnv => todo!(), + spk_schema::ident::PinnableValue::FromBuildEnvIfPresent => todo!(), + spk_schema::ident::PinnableValue::Pinned(value) => { + let SpkSolvable::GlobalVar { + key: record_key, + value: record_value, + } = &solvable.record + else { + if inverse { + selected.push(*candidate); + } + continue; + }; + if (var_request.var.base_name() == record_key + && value == record_value) + ^ inverse + { + selected.push(*candidate); + } + } + }, + } + } + RequestVS::GlobalVar { key, value } => { + let SpkSolvable::GlobalVar { + key: record_key, + value: record_value, + } = &solvable.record + else { + if inverse { selected.push(*candidate); } + continue; + }; + if (key == record_key && value == record_value) ^ inverse { + selected.push(*candidate); } - None => todo!("how do we handle 'global' vars?"), - }, + } } } selected @@ -141,6 +238,40 @@ impl DependencyProvider for SpkProvider { async fn get_candidates(&self, name: NameId) -> Option { let pkg_name = self.pool.resolve_package_name(name); + if let Some(key) = pkg_name.strip_prefix(PSEUDO_PKG_NAME_PREFIX) { + self.queried_global_var_values + .borrow_mut() + .insert(key.to_owned()); + + if let Some(values) = self.known_global_var_values.borrow().get(key) { + let mut candidates = Candidates { + candidates: Vec::with_capacity(values.len()), + ..Default::default() + }; + for value in values { + let solvable_id = *self + .interned_solvables + .borrow_mut() + .entry(SpkSolvable::GlobalVar { + key: key.to_owned(), + value: value.clone(), + }) + .or_insert_with(|| { + self.pool.intern_solvable( + name, + SpkSolvable::GlobalVar { + key: key.to_owned(), + value: value.clone(), + }, + ) + }); + candidates.candidates.push(solvable_id); + } + return Some(candidates); + } + return None; + } + let mut located_builds = Vec::new(); for repo in &self.repos { @@ -185,8 +316,11 @@ impl DependencyProvider for SpkProvider { let solvable_id = *self .interned_solvables .borrow_mut() - .entry(build.clone()) - .or_insert_with(|| self.pool.intern_solvable(name, build)); + .entry(SpkSolvable::LocatedBuildIdentWithComponent(build.clone())) + .or_insert_with(|| { + self.pool + .intern_solvable(name, SpkSolvable::LocatedBuildIdentWithComponent(build)) + }); candidates.candidates.push(solvable_id); } @@ -198,14 +332,45 @@ impl DependencyProvider for SpkProvider { solvables.sort_by(|a, b| { let a = self.pool.resolve_solvable(*a); let b = self.pool.resolve_solvable(*b); - b.record.ident.version().cmp(a.record.ident.version()) + match (&a.record, &b.record) { + ( + SpkSolvable::LocatedBuildIdentWithComponent(a), + SpkSolvable::LocatedBuildIdentWithComponent(b), + ) => b.ident.version().cmp(a.ident.version()), + ( + SpkSolvable::GlobalVar { + key: a_key, + value: a_value, + }, + SpkSolvable::GlobalVar { + key: b_key, + value: b_value, + }, + ) => { + if a_key == b_key { + a_value.cmp(b_value) + } else { + a_key.cmp(b_key) + } + } + (SpkSolvable::LocatedBuildIdentWithComponent(_), SpkSolvable::GlobalVar { .. }) => { + std::cmp::Ordering::Less + } + (SpkSolvable::GlobalVar { .. }, SpkSolvable::LocatedBuildIdentWithComponent(_)) => { + std::cmp::Ordering::Greater + } + } }); } async fn get_dependencies(&self, solvable: SolvableId) -> Dependencies { // TODO: get dependencies! let solvable = self.pool.resolve_solvable(solvable); - let located_build_ident_with_component = &solvable.record; + let SpkSolvable::LocatedBuildIdentWithComponent(located_build_ident_with_component) = + &solvable.record + else { + return Dependencies::Known(KnownDependencies::default()); + }; // XXX: This find runtime will add up. let repo = self .repos @@ -220,8 +385,49 @@ impl DependencyProvider for SpkProvider { let mut known_deps = KnownDependencies { requirements: Vec::with_capacity(package.runtime_requirements().len()), // This is where IfAlreadyPresent constraints would go. - constrains: Vec::new(), + constrains: Vec::with_capacity(package.get_build_options().len()), }; + for option in package.get_build_options() { + let Opt::Var(var_opt) = option else { + continue; + }; + if var_opt.var.namespace().is_some() { + continue; + } + let Some(value) = var_opt.get_value(None) else { + continue; + }; + let pseudo_pkg_name = + format!("{PSEUDO_PKG_NAME_PREFIX}{}", var_opt.var.base_name()); + if self + .known_global_var_values + .borrow_mut() + .entry(var_opt.var.base_name().to_owned()) + .or_default() + .insert(VarValue::Owned(value.clone())) + && self + .queried_global_var_values + .borrow() + .contains(var_opt.var.base_name()) + { + // Seeing a new value for a var that has already locked + // in the list of candidates. + self.cancel_solving.set(true); + } + let dep_name = self + .pool + .intern_package_name(pkg_name!(&pseudo_pkg_name).to_owned()); + // Add a constraint not a dependency because the package + // is targeting a specific global var value but there may + // not be a request for that var of a specific value. + known_deps.constrains.push(self.pool.intern_version_set( + dep_name, + RequestVS::GlobalVar { + key: var_opt.var.base_name().to_owned(), + value: VarValue::Owned(value), + }, + )); + } for requirement in package.runtime_requirements().iter() { match requirement { Request::Pkg(pkg_request) => { @@ -230,24 +436,79 @@ impl DependencyProvider for SpkProvider { .intern_package_name(pkg_request.pkg.name().to_owned()); known_deps.requirements.push( self.pool - .intern_version_set(dep_name, RequestVS(requirement.clone())) + .intern_version_set( + dep_name, + RequestVS::SpkRequest(requirement.clone()), + ) .into(), ); } - Request::Var(var_request) => match var_request.var.namespace() { - Some(pkg_name) => { - // If we end up adding pkg_name to the solve, - // it needs to satisfy this var request. - let dep_name = self.pool.intern_package_name(pkg_name.to_owned()); - known_deps.constrains.push( - self.pool.intern_version_set( + Request::Var(var_request) => { + match &var_request.value { + spk_schema::ident::PinnableValue::FromBuildEnv => todo!(), + spk_schema::ident::PinnableValue::FromBuildEnvIfPresent => todo!(), + spk_schema::ident::PinnableValue::Pinned(value) => { + let dep_name = match var_request.var.namespace() { + Some(pkg_name) => { + self.pool.intern_package_name(pkg_name.to_owned()) + } + None => { + // Since we will be adding + // constraints for global vars we + // need to add the pseudo-package + // to the dependency list so it + // will influence decisions. + let pseudo_pkg_name = format!( + "{PSEUDO_PKG_NAME_PREFIX}{}", + var_request.var.base_name() + ); + if self + .known_global_var_values + .borrow_mut() + .entry(var_request.var.base_name().to_owned()) + .or_default() + .insert(VarValue::ArcStr(Arc::clone(value))) + && self + .queried_global_var_values + .borrow() + .contains(var_request.var.base_name()) + { + // Seeing a new value for a var + // that has already locked in + // the list of candidates. + self.cancel_solving.set(true); + } + let dep_name = self.pool.intern_package_name( + pkg_name!(&pseudo_pkg_name).to_owned(), + ); + known_deps.requirements.push( + self.pool + .intern_version_set( + dep_name, + RequestVS::GlobalVar { + key: var_request + .var + .base_name() + .to_owned(), + value: VarValue::ArcStr(Arc::clone( + value, + )), + }, + ) + .into(), + ); + dep_name + } + }; + // If we end up adding pkg_name to the solve, + // it needs to satisfy this var request. + known_deps.constrains.push(self.pool.intern_version_set( dep_name, - RequestVS(requirement.clone()), - ), - ); + RequestVS::SpkRequest(requirement.clone()), + )); + } } - None => todo!("how do we handle 'global' vars?"), - }, + } } } Dependencies::Known(known_deps) @@ -258,12 +519,22 @@ impl DependencyProvider for SpkProvider { } } } + + fn should_cancel_with_value(&self) -> Option> { + if self.cancel_solving.get() { + // Eventually there will be more than one reason the solve is + // cancelled... + Some(Box::new(())) + } else { + None + } + } } impl Interner for SpkProvider { fn display_solvable(&self, solvable: SolvableId) -> impl std::fmt::Display + '_ { let solvable = self.pool.resolve_solvable(solvable); - format!("{}={}", solvable.record.ident.name(), solvable.record) + format!("{}", solvable.record) } fn display_name(&self, name: NameId) -> impl std::fmt::Display + '_ { @@ -271,7 +542,7 @@ impl Interner for SpkProvider { } fn display_version_set(&self, version_set: VersionSetId) -> impl std::fmt::Display + '_ { - self.pool.resolve_version_set(version_set).0.clone() + self.pool.resolve_version_set(version_set) } fn display_string(&self, string_id: StringId) -> impl std::fmt::Display + '_ { From 536013def9df0ada77baf18ad71a4f7929dfd596 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Mon, 3 Feb 2025 09:31:43 -0800 Subject: [PATCH 11/64] Trim down the debug output for SolvedRequest Since PackageSource::Repository includes the repo, the debug output would include the entire repository contents, making the debug output too verbose. Signed-off-by: J Robert Ray --- .../spk-solve/crates/solution/src/solution.rs | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/crates/spk-solve/crates/solution/src/solution.rs b/crates/spk-solve/crates/solution/src/solution.rs index d0ca5958e3..49bd1cbbcb 100644 --- a/crates/spk-solve/crates/solution/src/solution.rs +++ b/crates/spk-solve/crates/solution/src/solution.rs @@ -112,7 +112,7 @@ impl PartialOrd for PackageSource { } /// Represents a package request that has been resolved. -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct SolvedRequest { pub request: PkgRequest, pub spec: Arc, @@ -255,6 +255,28 @@ impl SolvedRequest { } } +impl std::fmt::Debug for SolvedRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SolvedRequest") + .field("request", &self.request.to_string()) + .field("spec", &format!("{}", self.spec.ident())) + .field( + "source", + &match &self.source { + PackageSource::Repository { repo, .. } => { + format!("Repository={}", repo.name()) + } + PackageSource::BuildFromSource { recipe } => { + format!("BuildFromSource={}", recipe.ident()) + } + PackageSource::Embedded { parent, .. } => format!("Embedded={parent}"), + PackageSource::SpkInternalTest => "SpkInternalTest".to_string(), + }, + ) + .finish() + } +} + /// A pairing of a solved request and a list of the components (names) /// it provides. pub struct LayerPackageAndComponents<'a>(pub &'a SolvedRequest, pub Vec); From 1e8f85f8aaf15ab988f98b69022e34ca4289b600 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Mon, 3 Feb 2025 09:31:43 -0800 Subject: [PATCH 12/64] Exclude source packages from solves For now just focusing on solving for a runtime environment with existing builds. Signed-off-by: J Robert Ray --- Cargo.lock | 1 + crates/spk-solve/Cargo.toml | 11 +++++---- .../resolvo/pkg_request_version_set.rs | 3 ++- .../src/solvers/resolvo/resolvo_tests.rs | 23 +++++++++++++++++++ .../src/solvers/resolvo/spk_provider.rs | 10 +++++++- 5 files changed, 41 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ee740a0f9e..f044945b5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4540,6 +4540,7 @@ dependencies = [ "thiserror", "tokio", "tracing", + "variantly", ] [[package]] diff --git a/crates/spk-solve/Cargo.toml b/crates/spk-solve/Cargo.toml index e4cc9218d5..9da88e23a9 100644 --- a/crates/spk-solve/Cargo.toml +++ b/crates/spk-solve/Cargo.toml @@ -37,27 +37,28 @@ ctrlc = "3.2" dyn-clone = { workspace = true } futures = { workspace = true } itertools = { workspace = true } -once_cell = { workspace = true } -priority-queue = "1.2" +miette = { workspace = true } num-bigint = "0.4.3" num-format = { version = "0.4.4", features = ["with-num-bigint"] } +once_cell = { workspace = true } +priority-queue = "1.2" resolvo = { workspace = true, features = ["tokio"] } -serde_json = { workspace = true } sentry = { workspace = true, optional = true } +serde_json = { workspace = true } signal-hook = "0.3" spfs = { workspace = true } spk-config = { workspace = true } +spk-schema = { workspace = true } spk-solve-graph = { workspace = true } spk-solve-package-iterator = { workspace = true } spk-solve-solution = { workspace = true } spk-solve-validation = { workspace = true } -spk-schema = { workspace = true } spk-storage = { workspace = true } statsd = { version = "0.15.0", optional = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt"] } tracing = { workspace = true } -miette = { workspace = true } +variantly = { workspace = true } [dev-dependencies] rstest = { workspace = true } diff --git a/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs b/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs index 64e0ecb6dd..644691d4aa 100644 --- a/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs +++ b/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs @@ -8,6 +8,7 @@ use resolvo::utils::VersionSet; use spk_schema::Request; use spk_schema::ident::LocatedBuildIdent; use spk_schema::ident_component::Component; +use variantly::Variantly; /// This allows for storing strings of different types but hash and compare by /// the underlying strings. @@ -87,7 +88,7 @@ impl std::fmt::Display for RequestVS { } /// Like `Component` but without the `All` variant. -#[derive(Clone, Debug, Eq, Hash, PartialEq)] +#[derive(Clone, Debug, Eq, Hash, PartialEq, Variantly)] pub(crate) enum ComponentWithoutAll { Build, Run, diff --git a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs index 2351a9d7bc..dd64eb3b51 100644 --- a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs +++ b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs @@ -192,3 +192,26 @@ async fn global_vars(#[case] global_spec: &str, #[case] expected_color: &str) { expected_color ); } + +#[rstest] +#[tokio::test] +async fn package_with_source_build() { + let repo = make_repo!( + [ + {"pkg": "dep/1.0.0/src"}, + {"pkg": "needs-dep/1.0.0", + "install": { + "requirements": [ + {"pkg": "dep"} + ] + } + }, + ] + ); + + let mut solver = Solver::new(vec![repo.into()], Cow::Borrowed(&[])); + solver + .solve(&[request!("needs-dep/1.0.0")]) + .await + .expect_err("src build should not satisfy dependency"); +} diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index 69b97b2592..f15a7cdeff 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -313,6 +313,7 @@ impl DependencyProvider for SpkProvider { }; for build in located_builds { + let is_src = build.component.is_source(); let solvable_id = *self .interned_solvables .borrow_mut() @@ -321,7 +322,14 @@ impl DependencyProvider for SpkProvider { self.pool .intern_solvable(name, SpkSolvable::LocatedBuildIdentWithComponent(build)) }); - candidates.candidates.push(solvable_id); + if is_src { + candidates.excluded.push(( + solvable_id, + self.pool.intern_string("source builds are excluded"), + )); + } else { + candidates.candidates.push(solvable_id); + } } Some(candidates) From d233bc9d3040c1e4da176536468077cdc9180b34 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Mon, 3 Feb 2025 12:03:27 -0800 Subject: [PATCH 13/64] Create an abstraction interface for solvers Start modifying the original solver tests to run the test against both solvers. Signed-off-by: J Robert Ray --- Cargo.lock | 4 ++ crates/spk-build/src/build/binary.rs | 2 +- crates/spk-cli/cmd-env/src/cmd_env.rs | 1 + crates/spk-cli/cmd-explain/Cargo.toml | 1 + crates/spk-cli/cmd-explain/src/cmd_explain.rs | 1 + crates/spk-cli/cmd-install/Cargo.toml | 1 + crates/spk-cli/cmd-install/src/cmd_install.rs | 1 + crates/spk-cli/cmd-render/Cargo.toml | 1 + crates/spk-cli/cmd-render/src/cmd_render.rs | 1 + crates/spk-cli/cmd-test/src/test/build.rs | 2 +- crates/spk-cli/cmd-test/src/test/install.rs | 2 +- crates/spk-cli/cmd-test/src/test/sources.rs | 2 +- crates/spk-cli/common/src/flags.rs | 1 + crates/spk-cli/group1/src/cmd_bake.rs | 1 + crates/spk-cli/group4/src/cmd_view.rs | 2 +- crates/spk-exec/src/exec_test.rs | 2 +- crates/spk-solve/Cargo.toml | 1 + crates/spk-solve/src/lib.rs | 6 +- crates/spk-solve/src/solver.rs | 31 +++++++++ crates/spk-solve/src/solvers/mod.rs | 1 + crates/spk-solve/src/solvers/resolvo/mod.rs | 29 +++++++-- .../src/solvers/resolvo/resolvo_tests.rs | 28 +++++---- crates/spk-solve/src/solvers/solver_test.rs | 38 ++++++++++- crates/spk-solve/src/solvers/step/solver.rs | 63 ++++++++++--------- 24 files changed, 164 insertions(+), 58 deletions(-) create mode 100644 crates/spk-solve/src/solver.rs diff --git a/Cargo.lock b/Cargo.lock index f044945b5d..65c9b582d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4220,6 +4220,7 @@ dependencies = [ "miette", "spfs", "spk-cli-common", + "spk-solve", "tokio", "tracing", ] @@ -4236,6 +4237,7 @@ dependencies = [ "spk-cli-common", "spk-exec", "spk-schema", + "spk-solve", "tokio", ] @@ -4300,6 +4302,7 @@ dependencies = [ "spfs", "spk-cli-common", "spk-exec", + "spk-solve", "spk-storage", "tokio", "tracing", @@ -4514,6 +4517,7 @@ dependencies = [ "crossterm", "ctrlc", "dyn-clone", + "enum_dispatch", "futures", "itertools 0.14.0", "miette", diff --git a/crates/spk-build/src/build/binary.rs b/crates/spk-build/src/build/binary.rs index 7f6d5aa54d..31198a8f27 100644 --- a/crates/spk-build/src/build/binary.rs +++ b/crates/spk-build/src/build/binary.rs @@ -37,7 +37,7 @@ use spk_schema::{ }; use spk_solve::graph::Graph; use spk_solve::solution::Solution; -use spk_solve::{BoxedResolverCallback, Named, ResolverCallback, StepSolver}; +use spk_solve::{BoxedResolverCallback, Named, ResolverCallback, Solver, StepSolver}; use spk_storage as storage; use crate::report::{BuildOutputReport, BuildReport, BuildSetupReport}; diff --git a/crates/spk-cli/cmd-env/src/cmd_env.rs b/crates/spk-cli/cmd-env/src/cmd_env.rs index caaac453f1..25d815878a 100644 --- a/crates/spk-cli/cmd-env/src/cmd_env.rs +++ b/crates/spk-cli/cmd-env/src/cmd_env.rs @@ -11,6 +11,7 @@ use spfs::tracking::SpecFile; use spfs_cli_common::Progress; use spk_cli_common::{CommandArgs, Run, build_required_packages, flags}; use spk_exec::setup_runtime_with_reporter; +use spk_solve::Solver; #[cfg(feature = "statsd")] use spk_solve::{SPK_RUN_TIME_METRIC, get_metrics_client}; diff --git a/crates/spk-cli/cmd-explain/Cargo.toml b/crates/spk-cli/cmd-explain/Cargo.toml index 7522bacb99..82a0e8bcdb 100644 --- a/crates/spk-cli/cmd-explain/Cargo.toml +++ b/crates/spk-cli/cmd-explain/Cargo.toml @@ -17,6 +17,7 @@ miette = { workspace = true, features = ["fancy"] } async-trait = { workspace = true } clap = { workspace = true } spk-cli-common = { workspace = true } +spk-solve = { workspace = true } # The dependency on spfs can be removed after the deprecated runtime flags are # removed. spfs = { workspace = true } diff --git a/crates/spk-cli/cmd-explain/src/cmd_explain.rs b/crates/spk-cli/cmd-explain/src/cmd_explain.rs index a02da56a8b..cc943762bf 100644 --- a/crates/spk-cli/cmd-explain/src/cmd_explain.rs +++ b/crates/spk-cli/cmd-explain/src/cmd_explain.rs @@ -5,6 +5,7 @@ use clap::Args; use miette::Result; use spk_cli_common::{CommandArgs, Run, flags}; +use spk_solve::Solver; /// Show the resolve process for a set of packages. #[derive(Args)] diff --git a/crates/spk-cli/cmd-install/Cargo.toml b/crates/spk-cli/cmd-install/Cargo.toml index 5d63928b26..fe0e204f31 100644 --- a/crates/spk-cli/cmd-install/Cargo.toml +++ b/crates/spk-cli/cmd-install/Cargo.toml @@ -21,4 +21,5 @@ futures = { workspace = true } spk-cli-common = { workspace = true } spk-exec = { workspace = true } spk-schema = { workspace = true } +spk-solve = { workspace = true } tokio = { workspace = true, features = ["rt"] } diff --git a/crates/spk-cli/cmd-install/src/cmd_install.rs b/crates/spk-cli/cmd-install/src/cmd_install.rs index 9dbb50d9ed..4ae4dcd6a2 100644 --- a/crates/spk-cli/cmd-install/src/cmd_install.rs +++ b/crates/spk-cli/cmd-install/src/cmd_install.rs @@ -14,6 +14,7 @@ use spk_exec::setup_current_runtime; use spk_schema::Package; use spk_schema::foundation::format::FormatIdent; use spk_schema::foundation::spec_ops::Named; +use spk_solve::Solver; /// Install a package into the current environment #[derive(Args)] diff --git a/crates/spk-cli/cmd-render/Cargo.toml b/crates/spk-cli/cmd-render/Cargo.toml index 30f14179bd..a709f34853 100644 --- a/crates/spk-cli/cmd-render/Cargo.toml +++ b/crates/spk-cli/cmd-render/Cargo.toml @@ -20,6 +20,7 @@ dunce = { workspace = true } spfs = { workspace = true } spk-cli-common = { workspace = true } spk-exec = { workspace = true } +spk-solve = { workspace = true } spk-storage = { workspace = true } tokio = { workspace = true, features = ["rt"] } tracing = { workspace = true } diff --git a/crates/spk-cli/cmd-render/src/cmd_render.rs b/crates/spk-cli/cmd-render/src/cmd_render.rs index e35a09482c..b64670b695 100644 --- a/crates/spk-cli/cmd-render/src/cmd_render.rs +++ b/crates/spk-cli/cmd-render/src/cmd_render.rs @@ -9,6 +9,7 @@ use miette::{Context, IntoDiagnostic, Result, bail}; use spfs::storage::fallback::FallbackProxy; use spk_cli_common::{CommandArgs, Run, build_required_packages, flags}; use spk_exec::resolve_runtime_layers; +use spk_solve::Solver; /// Output the contents of an spk environment (/spfs) to a folder #[derive(Args)] diff --git a/crates/spk-cli/cmd-test/src/test/build.rs b/crates/spk-cli/cmd-test/src/test/build.rs index d3c4e547c0..d26b484ef7 100644 --- a/crates/spk-cli/cmd-test/src/test/build.rs +++ b/crates/spk-cli/cmd-test/src/test/build.rs @@ -15,7 +15,7 @@ use spk_schema::foundation::option_map::OptionMap; use spk_schema::ident::{PkgRequest, PreReleasePolicy, RangeIdent, Request, RequestedBy}; use spk_schema::{AnyIdent, Recipe, SpecRecipe}; use spk_solve::solution::Solution; -use spk_solve::{BoxedResolverCallback, DefaultResolver, ResolverCallback, StepSolver}; +use spk_solve::{BoxedResolverCallback, DefaultResolver, ResolverCallback, Solver, StepSolver}; use spk_storage as storage; use super::Tester; diff --git a/crates/spk-cli/cmd-test/src/test/install.rs b/crates/spk-cli/cmd-test/src/test/install.rs index eac4b64ea5..f9dee2e4d9 100644 --- a/crates/spk-cli/cmd-test/src/test/install.rs +++ b/crates/spk-cli/cmd-test/src/test/install.rs @@ -12,7 +12,7 @@ use spk_schema::foundation::option_map::OptionMap; use spk_schema::ident::{PkgRequest, PreReleasePolicy, RangeIdent, Request, RequestedBy}; use spk_schema::ident_build::Build; use spk_schema::{Recipe, SpecRecipe, Variant, VariantExt}; -use spk_solve::{BoxedResolverCallback, DefaultResolver, ResolverCallback, StepSolver}; +use spk_solve::{BoxedResolverCallback, DefaultResolver, ResolverCallback, Solver, StepSolver}; use spk_storage as storage; use super::Tester; diff --git a/crates/spk-cli/cmd-test/src/test/sources.rs b/crates/spk-cli/cmd-test/src/test/sources.rs index bef95c641a..7282fe53e7 100644 --- a/crates/spk-cli/cmd-test/src/test/sources.rs +++ b/crates/spk-cli/cmd-test/src/test/sources.rs @@ -13,7 +13,7 @@ use spk_schema::foundation::ident_component::Component; use spk_schema::foundation::option_map::OptionMap; use spk_schema::ident::{PkgRequest, PreReleasePolicy, RangeIdent, Request, RequestedBy}; use spk_schema::{Recipe, SpecRecipe}; -use spk_solve::{BoxedResolverCallback, DefaultResolver, ResolverCallback, StepSolver}; +use spk_solve::{BoxedResolverCallback, DefaultResolver, ResolverCallback, Solver, StepSolver}; use spk_storage as storage; use super::Tester; diff --git a/crates/spk-cli/common/src/flags.rs b/crates/spk-cli/common/src/flags.rs index 3c5954313a..ab06c67af5 100644 --- a/crates/spk-cli/common/src/flags.rs +++ b/crates/spk-cli/common/src/flags.rs @@ -15,6 +15,7 @@ use solve::{ DecisionFormatter, DecisionFormatterBuilder, MultiSolverKind, + Solver as SolverTrait, }; use spk_schema::foundation::format::FormatIdent; use spk_schema::foundation::ident_build::Build; diff --git a/crates/spk-cli/group1/src/cmd_bake.rs b/crates/spk-cli/group1/src/cmd_bake.rs index 0f992f3a1a..a26a0396c5 100644 --- a/crates/spk-cli/group1/src/cmd_bake.rs +++ b/crates/spk-cli/group1/src/cmd_bake.rs @@ -9,6 +9,7 @@ use serde::Serialize; use spk_cli_common::{CommandArgs, Run, current_env, flags}; use spk_schema::Package; use spk_schema::ident::RequestedBy; +use spk_solve::Solver; use spk_solve::solution::{LayerPackageAndComponents, PackageSource, get_spfs_layers_to_packages}; #[cfg(test)] diff --git a/crates/spk-cli/group4/src/cmd_view.rs b/crates/spk-cli/group4/src/cmd_view.rs index 18d96a2201..b97ee88726 100644 --- a/crates/spk-cli/group4/src/cmd_view.rs +++ b/crates/spk-cli/group4/src/cmd_view.rs @@ -33,8 +33,8 @@ use spk_schema::{ Variant, VersionIdent, }; -use spk_solve::Recipe; use spk_solve::solution::{LayerPackageAndComponents, get_spfs_layers_to_packages}; +use spk_solve::{Recipe, Solver}; use spk_storage; use strum::{Display, EnumString, IntoEnumIterator, VariantNames}; diff --git a/crates/spk-exec/src/exec_test.rs b/crates/spk-exec/src/exec_test.rs index bb085ef1c4..2444a5d724 100644 --- a/crates/spk-exec/src/exec_test.rs +++ b/crates/spk-exec/src/exec_test.rs @@ -9,7 +9,7 @@ use rstest::{fixture, rstest}; use spk_cmd_build::build_package; use spk_schema::foundation::fixtures::*; use spk_schema::ident::build_ident; -use spk_solve::{DecisionFormatterBuilder, StepSolver}; +use spk_solve::{DecisionFormatterBuilder, Solver, StepSolver}; use spk_solve_macros::request; use spk_storage::fixtures::*; diff --git a/crates/spk-solve/Cargo.toml b/crates/spk-solve/Cargo.toml index 9da88e23a9..83b98d8822 100644 --- a/crates/spk-solve/Cargo.toml +++ b/crates/spk-solve/Cargo.toml @@ -35,6 +35,7 @@ console = { workspace = true } crossterm = "0.28.1" ctrlc = "3.2" dyn-clone = { workspace = true } +enum_dispatch = { workspace = true } futures = { workspace = true } itertools = { workspace = true } miette = { workspace = true } diff --git a/crates/spk-solve/src/lib.rs b/crates/spk-solve/src/lib.rs index d65a6e5d45..991a78be83 100644 --- a/crates/spk-solve/src/lib.rs +++ b/crates/spk-solve/src/lib.rs @@ -7,6 +7,7 @@ mod io; #[cfg(feature = "statsd")] mod metrics; mod search_space; +mod solver; mod solvers; mod status_line; @@ -34,8 +35,9 @@ pub use metrics::{ get_metrics_client, }; pub(crate) use search_space::show_search_space_stats; -// Publicly exported CdclSolver to stop dead code warnings -pub use solvers::resolvo::Solver as CdclSolver; +pub use solver::Solver; +// Publicly exported ResolvoSolver to stop dead code warnings +pub use solvers::resolvo::Solver as ResolvoSolver; pub use solvers::{StepSolver, StepSolverRuntime}; pub use spk_schema::foundation::ident_build::Build; pub use spk_schema::foundation::ident_component::Component; diff --git a/crates/spk-solve/src/solver.rs b/crates/spk-solve/src/solver.rs new file mode 100644 index 0000000000..a3a62e4410 --- /dev/null +++ b/crates/spk-solve/src/solver.rs @@ -0,0 +1,31 @@ +// Copyright (c) Contributors to the SPK project. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/spkenv/spk + +use std::sync::Arc; + +use enum_dispatch::enum_dispatch; +use spk_schema::{OptionMap, Request}; +use spk_storage::RepositoryHandle; +use variantly::Variantly; + +#[enum_dispatch(Solver)] +#[derive(Variantly)] +pub(crate) enum SolverImpl { + Step(crate::StepSolver), + Resolvo(crate::solvers::ResolvoSolver), +} + +#[async_trait::async_trait] +#[enum_dispatch] +pub trait Solver { + /// Add a repository where the solver can get packages. + fn add_repository(&mut self, repo: R) + where + R: Into>; + + /// Add a request to this solver. + fn add_request(&mut self, request: Request); + + fn update_options(&mut self, options: OptionMap); +} diff --git a/crates/spk-solve/src/solvers/mod.rs b/crates/spk-solve/src/solvers/mod.rs index 864cf1c32c..e2de48444c 100644 --- a/crates/spk-solve/src/solvers/mod.rs +++ b/crates/spk-solve/src/solvers/mod.rs @@ -7,6 +7,7 @@ pub(crate) mod resolvo; pub(crate) mod step; +pub use resolvo::Solver as ResolvoSolver; pub use step::{ErrorFreq, Solver as StepSolver, SolverRuntime as StepSolverRuntime}; // Public to allow other tests to use its macros diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index 2e4cf8e7bd..1ddf820a3a 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -22,22 +22,24 @@ use std::sync::Arc; use pkg_request_version_set::SpkSolvable; use spk_provider::SpkProvider; -use spk_schema::Request; use spk_schema::ident::{InclusionPolicy, PinPolicy, PkgRequest, RangeIdent}; use spk_schema::version_range::VersionFilter; +use spk_schema::{OptionMap, Request}; use spk_solve_solution::{PackageSource, Solution}; use spk_solve_validation::Validators; use spk_storage::RepositoryHandle; +use crate::solver::Solver as SolverTrait; use crate::{Error, Result}; #[cfg(test)] #[path = "resolvo_tests.rs"] mod resolvo_tests; -#[derive(Clone)] +#[derive(Clone, Default)] pub struct Solver { repos: Vec>, + requests: Vec, _validators: Cow<'static, [Validators]>, } @@ -45,14 +47,14 @@ impl Solver { pub fn new(repos: Vec>, validators: Cow<'static, [Validators]>) -> Self { Self { repos, + requests: Vec::new(), _validators: validators, } } - pub async fn solve(&mut self, requests: &[Request]) -> Result { + pub async fn solve(&self) -> Result { let repos = self.repos.clone(); - // XXX: Taking a slice reference doesn't make sense anymore. - let requests = requests.to_vec(); + let requests = self.requests.clone(); // Use a blocking thread so resolvo can call `block_on` on the runtime. let solvables = tokio::task::spawn_blocking(move || { let mut provider = Some(SpkProvider::new(repos.clone())); @@ -149,3 +151,20 @@ impl Solver { Ok(solution) } } + +impl SolverTrait for Solver { + fn add_repository(&mut self, repo: R) + where + R: Into>, + { + self.repos.push(repo.into()); + } + + fn add_request(&mut self, request: Request) { + self.requests.push(request); + } + + fn update_options(&mut self, _options: OptionMap) { + // TODO + } +} diff --git a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs index dd64eb3b51..fba0e89e08 100644 --- a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs +++ b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs @@ -10,6 +10,7 @@ use spk_schema::{Package, opt_name}; use spk_solve_macros::{make_repo, request}; use super::Solver; +use crate::Solver as SolverTrait; #[rstest] #[tokio::test] @@ -21,7 +22,8 @@ async fn basic() { ); let mut solver = Solver::new(vec![repo.into()], Cow::Borrowed(&[])); - let solution = solver.solve(&[request!("basic")]).await.unwrap(); + solver.add_request(request!("basic")); + let solution = solver.solve().await.unwrap(); assert_eq!(solution.len(), 1); } @@ -36,7 +38,8 @@ async fn two_choices() { ); let mut solver = Solver::new(vec![repo.into()], Cow::Borrowed(&[])); - let solution = solver.solve(&[request!("basic")]).await.unwrap(); + solver.add_request(request!("basic")); + let solution = solver.solve().await.unwrap(); assert_eq!(solution.len(), 1); // All things being equal it should pick the higher version assert_eq!( @@ -56,7 +59,8 @@ async fn two_choices_request_lower() { ); let mut solver = Solver::new(vec![repo.into()], Cow::Borrowed(&[])); - let solution = solver.solve(&[request!("basic/1.0.0")]).await.unwrap(); + solver.add_request(request!("basic/1.0.0")); + let solution = solver.solve().await.unwrap(); assert_eq!(solution.len(), 1); assert_eq!( solution.items().next().unwrap().spec.version().to_string(), @@ -75,10 +79,8 @@ async fn two_choices_request_missing() { ); let mut solver = Solver::new(vec![repo.into()], Cow::Borrowed(&[])); - let _solution = solver - .solve(&[request!("basic/1.0.0")]) - .await - .expect_err("Nothing satisfies 1.0.0"); + solver.add_request(request!("basic/1.0.0")); + let _solution = solver.solve().await.expect_err("Nothing satisfies 1.0.0"); } #[rstest] @@ -98,7 +100,8 @@ async fn package_with_dependency() { ); let mut solver = Solver::new(vec![repo.into()], Cow::Borrowed(&[])); - let solution = solver.solve(&[request!("needs-dep/1.0.0")]).await.unwrap(); + solver.add_request(request!("needs-dep/1.0.0")); + let solution = solver.solve().await.unwrap(); assert_eq!(solution.len(), 2); } @@ -140,7 +143,8 @@ async fn package_with_dependency_on_variant( ); let mut solver = Solver::new(vec![repo.into()], Cow::Borrowed(&[])); - let solution = solver.solve(&[request!("needs-dep/1.0.0")]).await.unwrap(); + solver.add_request(request!("needs-dep/1.0.0")); + let solution = solver.solve().await.unwrap(); assert_eq!(solution.len(), 2); let dep = solution.get("dep").unwrap(); assert_eq!( @@ -184,7 +188,8 @@ async fn global_vars(#[case] global_spec: &str, #[case] expected_color: &str) { ); let mut solver = Solver::new(vec![repo.into()], Cow::Borrowed(&[])); - let solution = solver.solve(&[request!("needs-dep/1.0.0")]).await.unwrap(); + solver.add_request(request!("needs-dep/1.0.0")); + let solution = solver.solve().await.unwrap(); assert_eq!(solution.len(), 2); let dep = solution.get("dep").unwrap(); assert_eq!( @@ -210,8 +215,9 @@ async fn package_with_source_build() { ); let mut solver = Solver::new(vec![repo.into()], Cow::Borrowed(&[])); + solver.add_request(request!("needs-dep/1.0.0")); solver - .solve(&[request!("needs-dep/1.0.0")]) + .solve() .await .expect_err("src build should not satisfy dependency"); } diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index 0e716d9b4b..f5c5255ab0 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -28,8 +28,9 @@ use spk_storage::RepositoryHandle; use spk_storage::fixtures::*; use crate::io::DecisionFormatterBuilder; +use crate::solver::{Solver, SolverImpl}; use crate::solvers::step::{ErrorDetails, ErrorFreq}; -use crate::{Error, Result, StepSolver, option_map, spec}; +use crate::{Error, ResolvoSolver, Result, StepSolver, option_map, spec}; #[fixture] fn solver() -> StepSolver { @@ -115,6 +116,25 @@ async fn run_and_print_resolve_for_tests(solver: &StepSolver) -> Result Result { + match solver { + SolverImpl::Step(solver) => { + let formatter = DecisionFormatterBuilder::default() + .with_verbosity(100) + .build(); + + let (solution, _) = formatter.run_and_print_resolve(solver).await?; + Ok(solution) + } + + SolverImpl::Resolvo(solver) => solver.solve().await, + } +} + /// Runs the given solver, logging the output with reasonable output settings /// for unit test debugging and inspection. async fn run_and_log_resolve_for_tests(solver: &StepSolver) -> Result { @@ -296,9 +316,19 @@ async fn test_solver_package_with_no_recipe_from_cmd_line_and_impossible_initial } } +fn step_solver() -> SolverImpl { + SolverImpl::Step(StepSolver::default()) +} + +fn resolvo_solver() -> SolverImpl { + SolverImpl::Resolvo(ResolvoSolver::default()) +} + #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_single_package_no_deps(mut solver: StepSolver) { +async fn test_solver_single_package_no_deps(#[case] mut solver: SolverImpl) { let options = option_map! {}; let repo = make_repo!([{"pkg": "my-pkg/1.0.0"}], options=options.clone()); @@ -306,7 +336,9 @@ async fn test_solver_single_package_no_deps(mut solver: StepSolver) { solver.add_repository(Arc::new(repo)); solver.add_request(request!("my-pkg")); - let packages = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let packages = run_and_print_resolve_for_tests_with_abstract_solver(&solver) + .await + .unwrap(); assert_eq!(packages.len(), 1, "expected one resolved package"); let resolved = packages.get("my-pkg").unwrap(); assert_eq!(&resolved.spec.version().to_string(), "1.0.0"); diff --git a/crates/spk-solve/src/solvers/step/solver.rs b/crates/spk-solve/src/solvers/step/solver.rs index 868472f812..64bc565841 100644 --- a/crates/spk-solve/src/solvers/step/solver.rs +++ b/crates/spk-solve/src/solvers/step/solver.rs @@ -55,6 +55,7 @@ use spk_storage::RepositoryHandle; use crate::error::OutOfOptions; use crate::option_map::OptionMap; +use crate::solver::Solver as SolverTrait; use crate::{Error, Result, error}; /// Structure to hold whether the three kinds of impossible checks are @@ -205,32 +206,6 @@ impl ErrorFreq { } impl Solver { - /// Add a request to this solver. - pub fn add_request(&mut self, request: Request) { - let request = match request { - Request::Pkg(mut request) => { - if request.pkg.components.is_empty() { - if request.pkg.is_source() { - request.pkg.components.insert(Component::Source); - } else { - request.pkg.components.insert(Component::default_for_run()); - } - } - Change::RequestPackage(RequestPackage::new(request)) - } - Request::Var(request) => Change::RequestVar(RequestVar::new(request)), - }; - self.initial_state_builders.push(request); - } - - /// Add a repository where the solver can get packages. - pub fn add_repository(&mut self, repo: R) - where - R: Into>, - { - self.repos.push(repo.into()); - } - /// Return a reference to the solver's list of repositories. pub fn repositories(&self) -> &Vec> { &self.repos @@ -1164,11 +1139,6 @@ impl Solver { self.solve().await } - pub fn update_options(&mut self, options: OptionMap) { - self.initial_state_builders - .push(Change::SetOptions(SetOptions::new(options))) - } - /// Get the number of steps (forward) taken in the solve pub fn get_number_of_steps(&self) -> usize { self.number_of_steps @@ -1200,6 +1170,37 @@ impl Solver { } } +impl SolverTrait for Solver { + fn add_repository(&mut self, repo: R) + where + R: Into>, + { + self.repos.push(repo.into()); + } + + fn add_request(&mut self, request: Request) { + let request = match request { + Request::Pkg(mut request) => { + if request.pkg.components.is_empty() { + if request.pkg.is_source() { + request.pkg.components.insert(Component::Source); + } else { + request.pkg.components.insert(Component::default_for_run()); + } + } + Change::RequestPackage(RequestPackage::new(request)) + } + Request::Var(request) => Change::RequestVar(RequestVar::new(request)), + }; + self.initial_state_builders.push(request); + } + + fn update_options(&mut self, options: OptionMap) { + self.initial_state_builders + .push(Change::SetOptions(SetOptions::new(options))) + } +} + // This is needed so `PriorityQueue` doesn't need to hash the node itself. struct NodeWrapper { pub(crate) node: Arc>>, From 668878a2b5c3e296274f4d07924d21f99380409e Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Wed, 5 Feb 2025 19:58:51 -0800 Subject: [PATCH 14/64] Finish integrating cdcl solver into existing solver tests Not all solver tests are applicable. Where possible, for tests the cdcl solver doesn't pass yet, add #[should_panic] so the test suite still passes as a whole. As the cdcl solver can pass more tests, this will highlight the tests that can then be enabled for the cdcl solver. Signed-off-by: J Robert Ray --- crates/spk-solve/src/solver.rs | 18 + crates/spk-solve/src/solvers/resolvo/mod.rs | 41 +- crates/spk-solve/src/solvers/solver_test.rs | 480 +++++++++++++------- crates/spk-solve/src/solvers/step/solver.rs | 117 +++-- 4 files changed, 423 insertions(+), 233 deletions(-) diff --git a/crates/spk-solve/src/solver.rs b/crates/spk-solve/src/solver.rs index a3a62e4410..1e7745fe62 100644 --- a/crates/spk-solve/src/solver.rs +++ b/crates/spk-solve/src/solver.rs @@ -6,9 +6,12 @@ use std::sync::Arc; use enum_dispatch::enum_dispatch; use spk_schema::{OptionMap, Request}; +use spk_solve_solution::Solution; use spk_storage::RepositoryHandle; use variantly::Variantly; +use crate::Result; + #[enum_dispatch(Solver)] #[derive(Variantly)] pub(crate) enum SolverImpl { @@ -27,5 +30,20 @@ pub trait Solver { /// Add a request to this solver. fn add_request(&mut self, request: Request); + /// Put this solver back into its default state + fn reset(&mut self); + + /// If true, only solve pre-built binary packages. + /// + /// When false, the solver may return packages where the build is not set. + /// These packages are known to have a source package available, and the requested + /// options are valid for a new build of that source package. + /// These packages are not actually built as part of the solver process but their + /// build environments are fully resolved and dependencies included + fn set_binary_only(&mut self, binary_only: bool); + + /// Run the solver as configured. + async fn solve(&mut self) -> Result; + fn update_options(&mut self, options: OptionMap); } diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index 1ddf820a3a..aa58696b61 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -26,7 +26,7 @@ use spk_schema::ident::{InclusionPolicy, PinPolicy, PkgRequest, RangeIdent}; use spk_schema::version_range::VersionFilter; use spk_schema::{OptionMap, Request}; use spk_solve_solution::{PackageSource, Solution}; -use spk_solve_validation::Validators; +use spk_solve_validation::{Validators, default_validators}; use spk_storage::RepositoryHandle; use crate::solver::Solver as SolverTrait; @@ -51,8 +51,32 @@ impl Solver { _validators: validators, } } +} + +#[async_trait::async_trait] +impl SolverTrait for Solver { + fn add_repository(&mut self, repo: R) + where + R: Into>, + { + self.repos.push(repo.into()); + } + + fn add_request(&mut self, request: Request) { + self.requests.push(request); + } + + fn reset(&mut self) { + self.repos.truncate(0); + self.requests.truncate(0); + self._validators = Cow::from(default_validators()); + } - pub async fn solve(&self) -> Result { + fn set_binary_only(&mut self, _binary_only: bool) { + // TODO + } + + async fn solve(&mut self) -> Result { let repos = self.repos.clone(); let requests = self.requests.clone(); // Use a blocking thread so resolvo can call `block_on` on the runtime. @@ -150,19 +174,6 @@ impl Solver { } Ok(solution) } -} - -impl SolverTrait for Solver { - fn add_repository(&mut self, repo: R) - where - R: Into>, - { - self.repos.push(repo.into()); - } - - fn add_request(&mut self, request: Request) { - self.requests.push(request); - } fn update_options(&mut self, _options: OptionMap) { // TODO diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index f5c5255ab0..6d60944031 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -30,7 +30,7 @@ use spk_storage::fixtures::*; use crate::io::DecisionFormatterBuilder; use crate::solver::{Solver, SolverImpl}; use crate::solvers::step::{ErrorDetails, ErrorFreq}; -use crate::{Error, ResolvoSolver, Result, StepSolver, option_map, spec}; +use crate::{Error, ResolvoSolver, Result, Solution, StepSolver, option_map, spec}; #[fixture] fn solver() -> StepSolver { @@ -107,20 +107,7 @@ macro_rules! assert_not_resolved { /// Runs the given solver, printing the output with reasonable output settings /// for unit test debugging and inspection. -async fn run_and_print_resolve_for_tests(solver: &StepSolver) -> Result { - let formatter = DecisionFormatterBuilder::default() - .with_verbosity(100) - .build(); - - let (solution, _) = formatter.run_and_print_resolve(solver).await?; - Ok(solution) -} - -/// Runs the given solver, printing the output with reasonable output settings -/// for unit test debugging and inspection. -async fn run_and_print_resolve_for_tests_with_abstract_solver( - solver: &SolverImpl, -) -> Result { +async fn run_and_print_resolve_for_tests(solver: &mut SolverImpl) -> Result { match solver { SolverImpl::Step(solver) => { let formatter = DecisionFormatterBuilder::default() @@ -137,24 +124,44 @@ async fn run_and_print_resolve_for_tests_with_abstract_solver( /// Runs the given solver, logging the output with reasonable output settings /// for unit test debugging and inspection. -async fn run_and_log_resolve_for_tests(solver: &StepSolver) -> Result { - let formatter = DecisionFormatterBuilder::default() - .with_verbosity(100) - .build(); +async fn run_and_log_resolve_for_tests(solver: &mut SolverImpl) -> Result { + match solver { + SolverImpl::Step(solver) => { + let formatter = DecisionFormatterBuilder::default() + .with_verbosity(100) + .build(); - let (solution, _) = formatter.run_and_log_resolve(solver).await?; - Ok(solution) + let (solution, _) = formatter.run_and_log_resolve(solver).await?; + Ok(solution) + } + SolverImpl::Resolvo(solver) => solver.solve().await, + } +} + +fn step_solver() -> SolverImpl { + SolverImpl::Step(StepSolver::default()) +} + +fn resolvo_solver() -> SolverImpl { + SolverImpl::Resolvo(ResolvoSolver::default()) } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_no_requests(mut solver: StepSolver) { +async fn test_solver_no_requests(#[case] mut solver: SolverImpl) { solver.solve().await.unwrap(); } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_package_with_no_recipe(mut solver: StepSolver, random_build_id: BuildId) { +async fn test_solver_package_with_no_recipe( + #[case] mut solver: SolverImpl, + random_build_id: BuildId, +) { let repo = RepositoryHandle::new_mem(); let options = option_map! {}; @@ -176,7 +183,7 @@ async fn test_solver_package_with_no_recipe(mut solver: StepSolver, random_build solver.add_request(request!("my-pkg")); // Test - let res = run_and_print_resolve_for_tests(&solver).await; + let res = run_and_print_resolve_for_tests(&mut solver).await; assert!( res.is_ok(), "'{res:?}' should be an Ok(_) solution not an error.')" @@ -184,9 +191,11 @@ async fn test_solver_package_with_no_recipe(mut solver: StepSolver, random_build } #[rstest] +// This test is only applicable to the og solver +#[case::step(step_solver())] #[tokio::test] async fn test_solver_package_with_no_recipe_and_impossible_initial_checks( - mut solver: StepSolver, + #[case] mut solver: SolverImpl, random_build_id: BuildId, ) { init_logging(); @@ -204,10 +213,12 @@ async fn test_solver_package_with_no_recipe_and_impossible_initial_checks( solver.update_options(options); solver.add_repository(Arc::new(repo)); solver.add_request(request!("my-pkg")); - solver.set_initial_request_impossible_checks(true); + if let SolverImpl::Step(ref mut solver) = solver { + solver.set_initial_request_impossible_checks(true); + } // Test - let res = run_and_print_resolve_for_tests(&solver).await; + let res = run_and_print_resolve_for_tests(&mut solver).await; if cfg!(feature = "migration-to-components") { match res { Err(Error::InitialRequestsContainImpossibleError(_)) => { @@ -236,8 +247,10 @@ async fn test_solver_package_with_no_recipe_and_impossible_initial_checks( } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_package_with_no_recipe_from_cmd_line(mut solver: StepSolver) { +async fn test_solver_package_with_no_recipe_from_cmd_line(#[case] mut solver: SolverImpl) { let repo = RepositoryHandle::new_mem(); let spec = spec!({"pkg": "my-pkg/1.0.0/4OYMIQUY"}); @@ -260,7 +273,7 @@ async fn test_solver_package_with_no_recipe_from_cmd_line(mut solver: StepSolver solver.add_request(req); // Test - let res = run_and_print_resolve_for_tests(&solver).await; + let res = run_and_print_resolve_for_tests(&mut solver).await; assert!( res.is_ok(), "'{res:?}' should be an Ok(_) solution not an error.')" @@ -268,9 +281,11 @@ async fn test_solver_package_with_no_recipe_from_cmd_line(mut solver: StepSolver } #[rstest] +// This test is only applicable to the og solver +#[case::step(step_solver())] #[tokio::test] async fn test_solver_package_with_no_recipe_from_cmd_line_and_impossible_initial_checks( - mut solver: StepSolver, + #[case] mut solver: SolverImpl, ) { init_logging(); let repo = RepositoryHandle::new_mem(); @@ -290,10 +305,12 @@ async fn test_solver_package_with_no_recipe_from_cmd_line_and_impossible_initial RequestedBy::CommandLine, )); solver.add_request(req); - solver.set_initial_request_impossible_checks(true); + if let SolverImpl::Step(ref mut solver) = solver { + solver.set_initial_request_impossible_checks(true); + } // Test - let res = run_and_print_resolve_for_tests(&solver).await; + let res = run_and_print_resolve_for_tests(&mut solver).await; if cfg!(feature = "migration-to-components") { // with the 'migration-to-components' feature and impossible // request initial checks will fail because the feature turns @@ -316,14 +333,6 @@ async fn test_solver_package_with_no_recipe_from_cmd_line_and_impossible_initial } } -fn step_solver() -> SolverImpl { - SolverImpl::Step(StepSolver::default()) -} - -fn resolvo_solver() -> SolverImpl { - SolverImpl::Resolvo(ResolvoSolver::default()) -} - #[rstest] #[case::step(step_solver())] #[case::resolvo(resolvo_solver())] @@ -336,9 +345,7 @@ async fn test_solver_single_package_no_deps(#[case] mut solver: SolverImpl) { solver.add_repository(Arc::new(repo)); solver.add_request(request!("my-pkg")); - let packages = run_and_print_resolve_for_tests_with_abstract_solver(&solver) - .await - .unwrap(); + let packages = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_eq!(packages.len(), 1, "expected one resolved package"); let resolved = packages.get("my-pkg").unwrap(); assert_eq!(&resolved.spec.version().to_string(), "1.0.0"); @@ -346,8 +353,10 @@ async fn test_solver_single_package_no_deps(#[case] mut solver: SolverImpl) { } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_single_package_simple_deps(mut solver: StepSolver) { +async fn test_solver_single_package_simple_deps(#[case] mut solver: SolverImpl) { let options = option_map! {}; let repo = make_repo!( [ @@ -365,15 +374,17 @@ async fn test_solver_single_package_simple_deps(mut solver: StepSolver) { solver.add_repository(Arc::new(repo)); solver.add_request(request!("pkg-b/1.1")); - let packages = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let packages = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_eq!(packages.len(), 2, "expected two resolved packages"); assert_resolved!(packages, "pkg-a", "1.2.1"); assert_resolved!(packages, "pkg-b", "1.1.0"); } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_dependency_abi_compat(mut solver: StepSolver) { +async fn test_solver_dependency_abi_compat(#[case] mut solver: SolverImpl) { let options = option_map! {}; let repo = make_repo!( [ @@ -394,15 +405,17 @@ async fn test_solver_dependency_abi_compat(mut solver: StepSolver) { solver.add_repository(Arc::new(repo)); solver.add_request(request!("pkg-b/1.1")); - let packages = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let packages = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_eq!(packages.len(), 2, "expected two resolved packages"); assert_resolved!(packages, "pkg-a", "1.1.1"); assert_resolved!(packages, "pkg-b", "1.1.0"); } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_dependency_incompatible(mut solver: StepSolver) { +async fn test_solver_dependency_incompatible(#[case] mut solver: SolverImpl) { // test what happens when a dependency is added which is incompatible // with an existing request in the stack let repo = make_repo!( @@ -421,14 +434,16 @@ async fn test_solver_dependency_incompatible(mut solver: StepSolver) { // this one is incompatible with requirements of my-plugin but the solver doesn't know it yet solver.add_request(request!("maya/2019")); - let res = run_and_print_resolve_for_tests(&solver).await; + let res = run_and_print_resolve_for_tests(&mut solver).await; assert!(res.is_err()); } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_dependency_incompatible_stepback(mut solver: StepSolver) { +async fn test_solver_dependency_incompatible_stepback(#[case] mut solver: SolverImpl) { // test what happens when a dependency is added which is incompatible // with an existing request in the stack - in this case we want the solver // to successfully step back into an older package version with @@ -453,15 +468,17 @@ async fn test_solver_dependency_incompatible_stepback(mut solver: StepSolver) { // this one is incompatible with requirements of my-plugin/1.1.0 but not my-plugin/1.0 solver.add_request(request!("maya/2019")); - let packages = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let packages = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!(packages, "my-plugin", "1.0.0"); assert_resolved!(packages, "maya", "2019.0.0"); } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_dependency_already_satisfied(mut solver: StepSolver) { +async fn test_solver_dependency_already_satisfied(#[case] mut solver: SolverImpl) { // test what happens when a dependency is added which represents // a package which has already been resolved // - and the resolved version satisfies the request @@ -484,15 +501,21 @@ async fn test_solver_dependency_already_satisfied(mut solver: StepSolver) { solver.add_repository(Arc::new(repo)); solver.add_request(request!("pkg-top")); - let packages = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let packages = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!(packages, ["pkg-top", "dep-1", "dep-2"]); assert_resolved!(packages, "dep-1", "1.0.0"); } #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_dependency_already_satisfied_conflicting_components(mut solver: StepSolver) { +async fn test_solver_dependency_already_satisfied_conflicting_components( + #[case] mut solver: SolverImpl, +) { // like test_solver_dependency_already_satisfied but with conflicting components let repo = make_repo!( @@ -529,14 +552,16 @@ async fn test_solver_dependency_already_satisfied_conflicting_components(mut sol // How can this test code verify that the solver is actually hitting // that code path? - run_and_print_resolve_for_tests(&solver) + run_and_print_resolve_for_tests(&mut solver) .await .expect_err("solve should fail"); } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_dependency_reopen_solvable(mut solver: StepSolver) { +async fn test_solver_dependency_reopen_solvable(#[case] mut solver: SolverImpl) { // test what happens when a dependency is added which represents // a package which has already been resolved // - and the resolved version does not satisfy the request @@ -564,14 +589,16 @@ async fn test_solver_dependency_reopen_solvable(mut solver: StepSolver) { solver.add_repository(Arc::new(repo)); solver.add_request(request!("my-plugin")); - let packages = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let packages = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!(packages, ["my-plugin", "some-library", "maya"]); assert_resolved!(packages, "maya", "2019.0.0"); } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_dependency_reiterate(mut solver: StepSolver) { +async fn test_solver_dependency_reiterate(#[case] mut solver: SolverImpl) { // test what happens when a package iterator must be run through twice // - walking back up the solve graph should reset the iterator to where it was @@ -598,14 +625,16 @@ async fn test_solver_dependency_reiterate(mut solver: StepSolver) { solver.add_repository(Arc::new(repo)); solver.add_request(request!("my-plugin")); - let packages = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let packages = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!(packages, ["my-plugin", "some-library", "maya"]); assert_resolved!(packages, "maya", "2019.0.0"); } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_dependency_reopen_unsolvable(mut solver: StepSolver) { +async fn test_solver_dependency_reopen_unsolvable(#[case] mut solver: SolverImpl) { // test what happens when a dependency is added which represents // a package which has already been resolved // - and the resolved version does not satisfy the request @@ -631,13 +660,15 @@ async fn test_solver_dependency_reopen_unsolvable(mut solver: StepSolver) { solver.add_repository(Arc::new(repo)); solver.add_request(request!("pkg-top")); - let result = run_and_print_resolve_for_tests(&solver).await; + let result = run_and_print_resolve_for_tests(&mut solver).await; assert!(result.is_err()); } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_pre_release_config(mut solver: StepSolver) { +async fn test_solver_pre_release_config(#[case] mut solver: SolverImpl) { let repo = make_repo!( [ {"pkg": "my-pkg/0.9.0"}, @@ -651,7 +682,7 @@ async fn test_solver_pre_release_config(mut solver: StepSolver) { solver.add_repository(repo.clone()); solver.add_request(request!("my-pkg")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!( solution, "my-pkg", @@ -663,13 +694,17 @@ async fn test_solver_pre_release_config(mut solver: StepSolver) { solver.add_repository(repo); solver.add_request(request!({"pkg": "my-pkg", "prereleasePolicy": "IncludeAll"})); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!(solution, "my-pkg", "1.0.0-pre.2"); } #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_constraint_only(mut solver: StepSolver) { +async fn test_solver_constraint_only(#[case] mut solver: SolverImpl) { // test what happens when a dependency is marked as a constraint/optional // and no other request is added // - the constraint is noted @@ -690,13 +725,15 @@ async fn test_solver_constraint_only(mut solver: StepSolver) { solver.add_repository(Arc::new(repo)); solver.add_request(request!("vnp3")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert!(solution.get("python").is_none()); } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_constraint_and_request(mut solver: StepSolver) { +async fn test_solver_constraint_and_request(#[case] mut solver: SolverImpl) { // test what happens when a dependency is marked as a constraint/optional // and also requested by another package // - the constraint is noted @@ -723,14 +760,18 @@ async fn test_solver_constraint_and_request(mut solver: StepSolver) { solver.add_repository(Arc::new(repo)); solver.add_request(request!("my-tool")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!(solution, "python", "3.7.3"); } #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_option_compatibility(mut solver: StepSolver) { +async fn test_solver_option_compatibility(#[case] mut solver: SolverImpl) { // test what happens when an option is given in the solver // - the options for each build are checked // - the resolved build must have used the option @@ -787,7 +828,7 @@ async fn test_solver_option_compatibility(mut solver: StepSolver) { .into(), ); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); let resolved = solution.get("vnp3").unwrap(); let value = resolved @@ -810,8 +851,12 @@ async fn test_solver_option_compatibility(mut solver: StepSolver) { } #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_option_injection(mut solver: StepSolver) { +async fn test_solver_option_injection(#[case] mut solver: SolverImpl) { // test the options that are defined when a package is resolved // - options are namespaced and added to the environment init_logging(); @@ -842,7 +887,7 @@ async fn test_solver_option_injection(mut solver: StepSolver) { solver.add_repository(Arc::new(repo)); solver.add_request(request!("vnp3")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); let mut opts = solution.options().clone(); assert_eq!(opts.remove(opt_name!("vnp3")), Some("~2.0.0".to_string())); @@ -863,8 +908,12 @@ async fn test_solver_option_injection(mut solver: StepSolver) { } #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_build_from_source(mut solver: StepSolver) { +async fn test_solver_build_from_source(#[case] mut solver: SolverImpl) { init_logging(); // test when no appropriate build exists but the source is available // - the build is skipped @@ -893,7 +942,7 @@ async fn test_solver_build_from_source(mut solver: StepSolver) { solver.add_request(request!({"var": "debug/on"})); solver.add_request(request!("my-tool")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); let resolved = solution.get("my-tool").unwrap(); assert!( @@ -909,14 +958,18 @@ async fn test_solver_build_from_source(mut solver: StepSolver) { solver.set_binary_only(true); // Should fail when binary-only is specified - let res = run_and_print_resolve_for_tests(&solver).await; + let res = run_and_print_resolve_for_tests(&mut solver).await; assert!(res.is_err()); } #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_build_from_source_unsolvable(mut solver: StepSolver) { +async fn test_solver_build_from_source_unsolvable(#[case] mut solver: SolverImpl) { let log = init_logging(); // test when no appropriate build exists but the source is available // - if the requested pkg cannot resolve a build environment @@ -952,7 +1005,7 @@ async fn test_solver_build_from_source_unsolvable(mut solver: StepSolver) { solver.add_request(request!({"var": "gcc/6.3"})); solver.add_request(request!("my-tool:run")); - let res = run_and_log_resolve_for_tests(&solver).await; + let res = run_and_log_resolve_for_tests(&mut solver).await; assert!(res.is_err(), "should fail to resolve"); let log = log.lock(); @@ -973,8 +1026,12 @@ async fn test_solver_build_from_source_unsolvable(mut solver: StepSolver) { } #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_build_from_source_dependency(mut solver: StepSolver) { +async fn test_solver_build_from_source_dependency(#[case] mut solver: SolverImpl) { // test when no appropriate build exists but the source is available // - the existing build is skipped // - the source package is checked for current options @@ -1021,7 +1078,7 @@ async fn test_solver_build_from_source_dependency(mut solver: StepSolver) { solver.add_request(request!("my-tool")); solver.set_binary_only(false); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert!( solution.get("my-tool").unwrap().is_source_build(), @@ -1030,8 +1087,10 @@ async fn test_solver_build_from_source_dependency(mut solver: StepSolver) { } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_deprecated_build(mut solver: StepSolver) { +async fn test_solver_deprecated_build(#[case] mut solver: SolverImpl) { let deprecated = make_build!({"pkg": "my-pkg/1.0.0", "deprecated": true}); let deprecated_build = deprecated.ident().clone(); let repo = make_repo!([ @@ -1044,7 +1103,7 @@ async fn test_solver_deprecated_build(mut solver: StepSolver) { solver.add_repository(repo.clone()); solver.add_request(request!("my-pkg")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!( solution, "my-pkg", @@ -1062,7 +1121,7 @@ async fn test_solver_deprecated_build(mut solver: StepSolver) { .into(), ); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!( solution, "my-pkg", @@ -1072,8 +1131,10 @@ async fn test_solver_deprecated_build(mut solver: StepSolver) { } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_deprecated_version(mut solver: StepSolver) { +async fn test_solver_deprecated_version(#[case] mut solver: SolverImpl) { let deprecated = make_build!({"pkg": "my-pkg/1.0.0", "deprecated": true}); let repo = make_repo!( [{"pkg": "my-pkg/0.9.0"}, {"pkg": "my-pkg/1.0.0", "deprecated": true}, deprecated] @@ -1083,7 +1144,7 @@ async fn test_solver_deprecated_version(mut solver: StepSolver) { solver.add_repository(repo.clone()); solver.add_request(request!("my-pkg")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!( solution, "my-pkg", @@ -1101,7 +1162,7 @@ async fn test_solver_deprecated_version(mut solver: StepSolver) { .into(), ); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!( solution, "my-pkg", @@ -1111,8 +1172,12 @@ async fn test_solver_deprecated_version(mut solver: StepSolver) { } #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_build_from_source_deprecated(mut solver: StepSolver) { +async fn test_solver_build_from_source_deprecated(#[case] mut solver: SolverImpl) { // test when no appropriate build exists and the main package // has been deprecated, no source build should be allowed @@ -1141,7 +1206,7 @@ async fn test_solver_build_from_source_deprecated(mut solver: StepSolver) { solver.add_request(request!({"var": "debug/on"})); solver.add_request(request!("my-tool")); - let res = run_and_print_resolve_for_tests(&solver).await; + let res = run_and_print_resolve_for_tests(&mut solver).await; match res { Err(Error::GraphError(ref graph_err)) if matches!(&**graph_err, spk_solve_graph::Error::FailedToResolve(_)) => {} @@ -1153,9 +1218,11 @@ async fn test_solver_build_from_source_deprecated(mut solver: StepSolver) { } #[rstest] +// This test is only applicable to the og solver +#[case::step(step_solver())] #[tokio::test] async fn test_solver_build_from_source_deprecated_and_impossible_initial_checks( - mut solver: StepSolver, + #[case] mut solver: SolverImpl, ) { // test when no appropriate build exists and the main package // has been deprecated, no source build should be allowed @@ -1184,9 +1251,11 @@ async fn test_solver_build_from_source_deprecated_and_impossible_initial_checks( solver.add_repository(Arc::new(repo)); solver.add_request(request!({"var": "debug/on"})); solver.add_request(request!("my-tool")); - solver.set_initial_request_impossible_checks(true); + if let SolverImpl::Step(ref mut solver) = solver { + solver.set_initial_request_impossible_checks(true); + } - let res = run_and_print_resolve_for_tests(&solver).await; + let res = run_and_print_resolve_for_tests(&mut solver).await; match res { Err(Error::GraphError(ref graph_err)) if matches!(&**graph_err, spk_solve_graph::Error::FailedToResolve(_)) => @@ -1214,8 +1283,12 @@ async fn test_solver_build_from_source_deprecated_and_impossible_initial_checks( } #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_embedded_package_adds_request(mut solver: StepSolver) { +async fn test_solver_embedded_package_adds_request(#[case] mut solver: SolverImpl) { // test when there is an embedded package // - the embedded package is added to the solution // - the embedded package is also added as a request in the resolve @@ -1233,7 +1306,7 @@ async fn test_solver_embedded_package_adds_request(mut solver: StepSolver) { solver.add_repository(Arc::new(repo)); solver.add_request(request!("maya")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!( solution, @@ -1249,8 +1322,12 @@ async fn test_solver_embedded_package_adds_request(mut solver: StepSolver) { } #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_embedded_package_solvable(mut solver: StepSolver) { +async fn test_solver_embedded_package_solvable(#[case] mut solver: SolverImpl) { // test when there is an embedded package // - the embedded package is added to the solution // - the embedded package resolves existing requests @@ -1274,7 +1351,7 @@ async fn test_solver_embedded_package_solvable(mut solver: StepSolver) { solver.add_request(request!("qt")); solver.add_request(request!("maya")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!(solution, "qt", "5.12.6"); assert_resolved!( @@ -1285,8 +1362,12 @@ async fn test_solver_embedded_package_solvable(mut solver: StepSolver) { } #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_embedded_package_unsolvable(mut solver: StepSolver) { +async fn test_solver_embedded_package_unsolvable(#[case] mut solver: SolverImpl) { // test when there is an embedded package // - the embedded package is added to the solution // - the embedded package conflicts with existing requests @@ -1313,13 +1394,16 @@ async fn test_solver_embedded_package_unsolvable(mut solver: StepSolver) { solver.add_repository(Arc::new(repo)); solver.add_request(request!("my-plugin")); - let res = run_and_print_resolve_for_tests(&solver).await; + let res = run_and_print_resolve_for_tests(&mut solver).await; assert!(res.is_err()); } #[rstest] +#[case::step(step_solver())] +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_embedded_package_replaces_real_package(mut solver: StepSolver) { +async fn test_solver_embedded_package_replaces_real_package(#[case] mut solver: SolverImpl) { // test when there is an embedded package // - the embedded package is added to the solution // - any dependencies from the "real" package aren't part of the solution @@ -1362,7 +1446,7 @@ async fn test_solver_embedded_package_replaces_real_package(mut solver: StepSolv // "unwanted-dep" is added to solution. solver.add_request(request!("thing-needs-plugin")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); // At time of writing, this is a point where "unwanted-dep" is part of the // solution: @@ -1378,9 +1462,11 @@ async fn test_solver_embedded_package_replaces_real_package(mut solver: StepSolv } #[rstest] +// This test is only applicable to the og solver +#[case::step(step_solver())] #[tokio::test] async fn test_solver_initial_request_impossible_masks_embedded_package_solution( - mut solver: StepSolver, + #[case] mut solver: SolverImpl, ) { // test when an embedded package and its parent package are // requested and impossible checks are enabled for initial @@ -1410,9 +1496,11 @@ async fn test_solver_initial_request_impossible_masks_embedded_package_solution( // requests work correctly. solver.add_request(request!("qt/5.12.6")); solver.add_request(request!("maya")); - solver.set_initial_request_impossible_checks(true); + if let SolverImpl::Step(ref mut solver) = solver { + solver.set_initial_request_impossible_checks(true); + } - match run_and_print_resolve_for_tests(&solver).await { + match run_and_print_resolve_for_tests(&mut solver).await { Ok(solution) => { assert_resolved!(solution, "qt", "5.12.6"); assert_resolved!( @@ -1428,9 +1516,11 @@ async fn test_solver_initial_request_impossible_masks_embedded_package_solution( } #[rstest] +// This test is only applicable to the og solver +#[case::step(step_solver())] #[tokio::test] async fn test_solver_impossible_request_but_embedded_package_makes_solvable( - mut solver: StepSolver, + #[case] mut solver: SolverImpl, ) { // test when there is an embedded package // - the initial request depends on the same package as the embedded package @@ -1478,7 +1568,9 @@ async fn test_solver_impossible_request_but_embedded_package_makes_solvable( solver.add_repository(Arc::new(repo)); solver.add_request(request!("needs")); - solver.set_resolve_validation_impossible_checks(true); + if let SolverImpl::Step(ref mut solver) = solver { + solver.set_resolve_validation_impossible_checks(true); + } // The solutions is: needs/1.0.0 -> something/2.4.0 -> maya/2019.2 (embeds qt/5.12.6) // -> somethingelse/3.2.1 ----------------------^ @@ -1490,7 +1582,7 @@ async fn test_solver_impossible_request_but_embedded_package_makes_solvable( // that point because the solver does not process all unresolved // requests before stopping and this is not an embedded package // cache for it to check. - match run_and_print_resolve_for_tests(&solver).await { + match run_and_print_resolve_for_tests(&mut solver).await { Ok(solution) => { assert_resolved!(solution, "qt", "5.12.6"); assert_resolved!( @@ -1509,9 +1601,13 @@ async fn test_solver_impossible_request_but_embedded_package_makes_solvable( /// When multiple packages try to embed the same package the solver doesn't /// panic. #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_multiple_packages_embed_same_package( - mut solver: StepSolver, + #[case] mut solver: SolverImpl, #[values(true, false)] resolve_validation_impossible_checks: bool, ) { init_logging(); @@ -1549,9 +1645,11 @@ async fn test_multiple_packages_embed_same_package( solver.add_repository(Arc::new(repo)); solver.add_request(request!("top-level")); - solver.set_resolve_validation_impossible_checks(resolve_validation_impossible_checks); + if let SolverImpl::Step(ref mut solver) = solver { + solver.set_resolve_validation_impossible_checks(resolve_validation_impossible_checks); + } - match run_and_print_resolve_for_tests(&solver).await { + match run_and_print_resolve_for_tests(&mut solver).await { Err(Error::GraphError(ref graph_err)) if matches!(&**graph_err, spk_solve_graph::Error::FailedToResolve(_)) => {} Ok(_) => { @@ -1564,8 +1662,10 @@ async fn test_multiple_packages_embed_same_package( } #[rstest] +// This test is only applicable to the og solver +#[case::step(step_solver())] #[tokio::test] -async fn test_solver_with_impossible_checks_in_build_keys(mut solver: StepSolver) { +async fn test_solver_with_impossible_checks_in_build_keys(#[case] mut solver: SolverImpl) { let options1 = option_map! {"dep" => "1.0.0"}; let options2 = option_map! {"dep" => "2.0.0"}; @@ -1595,16 +1695,20 @@ async fn test_solver_with_impossible_checks_in_build_keys(mut solver: StepSolver solver.add_request(request!("pkg-top")); // This is to exercise the check. The missing dep2 package will // ensure that the package that depends on dep1 is chosen. - solver.set_build_key_impossible_checks(true); + if let SolverImpl::Step(ref mut solver) = solver { + solver.set_build_key_impossible_checks(true); + } - let packages = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let packages = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!(packages, "pkg-a", "1.0.0"); assert_resolved!(packages, "dep", "1.0.0"); } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_some_versions_conflicting_requests(mut solver: StepSolver) { +async fn test_solver_some_versions_conflicting_requests(#[case] mut solver: SolverImpl) { // test when there is a package with some version that have a conflicting dependency // - the solver passes over the one with conflicting // - the solver logs compat info for versions with conflicts @@ -1634,14 +1738,17 @@ async fn test_solver_some_versions_conflicting_requests(mut solver: StepSolver) solver.add_repository(Arc::new(repo)); solver.add_request(request!("my-lib")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!(solution, "dep", "2.0.0"); } #[rstest] +#[case::step(step_solver())] +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_embedded_request_invalidates(mut solver: StepSolver) { +async fn test_solver_embedded_request_invalidates(#[case] mut solver: SolverImpl) { // test when a package is resolved with an incompatible embedded pkg // - the solver tries to resolve the package // - there is a conflict in the embedded request @@ -1668,14 +1775,17 @@ async fn test_solver_embedded_request_invalidates(mut solver: StepSolver) { solver.add_request(request!("python")); solver.add_request(request!("my-lib")); - let res = run_and_print_resolve_for_tests(&solver).await; + let res = run_and_print_resolve_for_tests(&mut solver).await; assert!(res.is_err()); } #[rstest] +#[case::step(step_solver())] +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_unknown_package_options(mut solver: StepSolver) { +async fn test_solver_unknown_package_options(#[case] mut solver: SolverImpl) { // test when a package is requested with specific options (eg: pkg.opt) // - the solver ignores versions that don't define the option // - the solver resolves versions that do define the option @@ -1688,19 +1798,21 @@ async fn test_solver_unknown_package_options(mut solver: StepSolver) { solver.add_request(request!({"var": "my-lib.something/value"})); solver.add_request(request!("my-lib")); - let res = run_and_print_resolve_for_tests(&solver).await; + let res = run_and_print_resolve_for_tests(&mut solver).await; assert!(res.is_err()); // this time we don't request that option, and it should be ok solver.reset(); solver.add_repository(repo); solver.add_request(request!("my-lib")); - run_and_print_resolve_for_tests(&solver).await.unwrap(); + run_and_print_resolve_for_tests(&mut solver).await.unwrap(); } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_var_requirements(mut solver: StepSolver) { +async fn test_solver_var_requirements(#[case] mut solver: SolverImpl) { // test what happens when a dependency is added which is incompatible // with an existing request in the stack let repo = make_repo!( @@ -1732,7 +1844,7 @@ async fn test_solver_var_requirements(mut solver: StepSolver) { solver.add_repository(repo.clone()); solver.add_request(request!("my-app/2")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!(solution, "my-app", "2.0.0"); assert_resolved!(solution, "python", "3.7.3"); @@ -1742,14 +1854,16 @@ async fn test_solver_var_requirements(mut solver: StepSolver) { solver.add_repository(repo); solver.add_request(request!("my-app/1")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!(solution, "python", "2.7.5"); } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_var_requirements_unresolve(mut solver: StepSolver) { +async fn test_solver_var_requirements_unresolve(#[case] mut solver: SolverImpl) { // test when a package is resolved that conflicts in var requirements // - the solver should unresolve the solved package // - the solver should resolve a new version of the package with the right version @@ -1783,7 +1897,7 @@ async fn test_solver_var_requirements_unresolve(mut solver: StepSolver) { // the addition of this app constrains the python.abi to 2.7 solver.add_request(request!("my-app/1")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!(solution, "my-app", "1.0.0"); assert_resolved!(solution, "python", "2.7.5", "should re-resolve python"); @@ -1795,15 +1909,19 @@ async fn test_solver_var_requirements_unresolve(mut solver: StepSolver) { // the addition of this app constrains the global abi to 2.7 solver.add_request(request!("my-app/2")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!(solution, "my-app", "2.0.0"); assert_resolved!(solution, "python", "2.7.5", "should re-resolve python"); } #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_build_options_dont_affect_compat(mut solver: StepSolver) { +async fn test_solver_build_options_dont_affect_compat(#[case] mut solver: SolverImpl) { // test when a package is resolved with some build option // - that option can conflict with another packages build options // - as long as there is no explicit requirement on that option's value @@ -1835,7 +1953,7 @@ async fn test_solver_build_options_dont_affect_compat(mut solver: StepSolver) { // b is not affected and can still be resolved solver.add_request(request!("pkgb")); - run_and_print_resolve_for_tests(&solver).await.unwrap(); + run_and_print_resolve_for_tests(&mut solver).await.unwrap(); solver.reset(); solver.add_repository(repo.clone()); @@ -1845,13 +1963,15 @@ async fn test_solver_build_options_dont_affect_compat(mut solver: StepSolver) { // this time the explicit request will cause a failure solver.add_request(request!({"var": "build-dep/=1.0.0"})); - let res = run_and_print_resolve_for_tests(&solver).await; + let res = run_and_print_resolve_for_tests(&mut solver).await; assert!(res.is_err()); } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_option_compat_intersection(mut solver: StepSolver) { +async fn test_solver_option_compat_intersection(#[case] mut solver: SolverImpl) { // A var option for spi-platform/~2022.4.1.4 should be able to resolve // with a build of openimageio that requires spi-platform/~2022.4.1.3. @@ -1883,12 +2003,14 @@ async fn test_solver_option_compat_intersection(mut solver: StepSolver) { solver.add_request(request!({"var": "spi-platform/~2022.4.1.4"})); solver.add_request(request!({"pkg": "openimageio"})); - let _ = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let _ = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); } #[rstest] +#[case::step(step_solver())] +// #[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_components(mut solver: StepSolver) { +async fn test_solver_components(#[case] mut solver: SolverImpl) { // test when a package is requested with specific components // - all the aggregated components are selected in the resolve // - the final build has published layers for each component @@ -1922,7 +2044,7 @@ async fn test_solver_components(mut solver: StepSolver) { solver.add_request(request!("pkga")); solver.add_request(request!("pkgb")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); let resolved = solution .get("python") @@ -1940,8 +2062,12 @@ async fn test_solver_components(mut solver: StepSolver) { } #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_components_interaction_with_embeds(mut solver: StepSolver) { +async fn test_solver_components_interaction_with_embeds(#[case] mut solver: SolverImpl) { // Test that a package can have a component that embeds a specific // component of some other package. This package must be included in a // solution to satisfy a request for that package+component combo. @@ -2002,7 +2128,7 @@ async fn test_solver_components_interaction_with_embeds(mut solver: StepSolver) solver.add_request(request!("fake-pkg:comp1")); solver.add_request(request!("victim")); - let Ok(solution) = run_and_print_resolve_for_tests(&solver).await else { + let Ok(solution) = run_and_print_resolve_for_tests(&mut solver).await else { panic!("Expected a valid solution"); }; @@ -2022,8 +2148,11 @@ async fn test_solver_components_interaction_with_embeds(mut solver: StepSolver) } #[rstest] +#[case::step(step_solver())] +// This test can sometimes succeed by chance so can't use #[should_panic] here +// TODO #[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_components_when_no_components_requested(mut solver: StepSolver) { +async fn test_solver_components_when_no_components_requested(#[case] mut solver: SolverImpl) { // test when a package is requested with no components and the // package is one that has components // - the default component(s) should be the ones in the resolve @@ -2057,7 +2186,7 @@ async fn test_solver_components_when_no_components_requested(mut solver: StepSol solver.add_request(request!("pkga")); solver.add_request(request!("pkgb")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); let resolved = solution .get("python") @@ -2075,8 +2204,14 @@ async fn test_solver_components_when_no_components_requested(mut solver: StepSol } #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_src_package_request_when_no_components_requested(mut solver: StepSolver) { +async fn test_solver_src_package_request_when_no_components_requested( + #[case] mut solver: SolverImpl, +) { // test when a /src package build is requested with no components // and a matching package with a /src package build exists in the repo // - the solver should resolve to the /src package build @@ -2095,7 +2230,7 @@ async fn test_solver_src_package_request_when_no_components_requested(mut solver let req = request!("mypkg/1.2.3/src"); solver.add_request(req); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); let resolved = solution.get("mypkg").unwrap().spec.ident().clone(); let expected = build_ident!("mypkg/1.2.3/src"); @@ -2103,8 +2238,12 @@ async fn test_solver_src_package_request_when_no_components_requested(mut solver } #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_all_component(mut solver: StepSolver) { +async fn test_solver_all_component(#[case] mut solver: SolverImpl) { // test when a package is requested with the 'all' component // - all the specs components are selected in the resolve // - the final build has published layers for each component @@ -2128,7 +2267,7 @@ async fn test_solver_all_component(mut solver: StepSolver) { solver.add_repository(Arc::new(repo)); solver.add_request(request!("python:all")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); let resolved = solution.get("python").unwrap(); assert_eq!(resolved.request.pkg.components.len(), 1); @@ -2144,8 +2283,12 @@ async fn test_solver_all_component(mut solver: StepSolver) { } #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_component_availability(mut solver: StepSolver) { +async fn test_solver_component_availability(#[case] mut solver: SolverImpl) { // test when a package is requested with some component // - all the specs components are selected in the resolve // - the final build has published layers for each component @@ -2198,7 +2341,7 @@ async fn test_solver_component_availability(mut solver: StepSolver) { solver.add_repository(Arc::new(repo)); solver.add_request(request!("python:bin")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!( solution, @@ -2210,8 +2353,12 @@ async fn test_solver_component_availability(mut solver: StepSolver) { } #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_component_requirements(mut solver: StepSolver) { +async fn test_solver_component_requirements(#[case] mut solver: SolverImpl) { // test when a component has its own list of requirements // - the requirements are added to the existing set of requirements // - the additional requirements are resolved @@ -2240,7 +2387,7 @@ async fn test_solver_component_requirements(mut solver: StepSolver) { solver.add_repository(repo.clone()); solver.add_request(request!("mypkg:build")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); solution.get("dep").expect("should exist"); solution.get("depb").expect("should exist"); @@ -2250,7 +2397,7 @@ async fn test_solver_component_requirements(mut solver: StepSolver) { solver.add_repository(repo); solver.add_request(request!("mypkg:run")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); solution.get("dep").expect("should exist"); solution.get("depr").expect("should exist"); @@ -2258,8 +2405,12 @@ async fn test_solver_component_requirements(mut solver: StepSolver) { } #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_component_requirements_extending(mut solver: StepSolver) { +async fn test_solver_component_requirements_extending(#[case] mut solver: SolverImpl) { // test when an additional component is requested after a package is resolved // - the new components requirements are still added and resolved @@ -2285,14 +2436,18 @@ async fn test_solver_component_requirements_extending(mut solver: StepSolver) { // has a new requirement on depc solver.add_request(request!("depb")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); solution.get("depc").expect("should exist"); } #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_component_embedded(mut solver: StepSolver) { +async fn test_solver_component_embedded(#[case] mut solver: SolverImpl) { // test when a component has its own list of embedded packages // - the embedded package is immediately selected // - it must be compatible with any previous requirements @@ -2335,7 +2490,7 @@ async fn test_solver_component_embedded(mut solver: StepSolver) { solver.add_repository(repo.clone()); solver.add_request(request!("downstream1")); - let solution = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_resolved!( solution, @@ -2350,7 +2505,7 @@ async fn test_solver_component_embedded(mut solver: StepSolver) { // should fail because the one embedded package // does not meet the requirements in downstream spec - let res = run_and_print_resolve_for_tests(&solver).await; + let res = run_and_print_resolve_for_tests(&mut solver).await; assert!(res.is_err()); } @@ -2360,7 +2515,10 @@ async fn test_solver_component_embedded(mut solver: StepSolver) { #[case::comp2(&["mypkg:comp2", "dep-e1:comp2"], false)] #[tokio::test] async fn test_solver_component_embedded_component_requirements( - mut solver: StepSolver, + #[values(step_solver() + // TODO , resolvo_solver() + )] + mut solver: SolverImpl, #[case] packages_to_request: &[&str], #[case] expected_solve_result: bool, ) { @@ -2398,7 +2556,7 @@ async fn test_solver_component_embedded_component_requirements( solver.add_request(request!(package_to_request)); } - match run_and_print_resolve_for_tests(&solver).await { + match run_and_print_resolve_for_tests(&mut solver).await { Ok(solution) => { assert!(expected_solve_result, "expected solve to fail"); @@ -2416,7 +2574,10 @@ async fn test_solver_component_embedded_component_requirements( #[case::downstream3("downstream3", false)] #[tokio::test] async fn test_solver_component_embedded_multiple_versions( - mut solver: StepSolver, + #[values(step_solver() + // TODO , resolvo_solver() + )] + mut solver: SolverImpl, #[case] package_to_request: &str, #[case] expected_solve_result: bool, ) { @@ -2467,7 +2628,7 @@ async fn test_solver_component_embedded_multiple_versions( solver.add_repository(repo); solver.add_request(request!(package_to_request)); - match run_and_print_resolve_for_tests(&solver).await { + match run_and_print_resolve_for_tests(&mut solver).await { Ok(solution) => { assert!(expected_solve_result, "expected solve to fail"); @@ -2484,8 +2645,12 @@ async fn test_solver_component_embedded_multiple_versions( } #[rstest] +#[case::step(step_solver())] +// Remove #[should_panic] once resolvo handles this case +#[should_panic] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_component_embedded_incompatible_requests(mut solver: StepSolver) { +async fn test_solver_component_embedded_incompatible_requests(#[case] mut solver: SolverImpl) { // test when different components of a package embedded packages that // make incompatible requests @@ -2516,7 +2681,7 @@ async fn test_solver_component_embedded_incompatible_requests(mut solver: StepSo solver.add_request(request!("mypkg:comp1")); solver.add_request(request!("mypkg:comp2")); - run_and_print_resolve_for_tests(&solver) + run_and_print_resolve_for_tests(&mut solver) .await .expect_err("expected solve to fail"); } @@ -2684,7 +2849,10 @@ fn test_problem_packages() { #[case::resolve_two_part_flavor("blue", "1.0")] #[tokio::test] async fn test_version_number_masking( - mut solver: StepSolver, + #[values(step_solver() + // TODO , resolvo_solver() + )] + mut solver: SolverImpl, #[case] color_to_solve_for: &str, #[case] expected_resolved_version: &str, #[values(RepoKind::Mem, RepoKind::Spfs)] repo: RepoKind, @@ -2755,7 +2923,7 @@ async fn test_version_number_masking( .into(), ); - let packages = run_and_print_resolve_for_tests(&solver).await.unwrap(); + let packages = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); assert_eq!(packages.len(), 1, "expected one resolved package"); let resolved = packages.get("my-pkg").unwrap(); assert_eq!( diff --git a/crates/spk-solve/src/solvers/step/solver.rs b/crates/spk-solve/src/solvers/step/solver.rs index 64bc565841..bd51fff43b 100644 --- a/crates/spk-solve/src/solvers/step/solver.rs +++ b/crates/spk-solve/src/solvers/step/solver.rs @@ -1024,63 +1024,11 @@ impl Solver { Ok(()) } - /// Put this solver back into its default state - pub fn reset(&mut self) { - self.repos.truncate(0); - self.initial_state_builders.truncate(0); - self.validators = Cow::from(default_validators()); - (*self.request_validator).reset(); - - self.number_of_steps = 0; - self.number_builds_skipped = 0; - self.number_incompat_versions = 0; - self.number_incompat_builds = 0; - self.number_total_builds = 0; - self.number_of_steps_back.store(0, Ordering::SeqCst); - self.error_frequency.clear(); - self.problem_packages.clear(); - } - /// Run this solver pub fn run(&self) -> SolverRuntime { SolverRuntime::new(self.clone()) } - /// If true, only solve pre-built binary packages. - /// - /// When false, the solver may return packages where the build is not set. - /// These packages are known to have a source package available, and the requested - /// options are valid for a new build of that source package. - /// These packages are not actually built as part of the solver process but their - /// build environments are fully resolved and dependencies included - pub fn set_binary_only(&mut self, binary_only: bool) { - self.request_validator.set_binary_only(binary_only); - - let has_binary_only = self - .validators - .iter() - .find_map(|v| match v { - Validators::BinaryOnly(_) => Some(true), - _ => None, - }) - .unwrap_or(false); - if !(has_binary_only ^ binary_only) { - return; - } - if binary_only { - // Add BinaryOnly validator because it was missing. - self.validators - .to_mut() - .insert(0, Validators::BinaryOnly(BinaryOnlyValidator {})) - } else { - // Remove all BinaryOnly validators because one was found. - self.validators = take(self.validators.to_mut()) - .into_iter() - .filter(|v| !matches!(v, Validators::BinaryOnly(_))) - .collect(); - } - } - /// Enable or disable running impossible checks on the initial requests /// before the solve starts pub fn set_initial_request_impossible_checks(&mut self, enabled: bool) { @@ -1107,16 +1055,6 @@ impl Solver { || self.impossible_checks.use_in_build_keys } - pub async fn solve(&mut self) -> Result { - let mut runtime = self.run(); - { - let iter = runtime.iter(); - tokio::pin!(iter); - while let Some(_step) = iter.try_next().await? {} - } - runtime.current_solution().await - } - /// Adds requests for all build requirements pub fn configure_for_build_environment(&mut self, recipe: &T) -> Result<()> { let state = self.get_initial_state(); @@ -1170,6 +1108,7 @@ impl Solver { } } +#[async_trait::async_trait] impl SolverTrait for Solver { fn add_repository(&mut self, repo: R) where @@ -1195,6 +1134,60 @@ impl SolverTrait for Solver { self.initial_state_builders.push(request); } + fn reset(&mut self) { + self.repos.truncate(0); + self.initial_state_builders.truncate(0); + self.validators = Cow::from(default_validators()); + (*self.request_validator).reset(); + + self.number_of_steps = 0; + self.number_builds_skipped = 0; + self.number_incompat_versions = 0; + self.number_incompat_builds = 0; + self.number_total_builds = 0; + self.number_of_steps_back.store(0, Ordering::SeqCst); + self.error_frequency.clear(); + self.problem_packages.clear(); + } + + fn set_binary_only(&mut self, binary_only: bool) { + self.request_validator.set_binary_only(binary_only); + + let has_binary_only = self + .validators + .iter() + .find_map(|v| match v { + Validators::BinaryOnly(_) => Some(true), + _ => None, + }) + .unwrap_or(false); + if !(has_binary_only ^ binary_only) { + return; + } + if binary_only { + // Add BinaryOnly validator because it was missing. + self.validators + .to_mut() + .insert(0, Validators::BinaryOnly(BinaryOnlyValidator {})) + } else { + // Remove all BinaryOnly validators because one was found. + self.validators = take(self.validators.to_mut()) + .into_iter() + .filter(|v| !matches!(v, Validators::BinaryOnly(_))) + .collect(); + } + } + + async fn solve(&mut self) -> Result { + let mut runtime = self.run(); + { + let iter = runtime.iter(); + tokio::pin!(iter); + while let Some(_step) = iter.try_next().await? {} + } + runtime.current_solution().await + } + fn update_options(&mut self, options: OptionMap) { self.initial_state_builders .push(Change::SetOptions(SetOptions::new(options))) From 60e546258f7df38850078cbb4cc2afaf69cb6574 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Thu, 6 Feb 2025 11:58:35 -0800 Subject: [PATCH 15/64] Handle dependency inclusion policy Signed-off-by: J Robert Ray --- .../src/solvers/resolvo/spk_provider.rs | 18 +++++++++++------- crates/spk-solve/src/solvers/solver_test.rs | 2 -- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index f15a7cdeff..b499fc2d9c 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -442,14 +442,18 @@ impl DependencyProvider for SpkProvider { let dep_name = self .pool .intern_package_name(pkg_request.pkg.name().to_owned()); - known_deps.requirements.push( - self.pool - .intern_version_set( - dep_name, - RequestVS::SpkRequest(requirement.clone()), - ) - .into(), + let dep_vs = self.pool.intern_version_set( + dep_name, + RequestVS::SpkRequest(requirement.clone()), ); + match pkg_request.inclusion_policy { + spk_schema::ident::InclusionPolicy::Always => { + known_deps.requirements.push(dep_vs.into()); + } + spk_schema::ident::InclusionPolicy::IfAlreadyPresent => { + known_deps.constrains.push(dep_vs); + } + }; } Request::Var(var_request) => { match &var_request.value { diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index 6d60944031..c359ae9686 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -700,8 +700,6 @@ async fn test_solver_pre_release_config(#[case] mut solver: SolverImpl) { #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_constraint_only(#[case] mut solver: SolverImpl) { From 5274994fefc4e0aafb590b6a2861210fd76027c8 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Thu, 6 Feb 2025 13:11:48 -0800 Subject: [PATCH 16/64] Support global var options At least well enough to make a handful of tests pass. Signed-off-by: J Robert Ray --- crates/spk-solve/src/solvers/resolvo/mod.rs | 2 +- .../src/solvers/resolvo/spk_provider.rs | 79 ++++++++++- crates/spk-solve/src/solvers/solver_test.rs | 133 +++++++++--------- 3 files changed, 138 insertions(+), 76 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index aa58696b61..d845ce771b 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -83,7 +83,7 @@ impl SolverTrait for Solver { let solvables = tokio::task::spawn_blocking(move || { let mut provider = Some(SpkProvider::new(repos.clone())); let (solver, solved) = loop { - let this_iter_provider = provider.take().expect("provider is always Some"); + let mut this_iter_provider = provider.take().expect("provider is always Some"); let pkg_requirements = this_iter_provider.pkg_requirements(&requests); let var_requirements = this_iter_provider.var_requirements(&requests); let mut solver = resolvo::Solver::new(this_iter_provider) diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index b499fc2d9c..cd143db600 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -22,8 +22,8 @@ use resolvo::{ VersionSetUnionId, }; use spk_schema::foundation::pkg_name; -use spk_schema::ident::LocatedBuildIdent; -use spk_schema::name::PkgNameBuf; +use spk_schema::ident::{LocatedBuildIdent, PinnableValue, Satisfy, VarRequest}; +use spk_schema::name::{OptNameBuf, PkgNameBuf}; use spk_schema::{Opt, Package, Request, VersionIdent}; use spk_storage::RepositoryHandle; @@ -42,6 +42,10 @@ const PSEUDO_PKG_NAME_PREFIX: &str = "global-vars--"; pub(crate) struct SpkProvider { pub(crate) pool: Pool, repos: Vec>, + /// Global options, like what might be specified with `--opt` to `spk env`. + /// Indexed by name. If multiple requests happen to exist with the same + /// name, the last one is kept. + global_var_requests: HashMap>, interned_solvables: RefCell>, /// Track all the global var keys and values that have been witnessed while /// solving. @@ -58,6 +62,7 @@ impl SpkProvider { Self { pool: Pool::new(), repos, + global_var_requests: Default::default(), interned_solvables: Default::default(), known_global_var_values: Default::default(), queried_global_var_values: Default::default(), @@ -91,6 +96,7 @@ impl SpkProvider { Self { pool: Pool::new(), repos: self.repos.clone(), + global_var_requests: self.global_var_requests.clone(), interned_solvables: Default::default(), known_global_var_values: RefCell::new(self.known_global_var_values.take()), queried_global_var_values: Default::default(), @@ -98,9 +104,31 @@ impl SpkProvider { } } - pub fn var_requirements(&self, _requests: &[Request]) -> Vec { - // TODO - Vec::new() + pub fn var_requirements(&mut self, requests: &[Request]) -> Vec { + self.global_var_requests.reserve(requests.len()); + requests + .iter() + .filter_map(|req| match req { + Request::Var(var) => Some(var), + _ => None, + }) + .filter_map(|req| match req.var.namespace() { + Some(pkg_name) => { + // A global request applicable to a specific package. + let dep_name = self.pool.intern_package_name(pkg_name.to_owned()); + Some(self.pool.intern_version_set( + dep_name, + RequestVS::SpkRequest(Request::Var(req.clone())), + )) + } + None => { + // A global request affecting all packages. + self.global_var_requests + .insert(req.var.without_namespace().to_owned(), req.clone()); + None + } + }) + .collect() } } @@ -313,7 +341,10 @@ impl DependencyProvider for SpkProvider { }; for build in located_builds { + // What we need from build before it is moved into the pool. + let ident = build.ident.clone(); let is_src = build.component.is_source(); + let solvable_id = *self .interned_solvables .borrow_mut() @@ -322,13 +353,49 @@ impl DependencyProvider for SpkProvider { self.pool .intern_solvable(name, SpkSolvable::LocatedBuildIdentWithComponent(build)) }); + if is_src { candidates.excluded.push(( solvable_id, self.pool.intern_string("source builds are excluded"), )); } else { - candidates.candidates.push(solvable_id); + // Filter builds that don't conform to global options + // XXX: This find runtime will add up. + let repo = self + .repos + .iter() + .find(|repo| repo.name() == ident.repository_name()) + .expect( + "Expected solved package's repository to be in the list of repositories", + ); + match repo.read_package(ident.target()).await { + Ok(package) => { + for (opt_name, _value) in package.option_values() { + if let Some(request) = self.global_var_requests.get(&opt_name) { + if let spk_schema::version::Compatibility::Incompatible( + incompatible_reason, + ) = package.check_satisfies_request(request) + { + candidates.excluded.push(( + solvable_id, + self.pool.intern_string(format!( + "build does not satisfy global var request: {incompatible_reason}" + )), + )); + continue; + } + } + } + + candidates.candidates.push(solvable_id); + } + Err(err) => { + candidates + .excluded + .push((solvable_id, self.pool.intern_string(err.to_string()))); + } + } } } diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index c359ae9686..143e25b7b0 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -191,7 +191,7 @@ async fn test_solver_package_with_no_recipe( } #[rstest] -// This test is only applicable to the og solver +// This test is only applicable to the step solver #[case::step(step_solver())] #[tokio::test] async fn test_solver_package_with_no_recipe_and_impossible_initial_checks( @@ -281,7 +281,7 @@ async fn test_solver_package_with_no_recipe_from_cmd_line(#[case] mut solver: So } #[rstest] -// This test is only applicable to the og solver +// This test is only applicable to the step solver #[case::step(step_solver())] #[tokio::test] async fn test_solver_package_with_no_recipe_from_cmd_line_and_impossible_initial_checks( @@ -765,11 +765,12 @@ async fn test_solver_constraint_and_request(#[case] mut solver: SolverImpl) { #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_solver_option_compatibility(#[case] mut solver: SolverImpl) { +async fn test_solver_option_compatibility( + #[case] mut solver: SolverImpl, + #[values("~2.0", "~2.7", "~2.7.5", "2,<3", "2.7,<3", "3", "3.7", "3.7.3")] pyver: &str, +) { // test what happens when an option is given in the solver // - the options for each build are checked // - the resolved build must have used the option @@ -809,43 +810,37 @@ async fn test_solver_option_compatibility(#[case] mut solver: SolverImpl) { // added to some of the version ranges below force the solver to // work through the ordered builds until it finds an appropriate // 2.x.y values to both solve and pass the test. - for pyver in [ - // Uncomment this, when the '2,<3' parsing bug: https://github.com/spkenv/spk/issues/322 has been fixed - //"~2.0", "~2.7", "~2.7.5", "2,<3", "2.7,<3", "3", "3.7", "3.7.3", - "~2.0", "~2.7", "~2.7.5", "3", "3.7", "3.7.3", - ] { - solver.reset(); - solver.add_repository(repo.clone()); - solver.add_request(request!("vnp3")); - solver.add_request( - VarRequest { - var: opt_name!("python").to_owned(), - value: pyver.into(), - description: None, - } - .into(), - ); + solver.reset(); + solver.add_repository(repo.clone()); + solver.add_request(request!("vnp3")); + solver.add_request( + VarRequest { + var: opt_name!("python").to_owned(), + value: pyver.into(), + description: None, + } + .into(), + ); - let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); - - let resolved = solution.get("vnp3").unwrap(); - let value = resolved - .spec - .option_values() - .remove(opt_name!("python")) - .unwrap(); - - // Check the first digit component of the pyver value - let expected = if pyver.starts_with('~') { - format!("~{}", pyver.chars().nth(1).unwrap()).to_string() - } else { - format!("~{}", pyver.chars().next().unwrap()).to_string() - }; - assert!( - value.starts_with(&expected), - "{value} should start with ~{expected} to be valid for {pyver}" - ); - } + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); + + let resolved = solution.get("vnp3").unwrap(); + let value = resolved + .spec + .option_values() + .remove(opt_name!("python")) + .unwrap(); + + // Check the first digit component of the pyver value + let expected = if pyver.starts_with('~') { + format!("~{}", pyver.chars().nth(1).unwrap()).to_string() + } else { + format!("~{}", pyver.chars().next().unwrap()).to_string() + }; + assert!( + value.starts_with(&expected), + "{value} should start with {expected} to be valid for {pyver}" + ); } #[rstest] @@ -963,12 +958,11 @@ async fn test_solver_build_from_source(#[case] mut solver: SolverImpl) { #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_build_from_source_unsolvable(#[case] mut solver: SolverImpl) { let log = init_logging(); + // test when no appropriate build exists but the source is available // - if the requested pkg cannot resolve a build environment // - this is flagged by the solver as impossible @@ -1006,21 +1000,28 @@ async fn test_solver_build_from_source_unsolvable(#[case] mut solver: SolverImpl let res = run_and_log_resolve_for_tests(&mut solver).await; assert!(res.is_err(), "should fail to resolve"); - let log = log.lock(); - let event = log.all_events().find(|e| { - let Some(msg) = e.message() else { - return false; - }; - let msg = strip_ansi_escapes::strip(msg); - let msg = String::from_utf8_lossy(&msg); - msg.ends_with( - "TRY my-tool/1.2.0/src - cannot resolve build env for source build: Failed to resolve: there is no solution for these requests using the available packages", - ) - }); - assert!( - event.is_some(), - "should block because of failed build env resolve" - ); + + // This additional assert is specific to the output of the step solver. + // XXX The log isn't cleared between test cases, so this code would + // sometimes pass while testing the resolvo solver. There is no obvious way + // to clear the log. + if let SolverImpl::Step(_) = solver { + let log = log.lock(); + let event = log.all_events().find(|e| { + let Some(msg) = e.message() else { + return false; + }; + let msg = strip_ansi_escapes::strip(msg); + let msg = String::from_utf8_lossy(&msg); + msg.ends_with( + "TRY my-tool/1.2.0/src - cannot resolve build env for source build: Failed to resolve: there is no solution for these requests using the available packages", + ) + }); + assert!( + event.is_some(), + "should block because of failed build env resolve" + ); + } } #[rstest] @@ -1216,7 +1217,7 @@ async fn test_solver_build_from_source_deprecated(#[case] mut solver: SolverImpl } #[rstest] -// This test is only applicable to the og solver +// This test is only applicable to the step solver #[case::step(step_solver())] #[tokio::test] async fn test_solver_build_from_source_deprecated_and_impossible_initial_checks( @@ -1460,7 +1461,7 @@ async fn test_solver_embedded_package_replaces_real_package(#[case] mut solver: } #[rstest] -// This test is only applicable to the og solver +// This test is only applicable to the step solver #[case::step(step_solver())] #[tokio::test] async fn test_solver_initial_request_impossible_masks_embedded_package_solution( @@ -1514,7 +1515,7 @@ async fn test_solver_initial_request_impossible_masks_embedded_package_solution( } #[rstest] -// This test is only applicable to the og solver +// This test is only applicable to the step solver #[case::step(step_solver())] #[tokio::test] async fn test_solver_impossible_request_but_embedded_package_makes_solvable( @@ -1660,7 +1661,7 @@ async fn test_multiple_packages_embed_same_package( } #[rstest] -// This test is only applicable to the og solver +// This test is only applicable to the step solver #[case::step(step_solver())] #[tokio::test] async fn test_solver_with_impossible_checks_in_build_keys(#[case] mut solver: SolverImpl) { @@ -1780,7 +1781,6 @@ async fn test_solver_embedded_request_invalidates(#[case] mut solver: SolverImpl #[rstest] #[case::step(step_solver())] -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_unknown_package_options(#[case] mut solver: SolverImpl) { @@ -1915,8 +1915,6 @@ async fn test_solver_var_requirements_unresolve(#[case] mut solver: SolverImpl) #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_build_options_dont_affect_compat(#[case] mut solver: SolverImpl) { @@ -2847,10 +2845,7 @@ fn test_problem_packages() { #[case::resolve_two_part_flavor("blue", "1.0")] #[tokio::test] async fn test_version_number_masking( - #[values(step_solver() - // TODO , resolvo_solver() - )] - mut solver: SolverImpl, + #[values(step_solver(), resolvo_solver())] mut solver: SolverImpl, #[case] color_to_solve_for: &str, #[case] expected_resolved_version: &str, #[values(RepoKind::Mem, RepoKind::Spfs)] repo: RepoKind, From af16e859d77ab23ac0beae3b30c35ecc24ff494d Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Thu, 6 Feb 2025 14:48:09 -0800 Subject: [PATCH 17/64] Populate Solution options Well enough to satisfy the one test. Signed-off-by: J Robert Ray --- crates/spk-solve/src/solvers/resolvo/mod.rs | 46 ++++++++++++++++++--- crates/spk-solve/src/solvers/solver_test.rs | 2 - 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index d845ce771b..07c104897e 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -23,8 +23,9 @@ use std::sync::Arc; use pkg_request_version_set::SpkSolvable; use spk_provider::SpkProvider; use spk_schema::ident::{InclusionPolicy, PinPolicy, PkgRequest, RangeIdent}; +use spk_schema::prelude::{HasVersion, Named, Versioned}; use spk_schema::version_range::VersionFilter; -use spk_schema::{OptionMap, Request}; +use spk_schema::{OptionMap, Package, Request}; use spk_solve_solution::{PackageSource, Solution}; use spk_solve_validation::{Validators, default_validators}; use spk_storage::RepositoryHandle; @@ -132,7 +133,8 @@ impl SolverTrait for Solver { .await .map_err(|err| Error::String(format!("Tokio panicked? {err}")))??; - let mut solution = Solution::default(); + let mut solution_options = OptionMap::default(); + let mut solution_adds = Vec::with_capacity(solvables.len()); for located_build_ident_with_component in solvables { let pkg_request = PkgRequest { pkg: RangeIdent { @@ -159,10 +161,38 @@ impl SolverTrait for Solver { repo.name() == located_build_ident_with_component.ident.repository_name() }) .expect("Expected solved package's repository to be in the list of repositories"); - solution.add( + let package = repo + .read_package(located_build_ident_with_component.ident.target()) + .await?; + let rendered_version = package.compat().render(package.version()); + solution_options.insert(package.name().as_opt_name().to_owned(), rendered_version); + for option in package.get_build_options() { + match option { + spk_schema::Opt::Pkg(pkg_opt) => { + if let Some(value) = pkg_opt.get_value(None) { + solution_options.insert( + format!("{}.{}", package.name(), pkg_opt.pkg).try_into().expect("Two packages names separated by a period is a valid option name"), + value, + ); + } + } + spk_schema::Opt::Var(var_opt) => { + if let Some(value) = var_opt.get_value(None) { + if var_opt.var.namespace().is_none() { + solution_options.insert( + format!("{}.{}", package.name(), var_opt.var).try_into().expect("A package name, a period, and a non-namespaced option name is a valid option name"), + value, + ); + } else { + solution_options.insert(var_opt.var.clone(), value); + } + } + } + } + } + solution_adds.push(( pkg_request, - repo.read_package(located_build_ident_with_component.ident.target()) - .await?, + package, PackageSource::Repository { repo: Arc::clone(repo), // XXX: Why is this needed? @@ -170,7 +200,11 @@ impl SolverTrait for Solver { .read_components(located_build_ident_with_component.ident.target()) .await?, }, - ); + )); + } + let mut solution = Solution::new(solution_options); + for (pkg_request, package, source) in solution_adds { + solution.add(pkg_request, package, source); } Ok(solution) } diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index 143e25b7b0..20f6cf16f0 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -845,8 +845,6 @@ async fn test_solver_option_compatibility( #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_option_injection(#[case] mut solver: SolverImpl) { From 5747f7d286c8062a13711836ce8f91197c470cd4 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Thu, 6 Feb 2025 16:19:14 -0800 Subject: [PATCH 18/64] Implement requests for source builds Stop completely excluding all source builds. Signed-off-by: J Robert Ray --- .../src/solvers/resolvo/spk_provider.rs | 74 ++++++++++--------- crates/spk-solve/src/solvers/solver_test.rs | 2 - 2 files changed, 40 insertions(+), 36 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index cd143db600..a2a2345780 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -154,9 +154,25 @@ impl DependencyProvider for SpkProvider { } continue; }; + let compatible = pkg_request .is_version_applicable(located_build_ident_with_component.ident.version()); if compatible.is_ok() { + // Only select source builds for requests of source builds. + if located_build_ident_with_component.ident.build().is_source() { + if pkg_request + .pkg + .build + .as_ref() + .map(|build| build.is_source()) + .unwrap_or(false) + ^ inverse + { + selected.push(*candidate); + } + continue; + } + // XXX: This find runtime will add up. let repo = self .repos @@ -343,7 +359,6 @@ impl DependencyProvider for SpkProvider { for build in located_builds { // What we need from build before it is moved into the pool. let ident = build.ident.clone(); - let is_src = build.component.is_source(); let solvable_id = *self .interned_solvables @@ -354,47 +369,38 @@ impl DependencyProvider for SpkProvider { .intern_solvable(name, SpkSolvable::LocatedBuildIdentWithComponent(build)) }); - if is_src { - candidates.excluded.push(( - solvable_id, - self.pool.intern_string("source builds are excluded"), - )); - } else { - // Filter builds that don't conform to global options - // XXX: This find runtime will add up. - let repo = self - .repos - .iter() - .find(|repo| repo.name() == ident.repository_name()) - .expect( - "Expected solved package's repository to be in the list of repositories", - ); - match repo.read_package(ident.target()).await { - Ok(package) => { - for (opt_name, _value) in package.option_values() { - if let Some(request) = self.global_var_requests.get(&opt_name) { - if let spk_schema::version::Compatibility::Incompatible( - incompatible_reason, - ) = package.check_satisfies_request(request) - { - candidates.excluded.push(( + // Filter builds that don't conform to global options + // XXX: This find runtime will add up. + let repo = self + .repos + .iter() + .find(|repo| repo.name() == ident.repository_name()) + .expect("Expected solved package's repository to be in the list of repositories"); + match repo.read_package(ident.target()).await { + Ok(package) => { + for (opt_name, _value) in package.option_values() { + if let Some(request) = self.global_var_requests.get(&opt_name) { + if let spk_schema::version::Compatibility::Incompatible( + incompatible_reason, + ) = package.check_satisfies_request(request) + { + candidates.excluded.push(( solvable_id, self.pool.intern_string(format!( "build does not satisfy global var request: {incompatible_reason}" )), )); - continue; - } + continue; } } - - candidates.candidates.push(solvable_id); - } - Err(err) => { - candidates - .excluded - .push((solvable_id, self.pool.intern_string(err.to_string()))); } + + candidates.candidates.push(solvable_id); + } + Err(err) => { + candidates + .excluded + .push((solvable_id, self.pool.intern_string(err.to_string()))); } } } diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index 20f6cf16f0..0ee86fd2a9 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -2199,8 +2199,6 @@ async fn test_solver_components_when_no_components_requested(#[case] mut solver: #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_src_package_request_when_no_components_requested( From 53a0ca9be02ce3c1e5df9ec997512bbc0e298819 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Thu, 6 Feb 2025 16:51:01 -0800 Subject: [PATCH 19/64] Pass the "All" component test This required rethinking the type backing NameIds to subdivide a package into its components. For example, "python:run" and "python:build" are considered separate packages by the solver. It's not possible for a single request to resolve to multiple resolvables. Either a request for "All" would have to be expanded into requests for the individual components, or there has to be a [virtual] package "python:all" that represents a dependency on all the components of that package. Since it is not possible to know the full set of components for a request like "python:all", where the set of components may differ between the different versions of python, then only the latter approach is feasible. Dependencies are needed to ensure the solve only contains components from the same spk package build. Signed-off-by: J Robert Ray --- crates/spk-solve/src/solvers/resolvo/mod.rs | 13 +- .../resolvo/pkg_request_version_set.rs | 93 +++--- .../src/solvers/resolvo/spk_provider.rs | 274 +++++++++++++++--- crates/spk-solve/src/solvers/solver_test.rs | 2 - 4 files changed, 293 insertions(+), 89 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index 07c104897e..84a9f5a5c0 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -20,7 +20,7 @@ use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; -use pkg_request_version_set::SpkSolvable; +use pkg_request_version_set::{SpkSolvable, SyntheticComponent}; use spk_provider::SpkProvider; use spk_schema::ident::{InclusionPolicy, PinPolicy, PkgRequest, RangeIdent}; use spk_schema::prelude::{HasVersion, Named, Versioned}; @@ -136,14 +136,17 @@ impl SolverTrait for Solver { let mut solution_options = OptionMap::default(); let mut solution_adds = Vec::with_capacity(solvables.len()); for located_build_ident_with_component in solvables { + let SyntheticComponent::Actual(solvable_component) = + &located_build_ident_with_component.component + else { + continue; + }; + let pkg_request = PkgRequest { pkg: RangeIdent { repository_name: None, name: located_build_ident_with_component.ident.name().to_owned(), - components: BTreeSet::from_iter([located_build_ident_with_component - .component - .clone() - .into()]), + components: BTreeSet::from_iter([solvable_component.clone()]), version: VersionFilter::default(), build: None, }, diff --git a/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs b/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs index 644691d4aa..dbe78a107f 100644 --- a/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs +++ b/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs @@ -6,9 +6,8 @@ use std::sync::Arc; use resolvo::utils::VersionSet; use spk_schema::Request; -use spk_schema::ident::LocatedBuildIdent; +use spk_schema::ident::{LocatedBuildIdent, PkgRequest, PreReleasePolicy, RangeIdent, RequestedBy}; use spk_schema::ident_component::Component; -use variantly::Variantly; /// This allows for storing strings of different types but hash and compare by /// the underlying strings. @@ -87,57 +86,62 @@ impl std::fmt::Display for RequestVS { } } -/// Like `Component` but without the `All` variant. -#[derive(Clone, Debug, Eq, Hash, PartialEq, Variantly)] -pub(crate) enum ComponentWithoutAll { - Build, - Run, - Source, - Named(String), +/// The component portion of a package that can exist in a solution. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub(crate) enum SyntheticComponent { + /// This represents the "parent" of any components of a package, used to + /// prevent components from different versions of a package from getting + /// into the same solution. + Base, + /// Real components. + Actual(Component), } -impl From for Component { - fn from(c: ComponentWithoutAll) -> Self { - match c { - ComponentWithoutAll::Build => Component::Build, - ComponentWithoutAll::Run => Component::Run, - ComponentWithoutAll::Source => Component::Source, - ComponentWithoutAll::Named(name) => Component::Named(name), - } +impl SyntheticComponent { + #[inline] + pub(crate) fn is_all(&self) -> bool { + matches!(self, SyntheticComponent::Actual(Component::All)) } } -impl TryFrom for ComponentWithoutAll { - type Error = Component; - - fn try_from(c: Component) -> Result { - match c { - Component::All => Err(Component::All), - Component::Build => Ok(ComponentWithoutAll::Build), - Component::Run => Ok(ComponentWithoutAll::Run), - Component::Source => Ok(ComponentWithoutAll::Source), - Component::Named(name) => Ok(ComponentWithoutAll::Named(name)), - } +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub(crate) struct LocatedBuildIdentWithComponent { + pub(crate) ident: LocatedBuildIdent, + pub(crate) component: SyntheticComponent, +} + +impl LocatedBuildIdentWithComponent { + /// Create a request that will match this ident but with a different + /// component name. + pub(crate) fn as_request_with_components( + &self, + components: impl IntoIterator, + ) -> Request { + let mut range_ident = RangeIdent::double_equals(&self.ident.to_any_ident(), components); + range_ident.repository_name = Some(self.ident.repository_name().to_owned()); + + let mut pkg_request = PkgRequest::new( + range_ident, + RequestedBy::BinaryBuild(self.ident.target().clone()), + ); + // Since we're using double_equals, is it safe to always enable + // prereleases? If self represents a prerelease, then the Request + // needs to allow it. + pkg_request.prerelease_policy = Some(PreReleasePolicy::IncludeAll); + + Request::Pkg(pkg_request) } } -impl std::fmt::Display for ComponentWithoutAll { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - ComponentWithoutAll::Build => write!(f, "build"), - ComponentWithoutAll::Run => write!(f, "run"), - ComponentWithoutAll::Source => write!(f, "source"), - ComponentWithoutAll::Named(name) => write!(f, "{name}"), +impl PartialEq for Component { + fn eq(&self, other: &SyntheticComponent) -> bool { + match other { + SyntheticComponent::Base => false, + SyntheticComponent::Actual(other) => self == other, } } } -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub(crate) struct LocatedBuildIdentWithComponent { - pub(crate) ident: LocatedBuildIdent, - pub(crate) component: ComponentWithoutAll, -} - #[derive(Clone, Debug, Eq, Hash, PartialEq)] pub(crate) enum SpkSolvable { LocatedBuildIdentWithComponent(LocatedBuildIdentWithComponent), @@ -161,6 +165,11 @@ impl VersionSet for RequestVS { impl std::fmt::Display for LocatedBuildIdentWithComponent { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{}:{}", self.ident, self.component) + match &self.component { + SyntheticComponent::Base => self.ident.fmt(f), + SyntheticComponent::Actual(component) => { + write!(f, "{}:{component}", self.ident) + } + } } } diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index a2a2345780..604eecf08c 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -22,7 +22,8 @@ use resolvo::{ VersionSetUnionId, }; use spk_schema::foundation::pkg_name; -use spk_schema::ident::{LocatedBuildIdent, PinnableValue, Satisfy, VarRequest}; +use spk_schema::ident::{LocatedBuildIdent, PinnableValue, PkgRequest, Satisfy, VarRequest}; +use spk_schema::ident_component::Component; use spk_schema::name::{OptNameBuf, PkgNameBuf}; use spk_schema::{Opt, Package, Request, VersionIdent}; use spk_storage::RepositoryHandle; @@ -31,6 +32,7 @@ use super::pkg_request_version_set::{ LocatedBuildIdentWithComponent, RequestVS, SpkSolvable, + SyntheticComponent, VarValue, }; @@ -39,8 +41,30 @@ use super::pkg_request_version_set::{ // however since it is a legal package name that can't be guaranteed. const PSEUDO_PKG_NAME_PREFIX: &str = "global-vars--"; +// Using just the package name as a Resolvo "package name" prevents multiple +// components from the same package from existing in the same solution, since +// we consider the different components to be different "solvables". Instead, +// treat different components of a package as separate packages. There needs to +// be a relationship between every component of a package and a "base" +// component, to prevent a solve containing a mix of components from different +// versions of the same package. +#[derive(Clone, Eq, Hash, PartialEq)] +pub(crate) struct PkgNameBufWithComponent { + pub(crate) name: PkgNameBuf, + pub(crate) component: SyntheticComponent, +} + +impl std::fmt::Display for PkgNameBufWithComponent { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match &self.component { + SyntheticComponent::Base => write!(f, "{}", self.name), + SyntheticComponent::Actual(component) => write!(f, "{}:{component}", self.name), + } + } +} + pub(crate) struct SpkProvider { - pub(crate) pool: Pool, + pub(crate) pool: Pool, repos: Vec>, /// Global options, like what might be specified with `--opt` to `spk env`. /// Indexed by name. If multiple requests happen to exist with the same @@ -70,6 +94,38 @@ impl SpkProvider { } } + fn pkg_request_to_known_dependencies(&self, pkg_request: &PkgRequest) -> KnownDependencies { + let mut components = pkg_request.pkg.components.iter().peekable(); + let iter = if components.peek().is_some() { + itertools::Either::Right(components.cloned().map(SyntheticComponent::Actual)) + } else { + itertools::Either::Left(std::iter::once(SyntheticComponent::Base)) + }; + let mut known_deps = KnownDependencies { + requirements: Vec::new(), + constrains: Vec::new(), + }; + for component in iter { + let dep_name = self.pool.intern_package_name(PkgNameBufWithComponent { + name: pkg_request.pkg.name().to_owned(), + component, + }); + let dep_vs = self.pool.intern_version_set( + dep_name, + RequestVS::SpkRequest(Request::Pkg(pkg_request.clone())), + ); + match pkg_request.inclusion_policy { + spk_schema::ident::InclusionPolicy::Always => { + known_deps.requirements.push(dep_vs.into()); + } + spk_schema::ident::InclusionPolicy::IfAlreadyPresent => { + known_deps.constrains.push(dep_vs); + } + } + } + known_deps + } + pub fn pkg_requirements(&self, requests: &[Request]) -> Vec { requests .iter() @@ -77,12 +133,7 @@ impl SpkProvider { Request::Pkg(pkg) => Some(pkg), _ => None, }) - .map(|req| { - let dep_name = self.pool.intern_package_name(req.pkg.name().to_owned()); - self.pool - .intern_version_set(dep_name, RequestVS::SpkRequest(Request::Pkg(req.clone()))) - .into() - }) + .flat_map(|req| self.pkg_request_to_known_dependencies(req).requirements) .collect() } @@ -115,7 +166,10 @@ impl SpkProvider { .filter_map(|req| match req.var.namespace() { Some(pkg_name) => { // A global request applicable to a specific package. - let dep_name = self.pool.intern_package_name(pkg_name.to_owned()); + let dep_name = self.pool.intern_package_name(PkgNameBufWithComponent { + name: pkg_name.to_owned(), + component: SyntheticComponent::Base, + }); Some(self.pool.intern_version_set( dep_name, RequestVS::SpkRequest(Request::Var(req.clone())), @@ -173,6 +227,58 @@ impl DependencyProvider for SpkProvider { continue; } + // Only select All component for requests of All + // component. + if located_build_ident_with_component.component.is_all() { + if pkg_request.pkg.components.contains(&Component::All) ^ inverse { + selected.push(*candidate); + } + continue; + } else + // Only the All component can satisfy requests for All. + if pkg_request.pkg.components.contains(&Component::All) { + if inverse { + selected.push(*candidate); + } + continue; + } + + // Only the x component can satisfy requests for x. + let mut at_least_one_request_matched_this_solvable = None; + for component in pkg_request.pkg.components.iter() { + if component.is_all() { + continue; + } + if component == &located_build_ident_with_component.component { + at_least_one_request_matched_this_solvable = Some(true); + break; + } else { + at_least_one_request_matched_this_solvable = Some(false); + } + } + + match at_least_one_request_matched_this_solvable { + Some(true) => { + if inverse { + continue; + } + } + Some(false) => { + // The request is for specific components but + // this solvable doesn't match any of them. + if inverse { + selected.push(*candidate); + continue; + } + } + None => { + // TODO: if at_least_one_request_matched_this_solvable + // is None it means the request didn't specify a + // component. Decide which specific component this + // should match. + } + } + // XXX: This find runtime will add up. let repo = self .repos @@ -282,7 +388,7 @@ impl DependencyProvider for SpkProvider { async fn get_candidates(&self, name: NameId) -> Option { let pkg_name = self.pool.resolve_package_name(name); - if let Some(key) = pkg_name.strip_prefix(PSEUDO_PKG_NAME_PREFIX) { + if let Some(key) = pkg_name.name.strip_prefix(PSEUDO_PKG_NAME_PREFIX) { self.queried_global_var_values .borrow_mut() .insert(key.to_owned()); @@ -320,12 +426,12 @@ impl DependencyProvider for SpkProvider { for repo in &self.repos { let versions = repo - .list_package_versions(pkg_name) + .list_package_versions(&pkg_name.name) .await .unwrap_or_default(); for version in versions.iter() { // TODO: We need a borrowing version of this to avoid cloning. - let pkg_version = VersionIdent::new(pkg_name.clone(), (**version).clone()); + let pkg_version = VersionIdent::new(pkg_name.name.clone(), (**version).clone()); let builds = repo .list_package_builds(&pkg_version) @@ -333,14 +439,31 @@ impl DependencyProvider for SpkProvider { .unwrap_or_default(); for build in builds { - let components = repo.list_build_components(&build).await.unwrap_or_default(); - let located_build_ident = LocatedBuildIdent::new(repo.name().to_owned(), build); - for component in components { + let located_build_ident = + LocatedBuildIdent::new(repo.name().to_owned(), build.clone()); + if let SyntheticComponent::Actual(pkg_name_component) = &pkg_name.component { + let components = + repo.list_build_components(&build).await.unwrap_or_default(); + for component in components.into_iter().chain( + // A build representing the All component is included so + // when a request for it is found it can act as a + // surrogate that depends on all the individual + // components. + [Component::All], + ) { + if component != *pkg_name_component { + continue; + } + + located_builds.push(LocatedBuildIdentWithComponent { + ident: located_build_ident.clone(), + component: pkg_name.component.clone(), + }); + } + } else { located_builds.push(LocatedBuildIdentWithComponent { - ident: located_build_ident.clone(), - component: component - .try_into() - .expect("list_build_components will never return an All component"), + ident: located_build_ident, + component: SyntheticComponent::Base, }); } } @@ -410,6 +533,9 @@ impl DependencyProvider for SpkProvider { async fn sort_candidates(&self, _solver: &SolverCache, solvables: &mut [SolvableId]) { // This implementation just picks the highest version. + // TODO: The ordering should take component names into account, so + // the run component or the build component is tried first in the + // appropriate situations. solvables.sort_by(|a, b| { let a = self.pool.resolve_solvable(*a); let b = self.pool.resolve_solvable(*b); @@ -445,13 +571,41 @@ impl DependencyProvider for SpkProvider { } async fn get_dependencies(&self, solvable: SolvableId) -> Dependencies { - // TODO: get dependencies! let solvable = self.pool.resolve_solvable(solvable); let SpkSolvable::LocatedBuildIdentWithComponent(located_build_ident_with_component) = &solvable.record else { return Dependencies::Known(KnownDependencies::default()); }; + if let SyntheticComponent::Base = located_build_ident_with_component.component { + let component_to_target = + if located_build_ident_with_component.ident.build().is_source() { + Component::Source + } else { + // XXX: hard coding a dependency on run for now. + Component::Run + }; + + let dep_name = self.pool.intern_package_name(PkgNameBufWithComponent { + name: located_build_ident_with_component.ident.name().to_owned(), + component: SyntheticComponent::Actual(component_to_target.clone()), + }); + let known_deps = KnownDependencies { + requirements: vec![ + self.pool + .intern_version_set( + dep_name, + RequestVS::SpkRequest( + located_build_ident_with_component + .as_request_with_components([component_to_target]), + ), + ) + .into(), + ], + constrains: Vec::new(), + }; + return Dependencies::Known(known_deps); + } // XXX: This find runtime will add up. let repo = self .repos @@ -468,6 +622,51 @@ impl DependencyProvider for SpkProvider { // This is where IfAlreadyPresent constraints would go. constrains: Vec::with_capacity(package.get_build_options().len()), }; + if located_build_ident_with_component.component.is_all() { + // The only dependencies of the All component are the other + // components defined in the package. + for component_spec in package.components().iter() { + let dep_name = self.pool.intern_package_name(PkgNameBufWithComponent { + name: located_build_ident_with_component.ident.name().to_owned(), + component: SyntheticComponent::Actual(component_spec.name.clone()), + }); + known_deps.requirements.push( + self.pool + .intern_version_set( + dep_name, + RequestVS::SpkRequest( + located_build_ident_with_component + .as_request_with_components([component_spec + .name + .clone()]), + ), + ) + .into(), + ); + } + return Dependencies::Known(known_deps); + } else if let SyntheticComponent::Actual(ref _solvable_component) = + located_build_ident_with_component.component + { + // For any non-All/non-Base component, add a dependency on + // the base to ensure all components come from the same + // base version. + let dep_name = self.pool.intern_package_name(PkgNameBufWithComponent { + name: located_build_ident_with_component.ident.name().to_owned(), + component: SyntheticComponent::Base, + }); + known_deps.requirements.push( + self.pool + .intern_version_set( + dep_name, + RequestVS::SpkRequest( + located_build_ident_with_component + .as_request_with_components([]), + ), + ) + .into(), + ); + } for option in package.get_build_options() { let Opt::Var(var_opt) = option else { continue; @@ -495,9 +694,10 @@ impl DependencyProvider for SpkProvider { // in the list of candidates. self.cancel_solving.set(true); } - let dep_name = self - .pool - .intern_package_name(pkg_name!(&pseudo_pkg_name).to_owned()); + let dep_name = self.pool.intern_package_name(PkgNameBufWithComponent { + name: pkg_name!(&pseudo_pkg_name).to_owned(), + component: SyntheticComponent::Base, + }); // Add a constraint not a dependency because the package // is targeting a specific global var value but there may // not be a request for that var of a specific value. @@ -512,21 +712,9 @@ impl DependencyProvider for SpkProvider { for requirement in package.runtime_requirements().iter() { match requirement { Request::Pkg(pkg_request) => { - let dep_name = self - .pool - .intern_package_name(pkg_request.pkg.name().to_owned()); - let dep_vs = self.pool.intern_version_set( - dep_name, - RequestVS::SpkRequest(requirement.clone()), - ); - match pkg_request.inclusion_policy { - spk_schema::ident::InclusionPolicy::Always => { - known_deps.requirements.push(dep_vs.into()); - } - spk_schema::ident::InclusionPolicy::IfAlreadyPresent => { - known_deps.constrains.push(dep_vs); - } - }; + let kd = self.pkg_request_to_known_dependencies(pkg_request); + known_deps.requirements.extend(kd.requirements); + known_deps.constrains.extend(kd.constrains); } Request::Var(var_request) => { match &var_request.value { @@ -535,7 +723,10 @@ impl DependencyProvider for SpkProvider { spk_schema::ident::PinnableValue::Pinned(value) => { let dep_name = match var_request.var.namespace() { Some(pkg_name) => { - self.pool.intern_package_name(pkg_name.to_owned()) + self.pool.intern_package_name(PkgNameBufWithComponent { + name: pkg_name.to_owned(), + component: SyntheticComponent::Base, + }) } None => { // Since we will be adding @@ -564,7 +755,10 @@ impl DependencyProvider for SpkProvider { self.cancel_solving.set(true); } let dep_name = self.pool.intern_package_name( - pkg_name!(&pseudo_pkg_name).to_owned(), + PkgNameBufWithComponent { + name: pkg_name!(&pseudo_pkg_name).to_owned(), + component: SyntheticComponent::Base, + }, ); known_deps.requirements.push( self.pool diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index 0ee86fd2a9..6739667afc 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -2231,8 +2231,6 @@ async fn test_solver_src_package_request_when_no_components_requested( #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_all_component(#[case] mut solver: SolverImpl) { From 9027e5d56095e105a2fd2fcbcf53bb4259c6d37b Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Thu, 6 Feb 2025 23:27:14 -0800 Subject: [PATCH 20/64] This other component test passes reliably now Though the use of the run component is still hard coded. Signed-off-by: J Robert Ray --- crates/spk-solve/src/solvers/solver_test.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index 6739667afc..2ca4c8adb3 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -2143,8 +2143,7 @@ async fn test_solver_components_interaction_with_embeds(#[case] mut solver: Solv #[rstest] #[case::step(step_solver())] -// This test can sometimes succeed by chance so can't use #[should_panic] here -// TODO #[case::resolvo(resolvo_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_components_when_no_components_requested(#[case] mut solver: SolverImpl) { // test when a package is requested with no components and the From b946576ec69b32a8ef3cc25839249d2ca3fde696 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Thu, 6 Feb 2025 23:31:06 -0800 Subject: [PATCH 21/64] Implement tracking component "uses" as dependencies Pass another test. Signed-off-by: J Robert Ray --- Cargo.lock | 1 + crates/spk-solve/Cargo.toml | 1 + .../src/solvers/resolvo/spk_provider.rs | 74 +++++++++++-------- crates/spk-solve/src/solvers/solver_test.rs | 8 +- 4 files changed, 52 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 65c9b582d8..6d662a2905 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4541,6 +4541,7 @@ dependencies = [ "spk-storage", "statsd 0.15.0", "strip-ansi-escapes 0.2.0", + "tap", "thiserror", "tokio", "tracing", diff --git a/crates/spk-solve/Cargo.toml b/crates/spk-solve/Cargo.toml index 83b98d8822..f88dbbf4e4 100644 --- a/crates/spk-solve/Cargo.toml +++ b/crates/spk-solve/Cargo.toml @@ -65,3 +65,4 @@ variantly = { workspace = true } rstest = { workspace = true } spk-solve-macros = { workspace = true } strip-ansi-escapes = { workspace = true } +tap = { workspace = true } diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index 604eecf08c..573a1783f1 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -99,7 +99,21 @@ impl SpkProvider { let iter = if components.peek().is_some() { itertools::Either::Right(components.cloned().map(SyntheticComponent::Actual)) } else { - itertools::Either::Left(std::iter::once(SyntheticComponent::Base)) + itertools::Either::Left( + // A request with no components is assumed to be a request for + // the run (or source) component. + if pkg_request + .pkg + .build + .as_ref() + .map(|build| build.is_source()) + .unwrap_or(false) + { + std::iter::once(SyntheticComponent::Actual(Component::Source)) + } else { + std::iter::once(SyntheticComponent::Actual(Component::Run)) + }, + ) }; let mut known_deps = KnownDependencies { requirements: Vec::new(), @@ -578,33 +592,10 @@ impl DependencyProvider for SpkProvider { return Dependencies::Known(KnownDependencies::default()); }; if let SyntheticComponent::Base = located_build_ident_with_component.component { - let component_to_target = - if located_build_ident_with_component.ident.build().is_source() { - Component::Source - } else { - // XXX: hard coding a dependency on run for now. - Component::Run - }; - - let dep_name = self.pool.intern_package_name(PkgNameBufWithComponent { - name: located_build_ident_with_component.ident.name().to_owned(), - component: SyntheticComponent::Actual(component_to_target.clone()), - }); - let known_deps = KnownDependencies { - requirements: vec![ - self.pool - .intern_version_set( - dep_name, - RequestVS::SpkRequest( - located_build_ident_with_component - .as_request_with_components([component_to_target]), - ), - ) - .into(), - ], - constrains: Vec::new(), - }; - return Dependencies::Known(known_deps); + // Base can't depend on anything because we don't know what + // components actually exist or if requests exist for whatever it + // was we picked if we were to pick a component to depend on. + return Dependencies::Known(KnownDependencies::default()); } // XXX: This find runtime will add up. let repo = self @@ -645,7 +636,7 @@ impl DependencyProvider for SpkProvider { ); } return Dependencies::Known(known_deps); - } else if let SyntheticComponent::Actual(ref _solvable_component) = + } else if let SyntheticComponent::Actual(ref solvable_component) = located_build_ident_with_component.component { // For any non-All/non-Base component, add a dependency on @@ -666,6 +657,31 @@ impl DependencyProvider for SpkProvider { ) .into(), ); + // Also add dependencies on any components that this + // component "uses". + package + .components() + .iter() + .find(|component_spec| component_spec.name == *solvable_component) + .into_iter() + .flat_map(|component_spec| component_spec.uses.iter()) + .for_each(|uses| { + let dep_name = self.pool.intern_package_name(PkgNameBufWithComponent { + name: located_build_ident_with_component.ident.name().to_owned(), + component: SyntheticComponent::Actual(uses.clone()), + }); + known_deps.requirements.push( + self.pool + .intern_version_set( + dep_name, + RequestVS::SpkRequest( + located_build_ident_with_component + .as_request_with_components([uses.clone()]), + ), + ) + .into(), + ); + }); } for option in package.get_build_options() { let Opt::Var(var_opt) = option else { diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index 2ca4c8adb3..a699089afe 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -26,6 +26,7 @@ use spk_solve_macros::{make_build, make_build_and_components, make_package, make use spk_solve_solution::PackageSource; use spk_storage::RepositoryHandle; use spk_storage::fixtures::*; +use tap::prelude::*; use crate::io::DecisionFormatterBuilder; use crate::solver::{Solver, SolverImpl}; @@ -2273,8 +2274,6 @@ async fn test_solver_all_component(#[case] mut solver: SolverImpl) { #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_component_availability(#[case] mut solver: SolverImpl) { @@ -2330,7 +2329,10 @@ async fn test_solver_component_availability(#[case] mut solver: SolverImpl) { solver.add_repository(Arc::new(repo)); solver.add_request(request!("python:bin")); - let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver) + .await + .tap_err(|e| eprintln!("{e}")) + .unwrap(); assert_resolved!( solution, From 10b175fb26f61a21d4dc380fddcf334d1ddc19ce Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Fri, 7 Feb 2025 00:08:28 -0800 Subject: [PATCH 22/64] Implement tracking component requirements as dependencies Makes two tests pass. Signed-off-by: J Robert Ray --- .../spk-solve/src/solvers/resolvo/spk_provider.rs | 13 ++++++++----- crates/spk-solve/src/solvers/solver_test.rs | 4 ---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index 573a1783f1..58c71611d3 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -658,14 +658,13 @@ impl DependencyProvider for SpkProvider { .into(), ); // Also add dependencies on any components that this - // component "uses". - package + // component "uses" and its install requirements. + if let Some(component_spec) = package .components() .iter() .find(|component_spec| component_spec.name == *solvable_component) - .into_iter() - .flat_map(|component_spec| component_spec.uses.iter()) - .for_each(|uses| { + { + component_spec.uses.iter().for_each(|uses| { let dep_name = self.pool.intern_package_name(PkgNameBufWithComponent { name: located_build_ident_with_component.ident.name().to_owned(), component: SyntheticComponent::Actual(uses.clone()), @@ -682,6 +681,10 @@ impl DependencyProvider for SpkProvider { .into(), ); }); + known_deps + .requirements + .extend(self.pkg_requirements(&component_spec.requirements)); + } } for option in package.get_build_options() { let Opt::Var(var_opt) = option else { diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index a699089afe..c0f8e35f0e 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -2345,8 +2345,6 @@ async fn test_solver_component_availability(#[case] mut solver: SolverImpl) { #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_component_requirements(#[case] mut solver: SolverImpl) { @@ -2397,8 +2395,6 @@ async fn test_solver_component_requirements(#[case] mut solver: SolverImpl) { #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_component_requirements_extending(#[case] mut solver: SolverImpl) { From 2479b76272b53aec906d7a824d52c143aaa00a5f Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Fri, 7 Feb 2025 00:23:02 -0800 Subject: [PATCH 23/64] Start to implement handling embedded packages A handful of tests now pass but `todo!`s remain. One test was modified to be less picky about how the build is populated when the solve includes an embedded package. The new solver populates the embedded package parent info instead of leaving it as "Unknown". Signed-off-by: J Robert Ray --- .../src/solvers/resolvo/spk_provider.rs | 89 ++++++++++++++++++- crates/spk-solve/src/solvers/solver_test.rs | 28 +++--- 2 files changed, 103 insertions(+), 14 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index 58c71611d3..fd4bede40b 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -3,7 +3,7 @@ // https://github.com/spkenv/spk use std::cell::{Cell, RefCell}; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::sync::Arc; use resolvo::utils::Pool; @@ -22,9 +22,20 @@ use resolvo::{ VersionSetUnionId, }; use spk_schema::foundation::pkg_name; -use spk_schema::ident::{LocatedBuildIdent, PinnableValue, PkgRequest, Satisfy, VarRequest}; +use spk_schema::ident::{ + LocatedBuildIdent, + PinnableValue, + PkgRequest, + RangeIdent, + RequestedBy, + Satisfy, + VarRequest, +}; +use spk_schema::ident_build::{Build, EmbeddedSource, EmbeddedSourcePackage}; use spk_schema::ident_component::Component; use spk_schema::name::{OptNameBuf, PkgNameBuf}; +use spk_schema::prelude::{HasVersion, Named}; +use spk_schema::version_range::{DoubleEqualsVersion, VersionFilter}; use spk_schema::{Opt, Package, Request, VersionIdent}; use spk_storage::RepositoryHandle; @@ -456,6 +467,19 @@ impl DependencyProvider for SpkProvider { let located_build_ident = LocatedBuildIdent::new(repo.name().to_owned(), build.clone()); if let SyntheticComponent::Actual(pkg_name_component) = &pkg_name.component { + if let Build::Embedded(EmbeddedSource::Package(parent)) = build.build() { + // Embedded packages don't have components, but + // their parent has a component. + if parent.components.contains(pkg_name_component) { + located_builds.push(LocatedBuildIdentWithComponent { + ident: located_build_ident, + component: SyntheticComponent::Actual( + pkg_name_component.clone(), + ), + }); + } + continue; + } let components = repo.list_build_components(&build).await.unwrap_or_default(); for component in components.into_iter().chain( @@ -686,6 +710,67 @@ impl DependencyProvider for SpkProvider { .extend(self.pkg_requirements(&component_spec.requirements)); } } + // Also add dependencies on any packages embedded in this + // component. + // XXX: This ignores the detail of embeds inside components for + // now. + for embedded in package.embedded().iter() { + let dep_name = self.pool.intern_package_name(PkgNameBufWithComponent { + name: embedded.name().to_owned(), + component: located_build_ident_with_component.component.clone(), + }); + known_deps.requirements.push( + self.pool + .intern_version_set( + dep_name, + RequestVS::SpkRequest(Request::Pkg(PkgRequest::new( + RangeIdent { + repository_name: Some( + located_build_ident_with_component + .ident + .repository_name() + .to_owned(), + ), + name: embedded.name().to_owned(), + components: Default::default(), + version: VersionFilter::single( + DoubleEqualsVersion::version_range( + embedded.version().clone(), + ), + ), + // This needs to match the build of + // the stub for get_candidates to like + // it. + build: Some(Build::Embedded(EmbeddedSource::Package( + Box::new(EmbeddedSourcePackage { + ident: package.ident().into(), + // XXX: Hard coded to Run for + // now. + components: BTreeSet::from_iter([Component::Run]), + }), + ))), + }, + RequestedBy::Embedded( + located_build_ident_with_component.ident.target().clone(), + ), + ))), + ) + .into(), + ); + // Any install requirements of components inside embedded + // packages with the same name as this component also + // become dependencies. + for embedded_component_requirement in embedded + .components() + .iter() + .filter(|embedded_component| { + embedded_component.name == located_build_ident_with_component.component + }) + .flat_map(|embedded_component| embedded_component.requirements.iter()) + { + todo!("{embedded_component_requirement:?}"); + } + } for option in package.get_build_options() { let Opt::Var(var_opt) = option else { continue; diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index c0f8e35f0e..6d68b1f8b5 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -67,6 +67,16 @@ macro_rules! assert_resolved { assert_eq!(pkg.spec.ident().build(), &$build, $message); }}; + ($solution:ident, $pkg:literal, build =~ $build:pat) => { + assert_resolved!($solution, $pkg, build =~ $build, "wrong package build was resolved") + }; + ($solution:ident, $pkg:literal, build =~ $build:pat, $message:literal) => {{ + let pkg = $solution + .get($pkg) + .expect("expected package to be in solution"); + assert!(matches!(pkg.spec.ident().build(), $build), $message); + }}; + ($solution:ident, $pkg:literal, components = [$($component:literal),+ $(,)?]) => {{ let mut resolved = std::collections::HashSet::::new(); let pkg = $solution @@ -510,8 +520,6 @@ async fn test_solver_dependency_already_satisfied(#[case] mut solver: SolverImpl #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_dependency_already_satisfied_conflicting_components( @@ -1282,8 +1290,6 @@ async fn test_solver_build_from_source_deprecated_and_impossible_initial_checks( #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_embedded_package_adds_request(#[case] mut solver: SolverImpl) { @@ -1304,18 +1310,21 @@ async fn test_solver_embedded_package_adds_request(#[case] mut solver: SolverImp solver.add_repository(Arc::new(repo)); solver.add_request(request!("maya")); - let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver) + .await + .tap_err(|e| eprintln!("{e}")) + .unwrap(); assert_resolved!( solution, "qt", - build = Build::Embedded(EmbeddedSource::Unknown) + build =~ Build::Embedded(_) ); assert_resolved!(solution, "qt", "5.12.6"); assert_resolved!( solution, "qt", - build = Build::Embedded(EmbeddedSource::Unknown) + build =~ Build::Embedded(_) ); } @@ -1361,8 +1370,6 @@ async fn test_solver_embedded_package_solvable(#[case] mut solver: SolverImpl) { #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_embedded_package_unsolvable(#[case] mut solver: SolverImpl) { @@ -1743,7 +1750,6 @@ async fn test_solver_some_versions_conflicting_requests(#[case] mut solver: Solv #[rstest] #[case::step(step_solver())] -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_embedded_request_invalidates(#[case] mut solver: SolverImpl) { @@ -2633,8 +2639,6 @@ async fn test_solver_component_embedded_multiple_versions( #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_component_embedded_incompatible_requests(#[case] mut solver: SolverImpl) { From 9f7c0879655852bb21dcec9acdc923c76e939aaf Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Fri, 7 Feb 2025 17:13:17 -0800 Subject: [PATCH 24/64] More passing embedded package tests These tests pass after relaxing the build comparison like the other test. Signed-off-by: J Robert Ray --- crates/spk-solve/src/solvers/solver_test.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index 6d68b1f8b5..640445384d 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -1330,8 +1330,6 @@ async fn test_solver_embedded_package_adds_request(#[case] mut solver: SolverImp #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_embedded_package_solvable(#[case] mut solver: SolverImpl) { @@ -1364,7 +1362,7 @@ async fn test_solver_embedded_package_solvable(#[case] mut solver: SolverImpl) { assert_resolved!( solution, "qt", - build = Build::Embedded(EmbeddedSource::Unknown) + build =~ Build::Embedded(_) ); } @@ -1405,7 +1403,6 @@ async fn test_solver_embedded_package_unsolvable(#[case] mut solver: SolverImpl) #[rstest] #[case::step(step_solver())] -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_embedded_package_replaces_real_package(#[case] mut solver: SolverImpl) { @@ -1461,7 +1458,7 @@ async fn test_solver_embedded_package_replaces_real_package(#[case] mut solver: assert_resolved!( solution, "qt", - build = Build::Embedded(EmbeddedSource::Unknown) + build =~ Build::Embedded(_) ); assert_not_resolved!(solution, "unwanted-dep"); } From 9631f886c75c9ea143016108764611450b9a4edb Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Fri, 7 Feb 2025 17:21:29 -0800 Subject: [PATCH 25/64] Another embedded package test passing This test was expecting a solver error but the new solver doesn't fail with a GraphError. Signed-off-by: J Robert Ray --- crates/spk-solve/src/error.rs | 2 ++ crates/spk-solve/src/solvers/resolvo/mod.rs | 2 +- crates/spk-solve/src/solvers/solver_test.rs | 5 +++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/spk-solve/src/error.rs b/crates/spk-solve/src/error.rs index d4545a954f..47270af9d1 100644 --- a/crates/spk-solve/src/error.rs +++ b/crates/spk-solve/src/error.rs @@ -65,6 +65,8 @@ pub enum Error { SolverLogFileIOError(#[source] std::io::Error, PathBuf), #[error("Error: Flushing solver log file: {0}")] SolverLogFileFlushError(#[source] std::io::Error), + #[error("Failed to resolve: {0}")] + FailedToResolve(String), } impl From for Error { diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index 84a9f5a5c0..8255b47d96 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -106,7 +106,7 @@ impl SolverTrait for Solver { provider = Some(solver.provider().reset()); continue; } - return Err(Error::String(format!( + return Err(Error::FailedToResolve(format!( "{}", conflict.display_user_friendly(&solver) ))); diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index 640445384d..9203505994 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -1604,8 +1604,6 @@ async fn test_solver_impossible_request_but_embedded_package_makes_solvable( /// panic. #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_multiple_packages_embed_same_package( @@ -1652,8 +1650,11 @@ async fn test_multiple_packages_embed_same_package( } match run_and_print_resolve_for_tests(&mut solver).await { + // step solver's error Err(Error::GraphError(ref graph_err)) if matches!(&**graph_err, spk_solve_graph::Error::FailedToResolve(_)) => {} + // resolvo solver's error + Err(Error::FailedToResolve(_)) => {} Ok(_) => { panic!("No solution expected"); } From 2e6f4d3c4bf35b0ab9603b04d948686cd1b555a7 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Fri, 7 Feb 2025 17:38:08 -0800 Subject: [PATCH 26/64] More work on solving embedded packages Pass one more test, handling the mix of embedded packages and components. There's still that `todo!` in the middle of `get_dependencies` that no test has run into yet. Signed-off-by: J Robert Ray --- crates/spk-solve/src/solvers/resolvo/mod.rs | 37 +++- .../src/solvers/resolvo/spk_provider.rs | 168 ++++++++++++++---- crates/spk-solve/src/solvers/solver_test.rs | 7 +- 3 files changed, 177 insertions(+), 35 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index 8255b47d96..0005100d98 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -17,12 +17,13 @@ mod pkg_request_version_set; mod spk_provider; use std::borrow::Cow; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::Arc; use pkg_request_version_set::{SpkSolvable, SyntheticComponent}; use spk_provider::SpkProvider; use spk_schema::ident::{InclusionPolicy, PinPolicy, PkgRequest, RangeIdent}; +use spk_schema::ident_component::Component; use spk_schema::prelude::{HasVersion, Named, Versioned}; use spk_schema::version_range::VersionFilter; use spk_schema::{OptionMap, Package, Request}; @@ -135,6 +136,12 @@ impl SolverTrait for Solver { let mut solution_options = OptionMap::default(); let mut solution_adds = Vec::with_capacity(solvables.len()); + // Keep track of the index of each package added to `solution_adds` in + // order to merge components. Components of a package come out of the + // solver as separate solvables. The solver logic guarantees that any + // two entries in this list with the same package name are for the same + // package and this merging of components is valid. + let mut seen_packages = HashMap::new(); for located_build_ident_with_component in solvables { let SyntheticComponent::Actual(solvable_component) = &located_build_ident_with_component.component @@ -142,6 +149,29 @@ impl SolverTrait for Solver { continue; }; + if let Some(existing_index) = + seen_packages.get(located_build_ident_with_component.ident.name()) + { + if let Some(( + PkgRequest { + pkg: RangeIdent { components, .. }, + .. + }, + _, + _, + )) = solution_adds.get_mut(*existing_index) + { + // If we visit a solvable for the "All" component, the + // solver guarantees that we will have all the components. + if !components.contains(&Component::All) { + components.insert(solvable_component.clone()); + } else if solvable_component.is_all() { + *components = BTreeSet::from([Component::All]); + } + } + continue; + } + let pkg_request = PkgRequest { pkg: RangeIdent { repository_name: None, @@ -193,6 +223,11 @@ impl SolverTrait for Solver { } } } + let next_index = solution_adds.len(); + seen_packages.insert( + located_build_ident_with_component.ident.name().to_owned(), + next_index, + ); solution_adds.push(( pkg_request, package, diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index fd4bede40b..e987c615af 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -4,6 +4,7 @@ use std::cell::{Cell, RefCell}; use std::collections::{BTreeSet, HashMap, HashSet}; +use std::str::FromStr; use std::sync::Arc; use resolvo::utils::Pool; @@ -35,8 +36,9 @@ use spk_schema::ident_build::{Build, EmbeddedSource, EmbeddedSourcePackage}; use spk_schema::ident_component::Component; use spk_schema::name::{OptNameBuf, PkgNameBuf}; use spk_schema::prelude::{HasVersion, Named}; +use spk_schema::version::Version; use spk_schema::version_range::{DoubleEqualsVersion, VersionFilter}; -use spk_schema::{Opt, Package, Request, VersionIdent}; +use spk_schema::{BuildIdent, Opt, Package, Request, VersionIdent}; use spk_storage::RepositoryHandle; use super::pkg_request_version_set::{ @@ -467,21 +469,31 @@ impl DependencyProvider for SpkProvider { let located_build_ident = LocatedBuildIdent::new(repo.name().to_owned(), build.clone()); if let SyntheticComponent::Actual(pkg_name_component) = &pkg_name.component { - if let Build::Embedded(EmbeddedSource::Package(parent)) = build.build() { - // Embedded packages don't have components, but - // their parent has a component. - if parent.components.contains(pkg_name_component) { - located_builds.push(LocatedBuildIdentWithComponent { - ident: located_build_ident, - component: SyntheticComponent::Actual( - pkg_name_component.clone(), - ), - }); - } - continue; - } - let components = - repo.list_build_components(&build).await.unwrap_or_default(); + let components = if let Build::Embedded(EmbeddedSource::Package(_parent)) = + build.build() + { + // Does this embedded stub contain the component + // being requested? For whatever reason, + // list_build_components returns an empty list for + // embedded stubs. + itertools::Either::Right( + if let Ok(stub) = repo.read_embed_stub(&build).await { + itertools::Either::Right( + stub.components() + .iter() + .map(|component_spec| component_spec.name.clone()) + .collect::>() + .into_iter(), + ) + } else { + itertools::Either::Left(std::iter::empty()) + }, + ) + } else { + itertools::Either::Left( + repo.list_build_components(&build).await.unwrap_or_default(), + ) + }; for component in components.into_iter().chain( // A build representing the All component is included so // when a request for it is found it can act as a @@ -615,12 +627,15 @@ impl DependencyProvider for SpkProvider { else { return Dependencies::Known(KnownDependencies::default()); }; - if let SyntheticComponent::Base = located_build_ident_with_component.component { - // Base can't depend on anything because we don't know what - // components actually exist or if requests exist for whatever it - // was we picked if we were to pick a component to depend on. - return Dependencies::Known(KnownDependencies::default()); - } + let actual_component = match &located_build_ident_with_component.component { + SyntheticComponent::Base => { + // Base can't depend on anything because we don't know what + // components actually exist or if requests exist for whatever it + // was we picked if we were to pick a component to depend on. + return Dependencies::Known(KnownDependencies::default()); + } + SyntheticComponent::Actual(component) => component, + }; // XXX: This find runtime will add up. let repo = self .repos @@ -660,9 +675,7 @@ impl DependencyProvider for SpkProvider { ); } return Dependencies::Known(known_deps); - } else if let SyntheticComponent::Actual(ref solvable_component) = - located_build_ident_with_component.component - { + } else { // For any non-All/non-Base component, add a dependency on // the base to ensure all components come from the same // base version. @@ -686,7 +699,7 @@ impl DependencyProvider for SpkProvider { if let Some(component_spec) = package .components() .iter() - .find(|component_spec| component_spec.name == *solvable_component) + .find(|component_spec| component_spec.name == *actual_component) { component_spec.uses.iter().for_each(|uses| { let dep_name = self.pool.intern_package_name(PkgNameBufWithComponent { @@ -712,8 +725,6 @@ impl DependencyProvider for SpkProvider { } // Also add dependencies on any packages embedded in this // component. - // XXX: This ignores the detail of embeds inside components for - // now. for embedded in package.embedded().iter() { let dep_name = self.pool.intern_package_name(PkgNameBufWithComponent { name: embedded.name().to_owned(), @@ -740,12 +751,11 @@ impl DependencyProvider for SpkProvider { ), // This needs to match the build of // the stub for get_candidates to like - // it. + // it. Stub parents are always the Run + // component. build: Some(Build::Embedded(EmbeddedSource::Package( Box::new(EmbeddedSourcePackage { ident: package.ident().into(), - // XXX: Hard coded to Run for - // now. components: BTreeSet::from_iter([Component::Run]), }), ))), @@ -771,6 +781,102 @@ impl DependencyProvider for SpkProvider { todo!("{embedded_component_requirement:?}"); } } + // If this solvable is an embedded stub and it is + // representing that it provides a component that lives in a + // component of the parent, then that parent component needs + // to be included in the solution. + if let Build::Embedded(EmbeddedSource::Package(parent)) = + located_build_ident_with_component.ident.build() + { + match actual_component { + Component::Run => { + // The Run component is the default "home" of + // embedded packages, no dependency needed in this + // case. + } + component => 'invalid_parent: { + // XXX: Do we not have a convenient way to read the + // parent package from an embedded stub ident? + let Ok(pkg_name) = PkgNameBuf::from_str(&parent.ident.pkg_name) else { + break 'invalid_parent; + }; + let Some(version_str) = parent.ident.version_str.as_ref() else { + break 'invalid_parent; + }; + let Ok(version) = Version::from_str(version_str) else { + break 'invalid_parent; + }; + let Some(build_str) = parent.ident.build_str.as_ref() else { + break 'invalid_parent; + }; + let Ok(build) = Build::from_str(build_str) else { + break 'invalid_parent; + }; + let ident = BuildIdent::new( + VersionIdent::new(pkg_name, version), + build.clone(), + ); + let Ok(parent) = repo.read_package(&ident).await else { + break 'invalid_parent; + }; + // Look through the components of the parent to see + // if one (or more?) of them embeds this component. + for parent_component in parent.components().iter() { + parent_component + .embedded + .iter() + .filter(|embedded_package| { + embedded_package.pkg.name() + == located_build_ident_with_component.ident.name() + && embedded_package.components().contains(component) + }) + .for_each(|_embedded_package| { + let dep_name = self.pool.intern_package_name( + PkgNameBufWithComponent { + name: ident.name().to_owned(), + component: SyntheticComponent::Actual( + parent_component.name.clone(), + ), + }, + ); + known_deps.requirements.push( + self.pool.intern_version_set( + dep_name, + RequestVS::SpkRequest(Request::Pkg( + PkgRequest::new( + RangeIdent { + repository_name: Some( + located_build_ident_with_component + .ident + .repository_name() + .to_owned(), + ), + name: ident.name().to_owned(), + components: BTreeSet::from_iter([ + parent_component.name.clone(), + ]), + version: VersionFilter::single( + DoubleEqualsVersion::version_range( + ident.version().clone(), + ), + ), + build: Some(build.clone()), + }, + RequestedBy::Embedded( + located_build_ident_with_component + .ident + .target() + .clone(), + ), + ), + )) + ).into(), + ); + }); + } + } + } + } for option in package.get_build_options() { let Opt::Var(var_opt) = option else { continue; diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index 9203505994..043eab20c5 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -2062,8 +2062,6 @@ async fn test_solver_components(#[case] mut solver: SolverImpl) { #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_components_interaction_with_embeds(#[case] mut solver: SolverImpl) { @@ -2127,7 +2125,10 @@ async fn test_solver_components_interaction_with_embeds(#[case] mut solver: Solv solver.add_request(request!("fake-pkg:comp1")); solver.add_request(request!("victim")); - let Ok(solution) = run_and_print_resolve_for_tests(&mut solver).await else { + let Ok(solution) = run_and_print_resolve_for_tests(&mut solver) + .await + .tap_err(|e| eprintln!("{e}")) + else { panic!("Expected a valid solution"); }; From 5ac278c4387a0168fd0ba55dad12144b80404e5d Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Sat, 8 Feb 2025 00:33:13 -0800 Subject: [PATCH 27/64] Another test that passes after adjusting expected build Signed-off-by: J Robert Ray --- crates/spk-solve/src/solvers/solver_test.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index 043eab20c5..1f4212bb5e 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -2435,8 +2435,6 @@ async fn test_solver_component_requirements_extending(#[case] mut solver: Solver #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_component_embedded(#[case] mut solver: SolverImpl) { @@ -2487,7 +2485,7 @@ async fn test_solver_component_embedded(#[case] mut solver: SolverImpl) { assert_resolved!( solution, "dep-e1", - build = Build::Embedded(EmbeddedSource::Unknown) + build =~ Build::Embedded(_) ); solver.reset(); From e575f311c6fc6875efd20fcd9cdb3000e1b3731a Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Sat, 8 Feb 2025 00:33:13 -0800 Subject: [PATCH 28/64] Handle requirements of components in embedded packages This is the test that would run up against that `todo!`. Signed-off-by: J Robert Ray --- .../src/solvers/resolvo/spk_provider.rs | 166 +++++++++--------- crates/spk-solve/src/solvers/solver_test.rs | 10 +- 2 files changed, 88 insertions(+), 88 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index e987c615af..3a21978aa1 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -168,6 +168,82 @@ impl SpkProvider { self.cancel_solving.get() } + fn request_to_known_dependencies(&self, requirement: &Request) -> KnownDependencies { + let mut known_deps = KnownDependencies::default(); + match requirement { + Request::Pkg(pkg_request) => { + let kd = self.pkg_request_to_known_dependencies(pkg_request); + known_deps.requirements.extend(kd.requirements); + known_deps.constrains.extend(kd.constrains); + } + Request::Var(var_request) => { + match &var_request.value { + spk_schema::ident::PinnableValue::FromBuildEnv => todo!(), + spk_schema::ident::PinnableValue::FromBuildEnvIfPresent => todo!(), + spk_schema::ident::PinnableValue::Pinned(value) => { + let dep_name = match var_request.var.namespace() { + Some(pkg_name) => { + self.pool.intern_package_name(PkgNameBufWithComponent { + name: pkg_name.to_owned(), + component: SyntheticComponent::Base, + }) + } + None => { + // Since we will be adding constraints for + // global vars we need to add the pseudo-package + // to the dependency list so it will influence + // decisions. + let pseudo_pkg_name = format!( + "{PSEUDO_PKG_NAME_PREFIX}{}", + var_request.var.base_name() + ); + if self + .known_global_var_values + .borrow_mut() + .entry(var_request.var.base_name().to_owned()) + .or_default() + .insert(VarValue::ArcStr(Arc::clone(value))) + && self + .queried_global_var_values + .borrow() + .contains(var_request.var.base_name()) + { + // Seeing a new value for a var that has + // already locked in the list of candidates. + self.cancel_solving.set(true); + } + let dep_name = + self.pool.intern_package_name(PkgNameBufWithComponent { + name: pkg_name!(&pseudo_pkg_name).to_owned(), + component: SyntheticComponent::Base, + }); + known_deps.requirements.push( + self.pool + .intern_version_set( + dep_name, + RequestVS::GlobalVar { + key: var_request.var.base_name().to_owned(), + value: VarValue::ArcStr(Arc::clone(value)), + }, + ) + .into(), + ); + dep_name + } + }; + // If we end up adding pkg_name to the solve, it needs + // to satisfy this var request. + known_deps.constrains.push(self.pool.intern_version_set( + dep_name, + RequestVS::SpkRequest(requirement.clone()), + )); + } + } + } + } + known_deps + } + /// Return a new provider to restart the solve, preserving what was learned /// about global variables. pub fn reset(&self) -> Self { @@ -773,12 +849,12 @@ impl DependencyProvider for SpkProvider { for embedded_component_requirement in embedded .components() .iter() - .filter(|embedded_component| { - embedded_component.name == located_build_ident_with_component.component - }) + .filter(|embedded_component| embedded_component.name == *actual_component) .flat_map(|embedded_component| embedded_component.requirements.iter()) { - todo!("{embedded_component_requirement:?}"); + let kd = self.request_to_known_dependencies(embedded_component_requirement); + known_deps.requirements.extend(kd.requirements); + known_deps.constrains.extend(kd.constrains); } } // If this solvable is an embedded stub and it is @@ -920,85 +996,9 @@ impl DependencyProvider for SpkProvider { )); } for requirement in package.runtime_requirements().iter() { - match requirement { - Request::Pkg(pkg_request) => { - let kd = self.pkg_request_to_known_dependencies(pkg_request); - known_deps.requirements.extend(kd.requirements); - known_deps.constrains.extend(kd.constrains); - } - Request::Var(var_request) => { - match &var_request.value { - spk_schema::ident::PinnableValue::FromBuildEnv => todo!(), - spk_schema::ident::PinnableValue::FromBuildEnvIfPresent => todo!(), - spk_schema::ident::PinnableValue::Pinned(value) => { - let dep_name = match var_request.var.namespace() { - Some(pkg_name) => { - self.pool.intern_package_name(PkgNameBufWithComponent { - name: pkg_name.to_owned(), - component: SyntheticComponent::Base, - }) - } - None => { - // Since we will be adding - // constraints for global vars we - // need to add the pseudo-package - // to the dependency list so it - // will influence decisions. - let pseudo_pkg_name = format!( - "{PSEUDO_PKG_NAME_PREFIX}{}", - var_request.var.base_name() - ); - if self - .known_global_var_values - .borrow_mut() - .entry(var_request.var.base_name().to_owned()) - .or_default() - .insert(VarValue::ArcStr(Arc::clone(value))) - && self - .queried_global_var_values - .borrow() - .contains(var_request.var.base_name()) - { - // Seeing a new value for a var - // that has already locked in - // the list of candidates. - self.cancel_solving.set(true); - } - let dep_name = self.pool.intern_package_name( - PkgNameBufWithComponent { - name: pkg_name!(&pseudo_pkg_name).to_owned(), - component: SyntheticComponent::Base, - }, - ); - known_deps.requirements.push( - self.pool - .intern_version_set( - dep_name, - RequestVS::GlobalVar { - key: var_request - .var - .base_name() - .to_owned(), - value: VarValue::ArcStr(Arc::clone( - value, - )), - }, - ) - .into(), - ); - dep_name - } - }; - // If we end up adding pkg_name to the solve, - // it needs to satisfy this var request. - known_deps.constrains.push(self.pool.intern_version_set( - dep_name, - RequestVS::SpkRequest(requirement.clone()), - )); - } - } - } - } + let kd = self.request_to_known_dependencies(requirement); + known_deps.requirements.extend(kd.requirements); + known_deps.constrains.extend(kd.constrains); } Dependencies::Known(known_deps) } diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index 1f4212bb5e..230914153d 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -2505,10 +2505,7 @@ async fn test_solver_component_embedded(#[case] mut solver: SolverImpl) { #[case::comp2(&["mypkg:comp2", "dep-e1:comp2"], false)] #[tokio::test] async fn test_solver_component_embedded_component_requirements( - #[values(step_solver() - // TODO , resolvo_solver() - )] - mut solver: SolverImpl, + #[values(step_solver(), resolvo_solver())] mut solver: SolverImpl, #[case] packages_to_request: &[&str], #[case] expected_solve_result: bool, ) { @@ -2546,7 +2543,10 @@ async fn test_solver_component_embedded_component_requirements( solver.add_request(request!(package_to_request)); } - match run_and_print_resolve_for_tests(&mut solver).await { + match run_and_print_resolve_for_tests(&mut solver) + .await + .tap_err(|e| eprintln!("{e}")) + { Ok(solution) => { assert!(expected_solve_result, "expected solve to fail"); From 6ef3931ea8aa26221b8464e8d6a0b67321baa1b5 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Sat, 8 Feb 2025 00:33:13 -0800 Subject: [PATCH 29/64] Handle package embedding multiple versions of the same package Fill in missing version comparsion in the necessary places. All the tests involving embedded packages now pass! Signed-off-by: J Robert Ray --- .../src/solvers/resolvo/spk_provider.rs | 39 +++++++++++++++++++ crates/spk-solve/src/solvers/solver_test.rs | 12 +++--- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index 3a21978aa1..4b51a6e253 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -802,6 +802,34 @@ impl DependencyProvider for SpkProvider { // Also add dependencies on any packages embedded in this // component. for embedded in package.embedded().iter() { + // If this embedded package is configured to exist in + // specific components, then skip it if this solvable's + // component is not one of those. + let components_where_this_embedded_package_exists = package + .components() + .iter() + .filter_map(|component_spec| { + if component_spec.embedded.iter().any(|embedded_package| { + embedded_package.pkg.name() == embedded.name() + && embedded_package + .pkg + .target() + .as_ref() + .map(|version| version == embedded.version()) + .unwrap_or(true) + }) { + Some(component_spec.name.clone()) + } else { + None + } + }) + .collect::>(); + if !components_where_this_embedded_package_exists.is_empty() + && !components_where_this_embedded_package_exists.contains(actual_component) + { + continue; + } + let dep_name = self.pool.intern_package_name(PkgNameBufWithComponent { name: embedded.name().to_owned(), component: located_build_ident_with_component.component.clone(), @@ -904,6 +932,17 @@ impl DependencyProvider for SpkProvider { .filter(|embedded_package| { embedded_package.pkg.name() == located_build_ident_with_component.ident.name() + && embedded_package + .pkg + .target() + .as_ref() + .map(|version| { + version + == located_build_ident_with_component + .ident + .version() + }) + .unwrap_or(true) && embedded_package.components().contains(component) }) .for_each(|_embedded_package| { diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index 230914153d..0bfce0863c 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -2564,10 +2564,7 @@ async fn test_solver_component_embedded_component_requirements( #[case::downstream3("downstream3", false)] #[tokio::test] async fn test_solver_component_embedded_multiple_versions( - #[values(step_solver() - // TODO , resolvo_solver() - )] - mut solver: SolverImpl, + #[values(step_solver(), resolvo_solver())] mut solver: SolverImpl, #[case] package_to_request: &str, #[case] expected_solve_result: bool, ) { @@ -2618,14 +2615,17 @@ async fn test_solver_component_embedded_multiple_versions( solver.add_repository(repo); solver.add_request(request!(package_to_request)); - match run_and_print_resolve_for_tests(&mut solver).await { + match run_and_print_resolve_for_tests(&mut solver) + .await + .tap_err(|e| eprintln!("{e}")) + { Ok(solution) => { assert!(expected_solve_result, "expected solve to fail"); assert_resolved!( solution, "dep-e1", - build = Build::Embedded(EmbeddedSource::Unknown) + build =~ Build::Embedded(_) ); } Err(_) => { From 0379c8aa1440a9a6720f6f17f14033dae35b134f Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Sat, 8 Feb 2025 13:44:18 -0800 Subject: [PATCH 30/64] Handle basic build from source cases Signed-off-by: J Robert Ray --- crates/spk-solve/src/solvers/resolvo/mod.rs | 54 +++-- .../resolvo/pkg_request_version_set.rs | 20 +- .../src/solvers/resolvo/spk_provider.rs | 193 +++++++++++++++++- crates/spk-solve/src/solvers/solver_test.rs | 9 +- 4 files changed, 247 insertions(+), 29 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index 0005100d98..36e0f8f0c1 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -17,12 +17,12 @@ mod pkg_request_version_set; mod spk_provider; use std::borrow::Cow; -use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::sync::Arc; use pkg_request_version_set::{SpkSolvable, SyntheticComponent}; use spk_provider::SpkProvider; -use spk_schema::ident::{InclusionPolicy, PinPolicy, PkgRequest, RangeIdent}; +use spk_schema::ident::{InclusionPolicy, LocatedBuildIdent, PinPolicy, PkgRequest, RangeIdent}; use spk_schema::ident_component::Component; use spk_schema::prelude::{HasVersion, Named, Versioned}; use spk_schema::version_range::VersionFilter; @@ -42,7 +42,9 @@ mod resolvo_tests; pub struct Solver { repos: Vec>, requests: Vec, + binary_only: bool, _validators: Cow<'static, [Validators]>, + build_from_source_trail: HashSet, } impl Solver { @@ -50,9 +52,15 @@ impl Solver { Self { repos, requests: Vec::new(), + binary_only: true, _validators: validators, + build_from_source_trail: HashSet::new(), } } + + pub(crate) fn set_build_from_source_trail(&mut self, trail: HashSet) { + self.build_from_source_trail = trail; + } } #[async_trait::async_trait] @@ -74,16 +82,22 @@ impl SolverTrait for Solver { self._validators = Cow::from(default_validators()); } - fn set_binary_only(&mut self, _binary_only: bool) { - // TODO + fn set_binary_only(&mut self, binary_only: bool) { + self.binary_only = binary_only; } async fn solve(&mut self) -> Result { let repos = self.repos.clone(); let requests = self.requests.clone(); + let binary_only = self.binary_only; + let build_from_source_trail = self.build_from_source_trail.clone(); // Use a blocking thread so resolvo can call `block_on` on the runtime. let solvables = tokio::task::spawn_blocking(move || { - let mut provider = Some(SpkProvider::new(repos.clone())); + let mut provider = Some(SpkProvider::new( + repos.clone(), + binary_only, + build_from_source_trail, + )); let (solver, solved) = loop { let mut this_iter_provider = provider.take().expect("provider is always Some"); let pkg_requirements = this_iter_provider.pkg_requirements(&requests); @@ -228,17 +242,25 @@ impl SolverTrait for Solver { located_build_ident_with_component.ident.name().to_owned(), next_index, ); - solution_adds.push(( - pkg_request, - package, - PackageSource::Repository { - repo: Arc::clone(repo), - // XXX: Why is this needed? - components: repo - .read_components(located_build_ident_with_component.ident.target()) - .await?, - }, - )); + solution_adds.push((pkg_request, package, { + if located_build_ident_with_component.requires_build_from_source { + PackageSource::BuildFromSource { + recipe: repo + .read_recipe( + &located_build_ident_with_component.ident.to_version_ident(), + ) + .await?, + } + } else { + PackageSource::Repository { + repo: Arc::clone(repo), + // XXX: Why is this needed? + components: repo + .read_components(located_build_ident_with_component.ident.target()) + .await?, + } + } + })); } let mut solution = Solution::new(solution_options); for (pkg_request, package, source) in solution_adds { diff --git a/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs b/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs index dbe78a107f..0c3a1b1df9 100644 --- a/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs +++ b/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs @@ -104,10 +104,28 @@ impl SyntheticComponent { } } -#[derive(Clone, Debug, Eq, Hash, PartialEq)] +// The `requires_build_from_source` field is ignored for hashing and equality +// purposes. +#[derive(Clone, Debug)] pub(crate) struct LocatedBuildIdentWithComponent { pub(crate) ident: LocatedBuildIdent, pub(crate) component: SyntheticComponent, + pub(crate) requires_build_from_source: bool, +} + +impl std::hash::Hash for LocatedBuildIdentWithComponent { + fn hash(&self, state: &mut H) { + self.ident.hash(state); + self.component.hash(state); + } +} + +impl Eq for LocatedBuildIdentWithComponent {} + +impl PartialEq for LocatedBuildIdentWithComponent { + fn eq(&self, other: &Self) -> bool { + self.ident == other.ident && self.component == other.component + } } impl LocatedBuildIdentWithComponent { diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index 4b51a6e253..949d563fce 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/spkenv/spk +use std::borrow::Cow; use std::cell::{Cell, RefCell}; use std::collections::{BTreeSet, HashMap, HashSet}; use std::str::FromStr; @@ -37,9 +38,10 @@ use spk_schema::ident_component::Component; use spk_schema::name::{OptNameBuf, PkgNameBuf}; use spk_schema::prelude::{HasVersion, Named}; use spk_schema::version::Version; -use spk_schema::version_range::{DoubleEqualsVersion, VersionFilter}; -use spk_schema::{BuildIdent, Opt, Package, Request, VersionIdent}; +use spk_schema::version_range::{DoubleEqualsVersion, Ranged, VersionFilter, parse_version_range}; +use spk_schema::{BuildIdent, Opt, Package, Recipe, Request, VersionIdent}; use spk_storage::RepositoryHandle; +use tracing::{Instrument, debug_span}; use super::pkg_request_version_set::{ LocatedBuildIdentWithComponent, @@ -48,6 +50,7 @@ use super::pkg_request_version_set::{ SyntheticComponent, VarValue, }; +use crate::Solver; // "global-vars--" represents a pseudo- package that accumulates global var // constraints. The name is intended to never conflict with a real package name, @@ -76,6 +79,11 @@ impl std::fmt::Display for PkgNameBufWithComponent { } } +enum CanBuildFromSource { + Yes, + No(StringId), +} + pub(crate) struct SpkProvider { pub(crate) pool: Pool, repos: Vec>, @@ -92,10 +100,109 @@ pub(crate) struct SpkProvider { /// restarting the solve. queried_global_var_values: RefCell>, cancel_solving: Cell, + binary_only: bool, + /// When recursively exploring building packages from source, track chain + /// of packages to detect cycles. + build_from_source_trail: RefCell>, } impl SpkProvider { - pub fn new(repos: Vec>) -> Self { + /// Return if the given solvable is buildable from source considering the + /// existing requests. + async fn can_build_from_source(&self, ident: &LocatedBuildIdent) -> CanBuildFromSource { + if self.build_from_source_trail.borrow().contains(ident) { + return CanBuildFromSource::No( + self.pool + .intern_string(format!("cycle detected while building {ident} from source")), + ); + } + + // Get solver requirements from the recipe. + let recipe = match self + .repos + .iter() + .find(|repo| repo.name() == ident.repository_name()) + { + Some(repo) => match repo.read_recipe(&ident.clone().to_version_ident()).await { + Ok(recipe) => recipe, + Err(err) => { + return CanBuildFromSource::No( + self.pool + .intern_string(format!("failed to read recipe: {err}")), + ); + } + }, + None => { + return CanBuildFromSource::No( + self.pool + .intern_string("package's repository is not in the list of repositories"), + ); + } + }; + + // Do we try all the variants in the recipe? + let variants = recipe.default_variants( + // XXX: What should really go here? + &Default::default(), + ); + + let mut solve_errors = Vec::new(); + + for variant in variants.iter() { + let mut solver = super::Solver::new(self.repos.clone(), Cow::Borrowed(&[])); + solver.set_binary_only(false); + solver.set_build_from_source_trail(HashSet::from_iter( + self.build_from_source_trail + .borrow() + .iter() + .cloned() + .chain([ident.clone()]), + )); + + let build_requirements = match recipe.get_build_requirements(&variant) { + Ok(build_requirements) => build_requirements, + Err(err) => { + return CanBuildFromSource::No( + self.pool + .intern_string(format!("failed to get build requirements: {err}")), + ); + } + }; + + for request in build_requirements.iter() { + solver.add_request(request.clone()); + } + + // These are last to take priority over the requests in the recipe. + for request in self.global_var_requests.values() { + solver.add_request(Request::Var(request.clone())); + } + + match solver + .solve() + .instrument(debug_span!( + "recursive solve", + ident = ident.to_string(), + variant = variant.to_string() + )) + .await + { + Ok(_solution) => return CanBuildFromSource::Yes, + Err(err) => solve_errors.push(err), + }; + } + + CanBuildFromSource::No( + self.pool + .intern_string(format!("failed to build from source: {solve_errors:?}")), + ) + } + + pub fn new( + repos: Vec>, + binary_only: bool, + build_from_source_trail: HashSet, + ) -> Self { Self { pool: Pool::new(), repos, @@ -104,6 +211,8 @@ impl SpkProvider { known_global_var_values: Default::default(), queried_global_var_values: Default::default(), cancel_solving: Default::default(), + binary_only, + build_from_source_trail: RefCell::new(build_from_source_trail), } } @@ -255,6 +364,8 @@ impl SpkProvider { known_global_var_values: RefCell::new(self.known_global_var_values.take()), queried_global_var_values: Default::default(), cancel_solving: Default::default(), + binary_only: self.binary_only, + build_from_source_trail: self.build_from_source_trail.clone(), } } @@ -315,8 +426,21 @@ impl DependencyProvider for SpkProvider { let compatible = pkg_request .is_version_applicable(located_build_ident_with_component.ident.version()); if compatible.is_ok() { + let is_source = + located_build_ident_with_component.ident.build().is_source(); + + // If build from source is enabled, any source build is + // a candidate. Source builds that can't be built from + // source are filtered out in `get_candidates`. + if located_build_ident_with_component.requires_build_from_source { + if !inverse { + selected.push(*candidate); + } + continue; + } + // Only select source builds for requests of source builds. - if located_build_ident_with_component.ident.build().is_source() { + if is_source { if pkg_request .pkg .build @@ -577,19 +701,23 @@ impl DependencyProvider for SpkProvider { // components. [Component::All], ) { - if component != *pkg_name_component { + if component != *pkg_name_component + && (self.binary_only || !component.is_source()) + { continue; } located_builds.push(LocatedBuildIdentWithComponent { ident: located_build_ident.clone(), component: pkg_name.component.clone(), + requires_build_from_source: component != *pkg_name_component, }); } } else { located_builds.push(LocatedBuildIdentWithComponent { ident: located_build_ident, component: SyntheticComponent::Base, + requires_build_from_source: false, }); } } @@ -608,6 +736,7 @@ impl DependencyProvider for SpkProvider { for build in located_builds { // What we need from build before it is moved into the pool. let ident = build.ident.clone(); + let requires_build_from_source = build.requires_build_from_source; let solvable_id = *self .interned_solvables @@ -625,8 +754,47 @@ impl DependencyProvider for SpkProvider { .iter() .find(|repo| repo.name() == ident.repository_name()) .expect("Expected solved package's repository to be in the list of repositories"); + + if requires_build_from_source { + match self.can_build_from_source(&ident).await { + CanBuildFromSource::Yes => { + candidates.candidates.push(solvable_id); + } + CanBuildFromSource::No(reason) => { + candidates.excluded.push((solvable_id, reason)); + } + } + continue; + } + match repo.read_package(ident.target()).await { Ok(package) => { + // Filter builds that don't satisfy global var requests + if let Some(VarRequest { + value: PinnableValue::Pinned(expected_version), + .. + }) = self.global_var_requests.get(ident.name().as_opt_name()) + { + if let Ok(expected_version) = parse_version_range(expected_version) { + if let spk_schema::version::Compatibility::Incompatible( + incompatible_reason, + ) = expected_version.is_applicable(package.version()) + { + candidates.excluded.push(( + solvable_id, + self.pool.intern_string(format!( + "build version does not satisfy global var request: {incompatible_reason}" + )), + )); + continue; + } + } + } + + // XXX: `package.check_satisfies_request` walks the + // package's build options, so is it better to do this loop + // over `option_values` here, or loop over all the + // global_var_requests instead? for (opt_name, _value) in package.option_values() { if let Some(request) = self.global_var_requests.get(&opt_name) { if let spk_schema::version::Compatibility::Incompatible( @@ -636,7 +804,7 @@ impl DependencyProvider for SpkProvider { candidates.excluded.push(( solvable_id, self.pool.intern_string(format!( - "build does not satisfy global var request: {incompatible_reason}" + "build option {opt_name} does not satisfy global var request: {incompatible_reason}" )), )); continue; @@ -669,7 +837,18 @@ impl DependencyProvider for SpkProvider { ( SpkSolvable::LocatedBuildIdentWithComponent(a), SpkSolvable::LocatedBuildIdentWithComponent(b), - ) => b.ident.version().cmp(a.ident.version()), + ) => match b.ident.version().cmp(a.ident.version()) { + std::cmp::Ordering::Equal => { + // Sort source builds last + match (a.ident.build(), b.ident.build()) { + (Build::Source, Build::Source) => std::cmp::Ordering::Equal, + (Build::Source, _) => std::cmp::Ordering::Greater, + (_, Build::Source) => std::cmp::Ordering::Less, + _ => std::cmp::Ordering::Equal, + } + } + ord => ord, + }, ( SpkSolvable::GlobalVar { key: a_key, diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index 0bfce0863c..3b17e5b810 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -909,8 +909,6 @@ async fn test_solver_option_injection(#[case] mut solver: SolverImpl) { #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_build_from_source(#[case] mut solver: SolverImpl) { @@ -942,7 +940,10 @@ async fn test_solver_build_from_source(#[case] mut solver: SolverImpl) { solver.add_request(request!({"var": "debug/on"})); solver.add_request(request!("my-tool")); - let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); + let solution = run_and_print_resolve_for_tests(&mut solver) + .await + .tap_err(|e| eprintln!("{e}")) + .unwrap(); let resolved = solution.get("my-tool").unwrap(); assert!( @@ -1033,8 +1034,6 @@ async fn test_solver_build_from_source_unsolvable(#[case] mut solver: SolverImpl #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_build_from_source_dependency(#[case] mut solver: SolverImpl) { From 54588a83207eda6ef8d075574d8cade8d39845b4 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Sat, 8 Feb 2025 19:09:17 -0800 Subject: [PATCH 31/64] Explicitly enable build from source in tests Don't assume it is enabled by default. Currently it is not enabled by default in the new solver. Signed-off-by: J Robert Ray --- crates/spk-solve/src/solvers/solver_test.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index 3b17e5b810..3d8a91307f 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -1000,6 +1000,7 @@ async fn test_solver_build_from_source_unsolvable(#[case] mut solver: SolverImpl repo.publish_recipe(&recipe).await.unwrap(); solver.add_repository(Arc::new(repo)); + solver.set_binary_only(false); // the new option value should disqualify the existing build // and there is no 6.3 that can be resolved for this request solver.add_request(request!({"var": "gcc/6.3"})); @@ -1080,8 +1081,8 @@ async fn test_solver_build_from_source_dependency(#[case] mut solver: SolverImpl // but a new one should be generated for this set of options solver.update_options(option_map! {"debug" => "on"}); solver.add_repository(Arc::new(repo)); - solver.add_request(request!("my-tool")); solver.set_binary_only(false); + solver.add_request(request!("my-tool")); let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); @@ -1208,6 +1209,7 @@ async fn test_solver_build_from_source_deprecated(#[case] mut solver: SolverImpl repo.force_publish_recipe(&spec).await.unwrap(); solver.add_repository(Arc::new(repo)); + solver.set_binary_only(false); solver.add_request(request!({"var": "debug/on"})); solver.add_request(request!("my-tool")); @@ -1254,6 +1256,7 @@ async fn test_solver_build_from_source_deprecated_and_impossible_initial_checks( repo.force_publish_recipe(&spec).await.unwrap(); solver.add_repository(Arc::new(repo)); + solver.set_binary_only(false); solver.add_request(request!({"var": "debug/on"})); solver.add_request(request!("my-tool")); if let SolverImpl::Step(ref mut solver) = solver { From ef5f867c2a83ff6983d3ee8cbaa930eb21924121 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Sat, 8 Feb 2025 19:09:17 -0800 Subject: [PATCH 32/64] Don't build from source from deprecated recipes All the solver tests now pass! Signed-off-by: J Robert Ray --- crates/spk-solve/src/solvers/resolvo/spk_provider.rs | 8 +++++++- crates/spk-solve/src/solvers/solver_test.rs | 5 +++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index 949d563fce..6e982bb415 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -39,7 +39,7 @@ use spk_schema::name::{OptNameBuf, PkgNameBuf}; use spk_schema::prelude::{HasVersion, Named}; use spk_schema::version::Version; use spk_schema::version_range::{DoubleEqualsVersion, Ranged, VersionFilter, parse_version_range}; -use spk_schema::{BuildIdent, Opt, Package, Recipe, Request, VersionIdent}; +use spk_schema::{BuildIdent, Deprecate, Opt, Package, Recipe, Request, VersionIdent}; use spk_storage::RepositoryHandle; use tracing::{Instrument, debug_span}; @@ -124,6 +124,12 @@ impl SpkProvider { .find(|repo| repo.name() == ident.repository_name()) { Some(repo) => match repo.read_recipe(&ident.clone().to_version_ident()).await { + Ok(recipe) if recipe.is_deprecated() => { + return CanBuildFromSource::No( + self.pool + .intern_string(format!("recipe for {ident} is deprecated")), + ); + } Ok(recipe) => recipe, Err(err) => { return CanBuildFromSource::No( diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index 3d8a91307f..8c4d9fa255 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -1179,8 +1179,6 @@ async fn test_solver_deprecated_version(#[case] mut solver: SolverImpl) { #[rstest] #[case::step(step_solver())] -// Remove #[should_panic] once resolvo handles this case -#[should_panic] #[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_build_from_source_deprecated(#[case] mut solver: SolverImpl) { @@ -1215,8 +1213,11 @@ async fn test_solver_build_from_source_deprecated(#[case] mut solver: SolverImpl let res = run_and_print_resolve_for_tests(&mut solver).await; match res { + // step solver's error Err(Error::GraphError(ref graph_err)) if matches!(&**graph_err, spk_solve_graph::Error::FailedToResolve(_)) => {} + // resolvo solver's error + Err(Error::FailedToResolve(_)) => {} Err(err) => { panic!("expected solver spk_solver_graph::Error::FailedToResolve, got: '{err:?}'") } From e3f56ceaa0999c560b7fcb305aa5abf8a2aad769 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Sat, 8 Feb 2025 19:24:35 -0800 Subject: [PATCH 33/64] Enable "impossible checks" tests for new solver Although these tests are for exercising the "impossible checks" behavior in the og solver, it is still good to prove the new solver results in the same outcome. Signed-off-by: J Robert Ray --- crates/spk-solve/src/solvers/solver_test.rs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index 8c4d9fa255..26f6edc7f9 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -19,7 +19,7 @@ use spk_schema::ident::{ parse_ident_range, version_ident, }; -use spk_schema::ident_build::{Build, BuildId, EmbeddedSource}; +use spk_schema::ident_build::{Build, BuildId}; use spk_schema::prelude::*; use spk_schema::{recipe, v0}; use spk_solve_macros::{make_build, make_build_and_components, make_package, make_repo, request}; @@ -202,8 +202,8 @@ async fn test_solver_package_with_no_recipe( } #[rstest] -// This test is only applicable to the step solver #[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_package_with_no_recipe_and_impossible_initial_checks( #[case] mut solver: SolverImpl, @@ -292,8 +292,8 @@ async fn test_solver_package_with_no_recipe_from_cmd_line(#[case] mut solver: So } #[rstest] -// This test is only applicable to the step solver #[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_package_with_no_recipe_from_cmd_line_and_impossible_initial_checks( #[case] mut solver: SolverImpl, @@ -1226,8 +1226,8 @@ async fn test_solver_build_from_source_deprecated(#[case] mut solver: SolverImpl } #[rstest] -// This test is only applicable to the step solver #[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_build_from_source_deprecated_and_impossible_initial_checks( #[case] mut solver: SolverImpl, @@ -1277,6 +1277,9 @@ async fn test_solver_build_from_source_deprecated_and_impossible_initial_checks( // recipe is deprecated and refuses to build a binary from // the source package. } + Err(Error::FailedToResolve(_)) => { + // Success; same as above, but for the resolvo solver. + } Err(Error::InitialRequestsContainImpossibleError(_)) => { // Success, when the 'migration-to-components' feature is // disabled because: the initial checks for impossible @@ -1467,8 +1470,8 @@ async fn test_solver_embedded_package_replaces_real_package(#[case] mut solver: } #[rstest] -// This test is only applicable to the step solver #[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_initial_request_impossible_masks_embedded_package_solution( #[case] mut solver: SolverImpl, @@ -1511,7 +1514,7 @@ async fn test_solver_initial_request_impossible_masks_embedded_package_solution( assert_resolved!( solution, "qt", - build = Build::Embedded(EmbeddedSource::Unknown) + build =~ Build::Embedded(_) ); } Err(err) => { @@ -1521,8 +1524,8 @@ async fn test_solver_initial_request_impossible_masks_embedded_package_solution( } #[rstest] -// This test is only applicable to the step solver #[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_impossible_request_but_embedded_package_makes_solvable( #[case] mut solver: SolverImpl, @@ -1593,7 +1596,7 @@ async fn test_solver_impossible_request_but_embedded_package_makes_solvable( assert_resolved!( solution, "qt", - build = Build::Embedded(EmbeddedSource::Unknown) + build =~ Build::Embedded(_) ); } Err(err) => { @@ -1668,8 +1671,8 @@ async fn test_multiple_packages_embed_same_package( } #[rstest] -// This test is only applicable to the step solver #[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] async fn test_solver_with_impossible_checks_in_build_keys(#[case] mut solver: SolverImpl) { let options1 = option_map! {"dep" => "1.0.0"}; From 952b26ad84963ce8d1c81cde8da35f4cb32dd294 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Sat, 8 Feb 2025 19:37:55 -0800 Subject: [PATCH 34/64] Use Variantly to auto-generate these methods Signed-off-by: J Robert Ray --- Cargo.lock | 1 + crates/spk-cli/common/src/flags.rs | 2 +- crates/spk-schema/crates/ident/Cargo.toml | 1 + crates/spk-schema/crates/ident/src/request.rs | 27 ++-------------- .../crates/ident/src/request_test.rs | 32 ++++++++----------- .../crates/validation/src/validation_test.rs | 4 +-- 6 files changed, 20 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6d662a2905..a287dc67f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4486,6 +4486,7 @@ dependencies = [ "spk-schema-foundation", "tap", "thiserror", + "variantly", ] [[package]] diff --git a/crates/spk-cli/common/src/flags.rs b/crates/spk-cli/common/src/flags.rs index ab06c67af5..09dfe55f2c 100644 --- a/crates/spk-cli/common/src/flags.rs +++ b/crates/spk-cli/common/src/flags.rs @@ -630,7 +630,7 @@ impl Requests { let mut req = serde_yaml::from_value::(request_data.into()) .into_diagnostic() .wrap_err_with(|| format!("Failed to parse request {request}"))? - .into_pkg() + .pkg() .ok_or_else(|| miette!("Expected a package request, got None"))?; req.add_requester(RequestedBy::CommandLine); diff --git a/crates/spk-schema/crates/ident/Cargo.toml b/crates/spk-schema/crates/ident/Cargo.toml index 44e72b5e4e..c8e28fb572 100644 --- a/crates/spk-schema/crates/ident/Cargo.toml +++ b/crates/spk-schema/crates/ident/Cargo.toml @@ -31,6 +31,7 @@ spk-schema-foundation = { workspace = true } tap = { workspace = true } thiserror = { workspace = true } miette = { workspace = true } +variantly = { workspace = true } [dev-dependencies] data-encoding = "2.3" diff --git a/crates/spk-schema/crates/ident/src/request.rs b/crates/spk-schema/crates/ident/src/request.rs index 2a25da91e7..96f7532fb3 100644 --- a/crates/spk-schema/crates/ident/src/request.rs +++ b/crates/spk-schema/crates/ident/src/request.rs @@ -39,6 +39,7 @@ use spk_schema_foundation::version_range::{ VersionFilter, }; use tap::Tap; +use variantly::Variantly; use super::AnyIdent; use crate::{BuildIdent, Error, RangeIdent, Result, Satisfy, VersionIdent}; @@ -176,7 +177,7 @@ impl<'de> Deserialize<'de> for PinPolicy { } /// Represents a constraint added to a resolved environment. -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Variantly)] #[serde(untagged)] pub enum Request { Pkg(PkgRequest), @@ -192,30 +193,6 @@ impl spk_schema_foundation::spec_ops::Named for Request { } } -impl Request { - pub fn is_pkg(&self) -> bool { - matches!(self, Self::Pkg(_)) - } - - pub fn into_pkg(self) -> Option { - match self { - Self::Pkg(p) => Some(p), - _ => None, - } - } - - pub fn is_var(&self) -> bool { - matches!(self, Self::Var(_)) - } - - pub fn into_var(self) -> Option { - match self { - Self::Var(v) => Some(v), - _ => None, - } - } -} - impl std::fmt::Display for Request { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/crates/spk-schema/crates/ident/src/request_test.rs b/crates/spk-schema/crates/ident/src/request_test.rs index 4433f9061f..240d6a7bb9 100644 --- a/crates/spk-schema/crates/ident/src/request_test.rs +++ b/crates/spk-schema/crates/ident/src/request_test.rs @@ -73,11 +73,11 @@ fn test_prerelease_policy_restricts( ) { let mut a = serde_yaml::from_str::(request_a) .unwrap() - .into_pkg() + .pkg() .expect("expected pkg request"); let b = serde_yaml::from_str::(request_b) .unwrap() - .into_pkg() + .pkg() .expect("expected pkg request"); a.restrict(&b).unwrap(); @@ -149,11 +149,11 @@ fn test_prerelease_policy_contains( ) { let a = serde_yaml::from_str::(request_a) .unwrap() - .into_pkg() + .pkg() .expect("expected pkg request"); let b = serde_yaml::from_str::(request_b) .unwrap() - .into_pkg() + .pkg() .expect("expected pkg request"); let compat = a.contains(&b); @@ -164,11 +164,11 @@ fn test_prerelease_policy_contains( fn test_inclusion_policy() { let mut a = serde_yaml::from_str::("{pkg: something, include: IfAlreadyPresent}") .unwrap() - .into_pkg() + .pkg() .expect("expected pkg request"); let b = serde_yaml::from_str::("{pkg: something, include: Always}") .unwrap() - .into_pkg() + .pkg() .expect("expected pkg request"); a.restrict(&b).unwrap(); @@ -182,11 +182,11 @@ fn test_inclusion_policy() { fn test_compat_and_equals_restrict() { let mut a = serde_yaml::from_str::("{pkg: something/Binary:1.2.3}") .unwrap() - .into_pkg() + .pkg() .expect("expected pkg request"); let b = serde_yaml::from_str::("{pkg: something/=1.2.3}") .unwrap() - .into_pkg() + .pkg() .expect("expected pkg request"); a.restrict(&b).unwrap(); @@ -247,14 +247,8 @@ fn test_inclusion_policy_and_merge( #[case] expected_policy: InclusionPolicy, #[case] expected_merged_range: Option<&str>, ) { - let mut a = serde_yaml::from_str::(a) - .unwrap() - .into_pkg() - .unwrap(); - let b = serde_yaml::from_str::(b) - .unwrap() - .into_pkg() - .unwrap(); + let mut a = serde_yaml::from_str::(a).unwrap().pkg().unwrap(); + let b = serde_yaml::from_str::(b).unwrap().pkg().unwrap(); let r = a.restrict(&b); match expected_merged_range { @@ -306,7 +300,7 @@ fn test_var_request_pinned_roundtrip() { "should be able to round-trip serialize a var request with pin" ); assert!( - res.unwrap().into_var().unwrap().value.is_from_build_env(), + res.unwrap().var().unwrap().value.is_from_build_env(), "should preserve pin value" ); } @@ -347,7 +341,7 @@ fn test_pkg_request_pin_rendering( ) { let req = serde_yaml::from_str::(&format!("{{pkg: test, fromBuildEnv: {pin}}}")) .unwrap() - .into_pkg() + .pkg() .expect("expected package request"); let version = parse_build_ident(format!("test/{version}/src")).unwrap(); let res = req @@ -473,7 +467,7 @@ fn test_deserialize_pkg_pin_string_or_bool() { let reqs = Vec::::from_yaml(YAML).expect("expected yaml parsing to succeed"); let pins: Vec<_> = reqs .into_iter() - .map(|r| r.into_pkg().expect("expected a pkg request").pin) + .map(|r| r.pkg().expect("expected a pkg request").pin) .collect(); assert_eq!( pins, diff --git a/crates/spk-solve/crates/validation/src/validation_test.rs b/crates/spk-solve/crates/validation/src/validation_test.rs index 25aec0322a..73c27e15f7 100644 --- a/crates/spk-solve/crates/validation/src/validation_test.rs +++ b/crates/spk-solve/crates/validation/src/validation_test.rs @@ -103,11 +103,11 @@ fn test_qualified_var_supersedes_unqualified() { vec![ Request::from_yaml("{var: debug/off}") .unwrap() - .into_var() + .var() .unwrap(), Request::from_yaml("{var: my-package.debug/on}") .unwrap() - .into_var() + .var() .unwrap(), ], vec![], From 62cb045e8ac5db813b6509076395a8a2d5fdf16e Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Sat, 22 Feb 2025 14:43:21 -0800 Subject: [PATCH 35/64] Rework flags test for the Solver trait (pt. 1) Don't require the solver has a [initial] `State`. Signed-off-by: J Robert Ray Signed-off-by: J Robert Ray --- crates/spk-cli/common/src/flags_test.rs | 16 ++++++++++------ crates/spk-solve/src/solver.rs | 4 ++++ crates/spk-solve/src/solvers/resolvo/mod.rs | 17 ++++++++++++++++- crates/spk-solve/src/solvers/step/solver.rs | 8 ++++++++ 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/crates/spk-cli/common/src/flags_test.rs b/crates/spk-cli/common/src/flags_test.rs index 3069bf9aff..ec308803d4 100644 --- a/crates/spk-cli/common/src/flags_test.rs +++ b/crates/spk-cli/common/src/flags_test.rs @@ -7,6 +7,7 @@ use spk_schema::foundation::name::OptName; use spk_schema::foundation::option_map::OptionMap; use spk_schema::ident::VarRequest; use spk_schema::option_map::HOST_OPTIONS; +use spk_solve::Solver; #[rstest] #[case(&["hello:world"], &[("hello", "world")])] @@ -37,13 +38,13 @@ fn test_option_flags_parsing(#[case] args: &[&str], #[case] expected: &[(&str, & } #[rstest] -#[case::no_host_true(true)] -#[case::no_host_false(false)] #[tokio::test] -async fn test_get_solver_with_host_options(#[case] no_host: bool) { +async fn test_get_solver_with_host_options(#[values(true, false)] no_host: bool) { // Test the get_solver() method adds the host options to the solver // correctly. + use std::collections::HashSet; + let options_flags = crate::flags::Options { options: Vec::new(), no_host, @@ -66,7 +67,10 @@ async fn test_get_solver_with_host_options(#[case] no_host: bool) { }; let solver = solver_flags.get_solver(&options_flags).await.unwrap(); - let initial_state = solver.get_initial_state(); + let var_requests = solver + .get_var_requests() + .into_iter() + .collect::>(); assert!( !HOST_OPTIONS.get().unwrap().is_empty(), @@ -76,9 +80,9 @@ async fn test_get_solver_with_host_options(#[case] no_host: bool) { for (name, value) in HOST_OPTIONS.get().unwrap() { let var_request = VarRequest::new_with_value(name, value); if no_host { - assert!(!initial_state.contains_var_request(&var_request)); + assert!(!var_requests.contains(&var_request)); } else { - assert!(initial_state.contains_var_request(&var_request)); + assert!(var_requests.contains(&var_request)); } } } diff --git a/crates/spk-solve/src/solver.rs b/crates/spk-solve/src/solver.rs index 1e7745fe62..e143593674 100644 --- a/crates/spk-solve/src/solver.rs +++ b/crates/spk-solve/src/solver.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use enum_dispatch::enum_dispatch; +use spk_schema::ident::VarRequest; use spk_schema::{OptionMap, Request}; use spk_solve_solution::Solution; use spk_storage::RepositoryHandle; @@ -30,6 +31,9 @@ pub trait Solver { /// Add a request to this solver. fn add_request(&mut self, request: Request); + /// Return the VarRequests added to the solver. + fn get_var_requests(&self) -> Vec; + /// Put this solver back into its default state fn reset(&mut self); diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index 36e0f8f0c1..988d504293 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -22,7 +22,14 @@ use std::sync::Arc; use pkg_request_version_set::{SpkSolvable, SyntheticComponent}; use spk_provider::SpkProvider; -use spk_schema::ident::{InclusionPolicy, LocatedBuildIdent, PinPolicy, PkgRequest, RangeIdent}; +use spk_schema::ident::{ + InclusionPolicy, + LocatedBuildIdent, + PinPolicy, + PkgRequest, + RangeIdent, + VarRequest, +}; use spk_schema::ident_component::Component; use spk_schema::prelude::{HasVersion, Named, Versioned}; use spk_schema::version_range::VersionFilter; @@ -76,6 +83,14 @@ impl SolverTrait for Solver { self.requests.push(request); } + fn get_var_requests(&self) -> Vec { + self.requests + .iter() + .filter_map(|r| r.var_ref()) + .cloned() + .collect() + } + fn reset(&mut self) { self.repos.truncate(0); self.requests.truncate(0); diff --git a/crates/spk-solve/src/solvers/step/solver.rs b/crates/spk-solve/src/solvers/step/solver.rs index bd51fff43b..22e2aad4e1 100644 --- a/crates/spk-solve/src/solvers/step/solver.rs +++ b/crates/spk-solve/src/solvers/step/solver.rs @@ -1134,6 +1134,14 @@ impl SolverTrait for Solver { self.initial_state_builders.push(request); } + fn get_var_requests(&self) -> Vec { + self.get_initial_state() + .get_var_requests() + .iter() + .cloned() + .collect() + } + fn reset(&mut self) { self.repos.truncate(0); self.initial_state_builders.truncate(0); From 3363227c31a2335d606614dcdbf1f95d3df98260 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Sat, 8 Feb 2025 19:37:55 -0800 Subject: [PATCH 36/64] Work towards using resolvo for spk commands Change the solver factory to return an `impl Solver` instead of a concrete `spk_solve::StepSolver`. This leads to trouble with the different way to run the solver, so `downcast_ref` is used to get back the original type. Signed-off-by: J Robert Ray --- crates/spk-cli/cmd-env/src/cmd_env.rs | 11 +++- crates/spk-cli/cmd-explain/src/cmd_explain.rs | 19 ++++-- crates/spk-cli/cmd-install/src/cmd_install.rs | 12 +++- crates/spk-cli/cmd-render/src/cmd_render.rs | 11 +++- crates/spk-cli/common/src/flags.rs | 23 +++---- crates/spk-cli/group1/src/cmd_bake.rs | 12 +++- crates/spk-cli/group4/src/cmd_view.rs | 61 ++++++++++--------- crates/spk-solve/src/io.rs | 13 +++- crates/spk-solve/src/solver.rs | 11 +++- crates/spk-solve/src/solvers/mod.rs | 2 +- crates/spk-solve/src/solvers/resolvo/mod.rs | 12 ++++ crates/spk-solve/src/solvers/step/solver.rs | 17 ++++-- 12 files changed, 140 insertions(+), 64 deletions(-) diff --git a/crates/spk-cli/cmd-env/src/cmd_env.rs b/crates/spk-cli/cmd-env/src/cmd_env.rs index 25d815878a..f2b43997f7 100644 --- a/crates/spk-cli/cmd-env/src/cmd_env.rs +++ b/crates/spk-cli/cmd-env/src/cmd_env.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/spkenv/spk +use std::any::Any; use std::collections::HashSet; use std::ffi::OsString; @@ -87,8 +88,14 @@ impl Run for Env { solver.add_request(request) } - let formatter = self.formatter_settings.get_formatter(self.verbose)?; - let (solution, _) = formatter.run_and_print_resolve(&solver).await?; + let solution = + if let Some(solver) = (&solver as &dyn Any).downcast_ref::() { + let formatter = self.formatter_settings.get_formatter(self.verbose)?; + let (solution, _) = formatter.run_and_print_resolve(solver).await?; + solution + } else { + solver.solve().await? + }; let solution = build_required_packages(&solution).await?; diff --git a/crates/spk-cli/cmd-explain/src/cmd_explain.rs b/crates/spk-cli/cmd-explain/src/cmd_explain.rs index cc943762bf..7fab1c1721 100644 --- a/crates/spk-cli/cmd-explain/src/cmd_explain.rs +++ b/crates/spk-cli/cmd-explain/src/cmd_explain.rs @@ -2,6 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/spkenv/spk +use std::any::Any; + use clap::Args; use miette::Result; use spk_cli_common::{CommandArgs, Run, flags}; @@ -81,12 +83,17 @@ impl Run for Explain { } // Always show the solution packages for the solve - let formatter = self - .formatter_settings - .get_formatter_builder(self.verbose + 1)? - .with_solution(true) - .build(); - formatter.run_and_print_resolve(&solver).await?; + if let Some(solver) = (&solver as &dyn Any).downcast_ref::() { + let formatter = self + .formatter_settings + .get_formatter_builder(self.verbose + 1)? + .with_solution(true) + .build(); + formatter.run_and_print_resolve(solver).await?; + } else { + // TODO: print the solve + solver.solve().await?; + } Ok(0) } diff --git a/crates/spk-cli/cmd-install/src/cmd_install.rs b/crates/spk-cli/cmd-install/src/cmd_install.rs index 4ae4dcd6a2..f1c7170f16 100644 --- a/crates/spk-cli/cmd-install/src/cmd_install.rs +++ b/crates/spk-cli/cmd-install/src/cmd_install.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/spkenv/spk +use std::any::Any; use std::collections::HashSet; use std::io::Write; @@ -63,12 +64,17 @@ impl Run for Install { solver.add_request(request); } - let formatter = self.formatter_settings.get_formatter(self.verbose)?; - let (solution, _) = formatter.run_and_print_resolve(&solver).await?; + let solution = + if let Some(solver) = (&solver as &dyn Any).downcast_ref::() { + let formatter = self.formatter_settings.get_formatter(self.verbose)?; + let (solution, _) = formatter.run_and_print_resolve(solver).await?; + solution + } else { + solver.solve().await? + }; println!("The following packages will be installed:\n"); let requested: HashSet<_> = solver - .get_initial_state() .get_pkg_requests() .iter() .map(|r| r.pkg.name.clone()) diff --git a/crates/spk-cli/cmd-render/src/cmd_render.rs b/crates/spk-cli/cmd-render/src/cmd_render.rs index b64670b695..7bbb29494f 100644 --- a/crates/spk-cli/cmd-render/src/cmd_render.rs +++ b/crates/spk-cli/cmd-render/src/cmd_render.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/spkenv/spk +use std::any::Any; use std::path::PathBuf; use clap::Args; @@ -52,8 +53,14 @@ impl Run for Render { solver.add_request(name); } - let formatter = self.formatter_settings.get_formatter(self.verbose)?; - let (solution, _) = formatter.run_and_print_resolve(&solver).await?; + let solution = + if let Some(solver) = (&solver as &dyn Any).downcast_ref::() { + let formatter = self.formatter_settings.get_formatter(self.verbose)?; + let (solution, _) = formatter.run_and_print_resolve(solver).await?; + solution + } else { + solver.solve().await? + }; let solution = build_required_packages(&solution).await?; let stack = resolve_runtime_layers(true, &solution).await?; diff --git a/crates/spk-cli/common/src/flags.rs b/crates/spk-cli/common/src/flags.rs index 09dfe55f2c..fda4ec253b 100644 --- a/crates/spk-cli/common/src/flags.rs +++ b/crates/spk-cli/common/src/flags.rs @@ -258,10 +258,11 @@ pub struct Solver { } impl Solver { - pub async fn get_solver(&self, options: &Options) -> Result { + pub async fn get_solver(&self, options: &Options) -> Result { let option_map = options.get_options()?; - let mut solver = solve::StepSolver::default(); + //let mut solver = solve::StepSolver::default(); + let mut solver = solve::ResolvoSolver::default(); solver.update_options(option_map); for (name, repo) in self.repos.get_repos_for_non_destructive_operation().await? { @@ -269,15 +270,15 @@ impl Solver { solver.add_repository(repo); } solver.set_binary_only(!self.allow_builds); - solver.set_initial_request_impossible_checks( - self.check_impossible_initial || self.check_impossible_all, - ); - solver.set_resolve_validation_impossible_checks( - self.check_impossible_validation || self.check_impossible_all, - ); - solver.set_build_key_impossible_checks( - self.check_impossible_builds || self.check_impossible_all, - ); + //solver.set_initial_request_impossible_checks( + // self.check_impossible_initial || self.check_impossible_all, + //); + //solver.set_resolve_validation_impossible_checks( + // self.check_impossible_validation || self.check_impossible_all, + //); + //solver.set_build_key_impossible_checks( + // self.check_impossible_builds || self.check_impossible_all, + //); for r in options.get_var_requests()? { solver.add_request(r.into()); diff --git a/crates/spk-cli/group1/src/cmd_bake.rs b/crates/spk-cli/group1/src/cmd_bake.rs index a26a0396c5..300cb37937 100644 --- a/crates/spk-cli/group1/src/cmd_bake.rs +++ b/crates/spk-cli/group1/src/cmd_bake.rs @@ -2,6 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/spkenv/spk +use std::any::Any; + use clap::Args; use futures::TryFutureExt; use miette::IntoDiagnostic; @@ -238,8 +240,14 @@ impl Bake { solver.add_request(request) } - let formatter = self.formatter_settings.get_formatter(self.verbose)?; - let (solution, _) = formatter.run_and_print_resolve(&solver).await?; + let solution = + if let Some(solver) = (&solver as &dyn Any).downcast_ref::() { + let formatter = self.formatter_settings.get_formatter(self.verbose)?; + let (solution, _) = formatter.run_and_print_resolve(solver).await?; + solution + } else { + solver.solve().await? + }; // The solution order is the order things were found during // the solve. Need to reverse it to match up with the spfs diff --git a/crates/spk-cli/group4/src/cmd_view.rs b/crates/spk-cli/group4/src/cmd_view.rs index b97ee88726..76ddd40854 100644 --- a/crates/spk-cli/group4/src/cmd_view.rs +++ b/crates/spk-cli/group4/src/cmd_view.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/spkenv/spk +use std::any::Any; use std::borrow::Cow; use std::collections::BTreeMap; use std::sync::Arc; @@ -606,7 +607,7 @@ impl View { async fn get_package_versions( &self, name: &PkgNameBuf, - repos: &Vec>, + repos: &[std::sync::Arc], ) -> Result> { let mut versions = Vec::new(); for repo in repos { @@ -648,37 +649,41 @@ impl View { _ => bail!("Not a package request: {request:?}"), }; - let mut runtime = solver.run(); - - let formatter = self.formatter_settings.get_formatter(self.verbose)?; - - let result = formatter.run_and_print_decisions(&mut runtime).await; - let solution = match result { - Ok((s, _)) => s, - Err(err) => { - println!("{}", err.to_string().red()); - match self.verbose { - 0 => eprintln!("{}", "try '--verbose' for more info".yellow().dimmed(),), - v if v < 2 => { - eprintln!("{}", "try '-vv' for even more info".yellow().dimmed(),) - } - _v => { - let graph = runtime.graph(); - let graph = graph.read().await; - // Iter much? - let mut graph_walk = graph.walk(); - let walk_iter = graph_walk.iter().map(Ok); - let mut decision_iter = formatter.formatted_decisions_iter(walk_iter); - let iter = decision_iter.iter(); - tokio::pin!(iter); - while let Some(line) = iter.try_next().await? { - println!("{line}"); + let solution = if let Some(solver) = + (&solver as &dyn Any).downcast_ref::() + { + let mut runtime = solver.run(); + let formatter = self.formatter_settings.get_formatter(self.verbose)?; + let result = formatter.run_and_print_decisions(&mut runtime).await; + match result { + Ok((s, _)) => s, + Err(err) => { + println!("{}", err.to_string().red()); + match self.verbose { + 0 => eprintln!("{}", "try '--verbose' for more info".yellow().dimmed(),), + v if v < 2 => { + eprintln!("{}", "try '-vv' for even more info".yellow().dimmed(),) + } + _v => { + let graph = runtime.graph(); + let graph = graph.read().await; + // Iter much? + let mut graph_walk = graph.walk(); + let walk_iter = graph_walk.iter().map(Ok); + let mut decision_iter = formatter.formatted_decisions_iter(walk_iter); + let iter = decision_iter.iter(); + tokio::pin!(iter); + while let Some(line) = iter.try_next().await? { + println!("{line}"); + } } } - } - return Ok(1); + return Ok(1); + } } + } else { + solver.solve().await? }; for item in solution.items() { diff --git a/crates/spk-solve/src/io.rs b/crates/spk-solve/src/io.rs index e13620e628..3337a7c2c3 100644 --- a/crates/spk-solve/src/io.rs +++ b/crates/spk-solve/src/io.rs @@ -40,8 +40,17 @@ use spk_solve_graph::{ State, }; -use crate::solvers::{ErrorFreq, StepSolver, StepSolverRuntime}; -use crate::{Error, ResolverCallback, Result, Solution, StatusLine, show_search_space_stats}; +use crate::solvers::step::ErrorFreq; +use crate::solvers::{StepSolver, StepSolverRuntime}; +use crate::{ + Error, + ResolverCallback, + Result, + Solution, + Solver, + StatusLine, + show_search_space_stats, +}; #[cfg(feature = "statsd")] use crate::{ SPK_SOLUTION_PACKAGE_COUNT_METRIC, diff --git a/crates/spk-solve/src/solver.rs b/crates/spk-solve/src/solver.rs index e143593674..33af9913e7 100644 --- a/crates/spk-solve/src/solver.rs +++ b/crates/spk-solve/src/solver.rs @@ -2,10 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/spkenv/spk +use std::any::Any; use std::sync::Arc; use enum_dispatch::enum_dispatch; -use spk_schema::ident::VarRequest; +use spk_schema::ident::{PkgRequest, VarRequest}; use spk_schema::{OptionMap, Request}; use spk_solve_solution::Solution; use spk_storage::RepositoryHandle; @@ -22,7 +23,7 @@ pub(crate) enum SolverImpl { #[async_trait::async_trait] #[enum_dispatch] -pub trait Solver { +pub trait Solver: Any { /// Add a repository where the solver can get packages. fn add_repository(&mut self, repo: R) where @@ -31,9 +32,15 @@ pub trait Solver { /// Add a request to this solver. fn add_request(&mut self, request: Request); + /// Return the PkgRequests added to the solver. + fn get_pkg_requests(&self) -> Vec; + /// Return the VarRequests added to the solver. fn get_var_requests(&self) -> Vec; + /// Return a reference to the solver's list of repositories. + fn repositories(&self) -> &[Arc]; + /// Put this solver back into its default state fn reset(&mut self); diff --git a/crates/spk-solve/src/solvers/mod.rs b/crates/spk-solve/src/solvers/mod.rs index e2de48444c..50d9b83b34 100644 --- a/crates/spk-solve/src/solvers/mod.rs +++ b/crates/spk-solve/src/solvers/mod.rs @@ -8,7 +8,7 @@ pub(crate) mod resolvo; pub(crate) mod step; pub use resolvo::Solver as ResolvoSolver; -pub use step::{ErrorFreq, Solver as StepSolver, SolverRuntime as StepSolverRuntime}; +pub use step::{Solver as StepSolver, SolverRuntime as StepSolverRuntime}; // Public to allow other tests to use its macros #[cfg(test)] diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index 988d504293..ba7cd3d4ce 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -83,6 +83,14 @@ impl SolverTrait for Solver { self.requests.push(request); } + fn get_pkg_requests(&self) -> Vec { + self.requests + .iter() + .filter_map(|r| r.pkg_ref()) + .cloned() + .collect() + } + fn get_var_requests(&self) -> Vec { self.requests .iter() @@ -91,6 +99,10 @@ impl SolverTrait for Solver { .collect() } + fn repositories(&self) -> &[Arc] { + &self.repos + } + fn reset(&mut self) { self.repos.truncate(0); self.requests.truncate(0); diff --git a/crates/spk-solve/src/solvers/step/solver.rs b/crates/spk-solve/src/solvers/step/solver.rs index 22e2aad4e1..a70f2bb1b1 100644 --- a/crates/spk-solve/src/solvers/step/solver.rs +++ b/crates/spk-solve/src/solvers/step/solver.rs @@ -206,11 +206,6 @@ impl ErrorFreq { } impl Solver { - /// Return a reference to the solver's list of repositories. - pub fn repositories(&self) -> &Vec> { - &self.repos - } - pub fn get_initial_state(&self) -> Arc { let mut state = None; let base = State::default_state(); @@ -1134,6 +1129,14 @@ impl SolverTrait for Solver { self.initial_state_builders.push(request); } + fn get_pkg_requests(&self) -> Vec { + self.get_initial_state() + .get_pkg_requests() + .iter() + .map(|pkg_request| (***pkg_request).clone()) + .collect() + } + fn get_var_requests(&self) -> Vec { self.get_initial_state() .get_var_requests() @@ -1142,6 +1145,10 @@ impl SolverTrait for Solver { .collect() } + fn repositories(&self) -> &[Arc] { + &self.repos + } + fn reset(&mut self) { self.repos.truncate(0); self.initial_state_builders.truncate(0); From 1413825441532e643a3d7305f200a918eef6d90a Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Sun, 9 Feb 2025 02:13:29 -0800 Subject: [PATCH 37/64] Use new solver in more places Signed-off-by: J Robert Ray --- Cargo.lock | 1 + crates/spk-build/src/build/binary.rs | 36 ++++--- crates/spk-cli/cmd-make-binary/Cargo.toml | 9 +- .../cmd-make-binary/src/cmd_make_binary.rs | 9 +- crates/spk-cli/cmd-test/src/cmd_test.rs | 17 ++-- crates/spk-cli/cmd-test/src/test/build.rs | 33 ++++--- crates/spk-cli/cmd-test/src/test/install.rs | 20 ++-- crates/spk-cli/cmd-test/src/test/sources.rs | 20 ++-- crates/spk-solve/src/io.rs | 16 ++-- crates/spk-solve/src/lib.rs | 42 ++++++--- crates/spk-solve/src/solvers/resolvo/mod.rs | 94 ++++++++++--------- 11 files changed, 174 insertions(+), 123 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a287dc67f7..64d538e3b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4255,6 +4255,7 @@ dependencies = [ "spk-build", "spk-cli-common", "spk-schema", + "spk-solve", "spk-storage", "tempfile", "tokio", diff --git a/crates/spk-build/src/build/binary.rs b/crates/spk-build/src/build/binary.rs index 31198a8f27..8ed12490dc 100644 --- a/crates/spk-build/src/build/binary.rs +++ b/crates/spk-build/src/build/binary.rs @@ -37,7 +37,7 @@ use spk_schema::{ }; use spk_solve::graph::Graph; use spk_solve::solution::Solution; -use spk_solve::{BoxedResolverCallback, Named, ResolverCallback, Solver, StepSolver}; +use spk_solve::{BoxedCdclResolverCallback, Named, ResolverCallback, ResolvoSolver, Solver}; use spk_storage as storage; use crate::report::{BuildOutputReport, BuildReport, BuildSetupReport}; @@ -131,10 +131,10 @@ pub struct BinaryPackageBuilder<'a, Recipe> { prefix: PathBuf, recipe: Recipe, source: BuildSource, - solver: StepSolver, + solver: ResolvoSolver, environment: HashMap, - source_resolver: BoxedResolverCallback<'a>, - build_resolver: BoxedResolverCallback<'a>, + source_resolver: BoxedCdclResolverCallback<'a>, + build_resolver: BoxedCdclResolverCallback<'a>, last_solve_graph: Arc>, repos: Vec>, interactive: bool, @@ -155,16 +155,16 @@ where recipe, source, prefix: PathBuf::from("/spfs"), - solver: StepSolver::default(), + solver: ResolvoSolver::default(), environment: Default::default(), - #[cfg(test)] - source_resolver: Box::new(spk_solve::DecisionFormatter::new_testing()), - #[cfg(not(test))] - source_resolver: Box::new(spk_solve::DefaultResolver {}), - #[cfg(test)] - build_resolver: Box::new(spk_solve::DecisionFormatter::new_testing()), - #[cfg(not(test))] - build_resolver: Box::new(spk_solve::DefaultResolver {}), + //#[cfg(test)] + //source_resolver: Box::new(spk_solve::DecisionFormatter::new_testing()), + //#[cfg(not(test))] + source_resolver: Box::new(spk_solve::DefaultCdclResolver {}), + //#[cfg(test)] + //build_resolver: Box::new(spk_solve::DecisionFormatter::new_testing()), + //#[cfg(not(test))] + build_resolver: Box::new(spk_solve::DefaultCdclResolver {}), last_solve_graph: Arc::new(tokio::sync::RwLock::new(Graph::new())), repos: Default::default(), interactive: false, @@ -222,7 +222,7 @@ where /// process as needed. pub fn with_source_resolver(&mut self, resolver: F) -> &mut Self where - F: ResolverCallback + 'a, + F: ResolverCallback + 'a, { self.source_resolver = Box::new(resolver); self @@ -236,7 +236,7 @@ where /// process as needed. pub fn with_build_resolver(&mut self, resolver: F) -> &mut Self where - F: ResolverCallback + 'a, + F: ResolverCallback + 'a, { self.build_resolver = Box::new(resolver); self @@ -441,8 +441,7 @@ where self.solver.add_request(request.into()); - let (solution, graph) = self.source_resolver.solve(&self.solver).await?; - self.last_solve_graph = graph; + let solution = self.source_resolver.solve(&self.solver).await?; Ok(solution) } @@ -466,8 +465,7 @@ where self.solver.add_request(request.clone()); } - let (solution, graph) = self.build_resolver.solve(&self.solver).await?; - self.last_solve_graph = graph; + let solution = self.build_resolver.solve(&self.solver).await?; Ok(solution) } diff --git a/crates/spk-cli/cmd-make-binary/Cargo.toml b/crates/spk-cli/cmd-make-binary/Cargo.toml index 5e823eadeb..0fbf8dc8c0 100644 --- a/crates/spk-cli/cmd-make-binary/Cargo.toml +++ b/crates/spk-cli/cmd-make-binary/Cargo.toml @@ -14,10 +14,10 @@ workspace = true [features] migration-to-components = [ - "spk-build/migration-to-components", - "spk-cli-common/migration-to-components", - "spk-schema/migration-to-components", - "spk-storage/migration-to-components", + "spk-build/migration-to-components", + "spk-cli-common/migration-to-components", + "spk-schema/migration-to-components", + "spk-storage/migration-to-components", ] [dependencies] @@ -30,6 +30,7 @@ spfs = { workspace = true } spk-build = { workspace = true } spk-cli-common = { workspace = true } spk-schema = { workspace = true } +spk-solve = { workspace = true } spk-storage = { workspace = true } tokio = { workspace = true, features = ["rt"] } tracing = { workspace = true } diff --git a/crates/spk-cli/cmd-make-binary/src/cmd_make_binary.rs b/crates/spk-cli/cmd-make-binary/src/cmd_make_binary.rs index 4cf6104f0b..835caa091d 100644 --- a/crates/spk-cli/cmd-make-binary/src/cmd_make_binary.rs +++ b/crates/spk-cli/cmd-make-binary/src/cmd_make_binary.rs @@ -15,6 +15,7 @@ use spk_schema::foundation::format::FormatIdent; use spk_schema::ident::{PkgRequest, RequestedBy}; use spk_schema::option_map::HOST_OPTIONS; use spk_schema::prelude::*; +use spk_solve::DefaultCdclResolver; use spk_storage as storage; #[cfg(test)] @@ -166,11 +167,11 @@ impl Run for MakeBinary { let mut fmt_builder = self .formatter_settings .get_formatter_builder(self.verbose)?; - let src_formatter = fmt_builder + let _src_formatter = fmt_builder .with_solution(true) .with_header("Src Resolver ") .build(); - let build_formatter = fmt_builder + let _build_formatter = fmt_builder .with_solution(true) .with_header("Build Resolver ") .build(); @@ -179,8 +180,8 @@ impl Run for MakeBinary { builder .with_repositories(repos.iter().cloned()) .set_interactive(self.interactive) - .with_source_resolver(&src_formatter) - .with_build_resolver(&build_formatter) + .with_source_resolver(DefaultCdclResolver {}) + .with_build_resolver(DefaultCdclResolver {}) .with_allow_circular_dependencies(self.allow_circular_dependencies); if self.here { diff --git a/crates/spk-cli/cmd-test/src/cmd_test.rs b/crates/spk-cli/cmd-test/src/cmd_test.rs index fc402b693d..ce9bf3d246 100644 --- a/crates/spk-cli/cmd-test/src/cmd_test.rs +++ b/crates/spk-cli/cmd-test/src/cmd_test.rs @@ -15,6 +15,7 @@ use spk_schema::foundation::ident_build::Build; use spk_schema::foundation::option_map::{HOST_OPTIONS, OptionMap}; use spk_schema::prelude::*; use spk_schema::{Recipe, Request, TestStage}; +use spk_solve::DefaultCdclResolver; use crate::test::{PackageBuildTester, PackageInstallTester, PackageSourceTester, Tester}; @@ -160,11 +161,11 @@ impl Run for CmdTest { let mut builder = self .formatter_settings .get_formatter_builder(self.verbose)?; - let src_formatter = builder.with_header("Source Resolver ").build(); - let build_src_formatter = + let _src_formatter = builder.with_header("Source Resolver ").build(); + let _build_src_formatter = builder.with_header("Build Source Resolver ").build(); - let build_formatter = builder.with_header("Build Resolver ").build(); - let install_formatter = + let _build_formatter = builder.with_header("Build Resolver ").build(); + let _install_formatter = builder.with_header("Install Env Resolver ").build(); let mut tester: Box = match stage { @@ -177,7 +178,7 @@ impl Run for CmdTest { .with_repositories(repos.iter().cloned()) .with_requirements(test.additional_requirements()) .with_source(source.clone()) - .watch_environment_resolve(&src_formatter); + .watch_environment_resolve(DefaultCdclResolver {}); Box::new(tester) } @@ -208,8 +209,8 @@ impl Run for CmdTest { }, ), ) - .with_source_resolver(&build_src_formatter) - .with_build_resolver(&build_formatter); + .with_source_resolver(DefaultCdclResolver {}) + .with_build_resolver(DefaultCdclResolver {}); Box::new(tester) } @@ -227,7 +228,7 @@ impl Run for CmdTest { .with_requirements(test.additional_requirements()) .with_requirements(options_reqs.clone()) .with_source(source.clone()) - .watch_environment_resolve(&install_formatter); + .watch_environment_resolve(DefaultCdclResolver {}); Box::new(tester) } diff --git a/crates/spk-cli/cmd-test/src/test/build.rs b/crates/spk-cli/cmd-test/src/test/build.rs index d26b484ef7..67df2444c3 100644 --- a/crates/spk-cli/cmd-test/src/test/build.rs +++ b/crates/spk-cli/cmd-test/src/test/build.rs @@ -15,7 +15,13 @@ use spk_schema::foundation::option_map::OptionMap; use spk_schema::ident::{PkgRequest, PreReleasePolicy, RangeIdent, Request, RequestedBy}; use spk_schema::{AnyIdent, Recipe, SpecRecipe}; use spk_solve::solution::Solution; -use spk_solve::{BoxedResolverCallback, DefaultResolver, ResolverCallback, Solver, StepSolver}; +use spk_solve::{ + BoxedCdclResolverCallback, + DefaultCdclResolver, + ResolverCallback, + ResolvoSolver, + Solver, +}; use spk_storage as storage; use super::Tester; @@ -28,8 +34,8 @@ pub struct PackageBuildTester<'a> { options: OptionMap, additional_requirements: Vec, source: BuildSource, - source_resolver: BoxedResolverCallback<'a>, - build_resolver: BoxedResolverCallback<'a>, + source_resolver: BoxedCdclResolverCallback<'a>, + build_resolver: BoxedCdclResolverCallback<'a>, } impl<'a> PackageBuildTester<'a> { @@ -44,8 +50,8 @@ impl<'a> PackageBuildTester<'a> { options: OptionMap::default(), additional_requirements: Vec::new(), source, - source_resolver: Box::new(DefaultResolver {}), - build_resolver: Box::new(DefaultResolver {}), + source_resolver: Box::new(DefaultCdclResolver {}), + build_resolver: Box::new(DefaultCdclResolver {}), } } @@ -82,7 +88,7 @@ impl<'a> PackageBuildTester<'a> { /// process as needed. pub fn with_source_resolver(&mut self, resolver: F) -> &mut Self where - F: ResolverCallback + 'a, + F: ResolverCallback + 'a, { self.source_resolver = Box::new(resolver); self @@ -96,7 +102,7 @@ impl<'a> PackageBuildTester<'a> { /// process as needed. pub fn with_build_resolver(&mut self, resolver: F) -> &mut Self where - F: ResolverCallback + 'a, + F: ResolverCallback + 'a, { self.build_resolver = Box::new(resolver); self @@ -117,18 +123,20 @@ impl<'a> PackageBuildTester<'a> { } } - let mut solver = StepSolver::default(); + let mut solver = ResolvoSolver::default(); solver.set_binary_only(true); solver.update_options(self.options.clone()); for repo in self.repos.iter().cloned() { solver.add_repository(repo); } - solver.configure_for_build_environment(&self.recipe)?; + // TODO + // solver.configure_for_build_environment(&self.recipe)?; for request in self.additional_requirements.drain(..) { solver.add_request(request) } - let (solution, _) = self.build_resolver.solve(&solver).await?; + // let (solution, _) = self.build_resolver.solve(&solver).await?; + let solution = self.build_resolver.solve(&solver).await?; for layer in resolve_runtime_layers(requires_localization, &solution).await? { rt.push_digest(layer); @@ -154,7 +162,7 @@ impl<'a> PackageBuildTester<'a> { } async fn resolve_source_package(&mut self, package: &AnyIdent) -> Result { - let mut solver = StepSolver::default(); + let mut solver = ResolvoSolver::default(); solver.update_options(self.options.clone()); let local_repo: Arc = Arc::new(storage::local_repository().await?.into()); @@ -175,7 +183,8 @@ impl<'a> PackageBuildTester<'a> { solver.add_request(request.into()); - let (solution, _) = self.source_resolver.solve(&solver).await?; + // let (solution, _) = self.source_resolver.solve(&solver).await?; + let solution = self.source_resolver.solve(&solver).await?; Ok(solution) } } diff --git a/crates/spk-cli/cmd-test/src/test/install.rs b/crates/spk-cli/cmd-test/src/test/install.rs index f9dee2e4d9..f0998e036b 100644 --- a/crates/spk-cli/cmd-test/src/test/install.rs +++ b/crates/spk-cli/cmd-test/src/test/install.rs @@ -12,7 +12,14 @@ use spk_schema::foundation::option_map::OptionMap; use spk_schema::ident::{PkgRequest, PreReleasePolicy, RangeIdent, Request, RequestedBy}; use spk_schema::ident_build::Build; use spk_schema::{Recipe, SpecRecipe, Variant, VariantExt}; -use spk_solve::{BoxedResolverCallback, DefaultResolver, ResolverCallback, Solver, StepSolver}; +use spk_solve::{ + BoxedCdclResolverCallback, + DefaultCdclResolver, + ResolverCallback, + ResolvoSolver, + Solution, + Solver, +}; use spk_storage as storage; use super::Tester; @@ -25,7 +32,7 @@ pub struct PackageInstallTester<'a, V> { options: OptionMap, additional_requirements: Vec, source: Option, - env_resolver: BoxedResolverCallback<'a>, + env_resolver: BoxedCdclResolverCallback<'a>, variant: V, } @@ -42,7 +49,7 @@ where options: OptionMap::default(), additional_requirements: Vec::new(), source: None, - env_resolver: Box::new(DefaultResolver {}), + env_resolver: Box::new(DefaultCdclResolver {}), variant, } } @@ -80,7 +87,7 @@ where /// process as needed. pub fn watch_environment_resolve(&mut self, resolver: F) -> &mut Self where - F: ResolverCallback + 'a, + F: ResolverCallback + 'a, { self.env_resolver = Box::new(resolver); self @@ -94,7 +101,7 @@ where let requires_localization = rt.config.mount_backend.requires_localization(); - let mut solver = StepSolver::default(); + let mut solver = ResolvoSolver::default(); solver.set_binary_only(true); solver.update_options(self.options.clone()); for repo in self.repos.iter().cloned() { @@ -122,7 +129,8 @@ where solver.add_request(request) } - let (solution, _) = self.env_resolver.solve(&solver).await?; + // let (solution, _) = self.env_resolver.solve(&solver).await?; + let solution = self.env_resolver.solve(&solver).await?; for layer in resolve_runtime_layers(requires_localization, &solution).await? { rt.push_digest(layer); diff --git a/crates/spk-cli/cmd-test/src/test/sources.rs b/crates/spk-cli/cmd-test/src/test/sources.rs index 7282fe53e7..3ebd92d806 100644 --- a/crates/spk-cli/cmd-test/src/test/sources.rs +++ b/crates/spk-cli/cmd-test/src/test/sources.rs @@ -13,7 +13,14 @@ use spk_schema::foundation::ident_component::Component; use spk_schema::foundation::option_map::OptionMap; use spk_schema::ident::{PkgRequest, PreReleasePolicy, RangeIdent, Request, RequestedBy}; use spk_schema::{Recipe, SpecRecipe}; -use spk_solve::{BoxedResolverCallback, DefaultResolver, ResolverCallback, Solver, StepSolver}; +use spk_solve::{ + BoxedCdclResolverCallback, + DefaultCdclResolver, + ResolverCallback, + ResolvoSolver, + Solution, + Solver, +}; use spk_storage as storage; use super::Tester; @@ -26,7 +33,7 @@ pub struct PackageSourceTester<'a> { options: OptionMap, additional_requirements: Vec, source: Option, - env_resolver: BoxedResolverCallback<'a>, + env_resolver: BoxedCdclResolverCallback<'a>, } impl<'a> PackageSourceTester<'a> { @@ -39,7 +46,7 @@ impl<'a> PackageSourceTester<'a> { options: OptionMap::default(), additional_requirements: Vec::new(), source: None, - env_resolver: Box::new(DefaultResolver {}), + env_resolver: Box::new(DefaultCdclResolver {}), } } @@ -77,7 +84,7 @@ impl<'a> PackageSourceTester<'a> { /// process as needed. pub fn watch_environment_resolve(&mut self, resolver: F) -> &mut Self where - F: ResolverCallback + 'a, + F: ResolverCallback + 'a, { self.env_resolver = Box::new(resolver); self @@ -92,7 +99,7 @@ impl<'a> PackageSourceTester<'a> { let requires_localization = rt.config.mount_backend.requires_localization(); - let mut solver = StepSolver::default(); + let mut solver = ResolvoSolver::default(); solver.set_binary_only(true); solver.update_options(self.options.clone()); for repo in self.repos.iter().cloned() { @@ -116,7 +123,8 @@ impl<'a> PackageSourceTester<'a> { solver.add_request(request) } - let (solution, _) = self.env_resolver.solve(&solver).await?; + // let (solution, _) = self.env_resolver.solve(&solver).await?; + let solution = self.env_resolver.solve(&solver).await?; for layer in resolve_runtime_layers(requires_localization, &solution).await? { rt.push_digest(layer); diff --git a/crates/spk-solve/src/io.rs b/crates/spk-solve/src/io.rs index 3337a7c2c3..1aebcd63a2 100644 --- a/crates/spk-solve/src/io.rs +++ b/crates/spk-solve/src/io.rs @@ -2127,20 +2127,20 @@ impl DecisionFormatter { #[async_trait::async_trait] impl ResolverCallback for &DecisionFormatter { - async fn solve<'s, 'a: 's>( - &'s self, - r: &'a StepSolver, - ) -> Result<(Solution, Arc>)> { + type Solver = StepSolver; + type SolveResult = (Solution, Arc>); + + async fn solve<'s, 'a: 's>(&'s self, r: &'a Self::Solver) -> Result { self.run_and_print_resolve(r).await } } #[async_trait::async_trait] impl ResolverCallback for DecisionFormatter { - async fn solve<'s, 'a: 's>( - &'s self, - r: &'a StepSolver, - ) -> Result<(Solution, Arc>)> { + type Solver = StepSolver; + type SolveResult = (Solution, Arc>); + + async fn solve<'s, 'a: 's>(&'s self, r: &'a Self::Solver) -> Result { self.run_and_print_resolve(r).await } } diff --git a/crates/spk-solve/src/lib.rs b/crates/spk-solve/src/lib.rs index 991a78be83..fb77e7b6e8 100644 --- a/crates/spk-solve/src/lib.rs +++ b/crates/spk-solve/src/lib.rs @@ -66,12 +66,11 @@ pub use { #[async_trait::async_trait] pub trait ResolverCallback: Send + Sync { - /// Run a solve using the given [`crate::StepSolver`], - /// producing a [`crate::Solution`]. - async fn solve<'s, 'a: 's>( - &'s self, - r: &'a StepSolver, - ) -> Result<(Solution, Arc>)>; + type Solver; + type SolveResult; + + /// Run a solve using the given [`Self::Solver`] producing a [`Self::SolveResult`]. + async fn solve<'s, 'a: 's>(&'s self, r: &'a Self::Solver) -> Result; } /// A no-frills implementation of [`ResolverCallback`]. @@ -79,10 +78,10 @@ pub struct DefaultResolver {} #[async_trait::async_trait] impl ResolverCallback for DefaultResolver { - async fn solve<'s, 'a: 's>( - &'s self, - r: &'a StepSolver, - ) -> Result<(Solution, Arc>)> { + type Solver = StepSolver; + type SolveResult = (Solution, Arc>); + + async fn solve<'s, 'a: 's>(&'s self, r: &'a Self::Solver) -> Result { let mut runtime = r.run(); let solution = runtime.solution().await; match solution { @@ -92,4 +91,25 @@ impl ResolverCallback for DefaultResolver { } } -pub type BoxedResolverCallback<'a> = Box; +pub type BoxedResolverCallback<'a> = Box< + dyn ResolverCallback< + Solver = StepSolver, + SolveResult = (Solution, Arc>), + > + 'a, +>; + +/// Another no-frills implementation of [`ResolverCallback`]. +pub struct DefaultCdclResolver {} + +#[async_trait::async_trait] +impl ResolverCallback for DefaultCdclResolver { + type Solver = ResolvoSolver; + type SolveResult = Solution; + + async fn solve<'s, 'a: 's>(&'s self, r: &'a Self::Solver) -> Result { + r.solve().await + } +} + +pub type BoxedCdclResolverCallback<'a> = + Box + 'a>; diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index ba7cd3d4ce..072b9ecf87 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -68,52 +68,8 @@ impl Solver { pub(crate) fn set_build_from_source_trail(&mut self, trail: HashSet) { self.build_from_source_trail = trail; } -} - -#[async_trait::async_trait] -impl SolverTrait for Solver { - fn add_repository(&mut self, repo: R) - where - R: Into>, - { - self.repos.push(repo.into()); - } - - fn add_request(&mut self, request: Request) { - self.requests.push(request); - } - - fn get_pkg_requests(&self) -> Vec { - self.requests - .iter() - .filter_map(|r| r.pkg_ref()) - .cloned() - .collect() - } - - fn get_var_requests(&self) -> Vec { - self.requests - .iter() - .filter_map(|r| r.var_ref()) - .cloned() - .collect() - } - - fn repositories(&self) -> &[Arc] { - &self.repos - } - fn reset(&mut self) { - self.repos.truncate(0); - self.requests.truncate(0); - self._validators = Cow::from(default_validators()); - } - - fn set_binary_only(&mut self, binary_only: bool) { - self.binary_only = binary_only; - } - - async fn solve(&mut self) -> Result { + pub async fn solve(&self) -> Result { let repos = self.repos.clone(); let requests = self.requests.clone(); let binary_only = self.binary_only; @@ -295,6 +251,54 @@ impl SolverTrait for Solver { } Ok(solution) } +} + +#[async_trait::async_trait] +impl SolverTrait for Solver { + fn add_repository(&mut self, repo: R) + where + R: Into>, + { + self.repos.push(repo.into()); + } + + fn add_request(&mut self, request: Request) { + self.requests.push(request); + } + + fn get_pkg_requests(&self) -> Vec { + self.requests + .iter() + .filter_map(|r| r.pkg_ref()) + .cloned() + .collect() + } + + fn get_var_requests(&self) -> Vec { + self.requests + .iter() + .filter_map(|r| r.var_ref()) + .cloned() + .collect() + } + + fn repositories(&self) -> &[Arc] { + &self.repos + } + + fn reset(&mut self) { + self.repos.truncate(0); + self.requests.truncate(0); + self._validators = Cow::from(default_validators()); + } + + fn set_binary_only(&mut self, binary_only: bool) { + self.binary_only = binary_only; + } + + async fn solve(&mut self) -> Result { + Solver::solve(self).await + } fn update_options(&mut self, _options: OptionMap) { // TODO From 7568cbeb9e16944ac234755c2e8467cf690bd9a4 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Sun, 9 Feb 2025 02:57:23 -0800 Subject: [PATCH 38/64] Refactor type used for resolvo package names By using an enum here, the global var handling can use its own enum variant instead of the flawed string prefix approach. Now it is simply not possible for a package name to be confused for a global var. Signed-off-by: J Robert Ray --- .../resolvo/pkg_request_version_set.rs | 5 +- .../src/solvers/resolvo/spk_provider.rs | 606 ++++++++++-------- 2 files changed, 333 insertions(+), 278 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs b/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs index 0c3a1b1df9..3bed5c4c79 100644 --- a/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs +++ b/crates/spk-solve/src/solvers/resolvo/pkg_request_version_set.rs @@ -8,6 +8,7 @@ use resolvo::utils::VersionSet; use spk_schema::Request; use spk_schema::ident::{LocatedBuildIdent, PkgRequest, PreReleasePolicy, RangeIdent, RequestedBy}; use spk_schema::ident_component::Component; +use spk_schema::name::OptNameBuf; /// This allows for storing strings of different types but hash and compare by /// the underlying strings. @@ -74,7 +75,7 @@ impl std::fmt::Display for VarValue { #[derive(Clone, Debug, Eq, Hash, PartialEq)] pub(crate) enum RequestVS { SpkRequest(Request), - GlobalVar { key: String, value: VarValue }, + GlobalVar { key: OptNameBuf, value: VarValue }, } impl std::fmt::Display for RequestVS { @@ -163,7 +164,7 @@ impl PartialEq for Component { #[derive(Clone, Debug, Eq, Hash, PartialEq)] pub(crate) enum SpkSolvable { LocatedBuildIdentWithComponent(LocatedBuildIdentWithComponent), - GlobalVar { key: String, value: VarValue }, + GlobalVar { key: OptNameBuf, value: VarValue }, } impl std::fmt::Display for SpkSolvable { diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index 6e982bb415..96741c938b 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -23,7 +23,6 @@ use resolvo::{ VersionSetId, VersionSetUnionId, }; -use spk_schema::foundation::pkg_name; use spk_schema::ident::{ LocatedBuildIdent, PinnableValue, @@ -52,11 +51,6 @@ use super::pkg_request_version_set::{ }; use crate::Solver; -// "global-vars--" represents a pseudo- package that accumulates global var -// constraints. The name is intended to never conflict with a real package name, -// however since it is a legal package name that can't be guaranteed. -const PSEUDO_PKG_NAME_PREFIX: &str = "global-vars--"; - // Using just the package name as a Resolvo "package name" prevents multiple // components from the same package from existing in the same solution, since // we consider the different components to be different "solvables". Instead, @@ -79,13 +73,257 @@ impl std::fmt::Display for PkgNameBufWithComponent { } } +#[derive(Clone, Eq, Hash, PartialEq)] +pub(crate) enum ResolvoPackageName { + GlobalVar(OptNameBuf), + PkgNameBufWithComponent(PkgNameBufWithComponent), +} + +impl ResolvoPackageName { + async fn get_candidates(&self, name: NameId, provider: &SpkProvider) -> Option { + match self { + ResolvoPackageName::GlobalVar(key) => { + provider + .queried_global_var_values + .borrow_mut() + .insert(key.clone()); + + if let Some(values) = provider.known_global_var_values.borrow().get(key) { + let mut candidates = Candidates { + candidates: Vec::with_capacity(values.len()), + ..Default::default() + }; + for value in values { + let solvable_id = *provider + .interned_solvables + .borrow_mut() + .entry(SpkSolvable::GlobalVar { + key: key.clone(), + value: value.clone(), + }) + .or_insert_with(|| { + provider.pool.intern_solvable( + name, + SpkSolvable::GlobalVar { + key: key.clone(), + value: value.clone(), + }, + ) + }); + candidates.candidates.push(solvable_id); + } + return Some(candidates); + } + None + } + ResolvoPackageName::PkgNameBufWithComponent(pkg_name) => { + let mut located_builds = Vec::new(); + + for repo in &provider.repos { + let versions = repo + .list_package_versions(&pkg_name.name) + .await + .unwrap_or_default(); + for version in versions.iter() { + // TODO: We need a borrowing version of this to avoid cloning. + let pkg_version = + VersionIdent::new(pkg_name.name.clone(), (**version).clone()); + + let builds = repo + .list_package_builds(&pkg_version) + .await + .unwrap_or_default(); + + for build in builds { + let located_build_ident = + LocatedBuildIdent::new(repo.name().to_owned(), build.clone()); + if let SyntheticComponent::Actual(pkg_name_component) = + &pkg_name.component + { + let components = + if let Build::Embedded(EmbeddedSource::Package(_parent)) = + build.build() + { + // Does this embedded stub contain the component + // being requested? For whatever reason, + // list_build_components returns an empty list for + // embedded stubs. + itertools::Either::Right( + if let Ok(stub) = repo.read_embed_stub(&build).await { + itertools::Either::Right( + stub.components() + .iter() + .map(|component_spec| { + component_spec.name.clone() + }) + .collect::>() + .into_iter(), + ) + } else { + itertools::Either::Left(std::iter::empty()) + }, + ) + } else { + itertools::Either::Left( + repo.list_build_components(&build) + .await + .unwrap_or_default(), + ) + }; + for component in components.into_iter().chain( + // A build representing the All component is included so + // when a request for it is found it can act as a + // surrogate that depends on all the individual + // components. + [Component::All], + ) { + if component != *pkg_name_component + && (provider.binary_only || !component.is_source()) + { + continue; + } + + located_builds.push(LocatedBuildIdentWithComponent { + ident: located_build_ident.clone(), + component: pkg_name.component.clone(), + requires_build_from_source: component + != *pkg_name_component, + }); + } + } else { + located_builds.push(LocatedBuildIdentWithComponent { + ident: located_build_ident, + component: SyntheticComponent::Base, + requires_build_from_source: false, + }); + } + } + } + } + + if located_builds.is_empty() { + return None; + } + + let mut candidates = Candidates { + candidates: Vec::with_capacity(located_builds.len()), + ..Default::default() + }; + + for build in located_builds { + // What we need from build before it is moved into the pool. + let ident = build.ident.clone(); + let requires_build_from_source = build.requires_build_from_source; + + let solvable_id = *provider + .interned_solvables + .borrow_mut() + .entry(SpkSolvable::LocatedBuildIdentWithComponent(build.clone())) + .or_insert_with(|| { + provider.pool.intern_solvable( + name, + SpkSolvable::LocatedBuildIdentWithComponent(build), + ) + }); + + // Filter builds that don't conform to global options + // XXX: This find runtime will add up. + let repo = provider + .repos + .iter() + .find(|repo| repo.name() == ident.repository_name()) + .expect("Expected solved package's repository to be in the list of repositories"); + + if requires_build_from_source { + match provider.can_build_from_source(&ident).await { + CanBuildFromSource::Yes => { + candidates.candidates.push(solvable_id); + } + CanBuildFromSource::No(reason) => { + candidates.excluded.push((solvable_id, reason)); + } + } + continue; + } + + match repo.read_package(ident.target()).await { + Ok(package) => { + // Filter builds that don't satisfy global var requests + if let Some(VarRequest { + value: PinnableValue::Pinned(expected_version), + .. + }) = provider.global_var_requests.get(ident.name().as_opt_name()) + { + if let Ok(expected_version) = parse_version_range(expected_version) + { + if let spk_schema::version::Compatibility::Incompatible( + incompatible_reason, + ) = expected_version.is_applicable(package.version()) + { + candidates.excluded.push(( + solvable_id, + provider.pool.intern_string(format!( + "build version does not satisfy global var request: {incompatible_reason}" + )), + )); + continue; + } + } + } + + // XXX: `package.check_satisfies_request` walks the + // package's build options, so is it better to do this loop + // over `option_values` here, or loop over all the + // global_var_requests instead? + for (opt_name, _value) in package.option_values() { + if let Some(request) = provider.global_var_requests.get(&opt_name) { + if let spk_schema::version::Compatibility::Incompatible( + incompatible_reason, + ) = package.check_satisfies_request(request) + { + candidates.excluded.push(( + solvable_id, + provider.pool.intern_string(format!( + "build option {opt_name} does not satisfy global var request: {incompatible_reason}" + )), + )); + continue; + } + } + } + + candidates.candidates.push(solvable_id); + } + Err(err) => { + candidates + .excluded + .push((solvable_id, provider.pool.intern_string(err.to_string()))); + } + } + } + + Some(candidates) + } + } + } +} + +impl std::fmt::Display for ResolvoPackageName { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + ResolvoPackageName::GlobalVar(name) => write!(f, "{name}"), + ResolvoPackageName::PkgNameBufWithComponent(name) => write!(f, "{name}"), + } + } +} + enum CanBuildFromSource { Yes, No(StringId), } pub(crate) struct SpkProvider { - pub(crate) pool: Pool, + pub(crate) pool: Pool, repos: Vec>, /// Global options, like what might be specified with `--opt` to `spk env`. /// Indexed by name. If multiple requests happen to exist with the same @@ -94,11 +332,11 @@ pub(crate) struct SpkProvider { interned_solvables: RefCell>, /// Track all the global var keys and values that have been witnessed while /// solving. - known_global_var_values: RefCell>>, + known_global_var_values: RefCell>>, /// Track which global var candidates have been queried by the solver. Once /// queried, it is no longer possible to add more possible values without /// restarting the solve. - queried_global_var_values: RefCell>, + queried_global_var_values: RefCell>, cancel_solving: Cell, binary_only: bool, /// When recursively exploring building packages from source, track chain @@ -248,10 +486,14 @@ impl SpkProvider { constrains: Vec::new(), }; for component in iter { - let dep_name = self.pool.intern_package_name(PkgNameBufWithComponent { - name: pkg_request.pkg.name().to_owned(), - component, - }); + let dep_name = + self.pool + .intern_package_name(ResolvoPackageName::PkgNameBufWithComponent( + PkgNameBufWithComponent { + name: pkg_request.pkg.name().to_owned(), + component, + }, + )); let dep_vs = self.pool.intern_version_set( dep_name, RequestVS::SpkRequest(Request::Pkg(pkg_request.clone())), @@ -297,47 +539,44 @@ impl SpkProvider { spk_schema::ident::PinnableValue::FromBuildEnvIfPresent => todo!(), spk_schema::ident::PinnableValue::Pinned(value) => { let dep_name = match var_request.var.namespace() { - Some(pkg_name) => { - self.pool.intern_package_name(PkgNameBufWithComponent { - name: pkg_name.to_owned(), - component: SyntheticComponent::Base, - }) - } + Some(pkg_name) => self.pool.intern_package_name( + ResolvoPackageName::PkgNameBufWithComponent( + PkgNameBufWithComponent { + name: pkg_name.to_owned(), + component: SyntheticComponent::Base, + }, + ), + ), None => { // Since we will be adding constraints for // global vars we need to add the pseudo-package // to the dependency list so it will influence // decisions. - let pseudo_pkg_name = format!( - "{PSEUDO_PKG_NAME_PREFIX}{}", - var_request.var.base_name() - ); if self .known_global_var_values .borrow_mut() - .entry(var_request.var.base_name().to_owned()) + .entry(var_request.var.without_namespace().to_owned()) .or_default() .insert(VarValue::ArcStr(Arc::clone(value))) && self .queried_global_var_values .borrow() - .contains(var_request.var.base_name()) + .contains(var_request.var.without_namespace()) { // Seeing a new value for a var that has // already locked in the list of candidates. self.cancel_solving.set(true); } let dep_name = - self.pool.intern_package_name(PkgNameBufWithComponent { - name: pkg_name!(&pseudo_pkg_name).to_owned(), - component: SyntheticComponent::Base, - }); + self.pool.intern_package_name(ResolvoPackageName::GlobalVar( + var_request.var.without_namespace().to_owned(), + )); known_deps.requirements.push( self.pool .intern_version_set( dep_name, RequestVS::GlobalVar { - key: var_request.var.base_name().to_owned(), + key: var_request.var.without_namespace().to_owned(), value: VarValue::ArcStr(Arc::clone(value)), }, ) @@ -386,10 +625,14 @@ impl SpkProvider { .filter_map(|req| match req.var.namespace() { Some(pkg_name) => { // A global request applicable to a specific package. - let dep_name = self.pool.intern_package_name(PkgNameBufWithComponent { - name: pkg_name.to_owned(), - component: SyntheticComponent::Base, - }); + let dep_name = + self.pool + .intern_package_name(ResolvoPackageName::PkgNameBufWithComponent( + PkgNameBufWithComponent { + name: pkg_name.to_owned(), + component: SyntheticComponent::Base, + }, + )); Some(self.pool.intern_version_set( dep_name, RequestVS::SpkRequest(Request::Var(req.clone())), @@ -588,7 +831,7 @@ impl DependencyProvider for SpkProvider { } continue; }; - if (var_request.var.base_name() == record_key + if (var_request.var.without_namespace() == record_key && value == record_value) ^ inverse { @@ -619,216 +862,8 @@ impl DependencyProvider for SpkProvider { } async fn get_candidates(&self, name: NameId) -> Option { - let pkg_name = self.pool.resolve_package_name(name); - - if let Some(key) = pkg_name.name.strip_prefix(PSEUDO_PKG_NAME_PREFIX) { - self.queried_global_var_values - .borrow_mut() - .insert(key.to_owned()); - - if let Some(values) = self.known_global_var_values.borrow().get(key) { - let mut candidates = Candidates { - candidates: Vec::with_capacity(values.len()), - ..Default::default() - }; - for value in values { - let solvable_id = *self - .interned_solvables - .borrow_mut() - .entry(SpkSolvable::GlobalVar { - key: key.to_owned(), - value: value.clone(), - }) - .or_insert_with(|| { - self.pool.intern_solvable( - name, - SpkSolvable::GlobalVar { - key: key.to_owned(), - value: value.clone(), - }, - ) - }); - candidates.candidates.push(solvable_id); - } - return Some(candidates); - } - return None; - } - - let mut located_builds = Vec::new(); - - for repo in &self.repos { - let versions = repo - .list_package_versions(&pkg_name.name) - .await - .unwrap_or_default(); - for version in versions.iter() { - // TODO: We need a borrowing version of this to avoid cloning. - let pkg_version = VersionIdent::new(pkg_name.name.clone(), (**version).clone()); - - let builds = repo - .list_package_builds(&pkg_version) - .await - .unwrap_or_default(); - - for build in builds { - let located_build_ident = - LocatedBuildIdent::new(repo.name().to_owned(), build.clone()); - if let SyntheticComponent::Actual(pkg_name_component) = &pkg_name.component { - let components = if let Build::Embedded(EmbeddedSource::Package(_parent)) = - build.build() - { - // Does this embedded stub contain the component - // being requested? For whatever reason, - // list_build_components returns an empty list for - // embedded stubs. - itertools::Either::Right( - if let Ok(stub) = repo.read_embed_stub(&build).await { - itertools::Either::Right( - stub.components() - .iter() - .map(|component_spec| component_spec.name.clone()) - .collect::>() - .into_iter(), - ) - } else { - itertools::Either::Left(std::iter::empty()) - }, - ) - } else { - itertools::Either::Left( - repo.list_build_components(&build).await.unwrap_or_default(), - ) - }; - for component in components.into_iter().chain( - // A build representing the All component is included so - // when a request for it is found it can act as a - // surrogate that depends on all the individual - // components. - [Component::All], - ) { - if component != *pkg_name_component - && (self.binary_only || !component.is_source()) - { - continue; - } - - located_builds.push(LocatedBuildIdentWithComponent { - ident: located_build_ident.clone(), - component: pkg_name.component.clone(), - requires_build_from_source: component != *pkg_name_component, - }); - } - } else { - located_builds.push(LocatedBuildIdentWithComponent { - ident: located_build_ident, - component: SyntheticComponent::Base, - requires_build_from_source: false, - }); - } - } - } - } - - if located_builds.is_empty() { - return None; - } - - let mut candidates = Candidates { - candidates: Vec::with_capacity(located_builds.len()), - ..Default::default() - }; - - for build in located_builds { - // What we need from build before it is moved into the pool. - let ident = build.ident.clone(); - let requires_build_from_source = build.requires_build_from_source; - - let solvable_id = *self - .interned_solvables - .borrow_mut() - .entry(SpkSolvable::LocatedBuildIdentWithComponent(build.clone())) - .or_insert_with(|| { - self.pool - .intern_solvable(name, SpkSolvable::LocatedBuildIdentWithComponent(build)) - }); - - // Filter builds that don't conform to global options - // XXX: This find runtime will add up. - let repo = self - .repos - .iter() - .find(|repo| repo.name() == ident.repository_name()) - .expect("Expected solved package's repository to be in the list of repositories"); - - if requires_build_from_source { - match self.can_build_from_source(&ident).await { - CanBuildFromSource::Yes => { - candidates.candidates.push(solvable_id); - } - CanBuildFromSource::No(reason) => { - candidates.excluded.push((solvable_id, reason)); - } - } - continue; - } - - match repo.read_package(ident.target()).await { - Ok(package) => { - // Filter builds that don't satisfy global var requests - if let Some(VarRequest { - value: PinnableValue::Pinned(expected_version), - .. - }) = self.global_var_requests.get(ident.name().as_opt_name()) - { - if let Ok(expected_version) = parse_version_range(expected_version) { - if let spk_schema::version::Compatibility::Incompatible( - incompatible_reason, - ) = expected_version.is_applicable(package.version()) - { - candidates.excluded.push(( - solvable_id, - self.pool.intern_string(format!( - "build version does not satisfy global var request: {incompatible_reason}" - )), - )); - continue; - } - } - } - - // XXX: `package.check_satisfies_request` walks the - // package's build options, so is it better to do this loop - // over `option_values` here, or loop over all the - // global_var_requests instead? - for (opt_name, _value) in package.option_values() { - if let Some(request) = self.global_var_requests.get(&opt_name) { - if let spk_schema::version::Compatibility::Incompatible( - incompatible_reason, - ) = package.check_satisfies_request(request) - { - candidates.excluded.push(( - solvable_id, - self.pool.intern_string(format!( - "build option {opt_name} does not satisfy global var request: {incompatible_reason}" - )), - )); - continue; - } - } - } - - candidates.candidates.push(solvable_id); - } - Err(err) => { - candidates - .excluded - .push((solvable_id, self.pool.intern_string(err.to_string()))); - } - } - } - - Some(candidates) + let resolvo_package_name = self.pool.resolve_package_name(name); + resolvo_package_name.get_candidates(name, self).await } async fn sort_candidates(&self, _solver: &SolverCache, solvables: &mut [SolvableId]) { @@ -917,10 +952,12 @@ impl DependencyProvider for SpkProvider { // The only dependencies of the All component are the other // components defined in the package. for component_spec in package.components().iter() { - let dep_name = self.pool.intern_package_name(PkgNameBufWithComponent { - name: located_build_ident_with_component.ident.name().to_owned(), - component: SyntheticComponent::Actual(component_spec.name.clone()), - }); + let dep_name = self.pool.intern_package_name( + ResolvoPackageName::PkgNameBufWithComponent(PkgNameBufWithComponent { + name: located_build_ident_with_component.ident.name().to_owned(), + component: SyntheticComponent::Actual(component_spec.name.clone()), + }), + ); known_deps.requirements.push( self.pool .intern_version_set( @@ -940,10 +977,17 @@ impl DependencyProvider for SpkProvider { // For any non-All/non-Base component, add a dependency on // the base to ensure all components come from the same // base version. - let dep_name = self.pool.intern_package_name(PkgNameBufWithComponent { - name: located_build_ident_with_component.ident.name().to_owned(), - component: SyntheticComponent::Base, - }); + let dep_name = + self.pool + .intern_package_name(ResolvoPackageName::PkgNameBufWithComponent( + PkgNameBufWithComponent { + name: located_build_ident_with_component + .ident + .name() + .to_owned(), + component: SyntheticComponent::Base, + }, + )); known_deps.requirements.push( self.pool .intern_version_set( @@ -963,10 +1007,17 @@ impl DependencyProvider for SpkProvider { .find(|component_spec| component_spec.name == *actual_component) { component_spec.uses.iter().for_each(|uses| { - let dep_name = self.pool.intern_package_name(PkgNameBufWithComponent { - name: located_build_ident_with_component.ident.name().to_owned(), - component: SyntheticComponent::Actual(uses.clone()), - }); + let dep_name = self.pool.intern_package_name( + ResolvoPackageName::PkgNameBufWithComponent( + PkgNameBufWithComponent { + name: located_build_ident_with_component + .ident + .name() + .to_owned(), + component: SyntheticComponent::Actual(uses.clone()), + }, + ), + ); known_deps.requirements.push( self.pool .intern_version_set( @@ -1015,10 +1066,14 @@ impl DependencyProvider for SpkProvider { continue; } - let dep_name = self.pool.intern_package_name(PkgNameBufWithComponent { - name: embedded.name().to_owned(), - component: located_build_ident_with_component.component.clone(), - }); + let dep_name = + self.pool + .intern_package_name(ResolvoPackageName::PkgNameBufWithComponent( + PkgNameBufWithComponent { + name: embedded.name().to_owned(), + component: located_build_ident_with_component.component.clone(), + }, + )); known_deps.requirements.push( self.pool .intern_version_set( @@ -1132,12 +1187,14 @@ impl DependencyProvider for SpkProvider { }) .for_each(|_embedded_package| { let dep_name = self.pool.intern_package_name( - PkgNameBufWithComponent { - name: ident.name().to_owned(), - component: SyntheticComponent::Actual( - parent_component.name.clone(), - ), - }, + ResolvoPackageName::PkgNameBufWithComponent( + PkgNameBufWithComponent { + name: ident.name().to_owned(), + component: SyntheticComponent::Actual( + parent_component.name.clone(), + ), + }, + ), ); known_deps.requirements.push( self.pool.intern_version_set( @@ -1187,34 +1244,31 @@ impl DependencyProvider for SpkProvider { let Some(value) = var_opt.get_value(None) else { continue; }; - let pseudo_pkg_name = - format!("{PSEUDO_PKG_NAME_PREFIX}{}", var_opt.var.base_name()); if self .known_global_var_values .borrow_mut() - .entry(var_opt.var.base_name().to_owned()) + .entry(var_opt.var.without_namespace().to_owned()) .or_default() .insert(VarValue::Owned(value.clone())) && self .queried_global_var_values .borrow() - .contains(var_opt.var.base_name()) + .contains(var_opt.var.without_namespace()) { // Seeing a new value for a var that has already locked // in the list of candidates. self.cancel_solving.set(true); } - let dep_name = self.pool.intern_package_name(PkgNameBufWithComponent { - name: pkg_name!(&pseudo_pkg_name).to_owned(), - component: SyntheticComponent::Base, - }); + let dep_name = self.pool.intern_package_name(ResolvoPackageName::GlobalVar( + var_opt.var.without_namespace().to_owned(), + )); // Add a constraint not a dependency because the package // is targeting a specific global var value but there may // not be a request for that var of a specific value. known_deps.constrains.push(self.pool.intern_version_set( dep_name, RequestVS::GlobalVar { - key: var_opt.var.base_name().to_owned(), + key: var_opt.var.without_namespace().to_owned(), value: VarValue::Owned(value), }, )); From 7e35be38298231e39c4d05c64188dce3ee500dd5 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Mon, 10 Feb 2025 10:35:12 -0800 Subject: [PATCH 39/64] Add a crude solve result output to cmd_explain --- crates/spk-cli/cmd-explain/src/cmd_explain.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/spk-cli/cmd-explain/src/cmd_explain.rs b/crates/spk-cli/cmd-explain/src/cmd_explain.rs index 7fab1c1721..03047a9c84 100644 --- a/crates/spk-cli/cmd-explain/src/cmd_explain.rs +++ b/crates/spk-cli/cmd-explain/src/cmd_explain.rs @@ -7,7 +7,7 @@ use std::any::Any; use clap::Args; use miette::Result; use spk_cli_common::{CommandArgs, Run, flags}; -use spk_solve::Solver; +use spk_solve::{Package, Solver}; /// Show the resolve process for a set of packages. #[derive(Args)] @@ -92,7 +92,10 @@ impl Run for Explain { formatter.run_and_print_resolve(solver).await?; } else { // TODO: print the solve - solver.solve().await?; + let solution = solver.solve().await?; + for item in solution.items() { + println!("{}", item.spec.ident()); + } } Ok(0) From cfd61352afd4ad4f3539e834b2bbfe0adece6cae Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Mon, 10 Feb 2025 14:35:54 -0800 Subject: [PATCH 40/64] Fix requests for :all not filtering invalid versions Messed up the logic when checking :all cases. Add test for this case. Signed-off-by: J Robert Ray --- .../src/solvers/resolvo/spk_provider.rs | 83 ++++++++++--------- crates/spk-solve/src/solvers/solver_test.rs | 34 +++++++- 2 files changed, 77 insertions(+), 40 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index 96741c938b..a25173fee8 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -675,6 +675,7 @@ impl DependencyProvider for SpkProvider { let compatible = pkg_request .is_version_applicable(located_build_ident_with_component.ident.version()); if compatible.is_ok() { + tracing::trace!(pkg_request = %pkg_request.pkg, version = %located_build_ident_with_component.ident.version(), "version applicable"); let is_source = located_build_ident_with_component.ident.build().is_source(); @@ -706,52 +707,57 @@ impl DependencyProvider for SpkProvider { // Only select All component for requests of All // component. if located_build_ident_with_component.component.is_all() { - if pkg_request.pkg.components.contains(&Component::All) ^ inverse { - selected.push(*candidate); - } - continue; - } else - // Only the All component can satisfy requests for All. - if pkg_request.pkg.components.contains(&Component::All) { - if inverse { - selected.push(*candidate); - } - continue; - } - - // Only the x component can satisfy requests for x. - let mut at_least_one_request_matched_this_solvable = None; - for component in pkg_request.pkg.components.iter() { - if component.is_all() { - continue; - } - if component == &located_build_ident_with_component.component { - at_least_one_request_matched_this_solvable = Some(true); - break; - } else { - at_least_one_request_matched_this_solvable = Some(false); - } - } - - match at_least_one_request_matched_this_solvable { - Some(true) => { + // This can disqualify but not qualify; version + // compatibility check is still required. + if !pkg_request.pkg.components.contains(&Component::All) { if inverse { - continue; + selected.push(*candidate); } + continue; } - Some(false) => { - // The request is for specific components but - // this solvable doesn't match any of them. + } else { + // Only the All component can satisfy requests for All. + if pkg_request.pkg.components.contains(&Component::All) { if inverse { selected.push(*candidate); + } + continue; + } + + // Only the x component can satisfy requests for x. + let mut at_least_one_request_matched_this_solvable = None; + for component in pkg_request.pkg.components.iter() { + if component.is_all() { continue; } + if component == &located_build_ident_with_component.component { + at_least_one_request_matched_this_solvable = Some(true); + break; + } else { + at_least_one_request_matched_this_solvable = Some(false); + } } - None => { - // TODO: if at_least_one_request_matched_this_solvable - // is None it means the request didn't specify a - // component. Decide which specific component this - // should match. + + match at_least_one_request_matched_this_solvable { + Some(true) => { + if inverse { + continue; + } + } + Some(false) => { + // The request is for specific components but + // this solvable doesn't match any of them. + if inverse { + selected.push(*candidate); + continue; + } + } + None => { + // TODO: if at_least_one_request_matched_this_solvable + // is None it means the request didn't specify a + // component. Decide which specific component this + // should match. + } } } @@ -768,6 +774,7 @@ impl DependencyProvider for SpkProvider { .await { if pkg_request.is_satisfied_by(&package).is_ok() ^ inverse { + tracing::trace!(pkg_request = %pkg_request.pkg, package = %package.ident(), %inverse, "satisfied by"); selected.push(*candidate); } } else if inverse { diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index 26f6edc7f9..c1cf361277 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -45,12 +45,15 @@ fn solver() -> StepSolver { /// of resolved components, or the specific build of the package. macro_rules! assert_resolved { ($solution:ident, $pkg:literal, $version:literal) => { - assert_resolved!($solution, $pkg, $version, "wrong package version was resolved") + assert_resolved!($solution, $pkg, version = $version, "wrong package version was resolved") + }; + ($solution:ident, $pkg:literal, version = $version:expr) => { + assert_resolved!($solution, $pkg, version = $version, "wrong package version was resolved") }; ($solution:ident, $pkg:literal, $version:literal, $message:literal) => { assert_resolved!($solution, $pkg, version = $version, $message) }; - ($solution:ident, $pkg:literal, version = $version:literal, $message:literal) => {{ + ($solution:ident, $pkg:literal, version = $version:expr, $message:literal) => {{ let pkg = $solution .get($pkg) .expect("expected package to be in solution"); @@ -2923,3 +2926,30 @@ async fn test_version_number_masking( ); assert_ne!(resolved.spec.ident().build(), &Build::Source); } + +#[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] +#[tokio::test] +async fn request_for_all_component_picks_correct_version( + #[case] mut solver: SolverImpl, + #[values("1.0.0", "2.0.0", "3.0.0")] version: &str, +) { + // A request for :all component still controls for version compatibility + + let repo = make_repo!( + [ + { "pkg": "mypkg/1.0.0" }, + { "pkg": "mypkg/2.0.0" }, + { "pkg": "mypkg/3.0.0" }, + ] + ); + let repo = Arc::new(repo); + + solver.add_repository(repo); + let request_str = format!("mypkg:all/{version}"); + solver.add_request(request!(request_str)); + + let solution = run_and_print_resolve_for_tests(&mut solver).await.unwrap(); + assert_resolved!(solution, "mypkg", version = version); +} From 855ada4d2545729f11b5b8f093154be74a829539 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Mon, 10 Feb 2025 16:32:08 -0800 Subject: [PATCH 41/64] Implement update_options in new solver These are effectively turned into more requests. Now the host options are being looked at. Signed-off-by: J Robert Ray --- crates/spk-solve/src/solvers/resolvo/mod.rs | 13 +++++-- .../src/solvers/resolvo/spk_provider.rs | 35 ++++++++++++++++++- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index 072b9ecf87..b3d822b2a2 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -49,6 +49,7 @@ mod resolvo_tests; pub struct Solver { repos: Vec>, requests: Vec, + options: OptionMap, binary_only: bool, _validators: Cow<'static, [Validators]>, build_from_source_trail: HashSet, @@ -59,6 +60,7 @@ impl Solver { Self { repos, requests: Vec::new(), + options: Default::default(), binary_only: true, _validators: validators, build_from_source_trail: HashSet::new(), @@ -72,6 +74,7 @@ impl Solver { pub async fn solve(&self) -> Result { let repos = self.repos.clone(); let requests = self.requests.clone(); + let options = self.options.clone(); let binary_only = self.binary_only; let build_from_source_trail = self.build_from_source_trail.clone(); // Use a blocking thread so resolvo can call `block_on` on the runtime. @@ -84,7 +87,11 @@ impl Solver { let (solver, solved) = loop { let mut this_iter_provider = provider.take().expect("provider is always Some"); let pkg_requirements = this_iter_provider.pkg_requirements(&requests); - let var_requirements = this_iter_provider.var_requirements(&requests); + let mut var_requirements = this_iter_provider.var_requirements(&requests); + // XXX: Not sure if this will result in the desired precedence + // when options and var requests for the same thing exist. + var_requirements + .extend(this_iter_provider.var_requirements_from_options(options.clone())); let mut solver = resolvo::Solver::new(this_iter_provider) .with_runtime(tokio::runtime::Handle::current()); let problem = resolvo::Problem::new() @@ -300,7 +307,7 @@ impl SolverTrait for Solver { Solver::solve(self).await } - fn update_options(&mut self, _options: OptionMap) { - // TODO + fn update_options(&mut self, options: OptionMap) { + self.options.extend(options); } } diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index a25173fee8..d1b63d5836 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -38,7 +38,7 @@ use spk_schema::name::{OptNameBuf, PkgNameBuf}; use spk_schema::prelude::{HasVersion, Named}; use spk_schema::version::Version; use spk_schema::version_range::{DoubleEqualsVersion, Ranged, VersionFilter, parse_version_range}; -use spk_schema::{BuildIdent, Deprecate, Opt, Package, Recipe, Request, VersionIdent}; +use spk_schema::{BuildIdent, Deprecate, Opt, OptionMap, Package, Recipe, Request, VersionIdent}; use spk_storage::RepositoryHandle; use tracing::{Instrument, debug_span}; @@ -647,6 +647,39 @@ impl SpkProvider { }) .collect() } + + pub fn var_requirements_from_options(&mut self, options: OptionMap) -> Vec { + self.global_var_requests.reserve(options.len()); + options + .into_iter() + .filter_map(|(var, value)| { + let req = VarRequest::new_with_value(var, value); + match req.var.namespace() { + Some(pkg_name) => { + // A global request applicable to a specific package. + let dep_name = self.pool.intern_package_name( + ResolvoPackageName::PkgNameBufWithComponent(PkgNameBufWithComponent { + name: pkg_name.to_owned(), + component: SyntheticComponent::Base, + }), + ); + Some( + self.pool.intern_version_set( + dep_name, + RequestVS::SpkRequest(Request::Var(req)), + ), + ) + } + None => { + // A global request affecting all packages. + self.global_var_requests + .insert(req.var.without_namespace().to_owned(), req); + None + } + } + }) + .collect() + } } impl DependencyProvider for SpkProvider { From 9362c6ab9d6b4bbee8d630a89919cb6c086815be Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Mon, 10 Feb 2025 17:59:15 -0800 Subject: [PATCH 42/64] Update known var values when processing var requirements It is safe to update the known values without restarting the solve here, since these functions are only called before starting the solve. But that's not enforced by the code in any way at the moment. Signed-off-by: J Robert Ray --- .../src/solvers/resolvo/spk_provider.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index d1b63d5836..b3cb63c301 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -642,6 +642,17 @@ impl SpkProvider { // A global request affecting all packages. self.global_var_requests .insert(req.var.without_namespace().to_owned(), req.clone()); + match &req.value { + PinnableValue::FromBuildEnv => {} + PinnableValue::FromBuildEnvIfPresent => {} + PinnableValue::Pinned(arc) => { + self.known_global_var_values + .borrow_mut() + .entry(req.var.without_namespace().to_owned()) + .or_default() + .insert(VarValue::ArcStr(Arc::clone(arc))); + } + }; None } }) @@ -653,6 +664,7 @@ impl SpkProvider { options .into_iter() .filter_map(|(var, value)| { + let var_value = VarValue::Owned(value.clone()); let req = VarRequest::new_with_value(var, value); match req.var.namespace() { Some(pkg_name) => { @@ -673,7 +685,12 @@ impl SpkProvider { None => { // A global request affecting all packages. self.global_var_requests - .insert(req.var.without_namespace().to_owned(), req); + .insert(req.var.without_namespace().to_owned(), req.clone()); + self.known_global_var_values + .borrow_mut() + .entry(req.var.without_namespace().to_owned()) + .or_default() + .insert(var_value); None } } From c2152b12beffc4f20fa6f6acb3ab49f2c8b9b9e7 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Tue, 11 Feb 2025 10:55:59 -0800 Subject: [PATCH 43/64] Wire up the normal solution output in cmd_explain The requested by column is not populated. Signed-off-by: J Robert Ray --- crates/spk-cli/cmd-explain/src/cmd_explain.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/spk-cli/cmd-explain/src/cmd_explain.rs b/crates/spk-cli/cmd-explain/src/cmd_explain.rs index 03047a9c84..a848c9f65e 100644 --- a/crates/spk-cli/cmd-explain/src/cmd_explain.rs +++ b/crates/spk-cli/cmd-explain/src/cmd_explain.rs @@ -7,7 +7,7 @@ use std::any::Any; use clap::Args; use miette::Result; use spk_cli_common::{CommandArgs, Run, flags}; -use spk_solve::{Package, Solver}; +use spk_solve::Solver; /// Show the resolve process for a set of packages. #[derive(Args)] @@ -91,11 +91,16 @@ impl Run for Explain { .build(); formatter.run_and_print_resolve(solver).await?; } else { - // TODO: print the solve let solution = solver.solve().await?; - for item in solution.items() { - println!("{}", item.spec.ident()); - } + let output = solution + .format_solution_with_highest_versions( + self.verbose + 1, + solver.repositories(), + // the order coming out of resolvo is ... random? + true, + ) + .await?; + println!("{output}"); } Ok(0) From 98ad4044e8d546450716a33ec7f3dac50db98926 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Tue, 11 Feb 2025 11:06:49 -0800 Subject: [PATCH 44/64] Improve sorting to avoid embedded packages At SPI the newest python version available on centos is an embedded package, but it is undesirable for `spk env python` to choose an embedded python package. It also follows that it would have preferred to build from source a newer version of a package instead of use an existing build of an older version. Signed-off-by: J Robert Ray --- .../src/solvers/resolvo/spk_provider.rs | 50 +++++++++++++++---- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index b3cb63c301..d596988a9d 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -935,18 +935,48 @@ impl DependencyProvider for SpkProvider { ( SpkSolvable::LocatedBuildIdentWithComponent(a), SpkSolvable::LocatedBuildIdentWithComponent(b), - ) => match b.ident.version().cmp(a.ident.version()) { - std::cmp::Ordering::Equal => { - // Sort source builds last - match (a.ident.build(), b.ident.build()) { - (Build::Source, Build::Source) => std::cmp::Ordering::Equal, - (Build::Source, _) => std::cmp::Ordering::Greater, - (_, Build::Source) => std::cmp::Ordering::Less, - _ => std::cmp::Ordering::Equal, + ) => { + // Sort source packages last to prefer using any existing + // build of whatever version over building from source. + match (a.ident.build(), b.ident.build()) { + (Build::Source, Build::Source) => {} + (Build::Source, _) => return std::cmp::Ordering::Greater, + (_, Build::Source) => return std::cmp::Ordering::Less, + _ => {} + }; + // Sort embedded packages second last, even if an embedded + // package has the highest version. + match (a.ident.build(), b.ident.build()) { + (Build::Embedded(_), Build::Embedded(_)) => {} + (Build::Embedded(_), _) => return std::cmp::Ordering::Greater, + (_, Build::Embedded(_)) => return std::cmp::Ordering::Less, + _ => {} + }; + // Then prefer higher versions... + match b.ident.version().cmp(a.ident.version()) { + std::cmp::Ordering::Equal => { + // Sort source builds last + match (a.ident.build(), b.ident.build()) { + (Build::Source, Build::Source) => {} + (Build::Source, _) => return std::cmp::Ordering::Greater, + (_, Build::Source) => return std::cmp::Ordering::Less, + _ => {} + }; + // Sort embedded packges second last + match (a.ident.build(), b.ident.build()) { + (Build::Embedded(_), Build::Embedded(_)) => { + // TODO: Could perhaps sort on the parent + // package to prefer a newer parent. + std::cmp::Ordering::Equal + } + (Build::Embedded(_), _) => std::cmp::Ordering::Greater, + (_, Build::Embedded(_)) => std::cmp::Ordering::Less, + _ => std::cmp::Ordering::Equal, + } } + ord => ord, } - ord => ord, - }, + } ( SpkSolvable::GlobalVar { key: a_key, From a5c558a7a1891e9ba69330f94559fe6a548e16dc Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Tue, 11 Feb 2025 11:31:19 -0800 Subject: [PATCH 45/64] Avoid reading packages where possible Improve performance by not reading builds for packages that are for versions that are not applicable to the root requests. Signed-off-by: J Robert Ray --- crates/spk-solve/src/solvers/resolvo/mod.rs | 2 +- .../src/solvers/resolvo/spk_provider.rs | 44 ++++++++++++++++++- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index b3d822b2a2..6aac32323e 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -86,7 +86,7 @@ impl Solver { )); let (solver, solved) = loop { let mut this_iter_provider = provider.take().expect("provider is always Some"); - let pkg_requirements = this_iter_provider.pkg_requirements(&requests); + let pkg_requirements = this_iter_provider.root_pkg_requirements(&requests); let mut var_requirements = this_iter_provider.var_requirements(&requests); // XXX: Not sure if this will result in the desired precedence // when options and var requests for the same thing exist. diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index d596988a9d..b94e8816d0 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -119,12 +119,27 @@ impl ResolvoPackageName { ResolvoPackageName::PkgNameBufWithComponent(pkg_name) => { let mut located_builds = Vec::new(); + let root_pkg_request = provider.global_pkg_requests.get(&pkg_name.name); + for repo in &provider.repos { let versions = repo .list_package_versions(&pkg_name.name) .await .unwrap_or_default(); for version in versions.iter() { + // Skip versions known to be excluded to avoid the + // overhead of reading all the builds and their package + // specs. The tradeoff is that resolvo doesn't learn + // these builds exist to use them when constructing + // error messages. However from observation solver + // errors already don't mention versions that aren't + // applicable to root requests. + if let Some(pkg_request) = root_pkg_request { + if !pkg_request.pkg.version.is_applicable(version).is_ok() { + continue; + } + } + // TODO: We need a borrowing version of this to avoid cloning. let pkg_version = VersionIdent::new(pkg_name.name.clone(), (**version).clone()); @@ -325,6 +340,9 @@ enum CanBuildFromSource { pub(crate) struct SpkProvider { pub(crate) pool: Pool, repos: Vec>, + /// Global package requests. These can be used to constrain the candidates + /// returned for these packages. + global_pkg_requests: HashMap, /// Global options, like what might be specified with `--opt` to `spk env`. /// Indexed by name. If multiple requests happen to exist with the same /// name, the last one is kept. @@ -450,6 +468,7 @@ impl SpkProvider { Self { pool: Pool::new(), repos, + global_pkg_requests: Default::default(), global_var_requests: Default::default(), interned_solvables: Default::default(), known_global_var_values: Default::default(), @@ -510,7 +529,27 @@ impl SpkProvider { known_deps } - pub fn pkg_requirements(&self, requests: &[Request]) -> Vec { + /// Add any package requests found in the given requests to the global + /// package requests, returning a list of Requirement. + pub(crate) fn root_pkg_requirements(&mut self, requests: &[Request]) -> Vec { + self.global_pkg_requests.reserve(requests.len()); + requests + .iter() + .filter_map(|req| match req { + Request::Pkg(pkg) => Some(pkg), + _ => None, + }) + .flat_map(|req| { + self.global_pkg_requests + .insert(req.pkg.name().to_owned(), req.clone()); + self.pkg_request_to_known_dependencies(req).requirements + }) + .collect() + } + + /// Return a list of requirements for all the package requests found in the + /// given requests. + fn dep_pkg_requirements(&self, requests: &[Request]) -> Vec { requests .iter() .filter_map(|req| match req { @@ -604,6 +643,7 @@ impl SpkProvider { Self { pool: Pool::new(), repos: self.repos.clone(), + global_pkg_requests: self.global_pkg_requests.clone(), global_var_requests: self.global_var_requests.clone(), interned_solvables: Default::default(), known_global_var_values: RefCell::new(self.known_global_var_values.take()), @@ -1119,7 +1159,7 @@ impl DependencyProvider for SpkProvider { }); known_deps .requirements - .extend(self.pkg_requirements(&component_spec.requirements)); + .extend(self.dep_pkg_requirements(&component_spec.requirements)); } } // Also add dependencies on any packages embedded in this From 99f93dc5194d17997a43c475843440d8480189b5 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Tue, 11 Feb 2025 12:43:23 -0800 Subject: [PATCH 46/64] Add feedback for solver retries Log an info message for now, in order to gauge how many retries are happening in practice. Signed-off-by: J Robert Ray Signed-off-by: J Robert Ray --- crates/spk-solve/src/solvers/resolvo/mod.rs | 10 +++++++++- .../src/solvers/resolvo/spk_provider.rs | 20 ++++++++++++------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index 6aac32323e..bb36fbe18b 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -84,7 +84,9 @@ impl Solver { binary_only, build_from_source_trail, )); + let mut loop_counter = 0; let (solver, solved) = loop { + loop_counter += 1; let mut this_iter_provider = provider.take().expect("provider is always Some"); let pkg_requirements = this_iter_provider.root_pkg_requirements(&requests); let mut var_requirements = this_iter_provider.var_requirements(&requests); @@ -99,8 +101,13 @@ impl Solver { .constraints(var_requirements); match solver.solve(problem) { Ok(solved) => break (solver, solved), - Err(resolvo::UnsolvableOrCancelled::Cancelled(_)) => { + Err(resolvo::UnsolvableOrCancelled::Cancelled(msg)) => { + let msg = msg.downcast_ref::(); provider = Some(solver.provider().reset()); + tracing::info!( + "Solver retry {loop_counter}: {msg:?}", + msg = msg.map_or("unknown", |v| v) + ); continue; } Err(resolvo::UnsolvableOrCancelled::Unsolvable(conflict)) => { @@ -109,6 +116,7 @@ impl Solver { // needs to cancel (unknown if this ever happens). if solver.provider().is_canceled() { provider = Some(solver.provider().reset()); + tracing::info!("Solver retry {loop_counter}"); continue; } return Err(Error::FailedToResolve(format!( diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index b94e8816d0..0e7efff9f7 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -3,7 +3,7 @@ // https://github.com/spkenv/spk use std::borrow::Cow; -use std::cell::{Cell, RefCell}; +use std::cell::RefCell; use std::collections::{BTreeSet, HashMap, HashSet}; use std::str::FromStr; use std::sync::Arc; @@ -355,7 +355,7 @@ pub(crate) struct SpkProvider { /// queried, it is no longer possible to add more possible values without /// restarting the solve. queried_global_var_values: RefCell>, - cancel_solving: Cell, + cancel_solving: RefCell>, binary_only: bool, /// When recursively exploring building packages from source, track chain /// of packages to detect cycles. @@ -561,7 +561,7 @@ impl SpkProvider { } pub fn is_canceled(&self) -> bool { - self.cancel_solving.get() + self.cancel_solving.borrow().is_some() } fn request_to_known_dependencies(&self, requirement: &Request) -> KnownDependencies { @@ -604,7 +604,10 @@ impl SpkProvider { { // Seeing a new value for a var that has // already locked in the list of candidates. - self.cancel_solving.set(true); + *self.cancel_solving.borrow_mut() = Some(format!( + "Saw new value for global var: {}/{value}", + var_request.var.without_namespace() + )); } let dep_name = self.pool.intern_package_name(ResolvoPackageName::GlobalVar( @@ -1384,7 +1387,10 @@ impl DependencyProvider for SpkProvider { { // Seeing a new value for a var that has already locked // in the list of candidates. - self.cancel_solving.set(true); + *self.cancel_solving.borrow_mut() = Some(format!( + "Saw new value for global var: {}/{value}", + var_opt.var.without_namespace() + )); } let dep_name = self.pool.intern_package_name(ResolvoPackageName::GlobalVar( var_opt.var.without_namespace().to_owned(), @@ -1415,10 +1421,10 @@ impl DependencyProvider for SpkProvider { } fn should_cancel_with_value(&self) -> Option> { - if self.cancel_solving.get() { + if let Some(msg) = self.cancel_solving.borrow().as_ref() { // Eventually there will be more than one reason the solve is // cancelled... - Some(Box::new(())) + Some(Box::new(msg.clone())) } else { None } From 124319c1fcd2be71fb2ab8b19130cb2b1e782570 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Tue, 11 Feb 2025 14:05:36 -0800 Subject: [PATCH 47/64] Add a replacement for running solve through formatter The abstract solver now has a `run_and_print_resolve` that takes a formatter, which makes it possible to run either the old or new solver in a consistent way. The original `run_and_print_resolve` function has become `pub(crate)` to catch all the places that needed to be modified to use the new function. Signed-off-by: J Robert Ray Signed-off-by: J Robert Ray --- crates/spk-cli/cmd-env/src/cmd_env.rs | 11 ++------ crates/spk-cli/cmd-explain/src/cmd_explain.rs | 27 +++++-------------- crates/spk-cli/cmd-install/src/cmd_install.rs | 11 ++------ crates/spk-cli/cmd-render/src/cmd_render.rs | 11 ++------ crates/spk-cli/group1/src/cmd_bake.rs | 12 ++------- crates/spk-solve/src/io.rs | 2 +- crates/spk-solve/src/solver.rs | 13 ++++++++- crates/spk-solve/src/solvers/resolvo/mod.rs | 18 ++++++++++++- crates/spk-solve/src/solvers/step/solver.rs | 9 +++++-- 9 files changed, 51 insertions(+), 63 deletions(-) diff --git a/crates/spk-cli/cmd-env/src/cmd_env.rs b/crates/spk-cli/cmd-env/src/cmd_env.rs index f2b43997f7..9f12217ce6 100644 --- a/crates/spk-cli/cmd-env/src/cmd_env.rs +++ b/crates/spk-cli/cmd-env/src/cmd_env.rs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/spkenv/spk -use std::any::Any; use std::collections::HashSet; use std::ffi::OsString; @@ -88,14 +87,8 @@ impl Run for Env { solver.add_request(request) } - let solution = - if let Some(solver) = (&solver as &dyn Any).downcast_ref::() { - let formatter = self.formatter_settings.get_formatter(self.verbose)?; - let (solution, _) = formatter.run_and_print_resolve(solver).await?; - solution - } else { - solver.solve().await? - }; + let formatter = self.formatter_settings.get_formatter(self.verbose)?; + let solution = solver.run_and_print_resolve(&formatter).await?; let solution = build_required_packages(&solution).await?; diff --git a/crates/spk-cli/cmd-explain/src/cmd_explain.rs b/crates/spk-cli/cmd-explain/src/cmd_explain.rs index a848c9f65e..8aaf46590b 100644 --- a/crates/spk-cli/cmd-explain/src/cmd_explain.rs +++ b/crates/spk-cli/cmd-explain/src/cmd_explain.rs @@ -2,8 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/spkenv/spk -use std::any::Any; - use clap::Args; use miette::Result; use spk_cli_common::{CommandArgs, Run, flags}; @@ -83,25 +81,12 @@ impl Run for Explain { } // Always show the solution packages for the solve - if let Some(solver) = (&solver as &dyn Any).downcast_ref::() { - let formatter = self - .formatter_settings - .get_formatter_builder(self.verbose + 1)? - .with_solution(true) - .build(); - formatter.run_and_print_resolve(solver).await?; - } else { - let solution = solver.solve().await?; - let output = solution - .format_solution_with_highest_versions( - self.verbose + 1, - solver.repositories(), - // the order coming out of resolvo is ... random? - true, - ) - .await?; - println!("{output}"); - } + let formatter = self + .formatter_settings + .get_formatter_builder(self.verbose + 1)? + .with_solution(true) + .build(); + solver.run_and_print_resolve(&formatter).await?; Ok(0) } diff --git a/crates/spk-cli/cmd-install/src/cmd_install.rs b/crates/spk-cli/cmd-install/src/cmd_install.rs index f1c7170f16..e9ada0b2a2 100644 --- a/crates/spk-cli/cmd-install/src/cmd_install.rs +++ b/crates/spk-cli/cmd-install/src/cmd_install.rs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/spkenv/spk -use std::any::Any; use std::collections::HashSet; use std::io::Write; @@ -64,14 +63,8 @@ impl Run for Install { solver.add_request(request); } - let solution = - if let Some(solver) = (&solver as &dyn Any).downcast_ref::() { - let formatter = self.formatter_settings.get_formatter(self.verbose)?; - let (solution, _) = formatter.run_and_print_resolve(solver).await?; - solution - } else { - solver.solve().await? - }; + let formatter = self.formatter_settings.get_formatter(self.verbose)?; + let solution = solver.run_and_print_resolve(&formatter).await?; println!("The following packages will be installed:\n"); let requested: HashSet<_> = solver diff --git a/crates/spk-cli/cmd-render/src/cmd_render.rs b/crates/spk-cli/cmd-render/src/cmd_render.rs index 7bbb29494f..eb0e88df44 100644 --- a/crates/spk-cli/cmd-render/src/cmd_render.rs +++ b/crates/spk-cli/cmd-render/src/cmd_render.rs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/spkenv/spk -use std::any::Any; use std::path::PathBuf; use clap::Args; @@ -53,14 +52,8 @@ impl Run for Render { solver.add_request(name); } - let solution = - if let Some(solver) = (&solver as &dyn Any).downcast_ref::() { - let formatter = self.formatter_settings.get_formatter(self.verbose)?; - let (solution, _) = formatter.run_and_print_resolve(solver).await?; - solution - } else { - solver.solve().await? - }; + let formatter = self.formatter_settings.get_formatter(self.verbose)?; + let solution = solver.run_and_print_resolve(&formatter).await?; let solution = build_required_packages(&solution).await?; let stack = resolve_runtime_layers(true, &solution).await?; diff --git a/crates/spk-cli/group1/src/cmd_bake.rs b/crates/spk-cli/group1/src/cmd_bake.rs index 300cb37937..2b7ca54d41 100644 --- a/crates/spk-cli/group1/src/cmd_bake.rs +++ b/crates/spk-cli/group1/src/cmd_bake.rs @@ -2,8 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/spkenv/spk -use std::any::Any; - use clap::Args; use futures::TryFutureExt; use miette::IntoDiagnostic; @@ -240,14 +238,8 @@ impl Bake { solver.add_request(request) } - let solution = - if let Some(solver) = (&solver as &dyn Any).downcast_ref::() { - let formatter = self.formatter_settings.get_formatter(self.verbose)?; - let (solution, _) = formatter.run_and_print_resolve(solver).await?; - solution - } else { - solver.solve().await? - }; + let formatter = self.formatter_settings.get_formatter(self.verbose)?; + let solution = solver.run_and_print_resolve(&formatter).await?; // The solution order is the order things were found during // the solve. Need to reverse it to match up with the spfs diff --git a/crates/spk-solve/src/io.rs b/crates/spk-solve/src/io.rs index 1aebcd63a2..beb936083b 100644 --- a/crates/spk-solve/src/io.rs +++ b/crates/spk-solve/src/io.rs @@ -1161,7 +1161,7 @@ impl DecisionFormatter { /// appropriate. This runs two solvers in parallel (one based on /// the given solver, one with additional options) and takes the /// result from the first to finish. - pub async fn run_and_print_resolve( + pub(crate) async fn run_and_print_resolve( &self, solver: &StepSolver, ) -> Result<(Solution, Arc>)> { diff --git a/crates/spk-solve/src/solver.rs b/crates/spk-solve/src/solver.rs index 33af9913e7..6bf0e19a3c 100644 --- a/crates/spk-solve/src/solver.rs +++ b/crates/spk-solve/src/solver.rs @@ -12,7 +12,7 @@ use spk_solve_solution::Solution; use spk_storage::RepositoryHandle; use variantly::Variantly; -use crate::Result; +use crate::{DecisionFormatter, Result}; #[enum_dispatch(Solver)] #[derive(Variantly)] @@ -44,6 +44,17 @@ pub trait Solver: Any { /// Put this solver back into its default state fn reset(&mut self); + /// Run the solver as configured using the given formatter. + /// + /// "print" means that solver progress is printed to the console, as + /// configured by the formatter. + /// + /// The solution may also be printed, if found, as configured by the + /// formatter. + /// + /// Not all formatter settings may be supported by every solver. + async fn run_and_print_resolve(&mut self, formatter: &DecisionFormatter) -> Result; + /// If true, only solve pre-built binary packages. /// /// When false, the solver may return packages where the build is not set. diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index bb36fbe18b..6149c813d5 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -39,7 +39,7 @@ use spk_solve_validation::{Validators, default_validators}; use spk_storage::RepositoryHandle; use crate::solver::Solver as SolverTrait; -use crate::{Error, Result}; +use crate::{DecisionFormatter, Error, Result}; #[cfg(test)] #[path = "resolvo_tests.rs"] @@ -307,6 +307,22 @@ impl SolverTrait for Solver { self._validators = Cow::from(default_validators()); } + async fn run_and_print_resolve(&mut self, formatter: &DecisionFormatter) -> Result { + let solution = self.solve().await?; + let output = solution + .format_solution_with_highest_versions( + formatter.settings.verbosity, + self.repositories(), + // the order coming out of resolvo is ... random? + true, + ) + .await?; + if formatter.settings.show_solution { + println!("{output}"); + } + Ok(solution) + } + fn set_binary_only(&mut self, binary_only: bool) { self.binary_only = binary_only; } diff --git a/crates/spk-solve/src/solvers/step/solver.rs b/crates/spk-solve/src/solvers/step/solver.rs index a70f2bb1b1..5212857ddc 100644 --- a/crates/spk-solve/src/solvers/step/solver.rs +++ b/crates/spk-solve/src/solvers/step/solver.rs @@ -53,10 +53,10 @@ use spk_solve_validation::{ }; use spk_storage::RepositoryHandle; -use crate::error::OutOfOptions; +use crate::error::{self, OutOfOptions}; use crate::option_map::OptionMap; use crate::solver::Solver as SolverTrait; -use crate::{Error, Result, error}; +use crate::{DecisionFormatter, Error, Result}; /// Structure to hold whether the three kinds of impossible checks are /// enabled or disabled in a solver. @@ -1165,6 +1165,11 @@ impl SolverTrait for Solver { self.problem_packages.clear(); } + async fn run_and_print_resolve(&mut self, formatter: &DecisionFormatter) -> Result { + let (solution, _graph) = formatter.run_and_print_resolve(self).await?; + Ok(solution) + } + fn set_binary_only(&mut self, binary_only: bool) { self.request_validator.set_binary_only(binary_only); From 0f8b663cce1cd403fede348e5fd3dc37b62f092a Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Tue, 11 Feb 2025 15:10:27 -0800 Subject: [PATCH 48/64] Also add run_and_log_resolve to the Solver trait Making the original function `pub(crate)` to force code to change to use the trait instead. Remove the unused `run_and_log_decisions` function. Signed-off-by: J Robert Ray Signed-off-by: J Robert Ray --- crates/spk-exec/src/exec_test.rs | 2 +- crates/spk-solve/src/io.rs | 23 +-------------------- crates/spk-solve/src/solver.rs | 11 ++++++++++ crates/spk-solve/src/solvers/resolvo/mod.rs | 5 +++++ crates/spk-solve/src/solvers/step/solver.rs | 5 +++++ 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/crates/spk-exec/src/exec_test.rs b/crates/spk-exec/src/exec_test.rs index 2444a5d724..2033cac487 100644 --- a/crates/spk-exec/src/exec_test.rs +++ b/crates/spk-exec/src/exec_test.rs @@ -66,7 +66,7 @@ build: solver.add_request(request!("one")); solver.add_request(request!("two")); - let (solution, _) = formatter.run_and_log_resolve(&solver).await.unwrap(); + let solution = solver.run_and_log_resolve(&formatter).await.unwrap(); let resolved_layers = solution_to_resolved_runtime_layers(&solution).unwrap(); diff --git a/crates/spk-solve/src/io.rs b/crates/spk-solve/src/io.rs index beb936083b..cb201f47da 100644 --- a/crates/spk-solve/src/io.rs +++ b/crates/spk-solve/src/io.rs @@ -1194,7 +1194,7 @@ impl DecisionFormatter { /// info-level event as appropriate. This runs two solvers in /// parallel (one based on the given solver, one with additional /// options) and takes the result from the first to finish. - pub async fn run_and_log_resolve( + pub(crate) async fn run_and_log_resolve( &self, solver: &StepSolver, ) -> Result<(Solution, Arc>)> { @@ -1202,27 +1202,6 @@ impl DecisionFormatter { self.run_multi_solve(solvers, OutputKind::Tracing).await } - /// Run the solver runtime to completion, logging each step as a - /// tracing info-level event as appropriate. This does not run - /// multiple solver and won't benefit from running solvers in - /// parallel. - pub async fn run_and_log_decisions( - &self, - runtime: &mut StepSolverRuntime, - ) -> Result<(Solution, Arc>)> { - // Note: this is not currently used directly. We may be able - // to remove this method. - let start = Instant::now(); - let loop_outcome = self.run_solver_loop(runtime, OutputKind::Tracing).await; - let solve_time = start.elapsed(); - - #[cfg(feature = "statsd")] - self.send_solver_end_metrics(solve_time); - - self.check_and_output_solver_results(loop_outcome, solve_time, runtime, OutputKind::Tracing) - .await - } - fn setup_solvers(&self, base_solver: &StepSolver) -> Vec { // Leave the first solver as is. let solver_with_no_change = base_solver.clone(); diff --git a/crates/spk-solve/src/solver.rs b/crates/spk-solve/src/solver.rs index 6bf0e19a3c..cc749f1888 100644 --- a/crates/spk-solve/src/solver.rs +++ b/crates/spk-solve/src/solver.rs @@ -44,6 +44,17 @@ pub trait Solver: Any { /// Put this solver back into its default state fn reset(&mut self); + /// Run the solver as configured using the given formatter. + /// + /// "log" means that solver progress is output via tracing, as + /// configured by the formatter. + /// + /// The solution may also be printed, if found, as configured by the + /// formatter. + /// + /// Not all formatter settings may be supported by every solver. + async fn run_and_log_resolve(&mut self, formatter: &DecisionFormatter) -> Result; + /// Run the solver as configured using the given formatter. /// /// "print" means that solver progress is printed to the console, as diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index 6149c813d5..8ea59761c2 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -307,6 +307,11 @@ impl SolverTrait for Solver { self._validators = Cow::from(default_validators()); } + async fn run_and_log_resolve(&mut self, formatter: &DecisionFormatter) -> Result { + // This solver doesn't currently support tracing. + self.run_and_print_resolve(formatter).await + } + async fn run_and_print_resolve(&mut self, formatter: &DecisionFormatter) -> Result { let solution = self.solve().await?; let output = solution diff --git a/crates/spk-solve/src/solvers/step/solver.rs b/crates/spk-solve/src/solvers/step/solver.rs index 5212857ddc..4e12a069b1 100644 --- a/crates/spk-solve/src/solvers/step/solver.rs +++ b/crates/spk-solve/src/solvers/step/solver.rs @@ -1165,6 +1165,11 @@ impl SolverTrait for Solver { self.problem_packages.clear(); } + async fn run_and_log_resolve(&mut self, formatter: &DecisionFormatter) -> Result { + let (solution, _graph) = formatter.run_and_log_resolve(self).await?; + Ok(solution) + } + async fn run_and_print_resolve(&mut self, formatter: &DecisionFormatter) -> Result { let (solution, _graph) = formatter.run_and_print_resolve(self).await?; Ok(solution) From f2c6c2dec2e56d26f77a053b29bd61ac1d4d43c1 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Tue, 11 Feb 2025 15:39:29 -0800 Subject: [PATCH 49/64] Rework how BinaryPackageBuilder invokes the solver The Solver trait was split into multiple traits with some blanket implementations for refs while chasing compile errors but this may not have actually been necessary. Add the "solver" flags to more commands so the user can choose what solver to use in more cases. Working towards being able to choose between the original solver and the new solver for all operations. More tests now run the test with both solves. Signed-off-by: J Robert Ray Signed-off-by: J Robert Ray --- Cargo.lock | 1 + crates/spk-build/src/archive_test.rs | 15 +- crates/spk-build/src/build/binary.rs | 98 ++++++------ crates/spk-build/src/build/binary_test.rs | 139 ++++++++++++------ crates/spk-cli/cmd-build/src/cmd_build.rs | 4 +- crates/spk-cli/cmd-du/Cargo.toml | 1 + crates/spk-cli/cmd-du/src/cmd_du_test.rs | 81 +++++++--- crates/spk-cli/cmd-env/src/cmd_env.rs | 4 +- crates/spk-cli/cmd-explain/src/cmd_explain.rs | 2 +- crates/spk-cli/cmd-install/src/cmd_install.rs | 4 +- .../cmd-make-binary/src/cmd_make_binary.rs | 17 ++- crates/spk-cli/cmd-render/src/cmd_render.rs | 4 +- crates/spk-cli/cmd-test/src/cmd_test.rs | 17 +-- crates/spk-cli/cmd-test/src/test/build.rs | 54 ++----- crates/spk-cli/cmd-test/src/test/install.rs | 35 ++--- crates/spk-cli/cmd-test/src/test/sources.rs | 35 ++--- crates/spk-cli/common/src/exec.rs | 18 ++- crates/spk-cli/common/src/flags.rs | 8 +- crates/spk-cli/group1/src/cmd_bake.rs | 2 +- crates/spk-cli/group3/src/cmd_export_test.rs | 26 +++- crates/spk-cli/group3/src/cmd_import_test.rs | 15 +- crates/spk-cli/group4/src/cmd_view.rs | 2 +- crates/spk-exec/src/exec_test.rs | 2 +- crates/spk-solve/src/io.rs | 37 +---- crates/spk-solve/src/lib.rs | 57 +------ crates/spk-solve/src/solver.rs | 113 ++++++++++++-- crates/spk-solve/src/solvers/resolvo/mod.rs | 31 ++-- .../src/solvers/resolvo/resolvo_tests.rs | 2 +- .../src/solvers/resolvo/spk_provider.rs | 2 +- crates/spk-solve/src/solvers/solver_test.rs | 2 +- crates/spk-solve/src/solvers/step/solver.rs | 57 +++---- 31 files changed, 500 insertions(+), 385 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 64d538e3b9..a459d81d1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4183,6 +4183,7 @@ dependencies = [ "futures", "itertools 0.14.0", "miette", + "rstest", "spfs", "spk-build", "spk-cli-common", diff --git a/crates/spk-build/src/archive_test.rs b/crates/spk-build/src/archive_test.rs index 2ed93eebbf..134f0cf9f4 100644 --- a/crates/spk-build/src/archive_test.rs +++ b/crates/spk-build/src/archive_test.rs @@ -5,14 +5,25 @@ use rstest::rstest; use spk_schema::foundation::option_map; use spk_schema::{Package, recipe}; +use spk_solve::SolverImpl; use spk_storage::export_package; use spk_storage::fixtures::*; use crate::{BinaryPackageBuilder, BuildSource}; +fn step_solver() -> SolverImpl { + SolverImpl::Step(spk_solve::StepSolver::default()) +} + +fn resolvo_solver() -> SolverImpl { + SolverImpl::Resolvo(spk_solve::ResolvoSolver::default()) +} + #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_archive_create_parents() { +async fn test_archive_create_parents(#[case] solver: SolverImpl) { let rt = spfs_runtime().await; let spec = recipe!( { @@ -21,7 +32,7 @@ async fn test_archive_create_parents() { } ); rt.tmprepo.publish_recipe(&spec).await.unwrap(); - let (spec, _) = BinaryPackageBuilder::from_recipe(spec) + let (spec, _) = BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_source(BuildSource::LocalPath(".".into())) .build_and_publish(option_map! {}, &*rt.tmprepo) .await diff --git a/crates/spk-build/src/build/binary.rs b/crates/spk-build/src/build/binary.rs index 8ed12490dc..0d96414803 100644 --- a/crates/spk-build/src/build/binary.rs +++ b/crates/spk-build/src/build/binary.rs @@ -37,7 +37,7 @@ use spk_schema::{ }; use spk_solve::graph::Graph; use spk_solve::solution::Solution; -use spk_solve::{BoxedCdclResolverCallback, Named, ResolverCallback, ResolvoSolver, Solver}; +use spk_solve::{DecisionFormatter, Named, SolverExt, SolverMut}; use spk_storage as storage; use crate::report::{BuildOutputReport, BuildReport, BuildSetupReport}; @@ -117,8 +117,9 @@ where /// /// ```no_run /// # use spk_schema::{recipe, foundation::option_map}; +/// # use spk_solve::StepSolver; /// # async fn demo() { -/// spk_build::BinaryPackageBuilder::from_recipe(recipe!({ +/// spk_build::BinaryPackageBuilder::<_, StepSolver>::from_recipe(recipe!({ /// "pkg": "my-pkg", /// "build": {"script": "echo hello, world"}, /// })) @@ -127,14 +128,14 @@ where /// .unwrap(); /// # } /// ``` -pub struct BinaryPackageBuilder<'a, Recipe> { +pub struct BinaryPackageBuilder { prefix: PathBuf, recipe: Recipe, source: BuildSource, - solver: ResolvoSolver, + solver: Solver, environment: HashMap, - source_resolver: BoxedCdclResolverCallback<'a>, - build_resolver: BoxedCdclResolverCallback<'a>, + source_solve_formatter: DecisionFormatter, + build_solve_formatter: DecisionFormatter, last_solve_graph: Arc>, repos: Vec>, interactive: bool, @@ -142,29 +143,30 @@ pub struct BinaryPackageBuilder<'a, Recipe> { allow_circular_dependencies: bool, } -impl<'a, Recipe> BinaryPackageBuilder<'a, Recipe> +impl BinaryPackageBuilder where Recipe: spk_schema::Recipe, - Recipe::Output: Package + serde::Serialize, { - /// Create a new builder that builds a binary package from the given recipe - pub fn from_recipe(recipe: Recipe) -> Self { + /// Create a new builder that builds a binary package from the given recipe. + /// + /// Use the provided solver. + pub fn from_recipe_with_solver(recipe: Recipe, solver: Solver) -> Self { let source = BuildSource::SourcePackage(recipe.ident().to_build_ident(Build::Source).into()); Self { recipe, source, prefix: PathBuf::from("/spfs"), - solver: ResolvoSolver::default(), + solver, environment: Default::default(), - //#[cfg(test)] - //source_resolver: Box::new(spk_solve::DecisionFormatter::new_testing()), - //#[cfg(not(test))] - source_resolver: Box::new(spk_solve::DefaultCdclResolver {}), - //#[cfg(test)] - //build_resolver: Box::new(spk_solve::DecisionFormatter::new_testing()), - //#[cfg(not(test))] - build_resolver: Box::new(spk_solve::DefaultCdclResolver {}), + #[cfg(test)] + source_solve_formatter: DecisionFormatter::new_testing(), + #[cfg(not(test))] + source_solve_formatter: DecisionFormatter::default(), + #[cfg(test)] + build_solve_formatter: DecisionFormatter::new_testing(), + #[cfg(not(test))] + build_solve_formatter: DecisionFormatter::default(), last_solve_graph: Arc::new(tokio::sync::RwLock::new(Graph::new())), repos: Default::default(), interactive: false, @@ -172,7 +174,27 @@ where allow_circular_dependencies: false, } } +} +impl BinaryPackageBuilder +where + Recipe: spk_schema::Recipe, + Solver: Default, +{ + /// Create a new builder that builds a binary package from the given recipe. + /// + /// This will use a default instance of the generic solver type. + pub fn from_recipe(recipe: Recipe) -> Self { + Self::from_recipe_with_solver(recipe, Solver::default()) + } +} + +impl BinaryPackageBuilder +where + Recipe: spk_schema::Recipe, + Recipe::Output: Package + serde::Serialize, + Solver: SolverExt + SolverMut, +{ /// Allow circular dependencies when resolving dependencies. /// /// Normally if a build dependency has a dependency on the package being @@ -214,31 +236,15 @@ where self } - /// Provide a function that will be called when resolving the source package. - /// - /// This function should run the provided solver runtime to - /// completion, returning the final result. This function - /// is useful for introspecting and reporting on the solve - /// process as needed. - pub fn with_source_resolver(&mut self, resolver: F) -> &mut Self - where - F: ResolverCallback + 'a, - { - self.source_resolver = Box::new(resolver); + /// Provide a formatter to use when resolving the source environment. + pub fn with_source_formatter(&mut self, formatter: DecisionFormatter) -> &mut Self { + self.source_solve_formatter = formatter; self } - /// Provide a function that will be called when resolving the build environment. - /// - /// This function should run the provided solver runtime to - /// completion, returning the final result. This function - /// is useful for introspecting and reporting on the solve - /// process as needed. - pub fn with_build_resolver(&mut self, resolver: F) -> &mut Self - where - F: ResolverCallback + 'a, - { - self.build_resolver = Box::new(resolver); + /// Provide a formatter to use when resolving the build environment. + pub fn with_build_formatter(&mut self, formatter: DecisionFormatter) -> &mut Self { + self.build_solve_formatter = formatter; self } @@ -441,7 +447,10 @@ where self.solver.add_request(request.into()); - let solution = self.source_resolver.solve(&self.solver).await?; + let solution = self + .solver + .run_and_print_resolve(&self.source_solve_formatter) + .await?; Ok(solution) } @@ -465,7 +474,10 @@ where self.solver.add_request(request.clone()); } - let solution = self.build_resolver.solve(&self.solver).await?; + let solution = self + .solver + .run_and_print_resolve(&self.build_solve_formatter) + .await?; Ok(solution) } diff --git a/crates/spk-build/src/build/binary_test.rs b/crates/spk-build/src/build/binary_test.rs index 8ee4583db4..ce0a5ec9b6 100644 --- a/crates/spk-build/src/build/binary_test.rs +++ b/crates/spk-build/src/build/binary_test.rs @@ -13,7 +13,7 @@ use spk_schema::foundation::ident_component::Component; use spk_schema::foundation::{opt_name, option_map}; use spk_schema::ident::{PkgRequest, RangeIdent, Request}; use spk_schema::{ComponentSpecList, FromYaml, OptionMap, Package, Recipe, SpecRecipe, recipe}; -use spk_solve::Solution; +use spk_solve::{Solution, SolverImpl}; use spk_storage::fixtures::*; use spk_storage::{self as storage, Repository}; @@ -97,9 +97,19 @@ fn test_var_with_build_assigns_build() { if name.as_str() == "my-dep" && digest.digest() == "QYB6QLCN")); } +fn step_solver() -> SolverImpl { + SolverImpl::Step(spk_solve::StepSolver::default()) +} + +fn resolvo_solver() -> SolverImpl { + SolverImpl::Resolvo(spk_solve::ResolvoSolver::default()) +} + #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_build_workdir(tmpdir: tempfile::TempDir) { +async fn test_build_workdir(tmpdir: tempfile::TempDir, #[case] solver: SolverImpl) { let rt = spfs_runtime().await; let out_file = tmpdir.path().join("out.log"); let recipe = recipe!({ @@ -113,7 +123,7 @@ async fn test_build_workdir(tmpdir: tempfile::TempDir) { }); rt.tmprepo.publish_recipe(&recipe).await.unwrap(); - BinaryPackageBuilder::from_recipe(recipe) + BinaryPackageBuilder::from_recipe_with_solver(recipe, solver) .with_source(BuildSource::LocalPath(tmpdir.path().to_owned())) .build_and_publish(&option_map! {}, &*rt.tmprepo) .await @@ -129,8 +139,10 @@ async fn test_build_workdir(tmpdir: tempfile::TempDir) { } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_build_package_options() { +async fn test_build_package_options(#[case] solver: SolverImpl) { let rt = spfs_runtime().await; let dep_spec = recipe!( {"pkg": "dep/1.0.0", "build": {"script": "touch /spfs/dep-file"}} @@ -161,7 +173,7 @@ async fn test_build_package_options() { rt.tmprepo.publish_recipe(&dep_spec).await.unwrap(); - BinaryPackageBuilder::from_recipe(dep_spec) + BinaryPackageBuilder::from_recipe_with_solver(dep_spec, solver.clone()) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(&option_map! {}, &*rt.tmprepo) @@ -175,7 +187,7 @@ async fn test_build_package_options() { // specific option takes precedence "top.dep" => "1.0.0", }; - let (spec, _) = BinaryPackageBuilder::from_recipe(spec) + let (spec, _) = BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(variant, &*rt.tmprepo) @@ -198,7 +210,10 @@ async fn test_build_package_options() { #[case::camel_case("fromBuildEnv")] #[case::lower_case("frombuildenv")] #[tokio::test] -async fn test_build_package_pinning(#[case] from_build_env_str: &str) { +async fn test_build_package_pinning( + #[case] from_build_env_str: &str, + #[values(step_solver(), resolvo_solver())] solver: SolverImpl, +) { let rt = spfs_runtime().await; let dep_spec = recipe!( {"pkg": "dep/1.0.0", "build": {"script": "touch /spfs/dep-file"}} @@ -217,14 +232,14 @@ async fn test_build_package_pinning(#[case] from_build_env_str: &str) { ); rt.tmprepo.publish_recipe(&dep_spec).await.unwrap(); - BinaryPackageBuilder::from_recipe(dep_spec) + BinaryPackageBuilder::from_recipe_with_solver(dep_spec, solver.clone()) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(option_map! {}, &*rt.tmprepo) .await .unwrap(); rt.tmprepo.publish_recipe(&spec).await.unwrap(); - let (spec, _) = BinaryPackageBuilder::from_recipe(spec) + let (spec, _) = BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(option_map! {}, &*rt.tmprepo) @@ -242,8 +257,10 @@ async fn test_build_package_pinning(#[case] from_build_env_str: &str) { } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_build_package_pinning_optional_requirement() { +async fn test_build_package_pinning_optional_requirement(#[case] solver: SolverImpl) { let rt = spfs_runtime().await; let dep1_spec = recipe!( {"pkg": "dep1/1.0.0", "build": {"script": "touch /spfs/dep-file"}} @@ -273,7 +290,7 @@ async fn test_build_package_pinning_optional_requirement() { for dep_spec in [dep1_spec, dep2_spec] { rt.tmprepo.publish_recipe(&dep_spec).await.unwrap(); - BinaryPackageBuilder::from_recipe(dep_spec) + BinaryPackageBuilder::from_recipe_with_solver(dep_spec, solver.clone()) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(option_map! {}, &*rt.tmprepo) @@ -285,7 +302,7 @@ async fn test_build_package_pinning_optional_requirement() { let default_variants = spec.default_variants(&OptionMap::default()); for (variant, expected_dep) in default_variants.iter().zip(["dep1", "dep2"].iter()) { - let (spec, _) = BinaryPackageBuilder::from_recipe(spec.clone()) + let (spec, _) = BinaryPackageBuilder::from_recipe_with_solver(spec.clone(), solver.clone()) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(variant, &*rt.tmprepo) @@ -304,8 +321,12 @@ async fn test_build_package_pinning_optional_requirement() { } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_build_package_pinning_optional_requirement_without_frombuildenv() { +async fn test_build_package_pinning_optional_requirement_without_frombuildenv( + #[case] solver: SolverImpl, +) { let rt = spfs_runtime().await; let dep1_spec = recipe!( {"pkg": "dep1/1.0.0", "build": {"script": "touch /spfs/dep-file"}} @@ -335,7 +356,7 @@ async fn test_build_package_pinning_optional_requirement_without_frombuildenv() for dep_spec in [dep1_spec, dep2_spec] { rt.tmprepo.publish_recipe(&dep_spec).await.unwrap(); - BinaryPackageBuilder::from_recipe(dep_spec) + BinaryPackageBuilder::from_recipe_with_solver(dep_spec, solver.clone()) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(option_map! {}, &*rt.tmprepo) @@ -347,7 +368,7 @@ async fn test_build_package_pinning_optional_requirement_without_frombuildenv() let default_variants = spec.default_variants(&OptionMap::default()); for (variant, expected_dep) in default_variants.iter().zip(["dep1", "dep2"].iter()) { - let (spec, _) = BinaryPackageBuilder::from_recipe(spec.clone()) + let (spec, _) = BinaryPackageBuilder::from_recipe_with_solver(spec.clone(), solver.clone()) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(variant, &*rt.tmprepo) @@ -366,8 +387,10 @@ async fn test_build_package_pinning_optional_requirement_without_frombuildenv() } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_build_var_pinning_optional_requirement() { +async fn test_build_var_pinning_optional_requirement(#[case] solver: SolverImpl) { let rt = spfs_runtime().await; let dep2_spec = recipe!( {"pkg": "dep2/1.0.0", "build": { @@ -397,7 +420,7 @@ async fn test_build_var_pinning_optional_requirement() { for dep_spec in [dep2_spec] { rt.tmprepo.publish_recipe(&dep_spec).await.unwrap(); - BinaryPackageBuilder::from_recipe(dep_spec) + BinaryPackageBuilder::from_recipe_with_solver(dep_spec, solver.clone()) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(option_map! {}, &*rt.tmprepo) @@ -417,7 +440,7 @@ async fn test_build_var_pinning_optional_requirement() { ] .into_iter(), ) { - let (spec, _) = BinaryPackageBuilder::from_recipe(spec.clone()) + let (spec, _) = BinaryPackageBuilder::from_recipe_with_solver(spec.clone(), solver.clone()) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(variant, &*rt.tmprepo) @@ -435,8 +458,10 @@ async fn test_build_var_pinning_optional_requirement() { } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_build_package_missing_deps() { +async fn test_build_package_missing_deps(#[case] solver: SolverImpl) { let rt = spfs_runtime().await; let spec = recipe!( { @@ -449,7 +474,7 @@ async fn test_build_package_missing_deps() { // should not fail to resolve build env and build even though // runtime dependency is missing in the current repos - BinaryPackageBuilder::from_recipe(spec) + BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(option_map! {}, &*rt.tmprepo) @@ -458,8 +483,10 @@ async fn test_build_package_missing_deps() { } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_build_var_pinning() { +async fn test_build_var_pinning(#[case] solver: SolverImpl) { let rt = spfs_runtime().await; let dep_spec = recipe!( { @@ -493,13 +520,13 @@ async fn test_build_var_pinning() { rt.tmprepo.publish_recipe(&dep_spec).await.unwrap(); rt.tmprepo.publish_recipe(&spec).await.unwrap(); - BinaryPackageBuilder::from_recipe(dep_spec) + BinaryPackageBuilder::from_recipe_with_solver(dep_spec, solver.clone()) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(option_map! {}, &*rt.tmprepo) .await .unwrap(); - let (spec, _) = BinaryPackageBuilder::from_recipe(spec) + let (spec, _) = BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(option_map! {}, &*rt.tmprepo) @@ -520,8 +547,10 @@ async fn test_build_var_pinning() { } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_build_bad_options() { +async fn test_build_bad_options(#[case] solver: SolverImpl) { let rt = spfs_runtime().await; let spec = recipe!( { @@ -536,7 +565,7 @@ async fn test_build_bad_options() { ); rt.tmprepo.publish_recipe(&spec).await.unwrap(); - let res = BinaryPackageBuilder::from_recipe(spec) + let res = BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_source(BuildSource::LocalPath(".".into())) .build_and_publish(option_map! {"debug" => "false"}, &*rt.tmprepo) .await; @@ -551,8 +580,10 @@ async fn test_build_bad_options() { } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_build_package_source_cleanup() { +async fn test_build_package_source_cleanup(#[case] solver: SolverImpl) { let rt = spfs_runtime().await; let spec = recipe!( { @@ -580,7 +611,7 @@ async fn test_build_package_source_cleanup() { .await .unwrap(); - let (pkg, _) = BinaryPackageBuilder::from_recipe(spec) + let (pkg, _) = BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_repository(rt.tmprepo.clone()) .build_and_publish(option_map! {}, &*rt.tmprepo) .await @@ -616,8 +647,10 @@ async fn test_build_package_source_cleanup() { } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_build_filters_reset_files() { +async fn test_build_filters_reset_files(#[case] solver: SolverImpl) { let rt = spfs_runtime().await; // Create a package that can be used as a dependency... @@ -642,7 +675,7 @@ async fn test_build_filters_reset_files() { .await .unwrap(); - let _ = BinaryPackageBuilder::from_recipe(spec) + let _ = BinaryPackageBuilder::from_recipe_with_solver(spec, solver.clone()) .with_repository(rt.tmprepo.clone()) .build_and_publish(option_map! {}, &*rt.tmprepo) .await @@ -676,7 +709,7 @@ async fn test_build_filters_reset_files() { .await .unwrap(); - let (pkg, _) = BinaryPackageBuilder::from_recipe(spec) + let (pkg, _) = BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_repository(rt.tmprepo.clone()) .build_and_publish(option_map! {}, &*rt.tmprepo) .await @@ -751,8 +784,10 @@ async fn test_default_build_component() { } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_build_components_metadata() { +async fn test_build_components_metadata(#[case] solver: SolverImpl) { let mut rt = spfs_runtime().await; let spec = recipe!( { @@ -767,7 +802,7 @@ async fn test_build_components_metadata() { } ); rt.tmprepo.publish_recipe(&spec).await.unwrap(); - let (spec, _) = BinaryPackageBuilder::from_recipe(spec.clone()) + let (spec, _) = BinaryPackageBuilder::from_recipe_with_solver(spec.clone(), solver) .with_source(BuildSource::LocalPath(".".into())) .build_and_publish(option_map! {}, &*rt.tmprepo) .await @@ -795,8 +830,10 @@ async fn test_build_components_metadata() { } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_build_add_startup_files(tmpdir: tempfile::TempDir) { +async fn test_build_add_startup_files(tmpdir: tempfile::TempDir, #[case] solver: SolverImpl) { let rt = spfs_runtime().await; let recipe = recipe!( { @@ -815,7 +852,7 @@ async fn test_build_add_startup_files(tmpdir: tempfile::TempDir) { let spec = recipe .generate_binary_build(&option_map! {}, &Solution::default()) .unwrap(); - BinaryPackageBuilder::from_recipe(recipe) + BinaryPackageBuilder::from_recipe_with_solver(recipe, solver) .with_prefix(tmpdir.path().into()) .generate_startup_scripts(&spec) .unwrap(); @@ -866,8 +903,10 @@ async fn test_build_multiple_priority_startup_files() { } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_build_priority_startup_files(tmpdir: tempfile::TempDir) { +async fn test_build_priority_startup_files(tmpdir: tempfile::TempDir, #[case] solver: SolverImpl) { let rt = spfs_runtime().await; let recipe = recipe!( { @@ -884,7 +923,7 @@ async fn test_build_priority_startup_files(tmpdir: tempfile::TempDir) { let spec = recipe .generate_binary_build(&option_map! {}, &Solution::default()) .unwrap(); - BinaryPackageBuilder::from_recipe(recipe) + BinaryPackageBuilder::from_recipe_with_solver(recipe, solver) .with_prefix(tmpdir.path().into()) .generate_startup_scripts(&spec) .unwrap(); @@ -896,8 +935,13 @@ async fn test_build_priority_startup_files(tmpdir: tempfile::TempDir) { } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_variable_substitution_in_build_env(tmpdir: tempfile::TempDir) { +async fn test_variable_substitution_in_build_env( + tmpdir: tempfile::TempDir, + #[case] solver: SolverImpl, +) { let rt = spfs_runtime().await; let dep_spec = recipe!( { @@ -931,14 +975,14 @@ async fn test_variable_substitution_in_build_env(tmpdir: tempfile::TempDir) { rt.tmprepo.publish_recipe(&dep_spec).await.unwrap(); rt.tmprepo.publish_recipe(&spec).await.unwrap(); - BinaryPackageBuilder::from_recipe(dep_spec) + BinaryPackageBuilder::from_recipe_with_solver(dep_spec, solver.clone()) .with_source(BuildSource::LocalPath(tmpdir.path().to_owned())) .with_repository(rt.tmprepo.clone()) .build_and_publish(option_map! {}, &*rt.tmprepo) .await .unwrap(); - BinaryPackageBuilder::from_recipe(spec) + BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_source(BuildSource::LocalPath(tmpdir.path().to_owned())) .with_repository(rt.tmprepo.clone()) .build_and_publish(option_map! {}, &*rt.tmprepo) @@ -994,9 +1038,14 @@ async fn test_variable_substitution_in_build_env(tmpdir: tempfile::TempDir) { } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] #[serial_test::serial(env)] // env manipulation must be reliable -async fn test_dependant_variable_substitution_in_startup_files(tmpdir: tempfile::TempDir) { +async fn test_dependant_variable_substitution_in_startup_files( + tmpdir: tempfile::TempDir, + #[case] solver: SolverImpl, +) { let rt = spfs_runtime().await; // Safety: this is unsafe. serial_test is used to prevent multiple tests @@ -1021,7 +1070,7 @@ async fn test_dependant_variable_substitution_in_startup_files(tmpdir: tempfile: let spec = recipe .generate_binary_build(&option_map! {}, &Solution::default()) .unwrap(); - BinaryPackageBuilder::from_recipe(recipe) + BinaryPackageBuilder::from_recipe_with_solver(recipe, solver) .with_prefix(tmpdir.path().into()) .generate_startup_scripts(&spec) .unwrap(); @@ -1066,8 +1115,10 @@ fn test_path_and_parents() { } #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_build_options_respect_components() { +async fn test_build_options_respect_components(#[case] solver: SolverImpl) { let rt = spfs_runtime().await; // Create a base package that has a couple components with unique // contents. @@ -1123,7 +1174,7 @@ async fn test_build_options_respect_components() { .build_and_publish(".", &*rt.tmprepo) .await .unwrap(); - let _base_pkg = BinaryPackageBuilder::from_recipe(base_spec) + let _base_pkg = BinaryPackageBuilder::from_recipe_with_solver(base_spec, solver.clone()) .with_repository(rt.tmprepo.clone()) .build_and_publish(option_map! {}, &*rt.tmprepo) .await @@ -1134,7 +1185,7 @@ async fn test_build_options_respect_components() { .await .unwrap(); - let r = BinaryPackageBuilder::from_recipe(top_spec) + let r = BinaryPackageBuilder::from_recipe_with_solver(top_spec, solver) .with_repository(rt.tmprepo.clone()) .build_and_publish(option_map! {}, &*rt.tmprepo) .await; diff --git a/crates/spk-cli/cmd-build/src/cmd_build.rs b/crates/spk-cli/cmd-build/src/cmd_build.rs index 1e5afd5d7b..652d34baca 100644 --- a/crates/spk-cli/cmd-build/src/cmd_build.rs +++ b/crates/spk-cli/cmd-build/src/cmd_build.rs @@ -18,7 +18,7 @@ pub struct Build { #[clap(flatten)] runtime: flags::Runtime, #[clap(flatten)] - repos: flags::Repositories, + solver: flags::Solver, #[clap(flatten)] options: flags::Options, @@ -107,8 +107,8 @@ impl Run for Build { let mut make_binary = spk_cmd_make_binary::cmd_make_binary::MakeBinary { verbose: self.verbose, runtime: self.runtime.clone(), - repos: self.repos.clone(), options: self.options.clone(), + solver: self.solver.clone(), here: self.here, interactive: self.interactive, env: self.env, diff --git a/crates/spk-cli/cmd-du/Cargo.toml b/crates/spk-cli/cmd-du/Cargo.toml index 0e417337bc..9e5d4ae2a5 100644 --- a/crates/spk-cli/cmd-du/Cargo.toml +++ b/crates/spk-cli/cmd-du/Cargo.toml @@ -20,6 +20,7 @@ clap = { workspace = true } colored = { workspace = true } futures = "0.3.9" itertools = { workspace = true } +rstest = { workspace = true } spfs = { workspace = true } spk-build = { workspace = true } spk-cli-common = { workspace = true } diff --git a/crates/spk-cli/cmd-du/src/cmd_du_test.rs b/crates/spk-cli/cmd-du/src/cmd_du_test.rs index e9d723aa6d..6500ed9eeb 100644 --- a/crates/spk-cli/cmd-du/src/cmd_du_test.rs +++ b/crates/spk-cli/cmd-du/src/cmd_du_test.rs @@ -8,6 +8,7 @@ use std::sync::Mutex; use clap::Parser; use colored::Colorize; use itertools::Itertools; +use rstest::rstest; use spfs::RemoteAddress; use spfs::config::Remote; use spfs::encoding::EMPTY_DIGEST; @@ -15,7 +16,7 @@ use spk_build::{BinaryPackageBuilder, BuildSource}; use spk_schema::foundation::ident_component::Component; use spk_schema::foundation::option_map; use spk_schema::recipe; -use spk_solve::spec; +use spk_solve::{SolverImpl, spec}; use spk_storage::fixtures::*; use super::{Du, Output, Run}; @@ -42,8 +43,19 @@ struct Opt { du: Du, } +fn step_solver() -> SolverImpl { + SolverImpl::Step(spk_solve::StepSolver::default()) +} + +fn resolvo_solver() -> SolverImpl { + SolverImpl::Resolvo(spk_solve::ResolvoSolver::default()) +} + +#[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_du_trivially_works() { +async fn test_du_trivially_works(#[case] solver: SolverImpl) { let mut rt = spfs_runtime().await; let remote_repo = spfsrepo().await; rt.add_remote_repo( @@ -65,7 +77,7 @@ async fn test_du_trivially_works() { rt.tmprepo.publish_recipe(&spec).await.unwrap(); - let (_spec, _) = BinaryPackageBuilder::from_recipe(spec) + let (_spec, _) = BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(&option_map! {}, &*rt.tmprepo) @@ -132,8 +144,11 @@ async fn test_du_warnings_when_object_is_tree_or_blob() { assert_eq!(opt.du.output.warnings.lock().unwrap().len(), 2); } +#[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_du_non_existing_version() { +async fn test_du_non_existing_version(#[case] solver: SolverImpl) { let mut rt = spfs_runtime().await; let remote_repo = spfsrepo().await; rt.add_remote_repo( @@ -150,7 +165,7 @@ async fn test_du_non_existing_version() { rt.tmprepo.publish_recipe(&spec).await.unwrap(); - let (_spec, _) = BinaryPackageBuilder::from_recipe(spec) + let (_spec, _) = BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(&option_map! {}, &*rt.tmprepo) @@ -162,8 +177,11 @@ async fn test_du_non_existing_version() { assert_eq!(opt.du.output.vec.lock().unwrap().len(), 0); } +#[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_du_out_of_range_input() { +async fn test_du_out_of_range_input(#[case] solver: SolverImpl) { let mut rt = spfs_runtime().await; let remote_repo = spfsrepo().await; rt.add_remote_repo( @@ -180,7 +198,7 @@ async fn test_du_out_of_range_input() { rt.tmprepo.publish_recipe(&spec).await.unwrap(); - let (_spec, _) = BinaryPackageBuilder::from_recipe(spec) + let (_spec, _) = BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(&option_map! {}, &*rt.tmprepo) @@ -196,8 +214,11 @@ async fn test_du_out_of_range_input() { assert_eq!(opt.du.output.vec.lock().unwrap().len(), 0); } +#[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_du_is_not_counting_links() { +async fn test_du_is_not_counting_links(#[case] solver: SolverImpl) { let mut rt = spfs_runtime().await; let remote_repo = spfsrepo().await; rt.add_remote_repo( @@ -219,7 +240,7 @@ async fn test_du_is_not_counting_links() { rt.tmprepo.publish_recipe(&spec).await.unwrap(); - let (_spec, _) = BinaryPackageBuilder::from_recipe(spec) + let (_spec, _) = BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(&option_map! {}, &*rt.tmprepo) @@ -239,8 +260,11 @@ async fn test_du_is_not_counting_links() { assert_ne!(build_component_output[0].parse::().unwrap(), 0); } +#[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_du_is_counting_links() { +async fn test_du_is_counting_links(#[case] solver: SolverImpl) { let mut rt = spfs_runtime().await; let remote_repo = spfsrepo().await; rt.add_remote_repo( @@ -262,7 +286,7 @@ async fn test_du_is_counting_links() { rt.tmprepo.publish_recipe(&spec).await.unwrap(); - let (_spec, _) = BinaryPackageBuilder::from_recipe(spec) + let (_spec, _) = BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(&option_map! {}, &*rt.tmprepo) @@ -284,8 +308,11 @@ async fn test_du_is_counting_links() { ); } +#[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_du_total_size() { +async fn test_du_total_size(#[case] solver: SolverImpl) { let mut rt = spfs_runtime().await; let remote_repo = spfsrepo().await; rt.add_remote_repo( @@ -302,7 +329,7 @@ async fn test_du_total_size() { rt.tmprepo.publish_recipe(&spec).await.unwrap(); - let (_spec, _) = BinaryPackageBuilder::from_recipe(spec) + let (_spec, _) = BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(&option_map! {}, &*rt.tmprepo) @@ -330,8 +357,11 @@ async fn test_du_total_size() { assert_eq!(total, calculated_total_size_from_output); } +#[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_du_summarize_output_enabled() { +async fn test_du_summarize_output_enabled(#[case] solver: SolverImpl) { let mut rt = spfs_runtime().await; let remote_repo = spfsrepo().await; rt.add_remote_repo( @@ -353,7 +383,7 @@ async fn test_du_summarize_output_enabled() { rt.tmprepo.publish_recipe(&spec).await.unwrap(); - let (_spec, _) = BinaryPackageBuilder::from_recipe(spec) + let (_spec, _) = BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(&option_map! {}, &*rt.tmprepo) @@ -370,8 +400,11 @@ async fn test_du_summarize_output_enabled() { assert_eq!(generated_output, expected_output); } +#[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_du_summarize_output_is_not_enabled() { +async fn test_du_summarize_output_is_not_enabled(#[case] solver: SolverImpl) { let mut rt = spfs_runtime().await; let remote_repo = spfsrepo().await; rt.add_remote_repo( @@ -393,7 +426,7 @@ async fn test_du_summarize_output_is_not_enabled() { rt.tmprepo.publish_recipe(&spec).await.unwrap(); - let (_spec, _) = BinaryPackageBuilder::from_recipe(spec) + let (_spec, _) = BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(&option_map! {}, &*rt.tmprepo) @@ -425,8 +458,11 @@ async fn test_du_summarize_output_is_not_enabled() { assert_eq!(expected_output.len(), 0); } +#[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_deprecate_flag() { +async fn test_deprecate_flag(#[case] solver: SolverImpl) { let mut rt = spfs_runtime().await; let remote_repo = spfsrepo().await; rt.add_remote_repo( @@ -448,7 +484,7 @@ async fn test_deprecate_flag() { rt.tmprepo.publish_recipe(&spec).await.unwrap(); - let (_spec, _) = BinaryPackageBuilder::from_recipe(spec) + let (_spec, _) = BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(&option_map! {}, &*rt.tmprepo) @@ -476,8 +512,11 @@ async fn test_deprecate_flag() { assert_eq!(expected_output, generated_output); } +#[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_human_readable_flag() { +async fn test_human_readable_flag(#[case] solver: SolverImpl) { let mut rt = spfs_runtime().await; let remote_repo = spfsrepo().await; rt.add_remote_repo( @@ -499,7 +538,7 @@ async fn test_human_readable_flag() { rt.tmprepo.publish_recipe(&spec).await.unwrap(); - let (_spec, _) = BinaryPackageBuilder::from_recipe(spec) + let (_spec, _) = BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_source(BuildSource::LocalPath(".".into())) .with_repository(rt.tmprepo.clone()) .build_and_publish(&option_map! {}, &*rt.tmprepo) diff --git a/crates/spk-cli/cmd-env/src/cmd_env.rs b/crates/spk-cli/cmd-env/src/cmd_env.rs index 9f12217ce6..16915ec93d 100644 --- a/crates/spk-cli/cmd-env/src/cmd_env.rs +++ b/crates/spk-cli/cmd-env/src/cmd_env.rs @@ -11,9 +11,9 @@ use spfs::tracking::SpecFile; use spfs_cli_common::Progress; use spk_cli_common::{CommandArgs, Run, build_required_packages, flags}; use spk_exec::setup_runtime_with_reporter; -use spk_solve::Solver; #[cfg(feature = "statsd")] use spk_solve::{SPK_RUN_TIME_METRIC, get_metrics_client}; +use spk_solve::{Solver, SolverMut}; /// Resolve and run an environment on-the-fly /// @@ -90,7 +90,7 @@ impl Run for Env { let formatter = self.formatter_settings.get_formatter(self.verbose)?; let solution = solver.run_and_print_resolve(&formatter).await?; - let solution = build_required_packages(&solution).await?; + let solution = build_required_packages(&solution, solver).await?; rt.status.editable = self.runtime.editable() || self.requests.any_build_stage_requests(&self.requested)?; diff --git a/crates/spk-cli/cmd-explain/src/cmd_explain.rs b/crates/spk-cli/cmd-explain/src/cmd_explain.rs index 8aaf46590b..2027bd1304 100644 --- a/crates/spk-cli/cmd-explain/src/cmd_explain.rs +++ b/crates/spk-cli/cmd-explain/src/cmd_explain.rs @@ -5,7 +5,7 @@ use clap::Args; use miette::Result; use spk_cli_common::{CommandArgs, Run, flags}; -use spk_solve::Solver; +use spk_solve::{Solver, SolverMut}; /// Show the resolve process for a set of packages. #[derive(Args)] diff --git a/crates/spk-cli/cmd-install/src/cmd_install.rs b/crates/spk-cli/cmd-install/src/cmd_install.rs index e9ada0b2a2..e1c3285edb 100644 --- a/crates/spk-cli/cmd-install/src/cmd_install.rs +++ b/crates/spk-cli/cmd-install/src/cmd_install.rs @@ -14,7 +14,7 @@ use spk_exec::setup_current_runtime; use spk_schema::Package; use spk_schema::foundation::format::FormatIdent; use spk_schema::foundation::spec_ops::Named; -use spk_solve::Solver; +use spk_solve::{Solver, SolverMut}; /// Install a package into the current environment #[derive(Args)] @@ -119,7 +119,7 @@ impl Run for Install { } } - let compiled_solution = build_required_packages(&solution) + let compiled_solution = build_required_packages(&solution, solver) .await .wrap_err("Failed to build one or more packages from source")?; setup_current_runtime(&compiled_solution).await?; diff --git a/crates/spk-cli/cmd-make-binary/src/cmd_make_binary.rs b/crates/spk-cli/cmd-make-binary/src/cmd_make_binary.rs index 835caa091d..ce010fd612 100644 --- a/crates/spk-cli/cmd-make-binary/src/cmd_make_binary.rs +++ b/crates/spk-cli/cmd-make-binary/src/cmd_make_binary.rs @@ -15,7 +15,6 @@ use spk_schema::foundation::format::FormatIdent; use spk_schema::ident::{PkgRequest, RequestedBy}; use spk_schema::option_map::HOST_OPTIONS; use spk_schema::prelude::*; -use spk_solve::DefaultCdclResolver; use spk_storage as storage; #[cfg(test)] @@ -27,7 +26,7 @@ mod cmd_make_binary_test; #[clap(visible_aliases = &["mkbinary", "mkbin", "mkb"])] pub struct MakeBinary { #[clap(flatten)] - pub repos: flags::Repositories, + pub solver: flags::Solver, #[clap(flatten)] pub options: flags::Options, #[clap(flatten)] @@ -94,7 +93,7 @@ impl Run for MakeBinary { let (_runtime, local, repos) = tokio::try_join!( self.runtime.ensure_active_runtime(&["make-binary", "mkbinary", "mkbin", "mkb"]), storage::local_repository().map_ok(storage::RepositoryHandle::from).map_err(miette::Error::from), - async { self.repos.get_repos_for_non_destructive_operation().await } + async { self.solver.repos.get_repos_for_non_destructive_operation().await } )?; let repos = repos .into_iter() @@ -167,21 +166,23 @@ impl Run for MakeBinary { let mut fmt_builder = self .formatter_settings .get_formatter_builder(self.verbose)?; - let _src_formatter = fmt_builder + let src_formatter = fmt_builder .with_solution(true) .with_header("Src Resolver ") .build(); - let _build_formatter = fmt_builder + let build_formatter = fmt_builder .with_solution(true) .with_header("Build Resolver ") .build(); - let mut builder = BinaryPackageBuilder::from_recipe((*recipe).clone()); + let solver = self.solver.get_solver(&self.options).await?; + let mut builder = + BinaryPackageBuilder::from_recipe_with_solver((*recipe).clone(), solver); builder .with_repositories(repos.iter().cloned()) .set_interactive(self.interactive) - .with_source_resolver(DefaultCdclResolver {}) - .with_build_resolver(DefaultCdclResolver {}) + .with_source_formatter(src_formatter) + .with_build_formatter(build_formatter) .with_allow_circular_dependencies(self.allow_circular_dependencies); if self.here { diff --git a/crates/spk-cli/cmd-render/src/cmd_render.rs b/crates/spk-cli/cmd-render/src/cmd_render.rs index eb0e88df44..60b133be90 100644 --- a/crates/spk-cli/cmd-render/src/cmd_render.rs +++ b/crates/spk-cli/cmd-render/src/cmd_render.rs @@ -9,7 +9,7 @@ use miette::{Context, IntoDiagnostic, Result, bail}; use spfs::storage::fallback::FallbackProxy; use spk_cli_common::{CommandArgs, Run, build_required_packages, flags}; use spk_exec::resolve_runtime_layers; -use spk_solve::Solver; +use spk_solve::{Solver, SolverMut}; /// Output the contents of an spk environment (/spfs) to a folder #[derive(Args)] @@ -55,7 +55,7 @@ impl Run for Render { let formatter = self.formatter_settings.get_formatter(self.verbose)?; let solution = solver.run_and_print_resolve(&formatter).await?; - let solution = build_required_packages(&solution).await?; + let solution = build_required_packages(&solution, solver.clone()).await?; let stack = resolve_runtime_layers(true, &solution).await?; std::fs::create_dir_all(&self.target) .into_diagnostic() diff --git a/crates/spk-cli/cmd-test/src/cmd_test.rs b/crates/spk-cli/cmd-test/src/cmd_test.rs index ce9bf3d246..8856fe11b7 100644 --- a/crates/spk-cli/cmd-test/src/cmd_test.rs +++ b/crates/spk-cli/cmd-test/src/cmd_test.rs @@ -15,7 +15,6 @@ use spk_schema::foundation::ident_build::Build; use spk_schema::foundation::option_map::{HOST_OPTIONS, OptionMap}; use spk_schema::prelude::*; use spk_schema::{Recipe, Request, TestStage}; -use spk_solve::DefaultCdclResolver; use crate::test::{PackageBuildTester, PackageInstallTester, PackageSourceTester, Tester}; @@ -161,11 +160,11 @@ impl Run for CmdTest { let mut builder = self .formatter_settings .get_formatter_builder(self.verbose)?; - let _src_formatter = builder.with_header("Source Resolver ").build(); - let _build_src_formatter = + let src_formatter = builder.with_header("Source Resolver ").build(); + let build_src_formatter = builder.with_header("Build Source Resolver ").build(); - let _build_formatter = builder.with_header("Build Resolver ").build(); - let _install_formatter = + let build_formatter = builder.with_header("Build Resolver ").build(); + let install_formatter = builder.with_header("Install Env Resolver ").build(); let mut tester: Box = match stage { @@ -178,7 +177,7 @@ impl Run for CmdTest { .with_repositories(repos.iter().cloned()) .with_requirements(test.additional_requirements()) .with_source(source.clone()) - .watch_environment_resolve(DefaultCdclResolver {}); + .watch_environment_formatter(src_formatter); Box::new(tester) } @@ -209,8 +208,8 @@ impl Run for CmdTest { }, ), ) - .with_source_resolver(DefaultCdclResolver {}) - .with_build_resolver(DefaultCdclResolver {}); + .with_source_formatter(build_src_formatter) + .with_build_formatter(build_formatter); Box::new(tester) } @@ -228,7 +227,7 @@ impl Run for CmdTest { .with_requirements(test.additional_requirements()) .with_requirements(options_reqs.clone()) .with_source(source.clone()) - .watch_environment_resolve(DefaultCdclResolver {}); + .watch_environment_formatter(install_formatter); Box::new(tester) } diff --git a/crates/spk-cli/cmd-test/src/test/build.rs b/crates/spk-cli/cmd-test/src/test/build.rs index 67df2444c3..d1f606208c 100644 --- a/crates/spk-cli/cmd-test/src/test/build.rs +++ b/crates/spk-cli/cmd-test/src/test/build.rs @@ -15,18 +15,12 @@ use spk_schema::foundation::option_map::OptionMap; use spk_schema::ident::{PkgRequest, PreReleasePolicy, RangeIdent, Request, RequestedBy}; use spk_schema::{AnyIdent, Recipe, SpecRecipe}; use spk_solve::solution::Solution; -use spk_solve::{ - BoxedCdclResolverCallback, - DefaultCdclResolver, - ResolverCallback, - ResolvoSolver, - Solver, -}; +use spk_solve::{DecisionFormatter, ResolvoSolver, SolverExt, SolverMut}; use spk_storage as storage; use super::Tester; -pub struct PackageBuildTester<'a> { +pub struct PackageBuildTester { prefix: PathBuf, recipe: SpecRecipe, script: String, @@ -34,11 +28,11 @@ pub struct PackageBuildTester<'a> { options: OptionMap, additional_requirements: Vec, source: BuildSource, - source_resolver: BoxedCdclResolverCallback<'a>, - build_resolver: BoxedCdclResolverCallback<'a>, + source_formatter: DecisionFormatter, + build_formatter: DecisionFormatter, } -impl<'a> PackageBuildTester<'a> { +impl PackageBuildTester { pub fn new(recipe: SpecRecipe, script: String) -> Self { let source = BuildSource::SourcePackage(recipe.ident().to_any_ident(Some(Build::Source)).into()); @@ -50,8 +44,8 @@ impl<'a> PackageBuildTester<'a> { options: OptionMap::default(), additional_requirements: Vec::new(), source, - source_resolver: Box::new(DefaultCdclResolver {}), - build_resolver: Box::new(DefaultCdclResolver {}), + source_formatter: DecisionFormatter::default(), + build_formatter: DecisionFormatter::default(), } } @@ -80,31 +74,15 @@ impl<'a> PackageBuildTester<'a> { self } - /// Provide a function that will be called when resolving the source package. - /// - /// This function should run the provided solver runtime to - /// completion, returning the final result. This function - /// is useful for introspecting and reporting on the solve - /// process as needed. - pub fn with_source_resolver(&mut self, resolver: F) -> &mut Self - where - F: ResolverCallback + 'a, - { - self.source_resolver = Box::new(resolver); + /// Provide a formatter to use when resolving the source package. + pub fn with_source_formatter(&mut self, formatter: DecisionFormatter) -> &mut Self { + self.source_formatter = formatter; self } - /// Provide a function that will be called when resolving the build environment. - /// - /// This function should run the provided solver runtime to - /// completion, returning the final result. This function - /// is useful for introspecting and reporting on the solve - /// process as needed. - pub fn with_build_resolver(&mut self, resolver: F) -> &mut Self - where - F: ResolverCallback + 'a, - { - self.build_resolver = Box::new(resolver); + /// Provide a formatter to use when resolving the build environment. + pub fn with_build_formatter(&mut self, formatter: DecisionFormatter) -> &mut Self { + self.build_formatter = formatter; self } @@ -136,7 +114,7 @@ impl<'a> PackageBuildTester<'a> { } // let (solution, _) = self.build_resolver.solve(&solver).await?; - let solution = self.build_resolver.solve(&solver).await?; + let solution = solver.run_and_print_resolve(&self.build_formatter).await?; for layer in resolve_runtime_layers(requires_localization, &solution).await? { rt.push_digest(layer); @@ -184,13 +162,13 @@ impl<'a> PackageBuildTester<'a> { solver.add_request(request.into()); // let (solution, _) = self.source_resolver.solve(&solver).await?; - let solution = self.source_resolver.solve(&solver).await?; + let solution = solver.run_and_print_resolve(&self.source_formatter).await?; Ok(solution) } } #[async_trait::async_trait] -impl Tester for PackageBuildTester<'_> { +impl Tester for PackageBuildTester { async fn test(&mut self) -> Result<()> { self.test().await } diff --git a/crates/spk-cli/cmd-test/src/test/install.rs b/crates/spk-cli/cmd-test/src/test/install.rs index f0998e036b..1cdd60200d 100644 --- a/crates/spk-cli/cmd-test/src/test/install.rs +++ b/crates/spk-cli/cmd-test/src/test/install.rs @@ -12,19 +12,12 @@ use spk_schema::foundation::option_map::OptionMap; use spk_schema::ident::{PkgRequest, PreReleasePolicy, RangeIdent, Request, RequestedBy}; use spk_schema::ident_build::Build; use spk_schema::{Recipe, SpecRecipe, Variant, VariantExt}; -use spk_solve::{ - BoxedCdclResolverCallback, - DefaultCdclResolver, - ResolverCallback, - ResolvoSolver, - Solution, - Solver, -}; +use spk_solve::{DecisionFormatter, ResolvoSolver, SolverExt, SolverMut}; use spk_storage as storage; use super::Tester; -pub struct PackageInstallTester<'a, V> { +pub struct PackageInstallTester { prefix: PathBuf, recipe: SpecRecipe, script: String, @@ -32,11 +25,11 @@ pub struct PackageInstallTester<'a, V> { options: OptionMap, additional_requirements: Vec, source: Option, - env_resolver: BoxedCdclResolverCallback<'a>, + env_formatter: DecisionFormatter, variant: V, } -impl<'a, V> PackageInstallTester<'a, V> +impl PackageInstallTester where V: Clone + Variant + Send, { @@ -49,7 +42,7 @@ where options: OptionMap::default(), additional_requirements: Vec::new(), source: None, - env_resolver: Box::new(DefaultCdclResolver {}), + env_formatter: DecisionFormatter::default(), variant, } } @@ -79,17 +72,9 @@ where self } - /// Provide a function that will be called when resolving the test environment. - /// - /// This function should run the provided solver runtime to - /// completion, returning the final result. This function - /// is useful for introspecting and reporting on the solve - /// process as needed. - pub fn watch_environment_resolve(&mut self, resolver: F) -> &mut Self - where - F: ResolverCallback + 'a, - { - self.env_resolver = Box::new(resolver); + /// Provide a formatter to use when resolving the test environment. + pub fn watch_environment_formatter(&mut self, formatter: DecisionFormatter) -> &mut Self { + self.env_formatter = formatter; self } @@ -130,7 +115,7 @@ where } // let (solution, _) = self.env_resolver.solve(&solver).await?; - let solution = self.env_resolver.solve(&solver).await?; + let solution = solver.run_and_print_resolve(&self.env_formatter).await?; for layer in resolve_runtime_layers(requires_localization, &solution).await? { rt.push_digest(layer); @@ -150,7 +135,7 @@ where } #[async_trait::async_trait] -impl Tester for PackageInstallTester<'_, V> +impl Tester for PackageInstallTester where V: Clone + Variant + Send, { diff --git a/crates/spk-cli/cmd-test/src/test/sources.rs b/crates/spk-cli/cmd-test/src/test/sources.rs index 3ebd92d806..1eef7fd315 100644 --- a/crates/spk-cli/cmd-test/src/test/sources.rs +++ b/crates/spk-cli/cmd-test/src/test/sources.rs @@ -13,19 +13,12 @@ use spk_schema::foundation::ident_component::Component; use spk_schema::foundation::option_map::OptionMap; use spk_schema::ident::{PkgRequest, PreReleasePolicy, RangeIdent, Request, RequestedBy}; use spk_schema::{Recipe, SpecRecipe}; -use spk_solve::{ - BoxedCdclResolverCallback, - DefaultCdclResolver, - ResolverCallback, - ResolvoSolver, - Solution, - Solver, -}; +use spk_solve::{DecisionFormatter, ResolvoSolver, SolverExt, SolverMut}; use spk_storage as storage; use super::Tester; -pub struct PackageSourceTester<'a> { +pub struct PackageSourceTester { prefix: PathBuf, recipe: SpecRecipe, script: String, @@ -33,10 +26,10 @@ pub struct PackageSourceTester<'a> { options: OptionMap, additional_requirements: Vec, source: Option, - env_resolver: BoxedCdclResolverCallback<'a>, + env_formatter: DecisionFormatter, } -impl<'a> PackageSourceTester<'a> { +impl PackageSourceTester { pub fn new(recipe: SpecRecipe, script: String) -> Self { Self { prefix: PathBuf::from("/spfs"), @@ -46,7 +39,7 @@ impl<'a> PackageSourceTester<'a> { options: OptionMap::default(), additional_requirements: Vec::new(), source: None, - env_resolver: Box::new(DefaultCdclResolver {}), + env_formatter: DecisionFormatter::default(), } } @@ -76,17 +69,9 @@ impl<'a> PackageSourceTester<'a> { self } - /// Provide a function that will be called when resolving the test environment. - /// - /// This function should run the provided solver runtime to - /// completion, returning the final result. This function - /// is useful for introspecting and reporting on the solve - /// process as needed. - pub fn watch_environment_resolve(&mut self, resolver: F) -> &mut Self - where - F: ResolverCallback + 'a, - { - self.env_resolver = Box::new(resolver); + /// Provide a formatter to use when resolving the test environment. + pub fn watch_environment_formatter(&mut self, formatter: DecisionFormatter) -> &mut Self { + self.env_formatter = formatter; self } @@ -124,7 +109,7 @@ impl<'a> PackageSourceTester<'a> { } // let (solution, _) = self.env_resolver.solve(&solver).await?; - let solution = self.env_resolver.solve(&solver).await?; + let solution = solver.run_and_print_resolve(&self.env_formatter).await?; for layer in resolve_runtime_layers(requires_localization, &solution).await? { rt.push_digest(layer); @@ -145,7 +130,7 @@ impl<'a> PackageSourceTester<'a> { } #[async_trait::async_trait] -impl Tester for PackageSourceTester<'_> { +impl Tester for PackageSourceTester { async fn test(&mut self) -> Result<()> { self.test().await } diff --git a/crates/spk-cli/common/src/exec.rs b/crates/spk-cli/common/src/exec.rs index 7a83024360..09650e4a47 100644 --- a/crates/spk-cli/common/src/exec.rs +++ b/crates/spk-cli/common/src/exec.rs @@ -8,6 +8,7 @@ use spk_build::BinaryPackageBuilder; use spk_schema::Package; use spk_schema::foundation::format::{FormatIdent, FormatOptionMap}; use spk_solve::solution::{PackageSource, Solution}; +use spk_solve::{SolverExt, SolverMut}; use spk_storage as storage; use crate::Result; @@ -15,7 +16,13 @@ use crate::Result; /// Build any packages in the given solution that need building. /// /// Returns a new solution of only binary packages. -pub async fn build_required_packages(solution: &Solution) -> Result { +pub async fn build_required_packages( + solution: &Solution, + _solver: Solver, +) -> Result +where + Solver: SolverExt + SolverMut + Default, +{ let handle: storage::RepositoryHandle = storage::local_repository().await?.into(); let local_repo = Arc::new(handle); let repos = solution.repositories(); @@ -35,10 +42,11 @@ pub async fn build_required_packages(solution: &Solution) -> Result { item.spec.ident().format_ident(), options.format_option_map() ); - let (package, components) = BinaryPackageBuilder::from_recipe((**recipe).clone()) - .with_repositories(repos.clone()) - .build_and_publish(&options, &*local_repo) - .await?; + let (package, components) = + BinaryPackageBuilder::from_recipe_with_solver((**recipe).clone(), Solver::default()) + .with_repositories(repos.clone()) + .build_and_publish(&options, &*local_repo) + .await?; let source = PackageSource::Repository { repo: local_repo.clone(), components, diff --git a/crates/spk-cli/common/src/flags.rs b/crates/spk-cli/common/src/flags.rs index fda4ec253b..611bf85f9b 100644 --- a/crates/spk-cli/common/src/flags.rs +++ b/crates/spk-cli/common/src/flags.rs @@ -15,7 +15,8 @@ use solve::{ DecisionFormatter, DecisionFormatterBuilder, MultiSolverKind, - Solver as SolverTrait, + SolverExt, + SolverMut, }; use spk_schema::foundation::format::FormatIdent; use spk_schema::foundation::ident_build::Build; @@ -258,7 +259,10 @@ pub struct Solver { } impl Solver { - pub async fn get_solver(&self, options: &Options) -> Result { + pub async fn get_solver( + &self, + options: &Options, + ) -> Result { let option_map = options.get_options()?; //let mut solver = solve::StepSolver::default(); diff --git a/crates/spk-cli/group1/src/cmd_bake.rs b/crates/spk-cli/group1/src/cmd_bake.rs index 2b7ca54d41..72e3512ff0 100644 --- a/crates/spk-cli/group1/src/cmd_bake.rs +++ b/crates/spk-cli/group1/src/cmd_bake.rs @@ -9,8 +9,8 @@ use serde::Serialize; use spk_cli_common::{CommandArgs, Run, current_env, flags}; use spk_schema::Package; use spk_schema::ident::RequestedBy; -use spk_solve::Solver; use spk_solve::solution::{LayerPackageAndComponents, PackageSource, get_spfs_layers_to_packages}; +use spk_solve::{Solver, SolverMut}; #[cfg(test)] #[path = "./cmd_bake_test.rs"] diff --git a/crates/spk-cli/group3/src/cmd_export_test.rs b/crates/spk-cli/group3/src/cmd_export_test.rs index 900d134ec0..c1abb116b6 100644 --- a/crates/spk-cli/group3/src/cmd_export_test.rs +++ b/crates/spk-cli/group3/src/cmd_export_test.rs @@ -7,11 +7,22 @@ use spfs::prelude::*; use spk_build::{BinaryPackageBuilder, BuildSource}; use spk_schema::foundation::option_map; use spk_schema::{Package, recipe}; +use spk_solve::SolverImpl; use spk_storage::fixtures::*; +fn step_solver() -> SolverImpl { + SolverImpl::Step(spk_solve::StepSolver::default()) +} + +fn resolvo_solver() -> SolverImpl { + SolverImpl::Resolvo(spk_solve::ResolvoSolver::default()) +} + #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_export_works_with_missing_builds() { +async fn test_export_works_with_missing_builds(#[case] solver: SolverImpl) { let rt = spfs_runtime().await; let spec = recipe!( @@ -26,12 +37,13 @@ async fn test_export_works_with_missing_builds() { } ); rt.tmprepo.publish_recipe(&spec).await.unwrap(); - let (blue_spec, _) = BinaryPackageBuilder::from_recipe(spec.clone()) - .with_source(BuildSource::LocalPath(".".into())) - .build_and_publish(option_map! {"color" => "blue"}, &*rt.tmprepo) - .await - .unwrap(); - let (red_spec, _) = BinaryPackageBuilder::from_recipe(spec) + let (blue_spec, _) = + BinaryPackageBuilder::from_recipe_with_solver(spec.clone(), solver.clone()) + .with_source(BuildSource::LocalPath(".".into())) + .build_and_publish(option_map! {"color" => "blue"}, &*rt.tmprepo) + .await + .unwrap(); + let (red_spec, _) = BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_source(BuildSource::LocalPath(".".into())) .build_and_publish(option_map! {"color" => "red"}, &*rt.tmprepo) .await diff --git a/crates/spk-cli/group3/src/cmd_import_test.rs b/crates/spk-cli/group3/src/cmd_import_test.rs index f0d72e0def..613c99fca1 100644 --- a/crates/spk-cli/group3/src/cmd_import_test.rs +++ b/crates/spk-cli/group3/src/cmd_import_test.rs @@ -7,11 +7,22 @@ use spk_build::{BinaryPackageBuilder, BuildSource}; use spk_cli_common::Run; use spk_schema::foundation::option_map; use spk_schema::{Package, recipe}; +use spk_solve::SolverImpl; use spk_storage::fixtures::*; +fn step_solver() -> SolverImpl { + SolverImpl::Step(spk_solve::StepSolver::default()) +} + +fn resolvo_solver() -> SolverImpl { + SolverImpl::Resolvo(spk_solve::ResolvoSolver::default()) +} + #[rstest] +#[case::step(step_solver())] +#[case::resolvo(resolvo_solver())] #[tokio::test] -async fn test_archive_io() { +async fn test_archive_io(#[case] solver: SolverImpl) { let rt = spfs_runtime().await; let spec = recipe!( { @@ -20,7 +31,7 @@ async fn test_archive_io() { } ); rt.tmprepo.publish_recipe(&spec).await.unwrap(); - let (spec, _) = BinaryPackageBuilder::from_recipe(spec) + let (spec, _) = BinaryPackageBuilder::from_recipe_with_solver(spec, solver) .with_source(BuildSource::LocalPath(".".into())) .build_and_publish(option_map! {}, &*rt.tmprepo) .await diff --git a/crates/spk-cli/group4/src/cmd_view.rs b/crates/spk-cli/group4/src/cmd_view.rs index 76ddd40854..0ca00e0133 100644 --- a/crates/spk-cli/group4/src/cmd_view.rs +++ b/crates/spk-cli/group4/src/cmd_view.rs @@ -35,7 +35,7 @@ use spk_schema::{ VersionIdent, }; use spk_solve::solution::{LayerPackageAndComponents, get_spfs_layers_to_packages}; -use spk_solve::{Recipe, Solver}; +use spk_solve::{Recipe, Solver, SolverMut}; use spk_storage; use strum::{Display, EnumString, IntoEnumIterator, VariantNames}; diff --git a/crates/spk-exec/src/exec_test.rs b/crates/spk-exec/src/exec_test.rs index 2033cac487..7bd12c7aa1 100644 --- a/crates/spk-exec/src/exec_test.rs +++ b/crates/spk-exec/src/exec_test.rs @@ -9,7 +9,7 @@ use rstest::{fixture, rstest}; use spk_cmd_build::build_package; use spk_schema::foundation::fixtures::*; use spk_schema::ident::build_ident; -use spk_solve::{DecisionFormatterBuilder, Solver, StepSolver}; +use spk_solve::{DecisionFormatterBuilder, SolverExt, SolverMut, StepSolver}; use spk_solve_macros::request; use spk_storage::fixtures::*; diff --git a/crates/spk-solve/src/io.rs b/crates/spk-solve/src/io.rs index cb201f47da..f230b58629 100644 --- a/crates/spk-solve/src/io.rs +++ b/crates/spk-solve/src/io.rs @@ -42,15 +42,7 @@ use spk_solve_graph::{ use crate::solvers::step::ErrorFreq; use crate::solvers::{StepSolver, StepSolverRuntime}; -use crate::{ - Error, - ResolverCallback, - Result, - Solution, - Solver, - StatusLine, - show_search_space_stats, -}; +use crate::{Error, Result, Solution, Solver, StatusLine, show_search_space_stats}; #[cfg(feature = "statsd")] use crate::{ SPK_SOLUTION_PACKAGE_COUNT_METRIC, @@ -1010,7 +1002,7 @@ impl OutputKind { } } -#[derive(Debug, Clone)] +#[derive(Debug, Default, Clone)] pub(crate) struct DecisionFormatterSettings { pub(crate) verbosity: u8, pub(crate) report_time: bool, @@ -1041,12 +1033,13 @@ enum LoopOutcome { Success, } -#[derive(PartialEq, Eq, Clone, Debug)] +#[derive(PartialEq, Eq, Clone, Debug, Default)] pub enum MultiSolverKind { Unchanged, AllImpossibleChecks, // This isn't a solver on its own. It indicates: the run all the // solvers in parallel but show the output from the unchanged one. + #[default] All, } @@ -1123,7 +1116,7 @@ struct SolverResult { pub(crate) result: Result<(Solution, Arc>)>, } -#[derive(Debug, Clone)] +#[derive(Debug, Default, Clone)] pub struct DecisionFormatter { pub(crate) settings: DecisionFormatterSettings, } @@ -2103,23 +2096,3 @@ impl DecisionFormatter { out } } - -#[async_trait::async_trait] -impl ResolverCallback for &DecisionFormatter { - type Solver = StepSolver; - type SolveResult = (Solution, Arc>); - - async fn solve<'s, 'a: 's>(&'s self, r: &'a Self::Solver) -> Result { - self.run_and_print_resolve(r).await - } -} - -#[async_trait::async_trait] -impl ResolverCallback for DecisionFormatter { - type Solver = StepSolver; - type SolveResult = (Solution, Arc>); - - async fn solve<'s, 'a: 's>(&'s self, r: &'a Self::Solver) -> Result { - self.run_and_print_resolve(r).await - } -} diff --git a/crates/spk-solve/src/lib.rs b/crates/spk-solve/src/lib.rs index fb77e7b6e8..933a5b3865 100644 --- a/crates/spk-solve/src/lib.rs +++ b/crates/spk-solve/src/lib.rs @@ -11,10 +11,7 @@ mod solver; mod solvers; mod status_line; -use std::sync::Arc; - pub use error::{Error, Result}; -use graph::Graph; pub use io::{ DEFAULT_SOLVER_RUN_FILE_PREFIX, DecisionFormatter, @@ -35,9 +32,9 @@ pub use metrics::{ get_metrics_client, }; pub(crate) use search_space::show_search_space_stats; -pub use solver::Solver; +pub use solver::{Solver, SolverExt, SolverImpl, SolverMut}; // Publicly exported ResolvoSolver to stop dead code warnings -pub use solvers::resolvo::Solver as ResolvoSolver; +pub use solvers::ResolvoSolver; pub use solvers::{StepSolver, StepSolverRuntime}; pub use spk_schema::foundation::ident_build::Build; pub use spk_schema::foundation::ident_component::Component; @@ -63,53 +60,3 @@ pub use { spk_solve_solution as solution, spk_solve_validation as validation, }; - -#[async_trait::async_trait] -pub trait ResolverCallback: Send + Sync { - type Solver; - type SolveResult; - - /// Run a solve using the given [`Self::Solver`] producing a [`Self::SolveResult`]. - async fn solve<'s, 'a: 's>(&'s self, r: &'a Self::Solver) -> Result; -} - -/// A no-frills implementation of [`ResolverCallback`]. -pub struct DefaultResolver {} - -#[async_trait::async_trait] -impl ResolverCallback for DefaultResolver { - type Solver = StepSolver; - type SolveResult = (Solution, Arc>); - - async fn solve<'s, 'a: 's>(&'s self, r: &'a Self::Solver) -> Result { - let mut runtime = r.run(); - let solution = runtime.solution().await; - match solution { - Err(err) => Err(err), - Ok(s) => Ok((s, runtime.graph())), - } - } -} - -pub type BoxedResolverCallback<'a> = Box< - dyn ResolverCallback< - Solver = StepSolver, - SolveResult = (Solution, Arc>), - > + 'a, ->; - -/// Another no-frills implementation of [`ResolverCallback`]. -pub struct DefaultCdclResolver {} - -#[async_trait::async_trait] -impl ResolverCallback for DefaultCdclResolver { - type Solver = ResolvoSolver; - type SolveResult = Solution; - - async fn solve<'s, 'a: 's>(&'s self, r: &'a Self::Solver) -> Result { - r.solve().await - } -} - -pub type BoxedCdclResolverCallback<'a> = - Box + 'a>; diff --git a/crates/spk-solve/src/solver.rs b/crates/spk-solve/src/solver.rs index cc749f1888..beceffe0ad 100644 --- a/crates/spk-solve/src/solver.rs +++ b/crates/spk-solve/src/solver.rs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/spkenv/spk -use std::any::Any; use std::sync::Arc; use enum_dispatch::enum_dispatch; @@ -14,24 +13,16 @@ use variantly::Variantly; use crate::{DecisionFormatter, Result}; -#[enum_dispatch(Solver)] -#[derive(Variantly)] -pub(crate) enum SolverImpl { +#[enum_dispatch(Solver, SolverExt, SolverMut)] +#[derive(Clone, Variantly)] +pub enum SolverImpl { Step(crate::StepSolver), Resolvo(crate::solvers::ResolvoSolver), } #[async_trait::async_trait] #[enum_dispatch] -pub trait Solver: Any { - /// Add a repository where the solver can get packages. - fn add_repository(&mut self, repo: R) - where - R: Into>; - - /// Add a request to this solver. - fn add_request(&mut self, request: Request); - +pub trait Solver { /// Return the PkgRequests added to the solver. fn get_pkg_requests(&self) -> Vec; @@ -40,6 +31,13 @@ pub trait Solver: Any { /// Return a reference to the solver's list of repositories. fn repositories(&self) -> &[Arc]; +} + +#[async_trait::async_trait] +#[enum_dispatch] +pub trait SolverMut { + /// Add a request to this solver. + fn add_request(&mut self, request: Request); /// Put this solver back into its default state fn reset(&mut self); @@ -80,3 +78,92 @@ pub trait Solver: Any { fn update_options(&mut self, options: OptionMap); } + +impl Solver for &T +where + T: Solver, +{ + fn get_pkg_requests(&self) -> Vec { + T::get_pkg_requests(self) + } + + fn get_var_requests(&self) -> Vec { + T::get_var_requests(self) + } + + fn repositories(&self) -> &[Arc] { + T::repositories(self) + } +} + +impl Solver for &mut T +where + T: Solver, +{ + fn get_pkg_requests(&self) -> Vec { + T::get_pkg_requests(self) + } + + fn get_var_requests(&self) -> Vec { + T::get_var_requests(self) + } + + fn repositories(&self) -> &[Arc] { + T::repositories(self) + } +} + +#[async_trait::async_trait] +impl SolverMut for &mut T +where + T: SolverMut + Send + Sync, +{ + fn add_request(&mut self, request: Request) { + T::add_request(self, request) + } + + fn reset(&mut self) { + T::reset(self) + } + + async fn run_and_log_resolve(&mut self, formatter: &DecisionFormatter) -> Result { + T::run_and_log_resolve(self, formatter).await + } + + async fn run_and_print_resolve(&mut self, formatter: &DecisionFormatter) -> Result { + T::run_and_print_resolve(self, formatter).await + } + + fn set_binary_only(&mut self, binary_only: bool) { + T::set_binary_only(self, binary_only) + } + + async fn solve(&mut self) -> Result { + T::solve(self).await + } + + fn update_options(&mut self, options: OptionMap) { + T::update_options(self, options) + } +} + +#[async_trait::async_trait] +#[enum_dispatch] +pub trait SolverExt: Solver { + /// Add a repository where the solver can get packages. + fn add_repository(&mut self, repo: R) + where + R: Into>; +} + +impl SolverExt for &mut T +where + T: SolverExt + Sync, +{ + fn add_repository(&mut self, repo: R) + where + R: Into>, + { + T::add_repository(self, repo); + } +} diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index 8ea59761c2..c1a4f3ec88 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -39,7 +39,7 @@ use spk_solve_validation::{Validators, default_validators}; use spk_storage::RepositoryHandle; use crate::solver::Solver as SolverTrait; -use crate::{DecisionFormatter, Error, Result}; +use crate::{DecisionFormatter, Error, Result, SolverExt, SolverMut}; #[cfg(test)] #[path = "resolvo_tests.rs"] @@ -268,19 +268,7 @@ impl Solver { } } -#[async_trait::async_trait] impl SolverTrait for Solver { - fn add_repository(&mut self, repo: R) - where - R: Into>, - { - self.repos.push(repo.into()); - } - - fn add_request(&mut self, request: Request) { - self.requests.push(request); - } - fn get_pkg_requests(&self) -> Vec { self.requests .iter() @@ -300,6 +288,13 @@ impl SolverTrait for Solver { fn repositories(&self) -> &[Arc] { &self.repos } +} + +#[async_trait::async_trait] +impl SolverMut for Solver { + fn add_request(&mut self, request: Request) { + self.requests.push(request); + } fn reset(&mut self) { self.repos.truncate(0); @@ -340,3 +335,13 @@ impl SolverTrait for Solver { self.options.extend(options); } } + +#[async_trait::async_trait] +impl SolverExt for Solver { + fn add_repository(&mut self, repo: R) + where + R: Into>, + { + self.repos.push(repo.into()); + } +} diff --git a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs index fba0e89e08..4d3038656a 100644 --- a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs +++ b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs @@ -10,7 +10,7 @@ use spk_schema::{Package, opt_name}; use spk_solve_macros::{make_repo, request}; use super::Solver; -use crate::Solver as SolverTrait; +use crate::SolverMut; #[rstest] #[tokio::test] diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index 0e7efff9f7..8a8f8023a6 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -49,7 +49,7 @@ use super::pkg_request_version_set::{ SyntheticComponent, VarValue, }; -use crate::Solver; +use crate::SolverMut; // Using just the package name as a Resolvo "package name" prevents multiple // components from the same package from existing in the same solution, since diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index c1cf361277..bd512ae1bf 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -29,7 +29,7 @@ use spk_storage::fixtures::*; use tap::prelude::*; use crate::io::DecisionFormatterBuilder; -use crate::solver::{Solver, SolverImpl}; +use crate::solver::{SolverExt, SolverImpl, SolverMut}; use crate::solvers::step::{ErrorDetails, ErrorFreq}; use crate::{Error, ResolvoSolver, Result, Solution, StepSolver, option_map, spec}; diff --git a/crates/spk-solve/src/solvers/step/solver.rs b/crates/spk-solve/src/solvers/step/solver.rs index 4e12a069b1..d339fa4553 100644 --- a/crates/spk-solve/src/solvers/step/solver.rs +++ b/crates/spk-solve/src/solvers/step/solver.rs @@ -56,7 +56,7 @@ use spk_storage::RepositoryHandle; use crate::error::{self, OutOfOptions}; use crate::option_map::OptionMap; use crate::solver::Solver as SolverTrait; -use crate::{DecisionFormatter, Error, Result}; +use crate::{DecisionFormatter, Error, Result, SolverExt, SolverMut}; /// Structure to hold whether the three kinds of impossible checks are /// enabled or disabled in a solver. @@ -1103,32 +1103,7 @@ impl Solver { } } -#[async_trait::async_trait] impl SolverTrait for Solver { - fn add_repository(&mut self, repo: R) - where - R: Into>, - { - self.repos.push(repo.into()); - } - - fn add_request(&mut self, request: Request) { - let request = match request { - Request::Pkg(mut request) => { - if request.pkg.components.is_empty() { - if request.pkg.is_source() { - request.pkg.components.insert(Component::Source); - } else { - request.pkg.components.insert(Component::default_for_run()); - } - } - Change::RequestPackage(RequestPackage::new(request)) - } - Request::Var(request) => Change::RequestVar(RequestVar::new(request)), - }; - self.initial_state_builders.push(request); - } - fn get_pkg_requests(&self) -> Vec { self.get_initial_state() .get_pkg_requests() @@ -1148,6 +1123,26 @@ impl SolverTrait for Solver { fn repositories(&self) -> &[Arc] { &self.repos } +} + +#[async_trait::async_trait] +impl SolverMut for Solver { + fn add_request(&mut self, request: Request) { + let request = match request { + Request::Pkg(mut request) => { + if request.pkg.components.is_empty() { + if request.pkg.is_source() { + request.pkg.components.insert(Component::Source); + } else { + request.pkg.components.insert(Component::default_for_run()); + } + } + Change::RequestPackage(RequestPackage::new(request)) + } + Request::Var(request) => Change::RequestVar(RequestVar::new(request)), + }; + self.initial_state_builders.push(request); + } fn reset(&mut self) { self.repos.truncate(0); @@ -1219,6 +1214,16 @@ impl SolverTrait for Solver { } } +#[async_trait::async_trait] +impl SolverExt for Solver { + fn add_repository(&mut self, repo: R) + where + R: Into>, + { + self.repos.push(repo.into()); + } +} + // This is needed so `PriorityQueue` doesn't need to hash the node itself. struct NodeWrapper { pub(crate) node: Arc>>, From 8adfb2658dfbd160ddaf575ac0f7686b21027642 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Tue, 11 Feb 2025 22:17:13 -0800 Subject: [PATCH 50/64] Add solver arg to `spk test` Making it possible to choose what solver to use when running spk tests. Signed-off-by: J Robert Ray --- crates/spk-cli/cmd-test/src/cmd_test.rs | 25 ++++++++++---- crates/spk-cli/cmd-test/src/test/build.rs | 37 ++++++++++++++------- crates/spk-cli/cmd-test/src/test/install.rs | 33 +++++++++++------- crates/spk-cli/cmd-test/src/test/sources.rs | 37 ++++++++++++++------- 4 files changed, 90 insertions(+), 42 deletions(-) diff --git a/crates/spk-cli/cmd-test/src/cmd_test.rs b/crates/spk-cli/cmd-test/src/cmd_test.rs index 8856fe11b7..cfdd3b432a 100644 --- a/crates/spk-cli/cmd-test/src/cmd_test.rs +++ b/crates/spk-cli/cmd-test/src/cmd_test.rs @@ -32,7 +32,7 @@ pub struct CmdTest { #[clap(flatten)] pub runtime: flags::Runtime, #[clap(flatten)] - pub repos: flags::Repositories, + pub solver: flags::Solver, #[clap(flatten)] pub workspace: flags::Workspace, @@ -69,7 +69,7 @@ impl Run for CmdTest { let options = self.options.get_options()?; let (_runtime, repos) = tokio::try_join!( self.runtime.ensure_active_runtime(&["test"]), - self.repos.get_repos_for_non_destructive_operation() + self.solver.repos.get_repos_for_non_destructive_operation() )?; let repos = repos .into_iter() @@ -169,8 +169,13 @@ impl Run for CmdTest { let mut tester: Box = match stage { TestStage::Sources => { - let mut tester = - PackageSourceTester::new((*recipe).clone(), test.script()); + let solver = self.solver.get_solver(&self.options).await?; + + let mut tester = PackageSourceTester::new( + (*recipe).clone(), + test.script(), + solver, + ); tester .with_options(variant.options().into_owned()) @@ -183,8 +188,13 @@ impl Run for CmdTest { } TestStage::Build => { - let mut tester = - PackageBuildTester::new((*recipe).clone(), test.script()); + let solver = self.solver.get_solver(&self.options).await?; + + let mut tester = PackageBuildTester::new( + (*recipe).clone(), + test.script(), + solver, + ); tester .with_options(variant.options().into_owned()) @@ -215,10 +225,13 @@ impl Run for CmdTest { } TestStage::Install => { + let solver = self.solver.get_solver(&self.options).await?; + let mut tester = PackageInstallTester::new( (*recipe).clone(), test.script(), &variant, + solver, ); tester diff --git a/crates/spk-cli/cmd-test/src/test/build.rs b/crates/spk-cli/cmd-test/src/test/build.rs index d1f606208c..e950abba43 100644 --- a/crates/spk-cli/cmd-test/src/test/build.rs +++ b/crates/spk-cli/cmd-test/src/test/build.rs @@ -15,16 +15,20 @@ use spk_schema::foundation::option_map::OptionMap; use spk_schema::ident::{PkgRequest, PreReleasePolicy, RangeIdent, Request, RequestedBy}; use spk_schema::{AnyIdent, Recipe, SpecRecipe}; use spk_solve::solution::Solution; -use spk_solve::{DecisionFormatter, ResolvoSolver, SolverExt, SolverMut}; +use spk_solve::{DecisionFormatter, SolverExt, SolverMut}; use spk_storage as storage; use super::Tester; -pub struct PackageBuildTester { +pub struct PackageBuildTester +where + Solver: Send, +{ prefix: PathBuf, recipe: SpecRecipe, script: String, repos: Vec>, + solver: Solver, options: OptionMap, additional_requirements: Vec, source: BuildSource, @@ -32,8 +36,11 @@ pub struct PackageBuildTester { build_formatter: DecisionFormatter, } -impl PackageBuildTester { - pub fn new(recipe: SpecRecipe, script: String) -> Self { +impl PackageBuildTester +where + Solver: SolverExt + SolverMut + Default + Send, +{ + pub fn new(recipe: SpecRecipe, script: String, solver: Solver) -> Self { let source = BuildSource::SourcePackage(recipe.ident().to_any_ident(Some(Build::Source)).into()); Self { @@ -41,6 +48,7 @@ impl PackageBuildTester { recipe, script, repos: Vec::new(), + solver, options: OptionMap::default(), additional_requirements: Vec::new(), source, @@ -101,20 +109,22 @@ impl PackageBuildTester { } } - let mut solver = ResolvoSolver::default(); - solver.set_binary_only(true); - solver.update_options(self.options.clone()); + self.solver.set_binary_only(true); + self.solver.update_options(self.options.clone()); for repo in self.repos.iter().cloned() { - solver.add_repository(repo); + self.solver.add_repository(repo); } // TODO // solver.configure_for_build_environment(&self.recipe)?; for request in self.additional_requirements.drain(..) { - solver.add_request(request) + self.solver.add_request(request) } // let (solution, _) = self.build_resolver.solve(&solver).await?; - let solution = solver.run_and_print_resolve(&self.build_formatter).await?; + let solution = self + .solver + .run_and_print_resolve(&self.build_formatter) + .await?; for layer in resolve_runtime_layers(requires_localization, &solution).await? { rt.push_digest(layer); @@ -140,7 +150,7 @@ impl PackageBuildTester { } async fn resolve_source_package(&mut self, package: &AnyIdent) -> Result { - let mut solver = ResolvoSolver::default(); + let mut solver = Solver::default(); solver.update_options(self.options.clone()); let local_repo: Arc = Arc::new(storage::local_repository().await?.into()); @@ -168,7 +178,10 @@ impl PackageBuildTester { } #[async_trait::async_trait] -impl Tester for PackageBuildTester { +impl Tester for PackageBuildTester +where + Solver: SolverExt + SolverMut + Default + Send, +{ async fn test(&mut self) -> Result<()> { self.test().await } diff --git a/crates/spk-cli/cmd-test/src/test/install.rs b/crates/spk-cli/cmd-test/src/test/install.rs index 1cdd60200d..0921a66e28 100644 --- a/crates/spk-cli/cmd-test/src/test/install.rs +++ b/crates/spk-cli/cmd-test/src/test/install.rs @@ -12,16 +12,20 @@ use spk_schema::foundation::option_map::OptionMap; use spk_schema::ident::{PkgRequest, PreReleasePolicy, RangeIdent, Request, RequestedBy}; use spk_schema::ident_build::Build; use spk_schema::{Recipe, SpecRecipe, Variant, VariantExt}; -use spk_solve::{DecisionFormatter, ResolvoSolver, SolverExt, SolverMut}; +use spk_solve::{DecisionFormatter, SolverExt, SolverMut}; use spk_storage as storage; use super::Tester; -pub struct PackageInstallTester { +pub struct PackageInstallTester +where + Solver: Send, +{ prefix: PathBuf, recipe: SpecRecipe, script: String, repos: Vec>, + solver: Solver, options: OptionMap, additional_requirements: Vec, source: Option, @@ -29,16 +33,18 @@ pub struct PackageInstallTester { variant: V, } -impl PackageInstallTester +impl PackageInstallTester where V: Clone + Variant + Send, + Solver: SolverExt + SolverMut + Send, { - pub fn new(recipe: SpecRecipe, script: String, variant: V) -> Self { + pub fn new(recipe: SpecRecipe, script: String, variant: V, solver: Solver) -> Self { Self { prefix: PathBuf::from("/spfs"), recipe, script, repos: Vec::new(), + solver, options: OptionMap::default(), additional_requirements: Vec::new(), source: None, @@ -86,11 +92,10 @@ where let requires_localization = rt.config.mount_backend.requires_localization(); - let mut solver = ResolvoSolver::default(); - solver.set_binary_only(true); - solver.update_options(self.options.clone()); + self.solver.set_binary_only(true); + self.solver.update_options(self.options.clone()); for repo in self.repos.iter().cloned() { - solver.add_repository(repo); + self.solver.add_repository(repo); } // Request the specific build that goes with the selected build variant. @@ -109,13 +114,16 @@ where .with_prerelease(Some(PreReleasePolicy::IncludeAll)) .with_pin(None) .with_compat(None); - solver.add_request(request.into()); + self.solver.add_request(request.into()); for request in self.additional_requirements.drain(..) { - solver.add_request(request) + self.solver.add_request(request) } // let (solution, _) = self.env_resolver.solve(&solver).await?; - let solution = solver.run_and_print_resolve(&self.env_formatter).await?; + let solution = self + .solver + .run_and_print_resolve(&self.env_formatter) + .await?; for layer in resolve_runtime_layers(requires_localization, &solution).await? { rt.push_digest(layer); @@ -135,9 +143,10 @@ where } #[async_trait::async_trait] -impl Tester for PackageInstallTester +impl Tester for PackageInstallTester where V: Clone + Variant + Send, + Solver: SolverExt + SolverMut + Send, { async fn test(&mut self) -> Result<()> { PackageInstallTester::test(self).await diff --git a/crates/spk-cli/cmd-test/src/test/sources.rs b/crates/spk-cli/cmd-test/src/test/sources.rs index 1eef7fd315..e77b17a7ab 100644 --- a/crates/spk-cli/cmd-test/src/test/sources.rs +++ b/crates/spk-cli/cmd-test/src/test/sources.rs @@ -13,29 +13,37 @@ use spk_schema::foundation::ident_component::Component; use spk_schema::foundation::option_map::OptionMap; use spk_schema::ident::{PkgRequest, PreReleasePolicy, RangeIdent, Request, RequestedBy}; use spk_schema::{Recipe, SpecRecipe}; -use spk_solve::{DecisionFormatter, ResolvoSolver, SolverExt, SolverMut}; +use spk_solve::{DecisionFormatter, SolverExt, SolverMut}; use spk_storage as storage; use super::Tester; -pub struct PackageSourceTester { +pub struct PackageSourceTester +where + Solver: Send, +{ prefix: PathBuf, recipe: SpecRecipe, script: String, repos: Vec>, + solver: Solver, options: OptionMap, additional_requirements: Vec, source: Option, env_formatter: DecisionFormatter, } -impl PackageSourceTester { - pub fn new(recipe: SpecRecipe, script: String) -> Self { +impl PackageSourceTester +where + Solver: SolverExt + SolverMut + Send, +{ + pub fn new(recipe: SpecRecipe, script: String, solver: Solver) -> Self { Self { prefix: PathBuf::from("/spfs"), recipe, script, repos: Vec::new(), + solver, options: OptionMap::default(), additional_requirements: Vec::new(), source: None, @@ -84,11 +92,10 @@ impl PackageSourceTester { let requires_localization = rt.config.mount_backend.requires_localization(); - let mut solver = ResolvoSolver::default(); - solver.set_binary_only(true); - solver.update_options(self.options.clone()); + self.solver.set_binary_only(true); + self.solver.update_options(self.options.clone()); for repo in self.repos.iter().cloned() { - solver.add_repository(repo); + self.solver.add_repository(repo); } if self.source.is_none() { @@ -101,15 +108,18 @@ impl PackageSourceTester { .with_prerelease(Some(PreReleasePolicy::IncludeAll)) .with_pin(None) .with_compat(None); - solver.add_request(request.into()); + self.solver.add_request(request.into()); } for request in self.additional_requirements.drain(..) { - solver.add_request(request) + self.solver.add_request(request) } // let (solution, _) = self.env_resolver.solve(&solver).await?; - let solution = solver.run_and_print_resolve(&self.env_formatter).await?; + let solution = self + .solver + .run_and_print_resolve(&self.env_formatter) + .await?; for layer in resolve_runtime_layers(requires_localization, &solution).await? { rt.push_digest(layer); @@ -130,7 +140,10 @@ impl PackageSourceTester { } #[async_trait::async_trait] -impl Tester for PackageSourceTester { +impl Tester for PackageSourceTester +where + Solver: SolverExt + SolverMut + Send, +{ async fn test(&mut self) -> Result<()> { self.test().await } From eae5253c82266b6165d69dddd1386e17db8d6560 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Tue, 11 Feb 2025 23:28:13 -0800 Subject: [PATCH 51/64] Change these calls to mimic the one in install.rs However, this can still become a recursive call that blows out the stack if the bounds on Solver don't fit the bounds on the "real" `test` implementation. Signed-off-by: J Robert Ray --- crates/spk-cli/cmd-test/src/test/build.rs | 2 +- crates/spk-cli/cmd-test/src/test/sources.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/spk-cli/cmd-test/src/test/build.rs b/crates/spk-cli/cmd-test/src/test/build.rs index e950abba43..8f192cb95f 100644 --- a/crates/spk-cli/cmd-test/src/test/build.rs +++ b/crates/spk-cli/cmd-test/src/test/build.rs @@ -183,7 +183,7 @@ where Solver: SolverExt + SolverMut + Default + Send, { async fn test(&mut self) -> Result<()> { - self.test().await + PackageBuildTester::test(self).await } fn prefix(&self) -> &Path { &self.prefix diff --git a/crates/spk-cli/cmd-test/src/test/sources.rs b/crates/spk-cli/cmd-test/src/test/sources.rs index e77b17a7ab..24272c08e5 100644 --- a/crates/spk-cli/cmd-test/src/test/sources.rs +++ b/crates/spk-cli/cmd-test/src/test/sources.rs @@ -145,7 +145,7 @@ where Solver: SolverExt + SolverMut + Send, { async fn test(&mut self) -> Result<()> { - self.test().await + PackageSourceTester::test(self).await } fn prefix(&self) -> &Path { &self.prefix From 68a685faadd8c7456da661de0af74c72eed5cfc3 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Wed, 12 Feb 2025 10:47:00 -0800 Subject: [PATCH 52/64] Add `flags::DecisionFormatterSettings` to `flags::Solver` In order for `flags::Solver::get_solver` to offer a choice in what solver is returned, it needs to have access to the `solver_to_run` setting in `flags::DecisionFormatterSettings`. This change continues the theme of changing from a DecisionFormatter "containing" a solver to a solver "containing" a DecisionFormatter. Signed-off-by: J Robert Ray --- crates/spk-cli/cmd-build/src/cmd_build.rs | 4 ---- crates/spk-cli/cmd-convert/src/cmd_convert.rs | 4 ---- crates/spk-cli/cmd-env/src/cmd_env.rs | 8 +++---- crates/spk-cli/cmd-explain/src/cmd_explain.rs | 6 ++--- crates/spk-cli/cmd-install/src/cmd_install.rs | 8 +++---- .../cmd-make-binary/src/cmd_make_binary.rs | 6 ++--- crates/spk-cli/cmd-render/src/cmd_render.rs | 8 +++---- crates/spk-cli/cmd-test/src/cmd_test.rs | 6 ++--- crates/spk-cli/common/src/flags.rs | 3 +++ crates/spk-cli/common/src/flags_test.rs | 22 +++++++++++++++++++ crates/spk-cli/group1/src/cmd_bake.rs | 8 +++---- crates/spk-cli/group4/src/cmd_view.rs | 8 +++---- 12 files changed, 51 insertions(+), 40 deletions(-) diff --git a/crates/spk-cli/cmd-build/src/cmd_build.rs b/crates/spk-cli/cmd-build/src/cmd_build.rs index 652d34baca..234f7f2ef0 100644 --- a/crates/spk-cli/cmd-build/src/cmd_build.rs +++ b/crates/spk-cli/cmd-build/src/cmd_build.rs @@ -44,9 +44,6 @@ pub struct Build { #[clap(flatten)] variant: flags::Variant, - #[clap(flatten)] - pub formatter_settings: flags::DecisionFormatterSettings, - /// Allow dependencies of the package being built to have a dependency on /// this package. #[clap(long)] @@ -114,7 +111,6 @@ impl Run for Build { env: self.env, packages, variant: self.variant.clone(), - formatter_settings: self.formatter_settings.clone(), allow_circular_dependencies: self.allow_circular_dependencies, created_builds: spk_cli_common::BuildResult::default(), }; diff --git a/crates/spk-cli/cmd-convert/src/cmd_convert.rs b/crates/spk-cli/cmd-convert/src/cmd_convert.rs index 06959873f6..73650f8e62 100644 --- a/crates/spk-cli/cmd-convert/src/cmd_convert.rs +++ b/crates/spk-cli/cmd-convert/src/cmd_convert.rs @@ -22,9 +22,6 @@ pub struct Convert { #[clap(short, long, global = true, action = clap::ArgAction::Count)] pub verbose: u8, - #[clap(flatten)] - pub formatter_settings: flags::DecisionFormatterSettings, - /// Options for showing progress #[clap(long, value_enum)] pub progress: Option, @@ -61,7 +58,6 @@ impl Run for Convert { runtime: self.runtime.clone(), requests: self.requests.clone(), verbose: self.verbose, - formatter_settings: self.formatter_settings.clone(), progress: self.progress, requested: vec![converter_package], command, diff --git a/crates/spk-cli/cmd-env/src/cmd_env.rs b/crates/spk-cli/cmd-env/src/cmd_env.rs index 16915ec93d..83b0f25750 100644 --- a/crates/spk-cli/cmd-env/src/cmd_env.rs +++ b/crates/spk-cli/cmd-env/src/cmd_env.rs @@ -34,9 +34,6 @@ pub struct Env { #[clap(short, long, global = true, action = clap::ArgAction::Count)] pub verbose: u8, - #[clap(flatten)] - pub formatter_settings: flags::DecisionFormatterSettings, - /// The requests to resolve and run #[clap(name = "REQUESTS")] pub requested: Vec, @@ -87,7 +84,10 @@ impl Run for Env { solver.add_request(request) } - let formatter = self.formatter_settings.get_formatter(self.verbose)?; + let formatter = self + .solver + .decision_formatter_settings + .get_formatter(self.verbose)?; let solution = solver.run_and_print_resolve(&formatter).await?; let solution = build_required_packages(&solution, solver).await?; diff --git a/crates/spk-cli/cmd-explain/src/cmd_explain.rs b/crates/spk-cli/cmd-explain/src/cmd_explain.rs index 2027bd1304..84de847414 100644 --- a/crates/spk-cli/cmd-explain/src/cmd_explain.rs +++ b/crates/spk-cli/cmd-explain/src/cmd_explain.rs @@ -20,9 +20,6 @@ pub struct Explain { #[clap(short, long, global = true, action = clap::ArgAction::Count)] pub verbose: u8, - #[clap(flatten)] - pub formatter_settings: flags::DecisionFormatterSettings, - /// The requests to resolve #[clap(name = "REQUESTS", required = true)] pub requested: Vec, @@ -82,7 +79,8 @@ impl Run for Explain { // Always show the solution packages for the solve let formatter = self - .formatter_settings + .solver + .decision_formatter_settings .get_formatter_builder(self.verbose + 1)? .with_solution(true) .build(); diff --git a/crates/spk-cli/cmd-install/src/cmd_install.rs b/crates/spk-cli/cmd-install/src/cmd_install.rs index e1c3285edb..b3a53ad49a 100644 --- a/crates/spk-cli/cmd-install/src/cmd_install.rs +++ b/crates/spk-cli/cmd-install/src/cmd_install.rs @@ -33,9 +33,6 @@ pub struct Install { #[clap(long, short)] yes: bool, - #[clap(flatten)] - pub formatter_settings: flags::DecisionFormatterSettings, - /// The packages to install #[clap(name = "PKG", required = true)] pub packages: Vec, @@ -63,7 +60,10 @@ impl Run for Install { solver.add_request(request); } - let formatter = self.formatter_settings.get_formatter(self.verbose)?; + let formatter = self + .solver + .decision_formatter_settings + .get_formatter(self.verbose)?; let solution = solver.run_and_print_resolve(&formatter).await?; println!("The following packages will be installed:\n"); diff --git a/crates/spk-cli/cmd-make-binary/src/cmd_make_binary.rs b/crates/spk-cli/cmd-make-binary/src/cmd_make_binary.rs index ce010fd612..78bdadf124 100644 --- a/crates/spk-cli/cmd-make-binary/src/cmd_make_binary.rs +++ b/crates/spk-cli/cmd-make-binary/src/cmd_make_binary.rs @@ -54,9 +54,6 @@ pub struct MakeBinary { #[clap(flatten)] pub variant: flags::Variant, - #[clap(flatten)] - pub formatter_settings: flags::DecisionFormatterSettings, - /// Allow dependencies of the package being built to have a dependency on /// this package. #[clap(long)] @@ -164,7 +161,8 @@ impl Run for MakeBinary { // Always show the solution packages for the solves let mut fmt_builder = self - .formatter_settings + .solver + .decision_formatter_settings .get_formatter_builder(self.verbose)?; let src_formatter = fmt_builder .with_solution(true) diff --git a/crates/spk-cli/cmd-render/src/cmd_render.rs b/crates/spk-cli/cmd-render/src/cmd_render.rs index 60b133be90..850e03ea91 100644 --- a/crates/spk-cli/cmd-render/src/cmd_render.rs +++ b/crates/spk-cli/cmd-render/src/cmd_render.rs @@ -24,9 +24,6 @@ pub struct Render { #[clap(short, long, global = true, action = clap::ArgAction::Count)] pub verbose: u8, - #[clap(flatten)] - pub formatter_settings: flags::DecisionFormatterSettings, - /// The packages to resolve and render #[clap(name = "PKG", required = true)] packages: Vec, @@ -52,7 +49,10 @@ impl Run for Render { solver.add_request(name); } - let formatter = self.formatter_settings.get_formatter(self.verbose)?; + let formatter = self + .solver + .decision_formatter_settings + .get_formatter(self.verbose)?; let solution = solver.run_and_print_resolve(&formatter).await?; let solution = build_required_packages(&solution, solver.clone()).await?; diff --git a/crates/spk-cli/cmd-test/src/cmd_test.rs b/crates/spk-cli/cmd-test/src/cmd_test.rs index cfdd3b432a..af0e62f5fc 100644 --- a/crates/spk-cli/cmd-test/src/cmd_test.rs +++ b/crates/spk-cli/cmd-test/src/cmd_test.rs @@ -39,9 +39,6 @@ pub struct CmdTest { #[clap(short, long, global = true, action = clap::ArgAction::Count)] pub verbose: u8, - #[clap(flatten)] - pub formatter_settings: flags::DecisionFormatterSettings, - /// Test in the current directory, instead of the source package /// /// This is mostly relevant when testing source and build stages @@ -158,7 +155,8 @@ impl Run for CmdTest { ); for (index, test) in selected.into_iter().enumerate() { let mut builder = self - .formatter_settings + .solver + .decision_formatter_settings .get_formatter_builder(self.verbose)?; let src_formatter = builder.with_header("Source Resolver ").build(); let build_src_formatter = diff --git a/crates/spk-cli/common/src/flags.rs b/crates/spk-cli/common/src/flags.rs index 611bf85f9b..34856c4511 100644 --- a/crates/spk-cli/common/src/flags.rs +++ b/crates/spk-cli/common/src/flags.rs @@ -234,6 +234,9 @@ pub struct Solver { #[clap(flatten)] pub repos: Repositories, + #[clap(flatten)] + pub decision_formatter_settings: DecisionFormatterSettings, + /// If true, build packages from source if needed #[clap(long)] pub allow_builds: bool, diff --git a/crates/spk-cli/common/src/flags_test.rs b/crates/spk-cli/common/src/flags_test.rs index ec308803d4..9bf4dce553 100644 --- a/crates/spk-cli/common/src/flags_test.rs +++ b/crates/spk-cli/common/src/flags_test.rs @@ -9,6 +9,8 @@ use spk_schema::ident::VarRequest; use spk_schema::option_map::HOST_OPTIONS; use spk_solve::Solver; +use crate::flags::{DecisionFormatterSettings, SolverToRun, SolverToShow}; + #[rstest] #[case(&["hello:world"], &[("hello", "world")])] #[case(&["hello=world"], &[("hello", "world")])] @@ -59,6 +61,26 @@ async fn test_get_solver_with_host_options(#[values(true, false)] no_host: bool) when: None, legacy_spk_version_tags: false, }, + decision_formatter_settings: DecisionFormatterSettings { + time: Default::default(), + increase_verbosity: Default::default(), + max_verbosity_increase_level: Default::default(), + timeout: Default::default(), + show_solution: Default::default(), + long_solves: Default::default(), + max_frequent_errors: Default::default(), + status_bar: Default::default(), + solver_to_run: SolverToRun::Cli, + solver_to_show: SolverToShow::Cli, + show_search_size: Default::default(), + compare_solvers: Default::default(), + stop_on_block: Default::default(), + step_on_block: Default::default(), + step_on_decision: Default::default(), + output_to_dir: Default::default(), + output_to_dir_min_verbosity: Default::default(), + output_file_prefix: Default::default(), + }, allow_builds: false, check_impossible_initial: false, check_impossible_validation: false, diff --git a/crates/spk-cli/group1/src/cmd_bake.rs b/crates/spk-cli/group1/src/cmd_bake.rs index 72e3512ff0..9612825671 100644 --- a/crates/spk-cli/group1/src/cmd_bake.rs +++ b/crates/spk-cli/group1/src/cmd_bake.rs @@ -64,9 +64,6 @@ pub struct Bake { #[clap(short, long, global = true, action = clap::ArgAction::Count)] pub verbose: u8, - #[clap(flatten)] - pub formatter_settings: flags::DecisionFormatterSettings, - /// The requests to resolve and bake #[clap(name = "REQUESTS")] pub requested: Vec, @@ -238,7 +235,10 @@ impl Bake { solver.add_request(request) } - let formatter = self.formatter_settings.get_formatter(self.verbose)?; + let formatter = self + .solver + .decision_formatter_settings + .get_formatter(self.verbose)?; let solution = solver.run_and_print_resolve(&formatter).await?; // The solution order is the order things were found during diff --git a/crates/spk-cli/group4/src/cmd_view.rs b/crates/spk-cli/group4/src/cmd_view.rs index 0ca00e0133..3d169973f6 100644 --- a/crates/spk-cli/group4/src/cmd_view.rs +++ b/crates/spk-cli/group4/src/cmd_view.rs @@ -90,9 +90,6 @@ pub struct View { #[clap(short = 'f', long)] pub format: Option, - #[clap(flatten)] - pub formatter_settings: flags::DecisionFormatterSettings, - /// Explicitly get info on a filepath #[clap(short = 'F', long)] filepath: Option, @@ -653,7 +650,10 @@ impl View { (&solver as &dyn Any).downcast_ref::() { let mut runtime = solver.run(); - let formatter = self.formatter_settings.get_formatter(self.verbose)?; + let formatter = self + .solver + .decision_formatter_settings + .get_formatter(self.verbose)?; let result = formatter.run_and_print_decisions(&mut runtime).await; match result { Ok((s, _)) => s, From 8a8ef8dabef8fab6e116844981947f33d20e11ff Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Wed, 12 Feb 2025 12:13:20 -0800 Subject: [PATCH 53/64] Make it possible to choose resolvo with --solver-to-run Signed-off-by: J Robert Ray --- Cargo.lock | 1 + crates/spk-cli/cmd-test/src/test/build.rs | 24 ++++++------ crates/spk-cli/common/src/exec.rs | 6 +-- crates/spk-cli/common/src/flags.rs | 47 +++++++++++++++-------- crates/spk-solve/Cargo.toml | 1 + crates/spk-solve/src/io.rs | 26 ++++++------- crates/spk-solve/src/solver.rs | 3 ++ 7 files changed, 65 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a459d81d1c..06cea3b940 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4544,6 +4544,7 @@ dependencies = [ "spk-storage", "statsd 0.15.0", "strip-ansi-escapes 0.2.0", + "strum", "tap", "thiserror", "tokio", diff --git a/crates/spk-cli/cmd-test/src/test/build.rs b/crates/spk-cli/cmd-test/src/test/build.rs index 8f192cb95f..52262ad2ec 100644 --- a/crates/spk-cli/cmd-test/src/test/build.rs +++ b/crates/spk-cli/cmd-test/src/test/build.rs @@ -38,7 +38,7 @@ where impl PackageBuildTester where - Solver: SolverExt + SolverMut + Default + Send, + Solver: SolverExt + SolverMut + Clone + Send, { pub fn new(recipe: SpecRecipe, script: String, solver: Solver) -> Self { let source = @@ -109,22 +109,22 @@ where } } - self.solver.set_binary_only(true); - self.solver.update_options(self.options.clone()); + // Use a clone of the solver before changing settings so + // `resolve_source_package` can do the same. + let mut solver = self.solver.clone(); + solver.set_binary_only(true); + solver.update_options(self.options.clone()); for repo in self.repos.iter().cloned() { - self.solver.add_repository(repo); + solver.add_repository(repo); } // TODO // solver.configure_for_build_environment(&self.recipe)?; for request in self.additional_requirements.drain(..) { - self.solver.add_request(request) + solver.add_request(request) } // let (solution, _) = self.build_resolver.solve(&solver).await?; - let solution = self - .solver - .run_and_print_resolve(&self.build_formatter) - .await?; + let solution = solver.run_and_print_resolve(&self.build_formatter).await?; for layer in resolve_runtime_layers(requires_localization, &solution).await? { rt.push_digest(layer); @@ -150,7 +150,9 @@ where } async fn resolve_source_package(&mut self, package: &AnyIdent) -> Result { - let mut solver = Solver::default(); + // Use a clone of the solver before changing settings so `test` can do + // the same. + let mut solver = self.solver.clone(); solver.update_options(self.options.clone()); let local_repo: Arc = Arc::new(storage::local_repository().await?.into()); @@ -180,7 +182,7 @@ where #[async_trait::async_trait] impl Tester for PackageBuildTester where - Solver: SolverExt + SolverMut + Default + Send, + Solver: SolverExt + SolverMut + Clone + Send, { async fn test(&mut self) -> Result<()> { PackageBuildTester::test(self).await diff --git a/crates/spk-cli/common/src/exec.rs b/crates/spk-cli/common/src/exec.rs index 09650e4a47..f753b6f7a0 100644 --- a/crates/spk-cli/common/src/exec.rs +++ b/crates/spk-cli/common/src/exec.rs @@ -18,10 +18,10 @@ use crate::Result; /// Returns a new solution of only binary packages. pub async fn build_required_packages( solution: &Solution, - _solver: Solver, + solver: Solver, ) -> Result where - Solver: SolverExt + SolverMut + Default, + Solver: SolverExt + SolverMut + Clone, { let handle: storage::RepositoryHandle = storage::local_repository().await?.into(); let local_repo = Arc::new(handle); @@ -43,7 +43,7 @@ where options.format_option_map() ); let (package, components) = - BinaryPackageBuilder::from_recipe_with_solver((**recipe).clone(), Solver::default()) + BinaryPackageBuilder::from_recipe_with_solver((**recipe).clone(), solver.clone()) .with_repositories(repos.clone()) .build_and_publish(&options, &*local_repo) .await?; diff --git a/crates/spk-cli/common/src/flags.rs b/crates/spk-cli/common/src/flags.rs index 34856c4511..724c6f98af 100644 --- a/crates/spk-cli/common/src/flags.rs +++ b/crates/spk-cli/common/src/flags.rs @@ -16,6 +16,7 @@ use solve::{ DecisionFormatterBuilder, MultiSolverKind, SolverExt, + SolverImpl, SolverMut, }; use spk_schema::foundation::format::FormatIdent; @@ -265,11 +266,27 @@ impl Solver { pub async fn get_solver( &self, options: &Options, - ) -> Result { + ) -> Result { let option_map = options.get_options()?; - //let mut solver = solve::StepSolver::default(); - let mut solver = solve::ResolvoSolver::default(); + let mut solver = match self.decision_formatter_settings.solver_to_run { + SolverToRun::Resolvo => SolverImpl::Resolvo(solve::ResolvoSolver::default()), + _ => { + let mut solver = solve::StepSolver::default(); + // These settings are only applicable to the Step solver. + solver.set_initial_request_impossible_checks( + self.check_impossible_initial || self.check_impossible_all, + ); + solver.set_resolve_validation_impossible_checks( + self.check_impossible_validation || self.check_impossible_all, + ); + solver.set_build_key_impossible_checks( + self.check_impossible_builds || self.check_impossible_all, + ); + SolverImpl::Step(solver) + } + }; + solver.update_options(option_map); for (name, repo) in self.repos.get_repos_for_non_destructive_operation().await? { @@ -277,15 +294,6 @@ impl Solver { solver.add_repository(repo); } solver.set_binary_only(!self.allow_builds); - //solver.set_initial_request_impossible_checks( - // self.check_impossible_initial || self.check_impossible_all, - //); - //solver.set_resolve_validation_impossible_checks( - // self.check_impossible_validation || self.check_impossible_all, - //); - //solver.set_build_key_impossible_checks( - // self.check_impossible_builds || self.check_impossible_all, - //); for r in options.get_var_requests()? { solver.add_request(r.into()); @@ -1181,17 +1189,22 @@ pub enum SolverToRun { Cli, /// Run and show output from the "impossible requests" checking solver Checks, - /// Run both solvers, showing the output from the basic solver, - /// unless overridden with --solver-to-run + /// Run both "cli" and "checks" solvers, showing the output from the "cli" + /// solver, unless overridden with --solver-to-run All, + /// Run the Resolvo-based SAT solver + Resolvo, } #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] pub enum SolverToShow { - /// Show output from the basic solver + /// Show output from the basic solver. Cli, - /// Show output from the "impossible requests" checking solver + /// Show output from the "impossible requests" checking solver. Checks, + /// Show output from the Resolvo SAT solver. This is only possible when + /// running with that solver. + Resolvo, } impl From for MultiSolverKind { @@ -1200,6 +1213,7 @@ impl From for MultiSolverKind { SolverToRun::Cli => MultiSolverKind::Unchanged, SolverToRun::Checks => MultiSolverKind::AllImpossibleChecks, SolverToRun::All => MultiSolverKind::All, + SolverToRun::Resolvo => MultiSolverKind::Resolvo, } } } @@ -1209,6 +1223,7 @@ impl From for MultiSolverKind { match item { SolverToShow::Cli => MultiSolverKind::Unchanged, SolverToShow::Checks => MultiSolverKind::AllImpossibleChecks, + SolverToShow::Resolvo => MultiSolverKind::Resolvo, } } } diff --git a/crates/spk-solve/Cargo.toml b/crates/spk-solve/Cargo.toml index f88dbbf4e4..8b6be31bcf 100644 --- a/crates/spk-solve/Cargo.toml +++ b/crates/spk-solve/Cargo.toml @@ -56,6 +56,7 @@ spk-solve-solution = { workspace = true } spk-solve-validation = { workspace = true } spk-storage = { workspace = true } statsd = { version = "0.15.0", optional = true } +strum = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt"] } tracing = { workspace = true } diff --git a/crates/spk-solve/src/io.rs b/crates/spk-solve/src/io.rs index f230b58629..1c95495d49 100644 --- a/crates/spk-solve/src/io.rs +++ b/crates/spk-solve/src/io.rs @@ -4,7 +4,7 @@ use std::cmp::max; use std::collections::VecDeque; -use std::fmt::{Display, Write}; +use std::fmt::Write; use std::fs::{File, OpenOptions}; use std::io::{BufWriter, ErrorKind, Write as IOWrite}; use std::path::{Path, PathBuf}; @@ -59,6 +59,7 @@ const BY_USER: &str = "by user"; const CLI_SOLVER: &str = "cli"; const IMPOSSIBLE_CHECKS_SOLVER: &str = "checks"; const ALL_SOLVERS: &str = "all"; +const RESOLVO_SOLVER: &str = "resolvo"; const UNABLE_TO_GET_OUTPUT_FILE_LOCK: &str = "Unable to get lock to write solver output to file"; const UNABLE_TO_WRITE_OUTPUT_MESSAGE: &str = "Unable to write solver output message to file"; @@ -1033,14 +1034,22 @@ enum LoopOutcome { Success, } -#[derive(PartialEq, Eq, Clone, Debug, Default)] +#[derive(PartialEq, Eq, Clone, Debug, Default, strum::Display)] pub enum MultiSolverKind { + #[strum(to_string = "Unchanged")] Unchanged, + #[strum(to_string = "All Impossible Checks")] AllImpossibleChecks, // This isn't a solver on its own. It indicates: the run all the // solvers in parallel but show the output from the unchanged one. + // This runs all the solvers implemented in the original solver. At least + // for now, it is not possible to run both the original solver and the + // new solver in parallel. #[default] + #[strum(to_string = "All")] All, + #[strum(to_string = "Resolvo")] + Resolvo, } impl MultiSolverKind { @@ -1055,6 +1064,7 @@ impl MultiSolverKind { MultiSolverKind::Unchanged => CLI_SOLVER, MultiSolverKind::AllImpossibleChecks => IMPOSSIBLE_CHECKS_SOLVER, MultiSolverKind::All => ALL_SOLVERS, + MultiSolverKind::Resolvo => RESOLVO_SOLVER, } } @@ -1082,17 +1092,6 @@ impl MultiSolverKind { } } -impl Display for MultiSolverKind { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let name = match self { - MultiSolverKind::Unchanged => "Unchanged", - MultiSolverKind::AllImpossibleChecks => "All Impossible Checks", - MultiSolverKind::All => "All", - }; - write!(f, "{name}") - } -} - struct SolverTaskSettings { solver: StepSolver, solver_kind: MultiSolverKind, @@ -1228,6 +1227,7 @@ impl DecisionFormatter { ignore_failure: false, }, ]), + MultiSolverKind::Resolvo => unreachable!(), } } diff --git a/crates/spk-solve/src/solver.rs b/crates/spk-solve/src/solver.rs index beceffe0ad..4e912ee17f 100644 --- a/crates/spk-solve/src/solver.rs +++ b/crates/spk-solve/src/solver.rs @@ -14,6 +14,9 @@ use variantly::Variantly; use crate::{DecisionFormatter, Result}; #[enum_dispatch(Solver, SolverExt, SolverMut)] +// Don't derive Default. If some code is generic on Solver and is given one of +// these, if it wants a "default" solver it needs to be given a new solver of +// the same variety and `SolverImpl::default()` can't do that. #[derive(Clone, Variantly)] pub enum SolverImpl { Step(crate::StepSolver), From 018408f331e68e0db00b787ccfc3a8be0b7c0f1e Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Thu, 13 Feb 2025 14:00:28 -0800 Subject: [PATCH 54/64] Remove unnecessary clone `key_entry_names` is unused after it is cloned into `ordered_names`. Signed-off-by: J Robert Ray --- .../crates/package-iterator/src/package_iterator.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/spk-solve/crates/package-iterator/src/package_iterator.rs b/crates/spk-solve/crates/package-iterator/src/package_iterator.rs index 3d4f1dd76d..b1e767f808 100644 --- a/crates/spk-solve/crates/package-iterator/src/package_iterator.rs +++ b/crates/spk-solve/crates/package-iterator/src/package_iterator.rs @@ -609,8 +609,7 @@ impl SortedBuildIterator { // names should be added to the configuration // BUILD_KEY_NAME_ORDER to ensure they fall in the correct // position for a site's spk setup. - let mut ordered_names = key_entry_names.clone(); - BUILD_KEY_NAME_ORDER.promote_names(ordered_names.as_mut_slice(), |n| n); + BUILD_KEY_NAME_ORDER.promote_names(key_entry_names.as_mut_slice(), |n| n); // Sort the builds by their generated keys generated from the // ordered names and values worth including. @@ -619,7 +618,7 @@ impl SortedBuildIterator { let spec = &hm.iter().next().expect("non-empty hashmap").1.0; SortedBuildIterator::make_option_values_build_key( spec, - &ordered_names, + &key_entry_names, &build_name_values, builds_with_impossible_requests.contains_key(&spec.ident().clone()), ) @@ -641,7 +640,7 @@ impl SortedBuildIterator { tracing::debug!( target: BUILD_SORT_TARGET, "Keys by build option values: built from: [{}]", - ordered_names + key_entry_names .iter() .map(|n| n.as_str()) .collect::>() @@ -659,7 +658,7 @@ impl SortedBuildIterator { spec.ident(), SortedBuildIterator::make_option_values_build_key( spec, - &ordered_names, + &key_entry_names, &build_name_values, builds_with_impossible_requests.contains_key(&spec.ident().clone()), ), From 28b1d0d5c59b06276dcf6bb2bf64764c23b46713 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Thu, 13 Feb 2025 14:01:19 -0800 Subject: [PATCH 55/64] Refactor some build key logic from SortedBuildIterator In order to reuse this logic for the resolvo solver, extract it to an external function. It is not practical for the resolvo solver to directly use a SortedBuildIterator. Signed-off-by: J Robert Ray --- .../package-iterator/src/package_iterator.rs | 130 +++++++++++------- 1 file changed, 77 insertions(+), 53 deletions(-) diff --git a/crates/spk-solve/crates/package-iterator/src/package_iterator.rs b/crates/spk-solve/crates/package-iterator/src/package_iterator.rs index b1e767f808..fcdd691de7 100644 --- a/crates/spk-solve/crates/package-iterator/src/package_iterator.rs +++ b/crates/spk-solve/crates/package-iterator/src/package_iterator.rs @@ -468,64 +468,17 @@ struct ChangeCounter { pub use_it: bool, } -impl SortedBuildIterator { - pub async fn new( - _options: OptionMap, - source: Arc>, - builds_with_impossible_requests: HashMap, - ) -> Result { - // Note: _options is unused in this implementation, it was used - // in the by_distance sorting implementation - let mut builds = VecDeque::::new(); - { - let mut source_lock = source.lock().await; - while let Some(item) = source_lock.next().await? { - builds.push_back(item); - } - } - - let mut sbi = SortedBuildIterator { builds }; - - sbi.sort_by_build_option_values(builds_with_impossible_requests) - .await; - Ok(sbi) - } - - /// Helper for making BuildKey structures used in the sorting in - /// sort_by_build_option_values() below - fn make_option_values_build_key( - spec: &Spec, - ordered_names: &Vec, - build_name_values: &HashMap, - makes_an_impossible_request: bool, - ) -> BuildKey { - let build_id = spec.ident(); - let empty = OptionMap::default(); - let name_values = match build_name_values.get(build_id) { - Some(nv) => nv, - None => &empty, - }; - BuildKey::new( - spec.ident(), - ordered_names, - name_values, - makes_an_impossible_request, - ) - } - - /// Sorts builds by keys based on ordered build option names and - /// differing values in those options - async fn sort_by_build_option_values( - &mut self, - builds_with_impossible_requests: HashMap, - ) { - let start = Instant::now(); +pub struct BuildToSortedOptName {} +impl BuildToSortedOptName { + pub fn sort_builds<'a>( + builds: impl Iterator, PackageSource)>, + ) -> (Vec, HashMap) { let mut number_non_src_builds: u64 = 0; let mut build_name_values: HashMap = HashMap::default(); let mut changes: HashMap = HashMap::new(); - for (build, _) in self.builds.iter().flat_map(|hm| hm.values()) { + for (build, _) in builds { // Skip this if it's a '/src' build because '/src' builds // won't use the build option values in their key, they // don't need to be looked at. They have a type of key @@ -611,6 +564,77 @@ impl SortedBuildIterator { // position for a site's spk setup. BUILD_KEY_NAME_ORDER.promote_names(key_entry_names.as_mut_slice(), |n| n); + (key_entry_names, build_name_values) + } +} + +impl SortedBuildIterator { + pub async fn new( + _options: OptionMap, + source: Arc>, + builds_with_impossible_requests: HashMap, + ) -> Result { + // Note: _options is unused in this implementation, it was used + // in the by_distance sorting implementation + let mut builds = VecDeque::::new(); + { + let mut source_lock = source.lock().await; + while let Some(item) = source_lock.next().await? { + builds.push_back(item); + } + } + + let mut sbi = SortedBuildIterator { builds }; + + sbi.sort_by_build_option_values(builds_with_impossible_requests) + .await; + Ok(sbi) + } + + pub async fn new_from_builds( + builds: VecDeque, + builds_with_impossible_requests: HashMap, + ) -> Result { + let mut sbi = SortedBuildIterator { builds }; + + sbi.sort_by_build_option_values(builds_with_impossible_requests) + .await; + Ok(sbi) + } + + /// Helper for making BuildKey structures used in the sorting in + /// sort_by_build_option_values() below + fn make_option_values_build_key( + spec: &Spec, + ordered_names: &Vec, + build_name_values: &HashMap, + makes_an_impossible_request: bool, + ) -> BuildKey { + let build_id = spec.ident(); + let empty = OptionMap::default(); + let name_values = match build_name_values.get(build_id) { + Some(nv) => nv, + None => &empty, + }; + BuildKey::new( + spec.ident(), + ordered_names, + name_values, + makes_an_impossible_request, + ) + } + + /// Sorts builds by keys based on ordered build option names and + /// differing values in those options + async fn sort_by_build_option_values( + &mut self, + builds_with_impossible_requests: HashMap, + ) { + let start = Instant::now(); + + let (key_entry_names, build_name_values) = + BuildToSortedOptName::sort_builds(self.builds.iter().flat_map(|hm| hm.values())); + // Sort the builds by their generated keys generated from the // ordered names and values worth including. self.builds.make_contiguous().sort_by_cached_key(|hm| { From 0bdcfdf2eb0bed5536b26704cf3881207d78c58d Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Thu, 13 Feb 2025 14:22:19 -0800 Subject: [PATCH 56/64] Do a reverse sort using std::cmp::Reverse Potential speedup by avoiding doing the reversal as a separate step. Signed-off-by: J Robert Ray --- .../package-iterator/src/package_iterator.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/crates/spk-solve/crates/package-iterator/src/package_iterator.rs b/crates/spk-solve/crates/package-iterator/src/package_iterator.rs index fcdd691de7..be9d2750c1 100644 --- a/crates/spk-solve/crates/package-iterator/src/package_iterator.rs +++ b/crates/spk-solve/crates/package-iterator/src/package_iterator.rs @@ -640,20 +640,18 @@ impl SortedBuildIterator { self.builds.make_contiguous().sort_by_cached_key(|hm| { // Pull an arbitrary spec out from the hashmap let spec = &hm.iter().next().expect("non-empty hashmap").1.0; - SortedBuildIterator::make_option_values_build_key( + // Reverse the sort to get the build with the highest + // "numbers" in the earlier parts of its key to come first, + // which also reverse sorts the text values, i.e. "on" will + // come before "off". + std::cmp::Reverse(SortedBuildIterator::make_option_values_build_key( spec, &key_entry_names, &build_name_values, builds_with_impossible_requests.contains_key(&spec.ident().clone()), - ) + )) }); - // Reverse the sort to get the build with the highest - // "numbers" in the earlier parts of its key to come first, - // which also reverse sorts the text values, i.e. "on" will - // come before "off". - self.builds.make_contiguous().reverse(); - let duration: Duration = start.elapsed(); tracing::info!( target: BUILD_SORT_TARGET, From cd1834e5a80eda08621dd1ceddc4d5a78ec9e2bd Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Thu, 13 Feb 2025 14:23:58 -0800 Subject: [PATCH 57/64] Rework build sorting in the new solver Reuse the same build sorting logic as the original solver in order to get a deterministic result out of the solve. The solve can change drastically based on which build is selected first. Signed-off-by: J Robert Ray --- .../crates/package-iterator/src/lib.rs | 2 + .../package-iterator/src/package_iterator.rs | 24 +- .../src/solvers/resolvo/spk_provider.rs | 218 ++++++++++++++++-- 3 files changed, 212 insertions(+), 32 deletions(-) diff --git a/crates/spk-solve/crates/package-iterator/src/lib.rs b/crates/spk-solve/crates/package-iterator/src/lib.rs index b81bd47433..38aa8680e7 100644 --- a/crates/spk-solve/crates/package-iterator/src/lib.rs +++ b/crates/spk-solve/crates/package-iterator/src/lib.rs @@ -7,10 +7,12 @@ mod error; mod package_iterator; mod promotion_patterns; +pub use build_key::BuildKey; pub use error::{Error, Result}; pub use package_iterator::{ BUILD_SORT_TARGET, BuildIterator, + BuildToSortedOptName, EmptyBuildIterator, PackageIterator, RepositoryPackageIterator, diff --git a/crates/spk-solve/crates/package-iterator/src/package_iterator.rs b/crates/spk-solve/crates/package-iterator/src/package_iterator.rs index be9d2750c1..15e14bcc3d 100644 --- a/crates/spk-solve/crates/package-iterator/src/package_iterator.rs +++ b/crates/spk-solve/crates/package-iterator/src/package_iterator.rs @@ -472,13 +472,13 @@ pub struct BuildToSortedOptName {} impl BuildToSortedOptName { pub fn sort_builds<'a>( - builds: impl Iterator, PackageSource)>, + builds: impl Iterator>, ) -> (Vec, HashMap) { let mut number_non_src_builds: u64 = 0; let mut build_name_values: HashMap = HashMap::default(); let mut changes: HashMap = HashMap::new(); - for (build, _) in builds { + for build in builds { // Skip this if it's a '/src' build because '/src' builds // won't use the build option values in their key, they // don't need to be looked at. They have a type of key @@ -591,20 +591,9 @@ impl SortedBuildIterator { Ok(sbi) } - pub async fn new_from_builds( - builds: VecDeque, - builds_with_impossible_requests: HashMap, - ) -> Result { - let mut sbi = SortedBuildIterator { builds }; - - sbi.sort_by_build_option_values(builds_with_impossible_requests) - .await; - Ok(sbi) - } - /// Helper for making BuildKey structures used in the sorting in /// sort_by_build_option_values() below - fn make_option_values_build_key( + pub fn make_option_values_build_key( spec: &Spec, ordered_names: &Vec, build_name_values: &HashMap, @@ -632,8 +621,11 @@ impl SortedBuildIterator { ) { let start = Instant::now(); - let (key_entry_names, build_name_values) = - BuildToSortedOptName::sort_builds(self.builds.iter().flat_map(|hm| hm.values())); + let (key_entry_names, build_name_values) = BuildToSortedOptName::sort_builds( + self.builds + .iter() + .flat_map(|hm| hm.values().map(|(spec, _src)| spec)), + ); // Sort the builds by their generated keys generated from the // ordered names and values worth including. diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index 8a8f8023a6..9a492c90ee 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -5,9 +5,11 @@ use std::borrow::Cow; use std::cell::RefCell; use std::collections::{BTreeSet, HashMap, HashSet}; +use std::ops::Not; use std::str::FromStr; use std::sync::Arc; +use itertools::Itertools; use resolvo::utils::Pool; use resolvo::{ Candidates, @@ -38,7 +40,18 @@ use spk_schema::name::{OptNameBuf, PkgNameBuf}; use spk_schema::prelude::{HasVersion, Named}; use spk_schema::version::Version; use spk_schema::version_range::{DoubleEqualsVersion, Ranged, VersionFilter, parse_version_range}; -use spk_schema::{BuildIdent, Deprecate, Opt, OptionMap, Package, Recipe, Request, VersionIdent}; +use spk_schema::{ + BuildIdent, + Deprecate, + Opt, + OptionMap, + Package, + Recipe, + Request, + Spec, + VersionIdent, +}; +use spk_solve_package_iterator::{BuildKey, BuildToSortedOptName, SortedBuildIterator}; use spk_storage::RepositoryHandle; use tracing::{Instrument, debug_span}; @@ -337,6 +350,51 @@ enum CanBuildFromSource { No(StringId), } +/// An iterator that yields slices of items that fall into the same partition. +/// +/// The partition is determined by the key function. +/// The items must be already sorted in ascending order by the key function. +struct PartitionIter<'a, I, F, K> +where + F: for<'i> Fn(&'i I) -> K, + K: PartialOrd, +{ + slice: &'a [I], + key_fn: F, +} + +impl<'a, I, F, K> PartitionIter<'a, I, F, K> +where + F: for<'i> Fn(&'i I) -> K, + K: PartialOrd, +{ + fn new(slice: &'a [I], key_fn: F) -> Self { + Self { slice, key_fn } + } +} + +impl<'a, I, F, K> Iterator for PartitionIter<'a, I, F, K> +where + F: for<'i> Fn(&'i I) -> K, + K: PartialOrd, +{ + type Item = &'a [I]; + + fn next(&mut self) -> Option { + let element = self.slice.first()?; + + // Is a binary search overkill? + let partition_key = (self.key_fn)(element); + // No need to check the first element again. + let p = + 1 + self.slice[1..].partition_point(|element| (self.key_fn)(element) <= partition_key); + + let part = &self.slice[..p]; + self.slice = &self.slice[p..]; + Some(part) + } +} + pub(crate) struct SpkProvider { pub(crate) pool: Pool, repos: Vec>, @@ -564,6 +622,21 @@ impl SpkProvider { self.cancel_solving.borrow().is_some() } + /// Return an iterator that yields slices of builds that are from the same + /// package version. + /// + /// The provided builds must already be sorted otherwise the behavior is + /// undefined. + fn find_version_runs<'a>( + builds: &'a [(SolvableId, &'a LocatedBuildIdentWithComponent, Arc)], + ) -> impl Iterator)]> + { + PartitionIter::new(builds, |(_, ident, _)| { + // partition by (name, version) ignoring repository + (ident.ident.name(), ident.ident.version()) + }) + } + fn request_to_known_dependencies(&self, requirement: &Request) -> KnownDependencies { let mut known_deps = KnownDependencies::default(); match requirement { @@ -657,6 +730,44 @@ impl SpkProvider { } } + /// Order two builds based on which should be preferred to include in a + /// solve as a candidate. + /// + /// Generally this means a build with newer dependencies is ordered first. + fn sort_builds( + &self, + build_key_index: &HashMap, + a: (SolvableId, &LocatedBuildIdentWithComponent), + b: (SolvableId, &LocatedBuildIdentWithComponent), + ) -> std::cmp::Ordering { + // This function should _not_ return `std::cmp::Ordering::Equal` unless + // `a` and `b` are the same build (in practice this function will never + // be called when that is true). + + // Embedded stubs are always ordered last. + match (a.1.ident.is_embedded(), b.1.ident.is_embedded()) { + (true, false) => return std::cmp::Ordering::Greater, + (false, true) => return std::cmp::Ordering::Less, + _ => {} + }; + + match (build_key_index.get(&a.0), build_key_index.get(&b.0)) { + (Some(a_key), Some(b_key)) => { + // BuildKey orders in reverse order from what is needed here. + return b_key.cmp(a_key); + } + (Some(_), None) => return std::cmp::Ordering::Less, + (None, Some(_)) => return std::cmp::Ordering::Greater, + _ => {} + }; + + // If neither build has a key, both packages failed to load? + // Add debug assert to see if this ever happens. + debug_assert!(false, "builds without keys"); + + a.1.ident.cmp(&b.1.ident) + } + pub fn var_requirements(&mut self, requests: &[Request]) -> Vec { self.global_var_requests.reserve(requests.len()); requests @@ -967,13 +1078,94 @@ impl DependencyProvider for SpkProvider { } async fn sort_candidates(&self, _solver: &SolverCache, solvables: &mut [SolvableId]) { - // This implementation just picks the highest version. + // Goal: Create a `BuildKey` for each build in `solvables`. + // The `BuildKey` factory needs as input the output from + // `BuildToSortedOptName::sort_builds`. + // `BuildToSortedOptName::sort_builds` needs to be fed builds from the + // same version. + // `solvables` can be builds from various versions so they need to be + // grouped by version. + let build_solvables = solvables + .iter() + .filter_map(|solvable_id| { + let solvable = self.pool.resolve_solvable(*solvable_id); + match &solvable.record { + SpkSolvable::LocatedBuildIdentWithComponent( + located_build_ident_with_component, + ) => + // sorting the source build (if any) is handled + // elsewhere; skip source builds. + { + located_build_ident_with_component + .ident + .is_source() + .not() + .then_some((*solvable_id, located_build_ident_with_component)) + } + _ => None, + } + }) + .sorted_by( + |(_, LocatedBuildIdentWithComponent { ident: a, .. }), + (_, LocatedBuildIdentWithComponent { ident: b, .. })| { + // build_solvables will be ordered by (pkg, version, build). + a.target().cmp(b.target()) + }, + ) + .collect::>(); + + // `BuildToSortedOptName::sort_builds` will need the package specs. + let mut build_solvables_and_specs = Vec::with_capacity(build_solvables.len()); + for build_solvable in build_solvables { + let (solvable_id, located_build_ident_with_component) = build_solvable; + let repo = self + .repos + .iter() + .find(|repo| { + repo.name() == located_build_ident_with_component.ident.repository_name() + }) + .expect("Expected solved package's repository to be in the list of repositories"); + let Ok(package) = repo + .read_package(located_build_ident_with_component.ident.target()) + .await + else { + // Any builds that can't load the spec will be sorted to the + // end. In most cases the package spec would already be loaded + // in cache at this point. + continue; + }; + build_solvables_and_specs.push(( + solvable_id, + located_build_ident_with_component, + package, + )); + } + + let mut build_key_index = HashMap::new(); + build_key_index.reserve(build_solvables_and_specs.len()); + + // Find runs of the same package version. + for version_run in SpkProvider::find_version_runs(&build_solvables_and_specs) { + let (ordered_names, build_name_values) = + BuildToSortedOptName::sort_builds(version_run.iter().map(|(_, _, spec)| spec)); + + for (solvable_id, _, spec) in version_run { + let build_key = SortedBuildIterator::make_option_values_build_key( + spec, + &ordered_names, + &build_name_values, + false, + ); + build_key_index.insert(*solvable_id, build_key); + } + } + // TODO: The ordering should take component names into account, so // the run component or the build component is tried first in the // appropriate situations. - solvables.sort_by(|a, b| { - let a = self.pool.resolve_solvable(*a); - let b = self.pool.resolve_solvable(*b); + solvables.sort_by(|solvable_id_a, solvable_id_b| { + let a = self.pool.resolve_solvable(*solvable_id_a); + let b = self.pool.resolve_solvable(*solvable_id_b); match (&a.record, &b.record) { ( SpkSolvable::LocatedBuildIdentWithComponent(a), @@ -1005,17 +1197,11 @@ impl DependencyProvider for SpkProvider { (_, Build::Source) => return std::cmp::Ordering::Less, _ => {} }; - // Sort embedded packges second last - match (a.ident.build(), b.ident.build()) { - (Build::Embedded(_), Build::Embedded(_)) => { - // TODO: Could perhaps sort on the parent - // package to prefer a newer parent. - std::cmp::Ordering::Equal - } - (Build::Embedded(_), _) => std::cmp::Ordering::Greater, - (_, Build::Embedded(_)) => std::cmp::Ordering::Less, - _ => std::cmp::Ordering::Equal, - } + self.sort_builds( + &build_key_index, + (*solvable_id_a, a), + (*solvable_id_b, b), + ) } ord => ord, } From a578048a4dc37cb3fbddb2eb3ecad7ecde992577 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Tue, 18 Feb 2025 17:39:05 -0800 Subject: [PATCH 58/64] Don't inject credentials for resolvo repo This is a public repo. Signed-off-by: J Robert Ray --- .site/spi/.spdev/overrides.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.site/spi/.spdev/overrides.py b/.site/spi/.spdev/overrides.py index a170a0fb63..6da5db9f4f 100644 --- a/.site/spi/.spdev/overrides.py +++ b/.site/spi/.spdev/overrides.py @@ -36,7 +36,7 @@ def inject_credentials(super_script_list: spdev.shell.Script) -> spdev.shell.Scr "1", "sed", "-i", - '"s|https://github.com|https://$GITHUB_SPFS_PULL_USERNAME:$GITHUB_SPFS_PULL_PASSWORD@github.com|"', + '"s|https://github.com/spkenv|https://$GITHUB_SPFS_PULL_USERNAME:$GITHUB_SPFS_PULL_PASSWORD@github.com/spkenv|"', ) ) From b124cf3273e46d6a66648057e0a30d63ee7dde61 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Wed, 19 Feb 2025 11:21:54 -0800 Subject: [PATCH 59/64] Convenience conversion from EmbeddedBuildSource to BuildIdent Signed-off-by: J Robert Ray --- .../foundation/src/ident_build/build.rs | 8 +++++ .../crates/foundation/src/version/mod.rs | 8 +++++ .../crates/ident/src/ident_build.rs | 30 ++++++++++++++++++- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/crates/spk-schema/crates/foundation/src/ident_build/build.rs b/crates/spk-schema/crates/foundation/src/ident_build/build.rs index fa2c0c012d..ce1da54845 100644 --- a/crates/spk-schema/crates/foundation/src/ident_build/build.rs +++ b/crates/spk-schema/crates/foundation/src/ident_build/build.rs @@ -177,6 +177,14 @@ impl std::fmt::Display for Build { } } +impl TryFrom for Build { + type Error = super::Error; + + fn try_from(value: String) -> Result { + Self::from_str(&value) + } +} + impl FromStr for Build { type Err = super::Error; diff --git a/crates/spk-schema/crates/foundation/src/version/mod.rs b/crates/spk-schema/crates/foundation/src/version/mod.rs index 6e27423822..b7ef793551 100644 --- a/crates/spk-schema/crates/foundation/src/version/mod.rs +++ b/crates/spk-schema/crates/foundation/src/version/mod.rs @@ -554,6 +554,14 @@ impl TryFrom<&str> for Version { } } +impl TryFrom for Version { + type Error = Error; + + fn try_from(value: String) -> Result { + parse_version(value) + } +} + impl FromStr for Version { type Err = Error; diff --git a/crates/spk-schema/crates/ident/src/ident_build.rs b/crates/spk-schema/crates/ident/src/ident_build.rs index f9dd39aee5..2eda97b46e 100644 --- a/crates/spk-schema/crates/ident/src/ident_build.rs +++ b/crates/spk-schema/crates/ident/src/ident_build.rs @@ -6,7 +6,7 @@ use std::fmt::Write; use std::str::FromStr; use relative_path::RelativePathBuf; -use spk_schema_foundation::ident_build::Build; +use spk_schema_foundation::ident_build::{Build, EmbeddedSourcePackage}; use spk_schema_foundation::ident_ops::parsing::IdentPartsBuf; use spk_schema_foundation::ident_ops::{MetadataPath, TagPath}; use spk_schema_foundation::name::{PkgName, PkgNameBuf, RepositoryNameBuf}; @@ -30,6 +30,34 @@ pub type BuildIdent = Ident; crate::ident_version::version_ident_methods!(BuildIdent, .base); +impl TryFrom for BuildIdent { + type Error = Error; + + fn try_from(value: EmbeddedSourcePackage) -> std::result::Result { + let IdentPartsBuf { + repository_name: _, + pkg_name, + version_str: Some(version), + build_str: Some(build), + } = value.ident + else { + return if value.ident.build_str.is_some() { + Err(Error::String( + "EmbeddedSourcePackage missing version".to_string(), + )) + } else { + Err(Error::String( + "EmbeddedSourcePackage missing build".to_string(), + )) + }; + }; + Ok(Self::new( + VersionIdent::new(pkg_name.try_into()?, version.try_into()?), + build.try_into()?, + )) + } +} + macro_rules! build_ident_methods { ($Ident:ty $(, .$($access:ident).+)?) => { impl $Ident { From cf5e258a57fdfe9aa53fb9d47325d8221e380c85 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Wed, 19 Feb 2025 11:21:54 -0800 Subject: [PATCH 60/64] Convert embedded packages properly from resolvo solve When populating the Solution object, embedded packages need to be categorized properly or else when calculating the layers for the solution it ends up with an empty blob as one of the layers, which is an error. Handle source builds in resolvo solve When not building from source. Signed-off-by: J Robert Ray --- .../cmd-build/src/cmd_build_test/mod.rs | 1 + crates/spk-solve/src/solvers/resolvo/mod.rs | 65 +++++++++++++++---- 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/crates/spk-cli/cmd-build/src/cmd_build_test/mod.rs b/crates/spk-cli/cmd-build/src/cmd_build_test/mod.rs index dab74d0e5f..05d0f1b23c 100644 --- a/crates/spk-cli/cmd-build/src/cmd_build_test/mod.rs +++ b/crates/spk-cli/cmd-build/src/cmd_build_test/mod.rs @@ -138,6 +138,7 @@ build: #[rstest] #[case::cli("cli")] #[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] async fn test_build_with_circular_dependency_allow_with_validation( tmpdir: tempfile::TempDir, diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index c1a4f3ec88..2eb4c23cc2 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -241,21 +241,58 @@ impl Solver { next_index, ); solution_adds.push((pkg_request, package, { - if located_build_ident_with_component.requires_build_from_source { - PackageSource::BuildFromSource { - recipe: repo - .read_recipe( - &located_build_ident_with_component.ident.to_version_ident(), - ) - .await?, + match located_build_ident_with_component.ident.build() { + spk_schema::ident_build::Build::Source + if located_build_ident_with_component.requires_build_from_source => + { + PackageSource::BuildFromSource { + recipe: repo + .read_recipe( + &located_build_ident_with_component.ident.to_version_ident(), + ) + .await?, + } + } + spk_schema::ident_build::Build::Source => { + // Not building this from source but just adding the + // source build to the Solution. + PackageSource::Repository { + repo: Arc::clone(repo), + // XXX: Why is this needed? + components: repo + .read_components(located_build_ident_with_component.ident.target()) + .await?, + } } - } else { - PackageSource::Repository { - repo: Arc::clone(repo), - // XXX: Why is this needed? - components: repo - .read_components(located_build_ident_with_component.ident.target()) - .await?, + spk_schema::ident_build::Build::Embedded(embedded_source) => { + match embedded_source { + spk_schema::ident_build::EmbeddedSource::Package( + embedded_source_package, + ) => { + PackageSource::Embedded { + parent: (**embedded_source_package).clone().try_into()?, + // XXX: Why is this needed? + components: repo + .read_components( + located_build_ident_with_component.ident.target(), + ) + .await? + .keys() + .cloned() + .collect(), + } + } + spk_schema::ident_build::EmbeddedSource::Unknown => todo!(), + } + } + spk_schema::ident_build::Build::BuildId(_build_id) => { + PackageSource::Repository { + repo: Arc::clone(repo), + // XXX: Why is this needed? + components: repo + .read_components(located_build_ident_with_component.ident.target()) + .await?, + } } } })); From b4f5690c76858f948c94ccec7d39b5222b63f623 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Thu, 20 Feb 2025 16:39:43 -0800 Subject: [PATCH 61/64] Resolvo: fix stubs getting added to solve without parent The case where a stub only depended on the Run component of the parent still needs to declare a dependency to bring it in. Signed-off-by: J Robert Ray --- .../src/solvers/resolvo/spk_provider.rs | 216 ++++++++++-------- 1 file changed, 119 insertions(+), 97 deletions(-) diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index 9a492c90ee..d72144a8b2 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -6,7 +6,6 @@ use std::borrow::Cow; use std::cell::RefCell; use std::collections::{BTreeSet, HashMap, HashSet}; use std::ops::Not; -use std::str::FromStr; use std::sync::Arc; use itertools::Itertools; @@ -38,7 +37,6 @@ use spk_schema::ident_build::{Build, EmbeddedSource, EmbeddedSourcePackage}; use spk_schema::ident_component::Component; use spk_schema::name::{OptNameBuf, PkgNameBuf}; use spk_schema::prelude::{HasVersion, Named}; -use spk_schema::version::Version; use spk_schema::version_range::{DoubleEqualsVersion, Ranged, VersionFilter, parse_version_range}; use spk_schema::{ BuildIdent, @@ -1448,106 +1446,130 @@ impl DependencyProvider for SpkProvider { if let Build::Embedded(EmbeddedSource::Package(parent)) = located_build_ident_with_component.ident.build() { - match actual_component { - Component::Run => { - // The Run component is the default "home" of - // embedded packages, no dependency needed in this - // case. + let parent_ident: BuildIdent = match (**parent).clone().try_into() { + Ok(ident) => ident, + Err(err) => { + let msg = self.pool.intern_string(format!( + "failed to get valid parent ident for '{}': {err}", + located_build_ident_with_component.ident + )); + return Dependencies::Unknown(msg); } - component => 'invalid_parent: { - // XXX: Do we not have a convenient way to read the - // parent package from an embedded stub ident? - let Ok(pkg_name) = PkgNameBuf::from_str(&parent.ident.pkg_name) else { - break 'invalid_parent; - }; - let Some(version_str) = parent.ident.version_str.as_ref() else { - break 'invalid_parent; - }; - let Ok(version) = Version::from_str(version_str) else { - break 'invalid_parent; - }; - let Some(build_str) = parent.ident.build_str.as_ref() else { - break 'invalid_parent; - }; - let Ok(build) = Build::from_str(build_str) else { - break 'invalid_parent; - }; - let ident = BuildIdent::new( - VersionIdent::new(pkg_name, version), - build.clone(), - ); - let Ok(parent) = repo.read_package(&ident).await else { - break 'invalid_parent; - }; - // Look through the components of the parent to see - // if one (or more?) of them embeds this component. - for parent_component in parent.components().iter() { - parent_component - .embedded - .iter() - .filter(|embedded_package| { - embedded_package.pkg.name() - == located_build_ident_with_component.ident.name() - && embedded_package - .pkg - .target() - .as_ref() - .map(|version| { - version - == located_build_ident_with_component + }; + let parent = match repo.read_package(&parent_ident).await { + Ok(spec) => spec, + Err(err) => { + let msg = self.pool.intern_string(format!( + "failed to read parent package for '{}': {err}", + located_build_ident_with_component.ident + )); + return Dependencies::Unknown(msg); + } + }; + // Look through the components of the parent to see + // if one (or more?) of them embeds this component. + let mut found = false; + for parent_component in parent.components().iter() { + parent_component + .embedded + .iter() + .filter(|embedded_package| { + embedded_package.pkg.name() + == located_build_ident_with_component.ident.name() + && embedded_package + .pkg + .target() + .as_ref() + .map(|version| { + version + == located_build_ident_with_component + .ident + .version() + }) + .unwrap_or(true) + && embedded_package.components().contains(actual_component) + }) + .for_each(|_embedded_package| { + found = true; + let dep_name = self.pool.intern_package_name( + ResolvoPackageName::PkgNameBufWithComponent( + PkgNameBufWithComponent { + name: parent_ident.name().to_owned(), + component: SyntheticComponent::Actual( + parent_component.name.clone(), + ), + }, + ), + ); + known_deps.requirements.push( + self.pool + .intern_version_set( + dep_name, + RequestVS::SpkRequest(Request::Pkg(PkgRequest::new( + RangeIdent { + repository_name: Some( + located_build_ident_with_component .ident - .version() - }) - .unwrap_or(true) - && embedded_package.components().contains(component) - }) - .for_each(|_embedded_package| { - let dep_name = self.pool.intern_package_name( - ResolvoPackageName::PkgNameBufWithComponent( - PkgNameBufWithComponent { - name: ident.name().to_owned(), - component: SyntheticComponent::Actual( - parent_component.name.clone(), + .repository_name() + .to_owned(), ), - }, - ), - ); - known_deps.requirements.push( - self.pool.intern_version_set( - dep_name, - RequestVS::SpkRequest(Request::Pkg( - PkgRequest::new( - RangeIdent { - repository_name: Some( - located_build_ident_with_component - .ident - .repository_name() - .to_owned(), - ), - name: ident.name().to_owned(), - components: BTreeSet::from_iter([ - parent_component.name.clone(), - ]), - version: VersionFilter::single( - DoubleEqualsVersion::version_range( - ident.version().clone(), - ), - ), - build: Some(build.clone()), - }, - RequestedBy::Embedded( - located_build_ident_with_component - .ident - .target() - .clone(), + name: parent_ident.name().to_owned(), + components: BTreeSet::from_iter([ + parent_component.name.clone(), + ]), + version: VersionFilter::single( + DoubleEqualsVersion::version_range( + parent_ident.version().clone(), ), ), - )) - ).into(), - ); - }); - } - } + build: Some(parent_ident.build().clone()), + }, + RequestedBy::Embedded( + located_build_ident_with_component + .ident + .target() + .clone(), + ), + ))), + ) + .into(), + ); + }); + } + if !found { + // In the event that no owning component was found, + // this stub must still bring in at least one + // component from the parent. By convention, bring + // in the Run component of the parent. + let dep_name = self.pool.intern_package_name( + ResolvoPackageName::PkgNameBufWithComponent(PkgNameBufWithComponent { + name: parent_ident.name().to_owned(), + component: SyntheticComponent::Actual(Component::Run), + }), + ); + let located_parent = LocatedBuildIdentWithComponent { + ident: parent_ident.clone().to_located( + located_build_ident_with_component + .ident + .repository_name() + .to_owned(), + ), + // as_request_with_components does not make use + // of the component field, assigning Base here + // does not imply anything. + component: SyntheticComponent::Base, + requires_build_from_source: false, + }; + known_deps.requirements.push( + self.pool + .intern_version_set( + dep_name, + RequestVS::SpkRequest( + located_parent.as_request_with_components([Component::Run]), + ), + ) + .into(), + ); } } for option in package.get_build_options() { From 1cb58b8ccda7acacb0fc2520cfb7057933617539 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Sat, 22 Feb 2025 13:53:52 -0800 Subject: [PATCH 62/64] Expand test coverage for resolvo solver To promote better test coverage, the build_package family of macros now require the name of the solver to use, and all the tests using these macros now run the test against all three solver choices (ignoring "all"). Workaround too many arguments lint and rstest https://github.com/la10736/rstest/issues/226#issuecomment-2695922123 Bumped to the latest version of rstest first to see if that would help. Signed-off-by: J Robert Ray Signed-off-by: J Robert Ray --- Cargo.toml | 2 +- .../src/cmd_build_test/environment.rs | 2 + .../cmd-build/src/cmd_build_test/mod.rs | 222 ++++++++++++------ .../src/cmd_build_test/variant_filter.rs | 111 ++++++++- crates/spk-cli/cmd-build/src/macros.rs | 24 +- crates/spk-cli/cmd-test/src/cmd_test_test.rs | 80 +++++-- crates/spk-cli/common/src/flags_test.rs | 13 +- crates/spk-exec/src/exec_test.rs | 7 + 8 files changed, 343 insertions(+), 118 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 96bbe8f52d..52a40a8820 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,7 +89,7 @@ regex = "1.6" relative-path = "1.3" resolvo = "0.9.1" ring = "0.17.14" -rstest = "0.25" +rstest = "0.25.0" sentry = { version = "0.34.0", default-features = false, features = [ # all the default features except `debug-images` which causes a deadlock on # centos 7: https://github.com/getsentry/sentry-rust/issues/358 diff --git a/crates/spk-cli/cmd-build/src/cmd_build_test/environment.rs b/crates/spk-cli/cmd-build/src/cmd_build_test/environment.rs index 61584d0590..79549d88da 100644 --- a/crates/spk-cli/cmd-build/src/cmd_build_test/environment.rs +++ b/crates/spk-cli/cmd-build/src/cmd_build_test/environment.rs @@ -25,6 +25,7 @@ async fn basic_environment_generation( tmpdir: tempfile::TempDir, #[case] env_spec: &str, #[case] expected: &str, + #[values("cli", "checks", "resolvo")] solver_to_run: &str, ) { let rt = spfs_runtime().await; @@ -43,6 +44,7 @@ install: {env_spec} "# ), + solver_to_run ); let mut result = result.expect("Expected build to succeed"); diff --git a/crates/spk-cli/cmd-build/src/cmd_build_test/mod.rs b/crates/spk-cli/cmd-build/src/cmd_build_test/mod.rs index 05d0f1b23c..033e1c25d6 100644 --- a/crates/spk-cli/cmd-build/src/cmd_build_test/mod.rs +++ b/crates/spk-cli/cmd-build/src/cmd_build_test/mod.rs @@ -26,8 +26,14 @@ struct Opt { } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_variant_options_contribute_to_build_hash(tmpdir: tempfile::TempDir) { +async fn test_variant_options_contribute_to_build_hash( + tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, +) { // A var that appears in the variant list and doesn't appear in the // build.options list should still affect the build hash / produce a // unique build. @@ -48,6 +54,7 @@ build: script: - "true" "#, + solver_to_run ); let ident = version_ident!("three-variants/1.0.0"); @@ -64,8 +71,14 @@ build: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_build_hash_not_affected_by_dependency_version(tmpdir: tempfile::TempDir) { +async fn test_build_hash_not_affected_by_dependency_version( + tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, +) { // The same recipe should produce the same build hash even if there is a // change in its dependencies (at resolve time). let rt = spfs_runtime().await; @@ -81,7 +94,8 @@ pkg: dependency/1.0.0 build: script: - "true" -"# +"#, + solver_to_run ); // Build a package that depends on "dependency". @@ -98,6 +112,7 @@ build: script: - "true" "#, + solver_to_run ); // Now build a newer version of the dependency. @@ -112,10 +127,11 @@ build: script: - "true" "#, + solver_to_run ); // And build the other package again. - build_package!(tmpdir, package_filename); + build_package!(tmpdir, package_filename, solver_to_run); // The second time building "package" we expect it to build something with // the _same_ build digest (e.g., the change in version of one of its @@ -160,7 +176,6 @@ build: script: - "true" "#, - "--solver-to-run", solver_to_run ); @@ -185,7 +200,6 @@ install: - pkg: one fromBuildEnv: true "#, - "--solver-to-run", solver_to_run ); @@ -213,7 +227,6 @@ install: - pkg: two fromBuildEnv: true "#, - "--solver-to-run", solver_to_run ); @@ -221,8 +234,14 @@ install: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_package_with_circular_dep_can_modify_files(tmpdir: tempfile::TempDir) { +async fn test_package_with_circular_dep_can_modify_files( + tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, +) { // A package that depends on itself should be able to modify files // belonging to itself. let _rt = spfs_runtime().await; @@ -237,7 +256,8 @@ build: script: - echo "1.0.0" > $PREFIX/a.txt - echo "1.0.0" > $PREFIX/z.txt -"# +"#, + solver_to_run ); build_package!( @@ -249,7 +269,8 @@ pkg: circ/1.0.0 build: script: - echo "1.0.0" > $PREFIX/version.txt -"# +"#, + solver_to_run ); // Force middle to pick up exactly 1.0.0 so for the multiple builds below @@ -270,6 +291,7 @@ install: requirements: - pkg: circ/=1.0.0 "#, + solver_to_run ); // Attempt to build a newer version of circ, but now it depends on `middle` @@ -294,6 +316,7 @@ build: rules: - allow: RecursiveBuild "#, + solver_to_run ); for other_file in ["a", "z"] { @@ -322,8 +345,9 @@ build: - echo "1.0.1" > $PREFIX/version.txt # try to modify a file belonging to 'other' too - echo "1.0.1" > $PREFIX/{other_file}.txt -"# +"#, ), + solver_to_run ) .1 .expect_err("Expected build to fail"); @@ -331,8 +355,14 @@ build: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_package_with_circular_dep_can_build_major_version_change(tmpdir: tempfile::TempDir) { +async fn test_package_with_circular_dep_can_build_major_version_change( + tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, +) { // A package that depends on itself should be able to build a new major // version of itself, as in something not compatible with the version // being brought in via the circular dependency. @@ -347,7 +377,8 @@ pkg: circ/1.0.0 build: script: - echo "1.0.0" > $PREFIX/version.txt -"# +"#, + solver_to_run ); build_package!( @@ -366,6 +397,7 @@ install: - pkg: circ fromBuildEnv: true "#, + solver_to_run ); // Attempt to build a 2.0.0 version of circ, which shouldn't prevent @@ -385,12 +417,19 @@ build: rules: - allow: RecursiveBuild "#, + solver_to_run ); } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_package_with_circular_dep_collects_all_files(tmpdir: tempfile::TempDir) { +async fn test_package_with_circular_dep_collects_all_files( + tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, +) { // Building a new version of a package that depends on itself should // produce a package containing all the expected files, even if the new // build creates files with the same content as the previous build. @@ -407,7 +446,8 @@ build: - echo "1.0.0" > $PREFIX/version.txt - echo "hello world" > $PREFIX/hello.txt - echo "unchanged" > $PREFIX/unchanged.txt -"# +"#, + solver_to_run ); build_package!( @@ -426,6 +466,7 @@ install: - pkg: circ fromBuildEnv: true "#, + solver_to_run ); // This build overwrites a file from the previous build, but it has the same @@ -447,6 +488,7 @@ build: rules: - allow: RecursiveBuild "#, + solver_to_run ); let build = rt @@ -495,8 +537,14 @@ build: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_package_with_circular_dep_does_not_collect_file_removals(tmpdir: tempfile::TempDir) { +async fn test_package_with_circular_dep_does_not_collect_file_removals( + tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, +) { // Building a new version of a package that depends on itself should not // collect "negative files" (e.g., files that were removed in the new // build). @@ -513,7 +561,8 @@ pkg: empty/1.0.0 build: script: - mkdir $PREFIX/subdir -"# +"#, + solver_to_run ); build_package!( @@ -527,7 +576,8 @@ build: - echo "1.0.0" > $PREFIX/version.txt - mkdir -p $PREFIX/subdir/v1 - echo "hello world" > $PREFIX/subdir/v1/hello.txt -"# +"#, + solver_to_run ); // This build deletes a subdir that is owned by the previous build. It should @@ -551,6 +601,7 @@ build: rules: - allow: RecursiveBuild "#, + solver_to_run ); let build = rt @@ -593,46 +644,51 @@ build: ); } -#[rstest] -// cases not involving host options -#[should_panic] -#[case::empty_value_fails("varname", "", &["yes", "no"], true)] -#[case::non_empty_value_succeeds("varname/yes", "", &["yes", "no"], true)] -#[should_panic] -#[case::non_empty_value_bad_value_fails("varname/what", "", &["yes", "no"], true)] -// cases involving host options -#[case::empty_value_for_host_option_succeeds("os", "", &["linux", "windows"], true)] -#[case::non_empty_value_for_host_option_succeeds("os", "linux", &["linux", "windows"], true)] -#[should_panic] -#[case::empty_value_for_host_option_fails_if_host_options_disabled("os", "", &["linux", "windows"], false)] -// this case verifies that the --no-host option is respected -#[case::non_empty_value_for_host_option_good_value_succeeds_with_host_options_disabled("os", "beos", &["beos"], false)] -#[should_panic] -#[case::non_empty_value_for_host_option_bad_value_fails_with_host_options_disabled("os", "beos", &["linux", "windows"], false)] -// this case passes because host options override default values, and the -// provided host option value of "linux" is a valid choice. -#[case::non_empty_value_for_host_option_bad_value_succeeds_with_host_options_enabled("os", "beos", &["linux", "windows"], true)] -#[serial_test::serial(host_options)] -#[tokio::test] -async fn test_options_with_choices_and_empty_values( - tmpdir: tempfile::TempDir, - #[case] name: &'static str, - #[case] value: &'static str, - #[case] choices: &'static [&'static str], - #[case] host_options_enabled: bool, -) { - let _rt = spfs_runtime().await; - - // Force "os" host option to "linux" to make this test pass on any OS. - HOST_OPTIONS - .scoped_options(Ok(option_map! { "os" => "linux" }), async move { - let name_maybe_value = if value.is_empty() { - name.to_string() - } else { - format!("{name}/{value}") - }; - let generated_spec = format!( - r#" +#[allow(clippy::too_many_arguments)] +mod workaround_rstest_not_preserving_attrs { + use super::*; + + #[rstest] + // cases not involving host options + #[should_panic] + #[case::empty_value_fails("varname", "", &["yes", "no"], true)] + #[case::non_empty_value_succeeds("varname/yes", "", &["yes", "no"], true)] + #[should_panic] + #[case::non_empty_value_bad_value_fails("varname/what", "", &["yes", "no"], true)] + // cases involving host options + #[case::empty_value_for_host_option_succeeds("os", "", &["linux", "windows"], true)] + #[case::non_empty_value_for_host_option_succeeds("os", "linux", &["linux", "windows"], true)] + #[should_panic] + #[case::empty_value_for_host_option_fails_if_host_options_disabled("os", "", &["linux", "windows"], false)] + // this case verifies that the --no-host option is respected + #[case::non_empty_value_for_host_option_good_value_succeeds_with_host_options_disabled("os", "beos", &["beos"], false)] + #[should_panic] + #[case::non_empty_value_for_host_option_bad_value_fails_with_host_options_disabled("os", "beos", &["linux", "windows"], false)] + // this case passes because host options override default values, and the + // provided host option value of "linux" is a valid choice. + #[case::non_empty_value_for_host_option_bad_value_succeeds_with_host_options_enabled("os", "beos", &["linux", "windows"], true)] + #[tokio::test] + #[serial_test::serial(host_options)] + async fn test_options_with_choices_and_empty_values( + tmpdir: tempfile::TempDir, + #[case] name: &'static str, + #[case] value: &'static str, + #[case] choices: &'static [&'static str], + #[case] host_options_enabled: bool, + #[values("cli", "checks", "resolvo")] solver_to_run: &'static str, + ) { + let _rt = spfs_runtime().await; + + // Force "os" host option to "linux" to make this test pass on any OS. + HOST_OPTIONS + .scoped_options(Ok(option_map! { "os" => "linux" }), async move { + let name_maybe_value = if value.is_empty() { + name.to_string() + } else { + format!("{name}/{value}") + }; + let generated_spec = format!( + r#" pkg: dummy/1.0.0 api: v0/package build: @@ -642,25 +698,35 @@ build: script: - "true" "#, - choices = choices.join(", ") - ); - - if !host_options_enabled { - build_package!(tmpdir, "dummy.spk.yaml", generated_spec, "--no-host"); - } else { - build_package!(tmpdir, "dummy.spk.yaml", generated_spec); - } - - Ok::<_, ()>(()) - }) - .await - .unwrap(); + choices = choices.join(", ") + ); + + if !host_options_enabled { + build_package!( + tmpdir, + "dummy.spk.yaml", + generated_spec, + solver_to_run, + "--no-host" + ); + } else { + build_package!(tmpdir, "dummy.spk.yaml", generated_spec, solver_to_run); + } + + Ok::<_, ()>(()) + }) + .await + .unwrap(); + } } /// A package may contain files/directories with a leading dot #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_dot_files_are_collected(tmpdir: tempfile::TempDir) { +async fn test_dot_files_are_collected(tmpdir: tempfile::TempDir, #[case] solver_to_run: &str) { let rt = spfs_runtime().await; build_package!( @@ -680,6 +746,7 @@ build: - touch /spfs/.dot2/.dot - ln -s .dot2 /spfs/.dot3 "#, + solver_to_run ); let build = rt @@ -733,8 +800,14 @@ build: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_package_with_environment_ops_preserves_ops_in_recipe(tmpdir: tempfile::TempDir) { +async fn test_package_with_environment_ops_preserves_ops_in_recipe( + tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, +) { let rt = spfs_runtime().await; build_package!( @@ -750,7 +823,8 @@ install: environment: - set: FOO value: bar -"# +"#, + solver_to_run ); let recipe = rt diff --git a/crates/spk-cli/cmd-build/src/cmd_build_test/variant_filter.rs b/crates/spk-cli/cmd-build/src/cmd_build_test/variant_filter.rs index d4d44a8984..4793e587ad 100644 --- a/crates/spk-cli/cmd-build/src/cmd_build_test/variant_filter.rs +++ b/crates/spk-cli/cmd-build/src/cmd_build_test/variant_filter.rs @@ -22,8 +22,14 @@ struct Opt { } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_build_with_variant_acts_as_variant_filter(tmpdir: tempfile::TempDir) { +async fn test_build_with_variant_acts_as_variant_filter( + tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, +) { let _rt = spfs_runtime().await; let (_, result) = try_build_package!( @@ -42,6 +48,7 @@ build: - { color: green } - { color: blue } "#, + solver_to_run, // By saying --variant color=green, we are asking for the second variant "--variant", "color=green", @@ -73,8 +80,14 @@ build: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_build_with_opts_acts_as_override(tmpdir: tempfile::TempDir) { +async fn test_build_with_opts_acts_as_override( + tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, +) { let _rt = spfs_runtime().await; let (_, result) = try_build_package!( @@ -93,6 +106,7 @@ build: - { color: green } - { color: blue } "#, + solver_to_run, // By saying --opt color=green, we are asking to override the color in // all the variants (pruning duplicates). "--opt", @@ -125,8 +139,14 @@ build: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_build_with_variant_acts_as_variant_filter_no_match(tmpdir: tempfile::TempDir) { +async fn test_build_with_variant_acts_as_variant_filter_no_match( + tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, +) { let _rt = spfs_runtime().await; let (_, result) = try_build_package!( @@ -145,6 +165,7 @@ build: - { color: green } - { color: blue } "#, + solver_to_run, // By saying --variant color=purple, we are asking for a variant that // doesn't exist. "--variant", @@ -155,9 +176,13 @@ build: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] async fn test_build_with_variant_on_recipe_with_no_variants_match_default( tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, ) { let _rt = spfs_runtime().await; @@ -173,6 +198,7 @@ build: script: - 'echo "color: $SPK_OPT_color" > "$PREFIX/color.txt"' "#, + solver_to_run, // By saying --variant color=blue, we are asking for the "default" // variant, because the default color is blue. "--variant", @@ -205,8 +231,14 @@ build: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_build_with_variant_on_recipe_with_no_variants_no_match(tmpdir: tempfile::TempDir) { +async fn test_build_with_variant_on_recipe_with_no_variants_no_match( + tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, +) { let _rt = spfs_runtime().await; let (_, result) = try_build_package!( @@ -221,6 +253,7 @@ build: script: - 'echo "color: $SPK_OPT_color" > "$PREFIX/color.txt"' "#, + solver_to_run, // By saying --variant color=green, we are asking for a variant that // doesn't exist. "--variant", @@ -231,8 +264,14 @@ build: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_build_with_new_variant_on_recipe_with_no_variants(tmpdir: tempfile::TempDir) { +async fn test_build_with_new_variant_on_recipe_with_no_variants( + tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, +) { let _rt = spfs_runtime().await; let (_, result) = try_build_package!( @@ -247,6 +286,7 @@ build: script: - 'echo "color: $SPK_OPT_color" > "$PREFIX/color.txt"' "#, + solver_to_run, // By saying --new-variant with color=green, we are asking for a bespoke // variant "--new-variant", @@ -279,8 +319,14 @@ build: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_build_with_variant_acts_as_variant_filter_two_opts(tmpdir: tempfile::TempDir) { +async fn test_build_with_variant_acts_as_variant_filter_two_opts( + tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, +) { let _rt = spfs_runtime().await; let (_, result) = try_build_package!( @@ -300,6 +346,7 @@ build: - { color: green, fruit: apple } - { color: blue, fruit: orange } "#, + solver_to_run, // By saying --variant color=green,fruit=apple we are asking for the // second variant "--variant", @@ -336,9 +383,13 @@ build: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] async fn test_build_with_variant_acts_as_variant_filter_two_opts_no_match( tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, ) { let _rt = spfs_runtime().await; @@ -359,6 +410,7 @@ build: - { color: green, fruit: apple } - { color: blue, fruit: orange } "#, + solver_to_run, // The first option matches, but the second doesn't "--variant", "color=green,fruit=orange", @@ -368,9 +420,13 @@ build: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] async fn test_build_with_variant_and_opts_acts_as_variant_filter_and_override( tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, ) { let _rt = spfs_runtime().await; @@ -391,6 +447,7 @@ build: - { color: green } - { color: blue } "#, + solver_to_run, // By saying --variant color=green, we are asking for the second variant "--variant", "color=green", @@ -429,8 +486,14 @@ build: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_build_with_opts_acts_as_an_override(tmpdir: tempfile::TempDir) { +async fn test_build_with_opts_acts_as_an_override( + tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, +) { let _rt = spfs_runtime().await; let (_, result) = try_build_package!( @@ -450,6 +513,7 @@ build: - { color: green } - { color: blue } "#, + solver_to_run, // Setting an option that doesn't appear in the variants will // not filter out any variants, but will override the default "--opt", @@ -484,8 +548,14 @@ build: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_build_with_opts_and_variant_index(tmpdir: tempfile::TempDir) { +async fn test_build_with_opts_and_variant_index( + tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, +) { let _rt = spfs_runtime().await; let (_, result) = try_build_package!( @@ -505,6 +575,7 @@ build: - { color: green, fruit: apple } - { color: blue, fruit: orange } "#, + solver_to_run, // By saying --variant 0, we are explicitly asking for the first variant "--variant", "0", @@ -544,8 +615,11 @@ build: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_build_with_variant_spec(tmpdir: tempfile::TempDir) { +async fn test_build_with_variant_spec(tmpdir: tempfile::TempDir, #[case] solver_to_run: &str) { let _rt = spfs_runtime().await; let (_, result) = try_build_package!( @@ -565,6 +639,7 @@ build: - { color: green, fruit: apple } - { color: blue, fruit: orange } "#, + solver_to_run, // By supplying a variant spec, we are asking for a bespoke variant "--new-variant", r#"{ "color": "brown", "fruit": "kiwi" }"#, @@ -601,8 +676,14 @@ build: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_build_with_variant_spec_and_override(tmpdir: tempfile::TempDir) { +async fn test_build_with_variant_spec_and_override( + tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, +) { let _rt = spfs_runtime().await; let (_, result) = try_build_package!( @@ -622,6 +703,7 @@ build: - { color: green, fruit: apple } - { color: blue, fruit: orange } "#, + solver_to_run, // By supplying a variant spec, we are asking for a bespoke variant "--new-variant", r#"{ "color": "brown", "fruit": "kiwi" }"#, @@ -661,9 +743,15 @@ build: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[serial_test::serial(host_options)] #[tokio::test] -async fn test_build_filters_variants_based_on_host_opts(tmpdir: tempfile::TempDir) { +async fn test_build_filters_variants_based_on_host_opts( + tmpdir: tempfile::TempDir, + #[case] solver_to_run: &'static str, +) { let _rt = spfs_runtime().await; // Force "distro" host option to "centos" to make this test pass on any OS. @@ -686,6 +774,7 @@ async fn test_build_filters_variants_based_on_host_opts(tmpdir: tempfile::TempDi - { distro: rocky, color: green } - { distro: centos, color: blue } "#, + solver_to_run ); let mut result = result.expect("Expected build to succeed"); diff --git a/crates/spk-cli/cmd-build/src/macros.rs b/crates/spk-cli/cmd-build/src/macros.rs index e1758a56af..ec9eb22cba 100644 --- a/crates/spk-cli/cmd-build/src/macros.rs +++ b/crates/spk-cli/cmd-build/src/macros.rs @@ -14,20 +14,20 @@ pub struct BuildOpt { #[macro_export] macro_rules! build_package { - ($tmpdir:ident, $filename:literal, $recipe:literal $(,$extra_build_args:expr)* $(,)?) => {{ - let (filename, r) = $crate::try_build_package!($tmpdir, $filename, $recipe, $($extra_build_args),*); + ($tmpdir:ident, $filename:literal, $recipe:literal, $solver_to_run:expr $(,$extra_build_args:expr)* $(,)?) => {{ + let (filename, r) = $crate::try_build_package!($tmpdir, $filename, $recipe, $solver_to_run, $($extra_build_args),*); r.unwrap(); filename }}; - ($tmpdir:ident, $filename:literal, $recipe:ident $(,$extra_build_args:expr)* $(,)?) => {{ - let (filename, r) = $crate::try_build_package!($tmpdir, $filename, $recipe, $($extra_build_args),*); + ($tmpdir:ident, $filename:literal, $recipe:ident, $solver_to_run:expr $(,$extra_build_args:expr)* $(,)?) => {{ + let (filename, r) = $crate::try_build_package!($tmpdir, $filename, $recipe, $solver_to_run, $($extra_build_args),*); r.unwrap(); filename }}; - ($tmpdir:ident, $filename:ident $(,$extra_build_args:expr)* $(,)?) => {{ - let (filename, r) = $crate::try_build_package!($tmpdir, $filename, $($extra_build_args),*); + ($tmpdir:ident, $filename:ident, $solver_to_run:expr $(,$extra_build_args:expr)* $(,)?) => {{ + let (filename, r) = $crate::try_build_package!($tmpdir, $filename, $solver_to_run, $($extra_build_args),*); r.unwrap(); filename }}; @@ -35,7 +35,7 @@ macro_rules! build_package { #[macro_export] macro_rules! try_build_package { - ($tmpdir:ident, $filename:literal, $recipe:literal $(,$extra_build_args:expr)* $(,)?) => {{ + ($tmpdir:ident, $filename:literal, $recipe:literal, $solver_to_run:expr $(,$extra_build_args:expr)* $(,)?) => {{ // Leak `filename` for convenience. let filename = Box::leak(Box::new($tmpdir.path().join($filename))); { @@ -46,10 +46,10 @@ macro_rules! try_build_package { let filename_str = filename.as_os_str().to_str().unwrap(); - $crate::try_build_package!($tmpdir, filename_str, $($extra_build_args),*) + $crate::try_build_package!($tmpdir, filename_str, $solver_to_run, $($extra_build_args),*) }}; - ($tmpdir:ident, $filename:literal, $recipe:expr $(,$extra_build_args:expr)* $(,)?) => {{ + ($tmpdir:ident, $filename:literal, $recipe:expr, $solver_to_run:expr $(,$extra_build_args:expr)* $(,)?) => {{ // Leak `filename` for convenience. let filename = Box::leak(Box::new($tmpdir.path().join($filename))); { @@ -60,10 +60,10 @@ macro_rules! try_build_package { let filename_str = filename.as_os_str().to_str().unwrap(); - $crate::try_build_package!($tmpdir, filename_str, $($extra_build_args),*) + $crate::try_build_package!($tmpdir, filename_str, $solver_to_run, $($extra_build_args),*) }}; - ($tmpdir:ident, $filename:ident $(,$extra_build_args:expr)* $(,)?) => {{ + ($tmpdir:ident, $filename:ident, $solver_to_run:expr $(,$extra_build_args:expr)* $(,)?) => {{ // Build the package so it can be tested. use clap::Parser; let mut opt = $crate::macros::BuildOpt::try_parse_from([ @@ -72,6 +72,8 @@ macro_rules! try_build_package { // coverage testing. "--no-runtime", "--disable-repo=origin", + "--solver-to-run", + $solver_to_run, $($extra_build_args,)* $filename, ]) diff --git a/crates/spk-cli/cmd-test/src/cmd_test_test.rs b/crates/spk-cli/cmd-test/src/cmd_test_test.rs index 2c7c63e734..c8bf76773b 100644 --- a/crates/spk-cli/cmd-test/src/cmd_test_test.rs +++ b/crates/spk-cli/cmd-test/src/cmd_test_test.rs @@ -18,8 +18,11 @@ struct TestOpt { } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_all_test_stages_succeed(tmpdir: tempfile::TempDir) { +async fn test_all_test_stages_succeed(tmpdir: tempfile::TempDir, #[case] solver_to_run: &str) { // A var that appears in the variant list and doesn't appear in the // build.options list should still affect the build hash / produce a // unique build. @@ -44,7 +47,8 @@ tests: - stage: install script: - "true" -"# +"#, + solver_to_run ); let mut opt = TestOpt::try_parse_from([ @@ -60,8 +64,14 @@ tests: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_install_test_picks_same_digest_as_build(tmpdir: tempfile::TempDir) { +async fn test_install_test_picks_same_digest_as_build( + tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, +) { let _rt = spfs_runtime().await; build_package!( @@ -72,7 +82,8 @@ pkg: a-pkg-with-no-version-specified/1.0.0 build: script: - "true" -"# +"#, + solver_to_run ); let filename_str = build_package!( @@ -90,7 +101,8 @@ tests: - stage: install script: - "true" -"# +"#, + solver_to_run ); let mut opt = TestOpt::try_parse_from([ @@ -112,9 +124,13 @@ tests: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] async fn test_install_test_picks_same_digest_as_build_with_new_dep_in_variant( tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, ) { let _rt = spfs_runtime().await; @@ -126,7 +142,8 @@ pkg: dep-a/1.2.3 build: script: - "true" -"# +"#, + solver_to_run ); build_package!( @@ -137,7 +154,8 @@ pkg: dep-b/1.2.3 build: script: - "true" -"# +"#, + solver_to_run ); // Note that "dep-b" is introduced as a new dependency in the variant. @@ -158,7 +176,8 @@ tests: - stage: install script: - "true" -"# +"#, + solver_to_run ); let mut opt = TestOpt::try_parse_from([ @@ -180,9 +199,13 @@ tests: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] async fn test_install_test_picks_same_digest_as_build_with_new_dep_in_variant_plus_command_line_overrides( tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, ) { let _rt = spfs_runtime().await; @@ -194,7 +217,8 @@ pkg: dep-a/1.2.5 build: script: - "true" -"# +"#, + solver_to_run ); build_package!( @@ -205,7 +229,8 @@ pkg: dep-b/1.2.3 build: script: - "true" -"# +"#, + solver_to_run ); let filename_str = build_package!( @@ -226,6 +251,7 @@ tests: script: - "true" "#, + solver_to_run, // Extra build options specified here. "--opt", "dep-a=1.2.4" @@ -253,9 +279,13 @@ tests: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] async fn test_install_test_picks_same_digest_as_build_with_circular_dependencies( tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, ) { let _rt = spfs_runtime().await; @@ -268,7 +298,8 @@ pkg: some-other/1.2.0 build: script: - "true" -"# +"#, + solver_to_run ); build_package!( @@ -285,7 +316,8 @@ install: requirements: - pkg: some-other fromBuildEnv: true -"# +"#, + solver_to_run ); build_package!( @@ -305,7 +337,8 @@ install: fromBuildEnv: true - pkg: some-other fromBuildEnv: true -"# +"#, + solver_to_run ); let filename_str = build_package!( @@ -333,6 +366,7 @@ tests: script: - "true" "#, + solver_to_run ); let mut opt = TestOpt::try_parse_from([ @@ -354,8 +388,14 @@ tests: } #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] -async fn test_selectors_with_component_names_match_correctly(tmpdir: tempfile::TempDir) { +async fn test_selectors_with_component_names_match_correctly( + tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, +) { let _rt = spfs_runtime().await; let _ = build_package!( @@ -376,7 +416,8 @@ install: - name: comp2 files: - comp2 -"# +"#, + solver_to_run ); // This package is expected to pass both tests. @@ -404,7 +445,8 @@ tests: - { "base:comp2": "1.0.0" } script: - test -f "$PREFIX"/comp2 -"# +"#, + solver_to_run ); let mut opt = TestOpt::try_parse_from([ @@ -442,7 +484,8 @@ tests: # If the whole test run fails we know that this selector matched as # expected. - test -f "$PREFIX"/comp2 -"# +"#, + solver_to_run ); let mut opt = TestOpt::try_parse_from([ @@ -480,7 +523,8 @@ tests: # If the whole test run fails we know that this selector matched as # expected. - test -f "$PREFIX"/comp1 -"# +"#, + solver_to_run ); let mut opt = TestOpt::try_parse_from([ diff --git a/crates/spk-cli/common/src/flags_test.rs b/crates/spk-cli/common/src/flags_test.rs index 9bf4dce553..fcd4c6c9bd 100644 --- a/crates/spk-cli/common/src/flags_test.rs +++ b/crates/spk-cli/common/src/flags_test.rs @@ -40,8 +40,15 @@ fn test_option_flags_parsing(#[case] args: &[&str], #[case] expected: &[(&str, & } #[rstest] +#[case::cli(SolverToRun::Cli, SolverToShow::Cli)] +#[case::cli(SolverToRun::Checks, SolverToShow::Checks)] +#[case::cli(SolverToRun::Resolvo, SolverToShow::Resolvo)] #[tokio::test] -async fn test_get_solver_with_host_options(#[values(true, false)] no_host: bool) { +async fn test_get_solver_with_host_options( + #[case] solver_to_run: SolverToRun, + #[case] solver_to_show: SolverToShow, + #[values(true, false)] no_host: bool, +) { // Test the get_solver() method adds the host options to the solver // correctly. @@ -70,8 +77,8 @@ async fn test_get_solver_with_host_options(#[values(true, false)] no_host: bool) long_solves: Default::default(), max_frequent_errors: Default::default(), status_bar: Default::default(), - solver_to_run: SolverToRun::Cli, - solver_to_show: SolverToShow::Cli, + solver_to_run, + solver_to_show, show_search_size: Default::default(), compare_solvers: Default::default(), stop_on_block: Default::default(), diff --git a/crates/spk-exec/src/exec_test.rs b/crates/spk-exec/src/exec_test.rs index 7bd12c7aa1..59f8604e45 100644 --- a/crates/spk-exec/src/exec_test.rs +++ b/crates/spk-exec/src/exec_test.rs @@ -23,10 +23,15 @@ fn solver() -> StepSolver { /// If two layers contribute files to the same subdirectory, the Manifest is /// expected to contain both files. #[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] #[tokio::test] async fn get_environment_filesystem_merges_directories( tmpdir: tempfile::TempDir, + // TODO: test with all solvers mut solver: StepSolver, + #[case] solver_to_run: &str, ) { let rt = spfs_runtime().await; @@ -42,6 +47,7 @@ build: - mkdir "$PREFIX"/subdir - touch "$PREFIX"/subdir/one.txt "#, + solver_to_run ); build_package!( @@ -56,6 +62,7 @@ build: - mkdir "$PREFIX"/subdir - touch "$PREFIX"/subdir/two.txt "#, + solver_to_run ); let formatter = DecisionFormatterBuilder::default() From 36647bc6a4df00e7f7b14c2970e83609673ca5a2 Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Wed, 5 Mar 2025 15:31:02 -0800 Subject: [PATCH 63/64] Fix resolvo solver with migration-to-components The solver tests were not passing if this feature is enabled, and this exposed a few problems in not handling ':all' requests correctly. When generating the "VersionSet" for dependencies, the PkgRequest needs to have the correct component populated. The logic for skipping or not skipping candidates when build from source needed to handle ':all' and it was restructured for clarity. When filtering candidates for a request of a component of a specific build, source builds needed to be filtered out. Signed-off-by: J Robert Ray --- .../foundation/src/ident_build/build.rs | 4 + crates/spk-solve/src/solvers/resolvo/mod.rs | 11 ++- .../src/solvers/resolvo/resolvo_tests.rs | 3 +- .../src/solvers/resolvo/spk_provider.rs | 94 +++++++++++++++---- crates/spk-solve/src/solvers/solver_test.rs | 9 +- cspell.json | 1 + 6 files changed, 101 insertions(+), 21 deletions(-) diff --git a/crates/spk-schema/crates/foundation/src/ident_build/build.rs b/crates/spk-schema/crates/foundation/src/ident_build/build.rs index ce1da54845..4b4f1daca4 100644 --- a/crates/spk-schema/crates/foundation/src/ident_build/build.rs +++ b/crates/spk-schema/crates/foundation/src/ident_build/build.rs @@ -130,6 +130,10 @@ impl Build { } } + pub fn is_buildid(&self) -> bool { + matches!(self, Build::BuildId(_)) + } + pub fn is_source(&self) -> bool { matches!(self, Build::Source) } diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index 2eb4c23cc2..e60dbc7422 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -329,7 +329,16 @@ impl SolverTrait for Solver { #[async_trait::async_trait] impl SolverMut for Solver { - fn add_request(&mut self, request: Request) { + fn add_request(&mut self, mut request: Request) { + if let Request::Pkg(request) = &mut request { + if request.pkg.components.is_empty() { + if request.pkg.is_source() { + request.pkg.components.insert(Component::Source); + } else { + request.pkg.components.insert(Component::default_for_run()); + } + } + } self.requests.push(request); } diff --git a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs index 4d3038656a..d8768d9b07 100644 --- a/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs +++ b/crates/spk-solve/src/solvers/resolvo/resolvo_tests.rs @@ -8,6 +8,7 @@ use rstest::rstest; use spk_schema::prelude::HasVersion; use spk_schema::{Package, opt_name}; use spk_solve_macros::{make_repo, request}; +use tap::TapFallible; use super::Solver; use crate::SolverMut; @@ -101,7 +102,7 @@ async fn package_with_dependency() { let mut solver = Solver::new(vec![repo.into()], Cow::Borrowed(&[])); solver.add_request(request!("needs-dep/1.0.0")); - let solution = solver.solve().await.unwrap(); + let solution = solver.solve().await.tap_err(|e| eprintln!("{e}")).unwrap(); assert_eq!(solution.len(), 2); } diff --git a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs index d72144a8b2..59ee4a7925 100644 --- a/crates/spk-solve/src/solvers/resolvo/spk_provider.rs +++ b/crates/spk-solve/src/solvers/resolvo/spk_provider.rs @@ -69,7 +69,7 @@ use crate::SolverMut; // be a relationship between every component of a package and a "base" // component, to prevent a solve containing a mix of components from different // versions of the same package. -#[derive(Clone, Eq, Hash, PartialEq)] +#[derive(Clone, Debug, Eq, Hash, PartialEq)] pub(crate) struct PkgNameBufWithComponent { pub(crate) name: PkgNameBuf, pub(crate) component: SyntheticComponent, @@ -128,7 +128,10 @@ impl ResolvoPackageName { None } ResolvoPackageName::PkgNameBufWithComponent(pkg_name) => { - let mut located_builds = Vec::new(); + // Prevent duplicate solvables by using a set. Ties (same build + // but "requires_build_from_source" differs) are resolved with + // "requires_build_from_source" being true. + let mut located_builds = HashSet::new(); let root_pkg_request = provider.global_pkg_requests.get(&pkg_name.name); @@ -201,23 +204,62 @@ impl ResolvoPackageName { // when a request for it is found it can act as a // surrogate that depends on all the individual // components. - [Component::All], + { + if !build.is_source() { + itertools::Either::Left([Component::All].into_iter()) + } else { + // XXX: Unclear if this is the right + // approach but without this special + // case the Solution can incorrectly + // end up with a src build marked as + // requires_build_from_source for + // requests that are asking for + // :src. + itertools::Either::Right([].into_iter()) + } + }, ) { - if component != *pkg_name_component - && (provider.binary_only || !component.is_source()) + let requires_build_from_source = build.is_source() + && (component != *pkg_name_component + || pkg_name_component.is_all()); + + if requires_build_from_source && provider.binary_only { + // Deny anything that requires build + // from source when binary_only is + // enabled. + continue; + } + + if (!requires_build_from_source || !build.is_source()) + && component != *pkg_name_component { + // Deny components that don't match + // unless it is possible to build from + // source. continue; } - located_builds.push(LocatedBuildIdentWithComponent { + let new_entry = LocatedBuildIdentWithComponent { ident: located_build_ident.clone(), component: pkg_name.component.clone(), - requires_build_from_source: component - != *pkg_name_component, - }); + requires_build_from_source, + }; + + if requires_build_from_source { + // _replace_ any existing entry, which + // might have + // requires_build_from_source == false, + // so it now is true. + located_builds.replace(new_entry); + } else { + // _insert_ to not overwrite any + // existing entry that might have + // requires_build_from_source == true. + located_builds.insert(new_entry); + } } } else { - located_builds.push(LocatedBuildIdentWithComponent { + located_builds.insert(LocatedBuildIdentWithComponent { ident: located_build_ident, component: SyntheticComponent::Base, requires_build_from_source: false, @@ -538,11 +580,11 @@ impl SpkProvider { fn pkg_request_to_known_dependencies(&self, pkg_request: &PkgRequest) -> KnownDependencies { let mut components = pkg_request.pkg.components.iter().peekable(); let iter = if components.peek().is_some() { - itertools::Either::Right(components.cloned().map(SyntheticComponent::Actual)) + itertools::Either::Right(components.cloned()) } else { itertools::Either::Left( // A request with no components is assumed to be a request for - // the run (or source) component. + // the default_for_run (or source) component. if pkg_request .pkg .build @@ -550,9 +592,9 @@ impl SpkProvider { .map(|build| build.is_source()) .unwrap_or(false) { - std::iter::once(SyntheticComponent::Actual(Component::Source)) + std::iter::once(Component::Source) } else { - std::iter::once(SyntheticComponent::Actual(Component::Run)) + std::iter::once(Component::default_for_run()) }, ) }; @@ -566,12 +608,14 @@ impl SpkProvider { .intern_package_name(ResolvoPackageName::PkgNameBufWithComponent( PkgNameBufWithComponent { name: pkg_request.pkg.name().to_owned(), - component, + component: SyntheticComponent::Actual(component.clone()), }, )); + let mut pkg_request_with_component = pkg_request.clone(); + pkg_request_with_component.pkg.components = BTreeSet::from_iter([component]); let dep_vs = self.pool.intern_version_set( dep_name, - RequestVS::SpkRequest(Request::Pkg(pkg_request.clone())), + RequestVS::SpkRequest(Request::Pkg(pkg_request_with_component)), ); match pkg_request.inclusion_policy { spk_schema::ident::InclusionPolicy::Always => { @@ -761,7 +805,7 @@ impl SpkProvider { // If neither build has a key, both packages failed to load? // Add debug assert to see if this ever happens. - debug_assert!(false, "builds without keys"); + debug_assert!(false, "builds without keys {a:?} {b:?}"); a.1.ident.cmp(&b.1.ident) } @@ -885,6 +929,22 @@ impl DependencyProvider for SpkProvider { // a candidate. Source builds that can't be built from // source are filtered out in `get_candidates`. if located_build_ident_with_component.requires_build_from_source { + // However, building from source is not a suitable + // candidate for a request for a specific component + // of an existing build, such as when finding the + // members of the :all component of a build. + if pkg_request + .pkg + .build + .as_ref() + .is_some_and(|b| b.is_buildid()) + { + if inverse { + selected.push(*candidate); + } + continue; + } + if !inverse { selected.push(*candidate); } diff --git a/crates/spk-solve/src/solvers/solver_test.rs b/crates/spk-solve/src/solvers/solver_test.rs index bd512ae1bf..553704857d 100644 --- a/crates/spk-solve/src/solvers/solver_test.rs +++ b/crates/spk-solve/src/solvers/solver_test.rs @@ -235,7 +235,8 @@ async fn test_solver_package_with_no_recipe_and_impossible_initial_checks( let res = run_and_print_resolve_for_tests(&mut solver).await; if cfg!(feature = "migration-to-components") { match res { - Err(Error::InitialRequestsContainImpossibleError(_)) => { + Err(Error::InitialRequestsContainImpossibleError(_)) + | Err(Error::FailedToResolve(_)) => { // Success, when the 'migration-to-components' feature // is enabled because the initial checks for // impossible requests fail because the package does @@ -332,7 +333,11 @@ async fn test_solver_package_with_no_recipe_from_cmd_line_and_impossible_initial // :build and a :run component to pass and it only has a :run // component assert!( - matches!(res, Err(Error::InitialRequestsContainImpossibleError(_))), + matches!( + res, + Err(Error::InitialRequestsContainImpossibleError(_)) + | Err(Error::FailedToResolve(_)) + ), "'{res:?}' should be a Error::String('Initial requests contain 1 impossible request.')", ); } else { diff --git a/cspell.json b/cspell.json index ca5b59870f..ad9ebab414 100644 --- a/cspell.json +++ b/cspell.json @@ -67,6 +67,7 @@ "builddir", "buildenv", "BUILDGST", + "buildid", "BUILDKEY", "buildroot", "buildvariabledescription", From cfba38f2ab1908ca544321e79350972ac2a12cce Mon Sep 17 00:00:00 2001 From: J Robert Ray Date: Thu, 20 Mar 2025 11:46:42 -0700 Subject: [PATCH 64/64] Add configure_for_build_environment to the Solver trait Fix commented out code in cmd_test for build stage tests and add a new test that checks the basic behavior of a build stage test that the test environment includes the build dependencies of the package being tested. Lift the old solver's configure_for_build_environment method to the Solver trait. Although the other test stages have code that only lives in the cmd_test crate to configure the solver properly, the build stage uses this method that was defined in the solver itself. While the logic of configure_for_build_environment could be duplicated in cmd_test, it is also reused by other code in the solver so it made sense to preserve the logic as a reusable solver method. Signed-off-by: J Robert Ray --- crates/spk-cli/cmd-test/src/cmd_test_test.rs | 55 ++++++++++++++++++++ crates/spk-cli/cmd-test/src/test/build.rs | 4 +- crates/spk-solve/src/solver.rs | 35 ++++++++++++- crates/spk-solve/src/solvers/resolvo/mod.rs | 4 ++ crates/spk-solve/src/solvers/step/solver.rs | 20 ++----- 5 files changed, 98 insertions(+), 20 deletions(-) diff --git a/crates/spk-cli/cmd-test/src/cmd_test_test.rs b/crates/spk-cli/cmd-test/src/cmd_test_test.rs index c8bf76773b..e1ecd169d7 100644 --- a/crates/spk-cli/cmd-test/src/cmd_test_test.rs +++ b/crates/spk-cli/cmd-test/src/cmd_test_test.rs @@ -541,3 +541,58 @@ tests: .await .expect_err("the test run should fail, otherwise the selectors aren't working properly"); } + +#[rstest] +#[case::cli("cli")] +#[case::checks("checks")] +#[case::resolvo("resolvo")] +#[tokio::test] +async fn build_stage_test_env_includes_build_deps( + tmpdir: tempfile::TempDir, + #[case] solver_to_run: &str, +) { + let _rt = spfs_runtime().await; + + let _ = build_package!( + tmpdir, + "base.spk.yaml", + br#" +pkg: base/1.0.0 +build: + script: + - touch "$PREFIX"/base +"#, + solver_to_run + ); + + let filename_str = build_package!( + tmpdir, + "simple1.spk.yaml", + br#" +pkg: simple1/1.0.0 +build: + options: + - pkg: base + script: + - "true" + +tests: + - stage: build + script: + # simple's build options should exist in a build stage test environment. + - test -f "$PREFIX"/base +"#, + solver_to_run + ); + + let mut opt = TestOpt::try_parse_from([ + "test", + // Don't exec a new process to move into a new runtime, this confuses + // coverage testing. + "--no-runtime", + "--disable-repo=origin", + filename_str, + ]) + .unwrap(); + opt.test.run().await.unwrap(); +} diff --git a/crates/spk-cli/cmd-test/src/test/build.rs b/crates/spk-cli/cmd-test/src/test/build.rs index 52262ad2ec..c42ef40a2d 100644 --- a/crates/spk-cli/cmd-test/src/test/build.rs +++ b/crates/spk-cli/cmd-test/src/test/build.rs @@ -117,8 +117,8 @@ where for repo in self.repos.iter().cloned() { solver.add_repository(repo); } - // TODO - // solver.configure_for_build_environment(&self.recipe)?; + // Configure solver for build environment. + solver.configure_for_build_environment(&self.recipe)?; for request in self.additional_requirements.drain(..) { solver.add_request(request) } diff --git a/crates/spk-solve/src/solver.rs b/crates/spk-solve/src/solver.rs index 4e912ee17f..c03956df73 100644 --- a/crates/spk-solve/src/solver.rs +++ b/crates/spk-solve/src/solver.rs @@ -2,11 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/spkenv/spk +use std::borrow::Cow; use std::sync::Arc; use enum_dispatch::enum_dispatch; use spk_schema::ident::{PkgRequest, VarRequest}; -use spk_schema::{OptionMap, Request}; +use spk_schema::{OptionMap, Recipe, Request}; use spk_solve_solution::Solution; use spk_storage::RepositoryHandle; use variantly::Variantly; @@ -26,6 +27,12 @@ pub enum SolverImpl { #[async_trait::async_trait] #[enum_dispatch] pub trait Solver { + /// Return the options that the solver is currently configured with. + /// + /// These are the options that have been set via + /// [`SolverMut::update_options`]. + fn get_options(&self) -> Cow<'_, OptionMap>; + /// Return the PkgRequests added to the solver. fn get_pkg_requests(&self) -> Vec; @@ -38,10 +45,26 @@ pub trait Solver { #[async_trait::async_trait] #[enum_dispatch] -pub trait SolverMut { +pub trait SolverMut: Solver { /// Add a request to this solver. fn add_request(&mut self, request: Request); + /// Adds requests for all build requirements of the given recipe. + fn configure_for_build_environment(&mut self, recipe: &T) -> Result<()> { + let options = self.get_options(); + + let build_options = recipe.resolve_options(&*options)?; + for req in recipe + .get_build_requirements(&build_options)? + .iter() + .cloned() + { + self.add_request(req) + } + + Ok(()) + } + /// Put this solver back into its default state fn reset(&mut self); @@ -86,6 +109,10 @@ impl Solver for &T where T: Solver, { + fn get_options(&self) -> Cow<'_, OptionMap> { + T::get_options(self) + } + fn get_pkg_requests(&self) -> Vec { T::get_pkg_requests(self) } @@ -103,6 +130,10 @@ impl Solver for &mut T where T: Solver, { + fn get_options(&self) -> Cow<'_, OptionMap> { + T::get_options(self) + } + fn get_pkg_requests(&self) -> Vec { T::get_pkg_requests(self) } diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index e60dbc7422..a8276561bf 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -306,6 +306,10 @@ impl Solver { } impl SolverTrait for Solver { + fn get_options(&self) -> Cow<'_, OptionMap> { + Cow::Borrowed(&self.options) + } + fn get_pkg_requests(&self) -> Vec { self.requests .iter() diff --git a/crates/spk-solve/src/solvers/step/solver.rs b/crates/spk-solve/src/solvers/step/solver.rs index d339fa4553..29fc3b3a31 100644 --- a/crates/spk-solve/src/solvers/step/solver.rs +++ b/crates/spk-solve/src/solvers/step/solver.rs @@ -1050,22 +1050,6 @@ impl Solver { || self.impossible_checks.use_in_build_keys } - /// Adds requests for all build requirements - pub fn configure_for_build_environment(&mut self, recipe: &T) -> Result<()> { - let state = self.get_initial_state(); - - let build_options = recipe.resolve_options(state.get_option_map())?; - for req in recipe - .get_build_requirements(&build_options)? - .iter() - .cloned() - { - self.add_request(req) - } - - Ok(()) - } - /// Adds requests for all build requirements and solves pub async fn solve_build_environment(&mut self, recipe: &SpecRecipe) -> Result { self.configure_for_build_environment(recipe)?; @@ -1104,6 +1088,10 @@ impl Solver { } impl SolverTrait for Solver { + fn get_options(&self) -> Cow<'_, OptionMap> { + Cow::Owned(self.get_initial_state().get_option_map().clone()) + } + fn get_pkg_requests(&self) -> Vec { self.get_initial_state() .get_pkg_requests()