diff --git a/Cargo.toml b/Cargo.toml index 5ae731806e..1bf5aae701 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -74,7 +74,7 @@ ordermap = "0.5.7" parking_lot = "0.12.4" pathdiff = "0.2.3" pep440_rs = "0.7.3" -pep508_rs = "0.9.2" +pep508_rs = { version = "0.9.2", features = ["non-pep508-extensions"] } percent-encoding = "2.3.1" pin-project-lite = "0.2.16" pixi = { path = "crates/pixi" } @@ -190,7 +190,9 @@ uv-install-wheel = { git = "https://github.com/astral-sh/uv", tag = "0.8.5" } uv-installer = { git = "https://github.com/astral-sh/uv", tag = "0.8.5" } uv-normalize = { git = "https://github.com/astral-sh/uv", tag = "0.8.5" } uv-pep440 = { git = "https://github.com/astral-sh/uv", tag = "0.8.5" } -uv-pep508 = { git = "https://github.com/astral-sh/uv", tag = "0.8.5" } +uv-pep508 = { git = "https://github.com/astral-sh/uv", tag = "0.8.5", features = [ + "non-pep508-extensions", +] } uv-platform-tags = { git = "https://github.com/astral-sh/uv", tag = "0.8.5" } uv-pypi-types = { git = "https://github.com/astral-sh/uv", tag = "0.8.5" } uv-python = { git = "https://github.com/astral-sh/uv", tag = "0.8.5" } diff --git a/crates/pixi_core/src/activation.rs b/crates/pixi_core/src/activation.rs index b79796e19c..144a979b06 100644 --- a/crates/pixi_core/src/activation.rs +++ b/crates/pixi_core/src/activation.rs @@ -489,7 +489,6 @@ pub(crate) async fn initialize_env_variables( mod tests { use super::*; use std::path::Path; - use std::str::FromStr; #[test] fn test_metadata_env() { @@ -703,7 +702,7 @@ packages: "#, platform = Platform::current() ); - let lock_file = LockFile::from_str(mock_lock).unwrap(); + let lock_file = LockFile::from_str_with_base_directory(mock_lock, None).unwrap(); let env = run_activation( &default_env, &CurrentEnvVarBehavior::Include, diff --git a/crates/pixi_core/src/lock_file/resolve/pypi.rs b/crates/pixi_core/src/lock_file/resolve/pypi.rs index 7ff17b9d82..d4cafd0c26 100644 --- a/crates/pixi_core/src/lock_file/resolve/pypi.rs +++ b/crates/pixi_core/src/lock_file/resolve/pypi.rs @@ -35,6 +35,7 @@ use pypi_modifiers::{ use rattler_digest::{Md5, Sha256, parse_digest_from_hex}; use rattler_lock::{ PackageHashes, PypiPackageData, PypiPackageEnvironmentData, PypiSourceTreeHashable, UrlOrPath, + Verbatim, }; use typed_path::Utf8TypedPathBuf; use url::Url; @@ -140,17 +141,18 @@ fn process_uv_path_url( path_url: &uv_pep508::VerbatimUrl, install_path: &Path, project_root: &Path, -) -> Result { +) -> Result<(String, Utf8TypedPathBuf), ProcessPathUrlError> { let given = path_url .given() - .ok_or_else(|| ProcessPathUrlError::NoGivenPath(path_url.to_string()))?; + .ok_or_else(|| ProcessPathUrlError::NoGivenPath(path_url.to_string()))? + .to_string(); let keep_abs = if given.starts_with("file://") { // Processed by UV this is a file url // don't keep it absolute as the origin is 1) and relative paths are impossible // we are assuming the intention was to keep it relative false } else { - let path = PathBuf::from(given); + let path = PathBuf::from(&given); // Determine if the path was given as an absolute path path.is_absolute() }; @@ -183,9 +185,9 @@ fn process_uv_path_url( Ok(if cfg!(windows) { // Replace backslashes with forward slashes on Windows because pathdiff can // return paths with backslashes. - Utf8TypedPathBuf::from(path_str.replace("\\", "/")) + (given, Utf8TypedPathBuf::from(path_str.replace("\\", "/"))) } else { - Utf8TypedPathBuf::from(path_str) + (given, Utf8TypedPathBuf::from(path_str)) }) } @@ -886,7 +888,7 @@ async fn lock_pypi_packages( ) .into_diagnostic() .context("cannot convert registry dist")?; - (url_or_path, hash) + (Verbatim::new(url_or_path), hash) } BuiltDist::DirectUrl(dist) => { let url = dist.url.to_url(); @@ -894,19 +896,17 @@ async fn lock_pypi_packages( .into_diagnostic() .context("cannot create direct url")?; - (UrlOrPath::Url(direct_url), None) + (Verbatim::new(UrlOrPath::Url(direct_url)), None) + } + BuiltDist::Path(dist) => { + let (given, path) = process_uv_path_url( + &dist.url, + &dist.install_path, + abs_project_root, + ) + .into_diagnostic()?; + (Verbatim::new_with_given(UrlOrPath::Path(path), given), None) } - BuiltDist::Path(dist) => ( - UrlOrPath::Path( - process_uv_path_url( - &dist.url, - &dist.install_path, - abs_project_root, - ) - .into_diagnostic()?, - ), - None, - ), }; let metadata = registry_client @@ -964,20 +964,22 @@ async fn lock_pypi_packages( get_url_or_path(®.index, ®.file.url, abs_project_root) .into_diagnostic() .context("cannot convert registry sdist")?; - (url_or_path, hash, false) + (Verbatim::new(url_or_path), hash, false) } SourceDist::DirectUrl(direct) => { let url = direct.url.to_url(); let direct_url = Url::parse(&format!("direct+{url}")) .into_diagnostic() .context("could not create direct-url")?; - (direct_url.into(), hash, false) + (Verbatim::new(direct_url.into()), hash, false) } SourceDist::Git(git) => { // convert resolved source dist into a pinned git spec let pinned_git_spec = into_pinned_git_spec(git.clone()); ( - pinned_git_spec.into_locked_git_url().to_url().into(), + Verbatim::new( + pinned_git_spec.into_locked_git_url().to_url().into(), + ), hash, false, ) @@ -996,7 +998,7 @@ async fn lock_pypi_packages( }; // process the path or url that we get back from uv - let install_path = process_uv_path_url( + let (given, install_path) = process_uv_path_url( &path.url, &path.install_path, abs_project_root, @@ -1006,7 +1008,8 @@ async fn lock_pypi_packages( // Create the url for the lock file. This is based on the passed in URL // instead of from the source path to copy the path that was passed in // from the requirement. - let url_or_path = UrlOrPath::Path(install_path); + let url_or_path = + Verbatim::new_with_given(UrlOrPath::Path(install_path), given); (url_or_path, hash, false) } SourceDist::Directory(dir) => { @@ -1023,14 +1026,15 @@ async fn lock_pypi_packages( }; // process the path or url that we get back from uv - let install_path = + let (given, install_path) = process_uv_path_url(&dir.url, &dir.install_path, abs_project_root) .into_diagnostic()?; // Create the url for the lock file. This is based on the passed in URL // instead of from the source path to copy the path that was passed in // from the requirement. - let url_or_path = UrlOrPath::Path(install_path); + let url_or_path = + Verbatim::new_with_given(UrlOrPath::Path(install_path), given); (url_or_path, hash, dir.editable.unwrap_or(false)) } }; @@ -1045,8 +1049,11 @@ async fn lock_pypi_packages( .transpose() .into_diagnostic()?, location, - requires_dist: to_requirements(metadata.requires_dist.iter()) - .into_diagnostic()?, + requires_dist: to_requirements( + metadata.requires_dist.iter(), + abs_project_root, + ) + .into_diagnostic()?, hash, editable, } @@ -1075,9 +1082,10 @@ mod tests { let url = uv_pep508::VerbatimUrl::parse_url("file:///a/b/c") .unwrap() .with_given("./b/c"); - let path = + let (given, path) = process_uv_path_url(&url, &PathBuf::from("/a/b/c"), &PathBuf::from("/a")).unwrap(); assert_eq!(path.as_str(), "./b/c"); + assert_eq!(given, "./b/c"); } // In this case we want to make the path relative to the project_root or lock @@ -1088,9 +1096,10 @@ mod tests { let url = uv_pep508::VerbatimUrl::parse_url("file:///a/b/c") .unwrap() .with_given("./b/c"); - let path = + let (given, path) = process_uv_path_url(&url, &PathBuf::from("/a/c/z"), &PathBuf::from("/a/b/f")).unwrap(); assert_eq!(path.as_str(), "../../c/z"); + assert_eq!(given, "./b/c"); } // In this case we want to make the path relative to the project_root or lock @@ -1101,7 +1110,7 @@ mod tests { let url = uv_pep508::VerbatimUrl::parse_url("file://C/a/b/c") .unwrap() .with_given("./b/c"); - let path = + let (given, path) = process_uv_path_url(&url, &PathBuf::from("C:\\a\\b\\c"), &PathBuf::from("C:\\a")) .unwrap(); assert_eq!(path.as_str(), "./b/c"); @@ -1115,13 +1124,14 @@ mod tests { let url = uv_pep508::VerbatimUrl::parse_url("file://C/a/b/c") .unwrap() .with_given("./b/c"); - let path = process_uv_path_url( + let (given, path) = process_uv_path_url( &url, &PathBuf::from("C:\\a\\c\\z"), &PathBuf::from("C:\\a\\b\\f"), ) .unwrap(); assert_eq!(path.as_str(), "../../c/z"); + assert_eq!(given, "./b/c"); } // In this case we want to keep the absolute path @@ -1131,9 +1141,10 @@ mod tests { let url = uv_pep508::VerbatimUrl::parse_url("file:///a/b/c") .unwrap() .with_given("/a/b/c"); - let path = + let (given, path) = process_uv_path_url(&url, &PathBuf::from("/a/b/c"), &PathBuf::from("/a")).unwrap(); assert_eq!(path.as_str(), "/a/b/c"); + assert_eq!(given, "/a/b/c"); } // In this case we want to keep the absolute path @@ -1143,9 +1154,10 @@ mod tests { let url = uv_pep508::VerbatimUrl::parse_url("file://C/a/b/c") .unwrap() .with_given("C:\\a\\b\\c"); - let path = + let (given, path) = process_uv_path_url(&url, &PathBuf::from("C:\\a\\b\\c"), &PathBuf::from("C:\\a")) .unwrap(); assert_eq!(path.as_str(), "C:/a/b/c"); + assert_eq!(given, "C:\\a\\b\\c"); } } diff --git a/crates/pixi_core/src/lock_file/satisfiability/mod.rs b/crates/pixi_core/src/lock_file/satisfiability/mod.rs index 9f876e71db..7e2f349737 100644 --- a/crates/pixi_core/src/lock_file/satisfiability/mod.rs +++ b/crates/pixi_core/src/lock_file/satisfiability/mod.rs @@ -566,7 +566,7 @@ fn verify_pypi_no_build( // violations for (_, packages) in locked_environment.pypi_packages_by_platform() { for (package, _) in packages { - let extension = match &package.location { + let extension = match package.location.inner() { // Get the extension from the url UrlOrPath::Url(url) => { if url.scheme().starts_with("git+") { @@ -778,7 +778,7 @@ pub(crate) fn pypi_satifisfies_editable( "editable requirement cannot be from registry, url, git or path (non-directory)" ) } - RequirementSource::Directory { install_path, .. } => match &locked_data.location { + RequirementSource::Directory { install_path, .. } => match locked_data.location.inner() { // If we have an url requirement locked, but the editable is requested, this does not // satifsfy UrlOrPath::Url(url) => Err(Box::new(PlatformUnsat::EditablePackageIsUrl( @@ -848,7 +848,7 @@ pub(crate) fn pypi_satifisfies_requirement( } } RequirementSource::Url { url: spec_url, .. } => { - if let UrlOrPath::Url(locked_url) = &locked_data.location { + if let UrlOrPath::Url(locked_url) = locked_data.location.inner() { // Url may not start with git, and must start with direct+ if locked_url.as_str().starts_with("git+") || !locked_url.as_str().starts_with("direct+") @@ -879,7 +879,7 @@ pub(crate) fn pypi_satifisfies_requirement( } => { let repository = git.repository(); let reference = git.reference(); - match &locked_data.location { + match locked_data.location.inner() { UrlOrPath::Url(url) => { if let Ok(pinned_git_spec) = LockedGitUrl::new(url.clone()).to_pinned_git_spec() { @@ -950,7 +950,7 @@ pub(crate) fn pypi_satifisfies_requirement( } RequirementSource::Path { install_path, .. } | RequirementSource::Directory { install_path, .. } => { - if let UrlOrPath::Path(locked_path) = &locked_data.location { + if let UrlOrPath::Path(locked_path) = locked_data.location.inner() { let install_path = Utf8TypedPathBuf::from(install_path.to_string_lossy().to_string()); let project_root = @@ -1361,7 +1361,7 @@ pub(crate) async fn verify_package_platform_satisfiability( if pypi_packages_visited.insert(idx) { // If this is path based package we need to check if the source tree hash still // matches. and if it is a directory - if let UrlOrPath::Path(path) = &record.0.location { + if let UrlOrPath::Path(path) = record.0.location.inner() { let absolute_path = if path.is_absolute() { Cow::Borrowed(Path::new(path.as_str())) } else { @@ -1854,7 +1854,7 @@ mod tests { use insta::Settings; use miette::{IntoDiagnostic, NarratableReportHandler}; use pep440_rs::{Operator, Version}; - use rattler_lock::LockFile; + use rattler_lock::{LockFile, Verbatim}; use rstest::rstest; use super::*; @@ -2094,7 +2094,7 @@ mod tests { let locked_data = PypiPackageData { name: "mypkg".parse().unwrap(), version: Version::from_str("0.1.0").unwrap(), - location: UrlOrPath::Path("C:\\Users\\username\\mypkg.tar.gz".into()), + location: Verbatim::new(UrlOrPath::Path("C:\\Users\\username\\mypkg.tar.gz".into())), hash: None, requires_dist: vec![], requires_python: None, diff --git a/crates/pixi_install_pypi/src/conversions.rs b/crates/pixi_install_pypi/src/conversions.rs index d3289905df..acb7a054bf 100644 --- a/crates/pixi_install_pypi/src/conversions.rs +++ b/crates/pixi_install_pypi/src/conversions.rs @@ -97,7 +97,7 @@ pub fn convert_to_dist( lock_file_dir: &Path, ) -> Result { // Figure out if it is a url from the registry or a direct url - let dist = match &pkg.location { + let dist = match pkg.location.inner() { UrlOrPath::Url(url) if is_direct_url(url.scheme()) => { let url_without_direct = strip_direct_scheme(url); let pkg_name = to_uv_normalize(&pkg.name)?; diff --git a/crates/pixi_install_pypi/src/plan/validation.rs b/crates/pixi_install_pypi/src/plan/validation.rs index f89a5dc40a..3dfae6fa9d 100644 --- a/crates/pixi_install_pypi/src/plan/validation.rs +++ b/crates/pixi_install_pypi/src/plan/validation.rs @@ -36,7 +36,7 @@ pub(crate) fn need_reinstall( // Check if the installed version is the same as the required version match installed { InstalledDist::Registry(reg) => { - if !matches!(locked.location, UrlOrPath::Url(_)) { + if !matches!(locked.location.inner(), UrlOrPath::Url(_)) { return Ok(ValidateCurrentInstall::Reinstall( NeedReinstall::SourceMismatch { locked_location: locked.location.to_string(), @@ -84,7 +84,7 @@ pub(crate) fn need_reinstall( match result { Ok(url) => { // Convert the locked location, which can be a path or a url, to a url - let locked_url = match &locked.location { + let locked_url = match locked.location.inner() { // Fine if it is already a url UrlOrPath::Url(url) => url.clone(), // Do some path mangling if it is actually a path to get it into a url @@ -157,7 +157,7 @@ pub(crate) fn need_reinstall( let lock_file_dir = typed_path::Utf8TypedPathBuf::from( lock_file_dir.to_string_lossy().as_ref(), ); - let locked_url = match &locked.location { + let locked_url = match locked.location.inner() { // Remove `direct+` scheme if it is there so we can compare the required to // the installed url UrlOrPath::Url(url) => strip_direct_scheme(url).into_owned(), @@ -225,7 +225,7 @@ pub(crate) fn need_reinstall( // Try to parse the locked git url, this can be any url, so this may fail // in practice it always seems to succeed, even with a non-git url - let locked_git_url = match &locked.location { + let locked_git_url = match locked.location.inner() { UrlOrPath::Url(url) => { // is it a git url? if LockedGitUrl::is_locked_git_url(url) { diff --git a/crates/pixi_pypi_spec/src/lib.rs b/crates/pixi_pypi_spec/src/lib.rs index 31cf018a91..5590306b97 100644 --- a/crates/pixi_pypi_spec/src/lib.rs +++ b/crates/pixi_pypi_spec/src/lib.rs @@ -11,7 +11,7 @@ use std::{ use pep440_rs::VersionSpecifiers; use pep508_rs::ExtraName; -use pixi_spec::GitSpec; +use pixi_spec::{GitSpec, Verbatim}; use serde::Serialize; use thiserror::Error; use url::Url; @@ -29,13 +29,13 @@ pub enum PixiPypiSpec { extras: Vec, }, Path { - path: PathBuf, + path: Verbatim, editable: Option, #[serde(default)] extras: Vec, }, Url { - url: Url, + url: Verbatim, subdirectory: Option, #[serde(default)] extras: Vec, diff --git a/crates/pixi_pypi_spec/src/pep508.rs b/crates/pixi_pypi_spec/src/pep508.rs index 5f8b91502c..cec81ea8c9 100644 --- a/crates/pixi_pypi_spec/src/pep508.rs +++ b/crates/pixi_pypi_spec/src/pep508.rs @@ -1,7 +1,7 @@ use crate::utils::extract_directory_from_url; use crate::{Pep508ToPyPiRequirementError, PixiPypiSpec, VersionOrStar}; use pixi_git::GitUrl; -use pixi_spec::GitSpec; +use pixi_spec::{GitSpec, Verbatim}; use std::path::Path; /// Implement from [`pep508_rs::Requirement`] to make the conversion easier. @@ -83,14 +83,14 @@ impl TryFrom for PixiPypiSpec { Pep508ToPyPiRequirementError::PathUrlIntoPath(url.clone()) })?; PixiPypiSpec::Path { - path: file, + path: Verbatim::new(file), editable: None, extras: req.extras, } } else { let subdirectory = extract_directory_from_url(&url); PixiPypiSpec::Url { - url, + url: Verbatim::new(url), extras: req.extras, subdirectory, } diff --git a/crates/pixi_pypi_spec/src/toml.rs b/crates/pixi_pypi_spec/src/toml.rs index b7cc646bae..1fc237742c 100644 --- a/crates/pixi_pypi_spec/src/toml.rs +++ b/crates/pixi_pypi_spec/src/toml.rs @@ -1,7 +1,7 @@ use crate::{PixiPypiSpec, VersionOrStar}; use itertools::Itertools; use pep508_rs::ExtraName; -use pixi_spec::{GitReference, GitSpec}; +use pixi_spec::{GitReference, GitSpec, Verbatim}; use pixi_toml::{TomlFromStr, TomlWith}; use std::fmt::Display; use std::path::PathBuf; @@ -59,7 +59,7 @@ struct RawPyPiRequirement { extras: Vec, // Path Only - pub path: Option, + pub path: Option>, pub editable: Option, // Git only @@ -69,7 +69,7 @@ struct RawPyPiRequirement { pub rev: Option, // Url only - pub url: Option, + pub url: Option>, // Git and Url only pub subdirectory: Option, @@ -170,8 +170,8 @@ impl<'de> toml_span::Deserialize<'de> for RawPyPiRequirement { .unwrap_or_default(); let path = th - .optional::>("path") - .map(TomlFromStr::into_inner); + .optional::("path") + .map(|path| Verbatim::new_with_given(PathBuf::from_str(&path).unwrap(), path)); let editable = th.optional("editable"); let git = th diff --git a/crates/pixi_spec/src/lib.rs b/crates/pixi_spec/src/lib.rs index 6c5f428c31..2b383e6be9 100644 --- a/crates/pixi_spec/src/lib.rs +++ b/crates/pixi_spec/src/lib.rs @@ -29,6 +29,8 @@ use thiserror::Error; pub use toml::{TomlLocationSpec, TomlSpec, TomlVersionSpecStr}; pub use url::{UrlBinarySpec, UrlSourceSpec, UrlSpec}; +pub use rattler_lock::Verbatim; + /// An error that is returned when a spec cannot be converted into another spec /// type. #[derive(Debug, Error)] diff --git a/crates/pixi_uv_conversions/src/conversions.rs b/crates/pixi_uv_conversions/src/conversions.rs index fb3180de24..e3038ce5e1 100644 --- a/crates/pixi_uv_conversions/src/conversions.rs +++ b/crates/pixi_uv_conversions/src/conversions.rs @@ -346,6 +346,7 @@ pub fn to_uv_specifiers( pub fn to_requirements<'req>( requirements: impl Iterator, + base_dir: &Path, ) -> Result, crate::ConversionError> { let requirements: Result, ConversionError> = requirements .map(|requirement| { @@ -403,17 +404,19 @@ pub fn to_requirements<'req>( writeln!(package_string, "#subdirectory={}", subdirectory.display())?; } } - uv_distribution_types::RequirementSource::Path { url, .. } => { - write!(package_string, " @ {url}")?; - } - uv_distribution_types::RequirementSource::Directory { url, .. } => { - write!(package_string, " @ {url}")?; + uv_distribution_types::RequirementSource::Path { url, .. } + | uv_distribution_types::RequirementSource::Directory { url, .. } => { + if let Some(g) = url.given() { + write!(package_string, " @ {g}")?; + } else { + write!(package_string, " @ {url}")?; + } } } if let Some(marker) = marker.contents() { write!(package_string, " ; {marker}")?; } - pep508_rs::Requirement::from_str(&package_string) + pep508_rs::Requirement::parse(&package_string, base_dir) .map_err(crate::Pep508Error::Pep508Error) .map_err(From::from) }) diff --git a/crates/pixi_uv_conversions/src/requirements.rs b/crates/pixi_uv_conversions/src/requirements.rs index fc7cdce66f..5837dd2b75 100644 --- a/crates/pixi_uv_conversions/src/requirements.rs +++ b/crates/pixi_uv_conversions/src/requirements.rs @@ -163,17 +163,19 @@ pub fn as_uv_req( editable, extras: _, } => { - let joined = project_root.join(path); + let joined = project_root.join(path.inner()); let canonicalized = dunce::canonicalize(&joined).map_err(|e| AsPep508Error::CanonicalizeError { source: e, path: joined.clone(), })?; - let given = path - .to_str() - .map(|s| s.to_owned()) - .unwrap_or_else(String::new); - let verbatim = VerbatimUrl::from_path(path, project_root)?.with_given(given); + let verbatim = { + let mut tmp = VerbatimUrl::from_path(path.inner(), project_root)?; + if let Some(g) = path.given() { + tmp = tmp.with_given(g) + } + tmp + }; if canonicalized.is_dir() { RequirementSource::Directory { @@ -181,7 +183,7 @@ pub fn as_uv_req( editable: Some(editable.unwrap_or_default()), url: verbatim, // TODO: we could see if we ever need this - // AFAICS it would be useful for constrainging dependencies + // AFAICS it would be useful for constraining dependencies r#virtual: Some(false), } } else if *editable == Some(true) { @@ -194,7 +196,7 @@ pub fn as_uv_req( RequirementSource::Path { install_path: canonicalized.into_boxed_path(), url: verbatim, - ext: DistExtension::from_path(path)?, + ext: DistExtension::from_path(path.inner())?, } } } @@ -203,9 +205,15 @@ pub fn as_uv_req( } => { // We will clone the original URL and strip it's SHA256 fragment, // So that we can normalize the URL for comparison. - let mut location_url = url.clone(); + let mut location_url = url.inner().clone(); location_url.set_fragment(None); - let verbatim_url = VerbatimUrl::from_url(url.clone().into()); + let verbatim_url = { + let mut tmp = VerbatimUrl::from_url(url.inner().clone().into()); + if let Some(g) = url.given() { + tmp = tmp.with_given(g); + } + tmp + }; RequirementSource::Url { subdirectory: subdirectory