Crate Universe: crate.spec() supports path (#3596)

This is inspired by #3464

Author: illicitonion

.bazelci/presubmit.yml

@@ -494,15 +494,15 @@ tasks:
     shell_commands: *no_bzlmod_shell_commands
     working_directory: examples/crate_universe_local_path
     run_targets:
-      - "//:vendor_edit_test_out_of_tree"
+      - "//:vendor_edit_test_out_of_tree_nobzlmod"
   crate_universe_local_path_in_tree_no_bzlmod:
     name: Crate Universe Local Path In Tree (No Bzlmod)
     platform: ubuntu2204
     bazel: *no_bzlmod_bazel_version
     shell_commands: *no_bzlmod_shell_commands
     working_directory: examples/crate_universe_local_path
     run_targets:
-      - "//:vendor_edit_test_in_tree"
+      - "//:vendor_edit_test_in_tree_nobzlmod"
   example_bindgen_toolchain:
     name: Example custom bindgen toolchain registration
     platform: ubuntu2204

crate_universe/extensions.bzl

@@ -646,6 +646,7 @@ def _generate_hub_and_spokes(
             output_dir = tag_path.get_child("splicing-output"),
             debug_workspace_dir = tag_path.get_child("splicing-workspace"),
             skip_cargo_lockfile_overwrite = cfg.skip_cargo_lockfile_overwrite,
+            nonhermetic_root_bazel_workspace_dir = nonhermetic_root_bazel_workspace_dir,
             repository_name = cfg.name,
         )
 
@@ -1399,7 +1400,7 @@ _spec = tag_class(
             doc = "A list of features to use for the crate.",
         ),
         "git": attr.string(
-            doc = "The Git url to use for the crate. Cannot be used with `version`.",
+            doc = "The Git url to use for the crate. Cannot be used with `version` or `path`.",
         ),
         "lib": attr.bool(
             doc = "If using `artifact = 'bin'`, additionally setting `lib = True` declares a dependency on both the package's library and binary, as opposed to just the binary.",
@@ -1408,6 +1409,9 @@ _spec = tag_class(
             doc = "The explicit name of the package.",
             mandatory = True,
         ),
+        "path": attr.string(
+            doc = "The local path of the remote crate. Cannot be used with `version` or `git`.",
+        ),
         "repositories": attr.string_list(
             doc = "A list of repository names specified from `crate.from_cargo(name=...)` that this spec is applied to. Defaults to all repositories.",
             default = [],
@@ -1419,7 +1423,7 @@ _spec = tag_class(
             doc = "The git tag of the remote crate. Tied with the `git` param. Only one of branch, tag or rev may be specified. Specifying `rev` is recommended for fully-reproducible builds.",
         ),
         "version": attr.string(
-            doc = "The exact version of the crate. Cannot be used with `git`.",
+            doc = "The exact version of the crate. Cannot be used with `git` or `path`.",
         ),
     },
 )

crate_universe/private/crate.bzl

@@ -27,7 +27,8 @@ def _spec(
         git = None,
         branch = None,
         tag = None,
-        rev = None):
+        rev = None,
+        path = None):
     """A constructor for a crate dependency.
 
     See [specifying dependencies][sd] in the Cargo book for more details.
@@ -36,15 +37,16 @@ def _spec(
 
     Args:
         package (str, optional): The explicit name of the package (used when attempting to alias a crate).
-        version (str, optional): The exact version of the crate. Cannot be used with `git`.
+        version (str, optional): The exact version of the crate. Cannot be used with `git` or `path`.
         artifact (str, optional): Set to "bin" to pull in a binary crate as an artifact dependency. Requires a nightly Cargo.
         lib (bool, optional): If using `artifact = "bin"`, additionally setting `lib = True` declares a dependency on both the package's library and binary, as opposed to just the binary.
         default_features (bool, optional): Maps to the `default-features` flag.
         features (list, optional): A list of features to use for the crate
-        git (str, optional): The Git url to use for the crate. Cannot be used with `version`.
+        git (str, optional): The Git url to use for the crate. Cannot be used with `version` or `path`.
         branch (str, optional): The git branch of the remote crate. Tied with the `git` param. Only one of branch, tag or rev may be specified. Specifying `rev` is recommended for fully-reproducible builds.
         tag (str, optional): The git tag of the remote crate. Tied with the `git` param. Only one of branch, tag or rev may be specified. Specifying `rev` is recommended for fully-reproducible builds.
         rev (str, optional): The git revision of the remote crate. Tied with the `git` param. Only one of branch, tag or rev may be specified.
+        path (str, optional): The local path of the remote crate. Cannot be used with `version` or `git`.
 
     Returns:
         string: A json encoded string of all inputs
@@ -59,6 +61,7 @@ def _spec(
             "git": git,
             "lib": lib,
             "package": package,
+            "path": path,
             "rev": rev,
             "tag": tag,
             "version": version,

crate_universe/private/crates_repository.bzl

@@ -92,6 +92,7 @@ def _crates_repository_impl(repository_ctx):
             config_path = config_path,
             output_dir = repository_ctx.path("splicing-output"),
             skip_cargo_lockfile_overwrite = repository_ctx.attr.skip_cargo_lockfile_overwrite,
+            nonhermetic_root_bazel_workspace_dir = nonhermetic_root_bazel_workspace_dir,
             repository_name = repository_ctx.name,
         )
 

crate_universe/private/splicing_utils.bzl

@@ -124,6 +124,7 @@ def splice_workspace_manifest(
         output_dir,
         repository_name,
         skip_cargo_lockfile_overwrite,
+        nonhermetic_root_bazel_workspace_dir,
         debug_workspace_dir = None):
     """Splice together a Cargo workspace from various other manifests and package definitions
 
@@ -138,6 +139,7 @@ def splice_workspace_manifest(
         skip_cargo_lockfile_overwrite (bool): Whether to skip writing the cargo lockfile back after resolving.
             You may want to set this if your dependency versions are maintained externally through a non-trivial set-up.
             But you probably don't want to set this.
+        nonhermetic_root_bazel_workspace_dir (path): The path to the current workspace root
         debug_workspace_dir (path): The location in which to save splicing outputs for future review.
 
     Returns:
@@ -155,6 +157,8 @@ def splice_workspace_manifest(
         config_path,
         "--repository-name",
         repository_name,
+        "--nonhermetic-root-bazel-workspace-dir",
+        nonhermetic_root_bazel_workspace_dir,
     ]
 
     if cargo_lockfile:

crate_universe/src/cli/splice.rs

@@ -70,6 +70,13 @@ pub struct SpliceOptions {
     /// But you probably don't want to set this.
     #[clap(long)]
     pub skip_cargo_lockfile_overwrite: bool,
+
+    /// The path to the Bazel root workspace (i.e. the directory containing the WORKSPACE.bazel file or similar).
+    /// BE CAREFUL with this value. We never want to include it in a lockfile hash (to keep lockfiles portable),
+    /// which means you also should not use it anywhere that _should_ be guarded by a lockfile hash.
+    /// You basically never want to use this value.
+    #[clap(long)]
+    pub nonhermetic_root_bazel_workspace_dir: Utf8PathBuf,
 }
 
 /// Combine a set of disjoint manifests into a single workspace.
@@ -97,7 +104,7 @@ pub fn splice(opt: SpliceOptions) -> Result<()> {
 
     // Splice together the manifest
     let manifest_path = prepared_splicer
-        .splice(&splicing_dir)
+        .splice(&splicing_dir, &opt.nonhermetic_root_bazel_workspace_dir)
         .with_context(|| format!("Failed to splice workspace {}", opt.repository_name))?;
 
     // Use the existing lockfile if possible, otherwise generate a new one.

crate_universe/src/cli/vendor.rs

@@ -217,7 +217,7 @@ pub fn vendor(opt: VendorOptions) -> anyhow::Result<()> {
 
     // Splice together the manifest
     let manifest_path = splicer
-        .splice_workspace()
+        .splice_workspace(&opt.nonhermetic_root_bazel_workspace_dir)
         .context("Failed to splice workspace")?;
 
     // Gather a cargo lockfile

crate_universe/src/config.rs

@@ -22,7 +22,7 @@ use crate::utils::starlark::Label;
 use crate::utils::target_triple::TargetTriple;
 
 /// Representations of different kinds of crate vendoring into workspaces.
-#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)]
 #[serde(rename_all = "lowercase")]
 pub(crate) enum VendorMode {
     /// Crates having full source being vendored into a workspace

crate_universe/src/rendering.rs

@@ -863,9 +863,15 @@ impl Renderer {
         extra_deps: Select<BTreeSet<Label>>,
     ) -> Select<BTreeSet<Label>> {
         Select::merge(
-            deps.map(|dep| match dep.local_path {
-                Some(path) => Label::from_str(&format!("//{}:{}", path, &dep.target)).unwrap(),
-                _ => self.crate_label(&dep.id.name, &dep.id.version.to_string(), &dep.target),
+            deps.map(|dep| {
+                match (dep.local_path, self.config.vendor_mode) {
+                    // In local vendor mode, we use paths within the the repo.
+                    (Some(path), Some(VendorMode::Local)) => {
+                        Label::from_str(&format!("//{}:{}", path, &dep.target)).unwrap()
+                    }
+                    // If we're not vendoring source, or don't have a path for the dep, construct the label we expect.
+                    _ => self.crate_label(&dep.id.name, &dep.id.version.to_string(), &dep.target),
+                }
             }),
             extra_deps,
         )