Add annotation support for transient crates. (#2392)

Author: matts1

crate_universe/extension.bzl

@@ -1,11 +1,34 @@
 """Module extension for generating third-party crates for use in bazel."""
 
+load("@bazel_skylib//lib:structs.bzl", "structs")
 load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("//crate_universe:defs.bzl", _crate_universe_crate = "crate")
 load("//crate_universe/private:crates_vendor.bzl", "CRATES_VENDOR_ATTRS", "generate_config_file", "generate_splicing_manifest")
 load("//crate_universe/private:generate_utils.bzl", "render_config")
 load("//crate_universe/private/module_extensions:cargo_bazel_bootstrap.bzl", "get_cargo_bazel_runner")
 
+# A list of labels which may be relative (and if so, is within the repo the rule is generated in).
+#
+# If I were to write ":foo", with attr.label_list, it would evaluate to
+# "@@//:foo". However, for a tag such as deps, ":foo" should refer to
+# "@@rules_rust~crates~<crate>//:foo".
+_relative_label_list = attr.string_list
+
+_OPT_BOOL_VALUES = {
+    "auto": None,
+    "off": False,
+    "on": True,
+}
+
+def optional_bool(doc):
+    return attr.string(doc = doc, values = _OPT_BOOL_VALUES.keys(), default = "auto")
+
+def _get_or_insert(d, key, value):
+    if key not in d:
+        d[key] = value
+    return d[key]
+
 def _generate_repo_impl(repo_ctx):
     for path, contents in repo_ctx.attr.contents.items():
         repo_ctx.file(path, contents)
@@ -17,7 +40,7 @@ _generate_repo = repository_rule(
     ),
 )
 
-def _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg):
+def _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg, annotations):
     cargo_lockfile = module_ctx.path(cfg.cargo_lockfile)
     tag_path = module_ctx.path(cfg.name)
 
@@ -31,7 +54,7 @@ def _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg):
         content = generate_config_file(
             module_ctx,
             mode = "remote",
-            annotations = {},
+            annotations = annotations,
             generate_build_scripts = cfg.generate_build_scripts,
             supported_platform_triples = cfg.supported_platform_triples,
             generate_target_compatible_with = True,
@@ -40,6 +63,7 @@ def _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg):
             workspace_name = cfg.name,
             generate_binaries = cfg.generate_binaries,
             render_config = rendering_config,
+            repository_ctx = module_ctx,
         ),
     )
 
@@ -160,16 +184,54 @@ def _crate_impl(module_ctx):
     cargo_bazel = get_cargo_bazel_runner(module_ctx)
     all_repos = []
     for mod in module_ctx.modules:
+        module_annotations = {}
+        repo_specific_annotations = {}
+        for annotation_tag in mod.tags.annotation:
+            annotation_dict = structs.to_dict(annotation_tag)
+            repositories = annotation_dict.pop("repositories")
+            crate = annotation_dict.pop("crate")
+
+            # The crate.annotation function can take in either a list or a bool.
+            # For the tag-based method, because it has type safety, we have to
+            # split it into two parameters.
+            if annotation_dict.pop("gen_all_binaries"):
+                annotation_dict["gen_binaries"] = True
+            annotation_dict["gen_build_script"] = _OPT_BOOL_VALUES[annotation_dict["gen_build_script"]]
+            annotation = _crate_universe_crate.annotation(**{
+                k: v
+                for k, v in annotation_dict.items()
+                # Tag classes can't take in None, but the function requires None
+                # instead of the empty values in many cases.
+                # https://github.com/bazelbuild/bazel/issues/20744
+                if v != "" and v != [] and v != {}
+            })
+            if not repositories:
+                _get_or_insert(module_annotations, crate, []).append(annotation)
+            for repo in repositories:
+                _get_or_insert(
+                    _get_or_insert(repo_specific_annotations, repo, {}),
+                    crate,
+                    [],
+                ).append(annotation)
+
         local_repos = []
         for cfg in mod.tags.from_cargo:
             if cfg.name in local_repos:
                 fail("Defined two crate universes with the same name in the same MODULE.bazel file. Use the name tag to give them different names.")
             elif cfg.name in all_repos:
                 fail("Defined two crate universes with the same name in different MODULE.bazel files. Either give one a different name, or use use_extension(isolate=True)")
-            _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg)
+
+            annotations = {k: v for k, v in module_annotations.items()}
+            for crate, values in repo_specific_annotations.get(cfg.name, {}).items():
+                _get_or_insert(annotations, crate, []).extend(values)
+            _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg, annotations)
             all_repos.append(cfg.name)
             local_repos.append(cfg.name)
 
+        for repo in repo_specific_annotations:
+            if repo not in local_repos:
+                fail("Annotation specified for repo %s, but the module defined repositories %s" % (repo, local_repos))
+
 _from_cargo = tag_class(
     doc = "Generates a repo @crates from a Cargo.toml / Cargo.lock pair",
     attrs = dict(
@@ -183,9 +245,54 @@ _from_cargo = tag_class(
     ),
 )
 
+# This should be kept in sync with crate_universe/private/crate.bzl.
+_annotation = tag_class(
+    attrs = dict(
+        repositories = attr.string_list(doc = "A list of repository names specified from `crate.from_cargo(name=...)` that this annotation is applied to. Defaults to all repositories.", default = []),
+        crate = attr.string(doc = "The name of the crate the annotation is applied to", mandatory = True),
+        version = attr.string(doc = "The versions of the crate the annotation is applied to. Defaults to all versions.", default = "*"),
+        additive_build_file_content = attr.string(doc = "Extra contents to write to the bottom of generated BUILD files."),
+        additive_build_file = attr.label(doc = "A file containing extra contents to write to the bottom of generated BUILD files."),
+        alias_rule = attr.string(doc = "Alias rule to use instead of `native.alias()`.  Overrides [render_config](#render_config)'s 'default_alias_rule'."),
+        build_script_data = _relative_label_list(doc = "A list of labels to add to a crate's `cargo_build_script::data` attribute."),
+        build_script_tools = _relative_label_list(doc = "A list of labels to add to a crate's `cargo_build_script::tools` attribute."),
+        build_script_data_glob = attr.string_list(doc = "A list of glob patterns to add to a crate's `cargo_build_script::data` attribute"),
+        build_script_deps = _relative_label_list(doc = "A list of labels to add to a crate's `cargo_build_script::deps` attribute."),
+        build_script_env = attr.string_dict(doc = "Additional environment variables to set on a crate's `cargo_build_script::env` attribute."),
+        build_script_proc_macro_deps = _relative_label_list(doc = "A list of labels to add to a crate's `cargo_build_script::proc_macro_deps` attribute."),
+        build_script_rundir = attr.string(doc = "An override for the build script's rundir attribute."),
+        build_script_rustc_env = attr.string_dict(doc = "Additional environment variables to set on a crate's `cargo_build_script::env` attribute."),
+        build_script_toolchains = attr.label_list(doc = "A list of labels to set on a crates's `cargo_build_script::toolchains` attribute."),
+        compile_data = _relative_label_list(doc = "A list of labels to add to a crate's `rust_library::compile_data` attribute."),
+        compile_data_glob = attr.string_list(doc = "A list of glob patterns to add to a crate's `rust_library::compile_data` attribute."),
+        crate_features = attr.string_list(doc = "A list of strings to add to a crate's `rust_library::crate_features` attribute."),
+        data = _relative_label_list(doc = "A list of labels to add to a crate's `rust_library::data` attribute."),
+        data_glob = attr.string_list(doc = "A list of glob patterns to add to a crate's `rust_library::data` attribute."),
+        deps = _relative_label_list(doc = "A list of labels to add to a crate's `rust_library::deps` attribute."),
+        extra_aliased_targets = attr.string_dict(doc = "A list of targets to add to the generated aliases in the root crate_universe repository."),
+        gen_binaries = attr.string_list(doc = "As a list, the subset of the crate's bins that should get `rust_binary` targets produced."),
+        gen_all_binaries = attr.bool(doc = "If true, generates `rust_binary` targets for all of the crates bins"),
+        disable_pipelining = attr.bool(doc = "If True, disables pipelining for library targets for this crate."),
+        gen_build_script = attr.string(
+            doc = "An authorative flag to determine whether or not to produce `cargo_build_script` targets for the current crate. Supported values are 'on', 'off', and 'auto'.",
+            values = _OPT_BOOL_VALUES.keys(),
+            default = "auto",
+        ),
+        patch_args = attr.string_list(doc = "The `patch_args` attribute of a Bazel repository rule. See [http_archive.patch_args](https://docs.bazel.build/versions/main/repo/http.html#http_archive-patch_args)"),
+        patch_tool = attr.string_list(doc = "The `patch_tool` attribute of a Bazel repository rule. See [http_archive.patch_tool](https://docs.bazel.build/versions/main/repo/http.html#http_archive-patch_tool)"),
+        patches = attr.label_list(doc = "The `patches` attribute of a Bazel repository rule. See [http_archive.patches](https://docs.bazel.build/versions/main/repo/http.html#http_archive-patches)"),
+        proc_macro_deps = _relative_label_list(doc = "A list of labels to add to a crate's `rust_library::proc_macro_deps` attribute."),
+        rustc_env = attr.string_dict(doc = "Additional variables to set on a crate's `rust_library::rustc_env` attribute."),
+        rustc_env_files = _relative_label_list(doc = "A list of labels to set on a crate's `rust_library::rustc_env_files` attribute."),
+        rustc_flags = attr.string_list(doc = "A list of strings to set on a crate's `rust_library::rustc_flags` attribute."),
+        shallow_since = attr.string(doc = "An optional timestamp used for crates originating from a git repository instead of a crate registry. This flag optimizes fetching the source code."),
+    ),
+)
+
 crate = module_extension(
     implementation = _crate_impl,
     tag_classes = dict(
         from_cargo = _from_cargo,
+        annotation = _annotation,
     ),
 )

crate_universe/private/crate.bzl

@@ -76,11 +76,12 @@ def _assert_absolute(label):
         label (Label): The label to check
     """
     label_str = str(label)
-    if not label.startswith("@"):
+    if not label_str.startswith("@"):
         fail("The labels must be absolute. Please update '{}'".format(
             label_str,
         ))
 
+# This should be kept in sync crate_universe/extension.bzl.
 def _annotation(
         version = "*",
         additive_build_file = None,
@@ -178,7 +179,7 @@ def _annotation(
     return json.encode((
         version,
         struct(
-            additive_build_file = additive_build_file,
+            additive_build_file = _stringify_label(additive_build_file),
             additive_build_file_content = additive_build_file_content,
             alias_rule = parse_alias_rule(alias_rule),
             build_script_data = _stringify_list(build_script_data),
@@ -211,6 +212,11 @@ def _annotation(
         ),
     ))
 
+def _stringify_label(value):
+    if not value:
+        return value
+    return str(value)
+
 # In bzlmod, attributes of type `attr.label_list` end up as `Label`s not `str`,
 # and the `json` module doesn't know how to serialize `Label`s,
 # so we proactively convert them to strings before serializing.

crate_universe/private/crates_vendor.bzl

@@ -176,7 +176,8 @@ def generate_config_file(
         repository_name,
         output_pkg,
         workspace_name,
-        render_config):
+        render_config,
+        repository_ctx = None):
     """Writes the rendering config to cargo-bazel-config.json.
 
     Args:
@@ -192,6 +193,8 @@ def generate_config_file(
         output_pkg: The path to the package containing the build files.
         workspace_name (str): The name of the workspace.
         render_config: The render config to use.
+        repository_ctx (repository_ctx, optional): A repository context object
+            used for enabling certain functionality.
 
     Returns:
         file: The cargo-bazel-config.json written.
@@ -244,6 +247,7 @@ def generate_config_file(
         render_config = render_config,
         supported_platform_triples = supported_platform_triples,
         repository_name = repository_name or ctx.label.name,
+        repository_ctx = repository_ctx,
     )
 
     return json.encode_indent(

examples/bzlmod/hello_world/MODULE.bazel

@@ -40,6 +40,17 @@ crate.from_cargo(
     manifests = ["//third-party:Cargo.toml"],
 )
 use_repo(crate, "crates")
+crate.annotation(
+    additive_build_file = "//:anyhow.BUILD.bazel",
+    crate = "anyhow",
+    # Defined in additive_build_file.
+    data = [":cargo_toml"],
+    # Optional, you probably don't need this. Defaults to all from_cargo
+    # invocations in this module.
+    repositories = ["crates"],
+    # Optional, you probably don't need this, defaults to "*".
+    version = "*",
+)
 
 # Option 2: Vendored crates
 crate_repositories = use_extension("//third-party:extension.bzl", "crate_repositories")

examples/bzlmod/hello_world/anyhow.BUILD.bazel

@@ -0,0 +1,7 @@
+# See MODULE.bazel's additive_build_file tag.
+# Extra build file content from file
+
+alias(
+    name = "cargo_toml",
+    actual = "Cargo.toml",
+)