Add setting always_enable_metadata_output_groups (#3524)

Right now, only library targets provide metadata-related output groups.
This makes sense for compilation purposes, but building the metadata is
also useful for other things; namely, building *only* the metadata is
the fastest way to get diagnostic information for your workspace like
`cargo check` does.

This PR adds a new setting, `always_enable_metadata_output_groups`, that
enables both the `build_metadata` and `rustc_rmeta_output` output groups
for non-library targets (such as binaries and tests). That means that
the presence of metadata in `CrateInfo` can no longer be used to
determine pipelind compilation support, so a new attribute,
`metadata_supports_pipelining`, is added for that purpose.

As an example of the performance increase, compare a `bazel build` on a
binary w/o `--output_groups=rustc_rmeta_output`:

```
INFO: Elapsed time: 2.247s, Critical Path: 2.11s
```

to with it:

```
INFO: Elapsed time: 1.034s, Critical Path: 0.90s
```

Author: refi64

extensions/prost/private/prost.bzl

@@ -18,7 +18,12 @@ load("@rules_rust//rust/private:rust_analyzer.bzl", "write_rust_analyzer_spec_fi
 load("@rules_rust//rust/private:rustc.bzl", "rustc_compile_action")
 
 # buildifier: disable=bzl-visibility
-load("@rules_rust//rust/private:utils.bzl", "can_build_metadata")
+load(
+    "@rules_rust//rust/private:utils.bzl",
+    "can_build_metadata",
+    "can_use_metadata_for_pipelining",
+    "generate_output_diagnostics",
+)
 load("//:providers.bzl", "ProstProtoInfo")
 load(":prost_transform.bzl", "ProstTransformInfo")
 
@@ -181,6 +186,8 @@ def _compile_rust(
 
     lib = ctx.actions.declare_file(lib_name)
     rmeta = None
+    rustc_rmeta_output = None
+    metadata_supports_pipelining = False
 
     if can_build_metadata(toolchain, ctx, "rlib"):
         rmeta_name = "{prefix}{name}-{lib_hash}{extension}".format(
@@ -190,6 +197,8 @@ def _compile_rust(
             extension = ".rmeta",
         )
         rmeta = ctx.actions.declare_file(rmeta_name)
+        rustc_rmeta_output = generate_output_diagnostics(ctx, rmeta)
+        metadata_supports_pipelining = can_use_metadata_for_pipelining(toolchain, "rlib")
 
     providers = rustc_compile_action(
         ctx = ctx,
@@ -205,6 +214,8 @@ def _compile_rust(
             aliases = {},
             output = lib,
             metadata = rmeta,
+            metadata_supports_pipelining = metadata_supports_pipelining,
+            rustc_rmeta_output = rustc_rmeta_output,
             edition = edition,
             is_test = False,
             rustc_env = {},
@@ -373,7 +384,12 @@ rust_prost_aspect = aspect(
             executable = True,
             default = Label("//private:protoc_wrapper"),
         ),
-    } | RUSTC_ATTRS,
+    } | RUSTC_ATTRS | {
+        # Need to override this attribute to explicitly set the workspace.
+        "_always_enable_metadata_output_groups": attr.label(
+            default = Label("@rules_rust//rust/settings:always_enable_metadata_output_groups"),
+        ),
+    },
     fragments = ["cpp"],
     toolchains = [
         TOOLCHAIN_TYPE,

extensions/protobuf/proto.bzl

@@ -20,7 +20,16 @@ load("@rules_proto//proto:defs.bzl", "ProtoInfo")
 load("@rules_rust//rust/private:rustc.bzl", "rustc_compile_action")
 
 # buildifier: disable=bzl-visibility
-load("@rules_rust//rust/private:utils.bzl", "can_build_metadata", "compute_crate_name", "determine_output_hash", "find_toolchain", "transform_deps")
+load(
+    "@rules_rust//rust/private:utils.bzl",
+    "can_build_metadata",
+    "can_use_metadata_for_pipelining",
+    "compute_crate_name",
+    "determine_output_hash",
+    "find_toolchain",
+    "generate_output_diagnostics",
+    "transform_deps",
+)
 load(
     "//:toolchain.bzl",
     _generate_proto = "rust_generate_proto",
@@ -194,12 +203,16 @@ def _rust_proto_compile(protos, descriptor_sets, imports, crate_name, ctx, is_gr
         output_hash,
     ))
     rust_metadata = None
+    rustc_rmeta_output = None
+    metadata_supports_pipelining = False
     if can_build_metadata(toolchain, ctx, "rlib"):
         rust_metadata = ctx.actions.declare_file("%s/lib%s-%s.rmeta" % (
             output_dir,
             crate_name,
             output_hash,
         ))
+        rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata)
+        metadata_supports_pipelining = can_use_metadata_for_pipelining(toolchain, "rlib")
 
     # Gather all dependencies for compilation
     compile_action_deps = depset(
@@ -223,6 +236,8 @@ def _rust_proto_compile(protos, descriptor_sets, imports, crate_name, ctx, is_gr
             aliases = {},
             output = rust_lib,
             metadata = rust_metadata,
+            metadata_supports_pipelining = metadata_supports_pipelining,
+            rustc_rmeta_output = rustc_rmeta_output,
             edition = proto_toolchain.edition,
             rustc_env = {},
             is_test = False,
@@ -311,6 +326,9 @@ rust_proto_library = rule(
                 file of arguments to rustc: `@$(location //package:target)`.
             """,
         ),
+        "_always_enable_metadata_output_groups": attr.label(
+            default = Label("@rules_rust//rust/settings:always_enable_metadata_output_groups"),
+        ),
         "_optional_output_wrapper": attr.label(
             executable = True,
             cfg = "exec",

rust/private/common.bzl

@@ -51,6 +51,8 @@ def _create_crate_info(**kwargs):
         kwargs.update({"wrapped_crate_type": None})
     if not "metadata" in kwargs:
         kwargs.update({"metadata": None})
+    if not "metadata_supports_pipelining" in kwargs:
+        kwargs.update({"metadata_supports_pipelining": False})
     if not "rustc_rmeta_output" in kwargs:
         kwargs.update({"rustc_rmeta_output": None})
     if not "rustc_output" in kwargs:

rust/private/providers.bzl

@@ -25,6 +25,8 @@ CrateInfo = provider(
         "edition": "str: The edition of this crate.",
         "is_test": "bool: If the crate is being compiled in a test context",
         "metadata": "File: The output from rustc from producing the output file. It is optional.",
+        "metadata_supports_pipelining": "bool: If the metadata in 'metadata' (if present) is " +
+                                        "usable for pipelined compilation.",
         "name": "str: The name of this crate.",
         "output": "File: The output File that will be produced, depends on crate type.",
         "owner": "Label: The label of the target that produced this CrateInfo",
@@ -98,6 +100,17 @@ DepVariantInfo = provider(
     },
 )
 
+AlwaysEnableMetadataOutputGroupsInfo = provider(
+    doc = (
+        "Enable the 'metadata' and 'rustc_rmeta_output' groups for all targets, " +
+        "even if not a library or if pipelining is disabled"
+    ),
+    fields = {
+        "always_enable_metadata_output_groups": ("bool: Whether or not to always enable " +
+                                                 "metadata-related output groups"),
+    },
+)
+
 RustcOutputDiagnosticsInfo = provider(
     doc = (
         "Save json diagnostics from rustc. Json diagnostics are able to be " +

rust/private/rust.bzl

@@ -28,6 +28,7 @@ load("//rust/private:rustc.bzl", "rustc_compile_action")
 load(
     "//rust/private:utils.bzl",
     "can_build_metadata",
+    "can_use_metadata_for_pipelining",
     "compute_crate_name",
     "crate_root_src",
     "dedent",
@@ -177,12 +178,22 @@ def _rust_library_common(ctx, crate_type):
     rust_lib = ctx.actions.declare_file(rust_lib_name)
     rust_metadata = None
     rustc_rmeta_output = None
-    if can_build_metadata(toolchain, ctx, crate_type) and not ctx.attr.disable_pipelining:
+    metadata_supports_pipelining = False
+    if can_build_metadata(
+        toolchain,
+        ctx,
+        crate_type,
+        disable_pipelining = getattr(ctx.attr, "disable_pipelining", False),
+    ):
         rust_metadata = ctx.actions.declare_file(
             paths.replace_extension(rust_lib_name, ".rmeta"),
             sibling = rust_lib,
         )
         rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata)
+        metadata_supports_pipelining = (
+            can_use_metadata_for_pipelining(toolchain, crate_type) and
+            not ctx.attr.disable_pipelining
+        )
 
     deps = transform_deps(ctx.attr.deps)
     proc_macro_deps = transform_deps(ctx.attr.proc_macro_deps + get_import_macro_deps(ctx))
@@ -203,6 +214,7 @@ def _rust_library_common(ctx, crate_type):
             output = rust_lib,
             rustc_output = generate_output_diagnostics(ctx, rust_lib),
             metadata = rust_metadata,
+            metadata_supports_pipelining = metadata_supports_pipelining,
             rustc_rmeta_output = rustc_rmeta_output,
             edition = get_edition(ctx.attr, toolchain, ctx.label),
             rustc_env = ctx.attr.rustc_env,
@@ -242,6 +254,15 @@ def _rust_binary_impl(ctx):
         crate_root = crate_root_src(ctx.attr.name, ctx.attr.crate_name, ctx.files.srcs, ctx.attr.crate_type)
     srcs, compile_data, crate_root = transform_sources(ctx, ctx.files.srcs, ctx.files.compile_data, crate_root)
 
+    rust_metadata = None
+    rustc_rmeta_output = None
+    if can_build_metadata(toolchain, ctx, ctx.attr.crate_type):
+        rust_metadata = ctx.actions.declare_file(
+            paths.replace_extension("lib" + crate_name, ".rmeta"),
+            sibling = output,
+        )
+        rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata)
+
     providers = rustc_compile_action(
         ctx = ctx,
         attr = ctx.attr,
@@ -256,6 +277,8 @@ def _rust_binary_impl(ctx):
             aliases = ctx.attr.aliases,
             output = output,
             rustc_output = generate_output_diagnostics(ctx, output),
+            metadata = rust_metadata,
+            rustc_rmeta_output = rustc_rmeta_output,
             edition = get_edition(ctx.attr, toolchain, ctx.label),
             rustc_env = ctx.attr.rustc_env,
             rustc_env_files = ctx.files.rustc_env_files,
@@ -337,6 +360,15 @@ def _rust_test_impl(ctx):
                 ),
             )
 
+        rust_metadata = None
+        rustc_rmeta_output = None
+        if can_build_metadata(toolchain, ctx, crate_type):
+            rust_metadata = ctx.actions.declare_file(
+                paths.replace_extension("lib" + crate_name, ".rmeta"),
+                sibling = output,
+            )
+            rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata)
+
         # Need to consider all src files together when transforming
         srcs = depset(ctx.files.srcs, transitive = [crate.srcs]).to_list()
         compile_data = depset(ctx.files.compile_data, transitive = [crate.compile_data]).to_list()
@@ -371,6 +403,8 @@ def _rust_test_impl(ctx):
             aliases = aliases,
             output = output,
             rustc_output = generate_output_diagnostics(ctx, output),
+            metadata = rust_metadata,
+            rustc_rmeta_output = rustc_rmeta_output,
             edition = crate.edition,
             rustc_env = rustc_env,
             rustc_env_files = rustc_env_files,
@@ -381,6 +415,8 @@ def _rust_test_impl(ctx):
             owner = ctx.label,
         )
     else:
+        crate_name = compute_crate_name(ctx.workspace_name, ctx.label, toolchain, ctx.attr.crate_name)
+
         crate_root = getattr(ctx.file, "crate_root", None)
 
         if not crate_root:
@@ -402,6 +438,15 @@ def _rust_test_impl(ctx):
                 ),
             )
 
+        rust_metadata = None
+        rustc_rmeta_output = None
+        if can_build_metadata(toolchain, ctx, crate_type):
+            rust_metadata = ctx.actions.declare_file(
+                paths.replace_extension("lib" + crate_name, ".rmeta"),
+                sibling = output,
+            )
+            rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata)
+
         data_paths = depset(direct = getattr(ctx.attr, "data", [])).to_list()
         rustc_env = expand_dict_value_locations(
             ctx,
@@ -412,7 +457,7 @@ def _rust_test_impl(ctx):
 
         # Target is a standalone crate. Build the test binary as its own crate.
         crate_info_dict = dict(
-            name = compute_crate_name(ctx.workspace_name, ctx.label, toolchain, ctx.attr.crate_name),
+            name = crate_name,
             type = crate_type,
             root = crate_root,
             srcs = depset(srcs),
@@ -421,6 +466,8 @@ def _rust_test_impl(ctx):
             aliases = ctx.attr.aliases,
             output = output,
             rustc_output = generate_output_diagnostics(ctx, output),
+            metadata = rust_metadata,
+            rustc_rmeta_output = rustc_rmeta_output,
             edition = get_edition(ctx.attr, toolchain, ctx.label),
             rustc_env = rustc_env,
             rustc_env_files = ctx.files.rustc_env_files,
@@ -534,6 +581,9 @@ def _stamp_attribute(default_value):
 
 # Internal attributes core to Rustc actions.
 RUSTC_ATTRS = {
+    "_always_enable_metadata_output_groups": attr.label(
+        default = Label("//rust/settings:always_enable_metadata_output_groups"),
+    ),
     "_error_format": attr.label(
         default = Label("//rust/settings:error_format"),
     ),

rust/private/rustc.bzl

@@ -27,7 +27,15 @@ load("@rules_cc//cc/common:cc_info.bzl", "CcInfo")
 load(":common.bzl", "rust_common")
 load(":compat.bzl", "abs")
 load(":lto.bzl", "construct_lto_arguments")
-load(":providers.bzl", "AllocatorLibrariesImplInfo", "AllocatorLibrariesInfo", "LintsInfo", "RustcOutputDiagnosticsInfo", _BuildInfo = "BuildInfo")
+load(
+    ":providers.bzl",
+    "AllocatorLibrariesImplInfo",
+    "AllocatorLibrariesInfo",
+    "AlwaysEnableMetadataOutputGroupsInfo",
+    "LintsInfo",
+    "RustcOutputDiagnosticsInfo",
+    _BuildInfo = "BuildInfo",
+)
 load(":rustc_resource_set.bzl", "get_rustc_resource_set", "is_codegen_units_enabled")
 load(":stamp.bzl", "is_stamping_enabled")
 load(
@@ -307,7 +315,7 @@ def collect_deps(
             # If it doesn't (for example a custom library that exports crate_info),
             # we depend on crate_info.output.
             depend_on = crate_info.metadata
-            if not crate_info.metadata:
+            if not crate_info.metadata or not crate_info.metadata_supports_pipelining:
                 depend_on = crate_info.output
 
             # If this dependency is a proc_macro, it still can be used for lib crates
@@ -2014,7 +2022,7 @@ def _crate_to_link_flag_metadata(crate):
         crate_info = crate
 
     lib_or_meta = crate_info.metadata
-    if not crate_info.metadata:
+    if not crate_info.metadata or not crate_info.metadata_supports_pipelining:
         lib_or_meta = crate_info.output
     return ["--extern={}={}".format(name, lib_or_meta.path)]
 
@@ -2329,6 +2337,30 @@ def get_error_format(attr, attr_name):
         return getattr(attr, attr_name)[ErrorFormatInfo].error_format
     return "human"
 
+def _always_enable_metadata_output_groups_impl(ctx):
+    """Implementation of the `always_enable_metadata_output_groups` rule
+
+    Args:
+        ctx (ctx): The rule's context object
+
+    Returns:
+        list: A list containing the AlwaysEnableMetadataOutputGroupsInfo provider
+    """
+    return [AlwaysEnableMetadataOutputGroupsInfo(
+        always_enable_metadata_output_groups = ctx.build_setting_value,
+    )]
+
+always_enable_metadata_output_groups = rule(
+    doc = (
+        "Setting this flag from the command line with `--@rules_rust//rust/settings:always_enable_metadata_output_groups` " +
+        "will cause all rules to generate the `metadata` and `rustc_rmeta_output` output groups, " +
+        "rather than the default behavior of only generated them for libraries when pipelined " +
+        "compilation is enabled."
+    ),
+    implementation = _always_enable_metadata_output_groups_impl,
+    build_setting = config.bool(flag = True),
+)
+
 def _rustc_output_diagnostics_impl(ctx):
     """Implementation of the `rustc_output_diagnostics` rule
 
@@ -2346,8 +2378,8 @@ rustc_output_diagnostics = rule(
     doc = (
         "Setting this flag from the command line with `--@rules_rust//rust/settings:rustc_output_diagnostics` " +
         "makes rules_rust save rustc json output(suitable for consumption by rust-analyzer) in a file. " +
-        "These are accessible via the " +
-        "`rustc_rmeta_output`(for pipelined compilation) and `rustc_output` output groups. " +
+        "These are accessible via the `rustc_rmeta_output` (for pipelined compilation or if " +
+        "`always_enable_metadata_output_groups` is enabled) and `rustc_output` output groups. " +
         "You can find these using `bazel cquery`"
     ),
     implementation = _rustc_output_diagnostics_impl,