feat: rustdoc depinfo rebuild detection via -Zrustdoc-depinfo (#15359)

### What does this PR try to resolve?

This leverages the unstable `--emit=depinfo` option from rustdoc,
so that rustdoc invocation rebuild can be better tracked
without traversing the entire directory.

Some design decisions made in the current implementation:

* Rustdoc's depinfo doesn't and shouldn't emit to `target/doc`,
  as the directory is considered part of the final artifact directory.
  In regard to that, we specify the dep-info output path to
  the fingerprint directory of rustdoc invocation.
  It looks like this
  `target/debug/.fingerprint/serde-12d29d32b3b8b38f/doc-lib-serde.d`.
* We also start supporting `-Zchecksum-freshness` as a side effect.
  Could make it a separate PR if desired.
* `-Zbinary-dep-depinfo` is not enabled along with this,
  since doc generations don't really require any binary dependencies.

### How should we test and review this PR?

The tests added has covered these cases:

* target src outside package root, e.g., `lib.path = "../lib.rs"`
* `#[doc = include_str!("../outside/pkgroot")]`
* `#[path = "../outside/pkgroot"]`
* `env!`

### Additional information

Fixes #12266
Closes #15205

Author: epage

src/cargo/core/compiler/fingerprint/mod.rs

@@ -1494,7 +1494,9 @@ fn calculate_normal(
 
     // Afterwards calculate our own fingerprint information.
     let build_root = build_root(build_runner);
-    let local = if unit.mode.is_doc() || unit.mode.is_doc_scrape() {
+    let is_any_doc_gen = unit.mode.is_doc() || unit.mode.is_doc_scrape();
+    let rustdoc_depinfo_enabled = build_runner.bcx.gctx.cli_unstable().rustdoc_depinfo;
+    let local = if is_any_doc_gen && !rustdoc_depinfo_enabled {
         // rustdoc does not have dep-info files.
         let fingerprint = pkg_fingerprint(build_runner.bcx, &unit.pkg).with_context(|| {
             format!(

src/cargo/core/compiler/mod.rs

@@ -763,6 +763,19 @@ fn prepare_rustdoc(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> CargoResu
     add_error_format_and_color(build_runner, &mut rustdoc);
     add_allow_features(build_runner, &mut rustdoc);
 
+    if build_runner.bcx.gctx.cli_unstable().rustdoc_depinfo {
+        // invocation-specific is required for keeping the original rustdoc emission
+        let mut arg = OsString::from("--emit=invocation-specific,dep-info=");
+        arg.push(rustdoc_dep_info_loc(build_runner, unit));
+        rustdoc.arg(arg);
+
+        if build_runner.bcx.gctx.cli_unstable().checksum_freshness {
+            rustdoc.arg("-Z").arg("checksum-hash-algorithm=blake3");
+        }
+
+        rustdoc.arg("-Zunstable-options");
+    }
+
     if let Some(trim_paths) = unit.profile.trim_paths.as_ref() {
         trim_paths_args_rustdoc(&mut rustdoc, build_runner, unit, trim_paths)?;
     }
@@ -838,6 +851,20 @@ fn rustdoc(build_runner: &mut BuildRunner<'_, '_>, unit: &Unit) -> CargoResult<W
     let package_id = unit.pkg.package_id();
     let manifest_path = PathBuf::from(unit.pkg.manifest_path());
     let target = Target::clone(&unit.target);
+
+    let rustdoc_dep_info_loc = rustdoc_dep_info_loc(build_runner, unit);
+    let dep_info_loc = fingerprint::dep_info_loc(build_runner, unit);
+    let build_dir = build_runner.bcx.ws.build_dir().into_path_unlocked();
+    let pkg_root = unit.pkg.root().to_path_buf();
+    let cwd = rustdoc
+        .get_cwd()
+        .unwrap_or_else(|| build_runner.bcx.gctx.cwd())
+        .to_path_buf();
+    let fingerprint_dir = build_runner.files().fingerprint_dir(unit);
+    let is_local = unit.is_local();
+    let env_config = Arc::clone(build_runner.bcx.gctx.env_config()?);
+    let rustdoc_depinfo_enabled = build_runner.bcx.gctx.cli_unstable().rustdoc_depinfo;
+
     let mut output_options = OutputOptions::new(build_runner, unit);
     let script_metadata = build_runner.find_build_script_metadata(unit);
     let scrape_outputs = if should_include_scrape_units(build_runner.bcx, unit) {
@@ -903,6 +930,7 @@ fn rustdoc(build_runner: &mut BuildRunner<'_, '_>, unit: &Unit) -> CargoResult<W
             paths::remove_dir_all(crate_dir)?;
         }
         state.running(&rustdoc);
+        let timestamp = paths::set_invocation_time(&fingerprint_dir)?;
 
         let result = rustdoc
             .exec_with_streaming(
@@ -930,6 +958,29 @@ fn rustdoc(build_runner: &mut BuildRunner<'_, '_>, unit: &Unit) -> CargoResult<W
             return Err(e);
         }
 
+        if rustdoc_depinfo_enabled && rustdoc_dep_info_loc.exists() {
+            fingerprint::translate_dep_info(
+                &rustdoc_dep_info_loc,
+                &dep_info_loc,
+                &cwd,
+                &pkg_root,
+                &build_dir,
+                &rustdoc,
+                // Should we track source file for doc gen?
+                is_local,
+                &env_config,
+            )
+            .with_context(|| {
+                internal(format_args!(
+                    "could not parse/generate dep info at: {}",
+                    rustdoc_dep_info_loc.display()
+                ))
+            })?;
+            // This mtime shift allows Cargo to detect if a source file was
+            // modified in the middle of the build.
+            paths::set_file_time_no_err(dep_info_loc, timestamp);
+        }
+
         Ok(())
     }))
 }
@@ -2012,3 +2063,10 @@ fn scrape_output_path(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> CargoR
         .outputs(unit)
         .map(|outputs| outputs[0].path.clone())
 }
+
+/// Gets the dep-info file emitted by rustdoc.
+fn rustdoc_dep_info_loc(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> PathBuf {
+    let mut loc = build_runner.files().fingerprint_file_path(unit, "");
+    loc.set_extension("d");
+    loc
+}

src/cargo/core/features.rs

@@ -790,6 +790,7 @@ unstable_cli_options!(
     public_dependency: bool = ("Respect a dependency's `public` field in Cargo.toml to control public/private dependencies"),
     publish_timeout: bool = ("Enable the `publish.timeout` key in .cargo/config.toml file"),
     root_dir: Option<PathBuf> = ("Set the root directory relative to which paths are printed (defaults to workspace root)"),
+    rustdoc_depinfo: bool = ("Use dep-info files in rustdoc rebuild detection"),
     rustdoc_map: bool = ("Allow passing external documentation mappings to rustdoc"),
     rustdoc_scrape_examples: bool = ("Allows Rustdoc to scrape code examples from reverse-dependencies"),
     sbom: bool = ("Enable the `sbom` option in build config in .cargo/config.toml file"),
@@ -1303,6 +1304,7 @@ impl CliUnstable {
             "trim-paths" => self.trim_paths = parse_empty(k, v)?,
             "publish-timeout" => self.publish_timeout = parse_empty(k, v)?,
             "root-dir" => self.root_dir = v.map(|v| v.into()),
+            "rustdoc-depinfo" => self.rustdoc_depinfo = parse_empty(k, v)?,
             "rustdoc-map" => self.rustdoc_map = parse_empty(k, v)?,
             "rustdoc-scrape-examples" => self.rustdoc_scrape_examples = parse_empty(k, v)?,
             "sbom" => self.sbom = parse_empty(k, v)?,

src/doc/src/reference/unstable.md

@@ -98,6 +98,7 @@ Each new feature described below should explain how to use it.
     * [rustdoc-map](#rustdoc-map) --- Provides mappings for documentation to link to external sites like [docs.rs](https://docs.rs/).
     * [scrape-examples](#scrape-examples) --- Shows examples within documentation.
     * [output-format](#output-format-for-rustdoc) --- Allows documentation to also be emitted in the experimental [JSON format](https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc_json_types/).
+    * [rustdoc-depinfo](#rustdoc-depinfo) --- Use dep-info files in rustdoc rebuild detection.
 * `Cargo.toml` extensions
     * [Profile `rustflags` option](#profile-rustflags-option) --- Passed directly to rustc.
     * [codegen-backend](#codegen-backend) --- Select the codegen backend used by rustc.
@@ -1904,6 +1905,14 @@ Requires `-Zunstable-options`.
 See [`cargo package --message-format`](../commands/cargo-package.md#option-cargo-package---message-format)
 for more information.
 
+## rustdoc depinfo
+
+* Tracking Issue: [#00000](https://github.com/rust-lang/cargo/issues/00000)
+
+The `-Z rustdoc-depinfo` flag leverages rustdoc's dep-info files to determine
+whether documentations are required to re-generate. This can be combined with
+`-Z checksum-freshness` to detect checksum changes rather than file mtime.
+
 # Stabilized and removed features
 
 ## Compile progress