feat(fingerprint): integrate rustdoc dep-info files

This leverages the unstable `--emit=depinfo` option from rustdoc,
so that rustdoc invocation rebuild can be better tracked
without traversing the entire directory.

Some design decisions:

* Rustdoc's depinfo doesn't and shouldn't emit to `target/doc`,
  as the directory is considered part of the final artifact directory.
  In regard to that, we specify the dep-info output path to
  the fingerprint directory of rustdoc invocation.
  It looks like this
  `target/debug/.fingerprint/serde-12d29d32b3b8b38f/doc-lib-serde.d`.
* We also start supporting `-Zchecksum-freshness` as a side effect.
  Could make it a separate PR if desired.
* `-Zbinary-dep-depinfo` is not enabled along with this,
  since doc generations don't really require any binary dependencies.

Author: weihanglo

src/cargo/core/compiler/fingerprint/mod.rs

@@ -1494,7 +1494,9 @@ fn calculate_normal(
 
     // Afterwards calculate our own fingerprint information.
     let build_root = build_root(build_runner);
-    let local = if unit.mode.is_doc() || unit.mode.is_doc_scrape() {
+    let is_any_doc_gen = unit.mode.is_doc() || unit.mode.is_doc_scrape();
+    let rustdoc_depinfo_enabled = build_runner.bcx.gctx.cli_unstable().rustdoc_depinfo;
+    let local = if is_any_doc_gen && !rustdoc_depinfo_enabled {
         // rustdoc does not have dep-info files.
         let fingerprint = pkg_fingerprint(build_runner.bcx, &unit.pkg).with_context(|| {
             format!(

src/cargo/core/compiler/mod.rs

@@ -763,6 +763,19 @@ fn prepare_rustdoc(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> CargoResu
     add_error_format_and_color(build_runner, &mut rustdoc);
     add_allow_features(build_runner, &mut rustdoc);
 
+    if build_runner.bcx.gctx.cli_unstable().rustdoc_depinfo {
+        // invocation-specific is required for keeping the original rustdoc emission
+        let mut arg = OsString::from("--emit=invocation-specific,dep-info=");
+        arg.push(rustdoc_dep_info_loc(build_runner, unit));
+        rustdoc.arg(arg);
+
+        if build_runner.bcx.gctx.cli_unstable().checksum_freshness {
+            rustdoc.arg("-Z").arg("checksum-hash-algorithm=blake3");
+        }
+
+        rustdoc.arg("-Zunstable-options");
+    }
+
     if let Some(trim_paths) = unit.profile.trim_paths.as_ref() {
         trim_paths_args_rustdoc(&mut rustdoc, build_runner, unit, trim_paths)?;
     }
@@ -838,6 +851,20 @@ fn rustdoc(build_runner: &mut BuildRunner<'_, '_>, unit: &Unit) -> CargoResult<W
     let package_id = unit.pkg.package_id();
     let manifest_path = PathBuf::from(unit.pkg.manifest_path());
     let target = Target::clone(&unit.target);
+
+    let rustdoc_dep_info_loc = rustdoc_dep_info_loc(build_runner, unit);
+    let dep_info_loc = fingerprint::dep_info_loc(build_runner, unit);
+    let build_dir = build_runner.bcx.ws.build_dir().into_path_unlocked();
+    let pkg_root = unit.pkg.root().to_path_buf();
+    let cwd = rustdoc
+        .get_cwd()
+        .unwrap_or_else(|| build_runner.bcx.gctx.cwd())
+        .to_path_buf();
+    let fingerprint_dir = build_runner.files().fingerprint_dir(unit);
+    let is_local = unit.is_local();
+    let env_config = Arc::clone(build_runner.bcx.gctx.env_config()?);
+    let rustdoc_depinfo_enabled = build_runner.bcx.gctx.cli_unstable().rustdoc_depinfo;
+
     let mut output_options = OutputOptions::new(build_runner, unit);
     let script_metadata = build_runner.find_build_script_metadata(unit);
     let scrape_outputs = if should_include_scrape_units(build_runner.bcx, unit) {
@@ -903,6 +930,7 @@ fn rustdoc(build_runner: &mut BuildRunner<'_, '_>, unit: &Unit) -> CargoResult<W
             paths::remove_dir_all(crate_dir)?;
         }
         state.running(&rustdoc);
+        let timestamp = paths::set_invocation_time(&fingerprint_dir)?;
 
         let result = rustdoc
             .exec_with_streaming(
@@ -930,6 +958,29 @@ fn rustdoc(build_runner: &mut BuildRunner<'_, '_>, unit: &Unit) -> CargoResult<W
             return Err(e);
         }
 
+        if rustdoc_depinfo_enabled && rustdoc_dep_info_loc.exists() {
+            fingerprint::translate_dep_info(
+                &rustdoc_dep_info_loc,
+                &dep_info_loc,
+                &cwd,
+                &pkg_root,
+                &build_dir,
+                &rustdoc,
+                // Should we track source file for doc gen?
+                is_local,
+                &env_config,
+            )
+            .with_context(|| {
+                internal(format_args!(
+                    "could not parse/generate dep info at: {}",
+                    rustdoc_dep_info_loc.display()
+                ))
+            })?;
+            // This mtime shift allows Cargo to detect if a source file was
+            // modified in the middle of the build.
+            paths::set_file_time_no_err(dep_info_loc, timestamp);
+        }
+
         Ok(())
     }))
 }
@@ -2012,3 +2063,10 @@ fn scrape_output_path(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> CargoR
         .outputs(unit)
         .map(|outputs| outputs[0].path.clone())
 }
+
+/// Gets the dep-info file emitted by rustdoc.
+fn rustdoc_dep_info_loc(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> PathBuf {
+    let mut loc = build_runner.files().fingerprint_file_path(unit, "");
+    loc.set_extension("d");
+    loc
+}

tests/testsuite/doc.rs

@@ -2949,15 +2949,17 @@ fn rebuild_tracks_target_src_outside_package_root() {
     p.cargo("doc --verbose -Zrustdoc-depinfo")
         .masquerade_as_nightly_cargo(&["rustdoc-depinfo"])
         .with_stderr_data(str![[r#"
-[FRESH] foo v0.0.0 ([ROOT]/parent/foo)
+[DIRTY] foo v0.0.0 ([ROOT]/parent/foo): the file `../lib.rs` has changed ([TIME_DIFF_AFTER_LAST_BUILD])
+[DOCUMENTING] foo v0.0.0 ([ROOT]/parent/foo)
+[RUNNING] `rustdoc [..]`
 [FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
 [GENERATED] [ROOT]/parent/foo/target/doc/foo/index.html
 
 "#]])
         .run();
 
     let doc_html = p.read_file("target/doc/foo/index.html");
-    assert!(!doc_html.contains("depinfo-after"));
+    assert!(doc_html.contains("depinfo-after"));
 }
 
 #[cargo_test(nightly, reason = "`rustdoc --emit` is unstable")]
@@ -2986,15 +2988,17 @@ fn rebuild_tracks_include_str() {
     p.cargo("doc --verbose -Zrustdoc-depinfo")
         .masquerade_as_nightly_cargo(&["rustdoc-depinfo"])
         .with_stderr_data(str![[r#"
-[FRESH] foo v0.5.0 ([ROOT]/parent/foo)
+[DIRTY] foo v0.5.0 ([ROOT]/parent/foo): the file `src/../../README` has changed ([TIME_DIFF_AFTER_LAST_BUILD])
+[DOCUMENTING] foo v0.5.0 ([ROOT]/parent/foo)
+[RUNNING] `rustdoc [..]`
 [FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
 [GENERATED] [ROOT]/parent/foo/target/doc/foo/index.html
 
 "#]])
         .run();
 
     let doc_html = p.read_file("target/doc/foo/index.html");
-    assert!(!doc_html.contains("depinfo-after"));
+    assert!(doc_html.contains("depinfo-after"));
 }
 
 #[cargo_test(nightly, reason = "`rustdoc --emit` is unstable")]
@@ -3023,15 +3027,17 @@ fn rebuild_tracks_path_attr() {
     p.cargo("doc --verbose -Zrustdoc-depinfo")
         .masquerade_as_nightly_cargo(&["rustdoc-depinfo"])
         .with_stderr_data(str![[r#"
-[FRESH] foo v0.5.0 ([ROOT]/parent/foo)
+[DIRTY] foo v0.5.0 ([ROOT]/parent/foo): the file `src/../../bar.rs` has changed ([TIME_DIFF_AFTER_LAST_BUILD])
+[DOCUMENTING] foo v0.5.0 ([ROOT]/parent/foo)
+[RUNNING] `rustdoc [..]`
 [FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
 [GENERATED] [ROOT]/parent/foo/target/doc/foo/index.html
 
 "#]])
         .run();
 
     let doc_html = p.read_file("target/doc/foo/index.html");
-    assert!(!doc_html.contains("depinfo-after"));
+    assert!(doc_html.contains("depinfo-after"));
 }
 
 #[cargo_test(nightly, reason = "`rustdoc --emit` is unstable")]
@@ -3060,15 +3066,17 @@ fn rebuild_tracks_env() {
         .env(env, "# depinfo-after")
         .masquerade_as_nightly_cargo(&["rustdoc-depinfo"])
         .with_stderr_data(str![[r#"
-[FRESH] foo v0.5.0 ([ROOT]/foo)
+[DIRTY] foo v0.5.0 ([ROOT]/foo): the environment variable __RUSTDOC_INJECTED changed
+[DOCUMENTING] foo v0.5.0 ([ROOT]/foo)
+[RUNNING] `rustdoc [..]`
 [FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
 [GENERATED] [ROOT]/foo/target/doc/foo/index.html
 
 "#]])
         .run();
 
     let doc_html = p.read_file("target/doc/foo/index.html");
-    assert!(!doc_html.contains("depinfo-after"));
+    assert!(doc_html.contains("depinfo-after"));
 }
 
 #[cargo_test(nightly, reason = "`rustdoc --emit` is unstable")]
@@ -3119,21 +3127,27 @@ fn rebuild_tracks_env_in_dep() {
     p.cargo("doc --verbose -Zrustdoc-depinfo")
         .env(env, "# depinfo-after")
         .masquerade_as_nightly_cargo(&["rustdoc-depinfo"])
-        .with_stderr_data(str![[r#"
+        .with_stderr_data(
+            str![[r#"
+[DIRTY] bar v0.1.0: the environment variable __RUSTDOC_INJECTED changed
+[DOCUMENTING] bar v0.1.0
 [DIRTY] bar v0.1.0: the environment variable __RUSTDOC_INJECTED changed
 [CHECKING] bar v0.1.0
 [RUNNING] `rustc --crate-name bar [..]`
+[RUNNING] `rustdoc [..]--crate-name bar [..]`
 [DIRTY] foo v0.0.0 ([ROOT]/foo): the dependency bar was rebuilt
 [DOCUMENTING] foo v0.0.0 ([ROOT]/foo)
 [RUNNING] `rustdoc [..]--crate-name foo [..]`
 [FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
 [GENERATED] [ROOT]/foo/target/doc/foo/index.html
 
-"#]])
+"#]]
+            .unordered(),
+        )
         .run();
 
     let doc_html = p.read_file("target/doc/bar/index.html");
-    assert!(!doc_html.contains("depinfo-after"));
+    assert!(doc_html.contains("depinfo-after"));
 }
 
 #[cargo_test(
@@ -3167,7 +3181,7 @@ fn rebuild_tracks_checksum() {
     p.cargo("doc --verbose -Zrustdoc-depinfo -Zchecksum-freshness")
         .masquerade_as_nightly_cargo(&["rustdoc-depinfo"])
         .with_stderr_data(str![[r#"
-[DIRTY] foo v0.5.0 ([ROOT]/parent/foo): the precalculated components changed
+[DIRTY] foo v0.5.0 ([ROOT]/parent/foo): file size changed (16 != 15) for `src/../../README`
 [DOCUMENTING] foo v0.5.0 ([ROOT]/parent/foo)
 [RUNNING] `rustdoc [..]`
 [FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s