feat: rebuild(-ish) rustdoc json for different version of same crates

Author: motorailgun

src/cargo/core/compiler/build_runner/compilation_files.rs

@@ -523,18 +523,33 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
                 vec![]
             }
             CompileMode::Doc => {
-                let path = if bcx.build_config.intent.wants_doc_json_output() {
-                    self.output_dir(unit)
-                        .join(format!("{}.json", unit.target.crate_name()))
+                let wants_json_doc = bcx.build_config.intent.wants_doc_json_output();
+
+                let path = if wants_json_doc {
+                    // Always use 'new' layout for '--output-format=json'.
+                    let crate_name = unit.target.crate_name();
+                    self.out_dir_new_layout(unit)
+                        .join(format!("{crate_name}.json"))
                 } else {
                     self.output_dir(unit)
                         .join(unit.target.crate_name())
                         .join("index.html")
                 };
 
+                // Uplift if output is json, from 'new' layout location for backward compatibility
+                // See #16773.
+                let hardlink = if wants_json_doc {
+                    Some(
+                        self.output_dir(unit)
+                            .join(format!("{}.json", unit.target.crate_name())),
+                    )
+                } else {
+                    None
+                };
+
                 let mut outputs = vec![OutputFile {
                     path,
-                    hardlink: None,
+                    hardlink,
                     export_path: None,
                     flavor: FileFlavor::Normal,
                 }];

src/cargo/core/compiler/mod.rs

@@ -868,7 +868,16 @@ fn prepare_rustdoc(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> CargoResu
     add_cap_lints(bcx, unit, &mut rustdoc);
 
     unit.kind.add_target_arg(&mut rustdoc);
-    let doc_dir = build_runner.files().output_dir(unit);
+
+    let doc_dir = if build_runner.bcx.build_config.intent.wants_doc_json_output() {
+        // Always use new layout for '--output-format=json'.
+        // In fix for https://github.com/rust-lang/cargo/issues/16291
+
+        build_runner.files().out_dir_new_layout(unit)
+    } else {
+        build_runner.files().output_dir(unit)
+    };
+
     rustdoc.arg("-o").arg(&doc_dir);
     rustdoc.args(&features_args(unit));
     rustdoc.args(&check_cfg_args(unit));
@@ -972,6 +981,7 @@ fn rustdoc(build_runner: &mut BuildRunner<'_, '_>, unit: &Unit) -> CargoResult<W
     let mut rustdoc = prepare_rustdoc(build_runner, unit)?;
 
     let crate_name = unit.target.crate_name();
+    let is_json_output = build_runner.bcx.build_config.intent.wants_doc_json_output();
     let doc_dir = build_runner.files().output_dir(unit);
     // Create the documentation directory ahead of time as rustdoc currently has
     // a bug where concurrent invocations will race to create this directory if
@@ -1055,13 +1065,15 @@ fn rustdoc(build_runner: &mut BuildRunner<'_, '_>, unit: &Unit) -> CargoResult<W
             }
         }
 
-        let crate_dir = doc_dir.join(&crate_name);
-        if crate_dir.exists() {
-            // Remove output from a previous build. This ensures that stale
-            // files for removed items are removed.
-            debug!("removing pre-existing doc directory {:?}", crate_dir);
-            paths::remove_dir_all(crate_dir)?;
-        }
+        if !is_json_output {
+            let crate_dir = doc_dir.join(&crate_name);
+            if crate_dir.exists() {
+                // Remove output from a previous build. This ensures that stale
+                // files for removed items are removed.
+                debug!("removing pre-existing doc directory {:?}", crate_dir);
+                paths::remove_dir_all(&crate_dir)?;
+            }
+        };
         state.running(&rustdoc);
         let timestamp = paths::set_invocation_time(&fingerprint_dir)?;
 

src/cargo/core/compiler/rustdoc.rs

@@ -250,12 +250,6 @@ pub fn add_output_format(
     build_runner: &BuildRunner<'_, '_>,
     rustdoc: &mut ProcessBuilder,
 ) -> CargoResult<()> {
-    let gctx = build_runner.bcx.gctx;
-    if !gctx.cli_unstable().unstable_options {
-        tracing::debug!("`unstable-options` is ignored, required -Zunstable-options flag");
-        return Ok(());
-    }
-
     if build_runner.bcx.build_config.intent.wants_doc_json_output() {
         rustdoc.arg("-Zunstable-options");
         rustdoc.arg("--output-format=json");

tests/testsuite/build_dir.rs

@@ -535,6 +535,7 @@ fn cargo_rustdoc_json_should_output_to_target_dir() {
 [ROOT]/foo/build-dir/debug/build/foo/[HASH]/fingerprint/doc-lib-foo
 [ROOT]/foo/build-dir/debug/build/foo/[HASH]/fingerprint/doc-lib-foo.json
 [ROOT]/foo/build-dir/debug/build/foo/[HASH]/fingerprint/invoked.timestamp
+[ROOT]/foo/build-dir/debug/build/foo/[HASH]/out/foo.json
 
 "#
     ]);

tests/testsuite/build_dir_legacy.rs

@@ -492,6 +492,7 @@ fn cargo_rustdoc_json_should_output_to_target_dir() {
 [ROOT]/foo/build-dir/debug/.fingerprint/foo-[HASH]/doc-lib-foo
 [ROOT]/foo/build-dir/debug/.fingerprint/foo-[HASH]/doc-lib-foo.json
 [ROOT]/foo/build-dir/debug/.fingerprint/foo-[HASH]/invoked.timestamp
+[ROOT]/foo/build-dir/debug/build/foo-[HASH]/out/foo.json
 
 "#
     ]);

tests/testsuite/rustdoc.rs

@@ -46,7 +46,7 @@ fn rustdoc_simple_json() {
         .masquerade_as_nightly_cargo(&["rustdoc-output-format"])
         .with_stderr_data(str![[r#"
 [DOCUMENTING] foo v0.0.1 ([ROOT]/foo)
-[RUNNING] `rustdoc [..] --crate-name foo [..]-o [ROOT]/foo/target/doc [..] --output-format=json[..]
+[RUNNING] `rustdoc [..] --crate-name foo [..]-o [ROOT]/foo/target/debug/build/foo-[HASH]/out [..] --output-format=json[..]
 [FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
 [GENERATED] [ROOT]/foo/target/doc/foo.json
 
@@ -63,7 +63,7 @@ fn rustdoc_json_with_new_layout() {
         .masquerade_as_nightly_cargo(&["rustdoc-output-format"])
         .with_stderr_data(str![[r#"
 [DOCUMENTING] foo v0.0.1 ([ROOT]/foo)
-[RUNNING] `rustdoc [..] --crate-name foo [..]-o [ROOT]/foo/target/doc [..] --output-format=json[..]
+[RUNNING] `rustdoc [..] --crate-name foo [..]-o [ROOT]/foo/target/debug/build/foo/[HASH]/out [..] --output-format=json[..]
 [FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
 [GENERATED] [ROOT]/foo/target/doc/foo.json
 
@@ -434,6 +434,6 @@ fn rustdoc_json_same_crate_different_version() {
         .run();
 
     let dep_json = fs::read_to_string(entry.root().join("target/doc/dep.json")).unwrap();
-    assert!(!dep_json.contains("dep_v1_fn"));
-    assert!(dep_json.contains("dep_v2_fn"));
+    assert!(dep_json.contains("dep_v1_fn"));
+    assert!(!dep_json.contains("dep_v2_fn"));
 }