rustdoc: parse item and reexport markdown separate

This change is a pre-requisite for LaTeX support, because the markdown
parser doesn't support enabling and disabling extensions in the middle
of a document, and there's no way to add that. This means string
concatenating a document that has LaTeX Math disabled with a document
that has it enabled can't be done.

As part of this change, a bug related to intra-doc links is fixed.
This shows up when the reexport and the item both have intra-doc links
with the same visible path, but where they resolve to different items.
The bug is demonstrated in
`tests/rustdoc-html/reexport/link-with-same-name-but-different-destination.rs`.

The other test case changes demonstrate that this is, technically, a
breaking change. When I ran a Crater test for docs that rely on this
behavior, though, it seemed most authors weren't relying on it.

Author: notriddle

compiler/rustc_resolve/src/rustdoc.rs

@@ -236,9 +236,7 @@ pub fn attrs_to_doc_fragments<'a, A: AttributeExt + Clone + 'a>(
 ///
 /// The last newline is not trimmed so the produced strings are reusable between
 /// early and late doc link resolution regardless of their position.
-pub fn prepare_to_doc_link_resolution(
-    doc_fragments: &[DocFragment],
-) -> FxIndexMap<Option<DefId>, String> {
+pub fn prepare_fragments(doc_fragments: &[DocFragment]) -> FxIndexMap<Option<DefId>, String> {
     let mut res = FxIndexMap::default();
     for fragment in doc_fragments {
         let out_str = res.entry(fragment.item_id).or_default();
@@ -412,7 +410,7 @@ pub fn may_be_doc_link(link_type: LinkType) -> bool {
 pub(crate) fn attrs_to_preprocessed_links(attrs: &[ast::Attribute]) -> Vec<Box<str>> {
     let (doc_fragments, other_attrs) =
         attrs_to_doc_fragments(attrs.iter().map(|attr| (attr, None)), false);
-    let doc = prepare_to_doc_link_resolution(&doc_fragments).into_values().next();
+    let doc = prepare_fragments(&doc_fragments).into_values().next();
     let mut links = doc.as_deref().map(parse_links).unwrap_or_default();
 
     for attr in other_attrs {

src/librustdoc/calculate_doc_coverage.rs

@@ -238,7 +238,9 @@ impl DocVisitor<'_> for CoverageCalculator<'_, '_> {
                 let has_docs = !i.attrs.doc_strings.is_empty();
                 let mut tests = Tests { found_tests: 0 };
 
-                find_testable_code(&i.doc_value(), &mut tests, ErrorCodes::No, None);
+                for text in i.doc_values().values() {
+                    find_testable_code(&text, &mut tests, ErrorCodes::No, None);
+                }
 
                 let has_doc_example = tests.found_tests != 0;
                 let hir_id = DocContext::as_local_hir_id(self.ctx.tcx, i.item_id).unwrap();

src/librustdoc/clean/types.rs

@@ -21,7 +21,7 @@ use rustc_metadata::rendered_const;
 use rustc_middle::ty::fast_reject::SimplifiedType;
 use rustc_middle::ty::{self, Ty, TyCtxt, Visibility};
 use rustc_resolve::rustdoc::{
-    DocFragment, add_doc_fragment, attrs_to_doc_fragments, inner_docs, span_of_fragments,
+    DocFragment, attrs_to_doc_fragments, inner_docs, prepare_fragments, span_of_fragments,
 };
 use rustc_session::Session;
 use rustc_span::def_id::{CRATE_DEF_ID, ModId};
@@ -122,6 +122,42 @@ impl ItemId {
             | ItemId::DefId(id) => id.krate,
         }
     }
+
+    pub(crate) fn links(&self, cx: &Context<'_>) -> Vec<RenderedLink> {
+        use crate::html::format::{href_with_path_check, link_tooltip};
+
+        let Some(links) = cx.cache().intra_doc_links.get(&self) else {
+            return vec![];
+        };
+        links
+            .iter()
+            .filter_map(|ItemLink { link: s, link_text, page_id: id, fragment }| {
+                debug!(?id);
+                if let Ok(HrefInfo { mut url, .. }) = href_with_path_check(*id, cx, link_text) {
+                    debug!(?url);
+                    match fragment {
+                        Some(UrlFragment::Item(def_id)) => {
+                            write!(url, "{}", crate::html::format::fragment(*def_id, cx.tcx()))
+                                .unwrap();
+                        }
+                        Some(UrlFragment::UserWritten(raw)) => {
+                            url.push('#');
+                            url.push_str(raw);
+                        }
+                        None => {}
+                    }
+                    Some(RenderedLink {
+                        original_text: s.clone(),
+                        new_text: link_text.clone(),
+                        tooltip: link_tooltip(*id, fragment, cx, Some(link_text)).to_string(),
+                        href: url,
+                    })
+                } else {
+                    None
+                }
+            })
+            .collect()
+    }
 }
 
 impl From<DefId> for ItemId {
@@ -363,7 +399,7 @@ impl fmt::Debug for Item {
             fmt.field("attrs", &self.attrs).field("kind", &self.kind).field("cfg", &self.cfg);
         } else {
             fmt.field("kind", &self.type_());
-            fmt.field("docs", &self.doc_value());
+            fmt.field("docs", &self.doc_values().values().cloned().collect::<Vec<String>>());
         }
         fmt.finish()
     }
@@ -507,16 +543,20 @@ impl Item {
             .unwrap_or_else(|| self.span(tcx).map_or(DUMMY_SP, |span| span.inner()))
     }
 
-    /// Combine all doc strings into a single value handling indentation and newlines as needed.
-    pub(crate) fn doc_value(&self) -> String {
-        self.attrs.doc_value()
+    /// Combine each reexport's docstrings into one docstring per item, handling indentation and
+    /// newlines as needed.
+    pub(crate) fn doc_values(&self) -> FxIndexMap<Option<DefId>, String> {
+        self.attrs.doc_values()
     }
 
-    /// Combine all doc strings into a single value handling indentation and newlines as needed.
+    /// Combine each reexport's docstrings into one docstring per item.
+    ///
     /// Returns `None` is there's no documentation at all, and `Some("")` if there is some
     /// documentation but it is empty (e.g. `#[doc = ""]`).
-    pub(crate) fn opt_doc_value(&self) -> Option<String> {
-        self.attrs.opt_doc_value()
+    ///
+    /// The trailing newline is trimmed.
+    pub(crate) fn opt_doc_values(&self) -> Option<FxIndexMap<Option<DefId>, String>> {
+        self.attrs.opt_doc_values()
     }
 
     pub(crate) fn from_def_id_and_parts(
@@ -559,58 +599,6 @@ impl Item {
         }
     }
 
-    /// If the item has doc comments from a reexport, returns the item id of that reexport,
-    /// otherwise returns returns the item id.
-    ///
-    /// This is used as a key for caching intra-doc link resolution,
-    /// to prevent two reexports of the same item from using the same cache.
-    pub(crate) fn item_or_reexport_id(&self) -> ItemId {
-        // added documentation on a reexport is always prepended.
-        self.attrs
-            .doc_strings
-            .first()
-            .map(|x| x.item_id)
-            .flatten()
-            .map(ItemId::from)
-            .unwrap_or(self.item_id)
-    }
-
-    pub(crate) fn links(&self, cx: &Context<'_>) -> Vec<RenderedLink> {
-        use crate::html::format::{href_with_path_check, link_tooltip};
-
-        let Some(links) = cx.cache().intra_doc_links.get(&self.item_or_reexport_id()) else {
-            return vec![];
-        };
-        links
-            .iter()
-            .filter_map(|ItemLink { link: s, link_text, page_id: id, fragment }| {
-                debug!(?id);
-                if let Ok(HrefInfo { mut url, .. }) = href_with_path_check(*id, cx, link_text) {
-                    debug!(?url);
-                    match fragment {
-                        Some(UrlFragment::Item(def_id)) => {
-                            write!(url, "{}", crate::html::format::fragment(*def_id, cx.tcx()))
-                                .unwrap();
-                        }
-                        Some(UrlFragment::UserWritten(raw)) => {
-                            url.push('#');
-                            url.push_str(raw);
-                        }
-                        None => {}
-                    }
-                    Some(RenderedLink {
-                        original_text: s.clone(),
-                        new_text: link_text.clone(),
-                        tooltip: link_tooltip(*id, fragment, cx, Some(link_text)).to_string(),
-                        href: url,
-                    })
-                } else {
-                    None
-                }
-            })
-            .collect()
-    }
-
     /// Find a list of all link names, without finding their href.
     ///
     /// This is used for generating summary text, which does not include
@@ -1095,21 +1083,26 @@ impl Attributes {
         Attributes { doc_strings, other_attrs }
     }
 
-    /// Combine all doc strings into a single value handling indentation and newlines as needed.
-    pub(crate) fn doc_value(&self) -> String {
-        self.opt_doc_value().unwrap_or_default()
+    /// Combine each reexport's docstrings into one docstring per item, handling indentation and
+    /// newlines as needed.
+    pub(crate) fn doc_values(&self) -> FxIndexMap<Option<DefId>, String> {
+        self.opt_doc_values().unwrap_or_default()
     }
 
-    /// Combine all doc strings into a single value handling indentation and newlines as needed.
+    /// Combine each reexport's docstrings into one docstring per item.
+    ///
     /// Returns `None` is there's no documentation at all, and `Some("")` if there is some
     /// documentation but it is empty (e.g. `#[doc = ""]`).
-    pub(crate) fn opt_doc_value(&self) -> Option<String> {
+    ///
+    /// The trailing newline is trimmed.
+    pub(crate) fn opt_doc_values(&self) -> Option<FxIndexMap<Option<DefId>, String>> {
         (!self.doc_strings.is_empty()).then(|| {
-            let mut res = String::new();
-            for frag in &self.doc_strings {
-                add_doc_fragment(&mut res, frag);
+            let mut res = prepare_fragments(&self.doc_strings);
+            for string in res.values_mut() {
+                if string.as_bytes().last() == Some(&b'\n') {
+                    string.pop();
+                }
             }
-            res.pop();
             res
         })
     }

src/librustdoc/clean/types/tests.rs

@@ -21,7 +21,9 @@ fn run_test(input: &str, expected: &str) {
         let mut s = create_doc_fragment(input);
         unindent_doc_fragments(&mut s);
         let attrs = Attributes { doc_strings: s, other_attrs: Default::default() };
-        assert_eq!(attrs.doc_value(), expected);
+        let doc_values = attrs.doc_values();
+        assert_eq!(doc_values.len(), 1);
+        assert_eq!(doc_values.values().next().expect("must have exactly one doc"), expected);
     });
 }
 

src/librustdoc/core.rs

@@ -409,7 +409,7 @@ pub(crate) fn run_global_ctxt(
 
     let mut krate = tcx.sess.time("clean_crate", || clean::krate(&mut ctxt));
 
-    if krate.module.doc_value().is_empty() {
+    if krate.module.doc_values().is_empty() {
         let help = format!(
             "The following guide may be of use:\n\
             {}/rustdoc/how-to-write-documentation.html",

src/librustdoc/doctest/rust.rs

@@ -211,7 +211,7 @@ impl HirCollector<'_> {
         // The collapse-docs pass won't combine sugared/raw doc attributes, or included files with
         // anything else, this will combine them for us.
         let attrs = Attributes::from_hir(hir_attrs);
-        if let Some(doc) = attrs.opt_doc_value() {
+        for doc in attrs.doc_values().values() {
             let span = span_of_fragments(&attrs.doc_strings).unwrap_or(sp);
             self.collector.position = if span.edition().at_least_rust_2024() {
                 span

src/librustdoc/html/render/context.rs

@@ -254,7 +254,10 @@ impl<'tcx> Context<'tcx> {
             };
             title.push_str(" - Rust");
             let tyname = it.type_();
-            let desc = plain_text_summary(&it.doc_value(), &it.link_names(self.cache()));
+            let desc = plain_text_summary(
+                &it.doc_values().values().next().map_or_default(|s| &s[..]),
+                &it.link_names(self.cache()),
+            );
             let desc = if !desc.is_empty() {
                 desc
             } else if it.is_crate() {