restore text w/ sourcepos of broken footnote references with smart chars.

Author: kivikakk

src/nodes.rs

@@ -173,7 +173,7 @@ pub enum NodeValue {
     Image(Box<NodeLink>),
 
     /// **Inline**.  A footnote reference.
-    FootnoteReference(NodeFootnoteReference),
+    FootnoteReference(Box<NodeFootnoteReference>),
 
     #[cfg(feature = "shortcodes")]
     /// **Inline**. An Emoji character generated from a shortcode. Enable with feature "shortcodes".
@@ -459,6 +459,9 @@ pub struct NodeFootnoteReference {
     /// The name of the footnote.
     pub name: String,
 
+    /// The original text elements of the footnote, including their source position spans.
+    pub texts: Vec<(String, usize)>,
+
     /// The index of reference to the same footnote
     pub ref_num: u32,
 

src/parser/inlines.rs

@@ -994,11 +994,12 @@ impl<'a, 'r, 'o, 'd, 'c, 'p> Subject<'a, 'r, 'o, 'd, 'c, 'p> {
 
         // Create the footnote reference node
         let ref_node = self.make_inline(
-            NodeValue::FootnoteReference(NodeFootnoteReference {
+            NodeValue::FootnoteReference(Box::new(NodeFootnoteReference {
                 name: name.clone(),
+                texts: vec![], // Unused.
                 ref_num: 0,
                 ix: 0,
-            }),
+            })),
             startpos,
             endpos,
         );
@@ -1734,13 +1735,33 @@ impl<'a, 'r, 'o, 'd, 'c, 'p> Subject<'a, 'r, 'o, 'd, 'c, 'p> {
             // do anything fancy here at all.
             let mut sussy = false;
 
+            let mut texts = vec![];
+
             for sibling in sibling_iterator {
-                match sibling.data().value {
+                let sibling_ast = sibling.data();
+                if sibling_ast.sourcepos.start.line != sibling_ast.sourcepos.end.line
+                    || sibling_ast.sourcepos.end.column < sibling_ast.sourcepos.start.column
+                {
+                    sussy = true;
+                    break;
+                }
+
+                match sibling_ast.value {
                     NodeValue::Text(ref literal) => {
                         text.push_str(literal);
+                        texts.push((
+                            literal.to_string(),
+                            sibling_ast.sourcepos.end.column - sibling_ast.sourcepos.start.column
+                                + 1,
+                        ));
                     }
                     NodeValue::HtmlInline(ref literal) => {
                         text.push_str(literal);
+                        texts.push((
+                            literal.to_string(),
+                            sibling_ast.sourcepos.end.column - sibling_ast.sourcepos.start.column
+                                + 1,
+                        ));
                     }
                     _ => {
                         sussy = true;
@@ -1751,11 +1772,12 @@ impl<'a, 'r, 'o, 'd, 'c, 'p> Subject<'a, 'r, 'o, 'd, 'c, 'p> {
 
             if !sussy && text.len() > 1 {
                 let inl = self.make_inline(
-                    NodeValue::FootnoteReference(NodeFootnoteReference {
+                    NodeValue::FootnoteReference(Box::new(NodeFootnoteReference {
                         name: text[1..].to_string(),
+                        texts,
                         ref_num: 0,
                         ix: 0,
-                    }),
+                    })),
                     // Overridden immediately below.
                     self.scanner.pos,
                     self.scanner.pos,

src/parser/mod.rs

@@ -1740,7 +1740,7 @@ where
         Self::find_footnote_definitions(self.root, &mut fd_map);
 
         let mut next_ix = 0;
-        Self::find_footnote_references(self.root, &mut fd_map, &mut next_ix);
+        self.find_footnote_references(&mut fd_map, &mut next_ix);
 
         let mut fds = fd_map.into_values().collect::<Vec<_>>();
         fds.sort_unstable_by(|a, b| a.ix.cmp(&b.ix));
@@ -1786,13 +1786,14 @@ where
     }
 
     fn find_footnote_references(
-        root: Node<'a>,
+        &mut self,
         map: &mut HashMap<String, FootnoteDefinition>,
         ixp: &mut u32,
     ) {
-        let mut stack = vec![root];
+        let mut stack = vec![self.root];
         while let Some(node) = stack.pop() {
             let mut ast = node.data_mut();
+            let sp = ast.sourcepos;
             match ast.value {
                 NodeValue::FootnoteReference(ref mut nfr) => {
                     let normalized = strings::normalize_label(&nfr.name, Case::Fold);
@@ -1810,7 +1811,41 @@ where
                         nfr.ix = ix;
                         nfr.name = strings::normalize_label(&footnote.name, Case::Preserve);
                     } else {
-                        ast.value = NodeValue::Text(format!("[^{}]", nfr.name).into());
+                        // Restore the nodes as they were-ish.  We restore each
+                        // Text node as it was found, preserving the sourcepos
+                        // spans.  This is important for accurate sourcepos
+                        // tracking; we assert when 'consuming' sourcepos
+                        // lengths in post-processing that either the span
+                        // length matches the byte count of the string (meaning
+                        // we can reliably subset them both), or that we're
+                        // consuming a whole span.  Trying to consume part of a
+                        // span without a matching length is undefined, and we
+                        // will crash; see Spx::consume.
+                        //
+                        // See HACK comment in
+                        // `inlines::Subject::handle_close_bracket` for the
+                        // producer of these values.
+                        assert!(!nfr.texts.is_empty());
+                        let mut lc = sp.start;
+                        let mut target = node;
+
+                        let mut texts = mem::take(&mut nfr.texts);
+                        texts.insert(0, ("[".into(), 1));
+                        texts.push(("]".into(), 1));
+
+                        for (text, span) in &mut texts {
+                            let inl = self.arena.alloc(
+                                Ast::new_with_sourcepos(
+                                    NodeValue::Text(mem::take(text).into()),
+                                    (lc, lc.column_add(*span as isize - 1)).into(),
+                                )
+                                .into(),
+                            );
+                            target.insert_after(inl);
+                            target = inl;
+                            lc = lc.column_add(*span as isize);
+                        }
+                        node.detach();
                     }
                 }
                 _ => {

src/tests/fuzz.rs

@@ -540,3 +540,14 @@ fn oye_siri_prende_las_luces() {
         ])
     );
 }
+
+#[test]
+fn cursed_lands() {
+    let mut opts = Options::default();
+    opts.extension.autolink = true;
+    opts.extension.footnotes = true;
+    opts.parse.smart = true;
+    opts.parse.relaxed_autolinks = true;
+
+    markdown_to_html("[^[email protected]]", &opts);
+}