Merge pull request #83 from brson/cleanup

Cleanup `feed` and a NUL replacement fix

Author: Ashe Connor

src/nodes.rs

@@ -315,15 +315,6 @@ pub struct Ast {
     /// The line in the input document the node starts at.
     pub start_line: u32,
 
-    /// The column in the input document the node starts at.
-    pub start_column: usize,
-
-    /// The line in the input document the node ends at.
-    pub end_line: u32,
-
-    /// The column in the input document the node ends at.
-    pub end_column: usize,
-
     #[doc(hidden)]
     pub content: Vec<u8>,
     #[doc(hidden)]
@@ -333,14 +324,11 @@ pub struct Ast {
 }
 
 #[doc(hidden)]
-pub fn make_block(value: NodeValue, start_line: u32, start_column: usize) -> Ast {
+pub fn make_block(value: NodeValue, start_line: u32) -> Ast {
     Ast {
         value: value,
         content: vec![],
         start_line: start_line,
-        start_column: start_column,
-        end_line: start_line,
-        end_column: 0,
         open: true,
         last_line_blank: false,
     }

src/parser/inlines.rs

@@ -1125,9 +1125,6 @@ pub fn make_inline<'a>(arena: &'a Arena<AstNode<'a>>, value: NodeValue) -> &'a A
         value: value,
         content: vec![],
         start_line: 0,
-        start_column: 0,
-        end_line: 0,
-        end_column: 0,
         open: false,
         last_line_blank: false,
     };

src/parser/mod.rs

@@ -33,14 +33,11 @@ pub fn parse_document<'a>(
         value: NodeValue::Document,
         content: vec![],
         start_line: 0,
-        start_column: 0,
-        end_line: 0,
-        end_column: 0,
         open: true,
         last_line_blank: false,
     })));
     let mut parser = Parser::new(arena, root, options);
-    parser.feed(buffer, true);
+    parser.feed(buffer);
     parser.finish()
 }
 
@@ -58,8 +55,6 @@ pub struct Parser<'a, 'o> {
     blank: bool,
     partially_consumed_tab: bool,
     last_line_length: usize,
-    linebuf: Vec<u8>,
-    last_buffer_ended_with_cr: bool,
     options: &'o ComrakOptions,
 }
 
@@ -305,7 +300,7 @@ struct FootnoteDefinition<'a> {
 }
 
 impl<'a, 'o> Parser<'a, 'o> {
-    pub fn new(
+    fn new(
         arena: &'a Arena<AstNode<'a>>,
         root: &'a AstNode<'a>,
         options: &'o ComrakOptions,
@@ -324,69 +319,53 @@ impl<'a, 'o> Parser<'a, 'o> {
             blank: false,
             partially_consumed_tab: false,
             last_line_length: 0,
-            linebuf: Vec::with_capacity(80),
-            last_buffer_ended_with_cr: false,
             options: options,
         }
     }
 
-    fn feed(&mut self, s: &str, eof: bool) {
+    fn feed(&mut self, s: &str) {
         let s = s.as_bytes();
         let mut i = 0;
-        let buffer = s;
-        let sz = buffer.len();
-
-        if self.last_buffer_ended_with_cr && buffer[i] == b'\n' {
-            i += 1;
-        }
-        self.last_buffer_ended_with_cr = false;
+        let sz = s.len();
+        let mut linebuf = vec![];
 
         while i < sz {
-            let mut process = false;
+            let mut process = true;
             let mut eol = i;
             while eol < sz {
-                if strings::is_line_end_char(buffer[eol]) {
-                    process = true;
+                if strings::is_line_end_char(s[eol]) {
                     break;
                 }
-                if buffer[eol] == 0 {
+                if s[eol] == 0 {
+                    process = false;
                     break;
                 }
                 eol += 1;
             }
 
-            if eol >= sz && eof {
-                process = true;
-            }
-
             if process {
-                if !self.linebuf.is_empty() {
-                    self.linebuf.extend_from_slice(&s[i..eol]);
-                    let linebuf = mem::replace(&mut self.linebuf, Vec::with_capacity(80));
+                if !linebuf.is_empty() {
+                    linebuf.extend_from_slice(&s[i..eol]);
                     self.process_line(&linebuf);
-                } else if sz > eol && buffer[eol] == b'\n' {
+                    linebuf.truncate(0);
+                } else if sz > eol && s[eol] == b'\n' {
                     self.process_line(&s[i..eol + 1]);
                 } else {
                     self.process_line(&s[i..eol]);
                 }
-            } else if eol < sz && buffer[eol] == b'\0' {
-                self.linebuf.extend_from_slice(&s[i..eol]);
-                self.linebuf
-                    .extend_from_slice(&"\u{fffd}".to_string().into_bytes());
-                eol += 1;
-            } else {
-                self.linebuf.extend_from_slice(&s[i..eol]);
-            }
 
-            i = eol;
-            if i < sz && buffer[i] == b'\r' {
-                i += 1;
-                if i == sz {
-                    self.last_buffer_ended_with_cr = true;
+                i = eol;
+                if i < sz && s[i] == b'\r' {
+                    i += 1;
                 }
-            }
-            if i < sz && buffer[i] == b'\n' {
-                i += 1;
+                if i < sz && s[i] == b'\n' {
+                    i += 1;
+                }
+            } else {
+                debug_assert!(eol < sz && s[eol] == b'\0');
+                linebuf.extend_from_slice(&s[i..eol]);
+                linebuf.extend_from_slice(&"\u{fffd}".to_string().into_bytes());
+                i = eol + 1;
             }
         }
     }
@@ -558,14 +537,13 @@ impl<'a, 'o> Parser<'a, 'o> {
             let indented = self.indent >= CODE_INDENT;
 
             if !indented && line[self.first_nonspace] == b'>' {
-                let blockquote_startpos = self.first_nonspace;
                 let offset = self.first_nonspace + 1 - self.offset;
                 self.advance_offset(line, offset, false);
                 if strings::is_space_or_tab(line[self.offset]) {
                     self.advance_offset(line, 1, true);
                 }
                 *container =
-                    self.add_child(*container, NodeValue::BlockQuote, blockquote_startpos + 1);
+                    self.add_child(*container, NodeValue::BlockQuote);
             } else if !indented
                 && unwrap_into(
                     scanners::atx_heading_start(&line[self.first_nonspace..]),
@@ -577,7 +555,6 @@ impl<'a, 'o> Parser<'a, 'o> {
                 *container = self.add_child(
                     *container,
                     NodeValue::Heading(NodeHeading::default()),
-                    heading_startpos + 1,
                 );
 
                 let mut hashpos = line[self.first_nonspace..]
@@ -610,7 +587,7 @@ impl<'a, 'o> Parser<'a, 'o> {
                     literal: Vec::new(),
                 };
                 *container =
-                    self.add_child(*container, NodeValue::CodeBlock(ncb), first_nonspace + 1);
+                    self.add_child(*container, NodeValue::CodeBlock(ncb));
                 self.advance_offset(line, first_nonspace + matched - offset, false);
             } else if !indented
                 && (unwrap_into(
@@ -623,13 +600,12 @@ impl<'a, 'o> Parser<'a, 'o> {
                         &mut matched,
                     ),
                 }) {
-                let offset = self.first_nonspace + 1;
                 let nhb = NodeHtmlBlock {
                     block_type: matched as u8,
                     literal: Vec::new(),
                 };
 
-                *container = self.add_child(*container, NodeValue::HtmlBlock(nhb), offset);
+                *container = self.add_child(*container, NodeValue::HtmlBlock(nhb));
             } else if !indented && match container.data.borrow().value {
                 NodeValue::Paragraph => unwrap_into(
                     scanners::setext_heading_line(&line[self.first_nonspace..]),
@@ -653,8 +629,7 @@ impl<'a, 'o> Parser<'a, 'o> {
                     &mut matched,
                 ),
             } {
-                let offset = self.first_nonspace + 1;
-                *container = self.add_child(*container, NodeValue::ThematicBreak, offset);
+                *container = self.add_child(*container, NodeValue::ThematicBreak);
                 let adv = line.len() - 1 - self.offset;
                 self.advance_offset(line, adv, false);
             } else if !indented && self.options.ext_footnotes
@@ -666,11 +641,9 @@ impl<'a, 'o> Parser<'a, 'o> {
                 c = c.split(|&e| e == b']').next().unwrap();
                 let offset = self.first_nonspace + matched - self.offset;
                 self.advance_offset(line, offset, false);
-                let offset = self.first_nonspace + matched + 1;
                 *container = self.add_child(
                     *container,
                     NodeValue::FootnoteDefinition(c.to_vec()),
-                    offset,
                 );
             } else if (!indented || match container.data.borrow().value {
                 NodeValue::List(..) => true,
@@ -714,16 +687,14 @@ impl<'a, 'o> Parser<'a, 'o> {
 
                 nl.marker_offset = self.indent;
 
-                let offset = self.first_nonspace + 1;
                 if match container.data.borrow().value {
                     NodeValue::List(ref mnl) => !lists_match(&nl, mnl),
                     _ => true,
                 } {
-                    *container = self.add_child(*container, NodeValue::List(nl), offset);
+                    *container = self.add_child(*container, NodeValue::List(nl));
                 }
 
-                let offset = self.first_nonspace + 1;
-                *container = self.add_child(*container, NodeValue::Item(nl), offset);
+                *container = self.add_child(*container, NodeValue::Item(nl));
             } else if indented && !maybe_lazy && !self.blank {
                 self.advance_offset(line, CODE_INDENT, true);
                 let ncb = NodeCodeBlock {
@@ -734,8 +705,7 @@ impl<'a, 'o> Parser<'a, 'o> {
                     info: vec![],
                     literal: Vec::new(),
                 };
-                let offset = self.offset + 1;
-                *container = self.add_child(*container, NodeValue::CodeBlock(ncb), offset);
+                *container = self.add_child(*container, NodeValue::CodeBlock(ncb));
             } else {
                 let new_container = if !indented && self.options.ext_table {
                     table::try_opening_block(self, *container, line)
@@ -898,13 +868,12 @@ impl<'a, 'o> Parser<'a, 'o> {
         &mut self,
         mut parent: &'a AstNode<'a>,
         value: NodeValue,
-        start_column: usize,
     ) -> &'a AstNode<'a> {
         while !nodes::can_contain_type(parent, &value) {
             parent = self.finalize(parent).unwrap();
         }
 
-        let child = make_block(value, self.line_number, start_column);
+        let child = make_block(value, self.line_number);
         let node = self.arena.alloc(Node::new(RefCell::new(child)));
         parent.append(node);
         node
@@ -1010,8 +979,7 @@ impl<'a, 'o> Parser<'a, 'o> {
                             self.add_line(container, &line);
                         }
                     } else {
-                        let start_column = self.first_nonspace + 1;
-                        container = self.add_child(container, NodeValue::Paragraph, start_column);
+                        container = self.add_child(container, NodeValue::Paragraph);
                         let count = self.first_nonspace - self.offset;
                         self.advance_offset(line, count, false);
                         self.add_line(container, line);
@@ -1038,12 +1006,7 @@ impl<'a, 'o> Parser<'a, 'o> {
         }
     }
 
-    pub fn finish(&mut self) -> &'a AstNode<'a> {
-        if !self.linebuf.is_empty() {
-            let linebuf = mem::replace(&mut self.linebuf, vec![]);
-            self.process_line(&linebuf);
-        }
-
+    fn finish(&mut self) -> &'a AstNode<'a> {
         self.finalize_document();
         self.postprocess_text_nodes(self.root);
         self.root
@@ -1073,28 +1036,6 @@ impl<'a, 'o> Parser<'a, 'o> {
         assert!(ast.open);
         ast.open = false;
 
-        if !self.linebuf.is_empty() {
-            ast.end_line = self.line_number;
-            ast.end_column = self.last_line_length;
-        } else if match ast.value {
-            NodeValue::Document => true,
-            NodeValue::CodeBlock(ref ncb) => ncb.fenced,
-            NodeValue::Heading(ref nh) => nh.setext,
-            _ => false,
-        } {
-            ast.end_line = self.line_number;
-            ast.end_column = self.linebuf.len();
-            if ast.end_column > 0 && self.linebuf[ast.end_column - 1] == b'\n' {
-                ast.end_column -= 1;
-            }
-            if ast.end_column > 0 && self.linebuf[ast.end_column - 1] == b'\r' {
-                ast.end_column -= 1;
-            }
-        } else {
-            ast.end_line = self.line_number - 1;
-            ast.end_column = self.last_line_length;
-        }
-
         let content = &mut ast.content;
         let mut pos = 0;
 

src/parser/table.rs

@@ -58,18 +58,16 @@ fn try_opening_header<'a, 'o>(
         });
     }
 
-    let start_column = container.data.borrow().start_column;
     let child = make_block(
         NodeValue::Table(alignments),
         parser.line_number,
-        start_column,
     );
     let table = parser.arena.alloc(Node::new(RefCell::new(child)));
     container.append(table);
 
-    let header = parser.add_child(table, NodeValue::TableRow(true), start_column);
+    let header = parser.add_child(table, NodeValue::TableRow(true));
     for header_str in header_row {
-        let header_cell = parser.add_child(header, NodeValue::TableCell, start_column);
+        let header_cell = parser.add_child(header, NodeValue::TableCell);
         header_cell.data.borrow_mut().content = header_str;
     }
 
@@ -92,15 +90,13 @@ fn try_opening_row<'a, 'o>(
     let new_row = parser.add_child(
         container,
         NodeValue::TableRow(false),
-        container.data.borrow().start_column,
     );
 
     let mut i = 0;
     while i < min(alignments.len(), this_row.len()) {
         let cell = parser.add_child(
             new_row,
             NodeValue::TableCell,
-            container.data.borrow().start_column,
         );
         cell.data.borrow_mut().content = this_row[i].clone();
         i += 1;
@@ -110,7 +106,6 @@ fn try_opening_row<'a, 'o>(
         parser.add_child(
             new_row,
             NodeValue::TableCell,
-            container.data.borrow().start_column,
         );
         i += 1;
     }