simplify and normalise newline handling.

Author: kivikakk

src/parser/inlines.rs

@@ -358,12 +358,14 @@ impl<'a, 'r, 'o, 'd, 'c, 'p> Subject<'a, 'r, 'o, 'd, 'c, 'p> {
 
     fn handle_newline(&mut self) -> Node<'a> {
         let nlpos = self.scanner.pos;
-        if self.input.as_bytes()[self.scanner.pos] == b'\r' {
+        if self.peek_byte() == Some(b'\r') {
             self.scanner.pos += 1;
         }
-        if self.input.as_bytes()[self.scanner.pos] == b'\n' {
+        if self.peek_byte() == Some(b'\n') {
             self.scanner.pos += 1;
         }
+        assert_ne!(nlpos, self.scanner.pos);
+
         let inl = if nlpos > 1
             && self.input.as_bytes()[nlpos - 1] == b' '
             && self.input.as_bytes()[nlpos - 2] == b' '
@@ -1012,10 +1014,22 @@ impl<'a, 'r, 'o, 'd, 'c, 'p> Subject<'a, 'r, 'o, 'd, 'c, 'p> {
         );
         // Build line_offsets by scanning for newlines in the content
         para_ast.line_offsets = vec![0];
-        for (i, &byte) in content.as_bytes().iter().enumerate() {
-            if byte == b'\n' {
-                para_ast.line_offsets.push(i + 1);
+
+        let mut i = 0;
+        let bytes = content.as_bytes();
+        let len = content.len();
+        while i < len {
+            match bytes[i] {
+                b'\r' if i + 1 < len && bytes[i + 1] == b'\n' => {
+                    i += 1;
+                    para_ast.line_offsets.push(i + 1);
+                }
+                b'\n' | b'\r' => {
+                    para_ast.line_offsets.push(i + 1);
+                }
+                _ => {}
             }
+            i += 1;
         }
 
         let para_node = self.arena.alloc(para_ast.into());
@@ -2264,7 +2278,7 @@ pub(crate) fn manual_scan_link_url(input: &str) -> Option<(&str, usize)> {
                 break;
             } else if b == b'\\' {
                 i += 2;
-            } else if b == b'\n' || b == b'<' {
+            } else if b == b'\n' || b == b'\r' || b == b'<' {
                 return None;
             } else {
                 i += 1;

src/parser/mod.rs

@@ -140,21 +140,27 @@ where
 
         while ix < end {
             let mut eol = ix;
-            let mut ate_line_end = false;
+            let mut ate_line_end = 0;
 
             while eol < end {
-                if strings::is_line_end_char(sb[eol]) {
-                    ate_line_end = true;
-                    eol += 1;
-                    break;
-                }
-                if sb[eol] == 0 {
-                    break;
+                match sb[eol] {
+                    b'\r' if eol + 1 < end && sb[eol + 1] == b'\n' => {
+                        ate_line_end = 2;
+                        eol += 2;
+                        break;
+                    }
+                    b'\n' | b'\r' => {
+                        ate_line_end = 1;
+                        eol += 1;
+                        break;
+                    }
+                    0 => break,
+                    _ => {}
                 }
                 eol += 1;
             }
 
-            if ate_line_end || eol == end {
+            if ate_line_end > 0 || eol == end {
                 if !linebuf.is_empty() {
                     linebuf.push_str(&s[ix..eol]);
                     let line = mem::take(&mut linebuf);
@@ -166,24 +172,10 @@ where
                 assert_eq!(sb[eol], b'\0');
                 linebuf.push_str(&s[ix..eol]);
                 linebuf.push('\u{fffd}');
+                eol += 1;
             }
 
             ix = eol;
-            if ix < end {
-                if sb[ix] == b'\0' {
-                    ix += 1;
-                } else {
-                    if ate_line_end {
-                        ix -= 1;
-                    }
-                    if sb[ix] == b'\r' {
-                        ix += 1;
-                    }
-                    if ix < end && sb[ix] == b'\n' {
-                        ix += 1;
-                    }
-                }
-            }
         }
 
         if !linebuf.is_empty() {
@@ -227,14 +219,11 @@ where
     }
 
     fn process_line(&mut self, mut line: Cow<str>) {
-        let last_byte = line.as_bytes().last();
-        if last_byte.map_or(true, |&b| !strings::is_line_end_char(b)) {
+        let &last_byte = line.as_bytes().last().unwrap();
+        if !strings::is_line_end_char(last_byte) {
             line.to_mut().push('\n');
-        } else if last_byte == Some(&b'\r') {
-            let line_mut = line.to_mut();
-            line_mut.pop();
-            line_mut.push('\n');
-        };
+        }
+
         let line = line.as_ref();
         let bytes = line.as_bytes();
 
@@ -1639,7 +1628,7 @@ where
                     if content.as_bytes()[pos] == b'\r' {
                         pos += 1;
                     }
-                    if content.as_bytes()[pos] == b'\n' {
+                    if content.as_bytes().get(pos) == Some(&b'\n') {
                         pos += 1;
                     }
 
@@ -2163,7 +2152,7 @@ fn parse_list_marker(
             while strings::is_space_or_tab(bytes[i]) {
                 i += 1;
             }
-            if bytes[i] == b'\n' {
+            if strings::is_line_end_char(bytes[i]) {
                 return None;
             }
         }

src/parser/table.rs

@@ -5,7 +5,7 @@ use std::mem;
 use crate::nodes::{Ast, Node, NodeTable, NodeValue, TableAlignment};
 use crate::parser::Parser;
 use crate::scanners;
-use crate::strings::{count_newlines, trim_cow};
+use crate::strings::{count_newlines, is_line_end_char, trim_cow};
 
 // Limit to prevent a malicious input from causing a denial of service.
 // See get_num_autocompleted_cells.
@@ -312,7 +312,7 @@ fn try_inserting_table_header_paragraph<'a>(
             .iter()
             .rev()
             .skip(1)
-            .take_while(|&&c| c != b'\n')
+            .take_while(|&&c| !is_line_end_char(c))
             .count();
 
     container_ast.sourcepos.start.line += newlines;

src/scanners.re

@@ -360,7 +360,7 @@ pub fn ipv6_relaxed_url_start(s: &str) -> Option<usize> {
 
     table_spoiler = ['|']['|'];
     table_spacechar = [ \t\v\f];
-    table_newline = [\r]?[\n];
+    table_newline = [\r\n];
 
     table_delimiter = (table_spacechar*[:]?[-]+[:]?table_spacechar*);
     table_cell = (escaped_char|[^\x00|\r\n])+;