Merge pull request #689 from kivikakk/push-nnqpnowtsnom

tasklists: don't munge first byte into unicode codepoint.

Author: kivikakk

src/parser/mod.rs

@@ -2019,12 +2019,21 @@ where
         sourcepos: &mut Sourcepos,
         spx: &mut Spx,
     ) {
-        let (end, symbol) = match scanners::tasklist(text) {
+        let (end, matched) = match scanners::tasklist(text) {
             Some(p) => p,
             None => return,
         };
 
-        let symbol = symbol as char;
+        let mut chars = matched.chars();
+        let Some(symbol) = chars.next() else {
+            return;
+        };
+
+        // There must be at most one `char`'s worth of content in `matched`,
+        // otherwise we ignore it.
+        if !chars.next().is_none() {
+            return;
+        }
 
         if !self.options.parse.relaxed_tasklist_matching && !matches!(symbol, ' ' | 'x' | 'X') {
             return;

src/scanners.re

@@ -462,25 +462,28 @@ pub fn close_multiline_block_quote_fence(s: &str) -> Option<usize> {
 */
 }
 
-// Returns both the length of the match, and the tasklist character.
-pub fn tasklist(s: &str) -> Option<(usize, u8)> {
+// Returns both the length of the match, and the tasklist item contents.
+// It is not guaranteed to be one byte, or one "character" long; the caller must ascertain
+// its fitness for purpose.
+pub fn tasklist(s: &str) -> Option<(usize, &str)> {
     let mut cursor = 0;
     let mut marker = 0;
     let len = s.len();
 
-    let mut t1;
+    let t1;
+    let mut t2;
 /*!stags:re2c format = 'let mut @@{tag} = 0;'; */
 
 /*!local:re2c
     re2c:define:YYSTAGP = "@@{tag} = cursor;";
     re2c:define:YYSHIFTSTAG = "@@{tag} = (@@{tag} as isize + @@{shift}) as usize;";
-    re2c:tags = 1;
+    re2c:tags = 2;
 
-    spacechar* [[] @t1 [^\xff\r\n] [\]] (spacechar | [\xff]) {
+    spacechar* [[] @t1 [^\xff\r\n\]]+ @t2 [\]] (spacechar | [\xff]) {
         if cursor == len + 1 {
             cursor -= 1;
         }
-        return Some((cursor, s.as_bytes()[t1]));
+        return Some((cursor, &s[t1..t2]));
     }
     * { return None; }
 */

src/scanners.rs

@@ -491,3 +491,28 @@ fn sourcepos() {
         ])
     );
 }
+
+#[test]
+fn tasklist_relaxed_unicode() {
+    assert_ast_match!(
+        [extension.tasklist, parse.relaxed_tasklist_matching],
+        "- [あ] xy\n" // U+3042
+        "  - [い] zw\n", // U+3044
+        (document (1:1-2:12) [
+            (list (1:1-2:12) [
+                (taskitem (1:1-2:12) [
+                    (paragraph (1:9-1:10) [
+                        (text (1:9-1:10) "xy")
+                    ])
+                    (list (2:3-2:12) [
+                        (taskitem (2:3-2:12) [
+                            (paragraph (2:11-2:12) [
+                                (text (2:11-2:12) "zw")
+                            ])
+                        ])
+                    ])
+                ])
+            ])
+        ])
+    );
+}