File tree Expand file tree Collapse file tree 1 file changed +19
-0
lines changed Expand file tree Collapse file tree 1 file changed +19
-0
lines changed Original file line number Diff line number Diff line change @@ -653,6 +653,10 @@ impl<'a> Tokenizer<'a> {
653
653
) ;
654
654
Ok ( Some ( Token :: Placeholder ( String :: from ( "$" ) + & s) ) )
655
655
}
656
+ //whitespace check (including unicode chars) should be last as it covers some of the chars above
657
+ ch if ch. is_whitespace ( ) => {
658
+ self . consume_and_return ( chars, Token :: Whitespace ( Whitespace :: Space ) )
659
+ }
656
660
other => self . consume_and_return ( chars, Token :: Char ( other) ) ,
657
661
} ,
658
662
None => Ok ( None ) ,
@@ -1254,6 +1258,21 @@ mod tests {
1254
1258
compare ( expected, tokens) ;
1255
1259
}
1256
1260
1261
+ #[ test]
1262
+ fn tokenize_unicode_whitespace ( ) {
1263
+ let sql = String :: from ( " \u{2003} \n " ) ;
1264
+
1265
+ let dialect = GenericDialect { } ;
1266
+ let mut tokenizer = Tokenizer :: new ( & dialect, & sql) ;
1267
+ let tokens = tokenizer. tokenize ( ) . unwrap ( ) ;
1268
+ let expected = vec ! [
1269
+ Token :: Whitespace ( Whitespace :: Space ) ,
1270
+ Token :: Whitespace ( Whitespace :: Space ) ,
1271
+ Token :: Whitespace ( Whitespace :: Newline ) ,
1272
+ ] ;
1273
+ compare ( expected, tokens) ;
1274
+ }
1275
+
1257
1276
#[ test]
1258
1277
fn tokenize_mismatched_quotes ( ) {
1259
1278
let sql = String :: from ( "\" foo" ) ;
You can’t perform that action at this time.
0 commit comments