@@ -1615,7 +1615,25 @@ mod tests {
1615
1615
compare ( expected, tokens) ;
1616
1616
}
1617
1617
1618
- fn compare ( expected : Vec < Token > , actual : Vec < Token > ) {
1618
+ #[ test]
1619
+ fn tokenize_with_location ( ) {
1620
+ let sql = "SELECT a,\n b" ;
1621
+ let dialect = GenericDialect { } ;
1622
+ let mut tokenizer = Tokenizer :: new ( & dialect, sql) ;
1623
+ let tokens = tokenizer. tokenize_with_location ( ) . unwrap ( ) ;
1624
+ let expected = vec ! [
1625
+ TokenWithLocation :: new( Token :: make_keyword( "SELECT" ) , 1 , 1 ) ,
1626
+ TokenWithLocation :: new( Token :: Whitespace ( Whitespace :: Space ) , 1 , 7 ) ,
1627
+ TokenWithLocation :: new( Token :: make_word( "a" , None ) , 1 , 8 ) ,
1628
+ TokenWithLocation :: new( Token :: Comma , 1 , 9 ) ,
1629
+ TokenWithLocation :: new( Token :: Whitespace ( Whitespace :: Newline ) , 1 , 10 ) ,
1630
+ TokenWithLocation :: new( Token :: Whitespace ( Whitespace :: Space ) , 2 , 1 ) ,
1631
+ TokenWithLocation :: new( Token :: make_word( "b" , None ) , 2 , 2 ) ,
1632
+ ] ;
1633
+ compare ( expected, tokens) ;
1634
+ }
1635
+
1636
+ fn compare < T : PartialEq + std:: fmt:: Debug > ( expected : Vec < T > , actual : Vec < T > ) {
1619
1637
//println!("------------------------------");
1620
1638
//println!("tokens = {:?}", actual);
1621
1639
//println!("expected = {:?}", expected);
0 commit comments