Support the string concat operator (#178)

The selected precedence is based on BigQuery documentation, where it is equal to `*` and `/`:

https://cloud.google.com/bigquery/docs/reference/standard-sql/operators

Author: Dandandan

src/ast/operator.rs

@@ -38,6 +38,7 @@ pub enum BinaryOperator {
     Multiply,
     Divide,
     Modulus,
+    StringConcat,
     Gt,
     Lt,
     GtEq,
@@ -58,6 +59,7 @@ impl fmt::Display for BinaryOperator {
             BinaryOperator::Multiply => "*",
             BinaryOperator::Divide => "/",
             BinaryOperator::Modulus => "%",
+            BinaryOperator::StringConcat => "||",
             BinaryOperator::Gt => ">",
             BinaryOperator::Lt => "<",
             BinaryOperator::GtEq => ">=",

src/parser.rs

@@ -577,6 +577,7 @@ impl Parser {
             Token::Minus => Some(BinaryOperator::Minus),
             Token::Mult => Some(BinaryOperator::Multiply),
             Token::Mod => Some(BinaryOperator::Modulus),
+            Token::StringConcat => Some(BinaryOperator::StringConcat),
             Token::Div => Some(BinaryOperator::Divide),
             Token::Word(ref k) => match k.keyword.as_ref() {
                 "AND" => Some(BinaryOperator::And),
@@ -708,7 +709,7 @@ impl Parser {
                     Ok(20)
                 }
                 Token::Plus | Token::Minus => Ok(Self::PLUS_MINUS_PREC),
-                Token::Mult | Token::Div | Token::Mod => Ok(40),
+                Token::Mult | Token::Div | Token::Mod | Token::StringConcat => Ok(40),
                 Token::DoubleColon => Ok(50),
                 _ => Ok(0),
             }

src/tokenizer.rs

@@ -64,6 +64,8 @@ pub enum Token {
     Div,
     /// Modulo Operator `%`
     Mod,
+    /// String concatenation `||`
+    StringConcat,
     /// Left parenthesis `(`
     LParen,
     /// Right parenthesis `)`
@@ -111,6 +113,7 @@ impl fmt::Display for Token {
             Token::Minus => f.write_str("-"),
             Token::Mult => f.write_str("*"),
             Token::Div => f.write_str("/"),
+            Token::StringConcat => f.write_str("||"),
             Token::Mod => f.write_str("%"),
             Token::LParen => f.write_str("("),
             Token::RParen => f.write_str(")"),
@@ -374,6 +377,16 @@ impl<'a> Tokenizer<'a> {
                 '+' => self.consume_and_return(chars, Token::Plus),
                 '*' => self.consume_and_return(chars, Token::Mult),
                 '%' => self.consume_and_return(chars, Token::Mod),
+                '|' => {
+                    chars.next(); // consume the '|'
+                    match chars.peek() {
+                        Some('|') => self.consume_and_return(chars, Token::StringConcat),
+                        _ => Err(TokenizerError(format!(
+                            "Expecting to see `||`. Bitwise or operator `|` is not supported. \nError at Line: {}, Col: {}",
+                            self.line, self.col
+                        ))),
+                    }
+                }
                 '=' => self.consume_and_return(chars, Token::Eq),
                 '.' => self.consume_and_return(chars, Token::Period),
                 '!' => {
@@ -562,6 +575,26 @@ mod tests {
         compare(expected, tokens);
     }
 
+    #[test]
+    fn tokenize_string_string_concat() {
+        let sql = String::from("SELECT 'a' || 'b'");
+        let dialect = GenericDialect {};
+        let mut tokenizer = Tokenizer::new(&dialect, &sql);
+        let tokens = tokenizer.tokenize().unwrap();
+
+        let expected = vec![
+            Token::make_keyword("SELECT"),
+            Token::Whitespace(Whitespace::Space),
+            Token::SingleQuotedString(String::from("a")),
+            Token::Whitespace(Whitespace::Space),
+            Token::StringConcat,
+            Token::Whitespace(Whitespace::Space),
+            Token::SingleQuotedString(String::from("b")),
+        ];
+
+        compare(expected, tokens);
+    }
+
     #[test]
     fn tokenize_simple_select() {
         let sql = String::from("SELECT * FROM customer WHERE id = 1 LIMIT 5");

tests/sqlparser_common.rs

@@ -665,6 +665,21 @@ fn parse_in_subquery() {
     );
 }
 
+#[test]
+fn parse_string_agg() {
+    let sql = "SELECT a || b";
+
+    let select = verified_only_select(sql);
+    assert_eq!(
+        SelectItem::UnnamedExpr(Expr::BinaryOp {
+            left: Box::new(Expr::Identifier(Ident::new("a"))),
+            op: BinaryOperator::StringConcat,
+            right: Box::new(Expr::Identifier(Ident::new("b"))),
+        }),
+        select.projection[0]
+    );
+}
+
 #[test]
 fn parse_between() {
     fn chk(negated: bool) {