Parse string literals into AST

Author: ncw

ast/ast.go

@@ -11,7 +11,6 @@ import (
 
 type Identifier py.String
 type String py.String
-type PyBytes py.Bytes
 type Object py.Object
 type Singleton py.Object
 
@@ -516,7 +515,7 @@ type Str struct {
 
 type Bytes struct {
 	ExprBase
-	S PyBytes
+	S py.Bytes
 }
 
 type NameConstant struct {

parser/grammar.y

@@ -17,6 +17,10 @@ import (
 
 // FIXME can put code blocks in not just at the end - help with list initialisation
 
+// FIXME is the expStack needed at all?  Aren't the yylval put into a
+// stack anyway by yacc?  And in rsc cc he sets yylval to empty every
+// lex.
+
 // A stack of []ast.Expr
 type exprsStack [][]ast.Expr
 
@@ -81,7 +85,7 @@ func (es *stmtsStack) Add(stmt ...ast.Stmt) {
 	comma		bool
 }
 
-%type <str> strings
+%type <obj> strings
 %type <mod> inputs file_input single_input eval_input
 %type <stmts> simple_stmt stmt 
 %type <stmtsStack> nl_or_stmt small_stmts stmts
@@ -99,8 +103,8 @@ func (es *stmtsStack) Add(stmt ...ast.Stmt) {
 %token <str> NAME
 %token INDENT
 %token DEDENT
-%token <str> STRING
-%token <str> NUMBER
+%token <obj> STRING
+%token <obj> NUMBER
 
 %token PLINGEQ // !=
 %token PERCEQ // %=
@@ -1051,9 +1055,27 @@ trailers:
 
 strings:
 	STRING
+	{
+		$$ = $1
+	}
 |	strings STRING
 	{
-		$$ += $2
+		switch a := $$.(type) {
+		case py.String:
+			switch b := $2.(type) {
+			case py.String:
+				$$ = a + b
+			default:
+				yylex.Error("SyntaxError: cannot mix string and nonstring literals")
+			}
+		case py.Bytes:
+			switch b := $2.(type) {
+			case py.Bytes:
+				$$ = append(a, b...)
+			default:
+				yylex.Error("SyntaxError: cannot mix bytes and nonbytes literals")
+			}
+		}
 	}
 
 atom:
@@ -1100,8 +1122,14 @@ atom:
 	}
 |	strings
 	{
-		// FIXME
-		$$ = nil
+		switch s := $1.(type) {
+		case py.String:
+			$$ = &ast.Str{ExprBase: ast.ExprBase{$<pos>$}, S: s}
+		case py.Bytes:
+			$$ = &ast.Bytes{ExprBase: ast.ExprBase{$<pos>$}, S: s}
+		default:
+			panic("not Bytes or String in strings")
+		}
 	}
 |	ELIPSIS
 	{

parser/grammar_test.go

@@ -25,6 +25,10 @@ func TestGrammar(t *testing.T) {
 		{"None\n", "eval", "Expression(body=NameConstant(value=None))"},
 		{"...", "eval", "Expression(body=Ellipsis())"},
 		{"abc123", "eval", "Expression(body=Name(id='abc123', ctx=Load()))"},
+		{"\"abc\"", "eval", "Expression(body=Str(s='abc'))"},
+		{"\"abc\" \"\"\"123\"\"\"", "eval", "Expression(body=Str(s='abc123'))"},
+		{"b'abc'", "eval", "Expression(body=Bytes(s=b'abc'))"},
+		{"b'abc' b'''123'''", "eval", "Expression(body=Bytes(s=b'abc123'))"},
 		// END TESTS
 	} {
 		Ast, err := ParseString(test.in, test.mode)

parser/make_grammar_test.py

@@ -17,6 +17,10 @@
     ("None\n", "eval"),
     ("...", "eval"),
     ("abc123", "eval"),
+    ('"abc"', "eval"),
+    ('"abc" """123"""', "eval"),
+    ("b'abc'", "eval"),
+    ("b'abc' b'''123'''", "eval"),
 ]
 
 def dump(source, mode):
@@ -26,7 +30,7 @@ def dump(source, mode):
 
 def escape(x):
     """Encode strings with backslashes for python/go"""
-    return x.encode("unicode_escape").decode("utf-8")
+    return x.replace('\\', "\\\\").replace('"', r'\"').replace("\n", r'\n').replace("\t", r'\t')
 
 def main():
     """Read in grammar_test.go, and re-write the tests section"""