From 42f20b784e0d0a349341b97fc54dcc9e5682fd7c Mon Sep 17 00:00:00 2001 From: wetor Date: Sun, 12 Mar 2023 14:50:51 +0800 Subject: [PATCH] parser: fix CRLF(\r\n) file parsing error, SyntaxError: 'invalid syntax' --- parser/lexer.go | 3 +++ parser/lexer_test.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/parser/lexer.go b/parser/lexer.go index 801215ba..119a741b 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -110,6 +110,9 @@ func (x *yyLex) dequeue() int { func (x *yyLex) refill() { var err error x.line, err = x.reader.ReadString('\n') + if strings.HasSuffix(x.line, "\r\n") { + x.line = x.line[:len(x.line)-2] + "\n" + } if yyDebug >= 2 { fmt.Printf("line = %q, err = %v\n", x.line, err) } diff --git a/parser/lexer_test.go b/parser/lexer_test.go index 9971c2d7..5e044d39 100644 --- a/parser/lexer_test.go +++ b/parser/lexer_test.go @@ -262,7 +262,7 @@ func TestLex(t *testing.T) { {"01", "illegal decimal with leading zero 1:0", "exec", LexTokens{ {FILE_INPUT, nil, ast.Pos{0, 0}}, }}, - {"1\n 2\n 3\n4\n", "", "exec", LexTokens{ + {"1\n 2\r\n 3\r\n4\n", "", "exec", LexTokens{ {FILE_INPUT, nil, ast.Pos{0, 0}}, {NUMBER, py.Int(1), ast.Pos{1, 0}}, {NEWLINE, nil, ast.Pos{1, 1}},