Skip to content

Commit 0adc14a

Browse files
ncwncw
committed
Fix grammar for multiple entry points for compile mode
1 parent c2c5db0 commit 0adc14a

File tree

4 files changed

+173
-60
lines changed

4 files changed

+173
-60
lines changed

‎ast/ast.go

+3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
package ast
22

3+
// FIXME make base AstNode with position in
4+
// also keep a list of children in the parent node to simplify walking the tree?
5+
36
import (
47
"fmt"
58

‎parser/grammar.y

+50-25
Original file line numberDiff line numberDiff line change
@@ -12,21 +12,21 @@ import (
1212
%}
1313

1414
%union {
15-
str string
16-
obj py.Object
17-
ast ast.Ast
18-
mod ast.Mod
19-
stmt ast.Stmt
20-
stmts []ast.Stmt
21-
stmts1 []ast.Stmt // nl_or_stmt accumulator
22-
stmts2 []ast.Stmt // small_stmts accumulator
23-
stmts3 []ast.Stmt // stmts accumulator
24-
pos ast.Pos // kept up to date by the lexer
15+
str string
16+
obj py.Object
17+
ast ast.Ast
18+
mod ast.Mod
19+
stmt ast.Stmt
20+
stmts []ast.Stmt
21+
stmts1 []ast.Stmt // nl_or_stmt accumulator
22+
stmts2 []ast.Stmt // small_stmts accumulator
23+
stmts3 []ast.Stmt // stmts accumulator
24+
pos ast.Pos // kept up to date by the lexer
2525
}
2626

2727
%type <str> strings
2828
%type <ast> atom
29-
%type <mod> inputs file_input
29+
%type <mod> inputs file_input single_input eval_input
3030
%type <stmts> simple_stmt stmt
3131
%type <stmts1> nl_or_stmt
3232
%type <stmts2> small_stmts
@@ -101,6 +101,8 @@ import (
101101

102102
%token '(' ')' '[' ']' ':' ',' ';' '+' '-' '*' '/' '|' '&' '<' '>' '=' '.' '%' '{' '}' '^' '~' '@'
103103

104+
%token SINGLE_INPUT FILE_INPUT EVAL_INPUT
105+
104106
// Note: Changing the grammar specified in this file will most likely
105107
// require corresponding changes in the parser module
106108
// (../Modules/parsermodule.c). If you can't make the changes to
@@ -119,18 +121,45 @@ import (
119121

120122
%%
121123

122-
// FIXME figure out how to tell the parser to start from a given node
123-
// inputs: single_input | file_input | eval_input
124-
// In the mean time just do file_input
125-
// inputs: single_input | file_input | eval_input
124+
// Start of grammar. This has 3 pseudo tokens which say which
125+
// direction through the rest of the grammar we take.
126126
inputs:
127-
file_input
127+
SINGLE_INPUT single_input
128+
{
129+
yylex.(*yyLex).mod = $2
130+
return 0
131+
}
132+
| FILE_INPUT file_input
133+
{
134+
yylex.(*yyLex).mod = $2
135+
return 0
136+
}
137+
| EVAL_INPUT eval_input
128138
{
129-
yylex.(*yyLex).mod = $1
139+
yylex.(*yyLex).mod = $2
130140
return 0
131141
}
132142

133-
single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
143+
single_input:
144+
NEWLINE
145+
{
146+
$$ = &ast.Interactive{ModBase: ast.ModBase{$<pos>$}}
147+
}
148+
| simple_stmt
149+
{
150+
$$ = &ast.Interactive{ModBase: ast.ModBase{$<pos>$}, Body: $1}
151+
}
152+
| compound_stmt NEWLINE
153+
{
154+
$$ = &ast.Interactive{ModBase: ast.ModBase{$<pos>$}, Body: []ast.Stmt{$1}}
155+
}
156+
157+
//file_input: (NEWLINE | stmt)* ENDMARKER
158+
file_input:
159+
nl_or_stmt ENDMARKER
160+
{
161+
$$ = &ast.Module{ModBase: ast.ModBase{$<pos>$}, Body: $1}
162+
}
134163

135164
// (NEWLINE | stmt)*
136165
nl_or_stmt:
@@ -145,19 +174,15 @@ nl_or_stmt:
145174
$$ = append($$, $2...)
146175
}
147176

148-
//file_input: (NEWLINE | stmt)* ENDMARKER
149-
file_input:
150-
nl_or_stmt ENDMARKER
177+
//eval_input: testlist NEWLINE* ENDMARKER
178+
eval_input:
179+
testlist nls ENDMARKER
151180
{
152-
$$ = &ast.Module{ModBase: ast.ModBase{$<pos>$}, Body: $1}
153181
}
154182

155183
// NEWLINE*
156184
nls: | nls NEWLINE
157185

158-
//eval_input: testlist NEWLINE* ENDMARKER
159-
eval_input: testlist nls ENDMARKER
160-
161186
optional_arglist: | arglist
162187

163188
optional_arglist_call: | '(' optional_arglist ')'

‎parser/lexer.go

+50-15
Original file line numberDiff line numberDiff line change
@@ -46,15 +46,33 @@ type yyLex struct {
4646
parenthesis int // number of open ( )
4747
brace int // number of open { }
4848
mod ast.Mod // output
49+
startToken int // initial token to output
4950
}
5051

51-
func NewLex(r io.Reader) *yyLex {
52+
// Create a new lexer
53+
//
54+
// The mode argument specifies what kind of code must be compiled; it
55+
// can be 'exec' if source consists of a sequence of statements,
56+
// 'eval' if it consists of a single expression, or 'single' if it
57+
// consists of a single interactive statement
58+
func NewLex(r io.Reader, mode string) (*yyLex, error) {
5259
x := &yyLex{
5360
reader: bufio.NewReader(r),
5461
indentStack: []int{0},
5562
state: readString,
5663
}
57-
return x
64+
switch mode {
65+
case "exec":
66+
x.startToken = FILE_INPUT
67+
case "eval":
68+
x.startToken = EVAL_INPUT
69+
case "single":
70+
x.startToken = SINGLE_INPUT
71+
x.interactive = true
72+
default:
73+
return nil, py.ExceptionNewf(py.ValueError, "compile mode must be 'exec', 'eval' or 'single'")
74+
}
75+
return x, nil
5876
}
5977

6078
// Refill line
@@ -222,6 +240,9 @@ func init() {
222240
tokenToString[DEDENT] = "DEDENT"
223241
tokenToString[STRING] = "STRING"
224242
tokenToString[NUMBER] = "NUMBER"
243+
tokenToString[FILE_INPUT] = "FILE_INPUT"
244+
tokenToString[SINGLE_INPUT] = "SINGLE_INPUT"
245+
tokenToString[EVAL_INPUT] = "EVAL_INPUT"
225246
}
226247

227248
// True if there are any open brackets
@@ -309,6 +330,13 @@ func (x *yyLex) Lex(yylval *yySymType) (ret int) {
309330
defer func() { fmt.Printf("LEX> %v\n", newLexToken(ret, yylval)) }()
310331
}
311332

333+
// Return initial token
334+
if x.startToken != eof {
335+
token := x.startToken
336+
x.startToken = eof
337+
return token
338+
}
339+
312340
// FIXME keep x.pos up to date
313341
x.pos.Lineno = 42
314342
x.pos.ColOffset = 43
@@ -318,13 +346,14 @@ func (x *yyLex) Lex(yylval *yySymType) (ret int) {
318346
// Read x.line
319347
x.refill()
320348
x.state++
321-
// an empty line while reading interactive input should return a NEWLINE
322-
if x.interactive && (x.line == "" || x.line == "\n") {
349+
if x.line == "" && x.eof {
350+
x.state = checkEof
351+
// an empty line while reading interactive input should return a NEWLINE
323352
// Don't output NEWLINE if brackets are open
324-
if x.openBrackets() {
325-
continue
353+
if x.interactive && !x.openBrackets() {
354+
return NEWLINE
326355
}
327-
return NEWLINE
356+
continue
328357
}
329358
case readIndent:
330359
// Read the initial indent and get rid of it
@@ -799,20 +828,26 @@ func SetDebug(level int) {
799828
}
800829

801830
// Parse a file
802-
func Parse(in io.Reader) (ast.Mod, error) {
803-
lex := NewLex(in)
831+
func Parse(in io.Reader, mode string) (ast.Mod, error) {
832+
lex, err := NewLex(in, mode)
833+
if err != nil {
834+
return nil, err
835+
}
804836
yyParse(lex)
805837
return lex.mod, lex.ErrorReturn()
806838
}
807839

808840
// Parse a string
809-
func ParseString(in string) (ast.Ast, error) {
810-
return Parse(bytes.NewBufferString(in))
841+
func ParseString(in string, mode string) (ast.Ast, error) {
842+
return Parse(bytes.NewBufferString(in), mode)
811843
}
812844

813845
// Lex a file only, returning a sequence of tokens
814-
func Lex(in io.Reader) (lts LexTokens, err error) {
815-
lex := NewLex(in)
846+
func Lex(in io.Reader, mode string) (lts LexTokens, err error) {
847+
lex, err := NewLex(in, mode)
848+
if err != nil {
849+
return nil, err
850+
}
816851
yylval := yySymType{}
817852
for {
818853
ret := lex.Lex(&yylval)
@@ -827,6 +862,6 @@ func Lex(in io.Reader) (lts LexTokens, err error) {
827862
}
828863

829864
// Lex a string
830-
func LexString(in string) (lts LexTokens, err error) {
831-
return Lex(bytes.NewBufferString(in))
865+
func LexString(in string, mode string) (lts LexTokens, err error) {
866+
return Lex(bytes.NewBufferString(in), mode)
832867
}

0 commit comments

Comments
 (0)