Skip to content

Commit 378d65e

Browse files
committed
Fix exec mode - add \n on end if not present
1 parent 41189eb commit 378d65e

File tree

2 files changed

+30
-22
lines changed

2 files changed

+30
-22
lines changed

parser/lexer.go

+8-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@ type yyLex struct {
4141
indentStack []int // indent stack to control INDENT / DEDENT tokens
4242
state int // current state of state machine
4343
currentIndent string // whitespace at start of current line
44-
interactive bool // set if reading interactive input
44+
interactive bool // set if mode "single" reading interactive input
45+
exec bool // set if mode "exec" reading from file
4546
bracket int // number of open [ ]
4647
parenthesis int // number of open ( )
4748
brace int // number of open { }
@@ -64,6 +65,7 @@ func NewLex(r io.Reader, mode string) (*yyLex, error) {
6465
switch mode {
6566
case "exec":
6667
x.startToken = FILE_INPUT
68+
x.exec = true
6769
case "eval":
6870
x.startToken = EVAL_INPUT
6971
case "single":
@@ -90,6 +92,11 @@ func (x *yyLex) refill() {
9092
x.eof = true
9193
x.Errorf("Error reading input: %v", err)
9294
}
95+
// If this is exec input, add a newline to the end of the
96+
// string if there isn't one already.
97+
if x.eof && x.exec && len(x.line) > 0 && x.line[len(x.line)-1] != '\n' {
98+
x.line += "\n"
99+
}
93100
}
94101

95102
// Finds the length of a space and tab seperated string

parser/lexer_test.go

+22-21
Original file line numberDiff line numberDiff line change
@@ -263,24 +263,25 @@ func TestLex(t *testing.T) {
263263
{'(', nil},
264264
{NUMBER, py.Int(1)},
265265
{')', nil},
266+
{NEWLINE, nil},
266267
{ENDMARKER, nil},
267268
}},
268-
{"{\n 1\n}", "", "exec", LexTokens{
269-
{FILE_INPUT, nil},
269+
{"{\n 1\n}", "", "single", LexTokens{
270+
{SINGLE_INPUT, nil},
270271
{'{', nil},
271272
{NUMBER, py.Int(1)},
272273
{'}', nil},
273274
{ENDMARKER, nil},
274275
}},
275-
{"[\n 1\n]", "", "exec", LexTokens{
276-
{FILE_INPUT, nil},
276+
{"[\n 1\n]", "", "eval", LexTokens{
277+
{EVAL_INPUT, nil},
277278
{'[', nil},
278279
{NUMBER, py.Int(1)},
279280
{']', nil},
280281
{ENDMARKER, nil},
281282
}},
282-
{"1\\\n2", "", "exec", LexTokens{
283-
{FILE_INPUT, nil},
283+
{"1\\\n2", "", "eval", LexTokens{
284+
{EVAL_INPUT, nil},
284285
{NUMBER, py.Int(1)},
285286
{NUMBER, py.Int(2)},
286287
{ENDMARKER, nil},
@@ -295,33 +296,33 @@ func TestLex(t *testing.T) {
295296
{NUMBER, py.Int(1)},
296297
{ENDMARKER, nil},
297298
}},
298-
{"'1\\\n2'", "", "exec", LexTokens{
299-
{FILE_INPUT, nil},
299+
{"'1\\\n2'", "", "single", LexTokens{
300+
{SINGLE_INPUT, nil},
300301
{STRING, py.String("12")},
301302
{ENDMARKER, nil},
302303
}},
303-
{"0x1234 +\t0.1-6.1j", "", "exec", LexTokens{
304-
{FILE_INPUT, nil},
304+
{"0x1234 +\t0.1-6.1j", "", "eval", LexTokens{
305+
{EVAL_INPUT, nil},
305306
{NUMBER, py.Int(0x1234)},
306307
{'+', nil},
307308
{NUMBER, py.Float(0.1)},
308309
{'-', nil},
309310
{NUMBER, py.Complex(complex(0, 6.1))},
310311
{ENDMARKER, nil},
311312
}},
312-
{"001", "illegal decimal with leading zero", "exec", LexTokens{
313-
{FILE_INPUT, nil},
313+
{"001", "illegal decimal with leading zero", "eval", LexTokens{
314+
{EVAL_INPUT, nil},
314315
}},
315-
{"u'''1\n2\n'''", "", "exec", LexTokens{
316-
{FILE_INPUT, nil},
316+
{"u'''1\n2\n'''", "", "eval", LexTokens{
317+
{EVAL_INPUT, nil},
317318
{STRING, py.String("1\n2\n")},
318319
{ENDMARKER, nil},
319320
}},
320-
{"\"hello\n", "Unterminated \"x\" string", "exec", LexTokens{
321-
{FILE_INPUT, nil},
321+
{"\"hello\n", "Unterminated \"x\" string", "eval", LexTokens{
322+
{EVAL_INPUT, nil},
322323
}},
323-
{"1 >>-3\na <<=+12", "", "exec", LexTokens{
324-
{FILE_INPUT, nil},
324+
{"1 >>-3\na <<=+12", "", "eval", LexTokens{
325+
{EVAL_INPUT, nil},
325326
{NUMBER, py.Int(1)},
326327
{GTGT, nil},
327328
{'-', nil},
@@ -333,8 +334,8 @@ func TestLex(t *testing.T) {
333334
{NUMBER, py.Int(12)},
334335
{ENDMARKER, nil},
335336
}},
336-
{"$asdasd", "invalid syntax", "exec", LexTokens{
337-
{FILE_INPUT, nil},
337+
{"$asdasd", "invalid syntax", "eval", LexTokens{
338+
{EVAL_INPUT, nil},
338339
}},
339340
} {
340341
lts, err := LexString(test.in, test.mode)
@@ -595,7 +596,7 @@ func TestLexerReadString(t *testing.T) {
595596
{`BR"""a\nc"""`, STRING, py.Bytes(string(`a\nc`)), ``},
596597
{`rB'''a\"c'''`, STRING, py.Bytes(string(`a\"c`)), ``},
597598
} {
598-
x, err := NewLex(bytes.NewBufferString(test.in), "exec")
599+
x, err := NewLex(bytes.NewBufferString(test.in), "eval")
599600
if err != nil {
600601
t.Fatal(err)
601602
}

0 commit comments

Comments
 (0)