Skip to content

Commit d0ad679

Browse files
alecthomasalecthomas
committed
feat: improve Go lexer
1 parent d2f8caa commit d0ad679

File tree

5 files changed

+67
-72
lines changed

5 files changed

+67
-72
lines changed

‎lexers/go.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,10 @@ var Go = Register(MustNewLexer(
2828
func goRules() Rules {
2929
return Rules{
3030
"root": {
31-
{`\n`, Text, nil},
32-
{`\s+`, Text, nil},
33-
{`\\\n`, Text, nil},
34-
{`//[^\n\r]*`, CommentSingle, nil},
31+
{`\n`, TextWhitespace, nil},
32+
{`\s+`, TextWhitespace, nil},
33+
{`//[^\s][^\n\r]*`, CommentPreproc, nil},
34+
{`//\s+[^\n\r]*`, CommentSingle, nil},
3535
{`/(\\\n)?[*](.|\n)*?[*](\\\n)?/`, CommentMultiline, nil},
3636
{`(import|package)\b`, KeywordNamespace, nil},
3737
{`(var|func|struct|map|chan|type|interface|const)\b`, KeywordDeclaration, nil},

‎lexers/raku.go

Lines changed: 6 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package lexers
22

33
import (
44
"regexp"
5+
"slices"
56
"strings"
67
"unicode/utf8"
78

@@ -1318,16 +1319,6 @@ func indexAt(str []rune, substr []rune, pos int) int {
13181319
return idx
13191320
}
13201321

1321-
// Tells if an array of string contains a string
1322-
func contains(s []string, e string) bool {
1323-
for _, value := range s {
1324-
if value == e {
1325-
return true
1326-
}
1327-
}
1328-
return false
1329-
}
1330-
13311322
type rulePosition int
13321323

13331324
const (
@@ -1625,15 +1616,15 @@ func quote(groups []string, state *LexerState) Iterator {
16251616
var tokenState string
16261617

16271618
switch {
1628-
case keyword == "qq" || contains(tokenStates, "qq"):
1619+
case keyword == "qq" || slices.Contains(tokenStates, "qq"):
16291620
tokenState = "qq"
1630-
case adverbsStr == "ww" || contains(tokenStates, "ww"):
1621+
case adverbsStr == "ww" || slices.Contains(tokenStates, "ww"):
16311622
tokenState = "ww"
1632-
case contains(tokenStates, "Q-closure") && contains(tokenStates, "Q-variable"):
1623+
case slices.Contains(tokenStates, "Q-closure") && slices.Contains(tokenStates, "Q-variable"):
16331624
tokenState = "qq"
1634-
case contains(tokenStates, "Q-closure"):
1625+
case slices.Contains(tokenStates, "Q-closure"):
16351626
tokenState = "Q-closure"
1636-
case contains(tokenStates, "Q-variable"):
1627+
case slices.Contains(tokenStates, "Q-variable"):
16371628
tokenState = "Q-variable"
16381629
default:
16391630
tokenState = "Q"

‎lexers/testdata/go.actual

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
//go:build wasm
12
package main
23

34
import "fmt"
@@ -19,4 +20,4 @@ func hello(a int) {
1920

2021
type Int interface {
2122
~int | ~int8 | ~int16 | ~int32 | ~int64
22-
} // The very last comment w/o LF
23+
} // The very last comment w/o LF

‎lexers/testdata/go.expected

Lines changed: 53 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,63 +1,65 @@
11
[
2+
{"type":"CommentPreproc","value":"//go:build wasm"},
3+
{"type":"TextWhitespace","value":"\n"},
24
{"type":"KeywordNamespace","value":"package"},
3-
{"type":"Text","value":" "},
5+
{"type":"TextWhitespace","value":" "},
46
{"type":"NameOther","value":"main"},
5-
{"type":"Text","value":"\n\n"},
7+
{"type":"TextWhitespace","value":"\n\n"},
68
{"type":"KeywordNamespace","value":"import"},
7-
{"type":"Text","value":" "},
9+
{"type":"TextWhitespace","value":" "},
810
{"type":"LiteralString","value":"\"fmt\""},
9-
{"type":"Text","value":"\n\n"},
11+
{"type":"TextWhitespace","value":"\n\n"},
1012
{"type":"KeywordDeclaration","value":"func"},
11-
{"type":"Text","value":" "},
13+
{"type":"TextWhitespace","value":" "},
1214
{"type":"NameFunction","value":"main"},
1315
{"type":"Punctuation","value":"()"},
14-
{"type":"Text","value":" "},
16+
{"type":"TextWhitespace","value":" "},
1517
{"type":"Punctuation","value":"{"},
16-
{"type":"Text","value":"\n "},
18+
{"type":"TextWhitespace","value":"\n "},
1719
{"type":"NameOther","value":"fmt"},
1820
{"type":"Punctuation","value":"."},
1921
{"type":"NameFunction","value":"Println"},
2022
{"type":"Punctuation","value":"("},
2123
{"type":"LiteralString","value":"\"Hello World!\""},
2224
{"type":"Punctuation","value":")"},
23-
{"type":"Text","value":"\n"},
25+
{"type":"TextWhitespace","value":"\n"},
2426
{"type":"Punctuation","value":"}"},
25-
{"type":"Text","value":"\n\n"},
27+
{"type":"TextWhitespace","value":"\n\n"},
2628
{"type":"KeywordDeclaration","value":"var"},
27-
{"type":"Text","value":" "},
29+
{"type":"TextWhitespace","value":" "},
2830
{"type":"NameOther","value":"n"},
29-
{"type":"Text","value":" "},
31+
{"type":"TextWhitespace","value":" "},
3032
{"type":"KeywordType","value":"int"},
31-
{"type":"Text","value":" "},
33+
{"type":"TextWhitespace","value":" "},
3234
{"type":"Punctuation","value":"="},
33-
{"type":"Text","value":" "},
35+
{"type":"TextWhitespace","value":" "},
3436
{"type":"LiteralNumberHex","value":"0x21"},
35-
{"type":"Text","value":" "},
37+
{"type":"TextWhitespace","value":" "},
3638
{"type":"Operator","value":"+"},
37-
{"type":"Text","value":" "},
39+
{"type":"TextWhitespace","value":" "},
3840
{"type":"LiteralNumberInteger","value":"1_000"},
39-
{"type":"Text","value":"\n"},
41+
{"type":"TextWhitespace","value":"\n"},
4042
{"type":"KeywordDeclaration","value":"var"},
41-
{"type":"Text","value":" "},
43+
{"type":"TextWhitespace","value":" "},
4244
{"type":"NameOther","value":"n2"},
43-
{"type":"Text","value":" "},
45+
{"type":"TextWhitespace","value":" "},
4446
{"type":"KeywordType","value":"float64"},
45-
{"type":"Text","value":" "},
47+
{"type":"TextWhitespace","value":" "},
4648
{"type":"Punctuation","value":"="},
47-
{"type":"Text","value":" "},
49+
{"type":"TextWhitespace","value":" "},
4850
{"type":"LiteralNumberFloat","value":"1e3"},
49-
{"type":"Text","value":"\n\n"},
51+
{"type":"TextWhitespace","value":"\n\n"},
5052
{"type":"KeywordDeclaration","value":"func"},
51-
{"type":"Text","value":" "},
53+
{"type":"TextWhitespace","value":" "},
5254
{"type":"NameFunction","value":"hello"},
5355
{"type":"Punctuation","value":"("},
5456
{"type":"NameOther","value":"a"},
55-
{"type":"Text","value":" "},
57+
{"type":"TextWhitespace","value":" "},
5658
{"type":"KeywordType","value":"int"},
5759
{"type":"Punctuation","value":")"},
58-
{"type":"Text","value":" "},
60+
{"type":"TextWhitespace","value":" "},
5961
{"type":"Punctuation","value":"{"},
60-
{"type":"Text","value":"\n\t"},
62+
{"type":"TextWhitespace","value":"\n\t"},
6163
{"type":"NameOther","value":"fmt"},
6264
{"type":"Punctuation","value":"."},
6365
{"type":"NameFunction","value":"Println"},
@@ -66,58 +68,59 @@
6668
{"type":"Punctuation","value":")."},
6769
{"type":"NameFunction","value":"Hello"},
6870
{"type":"Punctuation","value":"()"},
69-
{"type":"Text","value":"\n\n "},
71+
{"type":"TextWhitespace","value":"\n\n "},
7072
{"type":"Keyword","value":"return"},
71-
{"type":"Text","value":" "},
73+
{"type":"TextWhitespace","value":" "},
7274
{"type":"KeywordDeclaration","value":"func"},
7375
{"type":"Punctuation","value":"()"},
74-
{"type":"Text","value":" "},
76+
{"type":"TextWhitespace","value":" "},
7577
{"type":"KeywordType","value":"int"},
76-
{"type":"Text","value":" "},
78+
{"type":"TextWhitespace","value":" "},
7779
{"type":"Punctuation","value":"{"},
78-
{"type":"Text","value":"\n "},
80+
{"type":"TextWhitespace","value":"\n "},
7981
{"type":"Keyword","value":"return"},
80-
{"type":"Text","value":" "},
82+
{"type":"TextWhitespace","value":" "},
8183
{"type":"NameOther","value":"i"},
82-
{"type":"Text","value":"\n "},
84+
{"type":"TextWhitespace","value":"\n "},
8385
{"type":"Punctuation","value":"}"},
84-
{"type":"Text","value":"\n"},
86+
{"type":"TextWhitespace","value":"\n"},
8587
{"type":"Punctuation","value":"}"},
86-
{"type":"Text","value":" "},
88+
{"type":"TextWhitespace","value":" "},
8789
{"type":"CommentSingle","value":"// One last thing"},
88-
{"type":"Text","value":"\n\n"},
90+
{"type":"TextWhitespace","value":"\n\n"},
8991
{"type":"KeywordDeclaration","value":"type"},
90-
{"type":"Text","value":" "},
92+
{"type":"TextWhitespace","value":" "},
9193
{"type":"NameOther","value":"Int"},
92-
{"type":"Text","value":" "},
94+
{"type":"TextWhitespace","value":" "},
9395
{"type":"KeywordDeclaration","value":"interface"},
94-
{"type":"Text","value":" "},
96+
{"type":"TextWhitespace","value":" "},
9597
{"type":"Punctuation","value":"{"},
96-
{"type":"Text","value":"\n\t"},
98+
{"type":"TextWhitespace","value":"\n\t"},
9799
{"type":"Punctuation","value":"~"},
98100
{"type":"KeywordType","value":"int"},
99-
{"type":"Text","value":" "},
101+
{"type":"TextWhitespace","value":" "},
100102
{"type":"Punctuation","value":"|"},
101-
{"type":"Text","value":" "},
103+
{"type":"TextWhitespace","value":" "},
102104
{"type":"Punctuation","value":"~"},
103105
{"type":"KeywordType","value":"int8"},
104-
{"type":"Text","value":" "},
106+
{"type":"TextWhitespace","value":" "},
105107
{"type":"Punctuation","value":"|"},
106-
{"type":"Text","value":" "},
108+
{"type":"TextWhitespace","value":" "},
107109
{"type":"Punctuation","value":"~"},
108110
{"type":"KeywordType","value":"int16"},
109-
{"type":"Text","value":" "},
111+
{"type":"TextWhitespace","value":" "},
110112
{"type":"Punctuation","value":"|"},
111-
{"type":"Text","value":" "},
113+
{"type":"TextWhitespace","value":" "},
112114
{"type":"Punctuation","value":"~"},
113115
{"type":"KeywordType","value":"int32"},
114-
{"type":"Text","value":" "},
116+
{"type":"TextWhitespace","value":" "},
115117
{"type":"Punctuation","value":"|"},
116-
{"type":"Text","value":" "},
118+
{"type":"TextWhitespace","value":" "},
117119
{"type":"Punctuation","value":"~"},
118120
{"type":"KeywordType","value":"int64"},
119-
{"type":"Text","value":"\n"},
121+
{"type":"TextWhitespace","value":"\n"},
120122
{"type":"Punctuation","value":"}"},
121-
{"type":"Text","value":" "},
122-
{"type":"CommentSingle","value":"// The very last comment w/o LF"}
123+
{"type":"TextWhitespace","value":" "},
124+
{"type":"CommentSingle","value":"// The very last comment w/o LF"},
125+
{"type":"TextWhitespace","value":"\n"}
123126
]

‎lexers/testdata/raku/raku.expected

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,14 @@
2323
{"type":"Punctuation","value":"\u003c"},
2424
{"type":"LiteralString","value":"go"},
2525
{"type":"Punctuation","value":"\u003e"},
26-
{"type":"Text","value":"\n"},
26+
{"type":"TextWhitespace","value":"\n"},
2727
{"type":"NameOther","value":"fmt"},
2828
{"type":"Punctuation","value":"."},
2929
{"type":"NameFunction","value":"Println"},
3030
{"type":"Punctuation","value":"("},
3131
{"type":"LiteralString","value":"\"Hello from Go\""},
3232
{"type":"Punctuation","value":")"},
33-
{"type":"Text","value":"\n"},
33+
{"type":"TextWhitespace","value":"\n"},
3434
{"type":"Keyword","value":"=end code"},
3535
{"type":"LiteralStringDoc","value":"\n\n"},
3636
{"type":"Comment","value":" "},

0 commit comments

Comments
 (0)