Skip to content

Commit f1c2c69

Browse files
ncwncw
committed
compiler: symbol table framework, plus test machinery
1 parent 611c8dc commit f1c2c69

6 files changed

+535
-9
lines changed

‎compile/compile_data_test.go

+41-8
Original file line numberDiff line numberDiff line change
@@ -1151,7 +1151,7 @@ var compileTestData = []struct {
11511151
Name: "<module>",
11521152
Firstlineno: 1,
11531153
Lnotab: "",
1154-
}, " 1 0 LOAD_CONST 0 (<code object <lambda> at 0x7fe734191db0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('<lambda>')\n 6 MAKE_FUNCTION 0\n 9 RETURN_VALUE\n", nil, ""},
1154+
}, " 1 0 LOAD_CONST 0 (<code object <lambda> at 0x7fd7bccdcdb0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('<lambda>')\n 6 MAKE_FUNCTION 0\n 9 RETURN_VALUE\n", nil, ""},
11551155
{"pass", "exec", &py.Code{
11561156
Argcount: 0,
11571157
Kwonlyargcount: 0,
@@ -1951,7 +1951,7 @@ var compileTestData = []struct {
19511951
Name: "<module>",
19521952
Firstlineno: 1,
19531953
Lnotab: "",
1954-
}, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fe7341918a0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
1954+
}, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fd7bccdc8a0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
19551955
{"def fn(a): pass", "exec", &py.Code{
19561956
Argcount: 0,
19571957
Kwonlyargcount: 0,
@@ -1984,7 +1984,7 @@ var compileTestData = []struct {
19841984
Name: "<module>",
19851985
Firstlineno: 1,
19861986
Lnotab: "",
1987-
}, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fe73567c6f0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
1987+
}, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fd7be1c76f0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
19881988
{"def fn(a,b,c): pass", "exec", &py.Code{
19891989
Argcount: 0,
19901990
Kwonlyargcount: 0,
@@ -2017,7 +2017,7 @@ var compileTestData = []struct {
20172017
Name: "<module>",
20182018
Firstlineno: 1,
20192019
Lnotab: "",
2020-
}, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fe734191db0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
2020+
}, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fd7bccdcdb0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
20212021
{"def fn(a,b=1,c=2): pass", "exec", &py.Code{
20222022
Argcount: 0,
20232023
Kwonlyargcount: 0,
@@ -2050,7 +2050,7 @@ var compileTestData = []struct {
20502050
Name: "<module>",
20512051
Firstlineno: 1,
20522052
Lnotab: "",
2053-
}, " 1 0 LOAD_CONST 0 (1)\n 3 LOAD_CONST 1 (2)\n 6 LOAD_CONST 2 (<code object fn at 0x7fe7341b68a0, file \"<string>\", line 1>)\n 9 LOAD_CONST 3 ('fn')\n 12 MAKE_FUNCTION 2\n 15 STORE_NAME 0 (fn)\n 18 LOAD_CONST 4 (None)\n 21 RETURN_VALUE\n", nil, ""},
2053+
}, " 1 0 LOAD_CONST 0 (1)\n 3 LOAD_CONST 1 (2)\n 6 LOAD_CONST 2 (<code object fn at 0x7fd7bcd028a0, file \"<string>\", line 1>)\n 9 LOAD_CONST 3 ('fn')\n 12 MAKE_FUNCTION 2\n 15 STORE_NAME 0 (fn)\n 18 LOAD_CONST 4 (None)\n 21 RETURN_VALUE\n", nil, ""},
20542054
{"def fn(a,*arg,b=1,c=2): pass", "exec", &py.Code{
20552055
Argcount: 0,
20562056
Kwonlyargcount: 0,
@@ -2083,7 +2083,7 @@ var compileTestData = []struct {
20832083
Name: "<module>",
20842084
Firstlineno: 1,
20852085
Lnotab: "",
2086-
}, " 1 0 LOAD_CONST 0 ('b')\n 3 LOAD_CONST 1 (1)\n 6 LOAD_CONST 2 ('c')\n 9 LOAD_CONST 3 (2)\n 12 LOAD_CONST 4 (<code object fn at 0x7fe7341918a0, file \"<string>\", line 1>)\n 15 LOAD_CONST 5 ('fn')\n 18 MAKE_FUNCTION 512\n 21 STORE_NAME 0 (fn)\n 24 LOAD_CONST 6 (None)\n 27 RETURN_VALUE\n", nil, ""},
2086+
}, " 1 0 LOAD_CONST 0 ('b')\n 3 LOAD_CONST 1 (1)\n 6 LOAD_CONST 2 ('c')\n 9 LOAD_CONST 3 (2)\n 12 LOAD_CONST 4 (<code object fn at 0x7fd7bccdc8a0, file \"<string>\", line 1>)\n 15 LOAD_CONST 5 ('fn')\n 18 MAKE_FUNCTION 512\n 21 STORE_NAME 0 (fn)\n 24 LOAD_CONST 6 (None)\n 27 RETURN_VALUE\n", nil, ""},
20872087
{"def fn(a,*arg,b=1,c=2,**kwargs): pass", "exec", &py.Code{
20882088
Argcount: 0,
20892089
Kwonlyargcount: 0,
@@ -2116,7 +2116,7 @@ var compileTestData = []struct {
21162116
Name: "<module>",
21172117
Firstlineno: 1,
21182118
Lnotab: "",
2119-
}, " 1 0 LOAD_CONST 0 ('b')\n 3 LOAD_CONST 1 (1)\n 6 LOAD_CONST 2 ('c')\n 9 LOAD_CONST 3 (2)\n 12 LOAD_CONST 4 (<code object fn at 0x7fe73567c6f0, file \"<string>\", line 1>)\n 15 LOAD_CONST 5 ('fn')\n 18 MAKE_FUNCTION 512\n 21 STORE_NAME 0 (fn)\n 24 LOAD_CONST 6 (None)\n 27 RETURN_VALUE\n", nil, ""},
2119+
}, " 1 0 LOAD_CONST 0 ('b')\n 3 LOAD_CONST 1 (1)\n 6 LOAD_CONST 2 ('c')\n 9 LOAD_CONST 3 (2)\n 12 LOAD_CONST 4 (<code object fn at 0x7fd7be1c76f0, file \"<string>\", line 1>)\n 15 LOAD_CONST 5 ('fn')\n 18 MAKE_FUNCTION 512\n 21 STORE_NAME 0 (fn)\n 24 LOAD_CONST 6 (None)\n 27 RETURN_VALUE\n", nil, ""},
21202120
{"def fn(a:\"a\",*arg:\"arg\",b:\"b\"=1,c:\"c\"=2,**kwargs:\"kw\") -> \"ret\": pass", "exec", &py.Code{
21212121
Argcount: 0,
21222122
Kwonlyargcount: 0,
@@ -2149,5 +2149,38 @@ var compileTestData = []struct {
21492149
Name: "<module>",
21502150
Firstlineno: 1,
21512151
Lnotab: "",
2152-
}, " 1 0 LOAD_CONST 0 ('b')\n 3 LOAD_CONST 1 (1)\n 6 LOAD_CONST 2 ('c')\n 9 LOAD_CONST 3 (2)\n 12 LOAD_CONST 4 ('a')\n 15 LOAD_CONST 5 ('arg')\n 18 LOAD_CONST 0 ('b')\n 21 LOAD_CONST 2 ('c')\n 24 LOAD_CONST 6 ('kw')\n 27 LOAD_CONST 7 ('ret')\n 30 LOAD_CONST 8 (('a', 'arg', 'b', 'c', 'kwargs', 'return'))\n 33 LOAD_CONST 9 (<code object fn at 0x7fe734191db0, file \"<string>\", line 1>)\n 36 LOAD_CONST 10 ('fn')\n 39 EXTENDED_ARG 7\n 42 MAKE_FUNCTION 459264\n 45 STORE_NAME 0 (fn)\n 48 LOAD_CONST 11 (None)\n 51 RETURN_VALUE\n", nil, ""},
2152+
}, " 1 0 LOAD_CONST 0 ('b')\n 3 LOAD_CONST 1 (1)\n 6 LOAD_CONST 2 ('c')\n 9 LOAD_CONST 3 (2)\n 12 LOAD_CONST 4 ('a')\n 15 LOAD_CONST 5 ('arg')\n 18 LOAD_CONST 0 ('b')\n 21 LOAD_CONST 2 ('c')\n 24 LOAD_CONST 6 ('kw')\n 27 LOAD_CONST 7 ('ret')\n 30 LOAD_CONST 8 (('a', 'arg', 'b', 'c', 'kwargs', 'return'))\n 33 LOAD_CONST 9 (<code object fn at 0x7fd7bccdcdb0, file \"<string>\", line 1>)\n 36 LOAD_CONST 10 ('fn')\n 39 EXTENDED_ARG 7\n 42 MAKE_FUNCTION 459264\n 45 STORE_NAME 0 (fn)\n 48 LOAD_CONST 11 (None)\n 51 RETURN_VALUE\n", nil, ""},
2153+
{"def fn(): a+b", "exec", &py.Code{
2154+
Argcount: 0,
2155+
Kwonlyargcount: 0,
2156+
Nlocals: 0,
2157+
Stacksize: 2,
2158+
Flags: 64,
2159+
Code: "\x64\x00\x00\x64\x01\x00\x84\x00\x00\x5a\x00\x00\x64\x02\x00\x53",
2160+
Consts: []py.Object{&py.Code{
2161+
Argcount: 0,
2162+
Kwonlyargcount: 0,
2163+
Nlocals: 0,
2164+
Stacksize: 2,
2165+
Flags: 67,
2166+
Code: "\x74\x00\x00\x74\x01\x00\x17\x01\x64\x00\x00\x53",
2167+
Consts: []py.Object{py.None},
2168+
Names: []string{"a", "b"},
2169+
Varnames: []string{},
2170+
Freevars: []string{},
2171+
Cellvars: []string{},
2172+
Filename: "<string>",
2173+
Name: "fn",
2174+
Firstlineno: 1,
2175+
Lnotab: "",
2176+
}, py.String("fn"), py.None},
2177+
Names: []string{"fn"},
2178+
Varnames: []string{},
2179+
Freevars: []string{},
2180+
Cellvars: []string{},
2181+
Filename: "<string>",
2182+
Name: "<module>",
2183+
Firstlineno: 1,
2184+
Lnotab: "",
2185+
}, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fd7bcd028a0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
21532186
}

‎compile/make_compile_test.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@
164164
('''def fn(a,*arg,b=1,c=2): pass''', "exec"),
165165
('''def fn(a,*arg,b=1,c=2,**kwargs): pass''', "exec"),
166166
('''def fn(a:"a",*arg:"arg",b:"b"=1,c:"c"=2,**kwargs:"kw") -> "ret": pass''', "exec"),
167-
#('''def fn(): a+b''', "exec"),
167+
('''def fn(): a+b''', "exec"),
168168

169169
]
170170

‎compile/make_symtable_test.py

+158
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
#!/usr/bin/env python3.4
2+
"""
3+
Write symtable_data_test.go
4+
"""
5+
6+
import sys
7+
import ast
8+
import subprocess
9+
import dis
10+
from symtable import symtable
11+
12+
# FIXME test errors too
13+
14+
inp = [
15+
('''1''', "eval"),
16+
#('''a*b*c''', "eval"),
17+
#('''def fn(): pass''', "exec"),
18+
#('''def fn(a,b):\n e=1\n return a*b*c*d*e''', "exec"),
19+
#('''def fn(a,b):\n def nested(c,d):\n return a*b*c*d*e''', "exec"),
20+
# ('''def fn(a,b,c): pass''', "exec"),
21+
# ('''def fn(a,b=1,c=2): pass''', "exec"),
22+
# ('''def fn(a,*arg,b=1,c=2): pass''', "exec"),
23+
# ('''def fn(a,*arg,b=1,c=2,**kwargs): pass''', "exec"),
24+
# ('''def fn(a:"a",*arg:"arg",b:"b"=1,c:"c"=2,**kwargs:"kw") -> "ret": pass''', "exec"),
25+
# ('''def fn(): a+b''', "exec"),
26+
]
27+
28+
def dump_bool(b):
29+
return ("true" if b else "false")
30+
31+
def dump_strings(ss):
32+
return "[]string{"+",".join([ '"%s"' % s for s in ss ])+"}"
33+
34+
# Scope numbers to names (from symtable.h)
35+
SCOPES = {
36+
1: "scopeLocal",
37+
2: "scopeGlobalExplicit",
38+
3: "scopeGlobalImplicit",
39+
4: "scopeFree",
40+
5: "scopeCell",
41+
}
42+
43+
#def-use flags to names (from symtable.h)
44+
DEF_FLAGS = (
45+
("defGlobal", 1), # global stmt
46+
("defLocal", 2), # assignment in code block
47+
("defParam", 2<<1), # formal parameter
48+
("defNonlocal", 2<<2), # nonlocal stmt
49+
("defUse", 2<<3), # name is used
50+
("defFree", 2<<4), # name used but not defined in nested block
51+
("defFreeClass", 2<<5),# free variable from class's method
52+
("defImport", 2<<6), # assignment occurred via import
53+
)
54+
55+
BLOCK_TYPES = {
56+
"function": "FunctionBlock",
57+
"class": "ClassBlock",
58+
"module": "ModuleBlock",
59+
}
60+
61+
def dump_symtable(st):
62+
"""Dump the symtable"""
63+
out = "&SymTable{\n"
64+
out += 'Type:%s,\n' % BLOCK_TYPES[st.get_type()] # Return the type of the symbol table. Possible values are 'class', 'module', and 'function'.
65+
out += 'Name:"%s",\n' % st.get_name() # Return the table’s name. This is the name of the class if the table is for a class, the name of the function if the table is for a function, or 'top' if the table is global (get_type() returns 'module').
66+
67+
out += 'Lineno:%s,\n' % st.get_lineno() # Return the number of the first line in the block this table represents.
68+
out += 'Optimized:%s,\n' % dump_bool(st.is_optimized()) # Return True if the locals in this table can be optimized.
69+
out += 'Nested:%s,\n' % dump_bool(st.is_nested()) # Return True if the block is a nested class or function.
70+
out += 'Exec:%s,\n' % dump_bool(st.has_exec()) # Return True if the block uses exec.
71+
out += 'ImportStar:%s,\n' % dump_bool(st.has_import_star()) # Return True if the block uses a starred from-import.
72+
out += 'Symbols: Symbols{\n'
73+
for name in st.get_identifiers():
74+
value = st.lookup(name)
75+
out += '"%s":%s,\n' % (name, dump_symbol(value))
76+
out += '},\n'
77+
# out += 'children:"%s",\n' % st.get_children() # Return a list of the nested symbol tables.
78+
out += "}"
79+
return out
80+
81+
def dump_symbol(s):
82+
"""Dump a symbol"""
83+
#class symtable.Symbol
84+
# An entry in a SymbolTable corresponding to an identifier in the source. The constructor is not public.
85+
out = "Symbol{\n"
86+
out += 'Name:"%s",\n' % s.get_name() # Return the symbol’s name.
87+
flags = []
88+
flag_bits = s._Symbol__flags
89+
for name, mask in DEF_FLAGS:
90+
if (flag_bits & mask) != 0:
91+
flags.append(name)
92+
if not flags:
93+
flags = ["0"]
94+
out += 'Flags:%s,\n' % "|".join(flags)
95+
96+
scope = SCOPES.get(s._Symbol__scope, "scopeUnknown")
97+
out += 'Scope:%s,\n' % scope
98+
99+
# Return a namespace bound to this name.
100+
ns = s.get_namespaces()
101+
if len(ns) == 0:
102+
pass
103+
elif len(ns) == 1:
104+
out += 'Namespace:%s,\n' % dump_symtable(ns[0])
105+
else:
106+
raise AssertionError("More than one namespace")
107+
out += "}"
108+
return out
109+
110+
def escape(x):
111+
"""Encode strings with backslashes for python/go"""
112+
return x.replace('\\', "\\\\").replace('"', r'\"').replace("\n", r'\n').replace("\t", r'\t')
113+
114+
def main():
115+
"""Write symtable_data_test.go"""
116+
path = "symtable_data_test.go"
117+
out = ["""// Test data generated by make_symtable_test.py - do not edit
118+
119+
package compile
120+
121+
import (
122+
"github.com/ncw/gpython/py"
123+
)
124+
125+
var symtableTestData = []struct {
126+
in string
127+
mode string // exec, eval or single
128+
out *SymTable
129+
exceptionType *py.Type
130+
errString string
131+
}{"""]
132+
for x in inp:
133+
source, mode = x[:2]
134+
if len(x) > 2:
135+
exc = x[2]
136+
try:
137+
table = symtable(source, "<string>", mode)
138+
except exc as e:
139+
error = e.msg
140+
else:
141+
raise ValueError("Expecting exception %s" % exc)
142+
table = "nil"
143+
gostring = "nil"
144+
exc_name = "py.%s" % exc.__name__
145+
else:
146+
table = symtable(source, "<string>", mode)
147+
exc_name = "nil"
148+
error = ""
149+
out.append('{"%s", "%s", %s, %s, "%s"},' % (escape(source), mode, dump_symtable(table), exc_name, escape(error)))
150+
out.append("}\n")
151+
print("Writing %s" % path)
152+
with open(path, "w") as f:
153+
f.write("\n".join(out))
154+
f.write("\n")
155+
subprocess.check_call(["gofmt", "-w", path])
156+
157+
if __name__ == "__main__":
158+
main()

0 commit comments

Comments
 (0)