Skip to content

Commit 75a06f0

Browse files
msuozzoblurb-it[bot]pablogsal
authored
bpo-43798: Add source location attributes to alias (GH-25324)
* Add source location attributes to alias. * Move alias star construction to pegen helper. Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Pablo Galindo <[email protected]>
1 parent e05a703 commit 75a06f0

File tree

11 files changed

+199
-19
lines changed

11 files changed

+199
-19
lines changed

‎Grammar/python.gram

+3-3
Original file line numberDiff line numberDiff line change
@@ -144,20 +144,20 @@ import_from[stmt_ty]:
144144
import_from_targets[asdl_alias_seq*]:
145145
| '(' a=import_from_as_names [','] ')' { a }
146146
| import_from_as_names !','
147-
| '*' { (asdl_alias_seq*)_PyPegen_singleton_seq(p, CHECK(alias_ty, _PyPegen_alias_for_star(p))) }
147+
| '*' { (asdl_alias_seq*)_PyPegen_singleton_seq(p, CHECK(alias_ty, _PyPegen_alias_for_star(p, EXTRA))) }
148148
| invalid_import_from_targets
149149
import_from_as_names[asdl_alias_seq*]:
150150
| a[asdl_alias_seq*]=','.import_from_as_name+ { a }
151151
import_from_as_name[alias_ty]:
152152
| a=NAME b=['as' z=NAME { z }] { _PyAST_alias(a->v.Name.id,
153153
(b) ? ((expr_ty) b)->v.Name.id : NULL,
154-
p->arena) }
154+
EXTRA) }
155155
dotted_as_names[asdl_alias_seq*]:
156156
| a[asdl_alias_seq*]=','.dotted_as_name+ { a }
157157
dotted_as_name[alias_ty]:
158158
| a=dotted_name b=['as' z=NAME { z }] { _PyAST_alias(a->v.Name.id,
159159
(b) ? ((expr_ty) b)->v.Name.id : NULL,
160-
p->arena) }
160+
EXTRA) }
161161
dotted_name[expr_ty]:
162162
| a=dotted_name '.' b=NAME { _PyPegen_join_names_with_dot(p, a, b) }
163163
| NAME

‎Include/internal/pycore_ast.h

+7-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Lib/test/test_asdl_parser.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,9 @@ def test_product(self):
6262
alias = self.types['alias']
6363
self.assertEqual(
6464
str(alias),
65-
'Product([Field(identifier, name), Field(identifier, asname, opt=True)])')
65+
'Product([Field(identifier, name), Field(identifier, asname, opt=True)], '
66+
'[Field(int, lineno), Field(int, col_offset), '
67+
'Field(int, end_lineno, opt=True), Field(int, end_col_offset, opt=True)])')
6668

6769
def test_attributes(self):
6870
stmt = self.types['stmt']

‎Lib/test/test_ast.py

+25-3
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,26 @@ def test_non_interned_future_from_ast(self):
335335
mod.body[0].module = " __future__ ".strip()
336336
compile(mod, "<test>", "exec")
337337

338+
def test_alias(self):
339+
im = ast.parse("from bar import y").body[0]
340+
self.assertEqual(len(im.names), 1)
341+
alias = im.names[0]
342+
self.assertEqual(alias.name, 'y')
343+
self.assertIsNone(alias.asname)
344+
self.assertEqual(alias.lineno, 1)
345+
self.assertEqual(alias.end_lineno, 1)
346+
self.assertEqual(alias.col_offset, 16)
347+
self.assertEqual(alias.end_col_offset, 17)
348+
349+
im = ast.parse("from bar import *").body[0]
350+
alias = im.names[0]
351+
self.assertEqual(alias.name, '*')
352+
self.assertIsNone(alias.asname)
353+
self.assertEqual(alias.lineno, 1)
354+
self.assertEqual(alias.end_lineno, 1)
355+
self.assertEqual(alias.col_offset, 16)
356+
self.assertEqual(alias.end_col_offset, 17)
357+
338358
def test_base_classes(self):
339359
self.assertTrue(issubclass(ast.For, ast.stmt))
340360
self.assertTrue(issubclass(ast.Name, ast.expr))
@@ -1037,7 +1057,8 @@ def test_bad_integer(self):
10371057

10381058
def test_level_as_none(self):
10391059
body = [ast.ImportFrom(module='time',
1040-
names=[ast.alias(name='sleep')],
1060+
names=[ast.alias(name='sleep',
1061+
lineno=0, col_offset=0)],
10411062
level=None,
10421063
lineno=0, col_offset=0)]
10431064
mod = ast.Module(body, [])
@@ -1735,6 +1756,7 @@ def test_import_from_multi_line(self):
17351756
''').strip()
17361757
imp = ast.parse(s).body[0]
17371758
self._check_end_pos(imp, 3, 1)
1759+
self._check_end_pos(imp.names[2], 2, 16)
17381760

17391761
def test_slices(self):
17401762
s1 = 'f()[1, 2] [0]'
@@ -2095,8 +2117,8 @@ def main():
20952117
('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []),
20962118
('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [], [], [('Pass', (4, 2, 4, 6))])], []),
20972119
('Module', [('Assert', (1, 0, 1, 8), ('Name', (1, 7, 1, 8), 'v', ('Load',)), None)], []),
2098-
('Module', [('Import', (1, 0, 1, 10), [('alias', 'sys', None)])], []),
2099-
('Module', [('ImportFrom', (1, 0, 1, 17), 'sys', [('alias', 'v', None)], 0)], []),
2120+
('Module', [('Import', (1, 0, 1, 10), [('alias', (1, 7, 1, 10), 'sys', None)])], []),
2121+
('Module', [('ImportFrom', (1, 0, 1, 17), 'sys', [('alias', (1, 16, 1, 17), 'v', None)], 0)], []),
21002122
('Module', [('Global', (1, 0, 1, 8), ['v'])], []),
21012123
('Module', [('Expr', (1, 0, 1, 1), ('Constant', (1, 0, 1, 1), 1, None))], []),
21022124
('Module', [('Pass', (1, 0, 1, 4))], []),

‎Lib/test/test_peg_generator/test_c_parser.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -314,7 +314,7 @@ def test_same_name_different_types(self) -> None:
314314
)
315315
simple_name[expr_ty]: NAME
316316
import_as_names_from[asdl_alias_seq*]: a[asdl_alias_seq*]=','.import_as_name_from+ { a }
317-
import_as_name_from[alias_ty]: a=NAME 'as' b=NAME { _PyAST_alias(((expr_ty) a)->v.Name.id, ((expr_ty) b)->v.Name.id, p->arena) }
317+
import_as_name_from[alias_ty]: a=NAME 'as' b=NAME { _PyAST_alias(((expr_ty) a)->v.Name.id, ((expr_ty) b)->v.Name.id, EXTRA) }
318318
"""
319319
test_source = """
320320
for stmt in ("from a import b as c", "from . import a as b"):

‎Misc/NEWS.d/next/Core and Builtins/2021-04-10-00-01-43.bpo-43798.p_nJFM.rst

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
:class:`ast.alias` nodes now include source location metadata attributes e.g. lineno, col_offset.

‎Parser/Python.asdl

+1
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ module Python
124124

125125
-- import name with optional 'as' alias.
126126
alias = (identifier name, identifier? asname)
127+
attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset)
127128

128129
withitem = (expr context_expr, expr? optional_vars)
129130

‎Parser/parser.c

+57-3
Original file line numberDiff line numberDiff line change
@@ -3310,6 +3310,15 @@ import_from_targets_rule(Parser *p)
33103310
}
33113311
asdl_alias_seq* _res = NULL;
33123312
int _mark = p->mark;
3313+
if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) {
3314+
p->error_indicator = 1;
3315+
D(p->level--);
3316+
return NULL;
3317+
}
3318+
int _start_lineno = p->tokens[_mark]->lineno;
3319+
UNUSED(_start_lineno); // Only used by EXTRA macro
3320+
int _start_col_offset = p->tokens[_mark]->col_offset;
3321+
UNUSED(_start_col_offset); // Only used by EXTRA macro
33133322
{ // '(' import_from_as_names ','? ')'
33143323
if (p->error_indicator) {
33153324
D(p->level--);
@@ -3377,7 +3386,16 @@ import_from_targets_rule(Parser *p)
33773386
)
33783387
{
33793388
D(fprintf(stderr, "%*c+ import_from_targets[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'"));
3380-
_res = ( asdl_alias_seq * ) _PyPegen_singleton_seq ( p , CHECK ( alias_ty , _PyPegen_alias_for_star ( p ) ) );
3389+
Token *_token = _PyPegen_get_last_nonnwhitespace_token(p);
3390+
if (_token == NULL) {
3391+
D(p->level--);
3392+
return NULL;
3393+
}
3394+
int _end_lineno = _token->end_lineno;
3395+
UNUSED(_end_lineno); // Only used by EXTRA macro
3396+
int _end_col_offset = _token->end_col_offset;
3397+
UNUSED(_end_col_offset); // Only used by EXTRA macro
3398+
_res = ( asdl_alias_seq * ) _PyPegen_singleton_seq ( p , CHECK ( alias_ty , _PyPegen_alias_for_star ( p , EXTRA ) ) );
33813399
if (_res == NULL && PyErr_Occurred()) {
33823400
p->error_indicator = 1;
33833401
D(p->level--);
@@ -3466,6 +3484,15 @@ import_from_as_name_rule(Parser *p)
34663484
}
34673485
alias_ty _res = NULL;
34683486
int _mark = p->mark;
3487+
if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) {
3488+
p->error_indicator = 1;
3489+
D(p->level--);
3490+
return NULL;
3491+
}
3492+
int _start_lineno = p->tokens[_mark]->lineno;
3493+
UNUSED(_start_lineno); // Only used by EXTRA macro
3494+
int _start_col_offset = p->tokens[_mark]->col_offset;
3495+
UNUSED(_start_col_offset); // Only used by EXTRA macro
34693496
{ // NAME ['as' NAME]
34703497
if (p->error_indicator) {
34713498
D(p->level--);
@@ -3481,7 +3508,16 @@ import_from_as_name_rule(Parser *p)
34813508
)
34823509
{
34833510
D(fprintf(stderr, "%*c+ import_from_as_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME ['as' NAME]"));
3484-
_res = _PyAST_alias ( a -> v . Name . id , ( b ) ? ( ( expr_ty ) b ) -> v . Name . id : NULL , p -> arena );
3511+
Token *_token = _PyPegen_get_last_nonnwhitespace_token(p);
3512+
if (_token == NULL) {
3513+
D(p->level--);
3514+
return NULL;
3515+
}
3516+
int _end_lineno = _token->end_lineno;
3517+
UNUSED(_end_lineno); // Only used by EXTRA macro
3518+
int _end_col_offset = _token->end_col_offset;
3519+
UNUSED(_end_col_offset); // Only used by EXTRA macro
3520+
_res = _PyAST_alias ( a -> v . Name . id , ( b ) ? ( ( expr_ty ) b ) -> v . Name . id : NULL , EXTRA );
34853521
if (_res == NULL && PyErr_Occurred()) {
34863522
p->error_indicator = 1;
34873523
D(p->level--);
@@ -3551,6 +3587,15 @@ dotted_as_name_rule(Parser *p)
35513587
}
35523588
alias_ty _res = NULL;
35533589
int _mark = p->mark;
3590+
if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) {
3591+
p->error_indicator = 1;
3592+
D(p->level--);
3593+
return NULL;
3594+
}
3595+
int _start_lineno = p->tokens[_mark]->lineno;
3596+
UNUSED(_start_lineno); // Only used by EXTRA macro
3597+
int _start_col_offset = p->tokens[_mark]->col_offset;
3598+
UNUSED(_start_col_offset); // Only used by EXTRA macro
35543599
{ // dotted_name ['as' NAME]
35553600
if (p->error_indicator) {
35563601
D(p->level--);
@@ -3566,7 +3611,16 @@ dotted_as_name_rule(Parser *p)
35663611
)
35673612
{
35683613
D(fprintf(stderr, "%*c+ dotted_as_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name ['as' NAME]"));
3569-
_res = _PyAST_alias ( a -> v . Name . id , ( b ) ? ( ( expr_ty ) b ) -> v . Name . id : NULL , p -> arena );
3614+
Token *_token = _PyPegen_get_last_nonnwhitespace_token(p);
3615+
if (_token == NULL) {
3616+
D(p->level--);
3617+
return NULL;
3618+
}
3619+
int _end_lineno = _token->end_lineno;
3620+
UNUSED(_end_lineno); // Only used by EXTRA macro
3621+
int _end_col_offset = _token->end_col_offset;
3622+
UNUSED(_end_col_offset); // Only used by EXTRA macro
3623+
_res = _PyAST_alias ( a -> v . Name . id , ( b ) ? ( ( expr_ty ) b ) -> v . Name . id : NULL , EXTRA );
35703624
if (_res == NULL && PyErr_Occurred()) {
35713625
p->error_indicator = 1;
35723626
D(p->level--);

‎Parser/pegen.c

+3-3
Original file line numberDiff line numberDiff line change
@@ -1555,8 +1555,8 @@ _PyPegen_seq_count_dots(asdl_seq *seq)
15551555

15561556
/* Creates an alias with '*' as the identifier name */
15571557
alias_ty
1558-
_PyPegen_alias_for_star(Parser *p)
1559-
{
1558+
_PyPegen_alias_for_star(Parser *p, int lineno, int col_offset, int end_lineno,
1559+
int end_col_offset, PyArena *arena) {
15601560
PyObject *str = PyUnicode_InternFromString("*");
15611561
if (!str) {
15621562
return NULL;
@@ -1565,7 +1565,7 @@ _PyPegen_alias_for_star(Parser *p)
15651565
Py_DECREF(str);
15661566
return NULL;
15671567
}
1568-
return _PyAST_alias(str, NULL, p->arena);
1568+
return _PyAST_alias(str, NULL, lineno, col_offset, end_lineno, end_col_offset, arena);
15691569
}
15701570

15711571
/* Creates a new asdl_seq* with the identifiers of all the names in seq */

‎Parser/pegen.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,7 @@ asdl_seq *_PyPegen_seq_append_to_end(Parser *, asdl_seq *, void *);
237237
asdl_seq *_PyPegen_seq_flatten(Parser *, asdl_seq *);
238238
expr_ty _PyPegen_join_names_with_dot(Parser *, expr_ty, expr_ty);
239239
int _PyPegen_seq_count_dots(asdl_seq *);
240-
alias_ty _PyPegen_alias_for_star(Parser *);
240+
alias_ty _PyPegen_alias_for_star(Parser *, int, int, int, int, PyArena *);
241241
asdl_identifier_seq *_PyPegen_map_names_to_ids(Parser *, asdl_expr_seq *);
242242
CmpopExprPair *_PyPegen_cmpop_expr_pair(Parser *, cmpop_ty, expr_ty);
243243
asdl_int_seq *_PyPegen_get_cmpops(Parser *p, asdl_seq *);

0 commit comments

Comments
 (0)