From f56a805acf6b6a8a561f08c2ab123140c3afb557 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 7 May 2025 09:30:27 +0200 Subject: [PATCH 1/5] gh-133555: allow to regenerate the parser with Python < 3.14 --- Tools/peg_generator/pegen/parser_generator.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py index 6ce0649aefe7ff..3552ce4ea1e91b 100644 --- a/Tools/peg_generator/pegen/parser_generator.py +++ b/Tools/peg_generator/pegen/parser_generator.py @@ -81,6 +81,11 @@ def __init__(self, rules: Dict[str, Rule], tokens: Set[str]): self.tokens.add("FSTRING_START") self.tokens.add("FSTRING_END") self.tokens.add("FSTRING_MIDDLE") + # If python < 3.14 add the virtual tstring tokens + if sys.version_info < (3, 14): + self.tokens.add("TSTRING_START") + self.tokens.add("TSTRING_END") + self.tokens.add("TSTRING_MIDDLE") def visit_NameLeaf(self, node: NameLeaf) -> None: if node.value not in self.rules and node.value not in self.tokens: From bc0ffe819cabe2b4769d5c26a81b7b06a8319c5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Preng=C3=A8re?= <2138730+alexprengere@users.noreply.github.com> Date: Wed, 7 May 2025 11:58:22 +0200 Subject: [PATCH 2/5] Update Tools/peg_generator/pegen/parser_generator.py Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Tools/peg_generator/pegen/parser_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py index 3552ce4ea1e91b..52ae743c26b6b8 100644 --- a/Tools/peg_generator/pegen/parser_generator.py +++ b/Tools/peg_generator/pegen/parser_generator.py @@ -82,7 +82,7 @@ def __init__(self, rules: Dict[str, Rule], tokens: Set[str]): self.tokens.add("FSTRING_END") self.tokens.add("FSTRING_MIDDLE") # If python < 3.14 add the virtual tstring tokens - if sys.version_info < (3, 14): + if sys.version_info < (3, 14, 0, 'beta', 1): self.tokens.add("TSTRING_START") self.tokens.add("TSTRING_END") self.tokens.add("TSTRING_MIDDLE") From 5043a30fa45e819371970b9e06218237eaa364e2 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 7 May 2025 15:46:33 +0200 Subject: [PATCH 3/5] Add comment regarding updating the peg generator when adding tokens --- Grammar/Tokens | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Grammar/Tokens b/Grammar/Tokens index e40a4437afb009..898c6cc7883d0e 100644 --- a/Grammar/Tokens +++ b/Grammar/Tokens @@ -1,3 +1,7 @@ +# When adding new tokens, remember to update as well the peg generator in +# Tools/peg_generator/pegen/parser_generator.py +# This will ensure that older versions of Python can generate a Python parser +# using "python -m pegen python ". ENDMARKER NAME NUMBER From e336531648c27eefce25aff1de252acca7437d7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Preng=C3=A8re?= <2138730+alexprengere@users.noreply.github.com> Date: Wed, 7 May 2025 16:09:06 +0200 Subject: [PATCH 4/5] Update Grammar/Tokens Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Grammar/Tokens | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Grammar/Tokens b/Grammar/Tokens index 898c6cc7883d0e..450e6f7cdf241f 100644 --- a/Grammar/Tokens +++ b/Grammar/Tokens @@ -1,4 +1,4 @@ -# When adding new tokens, remember to update as well the peg generator in +# When adding new tokens, remember to update the PEG generator in # Tools/peg_generator/pegen/parser_generator.py # This will ensure that older versions of Python can generate a Python parser # using "python -m pegen python ". From 68066fd6e7fd968c1a5cd45c4ccdf957cec73aa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Preng=C3=A8re?= <2138730+alexprengere@users.noreply.github.com> Date: Wed, 7 May 2025 16:09:29 +0200 Subject: [PATCH 5/5] Update Grammar/Tokens Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Grammar/Tokens | 1 + 1 file changed, 1 insertion(+) diff --git a/Grammar/Tokens b/Grammar/Tokens index 450e6f7cdf241f..0547e6ed08f79a 100644 --- a/Grammar/Tokens +++ b/Grammar/Tokens @@ -2,6 +2,7 @@ # Tools/peg_generator/pegen/parser_generator.py # This will ensure that older versions of Python can generate a Python parser # using "python -m pegen python ". + ENDMARKER NAME NUMBER