Skip to content

Commit da5727a

Browse files
vstinnervstinner
authored
gh-92651: Remove the Include/token.h header file (#92652)
Remove the token.h header file. There was never any public tokenizer C API. The token.h header file was only designed to be used by Python internals. Move Include/token.h to Include/internal/pycore_token.h. Including this header file now requires that the Py_BUILD_CORE macro is defined. It no longer checks for the Py_LIMITED_API macro. Rename functions: * PyToken_OneChar() => _PyToken_OneChar() * PyToken_TwoChars() => _PyToken_TwoChars() * PyToken_ThreeChars() => _PyToken_ThreeChars()
1 parent b69297e commit da5727a

File tree

13 files changed

+52
-39
lines changed

13 files changed

+52
-39
lines changed

‎Doc/whatsnew/3.12.rst

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,3 +139,8 @@ Deprecated
139139

140140
Removed
141141
-------
142+
143+
* Remove the ``token.h`` header file. There was never any public tokenizer C
144+
API. The ``token.h`` header file was only designed to be used by Python
145+
internals.
146+
(Contributed by Victor Stinner in :gh:`92651`.)

‎Include/token.h renamed to ‎Include/internal/pycore_token.h

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,16 @@
11
/* Auto-generated by Tools/scripts/generate_token.py */
22

33
/* Token types */
4-
#ifndef Py_LIMITED_API
5-
#ifndef Py_TOKEN_H
6-
#define Py_TOKEN_H
4+
#ifndef Py_INTERNAL_TOKEN_H
5+
#define Py_INTERNAL_TOKEN_H
76
#ifdef __cplusplus
87
extern "C" {
98
#endif
109

10+
#ifndef Py_BUILD_CORE
11+
# error "this header requires Py_BUILD_CORE define"
12+
#endif
13+
1114
#undef TILDE /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */
1215

1316
#define ENDMARKER 0
@@ -85,13 +88,13 @@ extern "C" {
8588
(x) == DEDENT)
8689

8790

91+
// Symbols exported for test_peg_generator
8892
PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */
89-
PyAPI_FUNC(int) PyToken_OneChar(int);
90-
PyAPI_FUNC(int) PyToken_TwoChars(int, int);
91-
PyAPI_FUNC(int) PyToken_ThreeChars(int, int, int);
93+
PyAPI_FUNC(int) _PyToken_OneChar(int);
94+
PyAPI_FUNC(int) _PyToken_TwoChars(int, int);
95+
PyAPI_FUNC(int) _PyToken_ThreeChars(int, int, int);
9296

9397
#ifdef __cplusplus
9498
}
9599
#endif
96-
#endif /* !Py_TOKEN_H */
97-
#endif /* Py_LIMITED_API */
100+
#endif // !Py_INTERNAL_TOKEN_H

‎Makefile.pre.in

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1325,11 +1325,11 @@ regen-token:
13251325
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py rst \
13261326
$(srcdir)/Grammar/Tokens \
13271327
$(srcdir)/Doc/library/token-list.inc
1328-
# Regenerate Include/token.h from Grammar/Tokens
1328+
# Regenerate Include/internal/pycore_token.h from Grammar/Tokens
13291329
# using Tools/scripts/generate_token.py
13301330
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py h \
13311331
$(srcdir)/Grammar/Tokens \
1332-
$(srcdir)/Include/token.h
1332+
$(srcdir)/Include/internal/pycore_token.h
13331333
# Regenerate Parser/token.c from Grammar/Tokens
13341334
# using Tools/scripts/generate_token.py
13351335
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py c \
@@ -1521,7 +1521,6 @@ PYTHON_HEADERS= \
15211521
$(srcdir)/Include/structmember.h \
15221522
$(srcdir)/Include/structseq.h \
15231523
$(srcdir)/Include/sysmodule.h \
1524-
$(srcdir)/Include/token.h \
15251524
$(srcdir)/Include/traceback.h \
15261525
$(srcdir)/Include/tracemalloc.h \
15271526
$(srcdir)/Include/tupleobject.h \
@@ -1632,6 +1631,7 @@ PYTHON_HEADERS= \
16321631
$(srcdir)/Include/internal/pycore_structseq.h \
16331632
$(srcdir)/Include/internal/pycore_symtable.h \
16341633
$(srcdir)/Include/internal/pycore_sysmodule.h \
1634+
$(srcdir)/Include/internal/pycore_token.h \
16351635
$(srcdir)/Include/internal/pycore_traceback.h \
16361636
$(srcdir)/Include/internal/pycore_tuple.h \
16371637
$(srcdir)/Include/internal/pycore_typeobject.h \

‎Misc/NEWS.d/next/C API/2022-05-11-02-33-10.gh-issue-92651.FIXLf0.rst

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Remove the ``token.h`` header file. There was never any public tokenizer C
2+
API. The ``token.h`` header file was only designed to be used by Python
3+
internals. Patch by Victor Stinner.

‎PCbuild/pythoncore.vcxproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,7 @@
244244
<ClInclude Include="..\Include\internal\pycore_structseq.h" />
245245
<ClInclude Include="..\Include\internal\pycore_sysmodule.h" />
246246
<ClInclude Include="..\Include\internal\pycore_symtable.h" />
247+
<ClInclude Include="..\Include\internal\pycore_token.h" />
247248
<ClInclude Include="..\Include\internal\pycore_traceback.h" />
248249
<ClInclude Include="..\Include\internal\pycore_tuple.h" />
249250
<ClInclude Include="..\Include\internal\pycore_typeobject.h" />
@@ -291,7 +292,6 @@
291292
<ClInclude Include="..\Include\structseq.h" />
292293
<ClInclude Include="..\Include\symtable.h" />
293294
<ClInclude Include="..\Include\sysmodule.h" />
294-
<ClInclude Include="..\Include\token.h" />
295295
<ClInclude Include="..\Include\traceback.h" />
296296
<ClInclude Include="..\Include\tracemalloc.h" />
297297
<ClInclude Include="..\Include\tupleobject.h" />

‎PCbuild/pythoncore.vcxproj.filters

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -213,9 +213,6 @@
213213
<ClInclude Include="..\Include\sysmodule.h">
214214
<Filter>Include</Filter>
215215
</ClInclude>
216-
<ClInclude Include="..\Include\token.h">
217-
<Filter>Include</Filter>
218-
</ClInclude>
219216
<ClInclude Include="..\Include\traceback.h">
220217
<Filter>Include</Filter>
221218
</ClInclude>
@@ -633,6 +630,9 @@
633630
<ClInclude Include="..\Include\internal\pycore_symtable.h">
634631
<Filter>Include\internal</Filter>
635632
</ClInclude>
633+
<ClInclude Include="..\Include\internal\pycore_token.h">
634+
<Filter>Include\internal</Filter>
635+
</ClInclude>
636636
<ClInclude Include="..\Include\internal\pycore_traceback.h">
637637
<Filter>Include\internal</Filter>
638638
</ClInclude>

‎PCbuild/regen.targets

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
<_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc">
2020
<Format>rst</Format>
2121
</_TokenOutputs>
22-
<_TokenOutputs Include="$(PySourcePath)Include\token.h">
22+
<_TokenOutputs Include="$(PySourcePath)Include\internal\pycore_token.h">
2323
<Format>h</Format>
2424
</_TokenOutputs>
2525
<_TokenOutputs Include="$(PySourcePath)Parser\token.c">

‎Parser/pegen.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33

44
#define PY_SSIZE_T_CLEAN
55
#include <Python.h>
6-
#include <token.h>
76
#include <pycore_ast.h>
7+
#include <pycore_token.h>
88

99
#if 0
1010
#define PyPARSE_YIELD_IS_KEYWORD 0x0001

‎Parser/token.c

Lines changed: 4 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Parser/tokenizer.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1992,10 +1992,10 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
19921992
/* Check for two-character token */
19931993
{
19941994
int c2 = tok_nextc(tok);
1995-
int token = PyToken_TwoChars(c, c2);
1995+
int token = _PyToken_TwoChars(c, c2);
19961996
if (token != OP) {
19971997
int c3 = tok_nextc(tok);
1998-
int token3 = PyToken_ThreeChars(c, c2, c3);
1998+
int token3 = _PyToken_ThreeChars(c, c2, c3);
19991999
if (token3 != OP) {
20002000
token = token3;
20012001
}
@@ -2059,7 +2059,7 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
20592059
/* Punctuation character */
20602060
*p_start = tok->start;
20612061
*p_end = tok->cur;
2062-
return PyToken_OneChar(c);
2062+
return _PyToken_OneChar(c);
20632063
}
20642064

20652065
int

‎Parser/tokenizer.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ extern "C" {
88

99
/* Tokenizer interface */
1010

11-
#include "token.h" /* For token types */
11+
#include "pycore_token.h" /* For token types */
1212

1313
#define MAXINDENT 100 /* Max indentation level */
1414
#define MAXLEVEL 200 /* Max parentheses level */

‎Python/pythonrun.c

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
#include "pycore_sysmodule.h" // _PySys_Audit()
2525
#include "pycore_traceback.h" // _PyTraceBack_Print_Indented()
2626

27-
#include "token.h" // INDENT
2827
#include "errcode.h" // E_EOF
2928
#include "marshal.h" // PyMarshal_ReadLongFromFile()
3029

‎Tools/scripts/generate_token.py

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -51,13 +51,16 @@ def update_file(file, content):
5151
/* Auto-generated by Tools/scripts/generate_token.py */
5252
5353
/* Token types */
54-
#ifndef Py_LIMITED_API
55-
#ifndef Py_TOKEN_H
56-
#define Py_TOKEN_H
54+
#ifndef Py_INTERNAL_TOKEN_H
55+
#define Py_INTERNAL_TOKEN_H
5756
#ifdef __cplusplus
5857
extern "C" {
5958
#endif
6059
60+
#ifndef Py_BUILD_CORE
61+
# error "this header requires Py_BUILD_CORE define"
62+
#endif
63+
6164
#undef TILDE /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */
6265
6366
%s\
@@ -75,19 +78,19 @@ def update_file(file, content):
7578
(x) == DEDENT)
7679
7780
81+
// Symbols exported for test_peg_generator
7882
PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */
79-
PyAPI_FUNC(int) PyToken_OneChar(int);
80-
PyAPI_FUNC(int) PyToken_TwoChars(int, int);
81-
PyAPI_FUNC(int) PyToken_ThreeChars(int, int, int);
83+
PyAPI_FUNC(int) _PyToken_OneChar(int);
84+
PyAPI_FUNC(int) _PyToken_TwoChars(int, int);
85+
PyAPI_FUNC(int) _PyToken_ThreeChars(int, int, int);
8286
8387
#ifdef __cplusplus
8488
}
8589
#endif
86-
#endif /* !Py_TOKEN_H */
87-
#endif /* Py_LIMITED_API */
90+
#endif // !Py_INTERNAL_TOKEN_H
8891
"""
8992

90-
def make_h(infile, outfile='Include/token.h'):
93+
def make_h(infile, outfile='Include/internal/pycore_token.h'):
9194
tok_names, ERRORTOKEN, string_to_tok = load_tokens(infile)
9295

9396
defines = []
@@ -106,7 +109,7 @@ def make_h(infile, outfile='Include/token.h'):
106109
/* Auto-generated by Tools/scripts/generate_token.py */
107110
108111
#include "Python.h"
109-
#include "token.h"
112+
#include "pycore_token.h"
110113
111114
/* Token names */
112115
@@ -117,21 +120,21 @@ def make_h(infile, outfile='Include/token.h'):
117120
/* Return the token corresponding to a single character */
118121
119122
int
120-
PyToken_OneChar(int c1)
123+
_PyToken_OneChar(int c1)
121124
{
122125
%s\
123126
return OP;
124127
}
125128
126129
int
127-
PyToken_TwoChars(int c1, int c2)
130+
_PyToken_TwoChars(int c1, int c2)
128131
{
129132
%s\
130133
return OP;
131134
}
132135
133136
int
134-
PyToken_ThreeChars(int c1, int c2, int c3)
137+
_PyToken_ThreeChars(int c1, int c2, int c3)
135138
{
136139
%s\
137140
return OP;

0 commit comments

Comments
 (0)