From da5727a120e426ffaf68bf3a8016491205bd2f80 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 11 May 2022 23:22:50 +0200 Subject: gh-92651: Remove the Include/token.h header file (#92652) Remove the token.h header file. There was never any public tokenizer C API. The token.h header file was only designed to be used by Python internals. Move Include/token.h to Include/internal/pycore_token.h. Including this header file now requires that the Py_BUILD_CORE macro is defined. It no longer checks for the Py_LIMITED_API macro. Rename functions: * PyToken_OneChar() => _PyToken_OneChar() * PyToken_TwoChars() => _PyToken_TwoChars() * PyToken_ThreeChars() => _PyToken_ThreeChars() --- Doc/whatsnew/3.12.rst | 5 ++ Include/internal/pycore_token.h | 100 +++++++++++++++++++++ Include/token.h | 97 -------------------- Makefile.pre.in | 6 +- .../2022-05-11-02-33-10.gh-issue-92651.FIXLf0.rst | 3 + PCbuild/pythoncore.vcxproj | 2 +- PCbuild/pythoncore.vcxproj.filters | 6 +- PCbuild/regen.targets | 2 +- Parser/pegen.h | 2 +- Parser/token.c | 8 +- Parser/tokenizer.c | 6 +- Parser/tokenizer.h | 2 +- Python/pythonrun.c | 1 - Tools/scripts/generate_token.py | 29 +++--- 14 files changed, 141 insertions(+), 128 deletions(-) create mode 100644 Include/internal/pycore_token.h delete mode 100644 Include/token.h create mode 100644 Misc/NEWS.d/next/C API/2022-05-11-02-33-10.gh-issue-92651.FIXLf0.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 461d9db..bc354c3 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -139,3 +139,8 @@ Deprecated Removed ------- + +* Remove the ``token.h`` header file. There was never any public tokenizer C + API. The ``token.h`` header file was only designed to be used by Python + internals. + (Contributed by Victor Stinner in :gh:`92651`.) diff --git a/Include/internal/pycore_token.h b/Include/internal/pycore_token.h new file mode 100644 index 0000000..f9b8240 --- /dev/null +++ b/Include/internal/pycore_token.h @@ -0,0 +1,100 @@ +/* Auto-generated by Tools/scripts/generate_token.py */ + +/* Token types */ +#ifndef Py_INTERNAL_TOKEN_H +#define Py_INTERNAL_TOKEN_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#undef TILDE /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */ + +#define ENDMARKER 0 +#define NAME 1 +#define NUMBER 2 +#define STRING 3 +#define NEWLINE 4 +#define INDENT 5 +#define DEDENT 6 +#define LPAR 7 +#define RPAR 8 +#define LSQB 9 +#define RSQB 10 +#define COLON 11 +#define COMMA 12 +#define SEMI 13 +#define PLUS 14 +#define MINUS 15 +#define STAR 16 +#define SLASH 17 +#define VBAR 18 +#define AMPER 19 +#define LESS 20 +#define GREATER 21 +#define EQUAL 22 +#define DOT 23 +#define PERCENT 24 +#define LBRACE 25 +#define RBRACE 26 +#define EQEQUAL 27 +#define NOTEQUAL 28 +#define LESSEQUAL 29 +#define GREATEREQUAL 30 +#define TILDE 31 +#define CIRCUMFLEX 32 +#define LEFTSHIFT 33 +#define RIGHTSHIFT 34 +#define DOUBLESTAR 35 +#define PLUSEQUAL 36 +#define MINEQUAL 37 +#define STAREQUAL 38 +#define SLASHEQUAL 39 +#define PERCENTEQUAL 40 +#define AMPEREQUAL 41 +#define VBAREQUAL 42 +#define CIRCUMFLEXEQUAL 43 +#define LEFTSHIFTEQUAL 44 +#define RIGHTSHIFTEQUAL 45 +#define DOUBLESTAREQUAL 46 +#define DOUBLESLASH 47 +#define DOUBLESLASHEQUAL 48 +#define AT 49 +#define ATEQUAL 50 +#define RARROW 51 +#define ELLIPSIS 52 +#define COLONEQUAL 53 +#define OP 54 +#define AWAIT 55 +#define ASYNC 56 +#define TYPE_IGNORE 57 +#define TYPE_COMMENT 58 +#define SOFT_KEYWORD 59 +#define ERRORTOKEN 60 +#define N_TOKENS 64 +#define NT_OFFSET 256 + +/* Special definitions for cooperation with parser */ + +#define ISTERMINAL(x) ((x) < NT_OFFSET) +#define ISNONTERMINAL(x) ((x) >= NT_OFFSET) +#define ISEOF(x) ((x) == ENDMARKER) +#define ISWHITESPACE(x) ((x) == ENDMARKER || \ + (x) == NEWLINE || \ + (x) == INDENT || \ + (x) == DEDENT) + + +// Symbols exported for test_peg_generator +PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */ +PyAPI_FUNC(int) _PyToken_OneChar(int); +PyAPI_FUNC(int) _PyToken_TwoChars(int, int); +PyAPI_FUNC(int) _PyToken_ThreeChars(int, int, int); + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_TOKEN_H diff --git a/Include/token.h b/Include/token.h deleted file mode 100644 index eb1b9ea..0000000 --- a/Include/token.h +++ /dev/null @@ -1,97 +0,0 @@ -/* Auto-generated by Tools/scripts/generate_token.py */ - -/* Token types */ -#ifndef Py_LIMITED_API -#ifndef Py_TOKEN_H -#define Py_TOKEN_H -#ifdef __cplusplus -extern "C" { -#endif - -#undef TILDE /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */ - -#define ENDMARKER 0 -#define NAME 1 -#define NUMBER 2 -#define STRING 3 -#define NEWLINE 4 -#define INDENT 5 -#define DEDENT 6 -#define LPAR 7 -#define RPAR 8 -#define LSQB 9 -#define RSQB 10 -#define COLON 11 -#define COMMA 12 -#define SEMI 13 -#define PLUS 14 -#define MINUS 15 -#define STAR 16 -#define SLASH 17 -#define VBAR 18 -#define AMPER 19 -#define LESS 20 -#define GREATER 21 -#define EQUAL 22 -#define DOT 23 -#define PERCENT 24 -#define LBRACE 25 -#define RBRACE 26 -#define EQEQUAL 27 -#define NOTEQUAL 28 -#define LESSEQUAL 29 -#define GREATEREQUAL 30 -#define TILDE 31 -#define CIRCUMFLEX 32 -#define LEFTSHIFT 33 -#define RIGHTSHIFT 34 -#define DOUBLESTAR 35 -#define PLUSEQUAL 36 -#define MINEQUAL 37 -#define STAREQUAL 38 -#define SLASHEQUAL 39 -#define PERCENTEQUAL 40 -#define AMPEREQUAL 41 -#define VBAREQUAL 42 -#define CIRCUMFLEXEQUAL 43 -#define LEFTSHIFTEQUAL 44 -#define RIGHTSHIFTEQUAL 45 -#define DOUBLESTAREQUAL 46 -#define DOUBLESLASH 47 -#define DOUBLESLASHEQUAL 48 -#define AT 49 -#define ATEQUAL 50 -#define RARROW 51 -#define ELLIPSIS 52 -#define COLONEQUAL 53 -#define OP 54 -#define AWAIT 55 -#define ASYNC 56 -#define TYPE_IGNORE 57 -#define TYPE_COMMENT 58 -#define SOFT_KEYWORD 59 -#define ERRORTOKEN 60 -#define N_TOKENS 64 -#define NT_OFFSET 256 - -/* Special definitions for cooperation with parser */ - -#define ISTERMINAL(x) ((x) < NT_OFFSET) -#define ISNONTERMINAL(x) ((x) >= NT_OFFSET) -#define ISEOF(x) ((x) == ENDMARKER) -#define ISWHITESPACE(x) ((x) == ENDMARKER || \ - (x) == NEWLINE || \ - (x) == INDENT || \ - (x) == DEDENT) - - -PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */ -PyAPI_FUNC(int) PyToken_OneChar(int); -PyAPI_FUNC(int) PyToken_TwoChars(int, int); -PyAPI_FUNC(int) PyToken_ThreeChars(int, int, int); - -#ifdef __cplusplus -} -#endif -#endif /* !Py_TOKEN_H */ -#endif /* Py_LIMITED_API */ diff --git a/Makefile.pre.in b/Makefile.pre.in index e45d4fe..869c78e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1325,11 +1325,11 @@ regen-token: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py rst \ $(srcdir)/Grammar/Tokens \ $(srcdir)/Doc/library/token-list.inc - # Regenerate Include/token.h from Grammar/Tokens + # Regenerate Include/internal/pycore_token.h from Grammar/Tokens # using Tools/scripts/generate_token.py $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py h \ $(srcdir)/Grammar/Tokens \ - $(srcdir)/Include/token.h + $(srcdir)/Include/internal/pycore_token.h # Regenerate Parser/token.c from Grammar/Tokens # using Tools/scripts/generate_token.py $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py c \ @@ -1521,7 +1521,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/structmember.h \ $(srcdir)/Include/structseq.h \ $(srcdir)/Include/sysmodule.h \ - $(srcdir)/Include/token.h \ $(srcdir)/Include/traceback.h \ $(srcdir)/Include/tracemalloc.h \ $(srcdir)/Include/tupleobject.h \ @@ -1632,6 +1631,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_structseq.h \ $(srcdir)/Include/internal/pycore_symtable.h \ $(srcdir)/Include/internal/pycore_sysmodule.h \ + $(srcdir)/Include/internal/pycore_token.h \ $(srcdir)/Include/internal/pycore_traceback.h \ $(srcdir)/Include/internal/pycore_tuple.h \ $(srcdir)/Include/internal/pycore_typeobject.h \ diff --git a/Misc/NEWS.d/next/C API/2022-05-11-02-33-10.gh-issue-92651.FIXLf0.rst b/Misc/NEWS.d/next/C API/2022-05-11-02-33-10.gh-issue-92651.FIXLf0.rst new file mode 100644 index 0000000..60a8818 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-05-11-02-33-10.gh-issue-92651.FIXLf0.rst @@ -0,0 +1,3 @@ +Remove the ``token.h`` header file. There was never any public tokenizer C +API. The ``token.h`` header file was only designed to be used by Python +internals. Patch by Victor Stinner. diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 3ce116d..a35884b 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -244,6 +244,7 @@ + @@ -291,7 +292,6 @@ - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 542d551..ff42cc9 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -213,9 +213,6 @@ Include - - Include - Include @@ -633,6 +630,9 @@ Include\internal + + Include\internal + Include\internal diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 24b5ced..9073bb6 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -19,7 +19,7 @@ <_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc"> rst - <_TokenOutputs Include="$(PySourcePath)Include\token.h"> + <_TokenOutputs Include="$(PySourcePath)Include\internal\pycore_token.h"> h <_TokenOutputs Include="$(PySourcePath)Parser\token.c"> diff --git a/Parser/pegen.h b/Parser/pegen.h index fe0c327..d6a6e4e 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -3,8 +3,8 @@ #define PY_SSIZE_T_CLEAN #include -#include #include +#include #if 0 #define PyPARSE_YIELD_IS_KEYWORD 0x0001 diff --git a/Parser/token.c b/Parser/token.c index 74bca0e..fa03fbc 100644 --- a/Parser/token.c +++ b/Parser/token.c @@ -1,7 +1,7 @@ /* Auto-generated by Tools/scripts/generate_token.py */ #include "Python.h" -#include "token.h" +#include "pycore_token.h" /* Token names */ @@ -76,7 +76,7 @@ const char * const _PyParser_TokenNames[] = { /* Return the token corresponding to a single character */ int -PyToken_OneChar(int c1) +_PyToken_OneChar(int c1) { switch (c1) { case '%': return PERCENT; @@ -107,7 +107,7 @@ PyToken_OneChar(int c1) } int -PyToken_TwoChars(int c1, int c2) +_PyToken_TwoChars(int c1, int c2) { switch (c1) { case '!': @@ -191,7 +191,7 @@ PyToken_TwoChars(int c1, int c2) } int -PyToken_ThreeChars(int c1, int c2, int c3) +_PyToken_ThreeChars(int c1, int c2, int c3) { switch (c1) { case '*': diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index c450aa8..7c79718 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1992,10 +1992,10 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) /* Check for two-character token */ { int c2 = tok_nextc(tok); - int token = PyToken_TwoChars(c, c2); + int token = _PyToken_TwoChars(c, c2); if (token != OP) { int c3 = tok_nextc(tok); - int token3 = PyToken_ThreeChars(c, c2, c3); + int token3 = _PyToken_ThreeChars(c, c2, c3); if (token3 != OP) { token = token3; } @@ -2059,7 +2059,7 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) /* Punctuation character */ *p_start = tok->start; *p_end = tok->cur; - return PyToken_OneChar(c); + return _PyToken_OneChar(c); } int diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 0cb6651..dba71bd 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -8,7 +8,7 @@ extern "C" { /* Tokenizer interface */ -#include "token.h" /* For token types */ +#include "pycore_token.h" /* For token types */ #define MAXINDENT 100 /* Max indentation level */ #define MAXLEVEL 200 /* Max parentheses level */ diff --git a/Python/pythonrun.c b/Python/pythonrun.c index f12b9f6..202df58 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -24,7 +24,6 @@ #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_traceback.h" // _PyTraceBack_Print_Indented() -#include "token.h" // INDENT #include "errcode.h" // E_EOF #include "marshal.h" // PyMarshal_ReadLongFromFile() diff --git a/Tools/scripts/generate_token.py b/Tools/scripts/generate_token.py index 77bb5bd..d8be8b9 100755 --- a/Tools/scripts/generate_token.py +++ b/Tools/scripts/generate_token.py @@ -51,13 +51,16 @@ token_h_template = """\ /* Auto-generated by Tools/scripts/generate_token.py */ /* Token types */ -#ifndef Py_LIMITED_API -#ifndef Py_TOKEN_H -#define Py_TOKEN_H +#ifndef Py_INTERNAL_TOKEN_H +#define Py_INTERNAL_TOKEN_H #ifdef __cplusplus extern "C" { #endif +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + #undef TILDE /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */ %s\ @@ -75,19 +78,19 @@ extern "C" { (x) == DEDENT) +// Symbols exported for test_peg_generator PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */ -PyAPI_FUNC(int) PyToken_OneChar(int); -PyAPI_FUNC(int) PyToken_TwoChars(int, int); -PyAPI_FUNC(int) PyToken_ThreeChars(int, int, int); +PyAPI_FUNC(int) _PyToken_OneChar(int); +PyAPI_FUNC(int) _PyToken_TwoChars(int, int); +PyAPI_FUNC(int) _PyToken_ThreeChars(int, int, int); #ifdef __cplusplus } #endif -#endif /* !Py_TOKEN_H */ -#endif /* Py_LIMITED_API */ +#endif // !Py_INTERNAL_TOKEN_H """ -def make_h(infile, outfile='Include/token.h'): +def make_h(infile, outfile='Include/internal/pycore_token.h'): tok_names, ERRORTOKEN, string_to_tok = load_tokens(infile) defines = [] @@ -106,7 +109,7 @@ token_c_template = """\ /* Auto-generated by Tools/scripts/generate_token.py */ #include "Python.h" -#include "token.h" +#include "pycore_token.h" /* Token names */ @@ -117,21 +120,21 @@ const char * const _PyParser_TokenNames[] = { /* Return the token corresponding to a single character */ int -PyToken_OneChar(int c1) +_PyToken_OneChar(int c1) { %s\ return OP; } int -PyToken_TwoChars(int c1, int c2) +_PyToken_TwoChars(int c1, int c2) { %s\ return OP; } int -PyToken_ThreeChars(int c1, int c2, int c3) +_PyToken_ThreeChars(int c1, int c2, int c3) { %s\ return OP; -- cgit v0.12