bpo-30455: Generate all token related code and docs from Grammar/Tokens. (GH-10370)

"Include/token.h", "Lib/token.py" (containing now some data moved from
"Lib/tokenize.py") and new files "Parser/token.c" (containing the code
moved from "Parser/tokenizer.c") and "Doc/library/token-list.inc" (included
in "Doc/library/token.rst") are now generated from "Grammar/Tokens" by
"Tools/scripts/generate_token.py". The script overwrites files only if
needed and can be used on the read-only sources tree.

"Lib/symbol.py" is now generated by "Tools/scripts/generate_symbol_py.py"
instead of been executable itself.

Added new make targets "regen-token" and "regen-symbol" which are now
dependencies of "regen-all".

The documentation contains now strings for operators and punctuation tokens.

1 files changed, 233 insertions, 0 deletions

diff --git a/Parser/token.c b/Parser/token.c
new file mode 100644
index 0000000..35519aa
--- /dev/null
+++ b/Parser/token.c
@@ -0,0 +1,233 @@
+/* Auto-generated by Tools/scripts/generate_token.py */
+
+#include "Python.h"
+#include "token.h"
+
+/* Token names */
+
+const char * const _PyParser_TokenNames[] = {
+    "ENDMARKER",
+    "NAME",
+    "NUMBER",
+    "STRING",
+    "NEWLINE",
+    "INDENT",
+    "DEDENT",
+    "LPAR",
+    "RPAR",
+    "LSQB",
+    "RSQB",
+    "COLON",
+    "COMMA",
+    "SEMI",
+    "PLUS",
+    "MINUS",
+    "STAR",
+    "SLASH",
+    "VBAR",
+    "AMPER",
+    "LESS",
+    "GREATER",
+    "EQUAL",
+    "DOT",
+    "PERCENT",
+    "LBRACE",
+    "RBRACE",
+    "EQEQUAL",
+    "NOTEQUAL",
+    "LESSEQUAL",
+    "GREATEREQUAL",
+    "TILDE",
+    "CIRCUMFLEX",
+    "LEFTSHIFT",
+    "RIGHTSHIFT",
+    "DOUBLESTAR",
+    "PLUSEQUAL",
+    "MINEQUAL",
+    "STAREQUAL",
+    "SLASHEQUAL",
+    "PERCENTEQUAL",
+    "AMPEREQUAL",
+    "VBAREQUAL",
+    "CIRCUMFLEXEQUAL",
+    "LEFTSHIFTEQUAL",
+    "RIGHTSHIFTEQUAL",
+    "DOUBLESTAREQUAL",
+    "DOUBLESLASH",
+    "DOUBLESLASHEQUAL",
+    "AT",
+    "ATEQUAL",
+    "RARROW",
+    "ELLIPSIS",
+    "OP",
+    "<ERRORTOKEN>",
+    "<COMMENT>",
+    "<NL>",
+    "<ENCODING>",
+    "<N_TOKENS>",
+};
+
+/* Return the token corresponding to a single character */
+
+int
+PyToken_OneChar(int c1)
+{
+    switch (c1) {
+    case '%': return PERCENT;
+    case '&': return AMPER;
+    case '(': return LPAR;
+    case ')': return RPAR;
+    case '*': return STAR;
+    case '+': return PLUS;
+    case ',': return COMMA;
+    case '-': return MINUS;
+    case '.': return DOT;
+    case '/': return SLASH;
+    case ':': return COLON;
+    case ';': return SEMI;
+    case '<': return LESS;
+    case '=': return EQUAL;
+    case '>': return GREATER;
+    case '@': return AT;
+    case '[': return LSQB;
+    case ']': return RSQB;
+    case '^': return CIRCUMFLEX;
+    case '{': return LBRACE;
+    case '|': return VBAR;
+    case '}': return RBRACE;
+    case '~': return TILDE;
+    }
+    return OP;
+}
+
+int
+PyToken_TwoChars(int c1, int c2)
+{
+    switch (c1) {
+    case '!':
+        switch (c2) {
+        case '=': return NOTEQUAL;
+        }
+        break;
+    case '%':
+        switch (c2) {
+        case '=': return PERCENTEQUAL;
+        }
+        break;
+    case '&':
+        switch (c2) {
+        case '=': return AMPEREQUAL;
+        }
+        break;
+    case '*':
+        switch (c2) {
+        case '*': return DOUBLESTAR;
+        case '=': return STAREQUAL;
+        }
+        break;
+    case '+':
+        switch (c2) {
+        case '=': return PLUSEQUAL;
+        }
+        break;
+    case '-':
+        switch (c2) {
+        case '=': return MINEQUAL;
+        case '>': return RARROW;
+        }
+        break;
+    case '/':
+        switch (c2) {
+        case '/': return DOUBLESLASH;
+        case '=': return SLASHEQUAL;
+        }
+        break;
+    case '<':
+        switch (c2) {
+        case '<': return LEFTSHIFT;
+        case '=': return LESSEQUAL;
+        case '>': return NOTEQUAL;
+        }
+        break;
+    case '=':
+        switch (c2) {
+        case '=': return EQEQUAL;
+        }
+        break;
+    case '>':
+        switch (c2) {
+        case '=': return GREATEREQUAL;
+        case '>': return RIGHTSHIFT;
+        }
+        break;
+    case '@':
+        switch (c2) {
+        case '=': return ATEQUAL;
+        }
+        break;
+    case '^':
+        switch (c2) {
+        case '=': return CIRCUMFLEXEQUAL;
+        }
+        break;
+    case '|':
+        switch (c2) {
+        case '=': return VBAREQUAL;
+        }
+        break;
+    }
+    return OP;
+}
+
+int
+PyToken_ThreeChars(int c1, int c2, int c3)
+{
+    switch (c1) {
+    case '*':
+        switch (c2) {
+        case '*':
+            switch (c3) {
+            case '=': return DOUBLESTAREQUAL;
+            }
+            break;
+        }
+        break;
+    case '.':
+        switch (c2) {
+        case '.':
+            switch (c3) {
+            case '.': return ELLIPSIS;
+            }
+            break;
+        }
+        break;
+    case '/':
+        switch (c2) {
+        case '/':
+            switch (c3) {
+            case '=': return DOUBLESLASHEQUAL;
+            }
+            break;
+        }
+        break;
+    case '<':
+        switch (c2) {
+        case '<':
+            switch (c3) {
+            case '=': return LEFTSHIFTEQUAL;
+            }
+            break;
+        }
+        break;
+    case '>':
+        switch (c2) {
+        case '>':
+            switch (c3) {
+            case '=': return RIGHTSHIFTEQUAL;
+            }
+            break;
+        }
+        break;
+    }
+    return OP;
+}