* test_select.py: (some) tests for built-in select module

* test_grammar.py, testall.out: added test for funny things in string literals
* token.py, symbol.py: definitions used with built-in parser module.
* tokenize.py: added double-quote recognition

1 files changed, 50 insertions, 0 deletions

diff --git a/Lib/token.py b/Lib/token.py
new file mode 100755
index 0000000..75ac39a
--- /dev/null
+++ b/Lib/token.py
@@ -0,0 +1,50 @@
+# Tokens (from "token.h")
+
+ENDMARKER = 0
+NAME = 1
+NUMBER = 2
+STRING = 3
+NEWLINE = 4
+INDENT = 5
+DEDENT = 6
+LPAR = 7
+RPAR = 8
+LSQB = 9
+RSQB = 10
+COLON = 11
+COMMA = 12
+SEMI = 13
+PLUS = 14
+MINUS = 15
+STAR = 16
+SLASH = 17
+VBAR = 18
+AMPER = 19
+LESS = 20
+GREATER = 21
+EQUAL = 22
+DOT = 23
+PERCENT = 24
+BACKQUOTE = 25
+LBRACE = 26
+RBRACE = 27
+EQEQUAL = 28
+NOTEQUAL = 29
+LESSEQUAL = 30
+GREATEREQUAL = 31
+TILDE = 32
+CIRCUMFLEX = 33
+LEFTSHIFT = 34
+RIGHTSHIFT = 35
+OP = 36
+ERRORTOKEN = 37
+
+names = dir()
+tok_name = {}
+for name in names:
+	number = eval(name)
+	tok_name[number] = name
+
+N_TOKENS = 38				# Number of tokens including ERRORTOKEN
+
+NT_OFFSET = 256				# Start of non-terminal symbols