Initial revision

1 files changed, 45 insertions, 0 deletions

diff --git a/Include/token.h b/Include/token.h
new file mode 100644
index 0000000..d68d142
--- /dev/null
+++ b/Include/token.h
@@ -0,0 +1,45 @@
+/* Token types */
+
+#define ENDMARKER	0
+#define NAME		1
+#define NUMBER		2
+#define STRING		3
+#define NEWLINE		4
+#define INDENT		5
+#define DEDENT		6
+#define LPAR		7
+#define RPAR		8
+#define LSQB		9
+#define RSQB		10
+#define COLON		11
+#define COMMA		12
+#define SEMI		13
+#define PLUS		14
+#define MINUS		15
+#define STAR		16
+#define SLASH		17
+#define VBAR		18
+#define AMPER		19
+#define LESS		20
+#define GREATER		21
+#define EQUAL		22
+#define DOT		23
+#define PERCENT		24
+#define BACKQUOTE	25
+#define LBRACE		26
+#define RBRACE		27
+#define OP		28
+#define ERRORTOKEN	29
+#define N_TOKENS	30
+
+/* Special definitions for cooperation with parser */
+
+#define NT_OFFSET		256
+
+#define ISTERMINAL(x)		((x) < NT_OFFSET)
+#define ISNONTERMINAL(x)	((x) >= NT_OFFSET)
+#define ISEOF(x)		((x) == ENDMARKER)
+
+
+extern char *tok_name[]; /* Token names */
+extern int tok_1char PROTO((int));