summaryrefslogtreecommitdiffstats
path: root/Grammar/Tokens
diff options
context:
space:
mode:
Diffstat (limited to 'Grammar/Tokens')
-rw-r--r--Grammar/Tokens62
1 files changed, 62 insertions, 0 deletions
diff --git a/Grammar/Tokens b/Grammar/Tokens
new file mode 100644
index 0000000..9595673
--- /dev/null
+++ b/Grammar/Tokens
@@ -0,0 +1,62 @@
+ENDMARKER
+NAME
+NUMBER
+STRING
+NEWLINE
+INDENT
+DEDENT
+
+LPAR '('
+RPAR ')'
+LSQB '['
+RSQB ']'
+COLON ':'
+COMMA ','
+SEMI ';'
+PLUS '+'
+MINUS '-'
+STAR '*'
+SLASH '/'
+VBAR '|'
+AMPER '&'
+LESS '<'
+GREATER '>'
+EQUAL '='
+DOT '.'
+PERCENT '%'
+LBRACE '{'
+RBRACE '}'
+EQEQUAL '=='
+NOTEQUAL '!='
+LESSEQUAL '<='
+GREATEREQUAL '>='
+TILDE '~'
+CIRCUMFLEX '^'
+LEFTSHIFT '<<'
+RIGHTSHIFT '>>'
+DOUBLESTAR '**'
+PLUSEQUAL '+='
+MINEQUAL '-='
+STAREQUAL '*='
+SLASHEQUAL '/='
+PERCENTEQUAL '%='
+AMPEREQUAL '&='
+VBAREQUAL '|='
+CIRCUMFLEXEQUAL '^='
+LEFTSHIFTEQUAL '<<='
+RIGHTSHIFTEQUAL '>>='
+DOUBLESTAREQUAL '**='
+DOUBLESLASH '//'
+DOUBLESLASHEQUAL '//='
+AT '@'
+ATEQUAL '@='
+RARROW '->'
+ELLIPSIS '...'
+
+OP
+ERRORTOKEN
+
+# These aren't used by the C tokenizer but are needed for tokenize.py
+COMMENT
+NL
+ENCODING