summaryrefslogtreecommitdiffstats
path: root/Parser/tokenizer.c
diff options
context:
space:
mode:
authorGuido van Rossum <guido@python.org>1991-10-20 20:25:03 (GMT)
committerGuido van Rossum <guido@python.org>1991-10-20 20:25:03 (GMT)
commitfbab905ae1fe6320268301193d953d25d2acb5c1 (patch)
tree891a98c10ac20d64a894f074ae9477472a382cce /Parser/tokenizer.c
parent8883aaa5ebe70e53bc8f3d074706ce385810c24c (diff)
downloadcpython-fbab905ae1fe6320268301193d953d25d2acb5c1.zip
cpython-fbab905ae1fe6320268301193d953d25d2acb5c1.tar.gz
cpython-fbab905ae1fe6320268301193d953d25d2acb5c1.tar.bz2
Added 2-char tokens and new versions of comparisons
Diffstat (limited to 'Parser/tokenizer.c')
-rw-r--r--Parser/tokenizer.c47
1 files changed, 47 insertions, 0 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index d737e55..ccc6d6d 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -81,6 +81,11 @@ char *tok_name[] = {
"BACKQUOTE",
"LBRACE",
"RBRACE",
+ "EQEQUAL",
+ "NOTEQUAL",
+ "LESSEQUAL",
+ "GREATEREQUAL",
+ /* This table must match the #defines in token.h! */
"OP",
"<ERRORTOKEN>",
"<N_TOKENS>"
@@ -301,6 +306,37 @@ tok_1char(c)
}
+int
+tok_2char(c1, c2)
+ int c1, c2;
+{
+ switch (c1) {
+ case '=':
+ switch (c2) {
+ case '=': return EQEQUAL;
+ }
+ break;
+ case '!':
+ switch (c2) {
+ case '=': return NOTEQUAL;
+ }
+ break;
+ case '<':
+ switch (c2) {
+ case '>': return NOTEQUAL;
+ case '=': return LESSEQUAL;
+ }
+ break;
+ case '>':
+ switch (c2) {
+ case '=': return GREATEREQUAL;
+ }
+ break;
+ }
+ return OP;
+}
+
+
/* Get next token, after space stripping etc. */
int
@@ -531,6 +567,17 @@ tok_get(tok, p_start, p_end)
goto again; /* Read next line */
}
+ /* Check for two-character token */
+ {
+ int c2 = tok_nextc(tok);
+ int token = tok_2char(c, c2);
+ if (token != OP) {
+ *p_end = tok->cur;
+ return token;
+ }
+ tok_backup(tok, c2);
+ }
+
/* Punctuation character */
*p_end = tok->cur;
return tok_1char(c);