summaryrefslogtreecommitdiffstats
path: root/Parser
diff options
context:
space:
mode:
authorGuido van Rossum <guido@python.org>1991-10-24 14:59:40 (GMT)
committerGuido van Rossum <guido@python.org>1991-10-24 14:59:40 (GMT)
commitbaf0ebf43c45e85e9a47d25e5279c99ca9455838 (patch)
tree4a3ac155f280815525e76799c466a2fa89b8929a /Parser
parent7928cd7636060c09da613d6f226a54903b86740a (diff)
downloadcpython-baf0ebf43c45e85e9a47d25e5279c99ca9455838.zip
cpython-baf0ebf43c45e85e9a47d25e5279c99ca9455838.tar.gz
cpython-baf0ebf43c45e85e9a47d25e5279c99ca9455838.tar.bz2
Added shift and mask ops.
Allow numbers starting with a period.
Diffstat (limited to 'Parser')
-rw-r--r--Parser/tokenizer.c27
1 files changed, 25 insertions, 2 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index ccc6d6d..009c193 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -85,6 +85,10 @@ char *tok_name[] = {
"NOTEQUAL",
"LESSEQUAL",
"GREATEREQUAL",
+ "TILDE",
+ "CIRCUMFLEX",
+ "LEFTSHIFT",
+ "RIGHTSHIFT",
/* This table must match the #defines in token.h! */
"OP",
"<ERRORTOKEN>",
@@ -301,6 +305,8 @@ tok_1char(c)
case '`': return BACKQUOTE;
case '{': return LBRACE;
case '}': return RBRACE;
+ case '^': return CIRCUMFLEX;
+ case '~': return TILDE;
default: return OP;
}
}
@@ -325,11 +331,13 @@ tok_2char(c1, c2)
switch (c2) {
case '>': return NOTEQUAL;
case '=': return LESSEQUAL;
+ case '<': return LEFTSHIFT;
}
break;
case '>':
switch (c2) {
case '=': return GREATEREQUAL;
+ case '>': return RIGHTSHIFT;
}
break;
}
@@ -438,7 +446,7 @@ tok_get(tok, p_start, p_end)
beginning or end of the file. (Will vi never die...?)
For Python it must be at the beginning of the file! */
int x;
- /* XXX The case to (unsigned char *) is needed by THINK C 3.0 */
+ /* XXX The cast to (unsigned char *) is needed by THINK C 3.0 */
if (sscanf(/*(unsigned char *)*/tok->cur,
" vi:set tabsize=%d:", &x) == 1 &&
x >= 1 && x <= 40) {
@@ -451,8 +459,10 @@ tok_get(tok, p_start, p_end)
}
/* Check for EOF and errors now */
- if (c == EOF)
+ if (c == EOF) {
+ *p_start = *p_end = tok->cur;
return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
+ }
/* Identifier (most frequent token!) */
if (isalpha(c) || c == '_') {
@@ -473,6 +483,19 @@ tok_get(tok, p_start, p_end)
return NEWLINE;
}
+ /* Period or number starting with period? */
+ if (c == '.') {
+ c = tok_nextc(tok);
+ if (isdigit(c)) {
+ goto fraction;
+ }
+ else {
+ tok_backup(tok, c);
+ *p_end = tok->cur;
+ return DOT;
+ }
+ }
+
/* Number */
if (isdigit(c)) {
if (c == '0') {