diff options
Diffstat (limited to 'Parser/tokenizer.c')
| -rw-r--r-- | Parser/tokenizer.c | 28 |
1 files changed, 21 insertions, 7 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 8b1b085..7c25450 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -122,8 +122,7 @@ tok_setups(str) struct tok_state *tok = tok_new(); if (tok == NULL) return NULL; - tok->buf = tok->cur = str; - tok->end = tok->inp = strchr(str, '\0'); + tok->buf = tok->cur = tok->end = tok->inp = str; return tok; } @@ -170,13 +169,27 @@ tok_nextc(tok) register struct tok_state *tok; { for (;;) { - if (tok->cur != tok->inp) + if (tok->cur != tok->inp) { return *tok->cur++; /* Fast path */ + } if (tok->done != E_OK) return EOF; if (tok->fp == NULL) { - tok->done = E_EOF; - return EOF; + char *end = strchr(tok->inp, '\n'); + if (end != NULL) + end++; + else { + end = strchr(tok->inp, '\0'); + if (end == tok->inp) { + tok->done = E_EOF; + return EOF; + } + } + if (tok->start == NULL) + tok->buf = tok->cur; + tok->lineno++; + tok->inp = end; + return *tok->cur++; } if (tok->prompt != NULL) { char *new = my_readline(tok->prompt); @@ -478,9 +491,10 @@ tok_get(tok, p_start, p_end) This is also recognized by vi, when it occurs near the beginning or end of the file. (Will vi never die...?) For Python it must be at the beginning of the file! */ + /* XXX The real vi syntax is actually different :-( */ + /* XXX Should recognize Emacs syntax, too */ int x; - /* XXX The cast to (unsigned char *) is needed by THINK C 3.0 */ - if (sscanf(/*(unsigned char *)*/tok->cur, + if (sscanf(tok->cur, " vi:set tabsize=%d:", &x) == 1 && x >= 1 && x <= 40) { /* fprintf(stderr, "# vi:set tabsize=%d:\n", x); */ |