diff options
| author | Benjamin Peterson <benjamin@python.org> | 2010-04-03 22:55:48 (GMT) |
|---|---|---|
| committer | Benjamin Peterson <benjamin@python.org> | 2010-04-03 22:55:48 (GMT) |
| commit | 8daa49ee34667899af5687b322603fc71d848d3b (patch) | |
| tree | 8d670f64866c338f67a8df77cf754c14ead38ee6 /Parser/tokenizer.c | |
| parent | ace7108f86f9fcdd8ff9867a4375d0cd939b56aa (diff) | |
| download | cpython-8daa49ee34667899af5687b322603fc71d848d3b.zip cpython-8daa49ee34667899af5687b322603fc71d848d3b.tar.gz cpython-8daa49ee34667899af5687b322603fc71d848d3b.tar.bz2 | |
Merged revisions 79723 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/trunk
........
r79723 | benjamin.peterson | 2010-04-03 17:48:51 -0500 (Sat, 03 Apr 2010) | 1 line
ensure that the locale does not affect the tokenization of identifiers
........
Diffstat (limited to 'Parser/tokenizer.c')
| -rw-r--r-- | Parser/tokenizer.c | 12 |
1 files changed, 11 insertions, 1 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 0e2dc3a..be2940c 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -179,6 +179,16 @@ decode_str(const char *str, int exec_input, struct tok_state *tok) #else /* PGEN */ +/* Ensure that the locale does not interfere with tokenization. */ + +static int +ascii_isalnum(int c) +{ + return (('a' <= c && c <= 'z') || + ('A' <= c && c <= 'Z') || + ('0' <= c && c <= '9')); +} + static char * error_ret(struct tok_state *tok) /* XXX */ { @@ -245,7 +255,7 @@ get_coding_spec(const char *s, Py_ssize_t size) } while (t[0] == '\x20' || t[0] == '\t'); begin = t; - while (isalnum(Py_CHARMASK(t[0])) || + while (ascii_isalnum(Py_CHARMASK(t[0])) || t[0] == '-' || t[0] == '_' || t[0] == '.') t++; |