diff options
| -rw-r--r-- | Misc/NEWS | 2 | ||||
| -rw-r--r-- | Parser/tokenizer.c | 12 |
2 files changed, 13 insertions, 1 deletions
@@ -12,6 +12,8 @@ What's New in Python 3.2 Alpha 1? Core and Builtins ----------------- +- Ensure that tokenization of identifiers is not affected by locale. + - Issue #1222585: Added LDCXXSHARED for C++ support. Patch by Arfrever. - Raise a TypeError when trying to delete a T_STRING_INPLACE struct member. diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 0e2dc3a..be2940c 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -179,6 +179,16 @@ decode_str(const char *str, int exec_input, struct tok_state *tok) #else /* PGEN */ +/* Ensure that the locale does not interfere with tokenization. */ + +static int +ascii_isalnum(int c) +{ + return (('a' <= c && c <= 'z') || + ('A' <= c && c <= 'Z') || + ('0' <= c && c <= '9')); +} + static char * error_ret(struct tok_state *tok) /* XXX */ { @@ -245,7 +255,7 @@ get_coding_spec(const char *s, Py_ssize_t size) } while (t[0] == '\x20' || t[0] == '\t'); begin = t; - while (isalnum(Py_CHARMASK(t[0])) || + while (ascii_isalnum(Py_CHARMASK(t[0])) || t[0] == '-' || t[0] == '_' || t[0] == '.') t++; |