diff options
| author | Benjamin Peterson <benjamin@python.org> | 2015-04-21 16:07:06 (GMT) |
|---|---|---|
| committer | Benjamin Peterson <benjamin@python.org> | 2015-04-21 16:07:06 (GMT) |
| commit | 273a720f876e754013e17fab0ab9b599284239f7 (patch) | |
| tree | 3467634eaf7246dd125f8aa8ebad732e001a6915 /Parser/tokenizer.c | |
| parent | 8714cfdc4aeedce96fb40c73f7226535fbe7833f (diff) | |
| parent | d73aca769f1f6eebb46faa9161cbebe806db3659 (diff) | |
| download | cpython-273a720f876e754013e17fab0ab9b599284239f7.zip cpython-273a720f876e754013e17fab0ab9b599284239f7.tar.gz cpython-273a720f876e754013e17fab0ab9b599284239f7.tar.bz2 | |
merge 3.4 (#24022)
Diffstat (limited to 'Parser/tokenizer.c')
| -rw-r--r-- | Parser/tokenizer.c | 7 |
1 files changed, 3 insertions, 4 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index ef7b19f..ac413a8 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1307,6 +1307,8 @@ verify_identifier(struct tok_state *tok) { PyObject *s; int result; + if (tok->decoding_erred) + return 0; s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL); if (s == NULL || PyUnicode_READY(s) == -1) { if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) { @@ -1475,11 +1477,8 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) c = tok_nextc(tok); } tok_backup(tok, c); - if (nonascii && - !verify_identifier(tok)) { - tok->done = E_IDENTIFIER; + if (nonascii && !verify_identifier(tok)) return ERRORTOKEN; - } *p_start = tok->start; *p_end = tok->cur; return NAME; |