diff options
| author | Martin v. Löwis <martin@v.loewis.de> | 2011-09-28 05:41:54 (GMT) |
|---|---|---|
| committer | Martin v. Löwis <martin@v.loewis.de> | 2011-09-28 05:41:54 (GMT) |
| commit | d63a3b8beb4a0841cb59fb3515347ccaab34b733 (patch) | |
| tree | 3b4e3cc63151c5a5a910c3550a190aefaea96ad4 /Parser | |
| parent | 48d49497c50e79d14e9df9527d766ca3a0a38be5 (diff) | |
| download | cpython-d63a3b8beb4a0841cb59fb3515347ccaab34b733.zip cpython-d63a3b8beb4a0841cb59fb3515347ccaab34b733.tar.gz cpython-d63a3b8beb4a0841cb59fb3515347ccaab34b733.tar.bz2 | |
Implement PEP 393.
Diffstat (limited to 'Parser')
| -rw-r--r-- | Parser/tokenizer.c | 6 |
1 files changed, 4 insertions, 2 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index f4d7e3f..a0a3a67 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1258,14 +1258,16 @@ indenterror(struct tok_state *tok) #ifdef PGEN #define verify_identifier(tok) 1 #else -/* Verify that the identifier follows PEP 3131. */ +/* Verify that the identifier follows PEP 3131. + All identifier strings are guaranteed to be "ready" unicode objects. + */ static int verify_identifier(struct tok_state *tok) { PyObject *s; int result; s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL); - if (s == NULL) { + if (s == NULL || PyUnicode_READY(s) == -1) { if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) { PyErr_Clear(); tok->done = E_IDENTIFIER; |