diff options
| author | Serhiy Storchaka <storchaka@gmail.com> | 2015-11-14 13:12:04 (GMT) |
|---|---|---|
| committer | Serhiy Storchaka <storchaka@gmail.com> | 2015-11-14 13:12:04 (GMT) |
| commit | 0304729ec49f4673bc8c88740537e66643a6c44a (patch) | |
| tree | a7b875e7bfcdcbebc3280b3670c6add881b3b826 /Parser | |
| parent | 28a465c9e0d4f49eb2f344d8cbb986f61a47efbe (diff) | |
| parent | 7e2b870b8593233306594237cd5286c60ec7a1e1 (diff) | |
| download | cpython-0304729ec49f4673bc8c88740537e66643a6c44a.zip cpython-0304729ec49f4673bc8c88740537e66643a6c44a.tar.gz cpython-0304729ec49f4673bc8c88740537e66643a6c44a.tar.bz2 | |
Issue #25388: Fixed tokenizer crash when processing undecodable source code
with a null byte.
Diffstat (limited to 'Parser')
| -rw-r--r-- | Parser/tokenizer.c | 14 |
1 files changed, 6 insertions, 8 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 2369be4..cdb5103 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -196,7 +196,8 @@ error_ret(struct tok_state *tok) /* XXX */ tok->decoding_erred = 1; if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */ PyMem_FREE(tok->buf); - tok->buf = NULL; + tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL; + tok->done = E_DECODE; return NULL; /* as if it were EOF */ } @@ -952,11 +953,6 @@ tok_nextc(struct tok_state *tok) } buflen = PyBytes_GET_SIZE(u); buf = PyBytes_AS_STRING(u); - if (!buf) { - Py_DECREF(u); - tok->done = E_DECODE; - return EOF; - } newtok = PyMem_MALLOC(buflen+1); strcpy(newtok, buf); Py_DECREF(u); @@ -998,7 +994,6 @@ tok_nextc(struct tok_state *tok) if (tok->buf != NULL) PyMem_FREE(tok->buf); tok->buf = newtok; - tok->line_start = tok->buf; tok->cur = tok->buf; tok->line_start = tok->buf; tok->inp = strchr(tok->buf, '\0'); @@ -1021,7 +1016,8 @@ tok_nextc(struct tok_state *tok) } if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf), tok) == NULL) { - tok->done = E_EOF; + if (!tok->decoding_erred) + tok->done = E_EOF; done = 1; } else { @@ -1055,6 +1051,8 @@ tok_nextc(struct tok_state *tok) return EOF; } tok->buf = newbuf; + tok->cur = tok->buf + cur; + tok->line_start = tok->cur; tok->inp = tok->buf + curvalid; tok->end = tok->buf + newsize; tok->start = curstart < 0 ? NULL : |