diff options
| author | Pablo Galindo Salgado <Pablogsal@gmail.com> | 2021-12-12 07:06:50 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-12-12 07:06:50 (GMT) |
| commit | 4325a766f5f603ef6dfb8c4d5798e5e73cb5efd5 (patch) | |
| tree | af6a6984dcbce46cad39d8d3fc669909cb36ffc9 /Parser/tokenizer.c | |
| parent | 59435eea08d30796174552c0ca03c59b41adf8a5 (diff) | |
| download | cpython-4325a766f5f603ef6dfb8c4d5798e5e73cb5efd5.zip cpython-4325a766f5f603ef6dfb8c4d5798e5e73cb5efd5.tar.gz cpython-4325a766f5f603ef6dfb8c4d5798e5e73cb5efd5.tar.bz2 | |
bpo-46054: Fix parsing error when parsing non-utf8 characters in source files (GH-30068)
Diffstat (limited to 'Parser/tokenizer.c')
| -rw-r--r-- | Parser/tokenizer.c | 13 |
1 files changed, 5 insertions, 8 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 6358cdf..a560572 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -819,10 +819,10 @@ tok_readline_raw(struct tok_state *tok) tok_concatenate_interactive_new_line(tok, line) == -1) { return 0; } - if (*tok->inp == '\0') { + tok->inp = strchr(tok->inp, '\0'); + if (tok->inp == tok->buf) { return 0; } - tok->inp = strchr(tok->inp, '\0'); } while (tok->inp[-1] != '\n'); return 1; } @@ -984,12 +984,9 @@ tok_underflow_file(struct tok_state *tok) { } /* The default encoding is UTF-8, so make sure we don't have any non-UTF-8 sequences in it. */ - if (!tok->encoding - && (tok->decoding_state != STATE_NORMAL || tok->lineno >= 2)) { - if (!ensure_utf8(tok->cur, tok)) { - error_ret(tok); - return 0; - } + if (!tok->encoding && !ensure_utf8(tok->cur, tok)) { + error_ret(tok); + return 0; } assert(tok->done == E_OK); return tok->done == E_OK; |