diff options
| author | Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com> | 2019-06-25 09:12:16 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2019-06-25 09:12:16 (GMT) |
| commit | d32594ad27f48a898d42a0ea30b9d007b1c57de9 (patch) | |
| tree | 5a21486f006076a0596955e272c90726fdcc1201 /Objects | |
| parent | 26fe6c35374fa32577b230b856a92a3b094e08ed (diff) | |
| download | cpython-d32594ad27f48a898d42a0ea30b9d007b1c57de9.zip cpython-d32594ad27f48a898d42a0ea30b9d007b1c57de9.tar.gz cpython-d32594ad27f48a898d42a0ea30b9d007b1c57de9.tar.bz2 | |
bpo-24214: Fixed the UTF-8 and UTF-16 incremental decoders. (GH-14304)
* The UTF-8 incremental decoders fails now fast if encounter
a sequence that can't be handled by the error handler.
* The UTF-16 incremental decoders with the surrogatepass error
handler decodes now a lone low surrogate with final=False.
(cherry picked from commit 894263ba80af4b7733c2df95b527e96953922656)
Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
Diffstat (limited to 'Objects')
| -rw-r--r-- | Objects/stringlib/codecs.h | 6 | ||||
| -rw-r--r-- | Objects/unicodeobject.c | 10 |
2 files changed, 10 insertions, 6 deletions
diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index 8645bc2..d6f2b98 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -207,7 +207,7 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end, goto InvalidContinuation1; } else if (ch == 0xF4 && ch2 >= 0x90) { /* invalid sequence - \xF4\x90\x80\80- -- 110000- overflow */ + \xF4\x90\x80\x80- -- 110000- overflow */ goto InvalidContinuation1; } if (!IS_CONTINUATION_BYTE(ch3)) { @@ -573,10 +573,10 @@ STRINGLIB(utf16_decode)(const unsigned char **inptr, const unsigned char *e, } /* UTF-16 code pair: */ - if (q >= e) - goto UnexpectedEnd; if (!Py_UNICODE_IS_HIGH_SURROGATE(ch)) goto IllegalEncoding; + if (q >= e) + goto UnexpectedEnd; ch2 = (q[ihi] << 8) | q[ilo]; q += 2; if (!Py_UNICODE_IS_LOW_SURROGATE(ch2)) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 6ec4127..132f57a 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4937,11 +4937,15 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, endinpos = startinpos + 1; break; case 2: - case 3: - case 4: - if (s == end || consumed) { + if (consumed && (unsigned char)s[0] == 0xED && end - s == 2 + && (unsigned char)s[1] >= 0xA0 && (unsigned char)s[1] <= 0xBF) + { + /* Truncated surrogate code in range D800-DFFF */ goto End; } + /* fall through */ + case 3: + case 4: errmsg = "invalid continuation byte"; startinpos = s - starts; endinpos = startinpos + ch - 1; |