From 6aa278ee53647ceb63cfe5ce6c1a07b0e10cdf90 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 3 Mar 2010 00:18:49 +0000 Subject: Merged revisions 78603 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r78603 | victor.stinner | 2010-03-03 00:20:02 +0100 (mer., 03 mars 2010) | 5 lines Issue #7820: The parser tokenizer restores all bytes in the right if the BOM check fails. Fix an assertion in pydebug mode. ........ --- Lib/test/test_pep263.py | 11 +++++++++++ Misc/NEWS | 3 +++ Parser/tokenizer.c | 51 +++++++++++++++++++++++++++---------------------- 3 files changed, 42 insertions(+), 23 deletions(-) diff --git a/Lib/test/test_pep263.py b/Lib/test/test_pep263.py index 8c1fbe7..e136a9c 100644 --- a/Lib/test/test_pep263.py +++ b/Lib/test/test_pep263.py @@ -44,6 +44,17 @@ class PEP263Test(unittest.TestCase): self.assertEqual(len(d['a']), len(d['b'])) self.assertEqual(ascii(d['a']), ascii(d['b'])) + def test_issue7820(self): + # Ensure that check_bom() restores all bytes in the right order if + # check_bom() fails in pydebug mode: a buffer starts with the first + # byte of a valid BOM, but next bytes are different + + # one byte in common with the UTF-16-LE BOM + self.assertRaises(SyntaxError, eval, b'\xff\x20') + + # two bytes in common with the UTF-8 BOM + self.assertRaises(SyntaxError, eval, b'\xef\xbb\x20') + def test_main(): support.run_unittest(PEP263Test) diff --git a/Misc/NEWS b/Misc/NEWS index fddba76..e8f37ab 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,9 @@ What's New in Python 3.2 Alpha 1? Core and Builtins ----------------- +- Issue #7820: The parser tokenizer restores all bytes in the right if + the BOM check fails. + - Handle errors from looking up __prepare__ correctly. - Issue #5939: Add additional runtime checking to ensure a valid capsule diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 5a1c268..50b0ff1 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -318,46 +318,51 @@ check_bom(int get_char(struct tok_state *), int set_readline(struct tok_state *, const char *), struct tok_state *tok) { - int ch = get_char(tok); + int ch1, ch2, ch3; + ch1 = get_char(tok); tok->decoding_state = STATE_RAW; - if (ch == EOF) { + if (ch1 == EOF) { return 1; - } else if (ch == 0xEF) { - ch = get_char(tok); - if (ch != 0xBB) { - unget_char(ch, tok); - unget_char(0xEF, tok); - /* any token beginning with '\xEF' is a bad token */ + } else if (ch1 == 0xEF) { + ch2 = get_char(tok); + if (ch2 != 0xBB) { + unget_char(ch2, tok); + unget_char(ch1, tok); return 1; } - ch = get_char(tok); - if (ch != 0xBF) { - unget_char(ch, tok); - unget_char(0xBB, tok); - unget_char(0xEF, tok); - /* any token beginning with '\xEF' is a bad token */ + ch3 = get_char(tok); + if (ch3 != 0xBF) { + unget_char(ch3, tok); + unget_char(ch2, tok); + unget_char(ch1, tok); return 1; } #if 0 /* Disable support for UTF-16 BOMs until a decision is made whether this needs to be supported. */ - } else if (ch == 0xFE) { - ch = get_char(tok); - if (ch != 0xFF) - goto NON_BOM; + } else if (ch1 == 0xFE) { + ch2 = get_char(tok); + if (ch2 != 0xFF) { + unget_char(ch2, tok); + unget_char(ch1, tok); + return 1; + } if (!set_readline(tok, "utf-16-be")) return 0; tok->decoding_state = STATE_NORMAL; - } else if (ch == 0xFF) { - ch = get_char(tok); - if (ch != 0xFE) - goto NON_BOM; + } else if (ch1 == 0xFF) { + ch2 = get_char(tok); + if (ch2 != 0xFE) { + unget_char(ch2, tok); + unget_char(ch1, tok); + return 1; + } if (!set_readline(tok, "utf-16-le")) return 0; tok->decoding_state = STATE_NORMAL; #endif } else { - unget_char(ch, tok); + unget_char(ch1, tok); return 1; } if (tok->encoding != NULL) -- cgit v0.12