diff options
| -rw-r--r-- | Lib/test/test_builtin.py | 4 | ||||
| -rw-r--r-- | Parser/tokenizer.c | 4 |
2 files changed, 6 insertions, 2 deletions
diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 92e44d5..2e00632 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -190,6 +190,8 @@ class BuiltinTest(unittest.TestCase): def test_compile(self): compile('print 1\n', '', 'exec') + bom = '\xef\xbb\xbf' + compile(bom + 'print 1\n', '', 'exec') self.assertRaises(TypeError, compile) self.assertRaises(ValueError, compile, 'print 42\n', '<string>', 'badmode') self.assertRaises(ValueError, compile, 'print 42\n', '<string>', 'single', 0xff) @@ -305,6 +307,8 @@ class BuiltinTest(unittest.TestCase): self.assertEqual(eval(unicode('a'), globals, locals), 1) self.assertEqual(eval(unicode('b'), globals, locals), 200) self.assertEqual(eval(unicode('c'), globals, locals), 300) + bom = '\xef\xbb\xbf' + self.assertEqual(eval(bom + 'a', globals, locals), 1) self.assertRaises(TypeError, eval) self.assertRaises(TypeError, eval, ()) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index aaed637..4952a3c 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -506,14 +506,14 @@ decoding_feof(struct tok_state *tok) /* Fetch a byte from TOK, using the string buffer. */ static int buf_getc(struct tok_state *tok) { - return *tok->str++; + return Py_CHARMASK(*tok->str++); } /* Unfetch a byte from TOK, using the string buffer. */ static void buf_ungetc(int c, struct tok_state *tok) { tok->str--; - assert(*tok->str == c); /* tok->cur may point to read-only segment */ + assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */ } /* Set the readline function for TOK to ENC. For the string-based |