From 7279befccbd49fc6aa2f96c4b0dea1799695156d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 29 Nov 2015 13:12:10 +0200 Subject: Issue #25761: Added more test cases for testing unpickling broken data. Output raised exception at verbose level 2 (-vv). --- Lib/test/pickletester.py | 290 ++++++++++++++++++++++++++++++++--------------- Lib/test/test_pickle.py | 13 +++ 2 files changed, 210 insertions(+), 93 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index cd2423f..b948c55 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -11,8 +11,9 @@ import unittest import weakref from http.cookies import SimpleCookie +from test import support from test.support import ( - TestFailed, TESTFN, run_with_locale, no_tracing, captured_stdout, + TestFailed, TESTFN, run_with_locale, no_tracing, _2G, _4G, bigmemtest, ) @@ -679,6 +680,17 @@ class AbstractUnpickleTests(unittest.TestCase): self.assertEqual(getattr(obj, slot, None), getattr(objcopy, slot, None), msg=msg) + def check_unpickling_error(self, errors, data): + with self.subTest(data=data), \ + self.assertRaises(errors): + try: + self.loads(data) + except BaseException as exc: + if support.verbose > 1: + print('%-32r - %s: %s' % + (data, exc.__class__.__name__, exc)) + raise + def test_load_from_data0(self): self.assert_is_copy(self._testdata, self.loads(DATA0)) @@ -759,12 +771,7 @@ class AbstractUnpickleTests(unittest.TestCase): # Try too with a bogus literal. data = b'I' + str(maxint64).encode("ascii") + b'JUNK\n.' - self.assertRaises(ValueError, self.loads, data) - - def test_pop_empty_stack(self): - # Test issue7455 - s = b'0' - self.assertRaises((pickle.UnpicklingError, IndexError), self.loads, s) + self.check_unpickling_error(ValueError, data) def test_unpickle_from_2x(self): # Unpickle non-trivial data from Python 2.x. @@ -849,22 +856,22 @@ class AbstractUnpickleTests(unittest.TestCase): def test_negative_32b_binbytes(self): # On 32-bit builds, a BINBYTES of 2**31 or more is refused dumped = b'\x80\x03B\xff\xff\xff\xffxyzq\x00.' - with self.assertRaises((pickle.UnpicklingError, OverflowError)): - self.loads(dumped) + self.check_unpickling_error((pickle.UnpicklingError, OverflowError), + dumped) @requires_32b def test_negative_32b_binunicode(self): # On 32-bit builds, a BINUNICODE of 2**31 or more is refused dumped = b'\x80\x03X\xff\xff\xff\xffxyzq\x00.' - with self.assertRaises((pickle.UnpicklingError, OverflowError)): - self.loads(dumped) + self.check_unpickling_error((pickle.UnpicklingError, OverflowError), + dumped) def test_short_binunicode(self): dumped = b'\x80\x04\x8c\x04\xe2\x82\xac\x00.' self.assertEqual(self.loads(dumped), '\u20ac\x00') def test_misc_get(self): - self.assertRaises(KeyError, self.loads, b'g0\np0') + self.check_unpickling_error(KeyError, b'g0\np0') self.assert_is_copy([(100,), (100,)], self.loads(b'((Kdtp0\nh\x00l.))')) @@ -879,14 +886,14 @@ class AbstractUnpickleTests(unittest.TestCase): @requires_32b def test_large_32b_binbytes8(self): dumped = b'\x80\x04\x8e\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.' - with self.assertRaises((pickle.UnpicklingError, OverflowError)): - self.loads(dumped) + self.check_unpickling_error((pickle.UnpicklingError, OverflowError), + dumped) @requires_32b def test_large_32b_binunicode8(self): dumped = b'\x80\x04\x8d\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.' - with self.assertRaises((pickle.UnpicklingError, OverflowError)): - self.loads(dumped) + self.check_unpickling_error((pickle.UnpicklingError, OverflowError), + dumped) def test_get(self): pickled = b'((lp100000\ng100000\nt.' @@ -915,16 +922,16 @@ class AbstractUnpickleTests(unittest.TestCase): def test_negative_put(self): # Issue #12847 dumped = b'Va\np-1\n.' - self.assertRaises(ValueError, self.loads, dumped) + self.check_unpickling_error(ValueError, dumped) @requires_32b def test_negative_32b_binput(self): # Issue #12847 dumped = b'\x80\x03X\x01\x00\x00\x00ar\xff\xff\xff\xff.' - self.assertRaises(ValueError, self.loads, dumped) + self.check_unpickling_error(ValueError, dumped) def test_badly_escaped_string(self): - self.assertRaises(ValueError, self.loads, b"S'\\'\n.") + self.check_unpickling_error(ValueError, b"S'\\'\n.") def test_badly_quoted_string(self): # Issue #17710 @@ -942,7 +949,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'S\n.', b'S.'] for p in badpickles: - self.assertRaises(pickle.UnpicklingError, self.loads, p) + self.check_unpickling_error(pickle.UnpicklingError, p) def test_correctly_quoted_string(self): goodpickles = [(b"S''\n.", ''), @@ -989,86 +996,183 @@ class AbstractUnpickleTests(unittest.TestCase): def test_bad_stack(self): badpickles = [ - b'0.', # POP - b'1.', # POP_MARK - b'2.', # DUP - # b'(2.', # PyUnpickler doesn't raise - b'R.', # REDUCE - b')R.', - b'a.', # APPEND - b'Na.', - b'b.', # BUILD - b'Nb.', - b'd.', # DICT - b'e.', # APPENDS - # b'(e.', # PyUnpickler raises AttributeError - b'ibuiltins\nlist\n.', # INST - b'l.', # LIST - b'o.', # OBJ - b'(o.', - b'p1\n.', # PUT - b'q\x00.', # BINPUT - b'r\x00\x00\x00\x00.', # LONG_BINPUT - b's.', # SETITEM - b'Ns.', - b'NNs.', - b't.', # TUPLE - b'u.', # SETITEMS - b'(u.', - b'}(Nu.', - b'\x81.', # NEWOBJ - b')\x81.', - b'\x85.', # TUPLE1 - b'\x86.', # TUPLE2 - b'N\x86.', - b'\x87.', # TUPLE3 - b'N\x87.', - b'NN\x87.', - b'\x90.', # ADDITEMS - # b'(\x90.', # PyUnpickler raises AttributeError - b'\x91.', # FROZENSET - b'\x92.', # NEWOBJ_EX - b')}\x92.', - b'\x93.', # STACK_GLOBAL - b'Vlist\n\x93.', - b'\x94.', # MEMOIZE + b'.', # STOP + b'0', # POP + b'1', # POP_MARK + b'2', # DUP + # b'(2', # PyUnpickler doesn't raise + b'R', # REDUCE + b')R', + b'a', # APPEND + b'Na', + b'b', # BUILD + b'Nb', + b'd', # DICT + b'e', # APPENDS + # b'(e', # PyUnpickler raises AttributeError + b'ibuiltins\nlist\n', # INST + b'l', # LIST + b'o', # OBJ + b'(o', + b'p1\n', # PUT + b'q\x00', # BINPUT + b'r\x00\x00\x00\x00', # LONG_BINPUT + b's', # SETITEM + b'Ns', + b'NNs', + b't', # TUPLE + b'u', # SETITEMS + # b'(u', # PyUnpickler doesn't raise + b'}(Nu', + b'\x81', # NEWOBJ + b')\x81', + b'\x85', # TUPLE1 + b'\x86', # TUPLE2 + b'N\x86', + b'\x87', # TUPLE3 + b'N\x87', + b'NN\x87', + b'\x90', # ADDITEMS + # b'(\x90', # PyUnpickler raises AttributeError + b'\x91', # FROZENSET + b'\x92', # NEWOBJ_EX + b')}\x92', + b'\x93', # STACK_GLOBAL + b'Vlist\n\x93', + b'\x94', # MEMOIZE ] for p in badpickles: - with self.subTest(p): - self.assertRaises(self.bad_stack_errors, self.loads, p) + self.check_unpickling_error(self.bad_stack_errors, p) def test_bad_mark(self): badpickles = [ - b'cbuiltins\nlist\n)(R.', # REDUCE - b'cbuiltins\nlist\n()R.', - b']N(a.', # APPEND - b'cbuiltins\nValueError\n)R}(b.', # BUILD - b'cbuiltins\nValueError\n)R(}b.', - b'(Nd.', # DICT - b'}NN(s.', # SETITEM - b'}N(Ns.', - b'cbuiltins\nlist\n)(\x81.', # NEWOBJ - b'cbuiltins\nlist\n()\x81.', - b'N(\x85.', # TUPLE1 - b'NN(\x86.', # TUPLE2 - b'N(N\x86.', - b'NNN(\x87.', # TUPLE3 - b'NN(N\x87.', - b'N(NN\x87.', - b'cbuiltins\nlist\n)}(\x92.', # NEWOBJ_EX - b'cbuiltins\nlist\n)(}\x92.', - b'cbuiltins\nlist\n()}\x92.', - b'Vbuiltins\n(Vlist\n\x93.', # STACK_GLOBAL - b'Vbuiltins\nVlist\n(\x93.', + # b'N(.', # STOP + b'N(2', # DUP + b'cbuiltins\nlist\n)(R', # REDUCE + b'cbuiltins\nlist\n()R', + b']N(a', # APPEND + # BUILD + b'cbuiltins\nValueError\n)R}(b', + b'cbuiltins\nValueError\n)R(}b', + b'(Nd', # DICT + b'N(p1\n', # PUT + b'N(q\x00', # BINPUT + b'N(r\x00\x00\x00\x00', # LONG_BINPUT + b'}NN(s', # SETITEM + b'}N(Ns', + b'}(NNs', + b'}((u', # SETITEMS + b'cbuiltins\nlist\n)(\x81', # NEWOBJ + b'cbuiltins\nlist\n()\x81', + b'N(\x85', # TUPLE1 + b'NN(\x86', # TUPLE2 + b'N(N\x86', + b'NNN(\x87', # TUPLE3 + b'NN(N\x87', + b'N(NN\x87', + b']((\x90', # ADDITEMS + # NEWOBJ_EX + b'cbuiltins\nlist\n)}(\x92', + b'cbuiltins\nlist\n)(}\x92', + b'cbuiltins\nlist\n()}\x92', + # STACK_GLOBAL + b'Vbuiltins\n(Vlist\n\x93', + b'Vbuiltins\nVlist\n(\x93', + b'N(\x94', # MEMOIZE + ] + for p in badpickles: + self.check_unpickling_error(self.bad_mark_errors, p) + + def test_truncated_data(self): + self.check_unpickling_error(EOFError, b'') + self.check_unpickling_error(EOFError, b'N') + badpickles = [ + b'B', # BINBYTES + b'B\x03\x00\x00', + b'B\x03\x00\x00\x00', + b'B\x03\x00\x00\x00ab', + b'C', # SHORT_BINBYTES + b'C\x03', + b'C\x03ab', + b'F', # FLOAT + b'F0.0', + b'F0.00', + b'G', # BINFLOAT + b'G\x00\x00\x00\x00\x00\x00\x00', + b'I', # INT + b'I0', + b'J', # BININT + b'J\x00\x00\x00', + b'K', # BININT1 + b'L', # LONG + b'L0', + b'L10', + b'L0L', + b'L10L', + b'M', # BININT2 + b'M\x00', + # b'P', # PERSID + # b'Pabc', + b'S', # STRING + b"S'abc'", + b'T', # BINSTRING + b'T\x03\x00\x00', + b'T\x03\x00\x00\x00', + b'T\x03\x00\x00\x00ab', + b'U', # SHORT_BINSTRING + b'U\x03', + b'U\x03ab', + b'V', # UNICODE + b'Vabc', + b'X', # BINUNICODE + b'X\x03\x00\x00', + b'X\x03\x00\x00\x00', + b'X\x03\x00\x00\x00ab', + b'(c', # GLOBAL + b'(cbuiltins', + b'(cbuiltins\n', + b'(cbuiltins\nlist', + b'Ng', # GET + b'Ng0', + b'(i', # INST + b'(ibuiltins', + b'(ibuiltins\n', + b'(ibuiltins\nlist', + b'Nh', # BINGET + b'Nj', # LONG_BINGET + b'Nj\x00\x00\x00', + b'Np', # PUT + b'Np0', + b'Nq', # BINPUT + b'Nr', # LONG_BINPUT + b'Nr\x00\x00\x00', + b'\x80', # PROTO + b'\x82', # EXT1 + b'\x83', # EXT2 + b'\x84\x01', + b'\x84', # EXT4 + b'\x84\x01\x00\x00', + b'\x8a', # LONG1 + b'\x8b', # LONG4 + b'\x8b\x00\x00\x00', + b'\x8c', # SHORT_BINUNICODE + b'\x8c\x03', + b'\x8c\x03ab', + b'\x8d', # BINUNICODE8 + b'\x8d\x03\x00\x00\x00\x00\x00\x00', + b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00', + b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00ab', + b'\x8e', # BINBYTES8 + b'\x8e\x03\x00\x00\x00\x00\x00\x00', + b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00', + b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00ab', + b'\x95', # FRAME + b'\x95\x02\x00\x00\x00\x00\x00\x00', + b'\x95\x02\x00\x00\x00\x00\x00\x00\x00', + b'\x95\x02\x00\x00\x00\x00\x00\x00\x00N', ] for p in badpickles: - # PyUnpickler prints reduce errors to stdout - with self.subTest(p), captured_stdout(): - try: - self.loads(p) - except (IndexError, AttributeError, TypeError, - pickle.UnpicklingError): - pass + self.check_unpickling_error(self.truncated_errors, p) class AbstractPickleTests(unittest.TestCase): diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index f04c5ac..f6d9cc0 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -33,6 +33,11 @@ class PyUnpicklerTests(AbstractUnpickleTests): unpickler = pickle._Unpickler bad_stack_errors = (IndexError,) + bad_mark_errors = (IndexError, pickle.UnpicklingError, + TypeError, AttributeError, EOFError) + truncated_errors = (pickle.UnpicklingError, EOFError, + AttributeError, ValueError, + struct.error, IndexError, ImportError) def loads(self, buf, **kwds): f = io.BytesIO(buf) @@ -64,6 +69,11 @@ class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests, pickler = pickle._Pickler unpickler = pickle._Unpickler bad_stack_errors = (pickle.UnpicklingError, IndexError) + bad_mark_errors = (pickle.UnpicklingError, IndexError, + TypeError, AttributeError, EOFError) + truncated_errors = (pickle.UnpicklingError, EOFError, + AttributeError, ValueError, + struct.error, IndexError, ImportError) def dumps(self, arg, protocol=None): return pickle.dumps(arg, protocol) @@ -122,6 +132,9 @@ if has_c_implementation: class CUnpicklerTests(PyUnpicklerTests): unpickler = _pickle.Unpickler bad_stack_errors = (pickle.UnpicklingError,) + bad_mark_errors = (EOFError,) + truncated_errors = (pickle.UnpicklingError, EOFError, + AttributeError, ValueError) class CPicklerTests(PyPicklerTests): pickler = _pickle.Pickler -- cgit v0.12