Issue #25761: Added more test cases for testing unpickling broken data.

Output raised exception at verbose level 2 (-vv).

2 files changed, 210 insertions, 93 deletions

diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
index cd2423f..b948c55 100644
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -11,8 +11,9 @@ import unittest
 import weakref
 from http.cookies import SimpleCookie
 
+from test import support
 from test.support import (
-    TestFailed, TESTFN, run_with_locale, no_tracing, captured_stdout,
+    TestFailed, TESTFN, run_with_locale, no_tracing,
     _2G, _4G, bigmemtest,
     )
 
@@ -679,6 +680,17 @@ class AbstractUnpickleTests(unittest.TestCase):
                 self.assertEqual(getattr(obj, slot, None),
                                  getattr(objcopy, slot, None), msg=msg)
 
+    def check_unpickling_error(self, errors, data):
+        with self.subTest(data=data), \
+             self.assertRaises(errors):
+            try:
+                self.loads(data)
+            except BaseException as exc:
+                if support.verbose > 1:
+                    print('%-32r - %s: %s' %
+                          (data, exc.__class__.__name__, exc))
+                raise
+
     def test_load_from_data0(self):
         self.assert_is_copy(self._testdata, self.loads(DATA0))
 
@@ -759,12 +771,7 @@ class AbstractUnpickleTests(unittest.TestCase):
 
         # Try too with a bogus literal.
         data = b'I' + str(maxint64).encode("ascii") + b'JUNK\n.'
-        self.assertRaises(ValueError, self.loads, data)
-
-    def test_pop_empty_stack(self):
-        # Test issue7455
-        s = b'0'
-        self.assertRaises((pickle.UnpicklingError, IndexError), self.loads, s)
+        self.check_unpickling_error(ValueError, data)
 
     def test_unpickle_from_2x(self):
         # Unpickle non-trivial data from Python 2.x.
@@ -849,22 +856,22 @@ class AbstractUnpickleTests(unittest.TestCase):
     def test_negative_32b_binbytes(self):
         # On 32-bit builds, a BINBYTES of 2**31 or more is refused
         dumped = b'\x80\x03B\xff\xff\xff\xffxyzq\x00.'
-        with self.assertRaises((pickle.UnpicklingError, OverflowError)):
-            self.loads(dumped)
+        self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
+                                    dumped)
 
     @requires_32b
     def test_negative_32b_binunicode(self):
         # On 32-bit builds, a BINUNICODE of 2**31 or more is refused
         dumped = b'\x80\x03X\xff\xff\xff\xffxyzq\x00.'
-        with self.assertRaises((pickle.UnpicklingError, OverflowError)):
-            self.loads(dumped)
+        self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
+                                    dumped)
 
     def test_short_binunicode(self):
         dumped = b'\x80\x04\x8c\x04\xe2\x82\xac\x00.'
         self.assertEqual(self.loads(dumped), '\u20ac\x00')
 
     def test_misc_get(self):
-        self.assertRaises(KeyError, self.loads, b'g0\np0')
+        self.check_unpickling_error(KeyError, b'g0\np0')
         self.assert_is_copy([(100,), (100,)],
                             self.loads(b'((Kdtp0\nh\x00l.))'))
 
@@ -879,14 +886,14 @@ class AbstractUnpickleTests(unittest.TestCase):
     @requires_32b
     def test_large_32b_binbytes8(self):
         dumped = b'\x80\x04\x8e\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.'
-        with self.assertRaises((pickle.UnpicklingError, OverflowError)):
-            self.loads(dumped)
+        self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
+                                    dumped)
 
     @requires_32b
     def test_large_32b_binunicode8(self):
         dumped = b'\x80\x04\x8d\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.'
-        with self.assertRaises((pickle.UnpicklingError, OverflowError)):
-            self.loads(dumped)
+        self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
+                                    dumped)
 
     def test_get(self):
         pickled = b'((lp100000\ng100000\nt.'
@@ -915,16 +922,16 @@ class AbstractUnpickleTests(unittest.TestCase):
     def test_negative_put(self):
         # Issue #12847
         dumped = b'Va\np-1\n.'
-        self.assertRaises(ValueError, self.loads, dumped)
+        self.check_unpickling_error(ValueError, dumped)
 
     @requires_32b
     def test_negative_32b_binput(self):
         # Issue #12847
         dumped = b'\x80\x03X\x01\x00\x00\x00ar\xff\xff\xff\xff.'
-        self.assertRaises(ValueError, self.loads, dumped)
+        self.check_unpickling_error(ValueError, dumped)
 
     def test_badly_escaped_string(self):
-        self.assertRaises(ValueError, self.loads, b"S'\\'\n.")
+        self.check_unpickling_error(ValueError, b"S'\\'\n.")
 
     def test_badly_quoted_string(self):
         # Issue #17710
@@ -942,7 +949,7 @@ class AbstractUnpickleTests(unittest.TestCase):
                       b'S\n.',
                       b'S.']
         for p in badpickles:
-            self.assertRaises(pickle.UnpicklingError, self.loads, p)
+            self.check_unpickling_error(pickle.UnpicklingError, p)
 
     def test_correctly_quoted_string(self):
         goodpickles = [(b"S''\n.", ''),
@@ -989,86 +996,183 @@ class AbstractUnpickleTests(unittest.TestCase):
 
     def test_bad_stack(self):
         badpickles = [
-            b'0.',              # POP
-            b'1.',              # POP_MARK
-            b'2.',              # DUP
-            # b'(2.',           # PyUnpickler doesn't raise
-            b'R.',              # REDUCE
-            b')R.',
-            b'a.',              # APPEND
-            b'Na.',
-            b'b.',              # BUILD
-            b'Nb.',
-            b'd.',              # DICT
-            b'e.',              # APPENDS
-            # b'(e.',           # PyUnpickler raises AttributeError
-            b'ibuiltins\nlist\n.',  # INST
-            b'l.',              # LIST
-            b'o.',              # OBJ
-            b'(o.',
-            b'p1\n.',           # PUT
-            b'q\x00.',          # BINPUT
-            b'r\x00\x00\x00\x00.',  # LONG_BINPUT
-            b's.',              # SETITEM
-            b'Ns.',
-            b'NNs.',
-            b't.',              # TUPLE
-            b'u.',              # SETITEMS
-            b'(u.',
-            b'}(Nu.',
-            b'\x81.',           # NEWOBJ
-            b')\x81.',
-            b'\x85.',           # TUPLE1
-            b'\x86.',           # TUPLE2
-            b'N\x86.',
-            b'\x87.',           # TUPLE3
-            b'N\x87.',
-            b'NN\x87.',
-            b'\x90.',           # ADDITEMS
-            # b'(\x90.',        # PyUnpickler raises AttributeError
-            b'\x91.',           # FROZENSET
-            b'\x92.',           # NEWOBJ_EX
-            b')}\x92.',
-            b'\x93.',           # STACK_GLOBAL
-            b'Vlist\n\x93.',
-            b'\x94.',           # MEMOIZE
+            b'.',                       # STOP
+            b'0',                       # POP
+            b'1',                       # POP_MARK
+            b'2',                       # DUP
+            # b'(2',                    # PyUnpickler doesn't raise
+            b'R',                       # REDUCE
+            b')R',
+            b'a',                       # APPEND
+            b'Na',
+            b'b',                       # BUILD
+            b'Nb',
+            b'd',                       # DICT
+            b'e',                       # APPENDS
+            # b'(e',                    # PyUnpickler raises AttributeError
+            b'ibuiltins\nlist\n',       # INST
+            b'l',                       # LIST
+            b'o',                       # OBJ
+            b'(o',
+            b'p1\n',                    # PUT
+            b'q\x00',                   # BINPUT
+            b'r\x00\x00\x00\x00',       # LONG_BINPUT
+            b's',                       # SETITEM
+            b'Ns',
+            b'NNs',
+            b't',                       # TUPLE
+            b'u',                       # SETITEMS
+            # b'(u',                    # PyUnpickler doesn't raise
+            b'}(Nu',
+            b'\x81',                    # NEWOBJ
+            b')\x81',
+            b'\x85',                    # TUPLE1
+            b'\x86',                    # TUPLE2
+            b'N\x86',
+            b'\x87',                    # TUPLE3
+            b'N\x87',
+            b'NN\x87',
+            b'\x90',                    # ADDITEMS
+            # b'(\x90',                 # PyUnpickler raises AttributeError
+            b'\x91',                    # FROZENSET
+            b'\x92',                    # NEWOBJ_EX
+            b')}\x92',
+            b'\x93',                    # STACK_GLOBAL
+            b'Vlist\n\x93',
+            b'\x94',                    # MEMOIZE
         ]
         for p in badpickles:
-            with self.subTest(p):
-                self.assertRaises(self.bad_stack_errors, self.loads, p)
+            self.check_unpickling_error(self.bad_stack_errors, p)
 
     def test_bad_mark(self):
         badpickles = [
-            b'cbuiltins\nlist\n)(R.',           # REDUCE
-            b'cbuiltins\nlist\n()R.',
-            b']N(a.',                           # APPEND
-            b'cbuiltins\nValueError\n)R}(b.',   # BUILD
-            b'cbuiltins\nValueError\n)R(}b.',
-            b'(Nd.',                            # DICT
-            b'}NN(s.',                          # SETITEM
-            b'}N(Ns.',
-            b'cbuiltins\nlist\n)(\x81.',        # NEWOBJ
-            b'cbuiltins\nlist\n()\x81.',
-            b'N(\x85.',                         # TUPLE1
-            b'NN(\x86.',                        # TUPLE2
-            b'N(N\x86.',
-            b'NNN(\x87.',                       # TUPLE3
-            b'NN(N\x87.',
-            b'N(NN\x87.',
-            b'cbuiltins\nlist\n)}(\x92.',       # NEWOBJ_EX
-            b'cbuiltins\nlist\n)(}\x92.',
-            b'cbuiltins\nlist\n()}\x92.',
-            b'Vbuiltins\n(Vlist\n\x93.',        # STACK_GLOBAL
-            b'Vbuiltins\nVlist\n(\x93.',
+            # b'N(.',                     # STOP
+            b'N(2',                     # DUP
+            b'cbuiltins\nlist\n)(R',    # REDUCE
+            b'cbuiltins\nlist\n()R',
+            b']N(a',                    # APPEND
+                                        # BUILD
+            b'cbuiltins\nValueError\n)R}(b',
+            b'cbuiltins\nValueError\n)R(}b',
+            b'(Nd',                     # DICT
+            b'N(p1\n',                  # PUT
+            b'N(q\x00',                 # BINPUT
+            b'N(r\x00\x00\x00\x00',     # LONG_BINPUT
+            b'}NN(s',                   # SETITEM
+            b'}N(Ns',
+            b'}(NNs',
+            b'}((u',                    # SETITEMS
+            b'cbuiltins\nlist\n)(\x81', # NEWOBJ
+            b'cbuiltins\nlist\n()\x81',
+            b'N(\x85',                  # TUPLE1
+            b'NN(\x86',                 # TUPLE2
+            b'N(N\x86',
+            b'NNN(\x87',                # TUPLE3
+            b'NN(N\x87',
+            b'N(NN\x87',
+            b']((\x90',                 # ADDITEMS
+                                        # NEWOBJ_EX
+            b'cbuiltins\nlist\n)}(\x92',
+            b'cbuiltins\nlist\n)(}\x92',
+            b'cbuiltins\nlist\n()}\x92',
+                                        # STACK_GLOBAL
+            b'Vbuiltins\n(Vlist\n\x93',
+            b'Vbuiltins\nVlist\n(\x93',
+            b'N(\x94',                  # MEMOIZE
+        ]
+        for p in badpickles:
+            self.check_unpickling_error(self.bad_mark_errors, p)
+
+    def test_truncated_data(self):
+        self.check_unpickling_error(EOFError, b'')
+        self.check_unpickling_error(EOFError, b'N')
+        badpickles = [
+            b'B',                       # BINBYTES
+            b'B\x03\x00\x00',
+            b'B\x03\x00\x00\x00',
+            b'B\x03\x00\x00\x00ab',
+            b'C',                       # SHORT_BINBYTES
+            b'C\x03',
+            b'C\x03ab',
+            b'F',                       # FLOAT
+            b'F0.0',
+            b'F0.00',
+            b'G',                       # BINFLOAT
+            b'G\x00\x00\x00\x00\x00\x00\x00',
+            b'I',                       # INT
+            b'I0',
+            b'J',                       # BININT
+            b'J\x00\x00\x00',
+            b'K',                       # BININT1
+            b'L',                       # LONG
+            b'L0',
+            b'L10',
+            b'L0L',
+            b'L10L',
+            b'M',                       # BININT2
+            b'M\x00',
+            # b'P',                       # PERSID
+            # b'Pabc',
+            b'S',                       # STRING
+            b"S'abc'",
+            b'T',                       # BINSTRING
+            b'T\x03\x00\x00',
+            b'T\x03\x00\x00\x00',
+            b'T\x03\x00\x00\x00ab',
+            b'U',                       # SHORT_BINSTRING
+            b'U\x03',
+            b'U\x03ab',
+            b'V',                       # UNICODE
+            b'Vabc',
+            b'X',                       # BINUNICODE
+            b'X\x03\x00\x00',
+            b'X\x03\x00\x00\x00',
+            b'X\x03\x00\x00\x00ab',
+            b'(c',                      # GLOBAL
+            b'(cbuiltins',
+            b'(cbuiltins\n',
+            b'(cbuiltins\nlist',
+            b'Ng',                      # GET
+            b'Ng0',
+            b'(i',                      # INST
+            b'(ibuiltins',
+            b'(ibuiltins\n',
+            b'(ibuiltins\nlist',
+            b'Nh',                      # BINGET
+            b'Nj',                      # LONG_BINGET
+            b'Nj\x00\x00\x00',
+            b'Np',                      # PUT
+            b'Np0',
+            b'Nq',                      # BINPUT
+            b'Nr',                      # LONG_BINPUT
+            b'Nr\x00\x00\x00',
+            b'\x80',                    # PROTO
+            b'\x82',                    # EXT1
+            b'\x83',                    # EXT2
+            b'\x84\x01',
+            b'\x84',                    # EXT4
+            b'\x84\x01\x00\x00',
+            b'\x8a',                    # LONG1
+            b'\x8b',                    # LONG4
+            b'\x8b\x00\x00\x00',
+            b'\x8c',                    # SHORT_BINUNICODE
+            b'\x8c\x03',
+            b'\x8c\x03ab',
+            b'\x8d',                    # BINUNICODE8
+            b'\x8d\x03\x00\x00\x00\x00\x00\x00',
+            b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00',
+            b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00ab',
+            b'\x8e',                    # BINBYTES8
+            b'\x8e\x03\x00\x00\x00\x00\x00\x00',
+            b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00',
+            b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00ab',
+            b'\x95',                    # FRAME
+            b'\x95\x02\x00\x00\x00\x00\x00\x00',
+            b'\x95\x02\x00\x00\x00\x00\x00\x00\x00',
+            b'\x95\x02\x00\x00\x00\x00\x00\x00\x00N',
         ]
         for p in badpickles:
-            # PyUnpickler prints reduce errors to stdout
-            with self.subTest(p), captured_stdout():
-                try:
-                    self.loads(p)
-                except (IndexError, AttributeError, TypeError,
-                        pickle.UnpicklingError):
-                    pass
+            self.check_unpickling_error(self.truncated_errors, p)
 
 
 class AbstractPickleTests(unittest.TestCase):
diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py
index f04c5ac..f6d9cc0 100644
--- a/Lib/test/test_pickle.py
+++ b/Lib/test/test_pickle.py
@@ -33,6 +33,11 @@ class PyUnpicklerTests(AbstractUnpickleTests):
 
     unpickler = pickle._Unpickler
     bad_stack_errors = (IndexError,)
+    bad_mark_errors = (IndexError, pickle.UnpicklingError,
+                       TypeError, AttributeError, EOFError)
+    truncated_errors = (pickle.UnpicklingError, EOFError,
+                        AttributeError, ValueError,
+                        struct.error, IndexError, ImportError)
 
     def loads(self, buf, **kwds):
         f = io.BytesIO(buf)
@@ -64,6 +69,11 @@ class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests,
     pickler = pickle._Pickler
     unpickler = pickle._Unpickler
     bad_stack_errors = (pickle.UnpicklingError, IndexError)
+    bad_mark_errors = (pickle.UnpicklingError, IndexError,
+                       TypeError, AttributeError, EOFError)
+    truncated_errors = (pickle.UnpicklingError, EOFError,
+                        AttributeError, ValueError,
+                        struct.error, IndexError, ImportError)
 
     def dumps(self, arg, protocol=None):
         return pickle.dumps(arg, protocol)
@@ -122,6 +132,9 @@ if has_c_implementation:
     class CUnpicklerTests(PyUnpicklerTests):
         unpickler = _pickle.Unpickler
         bad_stack_errors = (pickle.UnpicklingError,)
+        bad_mark_errors = (EOFError,)
+        truncated_errors = (pickle.UnpicklingError, EOFError,
+                            AttributeError, ValueError)
 
     class CPicklerTests(PyPicklerTests):
         pickler = _pickle.Pickler