From 14a7d6389fb453cca18d61bbe39deff7062bc66f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 30 Mar 2016 20:43:06 +0300 Subject: Issue #26494: Fixed crash on iterating exhausting iterators. Affected classes are generic sequence iterators, iterators of bytearray, list, tuple, set, frozenset, dict, OrderedDict and corresponding views. --- Lib/test/seq_tests.py | 5 +++++ Lib/test/test_bytes.py | 4 ++++ Lib/test/test_deque.py | 4 ++++ Lib/test/test_dict.py | 9 +++++++++ Lib/test/test_iter.py | 6 +++++- Lib/test/test_ordered_dict.py | 9 +++++++++ Lib/test/test_set.py | 3 +++ Lib/test/test_support.py | 18 ++++++++++++++++++ Lib/test/test_unicode.py | 5 +++++ Misc/NEWS | 4 ++++ Objects/bytearrayobject.c | 2 +- Objects/dictobject.c | 6 +++--- Objects/iterobject.c | 2 +- Objects/listobject.c | 20 +++++++++++++------- Objects/setobject.c | 2 +- Objects/tupleobject.c | 2 +- 16 files changed, 86 insertions(+), 15 deletions(-) diff --git a/Lib/test/seq_tests.py b/Lib/test/seq_tests.py index f4673d4..f019e8d 100644 --- a/Lib/test/seq_tests.py +++ b/Lib/test/seq_tests.py @@ -4,6 +4,7 @@ Tests common to tuple, list and UserList.UserList import unittest import sys +from test import test_support as support # Various iterables # This is used for checking the constructor (here and in test_deque.py) @@ -402,3 +403,7 @@ class CommonTest(unittest.TestCase): self.assertEqual(a.index(0, -4*sys.maxint, 4*sys.maxint), 2) self.assertRaises(ValueError, a.index, 0, 4*sys.maxint,-4*sys.maxint) self.assertRaises(ValueError, a.index, 2, 0, -10) + + def test_free_after_iterating(self): + support.check_free_after_iterating(self, iter, self.type2test) + support.check_free_after_iterating(self, reversed, self.type2test) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 02fba38..82ad451 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -518,6 +518,10 @@ class BaseBytesTest(unittest.TestCase): self.assertRaisesRegexp(TypeError, r'\bendswith\b', b.endswith, x, None, None, None) + def test_free_after_iterating(self): + test.test_support.check_free_after_iterating(self, iter, self.type2test) + test.test_support.check_free_after_iterating(self, reversed, self.type2test) + class ByteArrayTest(BaseBytesTest): type2test = bytearray diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 13ea02f..b4efa65 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -669,6 +669,10 @@ class TestSubclassWithKwargs(unittest.TestCase): # SF bug #1486663 -- this used to erroneously raise a TypeError SubclassWithKwargs(newarg=1) + def test_free_after_iterating(self): + # For now, bypass tests that require slicing + self.skipTest("Exhausted deque iterator doesn't free a deque") + #============================================================================== libreftest = """ diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 1c63fc0..4474c9b 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -681,6 +681,15 @@ class DictTest(unittest.TestCase): self._tracked(MyDict()) + def test_free_after_iterating(self): + test_support.check_free_after_iterating(self, iter, dict) + test_support.check_free_after_iterating(self, lambda d: d.iterkeys(), dict) + test_support.check_free_after_iterating(self, lambda d: d.itervalues(), dict) + test_support.check_free_after_iterating(self, lambda d: d.iteritems(), dict) + test_support.check_free_after_iterating(self, lambda d: iter(d.viewkeys()), dict) + test_support.check_free_after_iterating(self, lambda d: iter(d.viewvalues()), dict) + test_support.check_free_after_iterating(self, lambda d: iter(d.viewitems()), dict) + from test import mapping_tests class GeneralMappingTests(mapping_tests.BasicTestMappingProtocol): diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index bd1b32d..4495fa1 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -2,7 +2,8 @@ import unittest from test.test_support import run_unittest, TESTFN, unlink, have_unicode, \ - check_py3k_warnings, cpython_only + check_py3k_warnings, cpython_only, \ + check_free_after_iterating # Test result of triple loop (too big to inline) TRIPLETS = [(0, 0, 0), (0, 0, 1), (0, 0, 2), @@ -921,6 +922,9 @@ class TestCase(unittest.TestCase): lst.extend(gen()) self.assertEqual(len(lst), 760) + def test_free_after_iterating(self): + check_free_after_iterating(self, iter, SequenceClass, (0,)) + def test_main(): run_unittest(TestCase) diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index 78b3e83..85e4841 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -267,6 +267,15 @@ class TestOrderedDict(unittest.TestCase): items = [('a', 1), ('c', 3), ('b', 2)] self.assertEqual(list(MyOD(items).items()), items) + def test_free_after_iterating(self): + test_support.check_free_after_iterating(self, iter, OrderedDict) + test_support.check_free_after_iterating(self, lambda d: d.iterkeys(), OrderedDict) + test_support.check_free_after_iterating(self, lambda d: d.itervalues(), OrderedDict) + test_support.check_free_after_iterating(self, lambda d: d.iteritems(), OrderedDict) + test_support.check_free_after_iterating(self, lambda d: iter(d.viewkeys()), OrderedDict) + test_support.check_free_after_iterating(self, lambda d: iter(d.viewvalues()), OrderedDict) + test_support.check_free_after_iterating(self, lambda d: iter(d.viewitems()), OrderedDict) + class GeneralMappingTests(mapping_tests.BasicTestMappingProtocol): type2test = OrderedDict diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index d9ea098..6baaccf 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -340,6 +340,9 @@ class TestJointOps(unittest.TestCase): gc.collect() self.assertTrue(ref() is None, "Cycle was not collected") + def test_free_after_iterating(self): + test_support.check_free_after_iterating(self, iter, self.thetype) + class TestSet(TestJointOps): thetype = set diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 956936f..85f9965 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -1659,3 +1659,21 @@ def strip_python_stderr(stderr): """ stderr = re.sub(br"\[\d+ refs\]\r?\n?$", b"", stderr).strip() return stderr + + +def check_free_after_iterating(test, iter, cls, args=()): + class A(cls): + def __del__(self): + done[0] = True + try: + next(it) + except StopIteration: + pass + + done = [False] + it = iter(A(*args)) + # Issue 26494: Shouldn't crash + test.assertRaises(StopIteration, next, it) + # The sequence should be deallocated just after the end of iterating + gc_collect() + test.assertTrue(done[0]) diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 63fb831..93224e1 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -1857,6 +1857,11 @@ class UnicodeTest( unicode_encodedecimal(u"123" + s, "xmlcharrefreplace"), '123' + exp) + def test_free_after_iterating(self): + test_support.check_free_after_iterating(self, iter, unicode) + test_support.check_free_after_iterating(self, reversed, unicode) + + def test_main(): test_support.run_unittest(__name__) diff --git a/Misc/NEWS b/Misc/NEWS index 11bc97e..20b0a62 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ What's New in Python 2.7.12? Core and Builtins ----------------- +- Issue #26494: Fixed crash on iterating exhausting iterators. + Affected classes are generic sequence iterators, iterators of bytearray, + list, tuple, set, frozenset, dict, OrderedDict and corresponding views. + - Issue #26581: If coding cookie is specified multiple times on a line in Python source code file, only the first one is taken to account. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 1fdd916..edb67a4 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2982,8 +2982,8 @@ bytearrayiter_next(bytesiterobject *it) return item; } - Py_DECREF(seq); it->it_seq = NULL; + Py_DECREF(seq); return NULL; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index fe19356..e3e4765 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2586,8 +2586,8 @@ static PyObject *dictiter_iternextkey(dictiterobject *di) return key; fail: - Py_DECREF(d); di->di_dict = NULL; + Py_DECREF(d); return NULL; } @@ -2658,8 +2658,8 @@ static PyObject *dictiter_iternextvalue(dictiterobject *di) return value; fail: - Py_DECREF(d); di->di_dict = NULL; + Py_DECREF(d); return NULL; } @@ -2744,8 +2744,8 @@ static PyObject *dictiter_iternextitem(dictiterobject *di) return result; fail: - Py_DECREF(d); di->di_dict = NULL; + Py_DECREF(d); return NULL; } diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 9c90abe..346d2d9 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -69,8 +69,8 @@ iter_iternext(PyObject *iterator) PyErr_ExceptionMatches(PyExc_StopIteration)) { PyErr_Clear(); - Py_DECREF(seq); it->it_seq = NULL; + Py_DECREF(seq); } return NULL; } diff --git a/Objects/listobject.c b/Objects/listobject.c index 27365b6..c414620 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2915,8 +2915,8 @@ listiter_next(listiterobject *it) return item; } - Py_DECREF(seq); it->it_seq = NULL; + Py_DECREF(seq); return NULL; } @@ -3018,9 +3018,17 @@ static PyObject * listreviter_next(listreviterobject *it) { PyObject *item; - Py_ssize_t index = it->it_index; - PyListObject *seq = it->it_seq; + Py_ssize_t index; + PyListObject *seq; + + assert(it != NULL); + seq = it->it_seq; + if (seq == NULL) { + return NULL; + } + assert(PyList_Check(seq)); + index = it->it_index; if (index>=0 && index < PyList_GET_SIZE(seq)) { item = PyList_GET_ITEM(seq, index); it->it_index--; @@ -3028,10 +3036,8 @@ listreviter_next(listreviterobject *it) return item; } it->it_index = -1; - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } + it->it_seq = NULL; + Py_DECREF(seq); return NULL; } diff --git a/Objects/setobject.c b/Objects/setobject.c index 18e6898..3958a6c 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -871,8 +871,8 @@ static PyObject *setiter_iternext(setiterobject *si) return key; fail: - Py_DECREF(so); si->si_set = NULL; + Py_DECREF(so); return NULL; } diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 5d72806..550719f 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -966,8 +966,8 @@ tupleiter_next(tupleiterobject *it) return item; } - Py_DECREF(seq); it->it_seq = NULL; + Py_DECREF(seq); return NULL; } -- cgit v0.12