From 78ae1fb1b94c0f614fd01ba5693aaaead3d4699d Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 14 Jan 2011 20:19:57 +0000 Subject: Merged revisions 87968,87971-87975 via svnmerge from svn+ssh://pythondev@svn.python.org/python/branches/py3k ........ r87968 | antoine.pitrou | 2011-01-12 21:46:37 +0100 (mer., 12 janv. 2011) | 4 lines Fix the expected memory use of utf-8 encoding. Also, release the one reference to a huge object even when an exception is raised. ........ r87971 | antoine.pitrou | 2011-01-12 22:19:59 +0100 (mer., 12 janv. 2011) | 3 lines Make test skipping message nicer, and remove the rather useless "overhead" parameter. ........ r87972 | antoine.pitrou | 2011-01-12 22:40:20 +0100 (mer., 12 janv. 2011) | 3 lines Fix @bigmemtest when no limit is given by the user (oops) ........ r87973 | antoine.pitrou | 2011-01-12 22:50:44 +0100 (mer., 12 janv. 2011) | 3 lines More informative skip message in @bigaddrspace ........ r87974 | antoine.pitrou | 2011-01-12 22:58:39 +0100 (mer., 12 janv. 2011) | 3 lines A better message again ........ r87975 | antoine.pitrou | 2011-01-12 23:02:45 +0100 (mer., 12 janv. 2011) | 3 lines Fix test_bigaddrspace (some tests didn't trigger the expected MemoryError) ........ --- Lib/test/support.py | 42 ++++++++++++-------------- Lib/test/test_bigaddrspace.py | 68 +++++++++++++++++++++++++++---------------- Lib/test/test_bigmem.py | 10 +++++-- 3 files changed, 68 insertions(+), 52 deletions(-) diff --git a/Lib/test/support.py b/Lib/test/support.py index 5dd8017..9a4e1b1 100644 --- a/Lib/test/support.py +++ b/Lib/test/support.py @@ -784,13 +784,12 @@ def set_memlimit(limit): raise ValueError('Memory limit %r too low to be useful' % (limit,)) max_memuse = memlimit -def bigmemtest(minsize, memuse, overhead=5*_1M): +def bigmemtest(minsize, memuse): """Decorator for bigmem tests. 'minsize' is the minimum useful size for the test (in arbitrary, test-interpreted units.) 'memuse' is the number of 'bytes per size' for - the test, or a good estimate of it. 'overhead' specifies fixed overhead, - independent of the testsize, and defaults to 5Mb. + the test, or a good estimate of it. The decorator tries to guess a good value for 'size' and passes it to the decorated test function. If minsize * memuse is more than the @@ -802,52 +801,43 @@ def bigmemtest(minsize, memuse, overhead=5*_1M): # Retrieve values in case someone decided to adjust them minsize = wrapper.minsize memuse = wrapper.memuse - overhead = wrapper.overhead if not max_memuse: # If max_memuse is 0 (the default), # we still want to run the tests with size set to a few kb, # to make sure they work. We still want to avoid using # too much memory, though, but we do that noisily. maxsize = 5147 - self.assertFalse(maxsize * memuse + overhead > 20 * _1M) + self.assertFalse(maxsize * memuse > 20 * _1M) else: - maxsize = int((max_memuse - overhead) / memuse) + maxsize = int(max_memuse / memuse) if maxsize < minsize: - # Really ought to print 'test skipped' or something - if verbose: - sys.stderr.write("Skipping %s because of memory " - "constraint\n" % (f.__name__,)) - return - # Try to keep some breathing room in memory use - maxsize = max(maxsize - 50 * _1M, minsize) + raise unittest.SkipTest( + "not enough memory: %.1fG minimum needed" + % (minsize * memuse / (1024 ** 3))) return f(self, maxsize) wrapper.minsize = minsize wrapper.memuse = memuse - wrapper.overhead = overhead return wrapper return decorator -def precisionbigmemtest(size, memuse, overhead=5*_1M): +def precisionbigmemtest(size, memuse): def decorator(f): def wrapper(self): size = wrapper.size memuse = wrapper.memuse - overhead = wrapper.overhead if not real_max_memuse: maxsize = 5147 else: maxsize = size if real_max_memuse and real_max_memuse < maxsize * memuse: - if verbose: - sys.stderr.write("Skipping %s because of memory " - "constraint\n" % (f.__name__,)) - return + raise unittest.SkipTest( + "not enough memory: %.1fG minimum needed" + % (size * memuse / (1024 ** 3))) return f(self, maxsize) wrapper.size = size wrapper.memuse = memuse - wrapper.overhead = overhead return wrapper return decorator @@ -855,9 +845,13 @@ def bigaddrspacetest(f): """Decorator for tests that fill the address space.""" def wrapper(self): if max_memuse < MAX_Py_ssize_t: - if verbose: - sys.stderr.write("Skipping %s because of memory " - "constraint\n" % (f.__name__,)) + if MAX_Py_ssize_t >= 2**63 - 1 and max_memuse >= 2**31: + raise unittest.SkipTest( + "not enough memory: try a 32-bit build instead") + else: + raise unittest.SkipTest( + "not enough memory: %.1fG minimum needed" + % (MAX_Py_ssize_t / (1024 ** 3))) else: return f(self) return wrapper diff --git a/Lib/test/test_bigaddrspace.py b/Lib/test/test_bigaddrspace.py index d383440..b8c59d4 100644 --- a/Lib/test/test_bigaddrspace.py +++ b/Lib/test/test_bigaddrspace.py @@ -23,25 +23,34 @@ class BytesTest(unittest.TestCase): # Allocate a bytestring that's near the maximum size allowed by # the address space, and then try to build a new, larger one through # concatenation. - x = b"x" * (MAX_Py_ssize_t - 128) - self.assertRaises(OverflowError, operator.add, x, b"x" * 128) + try: + x = b"x" * (MAX_Py_ssize_t - 128) + self.assertRaises(OverflowError, operator.add, x, b"x" * 128) + finally: + x = None @bigaddrspacetest def test_optimized_concat(self): - x = b"x" * (MAX_Py_ssize_t - 128) + try: + x = b"x" * (MAX_Py_ssize_t - 128) - with self.assertRaises(OverflowError) as cm: - # this statement uses a fast path in ceval.c - x = x + b"x" * 128 + with self.assertRaises(OverflowError) as cm: + # this statement used a fast path in ceval.c + x = x + b"x" * 128 - with self.assertRaises(OverflowError) as cm: - # this statement uses a fast path in ceval.c - x += b"x" * 128 + with self.assertRaises(OverflowError) as cm: + # this statement used a fast path in ceval.c + x += b"x" * 128 + finally: + x = None @bigaddrspacetest def test_repeat(self): - x = b"x" * (MAX_Py_ssize_t - 128) - self.assertRaises(OverflowError, operator.mul, x, 128) + try: + x = b"x" * (MAX_Py_ssize_t - 128) + self.assertRaises(OverflowError, operator.mul, x, 128) + finally: + x = None class StrTest(unittest.TestCase): @@ -50,28 +59,37 @@ class StrTest(unittest.TestCase): @bigaddrspacetest def test_concat(self): - # Create a string half the size that would fill the address space - x = "x" * (MAX_Py_ssize_t // (2 * self.unicodesize)) - # Unicode objects trigger MemoryError in case an operation that's - # going to cause a size overflow is executed - self.assertRaises(MemoryError, operator.add, x, x) + try: + # Create a string that would fill almost the address space + x = "x" * int(MAX_Py_ssize_t // (1.1 * self.unicodesize)) + # Unicode objects trigger MemoryError in case an operation that's + # going to cause a size overflow is executed + self.assertRaises(MemoryError, operator.add, x, x) + finally: + x = None @bigaddrspacetest def test_optimized_concat(self): - x = "x" * (MAX_Py_ssize_t // (2 * self.unicodesize)) + try: + x = "x" * int(MAX_Py_ssize_t // (1.1 * self.unicodesize)) - with self.assertRaises(MemoryError) as cm: - # this statement uses a fast path in ceval.c - x = x + x + with self.assertRaises(MemoryError) as cm: + # this statement uses a fast path in ceval.c + x = x + x - with self.assertRaises(MemoryError) as cm: - # this statement uses a fast path in ceval.c - x += x + with self.assertRaises(MemoryError) as cm: + # this statement uses a fast path in ceval.c + x += x + finally: + x = None @bigaddrspacetest def test_repeat(self): - x = "x" * (MAX_Py_ssize_t // (2 * self.unicodesize)) - self.assertRaises(MemoryError, operator.mul, x, 2) + try: + x = "x" * int(MAX_Py_ssize_t // (1.1 * self.unicodesize)) + self.assertRaises(MemoryError, operator.mul, x, 2) + finally: + x = None def test_main(): diff --git a/Lib/test/test_bigmem.py b/Lib/test/test_bigmem.py index 7def22b..aec3778 100644 --- a/Lib/test/test_bigmem.py +++ b/Lib/test/test_bigmem.py @@ -564,8 +564,11 @@ class StrTest(unittest.TestCase, BaseStrTest): if expectedsize is None: expectedsize = size - s = c * size - self.assertEqual(len(s.encode(enc)), expectedsize) + try: + s = c * size + self.assertEqual(len(s.encode(enc)), expectedsize) + finally: + s = None def setUp(self): # HACK: adjust memory use of tests inherited from BaseStrTest @@ -586,7 +589,8 @@ class StrTest(unittest.TestCase, BaseStrTest): for name, memuse in self._adjusted.items(): getattr(type(self), name).memuse = memuse - @bigmemtest(minsize=_2G + 2, memuse=character_size + 1) + # the utf8 encoder preallocates big time (4x the number of characters) + @bigmemtest(minsize=_2G + 2, memuse=character_size + 4) def test_encode(self, size): return self.basic_encode_test(size, 'utf-8') -- cgit v0.12