diff options
author | Antoine Pitrou <solipsis@pitrou.net> | 2011-10-04 08:28:37 (GMT) |
---|---|---|
committer | Antoine Pitrou <solipsis@pitrou.net> | 2011-10-04 08:28:37 (GMT) |
commit | 1a3ff48c55fcb4dd8151750976242d83a84a09b0 (patch) | |
tree | 74f4f275edb2206b64eb2aeaf0be9406814c1b71 | |
parent | 5a688dbf97f1537d3f2309bfe6be93dab2f10ce0 (diff) | |
parent | 94190bb6e7f3b1c7942b562fe0fad3e62b5386b9 (diff) | |
download | cpython-1a3ff48c55fcb4dd8151750976242d83a84a09b0.zip cpython-1a3ff48c55fcb4dd8151750976242d83a84a09b0.tar.gz cpython-1a3ff48c55fcb4dd8151750976242d83a84a09b0.tar.bz2 |
Start fixing test_bigmem:
- bigmemtest is replaced by precisionbigmemtest
- add a poor man's watchdog thread to print memory consumption
-rw-r--r-- | Lib/test/pickletester.py | 12 | ||||
-rw-r--r-- | Lib/test/support.py | 97 | ||||
-rw-r--r-- | Lib/test/test_bigmem.py | 241 | ||||
-rw-r--r-- | Lib/test/test_bz2.py | 6 | ||||
-rw-r--r-- | Lib/test/test_hashlib.py | 6 | ||||
-rw-r--r-- | Lib/test/test_xml_etree_c.py | 4 | ||||
-rw-r--r-- | Lib/test/test_zlib.py | 14 |
7 files changed, 207 insertions, 173 deletions
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index ca46457..061bce9 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -9,7 +9,7 @@ from http.cookies import SimpleCookie from test.support import ( TestFailed, TESTFN, run_with_locale, no_tracing, - _2G, _4G, precisionbigmemtest, + _2G, _4G, bigmemtest, ) from pickle import bytes_types @@ -1188,7 +1188,7 @@ class BigmemPickleTests(unittest.TestCase): # Binary protocols can serialize longs of up to 2GB-1 - @precisionbigmemtest(size=_2G, memuse=1 + 1, dry_run=False) + @bigmemtest(size=_2G, memuse=1 + 1, dry_run=False) def test_huge_long_32b(self, size): data = 1 << (8 * size) try: @@ -1204,7 +1204,7 @@ class BigmemPickleTests(unittest.TestCase): # (older protocols don't have a dedicated opcode for bytes and are # too inefficient) - @precisionbigmemtest(size=_2G, memuse=1 + 1, dry_run=False) + @bigmemtest(size=_2G, memuse=1 + 1, dry_run=False) def test_huge_bytes_32b(self, size): data = b"abcd" * (size // 4) try: @@ -1220,7 +1220,7 @@ class BigmemPickleTests(unittest.TestCase): finally: data = None - @precisionbigmemtest(size=_4G, memuse=1 + 1, dry_run=False) + @bigmemtest(size=_4G, memuse=1 + 1, dry_run=False) def test_huge_bytes_64b(self, size): data = b"a" * size try: @@ -1235,7 +1235,7 @@ class BigmemPickleTests(unittest.TestCase): # All protocols use 1-byte per printable ASCII character; we add another # byte because the encoded form has to be copied into the internal buffer. - @precisionbigmemtest(size=_2G, memuse=2 + character_size, dry_run=False) + @bigmemtest(size=_2G, memuse=2 + character_size, dry_run=False) def test_huge_str_32b(self, size): data = "abcd" * (size // 4) try: @@ -1252,7 +1252,7 @@ class BigmemPickleTests(unittest.TestCase): # BINUNICODE (protocols 1, 2 and 3) cannot carry more than # 2**32 - 1 bytes of utf-8 encoded unicode. - @precisionbigmemtest(size=_4G, memuse=1 + character_size, dry_run=False) + @bigmemtest(size=_4G, memuse=1 + character_size, dry_run=False) def test_huge_str_64b(self, size): data = "a" * size try: diff --git a/Lib/test/support.py b/Lib/test/support.py index 6545d1e..62ad606 100644 --- a/Lib/test/support.py +++ b/Lib/test/support.py @@ -1133,47 +1133,51 @@ def set_memlimit(limit): raise ValueError('Memory limit %r too low to be useful' % (limit,)) max_memuse = memlimit -def bigmemtest(minsize, memuse): +def _memory_watchdog(start_evt, finish_evt, period=10.0): + """A function which periodically watches the process' memory consumption + and prints it out. + """ + # XXX: because of the GIL, and because the very long operations tested + # in most bigmem tests are uninterruptible, the loop below gets woken up + # much less often than expected. + # The polling code should be rewritten in raw C, without holding the GIL, + # and push results onto an anonymous pipe. + try: + page_size = os.sysconf('SC_PAGESIZE') + except (ValueError, AttributeError): + try: + page_size = os.sysconf('SC_PAGE_SIZE') + except (ValueError, AttributeError): + page_size = 4096 + procfile = '/proc/{pid}/statm'.format(pid=os.getpid()) + try: + f = open(procfile, 'rb') + except IOError as e: + warnings.warn('/proc not available for stats: {}'.format(e), + RuntimeWarning) + sys.stderr.flush() + return + with f: + start_evt.set() + old_data = -1 + while not finish_evt.wait(period): + f.seek(0) + statm = f.read().decode('ascii') + data = int(statm.split()[5]) + if data != old_data: + old_data = data + print(" ... process data size: {data:.1f}G" + .format(data=data * page_size / (1024 ** 3))) + +def bigmemtest(size, memuse, dry_run=True): """Decorator for bigmem tests. 'minsize' is the minimum useful size for the test (in arbitrary, test-interpreted units.) 'memuse' is the number of 'bytes per size' for the test, or a good estimate of it. - The decorator tries to guess a good value for 'size' and passes it to - the decorated test function. If minsize * memuse is more than the - allowed memory use (as defined by max_memuse), the test is skipped. - Otherwise, minsize is adjusted upward to use up to max_memuse. - """ - def decorator(f): - def wrapper(self): - # Retrieve values in case someone decided to adjust them - minsize = wrapper.minsize - memuse = wrapper.memuse - if not max_memuse: - # If max_memuse is 0 (the default), - # we still want to run the tests with size set to a few kb, - # to make sure they work. We still want to avoid using - # too much memory, though, but we do that noisily. - maxsize = 5147 - self.assertFalse(maxsize * memuse > 20 * _1M) - else: - maxsize = int(max_memuse / memuse) - if maxsize < minsize: - raise unittest.SkipTest( - "not enough memory: %.1fG minimum needed" - % (minsize * memuse / (1024 ** 3))) - return f(self, maxsize) - wrapper.minsize = minsize - wrapper.memuse = memuse - return wrapper - return decorator - -def precisionbigmemtest(size, memuse, dry_run=True): - """Decorator for bigmem tests that need exact sizes. - - Like bigmemtest, but without the size scaling upward to fill available - memory. + if 'dry_run' is False, it means the test doesn't support dummy runs + when -M is not specified. """ def decorator(f): def wrapper(self): @@ -1190,7 +1194,28 @@ def precisionbigmemtest(size, memuse, dry_run=True): "not enough memory: %.1fG minimum needed" % (size * memuse / (1024 ** 3))) - return f(self, maxsize) + if real_max_memuse and verbose and threading: + print() + print(" ... expected peak memory use: {peak:.1f}G" + .format(peak=size * memuse / (1024 ** 3))) + sys.stdout.flush() + start_evt = threading.Event() + finish_evt = threading.Event() + t = threading.Thread(target=_memory_watchdog, + args=(start_evt, finish_evt, 0.5)) + t.daemon = True + t.start() + start_evt.set() + else: + t = None + + try: + return f(self, maxsize) + finally: + if t: + finish_evt.set() + t.join() + wrapper.size = size wrapper.memuse = memuse return wrapper diff --git a/Lib/test/test_bigmem.py b/Lib/test/test_bigmem.py index 91c62af..e2cf45d 100644 --- a/Lib/test/test_bigmem.py +++ b/Lib/test/test_bigmem.py @@ -9,7 +9,7 @@ high memory limit to regrtest, with the -M option. """ from test import support -from test.support import bigmemtest, _1G, _2G, _4G, precisionbigmemtest +from test.support import bigmemtest, _1G, _2G, _4G import unittest import operator @@ -50,11 +50,11 @@ import functools # a large object, make the subobject of a length that is not a power of # 2. That way, int-wrapping problems are more easily detected. # -# - While the bigmem decorators speak of 'minsize', all tests will actually -# be called with a much smaller number too, in the normal test run (5Kb -# currently.) This is so the tests themselves get frequent testing. -# Consequently, always make all large allocations based on the passed-in -# 'size', and don't rely on the size being very large. Also, +# - Despite the bigmemtest decorator, all tests will actually be called +# with a much smaller number too, in the normal test run (5Kb currently.) +# This is so the tests themselves get frequent testing. +# Consequently, always make all large allocations based on the +# passed-in 'size', and don't rely on the size being very large. Also, # memuse-per-size should remain sane (less than a few thousand); if your # test uses more, adjust 'size' upward, instead. @@ -67,7 +67,7 @@ character_size = 4 if sys.maxunicode > 0xFFFF else 2 class BaseStrTest: - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_capitalize(self, size): _ = self.from_latin1 SUBSTR = self.from_latin1(' abc def ghi') @@ -77,7 +77,7 @@ class BaseStrTest: SUBSTR.capitalize()) self.assertEqual(caps.lstrip(_('-')), SUBSTR) - @bigmemtest(minsize=_2G + 10, memuse=1) + @bigmemtest(size=_2G + 10, memuse=1) def test_center(self, size): SUBSTR = self.from_latin1(' abc def ghi') s = SUBSTR.center(size) @@ -88,7 +88,7 @@ class BaseStrTest: self.assertEqual(s[lpadsize:-rpadsize], SUBSTR) self.assertEqual(s.strip(), SUBSTR.strip()) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_count(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi') @@ -100,7 +100,7 @@ class BaseStrTest: self.assertEqual(s.count(_('i')), 1) self.assertEqual(s.count(_('j')), 0) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_endswith(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi') @@ -112,7 +112,7 @@ class BaseStrTest: self.assertFalse(s.endswith(_('a') + SUBSTR)) self.assertFalse(SUBSTR.endswith(s)) - @bigmemtest(minsize=_2G + 10, memuse=2) + @bigmemtest(size=_2G + 10, memuse=2) def test_expandtabs(self, size): _ = self.from_latin1 s = _('-') * size @@ -125,7 +125,7 @@ class BaseStrTest: self.assertEqual(len(s), size - remainder) self.assertEqual(len(s.strip(_(' '))), 0) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_find(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi') @@ -142,7 +142,7 @@ class BaseStrTest: sublen + size + SUBSTR.find(_('i'))) self.assertEqual(s.find(_('j')), -1) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_index(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi') @@ -159,7 +159,7 @@ class BaseStrTest: sublen + size + SUBSTR.index(_('i'))) self.assertRaises(ValueError, s.index, _('j')) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_isalnum(self, size): _ = self.from_latin1 SUBSTR = _('123456') @@ -168,7 +168,7 @@ class BaseStrTest: s += _('.') self.assertFalse(s.isalnum()) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_isalpha(self, size): _ = self.from_latin1 SUBSTR = _('zzzzzzz') @@ -177,7 +177,7 @@ class BaseStrTest: s += _('.') self.assertFalse(s.isalpha()) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_isdigit(self, size): _ = self.from_latin1 SUBSTR = _('123456') @@ -186,7 +186,7 @@ class BaseStrTest: s += _('z') self.assertFalse(s.isdigit()) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_islower(self, size): _ = self.from_latin1 chars = _(''.join( @@ -197,7 +197,7 @@ class BaseStrTest: s += _('A') self.assertFalse(s.islower()) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_isspace(self, size): _ = self.from_latin1 whitespace = _(' \f\n\r\t\v') @@ -207,7 +207,7 @@ class BaseStrTest: s += _('j') self.assertFalse(s.isspace()) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_istitle(self, size): _ = self.from_latin1 SUBSTR = _('123456') @@ -218,7 +218,7 @@ class BaseStrTest: s += _('aA') self.assertFalse(s.istitle()) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_isupper(self, size): _ = self.from_latin1 chars = _(''.join( @@ -229,7 +229,7 @@ class BaseStrTest: s += _('a') self.assertFalse(s.isupper()) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_join(self, size): _ = self.from_latin1 s = _('A') * size @@ -239,7 +239,7 @@ class BaseStrTest: self.assertTrue(x.startswith(_('aaaaaA'))) self.assertTrue(x.endswith(_('Abbbbb'))) - @bigmemtest(minsize=_2G + 10, memuse=1) + @bigmemtest(size=_2G + 10, memuse=1) def test_ljust(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi') @@ -248,7 +248,7 @@ class BaseStrTest: self.assertEqual(len(s), size) self.assertEqual(s.strip(), SUBSTR.strip()) - @bigmemtest(minsize=_2G + 10, memuse=2) + @bigmemtest(size=_2G + 10, memuse=2) def test_lower(self, size): _ = self.from_latin1 s = _('A') * size @@ -256,7 +256,7 @@ class BaseStrTest: self.assertEqual(len(s), size) self.assertEqual(s.count(_('a')), size) - @bigmemtest(minsize=_2G + 10, memuse=1) + @bigmemtest(size=_2G + 10, memuse=1) def test_lstrip(self, size): _ = self.from_latin1 SUBSTR = _('abc def ghi') @@ -271,7 +271,7 @@ class BaseStrTest: stripped = s.lstrip() self.assertTrue(stripped is s) - @bigmemtest(minsize=_2G + 10, memuse=2) + @bigmemtest(size=_2G + 10, memuse=2) def test_replace(self, size): _ = self.from_latin1 replacement = _('a') @@ -284,7 +284,7 @@ class BaseStrTest: self.assertEqual(s.count(replacement), 4) self.assertEqual(s[-10:], _(' aaaa')) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_rfind(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi') @@ -300,7 +300,7 @@ class BaseStrTest: SUBSTR.rfind(_('i'))) self.assertEqual(s.rfind(_('j')), -1) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_rindex(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi') @@ -319,7 +319,7 @@ class BaseStrTest: SUBSTR.rindex(_('i'))) self.assertRaises(ValueError, s.rindex, _('j')) - @bigmemtest(minsize=_2G + 10, memuse=1) + @bigmemtest(size=_2G + 10, memuse=1) def test_rjust(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi') @@ -328,7 +328,7 @@ class BaseStrTest: self.assertEqual(len(s), size) self.assertEqual(s.strip(), SUBSTR.strip()) - @bigmemtest(minsize=_2G + 10, memuse=1) + @bigmemtest(size=_2G + 10, memuse=1) def test_rstrip(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi') @@ -346,7 +346,7 @@ class BaseStrTest: # The test takes about size bytes to build a string, and then about # sqrt(size) substrings of sqrt(size) in size and a list to # hold sqrt(size) items. It's close but just over 2x size. - @bigmemtest(minsize=_2G, memuse=2.1) + @bigmemtest(size=_2G, memuse=2.1) def test_split_small(self, size): _ = self.from_latin1 # Crudely calculate an estimate so that the result of s.split won't @@ -372,7 +372,7 @@ class BaseStrTest: # suffer for the list size. (Otherwise, it'd cost another 48 times # size in bytes!) Nevertheless, a list of size takes # 8*size bytes. - @bigmemtest(minsize=_2G + 5, memuse=10) + @bigmemtest(size=_2G + 5, memuse=10) def test_split_large(self, size): _ = self.from_latin1 s = _(' a') * size + _(' ') @@ -384,7 +384,7 @@ class BaseStrTest: self.assertEqual(len(l), size + 1) self.assertEqual(set(l), set([_(' ')])) - @bigmemtest(minsize=_2G, memuse=2.1) + @bigmemtest(size=_2G, memuse=2.1) def test_splitlines(self, size): _ = self.from_latin1 # Crudely calculate an estimate so that the result of s.split won't @@ -398,7 +398,7 @@ class BaseStrTest: for item in l: self.assertEqual(item, expected) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_startswith(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi') @@ -407,7 +407,7 @@ class BaseStrTest: self.assertTrue(s.startswith(_('-') * size)) self.assertFalse(s.startswith(SUBSTR)) - @bigmemtest(minsize=_2G, memuse=1) + @bigmemtest(size=_2G, memuse=1) def test_strip(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi ') @@ -419,7 +419,7 @@ class BaseStrTest: self.assertEqual(len(s), size) self.assertEqual(s.strip(), SUBSTR.strip()) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_swapcase(self, size): _ = self.from_latin1 SUBSTR = _("aBcDeFG12.'\xa9\x00") @@ -431,7 +431,7 @@ class BaseStrTest: self.assertEqual(s[:sublen * 3], SUBSTR.swapcase() * 3) self.assertEqual(s[-sublen * 3:], SUBSTR.swapcase() * 3) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_title(self, size): _ = self.from_latin1 SUBSTR = _('SpaaHAaaAaham') @@ -440,7 +440,7 @@ class BaseStrTest: self.assertTrue(s.startswith((SUBSTR * 3).title())) self.assertTrue(s.endswith(SUBSTR.lower() * 3)) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_translate(self, size): _ = self.from_latin1 SUBSTR = _('aZz.z.Aaz.') @@ -463,7 +463,7 @@ class BaseStrTest: self.assertEqual(s.count(_('!')), repeats * 2) self.assertEqual(s.count(_('z')), repeats * 3) - @bigmemtest(minsize=_2G + 5, memuse=2) + @bigmemtest(size=_2G + 5, memuse=2) def test_upper(self, size): _ = self.from_latin1 s = _('a') * size @@ -471,7 +471,7 @@ class BaseStrTest: self.assertEqual(len(s), size) self.assertEqual(s.count(_('A')), size) - @bigmemtest(minsize=_2G + 20, memuse=1) + @bigmemtest(size=_2G + 20, memuse=1) def test_zfill(self, size): _ = self.from_latin1 SUBSTR = _('-568324723598234') @@ -483,7 +483,7 @@ class BaseStrTest: # This test is meaningful even with size < 2G, as long as the # doubled string is > 2G (but it tests more if both are > 2G :) - @bigmemtest(minsize=_1G + 2, memuse=3) + @bigmemtest(size=_1G + 2, memuse=3) def test_concat(self, size): _ = self.from_latin1 s = _('.') * size @@ -494,7 +494,7 @@ class BaseStrTest: # This test is meaningful even with size < 2G, as long as the # repeated string is > 2G (but it tests more if both are > 2G :) - @bigmemtest(minsize=_1G + 2, memuse=3) + @bigmemtest(size=_1G + 2, memuse=3) def test_repeat(self, size): _ = self.from_latin1 s = _('.') * size @@ -503,7 +503,7 @@ class BaseStrTest: self.assertEqual(len(s), size * 2) self.assertEqual(s.count(_('.')), size * 2) - @bigmemtest(minsize=_2G + 20, memuse=2) + @bigmemtest(size=_2G + 20, memuse=2) def test_slice_and_getitem(self, size): _ = self.from_latin1 SUBSTR = _('0123456789') @@ -537,7 +537,7 @@ class BaseStrTest: self.assertRaises(IndexError, operator.getitem, s, len(s) + 1) self.assertRaises(IndexError, operator.getitem, s, len(s) + 1<<31) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_contains(self, size): _ = self.from_latin1 SUBSTR = _('0123456789') @@ -551,7 +551,7 @@ class BaseStrTest: s += _('a') self.assertTrue(_('a') in s) - @bigmemtest(minsize=_2G + 10, memuse=2) + @bigmemtest(size=_2G + 10, memuse=2) def test_compare(self, size): _ = self.from_latin1 s1 = _('-') * size @@ -564,7 +564,7 @@ class BaseStrTest: s2 = _('.') * size self.assertFalse(s1 == s2) - @bigmemtest(minsize=_2G + 10, memuse=1) + @bigmemtest(size=_2G + 10, memuse=1) def test_hash(self, size): # Not sure if we can do any meaningful tests here... Even if we # start relying on the exact algorithm used, the result will be @@ -615,46 +615,36 @@ class StrTest(unittest.TestCase, BaseStrTest): getattr(type(self), name).memuse = memuse # the utf8 encoder preallocates big time (4x the number of characters) - @bigmemtest(minsize=_2G + 2, memuse=character_size + 4) + @bigmemtest(size=_2G + 2, memuse=character_size + 4) def test_encode(self, size): return self.basic_encode_test(size, 'utf-8') - @precisionbigmemtest(size=_4G // 6 + 2, memuse=character_size + 1) + @bigmemtest(size=_4G // 6 + 2, memuse=character_size + 1) def test_encode_raw_unicode_escape(self, size): try: return self.basic_encode_test(size, 'raw_unicode_escape') except MemoryError: pass # acceptable on 32-bit - @precisionbigmemtest(size=_4G // 5 + 70, memuse=character_size + 1) + @bigmemtest(size=_4G // 5 + 70, memuse=character_size + 1) def test_encode_utf7(self, size): try: return self.basic_encode_test(size, 'utf7') except MemoryError: pass # acceptable on 32-bit - @precisionbigmemtest(size=_4G // 4 + 5, memuse=character_size + 4) + @bigmemtest(size=_4G // 4 + 5, memuse=character_size + 4) def test_encode_utf32(self, size): try: return self.basic_encode_test(size, 'utf32', expectedsize=4*size+4) except MemoryError: pass # acceptable on 32-bit - @precisionbigmemtest(size=_2G - 1, memuse=character_size + 1) + @bigmemtest(size=_2G - 1, memuse=character_size + 1) def test_encode_ascii(self, size): return self.basic_encode_test(size, 'ascii', c='A') - @precisionbigmemtest(size=_4G // 5, memuse=character_size * (6 + 1)) - def test_unicode_repr_overflow(self, size): - try: - s = "\uDCBA"*size - r = repr(s) - except MemoryError: - pass # acceptable on 32-bit - else: - self.assertTrue(s == eval(r)) - - @bigmemtest(minsize=_2G + 10, memuse=character_size * 2) + @bigmemtest(size=_2G + 10, memuse=character_size * 2) def test_format(self, size): s = '-' * size sf = '%s' % (s,) @@ -675,7 +665,7 @@ class StrTest(unittest.TestCase, BaseStrTest): self.assertEqual(s.count('.'), 3) self.assertEqual(s.count('-'), size * 2) - @bigmemtest(minsize=_2G + 10, memuse=character_size * 2) + @bigmemtest(size=_2G + 10, memuse=character_size * 2) def test_repr_small(self, size): s = '-' * size s = repr(s) @@ -696,7 +686,7 @@ class StrTest(unittest.TestCase, BaseStrTest): self.assertEqual(s.count('\\'), size) self.assertEqual(s.count('0'), size * 2) - @bigmemtest(minsize=_2G + 10, memuse=character_size * 5) + @bigmemtest(size=_2G + 10, memuse=character_size * 5) def test_repr_large(self, size): s = '\x00' * size s = repr(s) @@ -706,27 +696,46 @@ class StrTest(unittest.TestCase, BaseStrTest): self.assertEqual(s.count('\\'), size) self.assertEqual(s.count('0'), size * 2) - @bigmemtest(minsize=2**32 / 5, memuse=character_size * 7) + @bigmemtest(size=_2G // 5 + 1, memuse=character_size * 7) def test_unicode_repr(self, size): # Use an assigned, but not printable code point. # It is in the range of the low surrogates \uDC00-\uDFFF. - s = "\uDCBA" * size - for f in (repr, ascii): - r = f(s) - self.assertTrue(len(r) > size) - self.assertTrue(r.endswith(r"\udcba'"), r[-10:]) - del r + char = "\uDCBA" + s = char * size + try: + for f in (repr, ascii): + r = f(s) + self.assertEqual(len(r), 2 + (len(f(char)) - 2) * size) + self.assertTrue(r.endswith(r"\udcba'"), r[-10:]) + r = None + finally: + r = s = None # The character takes 4 bytes even in UCS-2 builds because it will # be decomposed into surrogates. - @bigmemtest(minsize=2**32 / 5, memuse=4 + character_size * 9) + @bigmemtest(size=_2G // 5 + 1, memuse=4 + character_size * 9) def test_unicode_repr_wide(self, size): - s = "\U0001DCBA" * size - for f in (repr, ascii): - r = f(s) - self.assertTrue(len(r) > size) - self.assertTrue(r.endswith(r"\U0001dcba'"), r[-12:]) - del r + char = "\U0001DCBA" + s = char * size + try: + for f in (repr, ascii): + r = f(s) + self.assertEqual(len(r), 2 + (len(f(char)) - 2) * size) + self.assertTrue(r.endswith(r"\U0001dcba'"), r[-12:]) + r = None + finally: + r = s = None + + @bigmemtest(size=_4G // 5, memuse=character_size * (6 + 1)) + def _test_unicode_repr_overflow(self, size): + # XXX not sure what this test is about + char = "\uDCBA" + s = char * size + try: + r = repr(s) + self.assertTrue(s == eval(r)) + finally: + r = s = None class BytesTest(unittest.TestCase, BaseStrTest): @@ -734,7 +743,7 @@ class BytesTest(unittest.TestCase, BaseStrTest): def from_latin1(self, s): return s.encode("latin-1") - @bigmemtest(minsize=_2G + 2, memuse=1 + character_size) + @bigmemtest(size=_2G + 2, memuse=1 + character_size) def test_decode(self, size): s = self.from_latin1('.') * size self.assertEqual(len(s.decode('utf-8')), size) @@ -745,7 +754,7 @@ class BytearrayTest(unittest.TestCase, BaseStrTest): def from_latin1(self, s): return bytearray(s.encode("latin-1")) - @bigmemtest(minsize=_2G + 2, memuse=1 + character_size) + @bigmemtest(size=_2G + 2, memuse=1 + character_size) def test_decode(self, size): s = self.from_latin1('.') * size self.assertEqual(len(s.decode('utf-8')), size) @@ -764,7 +773,7 @@ class TupleTest(unittest.TestCase): # having more than 2<<31 references to any given object. Hence the # use of different types of objects as contents in different tests. - @bigmemtest(minsize=_2G + 2, memuse=16) + @bigmemtest(size=_2G + 2, memuse=16) def test_compare(self, size): t1 = ('',) * size t2 = ('',) * size @@ -787,15 +796,15 @@ class TupleTest(unittest.TestCase): t = t + t self.assertEqual(len(t), size * 2) - @bigmemtest(minsize=_2G // 2 + 2, memuse=24) + @bigmemtest(size=_2G // 2 + 2, memuse=24) def test_concat_small(self, size): return self.basic_concat_test(size) - @bigmemtest(minsize=_2G + 2, memuse=24) + @bigmemtest(size=_2G + 2, memuse=24) def test_concat_large(self, size): return self.basic_concat_test(size) - @bigmemtest(minsize=_2G // 5 + 10, memuse=8 * 5) + @bigmemtest(size=_2G // 5 + 10, memuse=8 * 5) def test_contains(self, size): t = (1, 2, 3, 4, 5) * size self.assertEqual(len(t), size * 5) @@ -803,7 +812,7 @@ class TupleTest(unittest.TestCase): self.assertFalse((1, 2, 3, 4, 5) in t) self.assertFalse(0 in t) - @bigmemtest(minsize=_2G + 10, memuse=8) + @bigmemtest(size=_2G + 10, memuse=8) def test_hash(self, size): t1 = (0,) * size h1 = hash(t1) @@ -811,7 +820,7 @@ class TupleTest(unittest.TestCase): t2 = (0,) * (size + 1) self.assertFalse(h1 == hash(t2)) - @bigmemtest(minsize=_2G + 10, memuse=8) + @bigmemtest(size=_2G + 10, memuse=8) def test_index_and_slice(self, size): t = (None,) * size self.assertEqual(len(t), size) @@ -836,19 +845,19 @@ class TupleTest(unittest.TestCase): t = t * 2 self.assertEqual(len(t), size * 2) - @bigmemtest(minsize=_2G // 2 + 2, memuse=24) + @bigmemtest(size=_2G // 2 + 2, memuse=24) def test_repeat_small(self, size): return self.basic_test_repeat(size) - @bigmemtest(minsize=_2G + 2, memuse=24) + @bigmemtest(size=_2G + 2, memuse=24) def test_repeat_large(self, size): return self.basic_test_repeat(size) - @bigmemtest(minsize=_1G - 1, memuse=12) + @bigmemtest(size=_1G - 1, memuse=12) def test_repeat_large_2(self, size): return self.basic_test_repeat(size) - @precisionbigmemtest(size=_1G - 1, memuse=9) + @bigmemtest(size=_1G - 1, memuse=9) def test_from_2G_generator(self, size): self.skipTest("test needs much more memory than advertised, see issue5438") try: @@ -862,7 +871,7 @@ class TupleTest(unittest.TestCase): count += 1 self.assertEqual(count, size) - @precisionbigmemtest(size=_1G - 25, memuse=9) + @bigmemtest(size=_1G - 25, memuse=9) def test_from_almost_2G_generator(self, size): self.skipTest("test needs much more memory than advertised, see issue5438") try: @@ -885,11 +894,11 @@ class TupleTest(unittest.TestCase): self.assertEqual(s[-5:], '0, 0)') self.assertEqual(s.count('0'), size) - @bigmemtest(minsize=_2G // 3 + 2, memuse=8 + 3 * character_size) + @bigmemtest(size=_2G // 3 + 2, memuse=8 + 3 * character_size) def test_repr_small(self, size): return self.basic_test_repr(size) - @bigmemtest(minsize=_2G + 2, memuse=8 + 3 * character_size) + @bigmemtest(size=_2G + 2, memuse=8 + 3 * character_size) def test_repr_large(self, size): return self.basic_test_repr(size) @@ -900,7 +909,7 @@ class ListTest(unittest.TestCase): # lists hold references to various objects to test their refcount # limits. - @bigmemtest(minsize=_2G + 2, memuse=16) + @bigmemtest(size=_2G + 2, memuse=16) def test_compare(self, size): l1 = [''] * size l2 = [''] * size @@ -923,11 +932,11 @@ class ListTest(unittest.TestCase): l = l + l self.assertEqual(len(l), size * 2) - @bigmemtest(minsize=_2G // 2 + 2, memuse=24) + @bigmemtest(size=_2G // 2 + 2, memuse=24) def test_concat_small(self, size): return self.basic_test_concat(size) - @bigmemtest(minsize=_2G + 2, memuse=24) + @bigmemtest(size=_2G + 2, memuse=24) def test_concat_large(self, size): return self.basic_test_concat(size) @@ -938,15 +947,15 @@ class ListTest(unittest.TestCase): self.assertTrue(l[0] is l[-1]) self.assertTrue(l[size - 1] is l[size + 1]) - @bigmemtest(minsize=_2G // 2 + 2, memuse=24) + @bigmemtest(size=_2G // 2 + 2, memuse=24) def test_inplace_concat_small(self, size): return self.basic_test_inplace_concat(size) - @bigmemtest(minsize=_2G + 2, memuse=24) + @bigmemtest(size=_2G + 2, memuse=24) def test_inplace_concat_large(self, size): return self.basic_test_inplace_concat(size) - @bigmemtest(minsize=_2G // 5 + 10, memuse=8 * 5) + @bigmemtest(size=_2G // 5 + 10, memuse=8 * 5) def test_contains(self, size): l = [1, 2, 3, 4, 5] * size self.assertEqual(len(l), size * 5) @@ -954,12 +963,12 @@ class ListTest(unittest.TestCase): self.assertFalse([1, 2, 3, 4, 5] in l) self.assertFalse(0 in l) - @bigmemtest(minsize=_2G + 10, memuse=8) + @bigmemtest(size=_2G + 10, memuse=8) def test_hash(self, size): l = [0] * size self.assertRaises(TypeError, hash, l) - @bigmemtest(minsize=_2G + 10, memuse=8) + @bigmemtest(size=_2G + 10, memuse=8) def test_index_and_slice(self, size): l = [None] * size self.assertEqual(len(l), size) @@ -1023,11 +1032,11 @@ class ListTest(unittest.TestCase): l = l * 2 self.assertEqual(len(l), size * 2) - @bigmemtest(minsize=_2G // 2 + 2, memuse=24) + @bigmemtest(size=_2G // 2 + 2, memuse=24) def test_repeat_small(self, size): return self.basic_test_repeat(size) - @bigmemtest(minsize=_2G + 2, memuse=24) + @bigmemtest(size=_2G + 2, memuse=24) def test_repeat_large(self, size): return self.basic_test_repeat(size) @@ -1043,11 +1052,11 @@ class ListTest(unittest.TestCase): self.assertEqual(len(l), size * 2) self.assertTrue(l[size - 1] is l[-1]) - @bigmemtest(minsize=_2G // 2 + 2, memuse=16) + @bigmemtest(size=_2G // 2 + 2, memuse=16) def test_inplace_repeat_small(self, size): return self.basic_test_inplace_repeat(size) - @bigmemtest(minsize=_2G + 2, memuse=16) + @bigmemtest(size=_2G + 2, memuse=16) def test_inplace_repeat_large(self, size): return self.basic_test_inplace_repeat(size) @@ -1060,17 +1069,17 @@ class ListTest(unittest.TestCase): self.assertEqual(s[-5:], '0, 0]') self.assertEqual(s.count('0'), size) - @bigmemtest(minsize=_2G // 3 + 2, memuse=8 + 3 * character_size) + @bigmemtest(size=_2G // 3 + 2, memuse=8 + 3 * character_size) def test_repr_small(self, size): return self.basic_test_repr(size) - @bigmemtest(minsize=_2G + 2, memuse=8 + 3 * character_size) + @bigmemtest(size=_2G + 2, memuse=8 + 3 * character_size) def test_repr_large(self, size): return self.basic_test_repr(size) # list overallocates ~1/8th of the total size (on first expansion) so # the single list.append call puts memuse at 9 bytes per size. - @bigmemtest(minsize=_2G, memuse=9) + @bigmemtest(size=_2G, memuse=9) def test_append(self, size): l = [object()] * size l.append(object()) @@ -1078,7 +1087,7 @@ class ListTest(unittest.TestCase): self.assertTrue(l[-3] is l[-2]) self.assertFalse(l[-2] is l[-1]) - @bigmemtest(minsize=_2G // 5 + 2, memuse=8 * 5) + @bigmemtest(size=_2G // 5 + 2, memuse=8 * 5) def test_count(self, size): l = [1, 2, 3, 4, 5] * size self.assertEqual(l.count(1), size) @@ -1091,15 +1100,15 @@ class ListTest(unittest.TestCase): self.assertTrue(l[0] is l[-1]) self.assertTrue(l[size - 1] is l[size + 1]) - @bigmemtest(minsize=_2G // 2 + 2, memuse=16) + @bigmemtest(size=_2G // 2 + 2, memuse=16) def test_extend_small(self, size): return self.basic_test_extend(size) - @bigmemtest(minsize=_2G + 2, memuse=16) + @bigmemtest(size=_2G + 2, memuse=16) def test_extend_large(self, size): return self.basic_test_extend(size) - @bigmemtest(minsize=_2G // 5 + 2, memuse=8 * 5) + @bigmemtest(size=_2G // 5 + 2, memuse=8 * 5) def test_index(self, size): l = [1, 2, 3, 4, 5] * size size *= 5 @@ -1110,7 +1119,7 @@ class ListTest(unittest.TestCase): self.assertRaises(ValueError, l.index, 6) # This tests suffers from overallocation, just like test_append. - @bigmemtest(minsize=_2G + 10, memuse=9) + @bigmemtest(size=_2G + 10, memuse=9) def test_insert(self, size): l = [1.0] * size l.insert(size - 1, "A") @@ -1129,7 +1138,7 @@ class ListTest(unittest.TestCase): self.assertEqual(l[:3], [1.0, "C", 1.0]) self.assertEqual(l[size - 3:], ["A", 1.0, "B"]) - @bigmemtest(minsize=_2G // 5 + 4, memuse=8 * 5) + @bigmemtest(size=_2G // 5 + 4, memuse=8 * 5) def test_pop(self, size): l = ["a", "b", "c", "d", "e"] * size size *= 5 @@ -1153,7 +1162,7 @@ class ListTest(unittest.TestCase): self.assertEqual(item, "c") self.assertEqual(l[-2:], ["b", "d"]) - @bigmemtest(minsize=_2G + 10, memuse=8) + @bigmemtest(size=_2G + 10, memuse=8) def test_remove(self, size): l = [10] * size self.assertEqual(len(l), size) @@ -1173,7 +1182,7 @@ class ListTest(unittest.TestCase): self.assertEqual(len(l), size) self.assertEqual(l[-2:], [10, 10]) - @bigmemtest(minsize=_2G // 5 + 2, memuse=8 * 5) + @bigmemtest(size=_2G // 5 + 2, memuse=8 * 5) def test_reverse(self, size): l = [1, 2, 3, 4, 5] * size l.reverse() @@ -1181,7 +1190,7 @@ class ListTest(unittest.TestCase): self.assertEqual(l[-5:], [5, 4, 3, 2, 1]) self.assertEqual(l[:5], [5, 4, 3, 2, 1]) - @bigmemtest(minsize=_2G // 5 + 2, memuse=8 * 5) + @bigmemtest(size=_2G // 5 + 2, memuse=8 * 5) def test_sort(self, size): l = [1, 2, 3, 4, 5] * size l.sort() diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index d003238..c324fb1 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 from test import support -from test.support import TESTFN, precisionbigmemtest, _4G +from test.support import TESTFN, bigmemtest, _4G import unittest from io import BytesIO @@ -497,7 +497,7 @@ class BZ2CompressorTest(BaseTest): data += bz2c.flush() self.assertEqual(self.decompress(data), self.TEXT) - @precisionbigmemtest(size=_4G + 100, memuse=2) + @bigmemtest(size=_4G + 100, memuse=2) def testCompress4G(self, size): # "Test BZ2Compressor.compress()/flush() with >4GiB input" bz2c = BZ2Compressor() @@ -548,7 +548,7 @@ class BZ2DecompressorTest(BaseTest): text = bz2d.decompress(self.DATA) self.assertRaises(EOFError, bz2d.decompress, b"anything") - @precisionbigmemtest(size=_4G + 100, memuse=3) + @bigmemtest(size=_4G + 100, memuse=3) def testDecompress4G(self, size): # "Test BZ2Decompressor.decompress() with >4GiB input" blocksize = 10 * 1024 * 1024 diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 17d752b..97981dd 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -17,7 +17,7 @@ except ImportError: import unittest import warnings from test import support -from test.support import _4G, precisionbigmemtest +from test.support import _4G, bigmemtest # Were we compiled --with-pydebug or with #define Py_DEBUG? COMPILED_WITH_PYDEBUG = hasattr(sys, 'gettotalrefcount') @@ -196,7 +196,7 @@ class HashLibTestCase(unittest.TestCase): b'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789', 'd174ab98d277d9f5a5611c2c9f419d9f') - @precisionbigmemtest(size=_4G + 5, memuse=1) + @bigmemtest(size=_4G + 5, memuse=1) def test_case_md5_huge(self, size): if size == _4G + 5: try: @@ -204,7 +204,7 @@ class HashLibTestCase(unittest.TestCase): except OverflowError: pass # 32-bit arch - @precisionbigmemtest(size=_4G - 1, memuse=1) + @bigmemtest(size=_4G - 1, memuse=1) def test_case_md5_uintmax(self, size): if size == _4G - 1: try: diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py index db69e5f..2ff118f 100644 --- a/Lib/test/test_xml_etree_c.py +++ b/Lib/test/test_xml_etree_c.py @@ -1,7 +1,7 @@ # xml.etree test for cElementTree from test import support -from test.support import precisionbigmemtest, _2G +from test.support import bigmemtest, _2G import unittest cET = support.import_module('xml.etree.cElementTree') @@ -35,7 +35,7 @@ def sanity(): class MiscTests(unittest.TestCase): # Issue #8651. - @support.precisionbigmemtest(size=support._2G + 100, memuse=1) + @support.bigmemtest(size=support._2G + 100, memuse=1) def test_length_overflow(self, size): if size < support._2G + 100: self.skipTest("not enough free memory, need at least 2 GB") diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 8d137ac..51a9d4b 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -3,7 +3,7 @@ from test import support import binascii import random import sys -from test.support import precisionbigmemtest, _1G, _4G +from test.support import bigmemtest, _1G, _4G zlib = support.import_module('zlib') @@ -188,16 +188,16 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase): # Memory use of the following functions takes into account overallocation - @precisionbigmemtest(size=_1G + 1024 * 1024, memuse=3) + @bigmemtest(size=_1G + 1024 * 1024, memuse=3) def test_big_compress_buffer(self, size): compress = lambda s: zlib.compress(s, 1) self.check_big_compress_buffer(size, compress) - @precisionbigmemtest(size=_1G + 1024 * 1024, memuse=2) + @bigmemtest(size=_1G + 1024 * 1024, memuse=2) def test_big_decompress_buffer(self, size): self.check_big_decompress_buffer(size, zlib.decompress) - @precisionbigmemtest(size=_4G + 100, memuse=1) + @bigmemtest(size=_4G + 100, memuse=1) def test_length_overflow(self, size): if size < _4G + 100: self.skipTest("not enough free memory, need at least 4 GB") @@ -542,19 +542,19 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): # Memory use of the following functions takes into account overallocation - @precisionbigmemtest(size=_1G + 1024 * 1024, memuse=3) + @bigmemtest(size=_1G + 1024 * 1024, memuse=3) def test_big_compress_buffer(self, size): c = zlib.compressobj(1) compress = lambda s: c.compress(s) + c.flush() self.check_big_compress_buffer(size, compress) - @precisionbigmemtest(size=_1G + 1024 * 1024, memuse=2) + @bigmemtest(size=_1G + 1024 * 1024, memuse=2) def test_big_decompress_buffer(self, size): d = zlib.decompressobj() decompress = lambda s: d.decompress(s) + d.flush() self.check_big_decompress_buffer(size, decompress) - @precisionbigmemtest(size=_4G + 100, memuse=1) + @bigmemtest(size=_4G + 100, memuse=1) def test_length_overflow(self, size): if size < _4G + 100: self.skipTest("not enough free memory, need at least 4 GB") |