diff options
| -rw-r--r-- | Lib/test/pickletester.py | 12 | ||||
| -rw-r--r-- | Lib/test/support.py | 92 | ||||
| -rw-r--r-- | Lib/test/test_bigmem.py | 239 | ||||
| -rw-r--r-- | Lib/test/test_hashlib.py | 6 | ||||
| -rw-r--r-- | Lib/test/test_xml_etree_c.py | 4 | ||||
| -rw-r--r-- | Lib/test/test_zlib.py | 14 |
6 files changed, 203 insertions, 164 deletions
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index a22acc0..49be720 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -8,7 +8,7 @@ from http.cookies import SimpleCookie from test.support import ( TestFailed, TESTFN, run_with_locale, - _2G, _4G, precisionbigmemtest, + _2G, _4G, bigmemtest, ) from pickle import bytes_types @@ -1159,7 +1159,7 @@ class BigmemPickleTests(unittest.TestCase): # Binary protocols can serialize longs of up to 2GB-1 - @precisionbigmemtest(size=_2G, memuse=1 + 1, dry_run=False) + @bigmemtest(size=_2G, memuse=1 + 1, dry_run=False) def test_huge_long_32b(self, size): data = 1 << (8 * size) try: @@ -1175,7 +1175,7 @@ class BigmemPickleTests(unittest.TestCase): # (older protocols don't have a dedicated opcode for bytes and are # too inefficient) - @precisionbigmemtest(size=_2G, memuse=1 + 1, dry_run=False) + @bigmemtest(size=_2G, memuse=1 + 1, dry_run=False) def test_huge_bytes_32b(self, size): data = b"abcd" * (size // 4) try: @@ -1191,7 +1191,7 @@ class BigmemPickleTests(unittest.TestCase): finally: data = None - @precisionbigmemtest(size=_4G, memuse=1 + 1, dry_run=False) + @bigmemtest(size=_4G, memuse=1 + 1, dry_run=False) def test_huge_bytes_64b(self, size): data = b"a" * size try: @@ -1206,7 +1206,7 @@ class BigmemPickleTests(unittest.TestCase): # All protocols use 1-byte per printable ASCII character; we add another # byte because the encoded form has to be copied into the internal buffer. - @precisionbigmemtest(size=_2G, memuse=2 + character_size, dry_run=False) + @bigmemtest(size=_2G, memuse=2 + character_size, dry_run=False) def test_huge_str_32b(self, size): data = "abcd" * (size // 4) try: @@ -1223,7 +1223,7 @@ class BigmemPickleTests(unittest.TestCase): # BINUNICODE (protocols 1, 2 and 3) cannot carry more than # 2**32 - 1 bytes of utf-8 encoded unicode. - @precisionbigmemtest(size=_4G, memuse=1 + character_size, dry_run=False) + @bigmemtest(size=_4G, memuse=1 + character_size, dry_run=False) def test_huge_str_64b(self, size): data = "a" * size try: diff --git a/Lib/test/support.py b/Lib/test/support.py index 8e6ca2a..ab71d47 100644 --- a/Lib/test/support.py +++ b/Lib/test/support.py @@ -1053,45 +1053,54 @@ def set_memlimit(limit): raise ValueError('Memory limit %r too low to be useful' % (limit,)) max_memuse = memlimit -def bigmemtest(minsize, memuse): +def _memory_watchdog(start_evt, finish_evt, period=10.0): + """A function which periodically watches the process' memory consumption + and prints it out. + """ + # XXX: because of the GIL, and because the very long operations tested + # in most bigmem tests are uninterruptible, the loop below gets woken up + # much less often than expected. + # The polling code should be rewritten in raw C, without holding the GIL, + # and push results onto an anonymous pipe. + try: + page_size = os.sysconf('SC_PAGESIZE') + except (ValueError, AttributeError): + try: + page_size = os.sysconf('SC_PAGE_SIZE') + except (ValueError, AttributeError): + page_size = 4096 + procfile = '/proc/{pid}/statm'.format(pid=os.getpid()) + try: + f = open(procfile, 'rb') + except IOError as e: + warnings.warn('/proc not available for stats: {}'.format(e), + RuntimeWarning) + sys.stderr.flush() + return + with f: + start_evt.set() + old_data = -1 + while not finish_evt.wait(period): + f.seek(0) + statm = f.read().decode('ascii') + data = int(statm.split()[5]) + if data != old_data: + old_data = data + print(" ... process data size: {data:.1f}G" + .format(data=data * page_size / (1024 ** 3))) + +def bigmemtest(size, memuse, dry_run=True): """Decorator for bigmem tests. 'minsize' is the minimum useful size for the test (in arbitrary, test-interpreted units.) 'memuse' is the number of 'bytes per size' for the test, or a good estimate of it. - The decorator tries to guess a good value for 'size' and passes it to - the decorated test function. If minsize * memuse is more than the - allowed memory use (as defined by max_memuse), the test is skipped. - Otherwise, minsize is adjusted upward to use up to max_memuse. + if 'dry_run' is False, it means the test doesn't support dummy runs + when -M is not specified. """ def decorator(f): def wrapper(self): - # Retrieve values in case someone decided to adjust them - minsize = wrapper.minsize - memuse = wrapper.memuse - if not max_memuse: - # If max_memuse is 0 (the default), - # we still want to run the tests with size set to a few kb, - # to make sure they work. We still want to avoid using - # too much memory, though, but we do that noisily. - maxsize = 5147 - self.assertFalse(maxsize * memuse > 20 * _1M) - else: - maxsize = int(max_memuse / memuse) - if maxsize < minsize: - raise unittest.SkipTest( - "not enough memory: %.1fG minimum needed" - % (minsize * memuse / (1024 ** 3))) - return f(self, maxsize) - wrapper.minsize = minsize - wrapper.memuse = memuse - return wrapper - return decorator - -def precisionbigmemtest(size, memuse, dry_run=True): - def decorator(f): - def wrapper(self): size = wrapper.size memuse = wrapper.memuse if not real_max_memuse: @@ -1105,7 +1114,28 @@ def precisionbigmemtest(size, memuse, dry_run=True): "not enough memory: %.1fG minimum needed" % (size * memuse / (1024 ** 3))) - return f(self, maxsize) + if real_max_memuse and verbose and threading: + print() + print(" ... expected peak memory use: {peak:.1f}G" + .format(peak=size * memuse / (1024 ** 3))) + sys.stdout.flush() + start_evt = threading.Event() + finish_evt = threading.Event() + t = threading.Thread(target=_memory_watchdog, + args=(start_evt, finish_evt, 0.5)) + t.daemon = True + t.start() + start_evt.set() + else: + t = None + + try: + return f(self, maxsize) + finally: + if t: + finish_evt.set() + t.join() + wrapper.size = size wrapper.memuse = memuse return wrapper diff --git a/Lib/test/test_bigmem.py b/Lib/test/test_bigmem.py index ac6b109..f3c6ebb 100644 --- a/Lib/test/test_bigmem.py +++ b/Lib/test/test_bigmem.py @@ -1,5 +1,5 @@ from test import support -from test.support import bigmemtest, _1G, _2G, _4G, precisionbigmemtest +from test.support import bigmemtest, _1G, _2G, _4G import unittest import operator @@ -25,10 +25,10 @@ import functools # a large object, make the subobject of a length that is not a power of # 2. That way, int-wrapping problems are more easily detected. # -# - While the bigmemtest decorator speaks of 'minsize', all tests will -# actually be called with a much smaller number too, in the normal -# test run (5Kb currently.) This is so the tests themselves get frequent -# testing. Consequently, always make all large allocations based on the +# - Despite the bigmemtest decorator, all tests will actually be called +# with a much smaller number too, in the normal test run (5Kb currently.) +# This is so the tests themselves get frequent testing. +# Consequently, always make all large allocations based on the # passed-in 'size', and don't rely on the size being very large. Also, # memuse-per-size should remain sane (less than a few thousand); if your # test uses more, adjust 'size' upward, instead. @@ -42,7 +42,7 @@ character_size = 4 if sys.maxunicode > 0xFFFF else 2 class BaseStrTest: - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_capitalize(self, size): _ = self.from_latin1 SUBSTR = self.from_latin1(' abc def ghi') @@ -52,7 +52,7 @@ class BaseStrTest: SUBSTR.capitalize()) self.assertEqual(caps.lstrip(_('-')), SUBSTR) - @bigmemtest(minsize=_2G + 10, memuse=1) + @bigmemtest(size=_2G + 10, memuse=1) def test_center(self, size): SUBSTR = self.from_latin1(' abc def ghi') s = SUBSTR.center(size) @@ -63,7 +63,7 @@ class BaseStrTest: self.assertEqual(s[lpadsize:-rpadsize], SUBSTR) self.assertEqual(s.strip(), SUBSTR.strip()) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_count(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi') @@ -75,7 +75,7 @@ class BaseStrTest: self.assertEqual(s.count(_('i')), 1) self.assertEqual(s.count(_('j')), 0) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_endswith(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi') @@ -87,7 +87,7 @@ class BaseStrTest: self.assertFalse(s.endswith(_('a') + SUBSTR)) self.assertFalse(SUBSTR.endswith(s)) - @bigmemtest(minsize=_2G + 10, memuse=2) + @bigmemtest(size=_2G + 10, memuse=2) def test_expandtabs(self, size): _ = self.from_latin1 s = _('-') * size @@ -100,7 +100,7 @@ class BaseStrTest: self.assertEqual(len(s), size - remainder) self.assertEqual(len(s.strip(_(' '))), 0) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_find(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi') @@ -117,7 +117,7 @@ class BaseStrTest: sublen + size + SUBSTR.find(_('i'))) self.assertEqual(s.find(_('j')), -1) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_index(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi') @@ -134,7 +134,7 @@ class BaseStrTest: sublen + size + SUBSTR.index(_('i'))) self.assertRaises(ValueError, s.index, _('j')) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_isalnum(self, size): _ = self.from_latin1 SUBSTR = _('123456') @@ -143,7 +143,7 @@ class BaseStrTest: s += _('.') self.assertFalse(s.isalnum()) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_isalpha(self, size): _ = self.from_latin1 SUBSTR = _('zzzzzzz') @@ -152,7 +152,7 @@ class BaseStrTest: s += _('.') self.assertFalse(s.isalpha()) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_isdigit(self, size): _ = self.from_latin1 SUBSTR = _('123456') @@ -161,7 +161,7 @@ class BaseStrTest: s += _('z') self.assertFalse(s.isdigit()) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_islower(self, size): _ = self.from_latin1 chars = _(''.join( @@ -172,7 +172,7 @@ class BaseStrTest: s += _('A') self.assertFalse(s.islower()) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_isspace(self, size): _ = self.from_latin1 whitespace = _(' \f\n\r\t\v') @@ -182,7 +182,7 @@ class BaseStrTest: s += _('j') self.assertFalse(s.isspace()) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_istitle(self, size): _ = self.from_latin1 SUBSTR = _('123456') @@ -193,7 +193,7 @@ class BaseStrTest: s += _('aA') self.assertFalse(s.istitle()) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_isupper(self, size): _ = self.from_latin1 chars = _(''.join( @@ -204,7 +204,7 @@ class BaseStrTest: s += _('a') self.assertFalse(s.isupper()) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_join(self, size): _ = self.from_latin1 s = _('A') * size @@ -214,7 +214,7 @@ class BaseStrTest: self.assertTrue(x.startswith(_('aaaaaA'))) self.assertTrue(x.endswith(_('Abbbbb'))) - @bigmemtest(minsize=_2G + 10, memuse=1) + @bigmemtest(size=_2G + 10, memuse=1) def test_ljust(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi') @@ -223,7 +223,7 @@ class BaseStrTest: self.assertEqual(len(s), size) self.assertEqual(s.strip(), SUBSTR.strip()) - @bigmemtest(minsize=_2G + 10, memuse=2) + @bigmemtest(size=_2G + 10, memuse=2) def test_lower(self, size): _ = self.from_latin1 s = _('A') * size @@ -231,7 +231,7 @@ class BaseStrTest: self.assertEqual(len(s), size) self.assertEqual(s.count(_('a')), size) - @bigmemtest(minsize=_2G + 10, memuse=1) + @bigmemtest(size=_2G + 10, memuse=1) def test_lstrip(self, size): _ = self.from_latin1 SUBSTR = _('abc def ghi') @@ -246,7 +246,7 @@ class BaseStrTest: stripped = s.lstrip() self.assertTrue(stripped is s) - @bigmemtest(minsize=_2G + 10, memuse=2) + @bigmemtest(size=_2G + 10, memuse=2) def test_replace(self, size): _ = self.from_latin1 replacement = _('a') @@ -259,7 +259,7 @@ class BaseStrTest: self.assertEqual(s.count(replacement), 4) self.assertEqual(s[-10:], _(' aaaa')) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_rfind(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi') @@ -275,7 +275,7 @@ class BaseStrTest: SUBSTR.rfind(_('i'))) self.assertEqual(s.rfind(_('j')), -1) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_rindex(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi') @@ -294,7 +294,7 @@ class BaseStrTest: SUBSTR.rindex(_('i'))) self.assertRaises(ValueError, s.rindex, _('j')) - @bigmemtest(minsize=_2G + 10, memuse=1) + @bigmemtest(size=_2G + 10, memuse=1) def test_rjust(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi') @@ -303,7 +303,7 @@ class BaseStrTest: self.assertEqual(len(s), size) self.assertEqual(s.strip(), SUBSTR.strip()) - @bigmemtest(minsize=_2G + 10, memuse=1) + @bigmemtest(size=_2G + 10, memuse=1) def test_rstrip(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi') @@ -321,7 +321,7 @@ class BaseStrTest: # The test takes about size bytes to build a string, and then about # sqrt(size) substrings of sqrt(size) in size and a list to # hold sqrt(size) items. It's close but just over 2x size. - @bigmemtest(minsize=_2G, memuse=2.1) + @bigmemtest(size=_2G, memuse=2.1) def test_split_small(self, size): _ = self.from_latin1 # Crudely calculate an estimate so that the result of s.split won't @@ -347,7 +347,7 @@ class BaseStrTest: # suffer for the list size. (Otherwise, it'd cost another 48 times # size in bytes!) Nevertheless, a list of size takes # 8*size bytes. - @bigmemtest(minsize=_2G + 5, memuse=10) + @bigmemtest(size=_2G + 5, memuse=10) def test_split_large(self, size): _ = self.from_latin1 s = _(' a') * size + _(' ') @@ -359,7 +359,7 @@ class BaseStrTest: self.assertEqual(len(l), size + 1) self.assertEqual(set(l), set([_(' ')])) - @bigmemtest(minsize=_2G, memuse=2.1) + @bigmemtest(size=_2G, memuse=2.1) def test_splitlines(self, size): _ = self.from_latin1 # Crudely calculate an estimate so that the result of s.split won't @@ -373,7 +373,7 @@ class BaseStrTest: for item in l: self.assertEqual(item, expected) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_startswith(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi') @@ -382,7 +382,7 @@ class BaseStrTest: self.assertTrue(s.startswith(_('-') * size)) self.assertFalse(s.startswith(SUBSTR)) - @bigmemtest(minsize=_2G, memuse=1) + @bigmemtest(size=_2G, memuse=1) def test_strip(self, size): _ = self.from_latin1 SUBSTR = _(' abc def ghi ') @@ -394,7 +394,7 @@ class BaseStrTest: self.assertEqual(len(s), size) self.assertEqual(s.strip(), SUBSTR.strip()) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_swapcase(self, size): _ = self.from_latin1 SUBSTR = _("aBcDeFG12.'\xa9\x00") @@ -406,7 +406,7 @@ class BaseStrTest: self.assertEqual(s[:sublen * 3], SUBSTR.swapcase() * 3) self.assertEqual(s[-sublen * 3:], SUBSTR.swapcase() * 3) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_title(self, size): _ = self.from_latin1 SUBSTR = _('SpaaHAaaAaham') @@ -415,7 +415,7 @@ class BaseStrTest: self.assertTrue(s.startswith((SUBSTR * 3).title())) self.assertTrue(s.endswith(SUBSTR.lower() * 3)) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_translate(self, size): _ = self.from_latin1 SUBSTR = _('aZz.z.Aaz.') @@ -438,7 +438,7 @@ class BaseStrTest: self.assertEqual(s.count(_('!')), repeats * 2) self.assertEqual(s.count(_('z')), repeats * 3) - @bigmemtest(minsize=_2G + 5, memuse=2) + @bigmemtest(size=_2G + 5, memuse=2) def test_upper(self, size): _ = self.from_latin1 s = _('a') * size @@ -446,7 +446,7 @@ class BaseStrTest: self.assertEqual(len(s), size) self.assertEqual(s.count(_('A')), size) - @bigmemtest(minsize=_2G + 20, memuse=1) + @bigmemtest(size=_2G + 20, memuse=1) def test_zfill(self, size): _ = self.from_latin1 SUBSTR = _('-568324723598234') @@ -458,7 +458,7 @@ class BaseStrTest: # This test is meaningful even with size < 2G, as long as the # doubled string is > 2G (but it tests more if both are > 2G :) - @bigmemtest(minsize=_1G + 2, memuse=3) + @bigmemtest(size=_1G + 2, memuse=3) def test_concat(self, size): _ = self.from_latin1 s = _('.') * size @@ -469,7 +469,7 @@ class BaseStrTest: # This test is meaningful even with size < 2G, as long as the # repeated string is > 2G (but it tests more if both are > 2G :) - @bigmemtest(minsize=_1G + 2, memuse=3) + @bigmemtest(size=_1G + 2, memuse=3) def test_repeat(self, size): _ = self.from_latin1 s = _('.') * size @@ -478,7 +478,7 @@ class BaseStrTest: self.assertEqual(len(s), size * 2) self.assertEqual(s.count(_('.')), size * 2) - @bigmemtest(minsize=_2G + 20, memuse=2) + @bigmemtest(size=_2G + 20, memuse=2) def test_slice_and_getitem(self, size): _ = self.from_latin1 SUBSTR = _('0123456789') @@ -512,7 +512,7 @@ class BaseStrTest: self.assertRaises(IndexError, operator.getitem, s, len(s) + 1) self.assertRaises(IndexError, operator.getitem, s, len(s) + 1<<31) - @bigmemtest(minsize=_2G, memuse=2) + @bigmemtest(size=_2G, memuse=2) def test_contains(self, size): _ = self.from_latin1 SUBSTR = _('0123456789') @@ -526,7 +526,7 @@ class BaseStrTest: s += _('a') self.assertIn(_('a'), s) - @bigmemtest(minsize=_2G + 10, memuse=2) + @bigmemtest(size=_2G + 10, memuse=2) def test_compare(self, size): _ = self.from_latin1 s1 = _('-') * size @@ -539,7 +539,7 @@ class BaseStrTest: s2 = _('.') * size self.assertFalse(s1 == s2) - @bigmemtest(minsize=_2G + 10, memuse=1) + @bigmemtest(size=_2G + 10, memuse=1) def test_hash(self, size): # Not sure if we can do any meaningful tests here... Even if we # start relying on the exact algorithm used, the result will be @@ -590,46 +590,36 @@ class StrTest(unittest.TestCase, BaseStrTest): getattr(type(self), name).memuse = memuse # the utf8 encoder preallocates big time (4x the number of characters) - @bigmemtest(minsize=_2G + 2, memuse=character_size + 4) + @bigmemtest(size=_2G + 2, memuse=character_size + 4) def test_encode(self, size): return self.basic_encode_test(size, 'utf-8') - @precisionbigmemtest(size=_4G // 6 + 2, memuse=character_size + 1) + @bigmemtest(size=_4G // 6 + 2, memuse=character_size + 1) def test_encode_raw_unicode_escape(self, size): try: return self.basic_encode_test(size, 'raw_unicode_escape') except MemoryError: pass # acceptable on 32-bit - @precisionbigmemtest(size=_4G // 5 + 70, memuse=character_size + 1) + @bigmemtest(size=_4G // 5 + 70, memuse=character_size + 1) def test_encode_utf7(self, size): try: return self.basic_encode_test(size, 'utf7') except MemoryError: pass # acceptable on 32-bit - @precisionbigmemtest(size=_4G // 4 + 5, memuse=character_size + 4) + @bigmemtest(size=_4G // 4 + 5, memuse=character_size + 4) def test_encode_utf32(self, size): try: return self.basic_encode_test(size, 'utf32', expectedsize=4*size+4) except MemoryError: pass # acceptable on 32-bit - @precisionbigmemtest(size=_2G - 1, memuse=character_size + 1) + @bigmemtest(size=_2G - 1, memuse=character_size + 1) def test_encode_ascii(self, size): return self.basic_encode_test(size, 'ascii', c='A') - @precisionbigmemtest(size=_4G // 5, memuse=character_size * (6 + 1)) - def test_unicode_repr_overflow(self, size): - try: - s = "\uDCBA"*size - r = repr(s) - except MemoryError: - pass # acceptable on 32-bit - else: - self.assertTrue(s == eval(r)) - - @bigmemtest(minsize=_2G + 10, memuse=character_size * 2) + @bigmemtest(size=_2G + 10, memuse=character_size * 2) def test_format(self, size): s = '-' * size sf = '%s' % (s,) @@ -650,7 +640,7 @@ class StrTest(unittest.TestCase, BaseStrTest): self.assertEqual(s.count('.'), 3) self.assertEqual(s.count('-'), size * 2) - @bigmemtest(minsize=_2G + 10, memuse=character_size * 2) + @bigmemtest(size=_2G + 10, memuse=character_size * 2) def test_repr_small(self, size): s = '-' * size s = repr(s) @@ -671,7 +661,7 @@ class StrTest(unittest.TestCase, BaseStrTest): self.assertEqual(s.count('\\'), size) self.assertEqual(s.count('0'), size * 2) - @bigmemtest(minsize=_2G + 10, memuse=character_size * 5) + @bigmemtest(size=_2G + 10, memuse=character_size * 5) def test_repr_large(self, size): s = '\x00' * size s = repr(s) @@ -681,27 +671,46 @@ class StrTest(unittest.TestCase, BaseStrTest): self.assertEqual(s.count('\\'), size) self.assertEqual(s.count('0'), size * 2) - @bigmemtest(minsize=2**32 / 5, memuse=character_size * 7) + @bigmemtest(size=_2G // 5 + 1, memuse=character_size * 7) def test_unicode_repr(self, size): # Use an assigned, but not printable code point. # It is in the range of the low surrogates \uDC00-\uDFFF. - s = "\uDCBA" * size - for f in (repr, ascii): - r = f(s) - self.assertTrue(len(r) > size) - self.assertTrue(r.endswith(r"\udcba'"), r[-10:]) - del r + char = "\uDCBA" + s = char * size + try: + for f in (repr, ascii): + r = f(s) + self.assertEqual(len(r), 2 + (len(f(char)) - 2) * size) + self.assertTrue(r.endswith(r"\udcba'"), r[-10:]) + r = None + finally: + r = s = None # The character takes 4 bytes even in UCS-2 builds because it will # be decomposed into surrogates. - @bigmemtest(minsize=2**32 / 5, memuse=4 + character_size * 9) + @bigmemtest(size=_2G // 5 + 1, memuse=4 + character_size * 9) def test_unicode_repr_wide(self, size): - s = "\U0001DCBA" * size - for f in (repr, ascii): - r = f(s) - self.assertTrue(len(r) > size) - self.assertTrue(r.endswith(r"\U0001dcba'"), r[-12:]) - del r + char = "\U0001DCBA" + s = char * size + try: + for f in (repr, ascii): + r = f(s) + self.assertEqual(len(r), 2 + (len(f(char)) - 2) * size) + self.assertTrue(r.endswith(r"\U0001dcba'"), r[-12:]) + r = None + finally: + r = s = None + + @bigmemtest(size=_4G // 5, memuse=character_size * (6 + 1)) + def _test_unicode_repr_overflow(self, size): + # XXX not sure what this test is about + char = "\uDCBA" + s = char * size + try: + r = repr(s) + self.assertTrue(s == eval(r)) + finally: + r = s = None class BytesTest(unittest.TestCase, BaseStrTest): @@ -709,7 +718,7 @@ class BytesTest(unittest.TestCase, BaseStrTest): def from_latin1(self, s): return s.encode("latin1") - @bigmemtest(minsize=_2G + 2, memuse=1 + character_size) + @bigmemtest(size=_2G + 2, memuse=1 + character_size) def test_decode(self, size): s = self.from_latin1('.') * size self.assertEqual(len(s.decode('utf-8')), size) @@ -720,7 +729,7 @@ class BytearrayTest(unittest.TestCase, BaseStrTest): def from_latin1(self, s): return bytearray(s.encode("latin1")) - @bigmemtest(minsize=_2G + 2, memuse=1 + character_size) + @bigmemtest(size=_2G + 2, memuse=1 + character_size) def test_decode(self, size): s = self.from_latin1('.') * size self.assertEqual(len(s.decode('utf-8')), size) @@ -739,7 +748,7 @@ class TupleTest(unittest.TestCase): # having more than 2<<31 references to any given object. Hence the # use of different types of objects as contents in different tests. - @bigmemtest(minsize=_2G + 2, memuse=16) + @bigmemtest(size=_2G + 2, memuse=16) def test_compare(self, size): t1 = ('',) * size t2 = ('',) * size @@ -762,15 +771,15 @@ class TupleTest(unittest.TestCase): t = t + t self.assertEqual(len(t), size * 2) - @bigmemtest(minsize=_2G // 2 + 2, memuse=24) + @bigmemtest(size=_2G // 2 + 2, memuse=24) def test_concat_small(self, size): return self.basic_concat_test(size) - @bigmemtest(minsize=_2G + 2, memuse=24) + @bigmemtest(size=_2G + 2, memuse=24) def test_concat_large(self, size): return self.basic_concat_test(size) - @bigmemtest(minsize=_2G // 5 + 10, memuse=8 * 5) + @bigmemtest(size=_2G // 5 + 10, memuse=8 * 5) def test_contains(self, size): t = (1, 2, 3, 4, 5) * size self.assertEqual(len(t), size * 5) @@ -778,7 +787,7 @@ class TupleTest(unittest.TestCase): self.assertNotIn((1, 2, 3, 4, 5), t) self.assertNotIn(0, t) - @bigmemtest(minsize=_2G + 10, memuse=8) + @bigmemtest(size=_2G + 10, memuse=8) def test_hash(self, size): t1 = (0,) * size h1 = hash(t1) @@ -786,7 +795,7 @@ class TupleTest(unittest.TestCase): t2 = (0,) * (size + 1) self.assertFalse(h1 == hash(t2)) - @bigmemtest(minsize=_2G + 10, memuse=8) + @bigmemtest(size=_2G + 10, memuse=8) def test_index_and_slice(self, size): t = (None,) * size self.assertEqual(len(t), size) @@ -811,19 +820,19 @@ class TupleTest(unittest.TestCase): t = t * 2 self.assertEqual(len(t), size * 2) - @bigmemtest(minsize=_2G // 2 + 2, memuse=24) + @bigmemtest(size=_2G // 2 + 2, memuse=24) def test_repeat_small(self, size): return self.basic_test_repeat(size) - @bigmemtest(minsize=_2G + 2, memuse=24) + @bigmemtest(size=_2G + 2, memuse=24) def test_repeat_large(self, size): return self.basic_test_repeat(size) - @bigmemtest(minsize=_1G - 1, memuse=12) + @bigmemtest(size=_1G - 1, memuse=12) def test_repeat_large_2(self, size): return self.basic_test_repeat(size) - @precisionbigmemtest(size=_1G - 1, memuse=9) + @bigmemtest(size=_1G - 1, memuse=9) def test_from_2G_generator(self, size): self.skipTest("test needs much more memory than advertised, see issue5438") try: @@ -837,7 +846,7 @@ class TupleTest(unittest.TestCase): count += 1 self.assertEqual(count, size) - @precisionbigmemtest(size=_1G - 25, memuse=9) + @bigmemtest(size=_1G - 25, memuse=9) def test_from_almost_2G_generator(self, size): self.skipTest("test needs much more memory than advertised, see issue5438") try: @@ -860,11 +869,11 @@ class TupleTest(unittest.TestCase): self.assertEqual(s[-5:], '0, 0)') self.assertEqual(s.count('0'), size) - @bigmemtest(minsize=_2G // 3 + 2, memuse=8 + 3 * character_size) + @bigmemtest(size=_2G // 3 + 2, memuse=8 + 3 * character_size) def test_repr_small(self, size): return self.basic_test_repr(size) - @bigmemtest(minsize=_2G + 2, memuse=8 + 3 * character_size) + @bigmemtest(size=_2G + 2, memuse=8 + 3 * character_size) def test_repr_large(self, size): return self.basic_test_repr(size) @@ -875,7 +884,7 @@ class ListTest(unittest.TestCase): # lists hold references to various objects to test their refcount # limits. - @bigmemtest(minsize=_2G + 2, memuse=16) + @bigmemtest(size=_2G + 2, memuse=16) def test_compare(self, size): l1 = [''] * size l2 = [''] * size @@ -898,11 +907,11 @@ class ListTest(unittest.TestCase): l = l + l self.assertEqual(len(l), size * 2) - @bigmemtest(minsize=_2G // 2 + 2, memuse=24) + @bigmemtest(size=_2G // 2 + 2, memuse=24) def test_concat_small(self, size): return self.basic_test_concat(size) - @bigmemtest(minsize=_2G + 2, memuse=24) + @bigmemtest(size=_2G + 2, memuse=24) def test_concat_large(self, size): return self.basic_test_concat(size) @@ -913,15 +922,15 @@ class ListTest(unittest.TestCase): self.assertTrue(l[0] is l[-1]) self.assertTrue(l[size - 1] is l[size + 1]) - @bigmemtest(minsize=_2G // 2 + 2, memuse=24) + @bigmemtest(size=_2G // 2 + 2, memuse=24) def test_inplace_concat_small(self, size): return self.basic_test_inplace_concat(size) - @bigmemtest(minsize=_2G + 2, memuse=24) + @bigmemtest(size=_2G + 2, memuse=24) def test_inplace_concat_large(self, size): return self.basic_test_inplace_concat(size) - @bigmemtest(minsize=_2G // 5 + 10, memuse=8 * 5) + @bigmemtest(size=_2G // 5 + 10, memuse=8 * 5) def test_contains(self, size): l = [1, 2, 3, 4, 5] * size self.assertEqual(len(l), size * 5) @@ -929,12 +938,12 @@ class ListTest(unittest.TestCase): self.assertNotIn([1, 2, 3, 4, 5], l) self.assertNotIn(0, l) - @bigmemtest(minsize=_2G + 10, memuse=8) + @bigmemtest(size=_2G + 10, memuse=8) def test_hash(self, size): l = [0] * size self.assertRaises(TypeError, hash, l) - @bigmemtest(minsize=_2G + 10, memuse=8) + @bigmemtest(size=_2G + 10, memuse=8) def test_index_and_slice(self, size): l = [None] * size self.assertEqual(len(l), size) @@ -998,11 +1007,11 @@ class ListTest(unittest.TestCase): l = l * 2 self.assertEqual(len(l), size * 2) - @bigmemtest(minsize=_2G // 2 + 2, memuse=24) + @bigmemtest(size=_2G // 2 + 2, memuse=24) def test_repeat_small(self, size): return self.basic_test_repeat(size) - @bigmemtest(minsize=_2G + 2, memuse=24) + @bigmemtest(size=_2G + 2, memuse=24) def test_repeat_large(self, size): return self.basic_test_repeat(size) @@ -1018,11 +1027,11 @@ class ListTest(unittest.TestCase): self.assertEqual(len(l), size * 2) self.assertTrue(l[size - 1] is l[-1]) - @bigmemtest(minsize=_2G // 2 + 2, memuse=16) + @bigmemtest(size=_2G // 2 + 2, memuse=16) def test_inplace_repeat_small(self, size): return self.basic_test_inplace_repeat(size) - @bigmemtest(minsize=_2G + 2, memuse=16) + @bigmemtest(size=_2G + 2, memuse=16) def test_inplace_repeat_large(self, size): return self.basic_test_inplace_repeat(size) @@ -1035,17 +1044,17 @@ class ListTest(unittest.TestCase): self.assertEqual(s[-5:], '0, 0]') self.assertEqual(s.count('0'), size) - @bigmemtest(minsize=_2G // 3 + 2, memuse=8 + 3 * character_size) + @bigmemtest(size=_2G // 3 + 2, memuse=8 + 3 * character_size) def test_repr_small(self, size): return self.basic_test_repr(size) - @bigmemtest(minsize=_2G + 2, memuse=8 + 3 * character_size) + @bigmemtest(size=_2G + 2, memuse=8 + 3 * character_size) def test_repr_large(self, size): return self.basic_test_repr(size) # list overallocates ~1/8th of the total size (on first expansion) so # the single list.append call puts memuse at 9 bytes per size. - @bigmemtest(minsize=_2G, memuse=9) + @bigmemtest(size=_2G, memuse=9) def test_append(self, size): l = [object()] * size l.append(object()) @@ -1053,7 +1062,7 @@ class ListTest(unittest.TestCase): self.assertTrue(l[-3] is l[-2]) self.assertFalse(l[-2] is l[-1]) - @bigmemtest(minsize=_2G // 5 + 2, memuse=8 * 5) + @bigmemtest(size=_2G // 5 + 2, memuse=8 * 5) def test_count(self, size): l = [1, 2, 3, 4, 5] * size self.assertEqual(l.count(1), size) @@ -1066,15 +1075,15 @@ class ListTest(unittest.TestCase): self.assertTrue(l[0] is l[-1]) self.assertTrue(l[size - 1] is l[size + 1]) - @bigmemtest(minsize=_2G // 2 + 2, memuse=16) + @bigmemtest(size=_2G // 2 + 2, memuse=16) def test_extend_small(self, size): return self.basic_test_extend(size) - @bigmemtest(minsize=_2G + 2, memuse=16) + @bigmemtest(size=_2G + 2, memuse=16) def test_extend_large(self, size): return self.basic_test_extend(size) - @bigmemtest(minsize=_2G // 5 + 2, memuse=8 * 5) + @bigmemtest(size=_2G // 5 + 2, memuse=8 * 5) def test_index(self, size): l = [1, 2, 3, 4, 5] * size size *= 5 @@ -1085,7 +1094,7 @@ class ListTest(unittest.TestCase): self.assertRaises(ValueError, l.index, 6) # This tests suffers from overallocation, just like test_append. - @bigmemtest(minsize=_2G + 10, memuse=9) + @bigmemtest(size=_2G + 10, memuse=9) def test_insert(self, size): l = [1.0] * size l.insert(size - 1, "A") @@ -1104,7 +1113,7 @@ class ListTest(unittest.TestCase): self.assertEqual(l[:3], [1.0, "C", 1.0]) self.assertEqual(l[size - 3:], ["A", 1.0, "B"]) - @bigmemtest(minsize=_2G // 5 + 4, memuse=8 * 5) + @bigmemtest(size=_2G // 5 + 4, memuse=8 * 5) def test_pop(self, size): l = ["a", "b", "c", "d", "e"] * size size *= 5 @@ -1128,7 +1137,7 @@ class ListTest(unittest.TestCase): self.assertEqual(item, "c") self.assertEqual(l[-2:], ["b", "d"]) - @bigmemtest(minsize=_2G + 10, memuse=8) + @bigmemtest(size=_2G + 10, memuse=8) def test_remove(self, size): l = [10] * size self.assertEqual(len(l), size) @@ -1148,7 +1157,7 @@ class ListTest(unittest.TestCase): self.assertEqual(len(l), size) self.assertEqual(l[-2:], [10, 10]) - @bigmemtest(minsize=_2G // 5 + 2, memuse=8 * 5) + @bigmemtest(size=_2G // 5 + 2, memuse=8 * 5) def test_reverse(self, size): l = [1, 2, 3, 4, 5] * size l.reverse() @@ -1156,7 +1165,7 @@ class ListTest(unittest.TestCase): self.assertEqual(l[-5:], [5, 4, 3, 2, 1]) self.assertEqual(l[:5], [5, 4, 3, 2, 1]) - @bigmemtest(minsize=_2G // 5 + 2, memuse=8 * 5) + @bigmemtest(size=_2G // 5 + 2, memuse=8 * 5) def test_sort(self, size): l = [1, 2, 3, 4, 5] * size l.sort() diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 17d752b..97981dd 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -17,7 +17,7 @@ except ImportError: import unittest import warnings from test import support -from test.support import _4G, precisionbigmemtest +from test.support import _4G, bigmemtest # Were we compiled --with-pydebug or with #define Py_DEBUG? COMPILED_WITH_PYDEBUG = hasattr(sys, 'gettotalrefcount') @@ -196,7 +196,7 @@ class HashLibTestCase(unittest.TestCase): b'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789', 'd174ab98d277d9f5a5611c2c9f419d9f') - @precisionbigmemtest(size=_4G + 5, memuse=1) + @bigmemtest(size=_4G + 5, memuse=1) def test_case_md5_huge(self, size): if size == _4G + 5: try: @@ -204,7 +204,7 @@ class HashLibTestCase(unittest.TestCase): except OverflowError: pass # 32-bit arch - @precisionbigmemtest(size=_4G - 1, memuse=1) + @bigmemtest(size=_4G - 1, memuse=1) def test_case_md5_uintmax(self, size): if size == _4G - 1: try: diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py index db69e5f..2ff118f 100644 --- a/Lib/test/test_xml_etree_c.py +++ b/Lib/test/test_xml_etree_c.py @@ -1,7 +1,7 @@ # xml.etree test for cElementTree from test import support -from test.support import precisionbigmemtest, _2G +from test.support import bigmemtest, _2G import unittest cET = support.import_module('xml.etree.cElementTree') @@ -35,7 +35,7 @@ def sanity(): class MiscTests(unittest.TestCase): # Issue #8651. - @support.precisionbigmemtest(size=support._2G + 100, memuse=1) + @support.bigmemtest(size=support._2G + 100, memuse=1) def test_length_overflow(self, size): if size < support._2G + 100: self.skipTest("not enough free memory, need at least 2 GB") diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 68dd3ea..dffa2ca 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -3,7 +3,7 @@ from test import support import binascii import random import sys -from test.support import precisionbigmemtest, _1G, _4G +from test.support import bigmemtest, _1G, _4G zlib = support.import_module('zlib') @@ -177,16 +177,16 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase): # Memory use of the following functions takes into account overallocation - @precisionbigmemtest(size=_1G + 1024 * 1024, memuse=3) + @bigmemtest(size=_1G + 1024 * 1024, memuse=3) def test_big_compress_buffer(self, size): compress = lambda s: zlib.compress(s, 1) self.check_big_compress_buffer(size, compress) - @precisionbigmemtest(size=_1G + 1024 * 1024, memuse=2) + @bigmemtest(size=_1G + 1024 * 1024, memuse=2) def test_big_decompress_buffer(self, size): self.check_big_decompress_buffer(size, zlib.decompress) - @precisionbigmemtest(size=_4G + 100, memuse=1) + @bigmemtest(size=_4G + 100, memuse=1) def test_length_overflow(self, size): if size < _4G + 100: self.skipTest("not enough free memory, need at least 4 GB") @@ -511,19 +511,19 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): # Memory use of the following functions takes into account overallocation - @precisionbigmemtest(size=_1G + 1024 * 1024, memuse=3) + @bigmemtest(size=_1G + 1024 * 1024, memuse=3) def test_big_compress_buffer(self, size): c = zlib.compressobj(1) compress = lambda s: c.compress(s) + c.flush() self.check_big_compress_buffer(size, compress) - @precisionbigmemtest(size=_1G + 1024 * 1024, memuse=2) + @bigmemtest(size=_1G + 1024 * 1024, memuse=2) def test_big_decompress_buffer(self, size): d = zlib.decompressobj() decompress = lambda s: d.decompress(s) + d.flush() self.check_big_decompress_buffer(size, decompress) - @precisionbigmemtest(size=_4G + 100, memuse=1) + @bigmemtest(size=_4G + 100, memuse=1) def test_length_overflow(self, size): if size < _4G + 100: self.skipTest("not enough free memory, need at least 4 GB") |