From 4847e4e1f47bae9ed1855801c70ce7fbdecfa32a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 10 Jan 2014 13:37:54 +0200 Subject: Issue #19886: Use better estimated memory requirements for bigmem tests. Incorrect requirements can cause memory swapping. --- Lib/test/pickletester.py | 12 +++++------- Lib/test/test_bz2.py | 2 +- Lib/test/test_hashlib.py | 18 ++++++------------ Lib/test/test_marshal.py | 10 +++++----- Lib/test/test_xml_etree_c.py | 4 +--- Misc/NEWS | 2 ++ 6 files changed, 20 insertions(+), 28 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 5ce46dd..4d59bde 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -19,8 +19,6 @@ from pickle import bytes_types # kind of outer loop. protocols = range(pickle.HIGHEST_PROTOCOL + 1) -ascii_char_size = 1 - # Return True if opcode code appears in the pickle, else False. def opcode_in_pickle(code, pickle): @@ -1331,7 +1329,7 @@ class BigmemPickleTests(unittest.TestCase): # Binary protocols can serialize longs of up to 2GB-1 - @bigmemtest(size=_2G, memuse=1 + 1, dry_run=False) + @bigmemtest(size=_2G, memuse=3.6, dry_run=False) def test_huge_long_32b(self, size): data = 1 << (8 * size) try: @@ -1347,7 +1345,7 @@ class BigmemPickleTests(unittest.TestCase): # (older protocols don't have a dedicated opcode for bytes and are # too inefficient) - @bigmemtest(size=_2G, memuse=1 + 1, dry_run=False) + @bigmemtest(size=_2G, memuse=2.5, dry_run=False) def test_huge_bytes_32b(self, size): data = b"abcd" * (size // 4) try: @@ -1363,7 +1361,7 @@ class BigmemPickleTests(unittest.TestCase): finally: data = None - @bigmemtest(size=_4G, memuse=1 + 1, dry_run=False) + @bigmemtest(size=_4G, memuse=2.5, dry_run=False) def test_huge_bytes_64b(self, size): data = b"a" * size try: @@ -1378,7 +1376,7 @@ class BigmemPickleTests(unittest.TestCase): # All protocols use 1-byte per printable ASCII character; we add another # byte because the encoded form has to be copied into the internal buffer. - @bigmemtest(size=_2G, memuse=2 + ascii_char_size, dry_run=False) + @bigmemtest(size=_2G, memuse=8, dry_run=False) def test_huge_str_32b(self, size): data = "abcd" * (size // 4) try: @@ -1395,7 +1393,7 @@ class BigmemPickleTests(unittest.TestCase): # BINUNICODE (protocols 1, 2 and 3) cannot carry more than # 2**32 - 1 bytes of utf-8 encoded unicode. - @bigmemtest(size=_4G, memuse=1 + ascii_char_size, dry_run=False) + @bigmemtest(size=_4G, memuse=8, dry_run=False) def test_huge_str_64b(self, size): data = "a" * size try: diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 98557f2..961053d 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -679,7 +679,7 @@ class BZ2DecompressorTest(BaseTest): self.assertRaises(EOFError, bz2d.decompress, b"anything") self.assertRaises(EOFError, bz2d.decompress, b"") - @bigmemtest(size=_4G + 100, memuse=3) + @bigmemtest(size=_4G + 100, memuse=3.3) def testDecompress4G(self, size): # "Test BZ2Decompressor.decompress() with >4GiB input" blocksize = 10 * 1024 * 1024 diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index f3385f6..c0470d6 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -235,21 +235,15 @@ class HashLibTestCase(unittest.TestCase): b'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789', 'd174ab98d277d9f5a5611c2c9f419d9f') - @bigmemtest(size=_4G + 5, memuse=1) + @unittest.skipIf(sys.maxsize < _4G + 5, 'test cannot run on 32-bit systems') + @bigmemtest(size=_4G + 5, memuse=1, dry_run=False) def test_case_md5_huge(self, size): - if size == _4G + 5: - try: - self.check('md5', b'A'*size, 'c9af2dff37468ce5dfee8f2cfc0a9c6d') - except OverflowError: - pass # 32-bit arch + self.check('md5', b'A'*size, 'c9af2dff37468ce5dfee8f2cfc0a9c6d') - @bigmemtest(size=_4G - 1, memuse=1) + @unittest.skipIf(sys.maxsize < _4G - 1, 'test cannot run on 32-bit systems') + @bigmemtest(size=_4G - 1, memuse=1, dry_run=False) def test_case_md5_uintmax(self, size): - if size == _4G - 1: - try: - self.check('md5', b'A'*size, '28138d306ff1b8281f1a9067e1a1a2b3') - except OverflowError: - pass # 32-bit arch + self.check('md5', b'A'*size, '28138d306ff1b8281f1a9067e1a1a2b3') # use the three examples from Federal Information Processing Standards # Publication 180-1, Secure Hash Standard, 1995 April 17 diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 7e37f39..af7a959 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -303,19 +303,19 @@ class LargeValuesTestCase(unittest.TestCase): def check_unmarshallable(self, data): self.assertRaises(ValueError, marshal.dump, data, NullWriter()) - @support.bigmemtest(size=LARGE_SIZE, memuse=1, dry_run=False) + @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False) def test_bytes(self, size): self.check_unmarshallable(b'x' * size) - @support.bigmemtest(size=LARGE_SIZE, memuse=1, dry_run=False) + @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False) def test_str(self, size): self.check_unmarshallable('x' * size) - @support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size, dry_run=False) + @support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size + 1, dry_run=False) def test_tuple(self, size): self.check_unmarshallable((None,) * size) - @support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size, dry_run=False) + @support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size + 1, dry_run=False) def test_list(self, size): self.check_unmarshallable([None] * size) @@ -331,7 +331,7 @@ class LargeValuesTestCase(unittest.TestCase): def test_frozenset(self, size): self.check_unmarshallable(frozenset(range(size))) - @support.bigmemtest(size=LARGE_SIZE, memuse=1, dry_run=False) + @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False) def test_bytearray(self, size): self.check_unmarshallable(bytearray(size)) diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py index bcaa724..d04b7dc 100644 --- a/Lib/test/test_xml_etree_c.py +++ b/Lib/test/test_xml_etree_c.py @@ -12,10 +12,8 @@ cET_alias = import_fresh_module('xml.etree.cElementTree', class MiscTests(unittest.TestCase): # Issue #8651. - @support.bigmemtest(size=support._2G + 100, memuse=1) + @support.bigmemtest(size=support._2G + 100, memuse=1, dry_run=False) def test_length_overflow(self, size): - if size < support._2G + 100: - self.skipTest("not enough free memory, need at least 2 GB") data = b'x' * size parser = cET.XMLParser() try: diff --git a/Misc/NEWS b/Misc/NEWS index 220289d..65cb245 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -234,6 +234,8 @@ IDLE Tests ----- +- Issue #19886: Use better estimated memory requirements for bigmem tests. + - Issue #20055: Fix test_shutil under Windows with symlink privileges held. Patch by Vajrasky Kok. -- cgit v0.12