diff options
| -rw-r--r-- | Lib/test/test_zlib.py | 29 | ||||
| -rw-r--r-- | Misc/NEWS | 6 | ||||
| -rw-r--r-- | Modules/zlibmodule.c | 24 |
3 files changed, 57 insertions, 2 deletions
diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 5615c2d..9aafffa 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -2,10 +2,16 @@ import unittest from test import support import binascii import random +import sys from test.support import precisionbigmemtest, _1G, _4G zlib = support.import_module('zlib') +try: + import mmap +except ImportError: + mmap = None + class ChecksumTestCase(unittest.TestCase): # checksum test cases @@ -57,6 +63,28 @@ class ChecksumTestCase(unittest.TestCase): self.assertEqual(binascii.crc32(b'spam'), zlib.crc32(b'spam')) +# Issue #10276 - check that inputs >=4GB are handled correctly. +class ChecksumBigBufferTestCase(unittest.TestCase): + + def setUp(self): + with open(support.TESTFN, "wb+") as f: + f.seek(_4G) + f.write(b"asdf") + f.flush() + self.mapping = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) + + def tearDown(self): + self.mapping.close() + support.unlink(support.TESTFN) + + @unittest.skipUnless(mmap, "mmap() is not available.") + @unittest.skipUnless(sys.maxsize > _4G, "Can't run on a 32-bit system.") + @unittest.skipUnless(support.is_resource_enabled("largefile"), + "May use lots of disk space.") + def test_big_buffer(self): + self.assertEqual(zlib.crc32(self.mapping), 3058686908) + self.assertEqual(zlib.adler32(self.mapping), 82837919) + class ExceptionTestCase(unittest.TestCase): # make sure we generate some expected errors @@ -577,6 +605,7 @@ LAERTES def test_main(): support.run_unittest( ChecksumTestCase, + ChecksumBigBufferTestCase, ExceptionTestCase, CompressTestCase, CompressObjectTestCase @@ -12,6 +12,12 @@ Core and Builtins - Check for NULL result in PyType_FromSpec. +Library +------- + +- Issue #10276: Fix the results of zlib.crc32() and zlib.adler32() on buffers + larger than 4GB. Patch by Nadeem Vawda. + What's New in Python 3.2? ========================= diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index a03045e..e439c5b 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -945,8 +945,18 @@ PyZlib_adler32(PyObject *self, PyObject *args) /* Releasing the GIL for very small buffers is inefficient and may lower performance */ if (pbuf.len > 1024*5) { + void *buf = pbuf.buf; + Py_ssize_t len = pbuf.len; + Py_BEGIN_ALLOW_THREADS - adler32val = adler32(adler32val, pbuf.buf, pbuf.len); + /* Avoid truncation of length for very large buffers. adler32() takes + length as an unsigned int, which may be narrower than Py_ssize_t. */ + while (len > (Py_ssize_t)UINT_MAX) { + adler32val = adler32(adler32val, buf, UINT_MAX); + buf += UINT_MAX; + len -= UINT_MAX; + } + adler32val = adler32(adler32val, buf, len); Py_END_ALLOW_THREADS } else { adler32val = adler32(adler32val, pbuf.buf, pbuf.len); @@ -973,8 +983,18 @@ PyZlib_crc32(PyObject *self, PyObject *args) /* Releasing the GIL for very small buffers is inefficient and may lower performance */ if (pbuf.len > 1024*5) { + void *buf = pbuf.buf; + Py_ssize_t len = pbuf.len; + Py_BEGIN_ALLOW_THREADS - signed_val = crc32(crc32val, pbuf.buf, pbuf.len); + /* Avoid truncation of length for very large buffers. crc32() takes + length as an unsigned int, which may be narrower than Py_ssize_t. */ + while (len > (Py_ssize_t)UINT_MAX) { + crc32val = crc32(crc32val, buf, UINT_MAX); + buf += UINT_MAX; + len -= UINT_MAX; + } + signed_val = crc32(crc32val, buf, len); Py_END_ALLOW_THREADS } else { signed_val = crc32(crc32val, pbuf.buf, pbuf.len); |