From a6df938fefd5b51d85a45cf3f29f60ee7bb21edf Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 8 Apr 2013 22:35:02 +0300 Subject: Close #17666: Fix reading gzip files with an extra field. --- Lib/gzip.py | 3 ++- Lib/test/test_gzip.py | 7 +++++++ Misc/NEWS | 17 ++++++++++++++--- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/Lib/gzip.py b/Lib/gzip.py index 8fb1ed0..0adfd3f 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -280,7 +280,8 @@ class GzipFile(io.BufferedIOBase): if flag & FEXTRA: # Read & discard the extra field, if present - self._read_exact(struct.unpack(" Date: Sun, 12 May 2013 11:09:11 +0200 Subject: Issue #15535: Fix pickling of named tuples. --- Lib/collections.py | 4 ++++ Lib/test/test_collections.py | 1 + Misc/NEWS | 3 +++ 3 files changed, 8 insertions(+) diff --git a/Lib/collections.py b/Lib/collections.py index eb20243..33aedd9 100644 --- a/Lib/collections.py +++ b/Lib/collections.py @@ -281,6 +281,10 @@ class {typename}(tuple): 'Return self as a plain tuple. Used by copy and pickle.' return tuple(self) + def __getstate__(self): + 'Exclude the OrderedDict from pickling' + return None + {field_defs} ''' diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index b2a5f05..c18256b 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -272,6 +272,7 @@ class TestNamedTuple(unittest.TestCase): q = loads(dumps(p, protocol)) self.assertEqual(p, q) self.assertEqual(p._fields, q._fields) + self.assertNotIn(b'OrderedDict', dumps(p, protocol)) def test_copy(self): p = TestNT(x=10, y=20, z=30) diff --git a/Misc/NEWS b/Misc/NEWS index e3e203f..a062175 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -16,6 +16,9 @@ Library - Issue #17666: Fix reading gzip files with an extra field. +- Issue #15535: Fix namedtuple pickles which were picking up the OrderedDict + instead of just the underlying tuple. + What's New in Python 3.2.4? =========================== -- cgit v0.12 From ba2f8be4c6ff755c79d4aeb6805de6bdd0354726 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 12 May 2013 11:11:51 +0200 Subject: Issue #17843: Remove bz2 test data that triggers antivirus warnings. --- Lib/test/test_bz2.py | 13 ++++++------- Lib/test/testbz2_bigmem.bz2 | Bin 3023 -> 0 bytes Misc/NEWS | 6 ++++++ 3 files changed, 12 insertions(+), 7 deletions(-) delete mode 100644 Lib/test/testbz2_bigmem.bz2 diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index fb104d7..bcf80dc 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -26,9 +26,6 @@ class BaseTest(unittest.TestCase): DATA_CRLF = b'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' EMPTY_DATA = b'BZh9\x17rE8P\x90\x00\x00\x00\x00' - with open(findfile("testbz2_bigmem.bz2"), "rb") as f: - DATA_BIGMEM = f.read() - if has_cmdline_bunzip2: def decompress(self, data): pop = subprocess.Popen("bunzip2", shell=True, @@ -395,9 +392,10 @@ class BZ2DecompressorTest(BaseTest): @bigmemtest(size=_4G, memuse=1.25, dry_run=False) def testBigmem(self, unused_size): # Issue #14398: decompression fails when output data is >=2GB. - text = bz2.BZ2Decompressor().decompress(self.DATA_BIGMEM) + compressed = bz2.compress(b"a" * _4G) + text = bz2.BZ2Decompressor().decompress(compressed) self.assertEqual(len(text), _4G) - self.assertEqual(text.strip(b"\0"), b"") + self.assertEqual(text.strip(b"a"), b"") class FuncTest(BaseTest): @@ -442,9 +440,10 @@ class FuncTest(BaseTest): @bigmemtest(size=_4G, memuse=1.25, dry_run=False) def testDecompressBigmem(self, unused_size): # Issue #14398: decompression fails when output data is >=2GB. - text = bz2.decompress(self.DATA_BIGMEM) + compressed = bz2.compress(b"a" * _4G) + text = bz2.BZ2Decompressor().decompress(compressed) self.assertEqual(len(text), _4G) - self.assertEqual(text.strip(b"\0"), b"") + self.assertEqual(text.strip(b"a"), b"") def test_main(): support.run_unittest( diff --git a/Lib/test/testbz2_bigmem.bz2 b/Lib/test/testbz2_bigmem.bz2 deleted file mode 100644 index c9a4616..0000000 Binary files a/Lib/test/testbz2_bigmem.bz2 and /dev/null differ diff --git a/Misc/NEWS b/Misc/NEWS index a062175..8710c10 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -19,6 +19,12 @@ Library - Issue #15535: Fix namedtuple pickles which were picking up the OrderedDict instead of just the underlying tuple. +Tests +----- + +- Issue #17843: Removed bz2 test data file that was triggering false-positive + virus warnings with certain antivirus software. + What's New in Python 3.2.4? =========================== -- cgit v0.12 From a9217a42e62dc8dcf9bb6a184bcf9d5de97d2d9e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 28 Apr 2013 14:10:27 +0300 Subject: Issue #17857: Prevent build failures with pre-3.5.0 versions of sqlite3, such as was shipped with Centos 5 and Mac OS X 10.4. --- Misc/NEWS | 6 ++++++ Modules/_sqlite/cursor.c | 2 +- Modules/_sqlite/util.c | 8 ++++---- Modules/_sqlite/util.h | 4 ++-- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 8710c10..cdf4f37 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -19,6 +19,12 @@ Library - Issue #15535: Fix namedtuple pickles which were picking up the OrderedDict instead of just the underlying tuple. +Build +----- + +- Issue #17857: Prevent build failures with pre-3.5.0 versions of sqlite3, + such as was shipped with Centos 5 and Mac OS X 10.4. + Tests ----- diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index fc4b608..30c2206 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -716,7 +716,7 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* Py_DECREF(self->lastrowid); if (!multiple && statement_type == STATEMENT_INSERT) { - sqlite3_int64 lastrowid; + sqlite_int64 lastrowid; Py_BEGIN_ALLOW_THREADS lastrowid = sqlite3_last_insert_rowid(self->connection->db); Py_END_ALLOW_THREADS diff --git a/Modules/_sqlite/util.c b/Modules/_sqlite/util.c index a1a462c..d88bc3f 100644 --- a/Modules/_sqlite/util.c +++ b/Modules/_sqlite/util.c @@ -111,7 +111,7 @@ int _pysqlite_seterror(sqlite3* db, sqlite3_stmt* st) #endif PyObject * -_pysqlite_long_from_int64(sqlite3_int64 value) +_pysqlite_long_from_int64(sqlite_int64 value) { #ifdef HAVE_LONG_LONG # if SIZEOF_LONG_LONG < 8 @@ -135,7 +135,7 @@ _pysqlite_long_from_int64(sqlite3_int64 value) return PyLong_FromLong(value); } -sqlite3_int64 +sqlite_int64 _pysqlite_long_as_int64(PyObject * py_val) { int overflow; @@ -158,8 +158,8 @@ _pysqlite_long_as_int64(PyObject * py_val) #endif return value; } - else if (sizeof(value) < sizeof(sqlite3_int64)) { - sqlite3_int64 int64val; + else if (sizeof(value) < sizeof(sqlite_int64)) { + sqlite_int64 int64val; if (_PyLong_AsByteArray((PyLongObject *)py_val, (unsigned char *)&int64val, sizeof(int64val), IS_LITTLE_ENDIAN, 1 /* signed */) >= 0) { diff --git a/Modules/_sqlite/util.h b/Modules/_sqlite/util.h index ca02149..88ea906 100644 --- a/Modules/_sqlite/util.h +++ b/Modules/_sqlite/util.h @@ -36,7 +36,7 @@ int pysqlite_step(sqlite3_stmt* statement, pysqlite_Connection* connection); */ int _pysqlite_seterror(sqlite3* db, sqlite3_stmt* st); -PyObject * _pysqlite_long_from_int64(sqlite3_int64 value); -sqlite3_int64 _pysqlite_long_as_int64(PyObject * value); +PyObject * _pysqlite_long_from_int64(sqlite_int64 value); +sqlite_int64 _pysqlite_long_as_int64(PyObject * value); #endif -- cgit v0.12 From 93b061bc3e1c9285ec1ce6405b85d3a1e072833f Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 12 May 2013 11:29:27 +0200 Subject: Issue #1159051: Back out a fix for handling corrupted gzip files that broke backwards compatibility. --- Lib/gzip.py | 73 +++++++++++++++++++++++++++------------------------ Lib/test/test_bz2.py | 18 ------------- Lib/test/test_gzip.py | 13 --------- Misc/NEWS | 5 ++-- 4 files changed, 41 insertions(+), 68 deletions(-) mode change 100644 => 100755 Lib/test/test_gzip.py diff --git a/Lib/gzip.py b/Lib/gzip.py index 0adfd3f..6aacc9a 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -33,6 +33,9 @@ def write32u(output, value): # or unsigned. output.write(struct.pack(" self.extrasize: - if not self._read(readsize): - if size > self.extrasize: - size = self.extrasize - break - readsize = min(self.max_read_chunk, readsize * 2) + try: + while size > self.extrasize: + self._read(readsize) + readsize = min(self.max_read_chunk, readsize * 2) + except EOFError: + if size > self.extrasize: + size = self.extrasize offset = self.offset - self.extrastart chunk = self.extrabuf[offset: offset + size] @@ -365,9 +366,12 @@ class GzipFile(io.BufferedIOBase): if self.extrasize == 0: if self.fileobj is None: return b'' - # Ensure that we don't return b"" if we haven't reached EOF. - # 1024 is the same buffering heuristic used in read() - while self.extrasize == 0 and self._read(max(n, 1024)): + try: + # Ensure that we don't return b"" if we haven't reached EOF. + while self.extrasize == 0: + # 1024 is the same buffering heuristic used in read() + self._read(max(n, 1024)) + except EOFError: pass offset = self.offset - self.extrastart remaining = self.extrasize @@ -380,14 +384,13 @@ class GzipFile(io.BufferedIOBase): def _read(self, size=1024): if self.fileobj is None: - return False + raise EOFError("Reached EOF") if self._new_member: # If the _new_member flag is set, we have to # jump to the next member, if there is one. self._init_read() - if not self._read_gzip_header(): - return False + self._read_gzip_header() self.decompress = zlib.decompressobj(-zlib.MAX_WBITS) self._new_member = False @@ -404,7 +407,7 @@ class GzipFile(io.BufferedIOBase): self.fileobj.prepend(self.decompress.unused_data, True) self._read_eof() self._add_read_data( uncompress ) - return False + raise EOFError('Reached EOF') uncompress = self.decompress.decompress(buf) self._add_read_data( uncompress ) @@ -420,7 +423,6 @@ class GzipFile(io.BufferedIOBase): # a new member on the next call self._read_eof() self._new_member = True - return True def _add_read_data(self, data): self.crc = zlib.crc32(data, self.crc) & 0xffffffff @@ -435,7 +437,8 @@ class GzipFile(io.BufferedIOBase): # We check the that the computed CRC and size of the # uncompressed data matches the stored values. Note that the size # stored is the true file size mod 2**32. - crc32, isize = struct.unpack("