From 77b286b2ccccc407d8ed17b0543b10f7c1ccc864 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 27 Jan 2014 00:53:38 +1000 Subject: Close #20105: set __traceback__ when chaining exceptions in C --- Doc/c-api/exceptions.rst | 10 ++++++++++ Doc/whatsnew/3.4.rst | 8 ++++++++ Lib/test/test_codecs.py | 1 + Misc/NEWS | 3 +++ Objects/exceptions.c | 7 +++++-- 5 files changed, 27 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 8658a58..d4065e0 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -90,6 +90,16 @@ in various ways. There is a separate error indicator for each thread. the class in that case. If the values are already normalized, nothing happens. The delayed normalization is implemented to improve performance. + .. note:: + + This function *does not* implicitly set the ``__traceback__`` + attribute on the exception value. If setting the traceback + appropriately is desired, the following additional snippet is needed:: + + if (tb != NULL) { + PyException_SetTraceback(val, tb); + } + .. c:function:: void PyErr_Clear() diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 5d397fe..1d88965 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -264,6 +264,9 @@ name of the codec responsible for producing the error:: >>> import codecs >>> codecs.decode(b"abcdefgh", "hex") + Traceback (most recent call last): + File "/usr/lib/python3.4/encodings/hex_codec.py", line 20, in hex_decode + return (binascii.a2b_hex(input), len(input)) binascii.Error: Non-hexadecimal digit found The above exception was the direct cause of the following exception: @@ -273,6 +276,11 @@ name of the codec responsible for producing the error:: binascii.Error: decoding with 'hex' codec failed (Error: Non-hexadecimal digit found) >>> codecs.encode("hello", "bz2") + Traceback (most recent call last): + File "/usr/lib/python3.4/encodings/bz2_codec.py", line 17, in bz2_encode + return (bz2.compress(input), len(input)) + File "/usr/lib/python3.4/bz2.py", line 498, in compress + return comp.compress(data) + comp.flush() TypeError: 'str' does not support the buffer interface The above exception was the direct cause of the following exception: diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index f9c9e69..a32ce76 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -2522,6 +2522,7 @@ class ExceptionChainingTest(unittest.TestCase): with self.assertRaisesRegex(exc_type, full_msg) as caught: yield caught self.assertIsInstance(caught.exception.__cause__, exc_type) + self.assertIsNotNone(caught.exception.__cause__.__traceback__) def raise_obj(self, *args, **kwds): # Helper to dynamically change the object raised by a test codec diff --git a/Misc/NEWS b/Misc/NEWS index c248d7c..2516735 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -36,6 +36,9 @@ Core and Builtins Library ------- +- Issue #20105: the codec exception chaining now correctly sets the + traceback of the original exception as its __traceback__ attribute. + - asyncio: Various improvements and small changes not all covered by issues listed below. E.g. wait_for() now cancels the inner task if the timeout occcurs; tweaked the set of exported symbols; renamed diff --git a/Objects/exceptions.c b/Objects/exceptions.c index bff7f08..2531ead 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -2689,8 +2689,11 @@ _PyErr_TrySetFromCause(const char *format, ...) * types as well, but that's quite a bit trickier due to the extra * state potentially stored on OSError instances. */ - - Py_XDECREF(tb); + /* Ensure the traceback is set correctly on the existing exception */ + if (tb != NULL) { + PyException_SetTraceback(val, tb); + Py_DECREF(tb); + } #ifdef HAVE_STDARG_PROTOTYPES va_start(vargs, format); -- cgit v0.12 From 08673c57f02cf85bc8336cdd7cc90ee530c41ecc Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 26 Jan 2014 10:24:24 -0500 Subject: fix refleak on error --- Modules/audioop.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Modules/audioop.c b/Modules/audioop.c index e0610f3..cc3a020 100644 --- a/Modules/audioop.c +++ b/Modules/audioop.c @@ -1608,7 +1608,7 @@ audioop_lin2adpcm_impl(PyModuleDef *module, Py_buffer *fragment, int width, PyOb Py_ssize_t i; int step, valpred, delta, index, sign, vpdiff, diff; - PyObject *rv, *str; + PyObject *rv = NULL, *str; int outputbuffer = 0, bufferstep; if (!audioop_check_parameters(fragment->len, width)) @@ -1626,9 +1626,10 @@ audioop_lin2adpcm_impl(PyModuleDef *module, Py_buffer *fragment, int width, PyOb index = 0; } else if (!PyTuple_Check(state)) { PyErr_SetString(PyExc_TypeError, "state must be a tuple or None"); - return NULL; - } else if (!PyArg_ParseTuple(state, "ii", &valpred, &index)) - return NULL; + goto exit; + } else if (!PyArg_ParseTuple(state, "ii", &valpred, &index)) { + goto exit; + } step = stepsizeTable[index]; bufferstep = 1; @@ -1704,6 +1705,8 @@ audioop_lin2adpcm_impl(PyModuleDef *module, Py_buffer *fragment, int width, PyOb bufferstep = !bufferstep; } rv = Py_BuildValue("(O(ii))", str, valpred, index); + + exit: Py_DECREF(str); return rv; } -- cgit v0.12 From b62deac9a3aa31b26a98ef6f846d4de506d9322f Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 26 Jan 2014 10:41:58 -0500 Subject: cleanup after custom buffer converter --- Modules/binascii.c | 5 +++++ Modules/clinic/binascii.c.h | 32 ++++++++++++++++++++++++++------ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/Modules/binascii.c b/Modules/binascii.c index 16f6361..d38182e 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -195,6 +195,11 @@ class ascii_buffer_converter(CConverter): type = 'Py_buffer' converter = 'ascii_buffer_converter' impl_by_reference = True + c_default = "{NULL, NULL}" + + def cleanup(self): + name = self.name + return "".join(["if (", name, ".obj)\n PyBuffer_Release(&", name, ");\n"]) [python start generated code]*/ /*[python end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/ diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h index 3e4a468..cbafc68 100644 --- a/Modules/clinic/binascii.c.h +++ b/Modules/clinic/binascii.c.h @@ -16,7 +16,7 @@ static PyObject * binascii_a2b_uu(PyModuleDef *module, PyObject *args) { PyObject *return_value = NULL; - Py_buffer data; + Py_buffer data = {NULL, NULL}; if (!PyArg_ParseTuple(args, "O&:a2b_uu", @@ -25,6 +25,10 @@ binascii_a2b_uu(PyModuleDef *module, PyObject *args) return_value = binascii_a2b_uu_impl(module, &data); exit: + /* Cleanup for data */ + if (data.obj) + PyBuffer_Release(&data); + return return_value; } @@ -72,7 +76,7 @@ static PyObject * binascii_a2b_base64(PyModuleDef *module, PyObject *args) { PyObject *return_value = NULL; - Py_buffer data; + Py_buffer data = {NULL, NULL}; if (!PyArg_ParseTuple(args, "O&:a2b_base64", @@ -81,6 +85,10 @@ binascii_a2b_base64(PyModuleDef *module, PyObject *args) return_value = binascii_a2b_base64_impl(module, &data); exit: + /* Cleanup for data */ + if (data.obj) + PyBuffer_Release(&data); + return return_value; } @@ -128,7 +136,7 @@ static PyObject * binascii_a2b_hqx(PyModuleDef *module, PyObject *args) { PyObject *return_value = NULL; - Py_buffer data; + Py_buffer data = {NULL, NULL}; if (!PyArg_ParseTuple(args, "O&:a2b_hqx", @@ -137,6 +145,10 @@ binascii_a2b_hqx(PyModuleDef *module, PyObject *args) return_value = binascii_a2b_hqx_impl(module, &data); exit: + /* Cleanup for data */ + if (data.obj) + PyBuffer_Release(&data); + return return_value; } @@ -350,7 +362,7 @@ static PyObject * binascii_a2b_hex(PyModuleDef *module, PyObject *args) { PyObject *return_value = NULL; - Py_buffer hexstr; + Py_buffer hexstr = {NULL, NULL}; if (!PyArg_ParseTuple(args, "O&:a2b_hex", @@ -359,6 +371,10 @@ binascii_a2b_hex(PyModuleDef *module, PyObject *args) return_value = binascii_a2b_hex_impl(module, &hexstr); exit: + /* Cleanup for hexstr */ + if (hexstr.obj) + PyBuffer_Release(&hexstr); + return return_value; } @@ -377,7 +393,7 @@ binascii_a2b_qp(PyModuleDef *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; static char *_keywords[] = {"data", "header", NULL}; - Py_buffer data; + Py_buffer data = {NULL, NULL}; int header = 0; if (!PyArg_ParseTupleAndKeywords(args, kwargs, @@ -387,6 +403,10 @@ binascii_a2b_qp(PyModuleDef *module, PyObject *args, PyObject *kwargs) return_value = binascii_a2b_qp_impl(module, &data, header); exit: + /* Cleanup for data */ + if (data.obj) + PyBuffer_Release(&data); + return return_value; } @@ -427,4 +447,4 @@ exit: return return_value; } -/*[clinic end generated code: checksum=abe48ca8020fa3ec25e13bd9fa7414f6b3ee2946]*/ +/*[clinic end generated code: checksum=8180e5be47a110ae8c89263a7c12a91d80754f60]*/ -- cgit v0.12 From 0742cae3357fcd7c41498b21060050e7cca788b1 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 26 Jan 2014 18:36:01 +0200 Subject: Merge latest Tulip into asyncio --- Lib/asyncio/base_subprocess.py | 1 - Lib/asyncio/unix_events.py | 3 --- Lib/asyncio/windows_events.py | 3 --- 3 files changed, 7 deletions(-) diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index d15fb15..c5efda7 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -114,7 +114,6 @@ class BaseSubprocessTransport(transports.SubprocessTransport): assert returncode is not None, returncode assert self._returncode is None, self._returncode self._returncode = returncode - self._loop._subprocess_closed(self) self._call(self._protocol.process_exited) self._try_finish() diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 2418642..219c88a 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -169,9 +169,6 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): def _child_watcher_callback(self, pid, returncode, transp): self.call_soon_threadsafe(transp._process_exited, returncode) - def _subprocess_closed(self, transp): - pass - def _set_nonblocking(fd): flags = fcntl.fcntl(fd, fcntl.F_GETFL) diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index 2e9ec69..3c21e43 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -178,9 +178,6 @@ class ProactorEventLoop(proactor_events.BaseProactorEventLoop): yield from transp._post_init() return transp - def _subprocess_closed(self, transport): - pass - class IocpProactor: """Proactor implementation using IOCP.""" -- cgit v0.12 From 8003850e22788c9e02b8dea7076858e01aa24f65 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 26 Jan 2014 19:21:00 +0200 Subject: Issue #8260: The read(), readline() and readlines() methods of codecs.StreamReader returned incomplete data when were called after readline() or read(size). Based on patch by Amaury Forgeot d'Arc. --- Lib/codecs.py | 13 ++++++------- Lib/test/test_codecs.py | 36 ++++++++++++++++++++++++++++++++++-- Misc/NEWS | 4 ++++ 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/Lib/codecs.py b/Lib/codecs.py index 6a6eb90..01ae0f3 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -463,15 +463,12 @@ class StreamReader(Codec): # read until we get the required number of characters (if available) while True: # can the request be satisfied from the character buffer? - if chars < 0: - if size < 0: - if self.charbuffer: - break - elif len(self.charbuffer) >= size: - break - else: + if chars >= 0: if len(self.charbuffer) >= chars: break + elif size >= 0: + if len(self.charbuffer) >= size: + break # we need more data if size < 0: newdata = self.stream.read() @@ -479,6 +476,8 @@ class StreamReader(Codec): newdata = self.stream.read(size) # decode bytes (those remaining from the last call included) data = self.bytebuffer + newdata + if not data: + break try: newchars, decodedbytes = self.decode(data, self.errors) except UnicodeDecodeError as exc: diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 3517057..5c51ef5 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -173,6 +173,40 @@ class ReadTest(MixInCheckStateHandling): size*"a", ) + def test_mixed_readline_and_read(self): + lines = ["Humpty Dumpty sat on a wall,\n", + "Humpty Dumpty had a great fall.\r\n", + "All the king's horses and all the king's men\r", + "Couldn't put Humpty together again."] + data = ''.join(lines) + def getreader(): + stream = io.BytesIO(data.encode(self.encoding)) + return codecs.getreader(self.encoding)(stream) + + # Issue #8260: Test readline() followed by read() + f = getreader() + self.assertEqual(f.readline(), lines[0]) + self.assertEqual(f.read(), ''.join(lines[1:])) + self.assertEqual(f.read(), '') + + # Issue #16636: Test readline() followed by readlines() + f = getreader() + self.assertEqual(f.readline(), lines[0]) + self.assertEqual(f.readlines(), lines[1:]) + self.assertEqual(f.read(), '') + + # Test read() followed by read() + f = getreader() + self.assertEqual(f.read(size=40, chars=5), data[:5]) + self.assertEqual(f.read(), data[5:]) + self.assertEqual(f.read(), '') + + # Issue #12446: Test read() followed by readlines() + f = getreader() + self.assertEqual(f.read(size=40, chars=5), data[:5]) + self.assertEqual(f.readlines(), [lines[0][5:]] + lines[1:]) + self.assertEqual(f.read(), '') + def test_bug1175396(self): s = [ '<%!--===================================================\r\n', @@ -2307,8 +2341,6 @@ class TransformCodecTest(unittest.TestCase): def test_readline(self): for encoding in bytes_transform_encodings: - if encoding in ['uu_codec', 'zlib_codec']: - continue sin = codecs.encode(b"\x80", encoding) reader = codecs.getreader(encoding)(io.BytesIO(sin)) sout = reader.readline() diff --git a/Misc/NEWS b/Misc/NEWS index ea6f1d7..89610c9 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -50,6 +50,10 @@ Core and Builtins Library ------- +- Issue #8260: The read(), readline() and readlines() methods of + codecs.StreamReader returned incomplete data when were called after + readline() or read(size). Based on patch by Amaury Forgeot d'Arc. + - Issue #20317: ExitStack.__exit__ could create a self-referential loop if an exception raised by a cleanup operation already had its context set correctly (for example, by the @contextmanager decorator). The infinite -- cgit v0.12 From dbe0982bc515cb1a881d4bf7728d265e58803bf0 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 26 Jan 2014 19:27:56 +0200 Subject: Issue #8260: The read(), readline() and readlines() methods of codecs.StreamReader returned incomplete data when were called after readline() or read(size). Based on patch by Amaury Forgeot d'Arc. --- Lib/codecs.py | 13 ++++++------- Lib/test/test_codecs.py | 36 ++++++++++++++++++++++++++++++++++-- Misc/NEWS | 4 ++++ 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/Lib/codecs.py b/Lib/codecs.py index 2e2e755..c2065da 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -475,15 +475,12 @@ class StreamReader(Codec): # read until we get the required number of characters (if available) while True: # can the request be satisfied from the character buffer? - if chars < 0: - if size < 0: - if self.charbuffer: - break - elif len(self.charbuffer) >= size: - break - else: + if chars >= 0: if len(self.charbuffer) >= chars: break + elif size >= 0: + if len(self.charbuffer) >= size: + break # we need more data if size < 0: newdata = self.stream.read() @@ -491,6 +488,8 @@ class StreamReader(Codec): newdata = self.stream.read(size) # decode bytes (those remaining from the last call included) data = self.bytebuffer + newdata + if not data: + break try: newchars, decodedbytes = self.decode(data, self.errors) except UnicodeDecodeError as exc: diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index a32ce76..3950c3b 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -175,6 +175,40 @@ class ReadTest(MixInCheckStateHandling): size*"a", ) + def test_mixed_readline_and_read(self): + lines = ["Humpty Dumpty sat on a wall,\n", + "Humpty Dumpty had a great fall.\r\n", + "All the king's horses and all the king's men\r", + "Couldn't put Humpty together again."] + data = ''.join(lines) + def getreader(): + stream = io.BytesIO(data.encode(self.encoding)) + return codecs.getreader(self.encoding)(stream) + + # Issue #8260: Test readline() followed by read() + f = getreader() + self.assertEqual(f.readline(), lines[0]) + self.assertEqual(f.read(), ''.join(lines[1:])) + self.assertEqual(f.read(), '') + + # Issue #16636: Test readline() followed by readlines() + f = getreader() + self.assertEqual(f.readline(), lines[0]) + self.assertEqual(f.readlines(), lines[1:]) + self.assertEqual(f.read(), '') + + # Test read() followed by read() + f = getreader() + self.assertEqual(f.read(size=40, chars=5), data[:5]) + self.assertEqual(f.read(), data[5:]) + self.assertEqual(f.read(), '') + + # Issue #12446: Test read() followed by readlines() + f = getreader() + self.assertEqual(f.read(size=40, chars=5), data[:5]) + self.assertEqual(f.readlines(), [lines[0][5:]] + lines[1:]) + self.assertEqual(f.read(), '') + def test_bug1175396(self): s = [ '<%!--===================================================\r\n', @@ -2370,8 +2404,6 @@ class TransformCodecTest(unittest.TestCase): def test_readline(self): for encoding in bytes_transform_encodings: - if encoding in ['uu_codec', 'zlib_codec']: - continue with self.subTest(encoding=encoding): sin = codecs.encode(b"\x80", encoding) reader = codecs.getreader(encoding)(io.BytesIO(sin)) diff --git a/Misc/NEWS b/Misc/NEWS index 2516735..9c0ed5a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -36,6 +36,10 @@ Core and Builtins Library ------- +- Issue #8260: The read(), readline() and readlines() methods of + codecs.StreamReader returned incomplete data when were called after + readline() or read(size). Based on patch by Amaury Forgeot d'Arc. + - Issue #20105: the codec exception chaining now correctly sets the traceback of the original exception as its __traceback__ attribute. -- cgit v0.12 From e6994ff6e341d049ca974d88f14ad608f5cf7a64 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 26 Jan 2014 09:57:51 -0800 Subject: Fix issue #20367: concurrent.futures.as_completed() for duplicate arguments. Patch by Glenn Langford. --- Doc/library/concurrent.futures.rst | 3 ++- Lib/concurrent/futures/_base.py | 6 ++++-- Lib/test/test_concurrent_futures.py | 7 +++++++ Misc/NEWS | 3 +++ 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 93538e4..0495737 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -371,7 +371,8 @@ Module Functions Returns an iterator over the :class:`Future` instances (possibly created by different :class:`Executor` instances) given by *fs* that yields futures as - they complete (finished or were cancelled). Any futures that completed + they complete (finished or were cancelled). Any futures given by *fs* that + are duplicated will be returned once. Any futures that completed before :func:`as_completed` is called will be yielded first. The returned iterator raises a :exc:`TimeoutError` if :meth:`~iterator.__next__` is called and the result isn't available after *timeout* seconds from the diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py index 3d03280..c3b1f01 100644 --- a/Lib/concurrent/futures/_base.py +++ b/Lib/concurrent/futures/_base.py @@ -181,7 +181,8 @@ def as_completed(fs, timeout=None): Returns: An iterator that yields the given Futures as they complete (finished or - cancelled). + cancelled). If any given Futures are duplicated, they will be returned + once. Raises: TimeoutError: If the entire result iterator could not be generated @@ -190,11 +191,12 @@ def as_completed(fs, timeout=None): if timeout is not None: end_time = timeout + time.time() + fs = set(fs) with _AcquireFutures(fs): finished = set( f for f in fs if f._state in [CANCELLED_AND_NOTIFIED, FINISHED]) - pending = set(fs) - finished + pending = fs - finished waiter = _create_and_install_waiters(fs, _AS_COMPLETED) try: diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index a7e945c..c74b2ca 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -350,6 +350,13 @@ class AsCompletedTests: SUCCESSFUL_FUTURE]), completed_futures) + def test_duplicate_futures(self): + # Issue 20367. Duplicate futures should not raise exceptions or give + # duplicate responses. + future1 = self.executor.submit(time.sleep, 2) + completed = [f for f in futures.as_completed([future1,future1])] + self.assertEqual(len(completed), 1) + class ThreadPoolAsCompletedTests(ThreadPoolMixin, AsCompletedTests, unittest.TestCase): pass diff --git a/Misc/NEWS b/Misc/NEWS index 9c0ed5a..7fb39c0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -36,6 +36,9 @@ Core and Builtins Library ------- +- Issue #20367: Fix behavior of concurrent.futures.as_completed() for + duplicate arguments. Patch by Glenn Langford. + - Issue #8260: The read(), readline() and readlines() methods of codecs.StreamReader returned incomplete data when were called after readline() or read(size). Based on patch by Amaury Forgeot d'Arc. -- cgit v0.12 From 1ac00950b28e91fc35fed6445722ade92732648e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 26 Jan 2014 23:48:38 +0200 Subject: Issue #19990: Added tests for the imghdr module. Based on patch by Claudiu Popa. --- Lib/test/imghdrdata/python.bmp | Bin 0 -> 1162 bytes Lib/test/imghdrdata/python.gif | Bin 0 -> 610 bytes Lib/test/imghdrdata/python.jpg | Bin 0 -> 543 bytes Lib/test/imghdrdata/python.pbm | 3 + Lib/test/imghdrdata/python.pgm | Bin 0 -> 269 bytes Lib/test/imghdrdata/python.png | Bin 0 -> 1020 bytes Lib/test/imghdrdata/python.ppm | Bin 0 -> 781 bytes Lib/test/imghdrdata/python.ras | Bin 0 -> 1056 bytes Lib/test/imghdrdata/python.sgi | Bin 0 -> 1967 bytes Lib/test/imghdrdata/python.tiff | Bin 0 -> 1326 bytes Lib/test/imghdrdata/python.xbm | 6 ++ Lib/test/test_imghdr.py | 131 ++++++++++++++++++++++++++++++++++++++++ Lib/test/test_sundry.py | 1 - Misc/NEWS | 3 + 14 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 Lib/test/imghdrdata/python.bmp create mode 100644 Lib/test/imghdrdata/python.gif create mode 100644 Lib/test/imghdrdata/python.jpg create mode 100644 Lib/test/imghdrdata/python.pbm create mode 100644 Lib/test/imghdrdata/python.pgm create mode 100644 Lib/test/imghdrdata/python.png create mode 100644 Lib/test/imghdrdata/python.ppm create mode 100644 Lib/test/imghdrdata/python.ras create mode 100644 Lib/test/imghdrdata/python.sgi create mode 100644 Lib/test/imghdrdata/python.tiff create mode 100644 Lib/test/imghdrdata/python.xbm create mode 100644 Lib/test/test_imghdr.py diff --git a/Lib/test/imghdrdata/python.bmp b/Lib/test/imghdrdata/python.bmp new file mode 100644 index 0000000..675f951 Binary files /dev/null and b/Lib/test/imghdrdata/python.bmp differ diff --git a/Lib/test/imghdrdata/python.gif b/Lib/test/imghdrdata/python.gif new file mode 100644 index 0000000..96fd9fe Binary files /dev/null and b/Lib/test/imghdrdata/python.gif differ diff --git a/Lib/test/imghdrdata/python.jpg b/Lib/test/imghdrdata/python.jpg new file mode 100644 index 0000000..21222c0 Binary files /dev/null and b/Lib/test/imghdrdata/python.jpg differ diff --git a/Lib/test/imghdrdata/python.pbm b/Lib/test/imghdrdata/python.pbm new file mode 100644 index 0000000..1848ba7 --- /dev/null +++ b/Lib/test/imghdrdata/python.pbm @@ -0,0 +1,3 @@ +P4 +16 16 +[a_X? \ No newline at end of file diff --git a/Lib/test/imghdrdata/python.pgm b/Lib/test/imghdrdata/python.pgm new file mode 100644 index 0000000..8349f2a Binary files /dev/null and b/Lib/test/imghdrdata/python.pgm differ diff --git a/Lib/test/imghdrdata/python.png b/Lib/test/imghdrdata/python.png new file mode 100644 index 0000000..1a987f7 Binary files /dev/null and b/Lib/test/imghdrdata/python.png differ diff --git a/Lib/test/imghdrdata/python.ppm b/Lib/test/imghdrdata/python.ppm new file mode 100644 index 0000000..7d9cdb3 Binary files /dev/null and b/Lib/test/imghdrdata/python.ppm differ diff --git a/Lib/test/imghdrdata/python.ras b/Lib/test/imghdrdata/python.ras new file mode 100644 index 0000000..130e96f Binary files /dev/null and b/Lib/test/imghdrdata/python.ras differ diff --git a/Lib/test/imghdrdata/python.sgi b/Lib/test/imghdrdata/python.sgi new file mode 100644 index 0000000..ffe9081 Binary files /dev/null and b/Lib/test/imghdrdata/python.sgi differ diff --git a/Lib/test/imghdrdata/python.tiff b/Lib/test/imghdrdata/python.tiff new file mode 100644 index 0000000..39d0bfc Binary files /dev/null and b/Lib/test/imghdrdata/python.tiff differ diff --git a/Lib/test/imghdrdata/python.xbm b/Lib/test/imghdrdata/python.xbm new file mode 100644 index 0000000..cfbee2e --- /dev/null +++ b/Lib/test/imghdrdata/python.xbm @@ -0,0 +1,6 @@ +#define python_width 16 +#define python_height 16 +static char python_bits[] = { + 0xDF, 0xFE, 0x8F, 0xFD, 0x5F, 0xFB, 0xAB, 0xFE, 0xB5, 0x8D, 0xDA, 0x8F, + 0xA5, 0x86, 0xFA, 0x83, 0x1A, 0x80, 0x0D, 0x80, 0x0D, 0x80, 0x0F, 0xE0, + 0x0F, 0xF8, 0x0F, 0xF8, 0x0F, 0xFC, 0xFF, 0xFF, }; diff --git a/Lib/test/test_imghdr.py b/Lib/test/test_imghdr.py new file mode 100644 index 0000000..0ad4343 --- /dev/null +++ b/Lib/test/test_imghdr.py @@ -0,0 +1,131 @@ +import imghdr +import io +import os +import unittest +import warnings +from test.support import findfile, TESTFN, unlink + +TEST_FILES = ( + ('python.png', 'png'), + ('python.gif', 'gif'), + ('python.bmp', 'bmp'), + ('python.ppm', 'ppm'), + ('python.pgm', 'pgm'), + ('python.pbm', 'pbm'), + ('python.jpg', 'jpeg'), + ('python.ras', 'rast'), + ('python.sgi', 'rgb'), + ('python.tiff', 'tiff'), + ('python.xbm', 'xbm') +) + +class UnseekableIO(io.FileIO): + def tell(self): + raise io.UnsupportedOperation + + def seek(self, *args, **kwargs): + raise io.UnsupportedOperation + +class TestImghdr(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.testfile = findfile('python.png', subdir='imghdrdata') + with open(cls.testfile, 'rb') as stream: + cls.testdata = stream.read() + + def tearDown(self): + unlink(TESTFN) + + def test_data(self): + for filename, expected in TEST_FILES: + filename = findfile(filename, subdir='imghdrdata') + self.assertEqual(imghdr.what(filename), expected) + with open(filename, 'rb') as stream: + self.assertEqual(imghdr.what(stream), expected) + with open(filename, 'rb') as stream: + data = stream.read() + self.assertEqual(imghdr.what(None, data), expected) + self.assertEqual(imghdr.what(None, bytearray(data)), expected) + + def test_register_test(self): + def test_jumbo(h, file): + if h.startswith(b'eggs'): + return 'ham' + imghdr.tests.append(test_jumbo) + self.addCleanup(imghdr.tests.pop) + self.assertEqual(imghdr.what(None, b'eggs'), 'ham') + + def test_file_pos(self): + with open(TESTFN, 'wb') as stream: + stream.write(b'ababagalamaga') + pos = stream.tell() + stream.write(self.testdata) + with open(TESTFN, 'rb') as stream: + stream.seek(pos) + self.assertEqual(imghdr.what(stream), 'png') + self.assertEqual(stream.tell(), pos) + + def test_bad_args(self): + with self.assertRaises(TypeError): + imghdr.what() + with self.assertRaises(AttributeError): + imghdr.what(None) + with self.assertRaises(TypeError): + imghdr.what(self.testfile, 1) + with self.assertRaises(AttributeError): + imghdr.what(os.fsencode(self.testfile)) + with open(self.testfile, 'rb') as f: + with self.assertRaises(AttributeError): + imghdr.what(f.fileno()) + + def test_invalid_headers(self): + for header in (b'\211PN\r\n', + b'\001\331', + b'\x59\xA6', + b'cutecat', + b'000000JFI', + b'GIF80'): + self.assertIsNone(imghdr.what(None, header)) + + def test_string_data(self): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", BytesWarning) + for filename, _ in TEST_FILES: + filename = findfile(filename, subdir='imghdrdata') + with open(filename, 'rb') as stream: + data = stream.read().decode('latin1') + with self.assertRaises(TypeError): + imghdr.what(io.StringIO(data)) + with self.assertRaises(TypeError): + imghdr.what(None, data) + + def test_missing_file(self): + with self.assertRaises(FileNotFoundError): + imghdr.what('missing') + + def test_closed_file(self): + stream = open(self.testfile, 'rb') + stream.close() + with self.assertRaises(ValueError) as cm: + imghdr.what(stream) + stream = io.BytesIO(self.testdata) + stream.close() + with self.assertRaises(ValueError) as cm: + imghdr.what(stream) + + def test_unseekable(self): + with open(TESTFN, 'wb') as stream: + stream.write(self.testdata) + with UnseekableIO(TESTFN, 'rb') as stream: + with self.assertRaises(io.UnsupportedOperation): + imghdr.what(stream) + + def test_output_stream(self): + with open(TESTFN, 'wb') as stream: + stream.write(self.testdata) + stream.seek(0) + with self.assertRaises(OSError) as cm: + imghdr.what(stream) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_sundry.py b/Lib/test/test_sundry.py index c6cca26..8808c47 100644 --- a/Lib/test/test_sundry.py +++ b/Lib/test/test_sundry.py @@ -42,7 +42,6 @@ class TestUntestedModules(unittest.TestCase): import encodings import formatter import html.entities - import imghdr import keyword import mailcap import nturl2path diff --git a/Misc/NEWS b/Misc/NEWS index 89610c9..10cb08e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -292,6 +292,9 @@ IDLE Tests ----- +- Issue #19990: Added tests for the imghdr module. Based on patch by + Claudiu Popa. + - Issue #19804: The test_find_mac test in test_uuid is now skipped if the ifconfig executable is not available. -- cgit v0.12 From 2c5ddbe030cf56434bca92f97517b941545337b4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 27 Jan 2014 00:03:31 +0200 Subject: Issue #20193: The zlib module now uses Argument Clinic. --- Modules/clinic/zlibmodule.c.h | 411 ++++++++++++++++++++++++++++++++++ Modules/zlibmodule.c | 503 ++++++++++++++++-------------------------- 2 files changed, 607 insertions(+), 307 deletions(-) create mode 100644 Modules/clinic/zlibmodule.c.h diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h new file mode 100644 index 0000000..0adeb01 --- /dev/null +++ b/Modules/clinic/zlibmodule.c.h @@ -0,0 +1,411 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(zlib_compress__doc__, +"compress(module, bytes, level=Z_DEFAULT_COMPRESSION)\n" +"Returns a bytes object containing compressed data.\n" +"\n" +" bytes\n" +" Binary data to be compressed.\n" +" level\n" +" Compression level, in 0-9."); + +#define ZLIB_COMPRESS_METHODDEF \ + {"compress", (PyCFunction)zlib_compress, METH_VARARGS, zlib_compress__doc__}, + +static PyObject * +zlib_compress_impl(PyModuleDef *module, Py_buffer *bytes, int level); + +static PyObject * +zlib_compress(PyModuleDef *module, PyObject *args) +{ + PyObject *return_value = NULL; + Py_buffer bytes = {NULL, NULL}; + int level = Z_DEFAULT_COMPRESSION; + + if (!PyArg_ParseTuple(args, + "y*|i:compress", + &bytes, &level)) + goto exit; + return_value = zlib_compress_impl(module, &bytes, level); + +exit: + /* Cleanup for bytes */ + if (bytes.obj) + PyBuffer_Release(&bytes); + + return return_value; +} + +PyDoc_STRVAR(zlib_decompress__doc__, +"decompress(module, data, wbits=MAX_WBITS, bufsize=DEF_BUF_SIZE)\n" +"Returns a bytes object containing the uncompressed data.\n" +"\n" +" data\n" +" Compressed data.\n" +" wbits\n" +" The window buffer size.\n" +" bufsize\n" +" The initial output buffer size."); + +#define ZLIB_DECOMPRESS_METHODDEF \ + {"decompress", (PyCFunction)zlib_decompress, METH_VARARGS, zlib_decompress__doc__}, + +static PyObject * +zlib_decompress_impl(PyModuleDef *module, Py_buffer *data, int wbits, unsigned int bufsize); + +static PyObject * +zlib_decompress(PyModuleDef *module, PyObject *args) +{ + PyObject *return_value = NULL; + Py_buffer data = {NULL, NULL}; + int wbits = MAX_WBITS; + unsigned int bufsize = DEF_BUF_SIZE; + + if (!PyArg_ParseTuple(args, + "y*|iO&:decompress", + &data, &wbits, uint_converter, &bufsize)) + goto exit; + return_value = zlib_decompress_impl(module, &data, wbits, bufsize); + +exit: + /* Cleanup for data */ + if (data.obj) + PyBuffer_Release(&data); + + return return_value; +} + +PyDoc_STRVAR(zlib_compressobj__doc__, +"compressobj(module, level=Z_DEFAULT_COMPRESSION, method=DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=Z_DEFAULT_STRATEGY, zdict=None)\n" +"Return a compressor object.\n" +"\n" +" level\n" +" The compression level (an integer in the range 0-9; default is 6).\n" +" Higher compression levels are slower, but produce smaller results.\n" +" method\n" +" The compression algorithm. If given, this must be DEFLATED.\n" +" wbits\n" +" The base two logarithm of the window size (range: 8..15).\n" +" memLevel\n" +" Controls the amount of memory used for internal compression state.\n" +" Valid values range from 1 to 9. Higher values result in higher memory\n" +" usage, faster compression, and smaller output.\n" +" strategy\n" +" Used to tune the compression algorithm. Possible values are\n" +" Z_DEFAULT_STRATEGY, Z_FILTERED, and Z_HUFFMAN_ONLY.\n" +" zdict\n" +" The predefined compression dictionary - a sequence of bytes\n" +" containing subsequences that are likely to occur in the input data."); + +#define ZLIB_COMPRESSOBJ_METHODDEF \ + {"compressobj", (PyCFunction)zlib_compressobj, METH_VARARGS|METH_KEYWORDS, zlib_compressobj__doc__}, + +static PyObject * +zlib_compressobj_impl(PyModuleDef *module, int level, int method, int wbits, int memLevel, int strategy, Py_buffer *zdict); + +static PyObject * +zlib_compressobj(PyModuleDef *module, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + static char *_keywords[] = {"level", "method", "wbits", "memLevel", "strategy", "zdict", NULL}; + int level = Z_DEFAULT_COMPRESSION; + int method = DEFLATED; + int wbits = MAX_WBITS; + int memLevel = DEF_MEM_LEVEL; + int strategy = Z_DEFAULT_STRATEGY; + Py_buffer zdict = {NULL, NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "|iiiiiy*:compressobj", _keywords, + &level, &method, &wbits, &memLevel, &strategy, &zdict)) + goto exit; + return_value = zlib_compressobj_impl(module, level, method, wbits, memLevel, strategy, &zdict); + +exit: + /* Cleanup for zdict */ + if (zdict.obj) + PyBuffer_Release(&zdict); + + return return_value; +} + +PyDoc_STRVAR(zlib_decompressobj__doc__, +"decompressobj(module, wbits=MAX_WBITS, zdict=b\'\')\n" +"Return a decompressor object.\n" +"\n" +" wbits\n" +" The window buffer size.\n" +" zdict\n" +" The predefined compression dictionary. This must be the same\n" +" dictionary as used by the compressor that produced the input data."); + +#define ZLIB_DECOMPRESSOBJ_METHODDEF \ + {"decompressobj", (PyCFunction)zlib_decompressobj, METH_VARARGS|METH_KEYWORDS, zlib_decompressobj__doc__}, + +static PyObject * +zlib_decompressobj_impl(PyModuleDef *module, int wbits, PyObject *zdict); + +static PyObject * +zlib_decompressobj(PyModuleDef *module, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + static char *_keywords[] = {"wbits", "zdict", NULL}; + int wbits = MAX_WBITS; + PyObject *zdict = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "|iO:decompressobj", _keywords, + &wbits, &zdict)) + goto exit; + return_value = zlib_decompressobj_impl(module, wbits, zdict); + +exit: + return return_value; +} + +PyDoc_STRVAR(zlib_Compress_compress__doc__, +"compress(self, data)\n" +"Returns a bytes object containing compressed data.\n" +"\n" +" data\n" +" Binary data to be compressed.\n" +"\n" +"After calling this function, some of the input data may still\n" +"be stored in internal buffers for later processing.\n" +"Call the flush() method to clear these buffers."); + +#define ZLIB_COMPRESS_COMPRESS_METHODDEF \ + {"compress", (PyCFunction)zlib_Compress_compress, METH_VARARGS, zlib_Compress_compress__doc__}, + +static PyObject * +zlib_Compress_compress_impl(compobject *self, Py_buffer *data); + +static PyObject * +zlib_Compress_compress(compobject *self, PyObject *args) +{ + PyObject *return_value = NULL; + Py_buffer data = {NULL, NULL}; + + if (!PyArg_ParseTuple(args, + "y*:compress", + &data)) + goto exit; + return_value = zlib_Compress_compress_impl(self, &data); + +exit: + /* Cleanup for data */ + if (data.obj) + PyBuffer_Release(&data); + + return return_value; +} + +PyDoc_STRVAR(zlib_Decompress_decompress__doc__, +"decompress(self, data, max_length=0)\n" +"Return a bytes object containing the decompressed version of the data.\n" +"\n" +" data\n" +" The binary data to decompress.\n" +" max_length\n" +" The maximum allowable length of the decompressed data.\n" +" Unconsumed input data will be stored in\n" +" the unconsumed_tail attribute.\n" +"\n" +"After calling this function, some of the input data may still be stored in\n" +"internal buffers for later processing.\n" +"Call the flush() method to clear these buffers."); + +#define ZLIB_DECOMPRESS_DECOMPRESS_METHODDEF \ + {"decompress", (PyCFunction)zlib_Decompress_decompress, METH_VARARGS, zlib_Decompress_decompress__doc__}, + +static PyObject * +zlib_Decompress_decompress_impl(compobject *self, Py_buffer *data, unsigned int max_length); + +static PyObject * +zlib_Decompress_decompress(compobject *self, PyObject *args) +{ + PyObject *return_value = NULL; + Py_buffer data = {NULL, NULL}; + unsigned int max_length = 0; + + if (!PyArg_ParseTuple(args, + "y*|O&:decompress", + &data, uint_converter, &max_length)) + goto exit; + return_value = zlib_Decompress_decompress_impl(self, &data, max_length); + +exit: + /* Cleanup for data */ + if (data.obj) + PyBuffer_Release(&data); + + return return_value; +} + +PyDoc_STRVAR(zlib_Compress_flush__doc__, +"flush(self, mode=Z_FINISH)\n" +"Return a bytes object containing any remaining compressed data.\n" +"\n" +" mode\n" +" One of the constants Z_SYNC_FLUSH, Z_FULL_FLUSH, Z_FINISH.\n" +" If mode == Z_FINISH, the compressor object can no longer be\n" +" used after calling the flush() method. Otherwise, more data\n" +" can still be compressed."); + +#define ZLIB_COMPRESS_FLUSH_METHODDEF \ + {"flush", (PyCFunction)zlib_Compress_flush, METH_VARARGS, zlib_Compress_flush__doc__}, + +static PyObject * +zlib_Compress_flush_impl(compobject *self, int mode); + +static PyObject * +zlib_Compress_flush(compobject *self, PyObject *args) +{ + PyObject *return_value = NULL; + int mode = Z_FINISH; + + if (!PyArg_ParseTuple(args, + "|i:flush", + &mode)) + goto exit; + return_value = zlib_Compress_flush_impl(self, mode); + +exit: + return return_value; +} + +PyDoc_STRVAR(zlib_Compress_copy__doc__, +"copy(self)\n" +"Return a copy of the compression object."); + +#define ZLIB_COMPRESS_COPY_METHODDEF \ + {"copy", (PyCFunction)zlib_Compress_copy, METH_NOARGS, zlib_Compress_copy__doc__}, + +static PyObject * +zlib_Compress_copy_impl(compobject *self); + +static PyObject * +zlib_Compress_copy(compobject *self, PyObject *Py_UNUSED(ignored)) +{ + return zlib_Compress_copy_impl(self); +} + +PyDoc_STRVAR(zlib_Decompress_copy__doc__, +"copy(self)\n" +"Return a copy of the decompression object."); + +#define ZLIB_DECOMPRESS_COPY_METHODDEF \ + {"copy", (PyCFunction)zlib_Decompress_copy, METH_NOARGS, zlib_Decompress_copy__doc__}, + +static PyObject * +zlib_Decompress_copy_impl(compobject *self); + +static PyObject * +zlib_Decompress_copy(compobject *self, PyObject *Py_UNUSED(ignored)) +{ + return zlib_Decompress_copy_impl(self); +} + +PyDoc_STRVAR(zlib_Decompress_flush__doc__, +"flush(self, length=DEF_BUF_SIZE)\n" +"Return a bytes object containing any remaining decompressed data.\n" +"\n" +" length\n" +" the initial size of the output buffer."); + +#define ZLIB_DECOMPRESS_FLUSH_METHODDEF \ + {"flush", (PyCFunction)zlib_Decompress_flush, METH_VARARGS, zlib_Decompress_flush__doc__}, + +static PyObject * +zlib_Decompress_flush_impl(compobject *self, unsigned int length); + +static PyObject * +zlib_Decompress_flush(compobject *self, PyObject *args) +{ + PyObject *return_value = NULL; + unsigned int length = DEF_BUF_SIZE; + + if (!PyArg_ParseTuple(args, + "|O&:flush", + uint_converter, &length)) + goto exit; + return_value = zlib_Decompress_flush_impl(self, length); + +exit: + return return_value; +} + +PyDoc_STRVAR(zlib_adler32__doc__, +"adler32(module, data, value=1)\n" +"Compute an Adler-32 checksum of data.\n" +"\n" +" value\n" +" Starting value of the checksum.\n" +"\n" +"The returned checksum is an integer."); + +#define ZLIB_ADLER32_METHODDEF \ + {"adler32", (PyCFunction)zlib_adler32, METH_VARARGS, zlib_adler32__doc__}, + +static PyObject * +zlib_adler32_impl(PyModuleDef *module, Py_buffer *data, unsigned int value); + +static PyObject * +zlib_adler32(PyModuleDef *module, PyObject *args) +{ + PyObject *return_value = NULL; + Py_buffer data = {NULL, NULL}; + unsigned int value = 1; + + if (!PyArg_ParseTuple(args, + "y*|I:adler32", + &data, &value)) + goto exit; + return_value = zlib_adler32_impl(module, &data, value); + +exit: + /* Cleanup for data */ + if (data.obj) + PyBuffer_Release(&data); + + return return_value; +} + +PyDoc_STRVAR(zlib_crc32__doc__, +"crc32(module, data, value=0)\n" +"Compute a CRC-32 checksum of data.\n" +"\n" +" value\n" +" Starting value of the checksum.\n" +"\n" +"The returned checksum is an integer."); + +#define ZLIB_CRC32_METHODDEF \ + {"crc32", (PyCFunction)zlib_crc32, METH_VARARGS, zlib_crc32__doc__}, + +static PyObject * +zlib_crc32_impl(PyModuleDef *module, Py_buffer *data, unsigned int value); + +static PyObject * +zlib_crc32(PyModuleDef *module, PyObject *args) +{ + PyObject *return_value = NULL; + Py_buffer data = {NULL, NULL}; + unsigned int value = 0; + + if (!PyArg_ParseTuple(args, + "y*|I:crc32", + &data, &value)) + goto exit; + return_value = zlib_crc32_impl(module, &data, value); + +exit: + /* Cleanup for data */ + if (data.obj) + PyBuffer_Release(&data); + + return return_value; +} +/*[clinic end generated code: checksum=04f94bbaf2652717753e237e4021bf6c92ddffdd]*/ diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 78ee10b..57c093c 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -28,10 +28,9 @@ #else # define DEF_MEM_LEVEL MAX_MEM_LEVEL #endif -#define DEF_WBITS MAX_WBITS -/* The output buffer will be increased in chunks of DEFAULTALLOC bytes. */ -#define DEFAULTALLOC (16*1024) +/* Initial buffer size. */ +#define DEF_BUF_SIZE (16*1024) static PyTypeObject Comptype; static PyTypeObject Decomptype; @@ -82,42 +81,13 @@ zlib_error(z_stream zst, int err, char *msg) } /*[clinic input] +output preset file module zlib class zlib.Compress "compobject *" "&Comptype" class zlib.Decompress "compobject *" "&Decomptype" [clinic start generated code]*/ /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/ -PyDoc_STRVAR(compressobj__doc__, -"compressobj(level=-1, method=DEFLATED, wbits=15, memlevel=8,\n" -" strategy=Z_DEFAULT_STRATEGY[, zdict])\n" -" -- Return a compressor object.\n" -"\n" -"level is the compression level (an integer in the range 0-9; default is 6).\n" -"Higher compression levels are slower, but produce smaller results.\n" -"\n" -"method is the compression algorithm. If given, this must be DEFLATED.\n" -"\n" -"wbits is the base two logarithm of the window size (range: 8..15).\n" -"\n" -"memlevel controls the amount of memory used for internal compression state.\n" -"Valid values range from 1 to 9. Higher values result in higher memory usage,\n" -"faster compression, and smaller output.\n" -"\n" -"strategy is used to tune the compression algorithm. Possible values are\n" -"Z_DEFAULT_STRATEGY, Z_FILTERED, and Z_HUFFMAN_ONLY.\n" -"\n" -"zdict is the predefined compression dictionary - a sequence of bytes\n" -"containing subsequences that are likely to occur in the input data."); - -PyDoc_STRVAR(decompressobj__doc__, -"decompressobj([wbits[, zdict]]) -- Return a decompressor object.\n" -"\n" -"Optional arg wbits is the window buffer size.\n" -"\n" -"Optional arg zdict is the predefined compression dictionary. This must be\n" -"the same dictionary as used by the compressor that produced the input data."); - static compobject * newcompobject(PyTypeObject *type) { @@ -165,70 +135,20 @@ PyZlib_Free(voidpf ctx, void *ptr) } /*[clinic input] - zlib.compress + bytes: Py_buffer Binary data to be compressed. - [ - level: int + level: int(c_default="Z_DEFAULT_COMPRESSION") = Z_DEFAULT_COMPRESSION Compression level, in 0-9. - ] / -Returns compressed string. - +Returns a bytes object containing compressed data. [clinic start generated code]*/ -PyDoc_STRVAR(zlib_compress__doc__, -"compress(module, bytes, [level])\n" -"Returns compressed string.\n" -"\n" -" bytes\n" -" Binary data to be compressed.\n" -" level\n" -" Compression level, in 0-9."); - -#define ZLIB_COMPRESS_METHODDEF \ - {"compress", (PyCFunction)zlib_compress, METH_VARARGS, zlib_compress__doc__}, - static PyObject * -zlib_compress_impl(PyModuleDef *module, Py_buffer *bytes, int group_right_1, int level); - -static PyObject * -zlib_compress(PyModuleDef *module, PyObject *args) -{ - PyObject *return_value = NULL; - Py_buffer bytes = {NULL, NULL}; - int group_right_1 = 0; - int level = 0; - - switch (PyTuple_GET_SIZE(args)) { - case 1: - if (!PyArg_ParseTuple(args, "y*:compress", &bytes)) - goto exit; - break; - case 2: - if (!PyArg_ParseTuple(args, "y*i:compress", &bytes, &level)) - goto exit; - group_right_1 = 1; - break; - default: - PyErr_SetString(PyExc_TypeError, "zlib.compress requires 1 to 2 arguments"); - goto exit; - } - return_value = zlib_compress_impl(module, &bytes, group_right_1, level); - -exit: - /* Cleanup for bytes */ - if (bytes.obj) - PyBuffer_Release(&bytes); - - return return_value; -} - -static PyObject * -zlib_compress_impl(PyModuleDef *module, Py_buffer *bytes, int group_right_1, int level) -/*[clinic end generated code: checksum=ce8d4c0a17ecd79c3ffcc032dcdf8ac6830ded1e]*/ +zlib_compress_impl(PyModuleDef *module, Py_buffer *bytes, int level) +/*[clinic end generated code: checksum=5d7dd4588788efd3516e5f4225050d6413632601]*/ { PyObject *ReturnVal = NULL; Byte *input, *output = NULL; @@ -236,9 +156,6 @@ zlib_compress_impl(PyModuleDef *module, Py_buffer *bytes, int group_right_1, int int err; z_stream zst; - if (!group_right_1) - level = Z_DEFAULT_COMPRESSION; - if ((size_t)bytes->len > UINT_MAX) { PyErr_SetString(PyExc_OverflowError, "Size does not fit in an unsigned int"); @@ -312,6 +229,7 @@ zlib_compress_impl(PyModuleDef *module, Py_buffer *bytes, int group_right_1, int class uint_converter(CConverter): type = 'unsigned int' converter = 'uint_converter' + c_ignored_default = "0" [python start generated code]*/ /*[python end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/ @@ -347,35 +265,38 @@ uint_converter(PyObject *obj, void *ptr) return 1; } -PyDoc_STRVAR(decompress__doc__, -"decompress(string[, wbits[, bufsize]]) -- Return decompressed string.\n" -"\n" -"Optional arg wbits is the window buffer size. Optional arg bufsize is\n" -"the initial output buffer size."); +/*[clinic input] +zlib.decompress + + data: Py_buffer + Compressed data. + wbits: int(c_default="MAX_WBITS") = MAX_WBITS + The window buffer size. + bufsize: uint(c_default="DEF_BUF_SIZE") = DEF_BUF_SIZE + The initial output buffer size. + / + +Returns a bytes object containing the uncompressed data. +[clinic start generated code]*/ static PyObject * -PyZlib_decompress(PyObject *self, PyObject *args) +zlib_decompress_impl(PyModuleDef *module, Py_buffer *data, int wbits, unsigned int bufsize) +/*[clinic end generated code: checksum=9e5464e72df9cb5fee73df662dbcaed867e01d32]*/ { PyObject *result_str = NULL; - Py_buffer pinput; Byte *input; unsigned int length; int err; - int wsize=DEF_WBITS; - unsigned int bufsize = DEFAULTALLOC, new_bufsize; + unsigned int new_bufsize; z_stream zst; - if (!PyArg_ParseTuple(args, "y*|iO&:decompress", - &pinput, &wsize, uint_converter, &bufsize)) - return NULL; - - if ((size_t)pinput.len > UINT_MAX) { + if ((size_t)data->len > UINT_MAX) { PyErr_SetString(PyExc_OverflowError, "Size does not fit in an unsigned int"); goto error; } - input = pinput.buf; - length = (unsigned int)pinput.len; + input = data->buf; + length = (unsigned int)data->len; if (bufsize == 0) bufsize = 1; @@ -391,7 +312,7 @@ PyZlib_decompress(PyObject *self, PyObject *args) zst.zfree = PyZlib_Free; zst.next_out = (Byte *)PyBytes_AS_STRING(result_str); zst.next_in = (Byte *)input; - err = inflateInit2(&zst, wsize); + err = inflateInit2(&zst, wbits); switch(err) { case(Z_OK): @@ -457,32 +378,45 @@ PyZlib_decompress(PyObject *self, PyObject *args) if (_PyBytes_Resize(&result_str, zst.total_out) < 0) goto error; - PyBuffer_Release(&pinput); return result_str; error: - PyBuffer_Release(&pinput); Py_XDECREF(result_str); return NULL; } +/*[clinic input] +zlib.compressobj + + level: int(c_default="Z_DEFAULT_COMPRESSION") = Z_DEFAULT_COMPRESSION + The compression level (an integer in the range 0-9; default is 6). + Higher compression levels are slower, but produce smaller results. + method: int(c_default="DEFLATED") = DEFLATED + The compression algorithm. If given, this must be DEFLATED. + wbits: int(c_default="MAX_WBITS") = MAX_WBITS + The base two logarithm of the window size (range: 8..15). + memLevel: int(c_default="DEF_MEM_LEVEL") = DEF_MEM_LEVEL + Controls the amount of memory used for internal compression state. + Valid values range from 1 to 9. Higher values result in higher memory + usage, faster compression, and smaller output. + strategy: int(c_default="Z_DEFAULT_STRATEGY") = Z_DEFAULT_STRATEGY + Used to tune the compression algorithm. Possible values are + Z_DEFAULT_STRATEGY, Z_FILTERED, and Z_HUFFMAN_ONLY. + zdict: Py_buffer = None + The predefined compression dictionary - a sequence of bytes + containing subsequences that are likely to occur in the input data. + +Return a compressor object. +[clinic start generated code]*/ + static PyObject * -PyZlib_compressobj(PyObject *selfptr, PyObject *args, PyObject *kwargs) +zlib_compressobj_impl(PyModuleDef *module, int level, int method, int wbits, int memLevel, int strategy, Py_buffer *zdict) +/*[clinic end generated code: checksum=89e5a6c1449caa9ed76f1baad066600e985151a9]*/ { compobject *self = NULL; - int level=Z_DEFAULT_COMPRESSION, method=DEFLATED; - int wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=0, err; - Py_buffer zdict; - static char *kwlist[] = {"level", "method", "wbits", - "memLevel", "strategy", "zdict", NULL}; - - zdict.buf = NULL; /* Sentinel, so we can tell whether zdict was supplied. */ - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iiiiiy*:compressobj", - kwlist, &level, &method, &wbits, - &memLevel, &strategy, &zdict)) - return NULL; + int err; - if (zdict.buf != NULL && (size_t)zdict.len > UINT_MAX) { + if (zdict->buf != NULL && (size_t)zdict->len > UINT_MAX) { PyErr_SetString(PyExc_OverflowError, "zdict length does not fit in an unsigned int"); goto error; @@ -500,11 +434,11 @@ PyZlib_compressobj(PyObject *selfptr, PyObject *args, PyObject *kwargs) switch(err) { case (Z_OK): self->is_initialised = 1; - if (zdict.buf == NULL) { + if (zdict->buf == NULL) { goto success; } else { err = deflateSetDictionary(&self->zst, - zdict.buf, (unsigned int)zdict.len); + zdict->buf, (unsigned int)zdict->len); switch (err) { case (Z_OK): goto success; @@ -532,22 +466,28 @@ PyZlib_compressobj(PyObject *selfptr, PyObject *args, PyObject *kwargs) Py_XDECREF(self); self = NULL; success: - if (zdict.buf != NULL) - PyBuffer_Release(&zdict); return (PyObject*)self; } +/*[clinic input] +zlib.decompressobj + + wbits: int(c_default="MAX_WBITS") = MAX_WBITS + The window buffer size. + zdict: object(c_default="NULL") = b'' + The predefined compression dictionary. This must be the same + dictionary as used by the compressor that produced the input data. + +Return a decompressor object. +[clinic start generated code]*/ + static PyObject * -PyZlib_decompressobj(PyObject *selfptr, PyObject *args, PyObject *kwargs) +zlib_decompressobj_impl(PyModuleDef *module, int wbits, PyObject *zdict) +/*[clinic end generated code: checksum=8ccd583fbd631798566d415933cd44440c8a74b5]*/ { - static char *kwlist[] = {"wbits", "zdict", NULL}; - int wbits=DEF_WBITS, err; + int err; compobject *self; - PyObject *zdict=NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iO:decompressobj", - kwlist, &wbits, &zdict)) - return NULL; if (zdict != NULL && !PyObject_CheckBuffer(zdict)) { PyErr_SetString(PyExc_TypeError, "zdict argument must support the buffer protocol"); @@ -615,37 +555,41 @@ Decomp_dealloc(compobject *self) Dealloc(self); } -PyDoc_STRVAR(comp_compress__doc__, -"compress(data) -- Return a string containing data compressed.\n" -"\n" -"After calling this function, some of the input data may still\n" -"be stored in internal buffers for later processing.\n" -"Call the flush() method to clear these buffers."); +/*[clinic input] +zlib.Compress.compress + + data: Py_buffer + Binary data to be compressed. + / +Returns a bytes object containing compressed data. + +After calling this function, some of the input data may still +be stored in internal buffers for later processing. +Call the flush() method to clear these buffers. +[clinic start generated code]*/ static PyObject * -PyZlib_objcompress(compobject *self, PyObject *args) +zlib_Compress_compress_impl(compobject *self, Py_buffer *data) +/*[clinic end generated code: checksum=5d5cd791cbc6a7f4b6de4ec12c085c88d4d3e31c]*/ { int err; unsigned int inplen; - unsigned int length = DEFAULTALLOC, new_length; - PyObject *RetVal = NULL; - Py_buffer pinput; + unsigned int length = DEF_BUF_SIZE, new_length; + PyObject *RetVal; Byte *input; unsigned long start_total_out; - if (!PyArg_ParseTuple(args, "y*:compress", &pinput)) - return NULL; - if ((size_t)pinput.len > UINT_MAX) { + if ((size_t)data->len > UINT_MAX) { PyErr_SetString(PyExc_OverflowError, "Size does not fit in an unsigned int"); - goto error_outer; + return NULL; } - input = pinput.buf; - inplen = (unsigned int)pinput.len; + input = data->buf; + inplen = (unsigned int)data->len; if (!(RetVal = PyBytes_FromStringAndSize(NULL, length))) - goto error_outer; + return NULL; ENTER_ZLIB(self); @@ -668,7 +612,7 @@ PyZlib_objcompress(compobject *self, PyObject *args) new_length = UINT_MAX; if (_PyBytes_Resize(&RetVal, new_length) < 0) { Py_CLEAR(RetVal); - goto error; + goto done; } self->zst.next_out = (unsigned char *)PyBytes_AS_STRING(RetVal) + length; @@ -686,18 +630,15 @@ PyZlib_objcompress(compobject *self, PyObject *args) if (err != Z_OK && err != Z_BUF_ERROR) { zlib_error(self->zst, err, "while compressing data"); - Py_DECREF(RetVal); - RetVal = NULL; - goto error; + Py_CLEAR(RetVal); + goto done; } if (_PyBytes_Resize(&RetVal, self->zst.total_out - start_total_out) < 0) { Py_CLEAR(RetVal); } - error: + done: LEAVE_ZLIB(self); - error_outer: - PyBuffer_Release(&pinput); return RetVal; } @@ -745,7 +686,6 @@ save_unconsumed_input(compobject *self, int err) } /*[clinic input] - zlib.Decompress.decompress data: Py_buffer @@ -756,61 +696,19 @@ zlib.Decompress.decompress the unconsumed_tail attribute. / -Return a string containing the decompressed version of the data. +Return a bytes object containing the decompressed version of the data. After calling this function, some of the input data may still be stored in internal buffers for later processing. Call the flush() method to clear these buffers. [clinic start generated code]*/ -PyDoc_STRVAR(zlib_Decompress_decompress__doc__, -"decompress(self, data, max_length=0)\n" -"Return a string containing the decompressed version of the data.\n" -"\n" -" data\n" -" The binary data to decompress.\n" -" max_length\n" -" The maximum allowable length of the decompressed data.\n" -" Unconsumed input data will be stored in\n" -" the unconsumed_tail attribute.\n" -"\n" -"After calling this function, some of the input data may still be stored in\n" -"internal buffers for later processing.\n" -"Call the flush() method to clear these buffers."); - -#define ZLIB_DECOMPRESS_DECOMPRESS_METHODDEF \ - {"decompress", (PyCFunction)zlib_Decompress_decompress, METH_VARARGS, zlib_Decompress_decompress__doc__}, - -static PyObject * -zlib_Decompress_decompress_impl(compobject *self, Py_buffer *data, unsigned int max_length); - -static PyObject * -zlib_Decompress_decompress(compobject *self, PyObject *args) -{ - PyObject *return_value = NULL; - Py_buffer data = {NULL, NULL}; - unsigned int max_length = 0; - - if (!PyArg_ParseTuple(args, - "y*|O&:decompress", - &data, uint_converter, &max_length)) - goto exit; - return_value = zlib_Decompress_decompress_impl(self, &data, max_length); - -exit: - /* Cleanup for data */ - if (data.obj) - PyBuffer_Release(&data); - - return return_value; -} - static PyObject * zlib_Decompress_decompress_impl(compobject *self, Py_buffer *data, unsigned int max_length) -/*[clinic end generated code: checksum=b7fd2e3b23430f57f5a84817189575bc46464901]*/ +/*[clinic end generated code: checksum=755cccc9087bfe55486b7e15fa7e2ab60b4c86d6]*/ { int err; - unsigned int old_length, length = DEFAULTALLOC; + unsigned int old_length, length = DEF_BUF_SIZE; PyObject *RetVal = NULL; unsigned long start_total_out; @@ -927,29 +825,31 @@ zlib_Decompress_decompress_impl(compobject *self, Py_buffer *data, unsigned int return RetVal; } -PyDoc_STRVAR(comp_flush__doc__, -"flush( [mode] ) -- Return a string containing any remaining compressed data.\n" -"\n" -"mode can be one of the constants Z_SYNC_FLUSH, Z_FULL_FLUSH, Z_FINISH; the\n" -"default value used when mode is not specified is Z_FINISH.\n" -"If mode == Z_FINISH, the compressor object can no longer be used after\n" -"calling the flush() method. Otherwise, more data can still be compressed."); +/*[clinic input] +zlib.Compress.flush + + mode: int(c_default="Z_FINISH") = Z_FINISH + One of the constants Z_SYNC_FLUSH, Z_FULL_FLUSH, Z_FINISH. + If mode == Z_FINISH, the compressor object can no longer be + used after calling the flush() method. Otherwise, more data + can still be compressed. + / + +Return a bytes object containing any remaining compressed data. +[clinic start generated code]*/ static PyObject * -PyZlib_flush(compobject *self, PyObject *args) +zlib_Compress_flush_impl(compobject *self, int mode) +/*[clinic end generated code: checksum=a203f4cefc9de727aa1d2ea39d11c0a16c32041a]*/ { int err; - unsigned int length = DEFAULTALLOC, new_length; + unsigned int length = DEF_BUF_SIZE, new_length; PyObject *RetVal; - int flushmode = Z_FINISH; unsigned long start_total_out; - if (!PyArg_ParseTuple(args, "|i:flush", &flushmode)) - return NULL; - /* Flushing with Z_NO_FLUSH is a no-op, so there's no point in doing any work at all; just return an empty string. */ - if (flushmode == Z_NO_FLUSH) { + if (mode == Z_NO_FLUSH) { return PyBytes_FromStringAndSize(NULL, 0); } @@ -964,7 +864,7 @@ PyZlib_flush(compobject *self, PyObject *args) self->zst.next_out = (unsigned char *)PyBytes_AS_STRING(RetVal); Py_BEGIN_ALLOW_THREADS - err = deflate(&(self->zst), flushmode); + err = deflate(&(self->zst), mode); Py_END_ALLOW_THREADS /* while Z_OK and the output buffer is full, there might be more output, @@ -984,14 +884,14 @@ PyZlib_flush(compobject *self, PyObject *args) length = new_length; Py_BEGIN_ALLOW_THREADS - err = deflate(&(self->zst), flushmode); + err = deflate(&(self->zst), mode); Py_END_ALLOW_THREADS } - /* If flushmode is Z_FINISH, we also have to call deflateEnd() to free + /* If mode is Z_FINISH, we also have to call deflateEnd() to free various data structures. Note we should only get Z_STREAM_END when - flushmode is Z_FINISH, but checking both for safety*/ - if (err == Z_STREAM_END && flushmode == Z_FINISH) { + mode is Z_FINISH, but checking both for safety*/ + if (err == Z_STREAM_END && mode == Z_FINISH) { err = deflateEnd(&(self->zst)); if (err != Z_OK) { zlib_error(self->zst, err, "while finishing compression"); @@ -1031,25 +931,9 @@ zlib.Compress.copy Return a copy of the compression object. [clinic start generated code]*/ -PyDoc_STRVAR(zlib_Compress_copy__doc__, -"copy(self)\n" -"Return a copy of the compression object."); - -#define ZLIB_COMPRESS_COPY_METHODDEF \ - {"copy", (PyCFunction)zlib_Compress_copy, METH_NOARGS, zlib_Compress_copy__doc__}, - -static PyObject * -zlib_Compress_copy_impl(compobject *self); - -static PyObject * -zlib_Compress_copy(compobject *self, PyObject *Py_UNUSED(ignored)) -{ - return zlib_Compress_copy_impl(self); -} - static PyObject * zlib_Compress_copy_impl(compobject *self) -/*[clinic end generated code: checksum=7aa841ad51297eb83250f511a76872e88fdc737e]*/ +/*[clinic end generated code: checksum=5144aa153c21e805afa5c19e5b48cf8e6480b5da]*/ { compobject *retval = NULL; int err; @@ -1099,11 +983,15 @@ error: return NULL; } -PyDoc_STRVAR(decomp_copy__doc__, -"copy() -- Return a copy of the decompression object."); +/*[clinic input] +zlib.Decompress.copy + +Return a copy of the decompression object. +[clinic start generated code]*/ static PyObject * -PyZlib_uncopy(compobject *self) +zlib_Decompress_copy_impl(compobject *self) +/*[clinic end generated code: checksum=02a883a2a510c8ccfeef3f89e317a275bfe8c094]*/ { compobject *retval = NULL; int err; @@ -1155,24 +1043,26 @@ error: } #endif -PyDoc_STRVAR(decomp_flush__doc__, -"flush( [length] ) -- Return a string containing any remaining\n" -"decompressed data. length, if given, is the initial size of the\n" -"output buffer.\n" -"\n" -"The decompressor object can no longer be used after this call."); +/*[clinic input] +zlib.Decompress.flush + + length: uint(c_default="DEF_BUF_SIZE") = DEF_BUF_SIZE + the initial size of the output buffer. + / + +Return a bytes object containing any remaining decompressed data. +[clinic start generated code]*/ static PyObject * -PyZlib_unflush(compobject *self, PyObject *args) +zlib_Decompress_flush_impl(compobject *self, unsigned int length) +/*[clinic end generated code: checksum=db6fb753ab698e22afe3957c9da9e5e77f4bfc08]*/ { int err; - unsigned int length = DEFAULTALLOC, new_length; + unsigned int new_length; PyObject * retval = NULL; unsigned long start_total_out; Py_ssize_t size; - if (!PyArg_ParseTuple(args, "|O&:flush", uint_converter, &length)) - return NULL; if (length == 0) { PyErr_SetString(PyExc_ValueError, "length must be greater than zero"); return NULL; @@ -1248,12 +1138,12 @@ error: return retval; } +#include "zlibmodule.clinic.c" + static PyMethodDef comp_methods[] = { - {"compress", (binaryfunc)PyZlib_objcompress, METH_VARARGS, - comp_compress__doc__}, - {"flush", (binaryfunc)PyZlib_flush, METH_VARARGS, - comp_flush__doc__}, + ZLIB_COMPRESS_COMPRESS_METHODDEF + ZLIB_COMPRESS_FLUSH_METHODDEF #ifdef HAVE_ZLIB_COPY ZLIB_COMPRESS_COPY_METHODDEF #endif @@ -1263,11 +1153,9 @@ static PyMethodDef comp_methods[] = static PyMethodDef Decomp_methods[] = { ZLIB_DECOMPRESS_DECOMPRESS_METHODDEF - {"flush", (binaryfunc)PyZlib_unflush, METH_VARARGS, - decomp_flush__doc__}, + ZLIB_DECOMPRESS_FLUSH_METHODDEF #ifdef HAVE_ZLIB_COPY - {"copy", (PyCFunction)PyZlib_uncopy, METH_NOARGS, - decomp_copy__doc__}, + ZLIB_DECOMPRESS_COPY_METHODDEF #endif {NULL, NULL} }; @@ -1280,95 +1168,95 @@ static PyMemberDef Decomp_members[] = { {NULL}, }; -PyDoc_STRVAR(adler32__doc__, -"adler32(string[, start]) -- Compute an Adler-32 checksum of string.\n" -"\n" -"An optional starting value can be specified. The returned checksum is\n" -"an integer."); +/*[clinic input] +zlib.adler32 + + data: Py_buffer + value: unsigned_int(bitwise=True) = 1 + Starting value of the checksum. + / + +Compute an Adler-32 checksum of data. + +The returned checksum is an integer. +[clinic start generated code]*/ static PyObject * -PyZlib_adler32(PyObject *self, PyObject *args) +zlib_adler32_impl(PyModuleDef *module, Py_buffer *data, unsigned int value) +/*[clinic end generated code: checksum=51d6d75ee655c78af8c968fdb4c11d97e62c67d5]*/ { - unsigned int adler32val = 1; /* adler32(0L, Z_NULL, 0) */ - Py_buffer pbuf; - - if (!PyArg_ParseTuple(args, "y*|I:adler32", &pbuf, &adler32val)) - return NULL; /* Releasing the GIL for very small buffers is inefficient and may lower performance */ - if (pbuf.len > 1024*5) { - unsigned char *buf = pbuf.buf; - Py_ssize_t len = pbuf.len; + if (data->len > 1024*5) { + unsigned char *buf = data->buf; + Py_ssize_t len = data->len; Py_BEGIN_ALLOW_THREADS /* Avoid truncation of length for very large buffers. adler32() takes length as an unsigned int, which may be narrower than Py_ssize_t. */ while ((size_t)len > UINT_MAX) { - adler32val = adler32(adler32val, buf, UINT_MAX); + value = adler32(value, buf, UINT_MAX); buf += (size_t) UINT_MAX; len -= (size_t) UINT_MAX; } - adler32val = adler32(adler32val, buf, (unsigned int)len); + value = adler32(value, buf, (unsigned int)len); Py_END_ALLOW_THREADS } else { - adler32val = adler32(adler32val, pbuf.buf, (unsigned int)pbuf.len); + value = adler32(value, data->buf, (unsigned int)data->len); } - PyBuffer_Release(&pbuf); - return PyLong_FromUnsignedLong(adler32val & 0xffffffffU); + return PyLong_FromUnsignedLong(value & 0xffffffffU); } -PyDoc_STRVAR(crc32__doc__, -"crc32(string[, start]) -- Compute a CRC-32 checksum of string.\n" -"\n" -"An optional starting value can be specified. The returned checksum is\n" -"an integer."); +/*[clinic input] +zlib.crc32 + + data: Py_buffer + value: unsigned_int(bitwise=True) = 0 + Starting value of the checksum. + / + +Compute a CRC-32 checksum of data. + +The returned checksum is an integer. +[clinic start generated code]*/ static PyObject * -PyZlib_crc32(PyObject *self, PyObject *args) +zlib_crc32_impl(PyModuleDef *module, Py_buffer *data, unsigned int value) +/*[clinic end generated code: checksum=c1e986e74fe7b62369998a71a81ebeb9b73e8d4c]*/ { - unsigned int crc32val = 0; /* crc32(0L, Z_NULL, 0) */ - Py_buffer pbuf; int signed_val; - if (!PyArg_ParseTuple(args, "y*|I:crc32", &pbuf, &crc32val)) - return NULL; /* Releasing the GIL for very small buffers is inefficient and may lower performance */ - if (pbuf.len > 1024*5) { - unsigned char *buf = pbuf.buf; - Py_ssize_t len = pbuf.len; + if (data->len > 1024*5) { + unsigned char *buf = data->buf; + Py_ssize_t len = data->len; Py_BEGIN_ALLOW_THREADS /* Avoid truncation of length for very large buffers. crc32() takes length as an unsigned int, which may be narrower than Py_ssize_t. */ while ((size_t)len > UINT_MAX) { - crc32val = crc32(crc32val, buf, UINT_MAX); + value = crc32(value, buf, UINT_MAX); buf += (size_t) UINT_MAX; len -= (size_t) UINT_MAX; } - signed_val = crc32(crc32val, buf, (unsigned int)len); + signed_val = crc32(value, buf, (unsigned int)len); Py_END_ALLOW_THREADS } else { - signed_val = crc32(crc32val, pbuf.buf, (unsigned int)pbuf.len); + signed_val = crc32(value, data->buf, (unsigned int)data->len); } - PyBuffer_Release(&pbuf); return PyLong_FromUnsignedLong(signed_val & 0xffffffffU); } static PyMethodDef zlib_methods[] = { - {"adler32", (PyCFunction)PyZlib_adler32, METH_VARARGS, - adler32__doc__}, + ZLIB_ADLER32_METHODDEF ZLIB_COMPRESS_METHODDEF - {"compressobj", (PyCFunction)PyZlib_compressobj, METH_VARARGS|METH_KEYWORDS, - compressobj__doc__}, - {"crc32", (PyCFunction)PyZlib_crc32, METH_VARARGS, - crc32__doc__}, - {"decompress", (PyCFunction)PyZlib_decompress, METH_VARARGS, - decompress__doc__}, - {"decompressobj", (PyCFunction)PyZlib_decompressobj, METH_VARARGS|METH_KEYWORDS, - decompressobj__doc__}, + ZLIB_COMPRESSOBJ_METHODDEF + ZLIB_CRC32_METHODDEF + ZLIB_DECOMPRESS_METHODDEF + ZLIB_DECOMPRESSOBJ_METHODDEF {NULL, NULL} }; @@ -1482,6 +1370,7 @@ PyInit_zlib(void) PyModule_AddIntMacro(m, MAX_WBITS); PyModule_AddIntMacro(m, DEFLATED); PyModule_AddIntMacro(m, DEF_MEM_LEVEL); + PyModule_AddIntMacro(m, DEF_BUF_SIZE); PyModule_AddIntMacro(m, Z_BEST_SPEED); PyModule_AddIntMacro(m, Z_BEST_COMPRESSION); PyModule_AddIntMacro(m, Z_DEFAULT_COMPRESSION); -- cgit v0.12 From 6834a65773edf0898df664b48b191c828f80d174 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 26 Jan 2014 23:33:49 +0100 Subject: Issue #20367: Add Glenn Langford to Misc/ACKS --- Misc/ACKS | 1 + 1 file changed, 1 insertion(+) diff --git a/Misc/ACKS b/Misc/ACKS index b10fcc2..12b5ab5 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -725,6 +725,7 @@ Ronan Lamy Torsten Landschoff Łukasz Langa Tino Lange +Glenn Langford Andrew Langmead Detlef Lannert Soren Larsen -- cgit v0.12 From a09daef09e787809dad237b9908e784cd0b014c1 Mon Sep 17 00:00:00 2001 From: Eli Bendersky Date: Sun, 26 Jan 2014 14:59:30 -0800 Subject: Update docstring to mention correct PEP number. This file hasn't been touched since its initial commit in 2006. In CPython default branch (3.4), the relevant PEP number is 3333. --- Lib/wsgiref/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/wsgiref/__init__.py b/Lib/wsgiref/__init__.py index 46c579f..1efbba0 100644 --- a/Lib/wsgiref/__init__.py +++ b/Lib/wsgiref/__init__.py @@ -1,4 +1,4 @@ -"""wsgiref -- a WSGI (PEP 333) Reference Library +"""wsgiref -- a WSGI (PEP 3333) Reference Library Current Contents: -- cgit v0.12 From 27ea78b352fc33aa9d79a298a4b313682a42ed21 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Mon, 27 Jan 2014 01:03:53 +0100 Subject: silence compiler warning that 's' may be used uninitialized in the load function. --- Modules/_pickle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index dabd81e..54a5ec5 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -6149,7 +6149,7 @@ static PyObject * load(UnpicklerObject *self) { PyObject *value = NULL; - char *s; + char *s = NULL; self->num_marks = 0; self->proto = 0; -- cgit v0.12 From 936e2f36ade1506d56dd5f10e1967936aabe70b3 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Mon, 27 Jan 2014 01:06:57 +0100 Subject: Issue #20193: Fix commit r6f217456b9ba by including clinic/zlibmodule.c.h instead of zlibmodule.clinic.c --- Modules/zlibmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 57c093c..a6b72bd 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -1138,7 +1138,7 @@ error: return retval; } -#include "zlibmodule.clinic.c" +#include "clinic/zlibmodule.c.h" static PyMethodDef comp_methods[] = { -- cgit v0.12 From c4ab9a4f1d923bb60a341856da1d273b17a545e0 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Mon, 27 Jan 2014 01:12:00 +0100 Subject: Issue #20394: Attempt to silence CID 1164423: Division or modulo by zero in audioop_ratecv_impl() Serhiy and I had the same idea so it's most likely right. ;) --- Modules/audioop.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/audioop.c b/Modules/audioop.c index cc3a020..159b2fb 100644 --- a/Modules/audioop.c +++ b/Modules/audioop.c @@ -1304,6 +1304,7 @@ audioop_ratecv_impl(PyModuleDef *module, Py_buffer *fragment, int width, int nch "weightA should be >= 1, weightB should be >= 0"); return NULL; } + assert(fragment->len >= 0); if (fragment->len % bytes_per_frame != 0) { PyErr_SetString(AudioopError, "not a whole number of frames"); return NULL; @@ -1370,7 +1371,7 @@ audioop_ratecv_impl(PyModuleDef *module, Py_buffer *fragment, int width, int nch case ceiling(len/inrate) * outrate. */ /* compute ceiling(len/inrate) without overflow */ - Py_ssize_t q = len > 0 ? 1 + (len - 1) / inrate : 0; + Py_ssize_t q = 1 + (len - 1) / inrate; if (outrate > PY_SSIZE_T_MAX / q / bytes_per_frame) str = NULL; else -- cgit v0.12 From a0f1e220684506f703f859350024a1365fa5cf7b Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 26 Jan 2014 19:55:34 -0500 Subject: Issue #20338: Increase allowed tip width slightly and wrap long signagure lines. Original patch by Serhiy Storchaka. --- Lib/idlelib/CallTips.py | 15 +++++++++------ Lib/idlelib/idle_test/test_calltips.py | 32 ++++++++++++++++++++++++-------- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/Lib/idlelib/CallTips.py b/Lib/idlelib/CallTips.py index 4f88da6..81bd5f1 100644 --- a/Lib/idlelib/CallTips.py +++ b/Lib/idlelib/CallTips.py @@ -5,16 +5,16 @@ parameter and docstring information when you type an opening parenthesis, and which disappear when you type a closing parenthesis. """ +import __main__ +import inspect import re import sys +import textwrap import types -import inspect from idlelib import CallTipWindow from idlelib.HyperParser import HyperParser -import __main__ - class CallTips: menudefs = [ @@ -117,8 +117,9 @@ def get_entity(expression): return None # The following are used in get_argspec and some in tests -_MAX_COLS = 79 +_MAX_COLS = 85 _MAX_LINES = 5 # enough for bytes +_INDENT = ' '*4 # for wrapped signatures _first_param = re.compile('(?<=\()\w*\,?\s*') _default_callable_argspec = "See source or doc" @@ -149,13 +150,15 @@ def get_argspec(ob): isinstance(ob_call, types.MethodType)): argspec = _first_param.sub("", argspec) + lines = (textwrap.wrap(argspec, _MAX_COLS, subsequent_indent=_INDENT) + if len(argspec) > _MAX_COLS else [argspec] if argspec else []) + if isinstance(ob_call, types.MethodType): doc = ob_call.__doc__ else: doc = getattr(ob, "__doc__", "") if doc: - lines = [argspec] if argspec else [] - for line in doc.split('\n', 5)[:_MAX_LINES]: + for line in doc.split('\n', _MAX_LINES)[:_MAX_LINES]: line = line.strip() if not line: break diff --git a/Lib/idlelib/idle_test/test_calltips.py b/Lib/idlelib/idle_test/test_calltips.py index eca24ec..e71656b 100644 --- a/Lib/idlelib/idle_test/test_calltips.py +++ b/Lib/idlelib/idle_test/test_calltips.py @@ -1,5 +1,6 @@ import unittest import idlelib.CallTips as ct +import textwrap import types default_tip = ct._default_callable_argspec @@ -64,20 +65,35 @@ class Get_signatureTest(unittest.TestCase): gtest(types.MethodType, "method(function, instance)") gtest(SB(), default_tip) + def test_signature_wrap(self): + self.assertEqual(signature(textwrap.TextWrapper), '''\ +(width=70, initial_indent='', subsequent_indent='', expand_tabs=True, + replace_whitespace=True, fix_sentence_endings=False, break_long_words=True, + drop_whitespace=True, break_on_hyphens=True, tabsize=8)''') + + def test_docline_truncation(self): + def f(): pass + f.__doc__ = 'a'*300 + self.assertEqual(signature(f), '()\n' + 'a' * (ct._MAX_COLS-3) + '...') + def test_multiline_docstring(self): # Test fewer lines than max. self.assertEqual(signature(list), "list() -> new empty list\n" "list(iterable) -> new list initialized from iterable's items") - # Test max lines and line (currently) too long. - self.assertEqual(signature(bytes), -"bytes(iterable_of_ints) -> bytes\n" -"bytes(string, encoding[, errors]) -> bytes\n" -"bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer\n" -#bytes(int) -> bytes object of size given by the parameter initialized with null bytes -"bytes(int) -> bytes object of size given by the parameter initialized with n...\n" -"bytes() -> empty bytes object") + # Test max lines + self.assertEqual(signature(bytes), '''\ +bytes(iterable_of_ints) -> bytes +bytes(string, encoding[, errors]) -> bytes +bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer +bytes(int) -> bytes object of size given by the parameter initialized with null bytes +bytes() -> empty bytes object''') + + # Test more than max lines + def f(): pass + f.__doc__ = 'a\n' * 15 + self.assertEqual(signature(f), '()' + '\na' * ct._MAX_LINES) def test_functions(self): def t1(): 'doc' -- cgit v0.12 From 76964877e3b40b79319dda095ba8ca6b50074c56 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 26 Jan 2014 20:24:35 -0500 Subject: White space and merge cleanup. --- Lib/idlelib/idle_test/test_calltips.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Lib/idlelib/idle_test/test_calltips.py b/Lib/idlelib/idle_test/test_calltips.py index 8edaf6b..4bf5e7e 100644 --- a/Lib/idlelib/idle_test/test_calltips.py +++ b/Lib/idlelib/idle_test/test_calltips.py @@ -69,12 +69,8 @@ class Get_signatureTest(unittest.TestCase): self.assertEqual(signature(textwrap.TextWrapper), '''\ (width=70, initial_indent='', subsequent_indent='', expand_tabs=True, replace_whitespace=True, fix_sentence_endings=False, break_long_words=True, -<<<<<<< local drop_whitespace=True, break_on_hyphens=True, tabsize=8, *, max_lines=None, placeholder=' [...]')''') -======= - drop_whitespace=True, break_on_hyphens=True, tabsize=8)''') ->>>>>>> other def test_docline_truncation(self): def f(): pass -- cgit v0.12 From 1b759bc49ed3958c64c7f2796df68b1c607acd69 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 26 Jan 2014 21:34:33 -0500 Subject: Idlelib.calltips: add test of starred first parameters. They should not be removed even for bound methods. (Inspect.signature does, see 20401.) --- Lib/idlelib/idle_test/test_calltips.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Lib/idlelib/idle_test/test_calltips.py b/Lib/idlelib/idle_test/test_calltips.py index e71656b..f363764 100644 --- a/Lib/idlelib/idle_test/test_calltips.py +++ b/Lib/idlelib/idle_test/test_calltips.py @@ -122,6 +122,16 @@ bytes() -> empty bytes object''') (tc.__call__, '(ci)'), (tc, '(ci)'), (TC.cm, "(a)"),): self.assertEqual(signature(meth), mtip + "\ndoc") + def test_starred_parameter(self): + # test that starred first parameter is *not* removed from argspec + class C: + def m1(*args): pass + def m2(**kwds): pass + c = C() + for meth, mtip in ((C.m1, '(*args)'), (c.m1, "(*args)"), + (C.m2, "(**kwds)"), (c.m2, "(**kwds)"),): + self.assertEqual(signature(meth), mtip) + def test_non_ascii_name(self): # test that re works to delete a first parameter name that # includes non-ascii chars, such as various forms of A. -- cgit v0.12 From 91d4278fba54610443cf7c1422f09e51d7aef567 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 26 Jan 2014 22:24:26 -0500 Subject: =?UTF-8?q?Issue=20#17721:=20Remove=20non-functional=20configurati?= =?UTF-8?q?on=20dialog=20help=20button=20until=20we=20make=20it=20actually?= =?UTF-8?q?=20gives=20some=20help=20when=20clicked.=20Patch=20by=20Guilher?= =?UTF-8?q?me=20Sim=C3=B5es.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/idlelib/configDialog.py | 9 +++++---- Misc/NEWS | 5 ++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Lib/idlelib/configDialog.py b/Lib/idlelib/configDialog.py index 1f4a3a5..efe5c43 100644 --- a/Lib/idlelib/configDialog.py +++ b/Lib/idlelib/configDialog.py @@ -82,9 +82,10 @@ class ConfigDialog(Toplevel): else: extraKwds=dict(padx=6, pady=3) - self.buttonHelp = Button(frameActionButtons,text='Help', - command=self.Help,takefocus=FALSE, - **extraKwds) +# Comment out button creation and packing until implement self.Help +## self.buttonHelp = Button(frameActionButtons,text='Help', +## command=self.Help,takefocus=FALSE, +## **extraKwds) self.buttonOk = Button(frameActionButtons,text='Ok', command=self.Ok,takefocus=FALSE, **extraKwds) @@ -98,7 +99,7 @@ class ConfigDialog(Toplevel): self.CreatePageHighlight() self.CreatePageKeys() self.CreatePageGeneral() - self.buttonHelp.pack(side=RIGHT,padx=5) +## self.buttonHelp.pack(side=RIGHT,padx=5) self.buttonOk.pack(side=LEFT,padx=5) self.buttonApply.pack(side=LEFT,padx=5) self.buttonCancel.pack(side=LEFT,padx=5) diff --git a/Misc/NEWS b/Misc/NEWS index 10cb08e..e8d1a7a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -276,7 +276,10 @@ Library IDLE ---- ---Issue #17390: Add Python version to Idle editor window title bar. +- Issue #17721: Remove non-functional configuration dialog help button until we + make it actually gives some help when clicked. Patch by Guilherme Simões. + +- Issue #17390: Add Python version to Idle editor window title bar. Original patches by Edmond Burnett and Kent Johnson. - Issue #18960: IDLE now ignores the source encoding declaration on the second -- cgit v0.12 From d1c85fd2835bafa0fde0b95b82b6e0fffd60649a Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 26 Jan 2014 22:52:08 -0500 Subject: eliminate redundancy between yield stmt and yield expr docs (closes #12704) Patch by Nikolaus Rath. --- Doc/reference/expressions.rst | 99 ++++++++++++++++++++---------------------- Doc/reference/simple_stmts.rst | 57 +++++++----------------- 2 files changed, 62 insertions(+), 94 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 66358c8..06baba0 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -319,27 +319,25 @@ Yield expressions yield_atom: "(" `yield_expression` ")" yield_expression: "yield" [`expression_list` | "from" `expression`] -The :keyword:`yield` expression is only used when defining a :term:`generator` -function, -and can only be used in the body of a function definition. Using a -:keyword:`yield` expression in a function definition is sufficient to cause that -definition to create a generator function instead of a normal function. +The yield expression is only used when defining a :term:`generator` function and +thus can only be used in the body of a function definition. Using a yield +expression in a function's body causes that function to be a generator. When a generator function is called, it returns an iterator known as a generator. That generator then controls the execution of a generator function. The execution starts when one of the generator's methods is called. At that -time, the execution proceeds to the first :keyword:`yield` expression, where it -is suspended again, returning the value of :token:`expression_list` to -generator's caller. By suspended we mean that all local state is retained, -including the current bindings of local variables, the instruction pointer, and -the internal evaluation stack. When the execution is resumed by calling one of -the generator's methods, the function can proceed exactly as if the -:keyword:`yield` expression was just another external call. The value of the -:keyword:`yield` expression after resuming depends on the method which resumed -the execution. If :meth:`~generator.__next__` is used (typically via either a -:keyword:`for` or the :func:`next` builtin) then the result is :const:`None`, -otherwise, if :meth:`~generator.send` is used, then the result will be the -value passed in to that method. +time, the execution proceeds to the first yield expression, where it is +suspended again, returning the value of :token:`expression_list` to generator's +caller. By suspended, we mean that all local state is retained, including the +current bindings of local variables, the instruction pointer, and the internal +evaluation stack. When the execution is resumed by calling one of the +generator's methods, the function can proceed exactly as if the yield expression +was just another external call. The value of the yield expression after +resuming depends on the method which resumed the execution. If +:meth:`~generator.__next__` is used (typically via either a :keyword:`for` or +the :func:`next` builtin) then the result is :const:`None`. Otherwise, if +:meth:`~generator.send` is used, then the result will be the value passed in to +that method. .. index:: single: coroutine @@ -349,11 +347,11 @@ suspended. The only difference is that a generator function cannot control where should the execution continue after it yields; the control is always transferred to the generator's caller. -:keyword:`yield` expressions are allowed in the :keyword:`try` clause of a -:keyword:`try` ... :keyword:`finally` construct. If the generator is not -resumed before it is finalized (by reaching a zero reference count or by being -garbage collected), the generator-iterator's :meth:`~generator.close` method -will be called, allowing any pending :keyword:`finally` clauses to execute. +yield expressions are allowed in the :keyword:`try` clause of a :keyword:`try` +... :keyword:`finally` construct. If the generator is not resumed before it is +finalized (by reaching a zero reference count or by being garbage collected), +the generator-iterator's :meth:`~generator.close` method will be called, +allowing any pending :keyword:`finally` clauses to execute. When ``yield from `` is used, it treats the supplied expression as a subiterator. All values produced by that subiterator are passed directly @@ -373,11 +371,23 @@ the yield expression. It can be either set explicitly when raising .. versionchanged:: 3.3 Added ``yield from `` to delegate control flow to a subiterator -The parentheses can be omitted when the :keyword:`yield` expression is the -sole expression on the right hand side of an assignment statement. +The parentheses may be omitted when the yield expression is the sole expression +on the right hand side of an assignment statement. -.. index:: object: generator +.. seealso:: + + :pep:`0255` - Simple Generators + The proposal for adding generators and the :keyword:`yield` statement to Python. + + :pep:`0342` - Coroutines via Enhanced Generators + The proposal to enhance the API and syntax of generators, making them + usable as simple coroutines. + :pep:`0380` - Syntax for Delegating to a Subgenerator + The proposal to introduce the :token:`yield_from` syntax, making delegation + to sub-generators easy. + +.. index:: object: generator Generator-iterator methods ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -395,13 +405,12 @@ is already executing raises a :exc:`ValueError` exception. .. method:: generator.__next__() Starts the execution of a generator function or resumes it at the last - executed :keyword:`yield` expression. When a generator function is resumed - with a :meth:`~generator.__next__` method, the current :keyword:`yield` - expression always evaluates to :const:`None`. The execution then continues - to the next :keyword:`yield` expression, where the generator is suspended - again, and the value of the :token:`expression_list` is returned to - :meth:`next`'s caller. - If the generator exits without yielding another value, a :exc:`StopIteration` + executed yield expression. When a generator function is resumed with a + :meth:`~generator.__next__` method, the current yield expression always + evaluates to :const:`None`. The execution then continues to the next yield + expression, where the generator is suspended again, and the value of the + :token:`expression_list` is returned to :meth:`next`'s caller. If the + generator exits without yielding another value, a :exc:`StopIteration` exception is raised. This method is normally called implicitly, e.g. by a :keyword:`for` loop, or @@ -411,12 +420,12 @@ is already executing raises a :exc:`ValueError` exception. .. method:: generator.send(value) Resumes the execution and "sends" a value into the generator function. The - ``value`` argument becomes the result of the current :keyword:`yield` - expression. The :meth:`send` method returns the next value yielded by the - generator, or raises :exc:`StopIteration` if the generator exits without - yielding another value. When :meth:`send` is called to start the generator, - it must be called with :const:`None` as the argument, because there is no - :keyword:`yield` expression that could receive the value. + *value* argument becomes the result of the current yield expression. The + :meth:`send` method returns the next value yielded by the generator, or + raises :exc:`StopIteration` if the generator exits without yielding another + value. When :meth:`send` is called to start the generator, it must be called + with :const:`None` as the argument, because there is no yield expression that + could receive the value. .. method:: generator.throw(type[, value[, traceback]]) @@ -478,20 +487,6 @@ For examples using ``yield from``, see :ref:`pep-380` in "What's New in Python." -.. seealso:: - - :pep:`0255` - Simple Generators - The proposal for adding generators and the :keyword:`yield` statement to Python. - - :pep:`0342` - Coroutines via Enhanced Generators - The proposal to enhance the API and syntax of generators, making them - usable as simple coroutines. - - :pep:`0380` - Syntax for Delegating to a Subgenerator - The proposal to introduce the :token:`yield_from` syntax, making delegation - to sub-generators easy. - - .. _primaries: Primaries diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index b9ebaaa..40bbc39 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -445,53 +445,26 @@ The :keyword:`yield` statement .. productionlist:: yield_stmt: `yield_expression` -The :keyword:`yield` statement is only used when defining a generator function, -and is only used in the body of the generator function. Using a :keyword:`yield` -statement in a function definition is sufficient to cause that definition to -create a generator function instead of a normal function. - -When a generator function is called, it returns an iterator known as a generator -iterator, or more commonly, a generator. The body of the generator function is -executed by calling the :func:`next` function on the generator repeatedly until -it raises an exception. - -When a :keyword:`yield` statement is executed, the state of the generator is -frozen and the value of :token:`expression_list` is returned to :meth:`next`'s -caller. By "frozen" we mean that all local state is retained, including the -current bindings of local variables, the instruction pointer, and the internal -evaluation stack: enough information is saved so that the next time :func:`next` -is invoked, the function can proceed exactly as if the :keyword:`yield` -statement were just another external call. - -The :keyword:`yield` statement is allowed in the :keyword:`try` clause of a -:keyword:`try` ... :keyword:`finally` construct. If the generator is not -resumed before it is finalized (by reaching a zero reference count or by being -garbage collected), the generator-iterator's :meth:`close` method will be -called, allowing any pending :keyword:`finally` clauses to execute. - -When ``yield from `` is used, it treats the supplied expression as -a subiterator, producing values from it until the underlying iterator is -exhausted. - - .. versionchanged:: 3.3 - Added ``yield from `` to delegate control flow to a subiterator - -For full details of :keyword:`yield` semantics, refer to the :ref:`yieldexpr` -section. +A :keyword:`yield` statement is semantically equivalent to a :ref:`yield +expression `. The yield statement can be used to omit the parentheses +that would otherwise be required in the equivalent yield expression +statement. For example, the yield statements :: -.. seealso:: + yield + yield from - :pep:`0255` - Simple Generators - The proposal for adding generators and the :keyword:`yield` statement to Python. +are equivalent to the yield expression statements :: - :pep:`0342` - Coroutines via Enhanced Generators - The proposal to enhance the API and syntax of generators, making them - usable as simple coroutines. + (yield ) + (yield from ) - :pep:`0380` - Syntax for Delegating to a Subgenerator - The proposal to introduce the :token:`yield_from` syntax, making delegation - to sub-generators easy. +Yield expressions and statements are only used when defining a :term:`generator` +function, and are only used in the body of the generator function. Using yield +in a function definition is sufficient to cause that definition to create a +generator function instead of a normal function. +For full details of :keyword:`yield` semantics, refer to the +:ref:`yieldexpr` section. .. _raise: -- cgit v0.12