diff options
| -rw-r--r-- | Doc/library/codecs.rst | 6 | ||||
| -rw-r--r-- | Doc/library/functions.rst | 40 | ||||
| -rw-r--r-- | Lib/codecs.py | 1 | ||||
| -rw-r--r-- | Lib/subprocess.py | 43 | ||||
| -rw-r--r-- | Lib/test/test_curses.py | 13 | ||||
| -rw-r--r-- | Misc/NEWS | 7 | ||||
| -rw-r--r-- | Modules/_curses_panel.c | 4 | ||||
| -rw-r--r-- | Modules/_io/_iomodule.c | 4 | ||||
| -rw-r--r-- | Modules/_io/textio.c | 5 |
9 files changed, 90 insertions, 33 deletions
diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 0d38253..e80fc3a 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -78,7 +78,11 @@ It defines the following functions: reference (for encoding only) * ``'backslashreplace'``: replace with backslashed escape sequences (for encoding only) - * ``'surrogateescape'``: replace with surrogate U+DCxx, see :pep:`383` + * ``'surrogateescape'``: on decoding, replace with code points in the Unicode + Private Use Area ranging from U+DC80 to U+DCFF. These private code + points will then be turned back into the same bytes when the + ``surrogateescape`` error handler is used when encoding the data. + (See :pep:`383` for more.) as well as any other error handling name defined via :func:`register_error`. diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 3059e17..04fb95e 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -895,16 +895,36 @@ are always available. They are listed here in alphabetical order. the list of supported encodings. *errors* is an optional string that specifies how encoding and decoding - errors are to be handled--this cannot be used in binary mode. Pass - ``'strict'`` to raise a :exc:`ValueError` exception if there is an encoding - error (the default of ``None`` has the same effect), or pass ``'ignore'`` to - ignore errors. (Note that ignoring encoding errors can lead to data loss.) - ``'replace'`` causes a replacement marker (such as ``'?'``) to be inserted - where there is malformed data. When writing, ``'xmlcharrefreplace'`` - (replace with the appropriate XML character reference) or - ``'backslashreplace'`` (replace with backslashed escape sequences) can be - used. Any other error handling name that has been registered with - :func:`codecs.register_error` is also valid. + errors are to be handled--this cannot be used in binary mode. + A variety of standard error handlers are available, though any + error handling name that has been registered with + :func:`codecs.register_error` is also valid. The standard names + are: + + * ``'strict'`` to raise a :exc:`ValueError` exception if there is + an encoding error. The default value of ``None`` has the same + effect. + + * ``'ignore'`` ignores errors. Note that ignoring encoding errors + can lead to data loss. + + * ``'replace'`` causes a replacement marker (such as ``'?'``) to be inserted + where there is malformed data. + + * ``'surrogateescape'`` will represent any incorrect bytes as code + points in the Unicode Private Use Area ranging from U+DC80 to + U+DCFF. These private code points will then be turned back into + the same bytes when the ``surrogateescape`` error handler is used + when writing data. This is useful for processing files in an + unknown encoding. + + * ``'xmlcharrefreplace'`` is only supported when writing to a file. + Characters not supported by the encoding are replaced with the + appropriate XML character reference ``&#nnn;``. + + * ``'backslashreplace'`` (also only supported when writing) + replaces unsupported characters with Python's backslashed escape + sequences. .. index:: single: universal newlines; open() built-in function diff --git a/Lib/codecs.py b/Lib/codecs.py index 48d4c9c..6a6eb90 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -105,6 +105,7 @@ class Codec: Python will use the official U+FFFD REPLACEMENT CHARACTER for the builtin Unicode codecs on decoding and '?' on encoding. + 'surrogateescape' - replace with private codepoints U+DCnn. 'xmlcharrefreplace' - Replace with the appropriate XML character reference (only for encoding). 'backslashreplace' - Replace with backslashed escape sequences diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 689046e..a62d9a4 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -810,6 +810,7 @@ class Popen(object): if universal_newlines: self.stderr = io.TextIOWrapper(self.stderr) + self._closed_child_pipe_fds = False try: self._execute_child(args, executable, preexec_fn, close_fds, pass_fds, cwd, env, @@ -826,19 +827,21 @@ class Popen(object): except EnvironmentError: pass # Ignore EBADF or other errors. - # Make sure the child pipes are closed as well. - to_close = [] - if stdin == PIPE: - to_close.append(p2cread) - if stdout == PIPE: - to_close.append(c2pwrite) - if stderr == PIPE: - to_close.append(errwrite) - for fd in to_close: - try: - os.close(fd) - except EnvironmentError: - pass + if not self._closed_child_pipe_fds: + to_close = [] + if stdin == PIPE: + to_close.append(p2cread) + if stdout == PIPE: + to_close.append(c2pwrite) + if stderr == PIPE: + to_close.append(errwrite) + if hasattr(self, '_devnull'): + to_close.append(self._devnull) + for fd in to_close: + try: + os.close(fd) + except EnvironmentError: + pass raise @@ -1383,14 +1386,18 @@ class Popen(object): # be sure the FD is closed no matter what os.close(errpipe_write) - if p2cread != -1 and p2cwrite != -1: + # self._devnull is not always defined. + devnull_fd = getattr(self, '_devnull', None) + if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd: os.close(p2cread) - if c2pwrite != -1 and c2pread != -1: + if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd: os.close(c2pwrite) - if errwrite != -1 and errread != -1: + if errwrite != -1 and errread != -1 and errwrite != devnull_fd: os.close(errwrite) - if hasattr(self, '_devnull'): - os.close(self._devnull) + if devnull_fd is not None: + os.close(devnull_fd) + # Prevent a double close of these fds from __init__ on error. + self._closed_child_pipe_fds = True # Wait for exec to fail or succeed; possibly raising an # exception (limited in size) diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index e959622..76812b7 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -252,6 +252,18 @@ def test_userptr_without_set(stdscr): except curses.panel.error: pass +def test_userptr_memory_leak(stdscr): + w = curses.newwin(10, 10) + p = curses.panel.new_panel(w) + obj = object() + nrefs = sys.getrefcount(obj) + for i in range(100): + p.set_userptr(obj) + + p.set_userptr(None) + if sys.getrefcount(obj) != nrefs: + raise RuntimeError("set_userptr leaked references") + def test_resize_term(stdscr): if hasattr(curses, 'resizeterm'): lines, cols = curses.LINES, curses.COLS @@ -317,6 +329,7 @@ def main(stdscr): module_funcs(stdscr) window_funcs(stdscr) test_userptr_without_set(stdscr) + test_userptr_memory_leak(stdscr) test_resize_term(stdscr) test_issue6243(stdscr) test_unget_wch(stdscr) @@ -32,6 +32,10 @@ Core and Builtins Library ------- +- subprocess: Prevent a possible double close of parent pipe fds when the + subprocess exec runs into an error. Prevent a regular multi-close of the + /dev/null fd when any of stdin, stdout and stderr was set to DEVNULL. + - Issue #16102: Make uuid._netbios_getnode() work again on Python 3. - Issue #18109: os.uname() now decodes fields from the locale encoding, and @@ -78,6 +82,9 @@ Library the default for linking if LDSHARED is not also overriden. This restores Distutils behavior introduced in 3.2.3 and inadvertently dropped in 3.3.0. +- Issue #18113: Fixed a refcount leak in the curses.panel module's + set_userptr() method. Reported by Atsuo Ishimoto. + IDLE ---- diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 1c7d084..c11f3d8 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -322,6 +322,10 @@ PyCursesPanel_replace_panel(PyCursesPanelObject *self, PyObject *args) static PyObject * PyCursesPanel_set_panel_userptr(PyCursesPanelObject *self, PyObject *obj) { + PyObject *oldobj; + PyCursesInitialised; + oldobj = (PyObject *) panel_userptr(self->pan); + Py_XDECREF(oldobj); Py_INCREF(obj); return PyCursesCheckERR(set_panel_userptr(self->pan, (void*)obj), "set_panel_userptr"); diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index b5cd176..4a7e758 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -168,8 +168,8 @@ PyDoc_STRVAR(open_doc, "'strict' to raise a ValueError exception if there is an encoding error\n" "(the default of None has the same effect), or pass 'ignore' to ignore\n" "errors. (Note that ignoring encoding errors can lead to data loss.)\n" -"See the documentation for codecs.register for a list of the permitted\n" -"encoding error strings.\n" +"See the documentation for codecs.register or run 'help(codecs.Codec)'\n" +"for a list of the permitted encoding error strings.\n" "\n" "newline controls how universal newlines works (it only applies to text\n" "mode). It can be None, '', '\\n', '\\r', and '\\r\\n'. It works as\n" diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index cff9c6e..cd751c1 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -642,8 +642,9 @@ PyDoc_STRVAR(textiowrapper_doc, "encoding gives the name of the encoding that the stream will be\n" "decoded or encoded with. It defaults to locale.getpreferredencoding(False).\n" "\n" - "errors determines the strictness of encoding and decoding (see the\n" - "codecs.register) and defaults to \"strict\".\n" + "errors determines the strictness of encoding and decoding (see\n" + "help(codecs.Codec) or the documentation for codecs.register) and\n" + "defaults to \"strict\".\n" "\n" "newline controls how line endings are handled. It can be None, '',\n" "'\\n', '\\r', and '\\r\\n'. It works as follows:\n" |