diff options
Diffstat (limited to 'Doc')
250 files changed, 19553 insertions, 3835 deletions
diff --git a/Doc/Makefile b/Doc/Makefile index 82f5bef..370c3fa 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -5,7 +5,7 @@ # You can set these variables from the command line. PYTHON = python -SVNROOT = http://svn.python.org/projects +SPHINXBUILD = sphinx-build SPHINXOPTS = PAPER = SOURCES = @@ -14,15 +14,15 @@ DISTVERSION = $(shell $(PYTHON) tools/sphinxext/patchlevel.py) ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_paper_size=$(PAPER) \ $(SPHINXOPTS) . build/$(BUILDER) $(SOURCES) -.PHONY: help checkout update build html htmlhelp latex text changes linkcheck \ +.PHONY: help build html htmlhelp latex text changes linkcheck \ suspicious coverage doctest pydoc-topics htmlview clean dist check serve \ autobuild-dev autobuild-stable help: @echo "Please use \`make <target>' where <target> is one of" @echo " clean to remove build files" - @echo " update to update build tools" @echo " html to make standalone HTML files" + @echo " htmlview to open the index page built by the html target in your browser" @echo " htmlhelp to make HTML files and a HTML help project" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " text to make plain text files" @@ -37,30 +37,8 @@ help: @echo " check to run a check for frequent markup errors" @echo " serve to serve the documentation on the localhost (8000)" -# Note: if you update versions here, do the same in make.bat and README.txt -checkout: - @if [ ! -d tools/sphinx ]; then \ - echo "Checking out Sphinx..."; \ - svn checkout $(SVNROOT)/external/Sphinx-1.2/sphinx tools/sphinx; \ - fi - @if [ ! -d tools/docutils ]; then \ - echo "Checking out Docutils..."; \ - svn checkout $(SVNROOT)/external/docutils-0.11/docutils tools/docutils; \ - fi - @if [ ! -d tools/jinja2 ]; then \ - echo "Checking out Jinja..."; \ - svn checkout $(SVNROOT)/external/Jinja-2.3.1/jinja2 tools/jinja2; \ - fi - @if [ ! -d tools/pygments ]; then \ - echo "Checking out Pygments..."; \ - svn checkout $(SVNROOT)/external/Pygments-1.6/pygments tools/pygments; \ - fi - -update: clean checkout - -build: checkout - mkdir -p build/$(BUILDER) build/doctrees - $(PYTHON) tools/sphinx-build.py $(ALLSPHINXOPTS) +build: + $(SPHINXBUILD) $(ALLSPHINXOPTS) @echo html: BUILDER = html @@ -120,10 +98,6 @@ htmlview: html clean: -rm -rf build/* - -rm -rf tools/sphinx - -rm -rf tools/pygments - -rm -rf tools/jinja2 - -rm -rf tools/docutils dist: rm -rf dist @@ -163,16 +137,10 @@ dist: cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-letter.zip cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-letter.tar.bz2 - # archive the epub build + # copy the epub build rm -rf build/epub make epub - mkdir -p dist/python-$(DISTVERSION)-docs-epub - cp -pPR build/epub/*.epub dist/python-$(DISTVERSION)-docs-epub/ - tar -C dist -cf dist/python-$(DISTVERSION)-docs-epub.tar python-$(DISTVERSION)-docs-epub - bzip2 -9 -k dist/python-$(DISTVERSION)-docs-epub.tar - (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-epub.zip python-$(DISTVERSION)-docs-epub) - rm -r dist/python-$(DISTVERSION)-docs-epub - rm dist/python-$(DISTVERSION)-docs-epub.tar + cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub check: $(PYTHON) tools/rstlint.py -i tools @@ -184,7 +152,6 @@ serve: # for development releases: always build autobuild-dev: - make update make dist SPHINXOPTS='-A daily=1 -A versionswitcher=1' -make suspicious diff --git a/Doc/README.txt b/Doc/README.txt index a494c89..6df12ab 100644 --- a/Doc/README.txt +++ b/Doc/README.txt @@ -3,34 +3,30 @@ Python Documentation README This directory contains the reStructuredText (reST) sources to the Python documentation. You don't need to build them yourself, prebuilt versions are -available at http://docs.python.org/download/. +available at <https://docs.python.org/3.4/download.html>. -Documentation on the authoring Python documentation, including information about +Documentation on authoring Python documentation, including information about both style and markup, is available in the "Documenting Python" chapter of the -documentation. +developers guide <http://docs.python.org/devguide/documenting.html>. Building the docs ================= -You need to have Python 2 installed; the toolset used to build the -docs is written in Python. It is called *Sphinx*, it is not included in this -tree, but maintained separately. Also needed are the docutils, supplying the -base markup that Sphinx uses, Jinja, a templating engine, and optionally -Pygments, a code highlighter. +You need to have Sphinx <http://sphinx-doc.org/> installed; it is the toolset +used to build the docs. It is not included in this tree, but maintained +separately and available from PyPI <http://pypi.python.org/pypi/Sphinx>. Using make ---------- -Luckily, a Makefile has been prepared so that on Unix, provided you have -installed Python and Subversion, you can just run :: +A Makefile has been prepared so that on Unix, provided you have installed +Sphinx, you can just run :: make html -to check out the necessary toolset in the `tools/` subdirectory and build the -HTML output files. To view the generated HTML, point your favorite browser at -the top-level index `build/html/index.html` after running "make". +to build the HTML output files. On Windows, we try to emulate the Makefile as closely as possible with a ``make.bat`` file. @@ -38,18 +34,29 @@ On Windows, we try to emulate the Makefile as closely as possible with a To use a Python interpreter that's not called ``python``, use the standard way to set Makefile variables, using e.g. :: - make html PYTHON=/usr/bin/python2.5 + make html PYTHON=python3 + +On Windows, set the PYTHON environment variable instead. + +To use a specific sphinx-build (something other than ``sphinx-build``), set +the SPHINXBUILD variable. Available make targets are: + * "clean", which removes all build files. + * "html", which builds standalone HTML files for offline viewing. + * "htmlview", which re-uses the "html" builder, but then opens the main page + in your default web browser. + * "htmlhelp", which builds HTML files and a HTML Help project file usable to convert them into a single Compiled HTML (.chm) file -- these are popular under Microsoft Windows, but very handy on every platform. - To create the CHM file, you need to run the Microsoft HTML Help Workshop over - the generated project (.hhp) file. + To create the CHM file, you need to run the Microsoft HTML Help Workshop + over the generated project (.hhp) file. The make.bat script does this for + you on Windows. * "latex", which builds LaTeX source files as input to "pdflatex" to produce PDF documents. @@ -78,7 +85,12 @@ Available make targets are: * "suspicious", which checks the parsed markup for text that looks like malformed and thus unconverted reST. -A "make update" updates the Subversion checkouts in `tools/`. + * "check", which checks for frequent markup errors. + + * "serve", which serves the build/html directory on port 8000. + + * "dist", (Unix only) which creates distributable archives of HTML, text, + PDF, and EPUB builds. Without make @@ -86,7 +98,7 @@ Without make Install the Sphinx package and its dependencies from PyPI. -Then, from the ``Docs`` directory, run :: +Then, from the ``Doc`` directory, run :: sphinx-build -b<builder> . build/<builder> diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 28a434e..2f02241 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -295,6 +295,8 @@ Other objects the object pointer is stored. If the Python object does not have the required type, :exc:`TypeError` is raised. +.. _o_ampersand: + ``O&`` (object) [*converter*, *anything*] Convert a Python object to a C variable through a *converter* function. This takes two arguments: the first is a function, the second is the address of a C diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index 65904ee..2d56386 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -74,26 +74,35 @@ intrinsic to the Python language. .. _mapobjects: -Mapping Objects -=============== +Container Objects +================= .. index:: object: mapping .. toctree:: dict.rst + set.rst .. _otherobjects: -Other Objects -============= +Function Objects +================ .. toctree:: - set.rst function.rst method.rst + cell.rst + code.rst + + +Other Objects +============= + +.. toctree:: + file.rst module.rst iterator.rst @@ -102,7 +111,6 @@ Other Objects memoryview.rst weakref.rst capsule.rst - cell.rst gen.rst datetime.rst - code.rst + diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 6bacc32..aeff640 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -84,7 +84,7 @@ Dictionary Objects on failure. -.. c:function:: int PyDict_DelItemString(PyObject *p, char *key) +.. c:function:: int PyDict_DelItemString(PyObject *p, const char *key) Remove the entry in dictionary *p* which has a key specified by the string *key*. Return ``0`` on success or ``-1`` on failure. @@ -110,6 +110,15 @@ Dictionary Objects :c:type:`char\*`, rather than a :c:type:`PyObject\*`. +.. c:function:: PyObject* PyDict_SetDefault(PyObject *p, PyObject *key, PyObject *default) + + This is the same as the Python-level :meth:`dict.setdefault`. If present, it + returns the value corresponding to *key* from the dictionary *p*. If the key + is not in the dict, it is inserted with value *defaultobj* and *defaultobj* + is returned. This function evaluates the hash function of *key* only once, + instead of evaluating it independently for the lookup and the insertion. + + .. c:function:: PyObject* PyDict_Items(PyObject *p) Return a :c:type:`PyListObject` containing all the items from the dictionary. @@ -192,8 +201,11 @@ Dictionary Objects .. c:function:: int PyDict_Update(PyObject *a, PyObject *b) - This is the same as ``PyDict_Merge(a, b, 1)`` in C, or ``a.update(b)`` in - Python. Return ``0`` on success or ``-1`` if an exception was raised. + This is the same as ``PyDict_Merge(a, b, 1)`` in C, and is similar to + ``a.update(b)`` in Python except that :c:func:`PyDict_Update` doesn't fall + back to the iterating over a sequence of key value pairs if the second + argument has no "keys" attribute. Return ``0`` on success or ``-1`` if an + exception was raised. .. c:function:: int PyDict_MergeFromSeq2(PyObject *a, PyObject *seq2, int override) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 0aa892d..33b4439 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -90,6 +90,16 @@ in various ways. There is a separate error indicator for each thread. the class in that case. If the values are already normalized, nothing happens. The delayed normalization is implemented to improve performance. + .. note:: + + This function *does not* implicitly set the ``__traceback__`` + attribute on the exception value. If setting the traceback + appropriately is desired, the following additional snippet is needed:: + + if (tb != NULL) { + PyException_SetTraceback(val, tb); + } + .. c:function:: void PyErr_Clear() @@ -226,16 +236,25 @@ in various ways. There is a separate error indicator for each thread. Similar to :c:func:`PyErr_SetFromErrno`, with the additional behavior that if *filenameObject* is not *NULL*, it is passed to the constructor of *type* as - a third parameter. In the case of exceptions such as :exc:`IOError` and - :exc:`OSError`, this is used to define the :attr:`filename` attribute of the + a third parameter. In the case of :exc:`OSError` exception, + this is used to define the :attr:`filename` attribute of the exception instance. +.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObjects(PyObject *type, PyObject *filenameObject, PyObject *filenameObject2) + + Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but takes a second + filename object, for raising errors when a function that takes two filenames + fails. + + .. versionadded:: 3.4 + + .. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename) Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but the filename is given as a C string. *filename* is decoded from the filesystem encoding - (:func:`sys.getfilesystemencoding`). + (:func:`os.fsdecode`). .. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr) @@ -256,18 +275,11 @@ in various ways. There is a separate error indicator for each thread. specifying the exception type to be raised. Availability: Windows. -.. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilenameObject(int ierr, PyObject *filenameObject) - - Similar to :c:func:`PyErr_SetFromWindowsErr`, with the additional behavior - that if *filenameObject* is not *NULL*, it is passed to the constructor of - :exc:`WindowsError` as a third parameter. Availability: Windows. - - .. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, but the filename is given as a C string. *filename* is decoded from the filesystem - encoding (:func:`sys.getfilesystemencoding`). Availability: Windows. + encoding (:func:`os.fsdecode`). Availability: Windows. .. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename) @@ -277,6 +289,15 @@ in various ways. There is a separate error indicator for each thread. Availability: Windows. +.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObjects(PyObject *type, int ierr, PyObject *filename, PyObject *filename2) + + Similar to :c:func:`PyErr_SetExcFromWindowsErrWithFilenameObject`, + but accepts a second filename object. + Availability: Windows. + + .. versionadded:: 3.4 + + .. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilename(PyObject *type, int ierr, const char *filename) Similar to :c:func:`PyErr_SetFromWindowsErrWithFilename`, with an additional @@ -293,13 +314,20 @@ in various ways. There is a separate error indicator for each thread. .. versionadded:: 3.3 -.. c:function:: void PyErr_SyntaxLocationEx(char *filename, int lineno, int col_offset) +.. c:function:: void PyErr_SyntaxLocationObject(PyObject *filename, int lineno, int col_offset) Set file, line, and offset information for the current exception. If the current exception is not a :exc:`SyntaxError`, then it sets additional attributes, which make the exception printing subsystem think the exception - is a :exc:`SyntaxError`. *filename* is decoded from the filesystem encoding - (:func:`sys.getfilesystemencoding`). + is a :exc:`SyntaxError`. + + .. versionadded:: 3.4 + + +.. c:function:: void PyErr_SyntaxLocationEx(char *filename, int lineno, int col_offset) + + Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string + decoded from the filesystem encoding (:func:`os.fsdecode`). .. versionadded:: 3.2 @@ -355,15 +383,22 @@ in various ways. There is a separate error indicator for each thread. documentation. There is no C API for warning control. -.. c:function:: int PyErr_WarnExplicit(PyObject *category, const char *message, const char *filename, int lineno, const char *module, PyObject *registry) +.. c:function:: int PyErr_WarnExplicitObject(PyObject *category, PyObject *message, PyObject *filename, int lineno, PyObject *module, PyObject *registry) Issue a warning message with explicit control over all warning attributes. This is a straightforward wrapper around the Python function :func:`warnings.warn_explicit`, see there for more information. The *module* and *registry* arguments may be set to *NULL* to get the default effect - described there. *message* and *module* are UTF-8 encoded strings, - *filename* is decoded from the filesystem encoding - (:func:`sys.getfilesystemencoding`). + described there. + + .. versionadded:: 3.4 + + +.. c:function:: int PyErr_WarnExplicit(PyObject *category, const char *message, const char *filename, int lineno, const char *module, PyObject *registry) + + Similar to :c:func:`PyErr_WarnExplicitObject` except that *message* and + *module* are UTF-8 encoded strings, and *filename* is decoded from the + filesystem encoding (:func:`os.fsdecode`). .. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index cc190c9..6f2ecee 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -17,7 +17,7 @@ error reporting in the interpreter; third-party code is advised to access the :mod:`io` APIs instead. -.. c:function:: PyFile_FromFd(int fd, char *name, char *mode, int buffering, char *encoding, char *errors, char *newline, int closefd) +.. c:function:: PyFile_FromFd(int fd, const char *name, const char *mode, int buffering, const char *encoding, const char *errors, const char *newline, int closefd) Create a Python file object from the file descriptor of an already opened file *fd*. The arguments *name*, *encoding*, *errors* and *newline* diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 270152e..3641fc6 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -118,7 +118,7 @@ Importing Modules encoded string instead of a Unicode object. -.. c:function:: PyObject* PyImport_ExecCodeModule(char *name, PyObject *co) +.. c:function:: PyObject* PyImport_ExecCodeModule(const char *name, PyObject *co) .. index:: builtin: compile @@ -132,8 +132,14 @@ Importing Modules such modules have no way to know that the module object is an unknown (and probably damaged with respect to the module author's intents) state. + The module's :attr:`__spec__` and :attr:`__loader__` will be set, if + not set already, with the appropriate values. The spec's loader will + be set to the module's ``__loader__`` (if set) and to an instance of + :class:`SourceFileLoader` otherwise. + The module's :attr:`__file__` attribute will be set to the code object's - :c:member:`co_filename`. + :c:member:`co_filename`. If applicable, :attr:`__cached__` will also + be set. This function will reload the module if it was already imported. See :c:func:`PyImport_ReloadModule` for the intended way to reload a module. @@ -145,7 +151,7 @@ Importing Modules :c:func:`PyImport_ExecCodeModuleWithPathnames`. -.. c:function:: PyObject* PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname) +.. c:function:: PyObject* PyImport_ExecCodeModuleEx(const char *name, PyObject *co, const char *pathname) Like :c:func:`PyImport_ExecCodeModule`, but the :attr:`__file__` attribute of the module object is set to *pathname* if it is non-``NULL``. @@ -162,7 +168,7 @@ Importing Modules .. versionadded:: 3.3 -.. c:function:: PyObject* PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, char *cpathname) +.. c:function:: PyObject* PyImport_ExecCodeModuleWithPathnames(const char *name, PyObject *co, const char *pathname, const char *cpathname) Like :c:func:`PyImport_ExecCodeModuleObject`, but *name*, *pathname* and *cpathname* are UTF-8 encoded strings. Attempts are also made to figure out @@ -245,8 +251,11 @@ Importing Modules .. versionadded:: 3.3 + .. versionchanged:: 3.4 + The ``__file__`` attribute is no longer set on the module. + -.. c:function:: int PyImport_ImportFrozenModule(char *name) +.. c:function:: int PyImport_ImportFrozenModule(const char *name) Similar to :c:func:`PyImport_ImportFrozenModuleObject`, but the name is a UTF-8 encoded string instead of a Unicode object. diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 4dad2c8..0587e15 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -86,6 +86,36 @@ Process-wide parameters ======================= +.. c:function:: int Py_SetStandardStreamEncoding(char *encoding, char *errors) + + .. index:: + single: Py_Initialize() + single: main() + triple: stdin; stdout; sdterr + + This function should be called before :c:func:`Py_Initialize`, if it is + called at all. It specifies which encoding and error handling to use + with standard IO, with the same meanings as in :func:`str.encode`. + + It overrides :envvar:`PYTHONIOENCODING` values, and allows embedding code + to control IO encoding when the environment variable does not work. + + ``encoding`` and/or ``errors`` may be NULL to use + :envvar:`PYTHONIOENCODING` and/or default values (depending on other + settings). + + Note that :data:`sys.stderr` always uses the "backslashreplace" error + handler, regardless of this (or any other) setting. + + If :c:func:`Py_Finalize` is called, this function will need to be called + again in order to affect subsequent calls to :c:func:`Py_Initialize`. + + Returns 0 if successful, a nonzero value on error (e.g. calling after the + interpreter has already been initialized). + + .. versionadded:: 3.4 + + .. c:function:: void Py_SetProgramName(wchar_t *name) .. index:: @@ -329,7 +359,11 @@ Process-wide parameters .. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) - This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set to 1. + This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set + to 1 unless the :program:`python` interpreter was started with the + :option:`-I`. + + .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. .. c:function:: void Py_SetPythonHome(wchar_t *home) @@ -658,6 +692,20 @@ with sub-interpreters: made on the main thread. This is mainly a helper/diagnostic function. +.. c:function:: int PyGILState_Check() + + Return 1 if the current thread is holding the GIL and 0 otherwise. + This function can be called from any thread at any time. + Only if it has had its Python thread state initialized and currently is + holding the GIL will it return 1. + This is mainly a helper/diagnostic function. It can be useful + for example in callback contexts or memory allocation functions when + knowing that the GIL is locked can allow the caller to perform sensitive + actions or otherwise behave differently. + + .. versionadded:: 3.4 + + The following macros are normally used without a trailing semicolon; look for example usage in the Python source distribution. diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index d5430fd..b348015 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -80,7 +80,7 @@ All integers are implemented as "long" integer objects of arbitrary size. *NULL* on failure. -.. c:function:: PyObject* PyLong_FromString(char *str, char **pend, int base) +.. c:function:: PyObject* PyLong_FromString(const char *str, char **pend, int base) Return a new :c:type:`PyLongObject` based on the string value in *str*, which is interpreted according to the radix in *base*. If *pend* is non-*NULL*, diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst index 0ef2961..2803fd0 100644 --- a/Doc/c-api/mapping.rst +++ b/Doc/c-api/mapping.rst @@ -22,7 +22,7 @@ Mapping Protocol expression ``len(o)``. -.. c:function:: int PyMapping_DelItemString(PyObject *o, char *key) +.. c:function:: int PyMapping_DelItemString(PyObject *o, const char *key) Remove the mapping for object *key* from the object *o*. Return ``-1`` on failure. This is equivalent to the Python statement ``del o[key]``. @@ -67,13 +67,13 @@ Mapping Protocol the Python expression ``list(o.items())``. -.. c:function:: PyObject* PyMapping_GetItemString(PyObject *o, char *key) +.. c:function:: PyObject* PyMapping_GetItemString(PyObject *o, const char *key) Return element of *o* corresponding to the object *key* or *NULL* on failure. This is the equivalent of the Python expression ``o[key]``. -.. c:function:: int PyMapping_SetItemString(PyObject *o, char *key, PyObject *v) +.. c:function:: int PyMapping_SetItemString(PyObject *o, const char *key, PyObject *v) Map the object *key* to the value *v* in object *o*. Returns ``-1`` on failure. This is the equivalent of the Python statement ``o[key] = v``. diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 8afa56a..a82e1c2 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -84,6 +84,48 @@ the C library allocator as shown in the previous example, the allocated memory for the I/O buffer escapes completely the Python memory manager. +Raw Memory Interface +==================== + +The following function sets are wrappers to the system allocator. These +functions are thread-safe, the :term:`GIL <global interpreter lock>` does not +need to be held. + +The default raw memory block allocator uses the following functions: +:c:func:`malloc`, :c:func:`realloc` and :c:func:`free`; call ``malloc(1)`` when +requesting zero bytes. + +.. versionadded:: 3.4 + +.. c:function:: void* PyMem_RawMalloc(size_t n) + + Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the + allocated memory, or *NULL* if the request fails. Requesting zero bytes + returns a distinct non-*NULL* pointer if possible, as if + ``PyMem_RawMalloc(1)`` had been called instead. The memory will not have + been initialized in any way. + + +.. c:function:: void* PyMem_RawRealloc(void *p, size_t n) + + Resizes the memory block pointed to by *p* to *n* bytes. The contents will + be unchanged to the minimum of the old and the new sizes. If *p* is *NULL*, + the call is equivalent to ``PyMem_RawMalloc(n)``; else if *n* is equal to + zero, the memory block is resized but is not freed, and the returned pointer + is non-*NULL*. Unless *p* is *NULL*, it must have been returned by a + previous call to :c:func:`PyMem_RawMalloc` or :c:func:`PyMem_RawRealloc`. If + the request fails, :c:func:`PyMem_RawRealloc` returns *NULL* and *p* remains + a valid pointer to the previous memory area. + + +.. c:function:: void PyMem_RawFree(void *p) + + Frees the memory block pointed to by *p*, which must have been returned by a + previous call to :c:func:`PyMem_RawMalloc` or :c:func:`PyMem_RawRealloc`. + Otherwise, or if ``PyMem_Free(p)`` has been called before, undefined + behavior occurs. If *p* is *NULL*, no operation is performed. + + .. _memoryinterface: Memory Interface @@ -91,8 +133,16 @@ Memory Interface The following function sets, modeled after the ANSI C standard, but specifying behavior when requesting zero bytes, are available for allocating and releasing -memory from the Python heap: +memory from the Python heap. + +The default memory block allocator uses the following functions: +:c:func:`malloc`, :c:func:`realloc` and :c:func:`free`; call ``malloc(1)`` when +requesting zero bytes. + +.. warning:: + The :term:`GIL <global interpreter lock>` must be held when using these + functions. .. c:function:: void* PyMem_Malloc(size_t n) @@ -155,6 +205,125 @@ versions and is therefore deprecated in extension modules. :c:func:`PyMem_NEW`, :c:func:`PyMem_RESIZE`, :c:func:`PyMem_DEL`. +Customize Memory Allocators +=========================== + +.. versionadded:: 3.4 + +.. c:type:: PyMemAllocator + + Structure used to describe a memory block allocator. The structure has + four fields: + + +----------------------------------------------------------+---------------------------------------+ + | Field | Meaning | + +==========================================================+=======================================+ + | ``void *ctx`` | user context passed as first argument | + +----------------------------------------------------------+---------------------------------------+ + | ``void* malloc(void *ctx, size_t size)`` | allocate a memory block | + +----------------------------------------------------------+---------------------------------------+ + | ``void* realloc(void *ctx, void *ptr, size_t new_size)`` | allocate or resize a memory block | + +----------------------------------------------------------+---------------------------------------+ + | ``void free(void *ctx, void *ptr)`` | free a memory block | + +----------------------------------------------------------+---------------------------------------+ + +.. c:type:: PyMemAllocatorDomain + + Enum used to identify an allocator domain. Domains: + + * :c:data:`PYMEM_DOMAIN_RAW`: functions :c:func:`PyMem_RawMalloc`, + :c:func:`PyMem_RawRealloc` and :c:func:`PyMem_RawFree` + * :c:data:`PYMEM_DOMAIN_MEM`: functions :c:func:`PyMem_Malloc`, + :c:func:`PyMem_Realloc` and :c:func:`PyMem_Free` + * :c:data:`PYMEM_DOMAIN_OBJ`: functions :c:func:`PyObject_Malloc`, + :c:func:`PyObject_Realloc` and :c:func:`PyObject_Free` + + +.. c:function:: void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator) + + Get the memory block allocator of the specified domain. + + +.. c:function:: void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator) + + Set the memory block allocator of the specified domain. + + The new allocator must return a distinct non-NULL pointer when requesting + zero bytes. + + For the :c:data:`PYMEM_DOMAIN_RAW` domain, the allocator must be + thread-safe: the :term:`GIL <global interpreter lock>` is not held when the + allocator is called. + + If the new allocator is not a hook (does not call the previous allocator), + the :c:func:`PyMem_SetupDebugHooks` function must be called to reinstall the + debug hooks on top on the new allocator. + + +.. c:function:: void PyMem_SetupDebugHooks(void) + + Setup hooks to detect bugs in the following Python memory allocator + functions: + + - :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc`, + :c:func:`PyMem_RawFree` + - :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc`, :c:func:`PyMem_Free` + - :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc`, + :c:func:`PyObject_Free` + + Newly allocated memory is filled with the byte ``0xCB``, freed memory is + filled with the byte ``0xDB``. Additionnal checks: + + - detect API violations, ex: :c:func:`PyObject_Free` called on a buffer + allocated by :c:func:`PyMem_Malloc` + - detect write before the start of the buffer (buffer underflow) + - detect write after the end of the buffer (buffer overflow) + + The function does nothing if Python is not compiled is debug mode. + + +Customize PyObject Arena Allocator +================================== + +Python has a *pymalloc* allocator for allocations smaller than 512 bytes. This +allocator is optimized for small objects with a short lifetime. It uses memory +mappings called "arenas" with a fixed size of 256 KB. It falls back to +:c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger +than 512 bytes. *pymalloc* is the default allocator used by +:c:func:`PyObject_Malloc`. + +The default arena allocator uses the following functions: + +* :c:func:`VirtualAlloc` and :c:func:`VirtualFree` on Windows, +* :c:func:`mmap` and :c:func:`munmap` if available, +* :c:func:`malloc` and :c:func:`free` otherwise. + +.. versionadded:: 3.4 + +.. c:type:: PyObjectArenaAllocator + + Structure used to describe an arena allocator. The structure has + three fields: + + +--------------------------------------------------+---------------------------------------+ + | Field | Meaning | + +==================================================+=======================================+ + | ``void *ctx`` | user context passed as first argument | + +--------------------------------------------------+---------------------------------------+ + | ``void* alloc(void *ctx, size_t size)`` | allocate an arena of size bytes | + +--------------------------------------------------+---------------------------------------+ + | ``void free(void *ctx, size_t size, void *ptr)`` | free an arena | + +--------------------------------------------------+---------------------------------------+ + +.. c:function:: PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) + + Get the arena allocator. + +.. c:function:: PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) + + Set the arena allocator. + + .. _memoryexamples: Examples diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index bd46170..985a347 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -35,13 +35,20 @@ There are only a few functions special to module objects. single: __name__ (module attribute) single: __doc__ (module attribute) single: __file__ (module attribute) + single: __package__ (module attribute) + single: __loader__ (module attribute) Return a new module object with the :attr:`__name__` attribute set to *name*. - Only the module's :attr:`__doc__` and :attr:`__name__` attributes are filled in; - the caller is responsible for providing a :attr:`__file__` attribute. + The module's :attr:`__name__`, :attr:`__doc__`, :attr:`__package__`, and + :attr:`__loader__` attributes are filled in (all but :attr:`__name__` are set + to ``None``); the caller is responsible for providing a :attr:`__file__` + attribute. .. versionadded:: 3.3 + .. versionchanged:: 3.4 + :attr:`__package__` and :attr:`__loader__` are set to ``None``. + .. c:function:: PyObject* PyModule_New(const char *name) diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 0aba360..3e7c3b6 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -149,6 +149,9 @@ Object Protocol representation on success, *NULL* on failure. This is the equivalent of the Python expression ``repr(o)``. Called by the :func:`repr` built-in function. + .. versionchanged:: 3.4 + This function now includes a debug assertion to help ensure that it + does not silently discard an active exception. .. c:function:: PyObject* PyObject_ASCII(PyObject *o) @@ -170,6 +173,10 @@ Object Protocol Python expression ``str(o)``. Called by the :func:`str` built-in function and, therefore, by the :func:`print` function. + .. versionchanged:: 3.4 + This function now includes a debug assertion to help ensure that it + does not silently discard an active exception. + .. c:function:: PyObject* PyObject_Bytes(PyObject *o) .. index:: builtin: bytes @@ -240,7 +247,7 @@ attribute is considered sufficient for this determination. of the Python expression ``callable_object(*args)``. -.. c:function:: PyObject* PyObject_CallFunction(PyObject *callable, char *format, ...) +.. c:function:: PyObject* PyObject_CallFunction(PyObject *callable, const char *format, ...) Call a callable Python object *callable*, with a variable number of C arguments. The C arguments are described using a :c:func:`Py_BuildValue` style format @@ -250,8 +257,11 @@ attribute is considered sufficient for this determination. pass :c:type:`PyObject \*` args, :c:func:`PyObject_CallFunctionObjArgs` is a faster alternative. + .. versionchanged:: 3.4 + The type of *format* was changed from ``char *``. + -.. c:function:: PyObject* PyObject_CallMethod(PyObject *o, char *method, char *format, ...) +.. c:function:: PyObject* PyObject_CallMethod(PyObject *o, const char *method, const char *format, ...) Call the method named *method* of object *o* with a variable number of C arguments. The C arguments are described by a :c:func:`Py_BuildValue` format @@ -261,6 +271,9 @@ attribute is considered sufficient for this determination. Note that if you only pass :c:type:`PyObject \*` args, :c:func:`PyObject_CallMethodObjArgs` is a faster alternative. + .. versionchanged:: 3.4 + The types of *method* and *format* were changed from ``char *``. + .. c:function:: PyObject* PyObject_CallFunctionObjArgs(PyObject *callable, ..., NULL) @@ -342,6 +355,16 @@ attribute is considered sufficient for this determination. returned. This is the equivalent to the Python expression ``len(o)``. +.. c:function:: Py_ssize_t PyObject_LengthHint(PyObject *o, Py_ssize_t default) + + Return an estimated length for the object *o*. First try to return its + actual length, then an estimate using :meth:`~object.__length_hint__`, and + finally return the default value. On error return ``-1``. This is the + equivalent to the Python expression ``operator.length_hint(o, default)``. + + .. versionadded:: 3.4 + + .. c:function:: PyObject* PyObject_GetItem(PyObject *o, PyObject *key) Return element of *o* corresponding to the object *key* or *NULL* on failure. diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index 0297ba3..cf1e142 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -123,10 +123,10 @@ Sequence Protocol .. c:function:: PyObject* PySequence_Fast(PyObject *o, const char *m) - Returns the sequence *o* as a tuple, unless it is already a tuple or list, in - which case *o* is returned. Use :c:func:`PySequence_Fast_GET_ITEM` to access the - members of the result. Returns *NULL* on failure. If the object is not a - sequence, raises :exc:`TypeError` with *m* as the message text. + Return the sequence *o* as a list, unless it is already a tuple or list, in + which case *o* is returned. Use :c:func:`PySequence_Fast_GET_ITEM` to access + the members of the result. Returns *NULL* on failure. If the object is not + a sequence, raises :exc:`TypeError` with *m* as the message text. .. c:function:: PyObject* PySequence_Fast_GET_ITEM(PyObject *o, Py_ssize_t i) diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index 184affb..3922d50 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -129,6 +129,14 @@ type. Initializes a struct sequence type *type* from *desc* in place. +.. c:function:: int PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) + + The same as ``PyStructSequence_InitType``, but returns ``0`` on success and ``-1`` on + failure. + + .. versionadded:: 3.4 + + .. c:type:: PyStructSequence_Desc Contains the meta information of a struct sequence type to create. diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 5d83254..8dc040b 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -97,3 +97,13 @@ Type Objects types. This allows the caller to reference other heap types as base types. .. versionadded:: 3.3 + +.. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) + + Return the function pointer stored in the given slot. If the + result is *NULL*, this indicates that either the slot is *NULL*, + or that the function was called with invalid parameters. + Callers will typically cast the result pointer into the appropriate + function type. + + .. versionadded:: 3.4 diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index f3089d0..3a64724 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -464,6 +464,32 @@ type objects) *must* have the :attr:`ob_size` field. :const:`Py_TPFLAGS_HAVE_VERSION_TAG`. + .. data:: Py_TPFLAGS_LONG_SUBCLASS + .. data:: Py_TPFLAGS_LIST_SUBCLASS + .. data:: Py_TPFLAGS_TUPLE_SUBCLASS + .. data:: Py_TPFLAGS_BYTES_SUBCLASS + .. data:: Py_TPFLAGS_UNICODE_SUBCLASS + .. data:: Py_TPFLAGS_DICT_SUBCLASS + .. data:: Py_TPFLAGS_BASE_EXC_SUBCLASS + .. data:: Py_TPFLAGS_TYPE_SUBCLASS + + These flags are used by functions such as + :c:func:`PyLong_Check` to quickly determine if a type is a subclass + of a built-in type; such specific checks are faster than a generic + check, like :c:func:`PyObject_IsInstance`. Custom types that inherit + from built-ins should have their :c:member:`~PyTypeObject.tp_flags` + set appropriately, or the code that interacts with such types + will behave differently depending on what kind of check is used. + + + .. data:: Py_TPFLAGS_HAVE_FINALIZE + + This bit is set when the :c:member:`~PyTypeObject.tp_finalize` slot is present in the + type structure. + + .. versionadded:: 3.4 + + .. c:member:: char* PyTypeObject.tp_doc An optional pointer to a NUL-terminated C string giving the docstring for this @@ -967,6 +993,47 @@ type objects) *must* have the :attr:`ob_size` field. This field is not inherited; it is calculated fresh by :c:func:`PyType_Ready`. +.. c:member:: destructor PyTypeObject.tp_finalize + + An optional pointer to an instance finalization function. Its signature is + :c:type:`destructor`:: + + void tp_finalize(PyObject *) + + If :c:member:`~PyTypeObject.tp_finalize` is set, the interpreter calls it once when + finalizing an instance. It is called either from the garbage + collector (if the instance is part of an isolated reference cycle) or + just before the object is deallocated. Either way, it is guaranteed + to be called before attempting to break reference cycles, ensuring + that it finds the object in a sane state. + + :c:member:`~PyTypeObject.tp_finalize` should not mutate the current exception status; + therefore, a recommended way to write a non-trivial finalizer is:: + + static void + local_finalize(PyObject *self) + { + PyObject *error_type, *error_value, *error_traceback; + + /* Save the current exception, if any. */ + PyErr_Fetch(&error_type, &error_value, &error_traceback); + + /* ... */ + + /* Restore the saved exception. */ + PyErr_Restore(error_type, error_value, error_traceback); + } + + For this field to be taken into account (even through inheritance), + you must also set the :const:`Py_TPFLAGS_HAVE_FINALIZE` flags bit. + + This field is inherited by subtypes. + + .. versionadded:: 3.4 + + .. seealso:: "Safe object finalization" (:pep:`442`) + + .. c:member:: PyObject* PyTypeObject.tp_cache Unused. Not inherited. Internal use only. @@ -1150,7 +1217,8 @@ Sequence Object Structures This function is used by :c:func:`PySequence_Repeat` and has the same signature. It is also used by the ``*`` operator, after trying numeric - multiplication via the :c:member:`~PyTypeObject.tp_as_number.nb_mul` slot. + multiplication via the :c:member:`~PyTypeObject.tp_as_number.nb_multiply` + slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_item diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 3649cfb..0f0419c 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -526,12 +526,23 @@ APIs: The `"%lld"` and `"%llu"` format specifiers are only available when :const:`HAVE_LONG_LONG` is defined. + .. note:: + The width formatter unit is number of characters rather than bytes. + The precision formatter unit is number of bytes for ``"%s"`` and + ``"%V"`` (if the ``PyObject*`` argument is NULL), and a number of + characters for ``"%A"``, ``"%U"``, ``"%S"``, ``"%R"`` and ``"%V"`` + (if the ``PyObject*`` argument is not NULL). + .. versionchanged:: 3.2 Support for ``"%lld"`` and ``"%llu"`` added. .. versionchanged:: 3.3 Support for ``"%li"``, ``"%lli"`` and ``"%zi"`` added. + .. versionchanged:: 3.4 + Support width and precision formatter for ``"%s"``, ``"%A"``, ``"%U"``, + ``"%V"``, ``"%S"``, ``"%R"`` added. + .. c:function:: PyObject* PyUnicode_FromFormatV(const char *format, va_list vargs) @@ -1613,7 +1624,7 @@ They all return *NULL* or ``-1`` if an exception occurs. Compare a unicode object, *uni*, with *string* and return -1, 0, 1 for less than, equal, and greater than, respectively. It is best to pass only ASCII-encoded strings, but the function interprets the input string as - ISO-8859-1 if it contains non-ASCII characters". + ISO-8859-1 if it contains non-ASCII characters. .. c:function:: PyObject* PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 14ef8df..f7ed4c7 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -144,6 +144,37 @@ the same library that the Python runtime is using. (:func:`sys.getfilesystemencoding`). Returns ``0`` at EOF. +.. c:var:: int (*PyOS_InputHook)(void) + + Can be set to point to a function with the prototype + ``int func(void)``. The function will be called when Python's + interpreter prompt is about to become idle and wait for user input + from the terminal. The return value is ignored. Overriding this + hook can be used to integrate the interpreter's prompt with other + event loops, as done in the :file:`Modules/_tkinter.c` in the + Python source code. + + +.. c:var:: char* (*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *) + + Can be set to point to a function with the prototype + ``char *func(FILE *stdin, FILE *stdout, char *prompt)``, + overriding the default function used to read a single line of input + at the interpreter's prompt. The function is expected to output + the string *prompt* if it's not *NULL*, and then read a line of + input from the provided standard input file, returning the + resulting string. For example, The :mod:`readline` module sets + this hook to provide line-editing and tab-completion features. + + The result must be a string allocated by :c:func:`PyMem_RawMalloc` or + :c:func:`PyMem_RawRealloc`, or *NULL* if an error occurred. + + .. versionchanged:: 3.4 + The result must be allocated by :c:func:`PyMem_RawMalloc` or + :c:func:`PyMem_RawRealloc`, instead of being allocated by + :c:func:`PyMem_Malloc` or :c:func:`PyMem_Realloc`. + + .. c:function:: struct _node* PyParser_SimpleParseString(const char *str, int start) This is a simplified interface to @@ -235,16 +266,15 @@ the same library that the Python runtime is using. *optimize* set to ``-1``. -.. c:function:: PyObject* Py_CompileStringExFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags, int optimize) +.. c:function:: PyObject* Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) Parse and compile the Python source code in *str*, returning the resulting code object. The start token is given by *start*; this can be used to constrain the code which can be compiled and should be :const:`Py_eval_input`, :const:`Py_file_input`, or :const:`Py_single_input`. The filename specified by *filename* is used to construct the code object and may appear in tracebacks or - :exc:`SyntaxError` exception messages, it is decoded from the filesystem - encoding (:func:`sys.getfilesystemencoding`). This returns *NULL* if the - code cannot be parsed or compiled. + :exc:`SyntaxError` exception messages. This returns *NULL* if the code + cannot be parsed or compiled. The integer *optimize* specifies the optimization level of the compiler; a value of ``-1`` selects the optimization level of the interpreter as given by @@ -252,8 +282,15 @@ the same library that the Python runtime is using. ``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false) or ``2`` (docstrings are removed too). - .. versionadded:: 3.2 + .. versionadded:: 3.4 + + +.. c:function:: PyObject* Py_CompileStringExFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags, int optimize) + Like :c:func:`Py_CompileStringExFlags`, but *filename* is a byte string + decoded from the filesystem encoding (:func:`os.fsdecode`). + + .. versionadded:: 3.2 .. c:function:: PyObject* PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals) @@ -285,6 +322,10 @@ the same library that the Python runtime is using. it causes an exception to immediately be thrown; this is used for the :meth:`~generator.throw` methods of generator objects. + .. versionchanged:: 3.4 + This function now includes a debug assertion to help ensure that it + does not silently discard an active exception. + .. c:function:: int PyEval_MergeCompilerFlags(PyCompilerFlags *cf) @@ -338,4 +379,3 @@ the same library that the Python runtime is using. This bit can be set in *flags* to cause division operator ``/`` to be interpreted as "true division" according to :pep:`238`. - diff --git a/Doc/conf.py b/Doc/conf.py index 5b63cad..c0ed8c6 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -61,6 +61,8 @@ add_module_names = True # By default, highlight as Python 3. highlight_language = 'python3' +needs_sphinx = '1.1' + # Options for HTML output # ----------------------- @@ -118,11 +120,11 @@ _stdauthor = r'Guido van Rossum\\Fred L. Drake, Jr., editor' latex_documents = [ ('c-api/index', 'c-api.tex', 'The Python/C API', _stdauthor, 'manual'), - ('distutils/index', 'distutils.tex', + ('distributing/index', 'distributing.tex', 'Distributing Python Modules', _stdauthor, 'manual'), ('extending/index', 'extending.tex', 'Extending and Embedding Python', _stdauthor, 'manual'), - ('install/index', 'install.tex', + ('installing/index', 'installing.tex', 'Installing Python Modules', _stdauthor, 'manual'), ('library/index', 'library.tex', 'The Python Library Reference', _stdauthor, 'manual'), diff --git a/Doc/contents.rst b/Doc/contents.rst index c0c6af3..29b07db 100644 --- a/Doc/contents.rst +++ b/Doc/contents.rst @@ -11,8 +11,8 @@ library/index.rst extending/index.rst c-api/index.rst - distutils/index.rst - install/index.rst + distributing/index.rst + installing/index.rst howto/index.rst faq/index.rst glossary.rst @@ -21,3 +21,11 @@ bugs.rst copyright.rst license.rst + +.. include legacy packaging docs in build + +.. toctree:: + :hidden: + + distutils/index.rst + install/index.rst diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index a42584c..6025617 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -29,7 +29,7 @@ # reference to the item argument! # The parameter names are as they appear in the API manual, not the source -# code. +# code. PyBool_FromLong:PyObject*::+1: PyBool_FromLong:long:v:0: @@ -220,6 +220,11 @@ PyDict_GetItemString:PyObject*::0: PyDict_GetItemString:PyObject*:p:0: PyDict_GetItemString:const char*:key:: +PyDict_SetDefault:PyObject*::0: +PyDict_SetDefault:PyObject*:p:0: +PyDict_SetDefault:PyObject*:key:0:conditionally +1 if inserted into the dict +PyDict_SetDefault:PyObject*:default:0:conditionally +1 if inserted into the dict + PyDict_Items:PyObject*::+1: PyDict_Items:PyObject*:p:0: @@ -503,13 +508,13 @@ PyImport_AddModule:const char*:name:: PyImport_Cleanup:void::: PyImport_ExecCodeModule:PyObject*::+1: -PyImport_ExecCodeModule:char*:name:: +PyImport_ExecCodeModule:const char*:name:: PyImport_ExecCodeModule:PyObject*:co:0: PyImport_ExecCodeModuleEx:PyObject*::+1: -PyImport_ExecCodeModuleEx:char*:name:: +PyImport_ExecCodeModuleEx:const char*:name:: PyImport_ExecCodeModuleEx:PyObject*:co:0: -PyImport_ExecCodeModuleEx:char*:pathname:: +PyImport_ExecCodeModuleEx:const char*:pathname:: PyImport_GetMagicNumber:long::: @@ -519,7 +524,7 @@ PyImport_Import:PyObject*::+1: PyImport_Import:PyObject*:name:0: PyImport_ImportFrozenModule:int::: -PyImport_ImportFrozenModule:char*::: +PyImport_ImportFrozenModule:const char*::: PyImport_ImportModule:PyObject*::+1: PyImport_ImportModule:const char*:name:: @@ -668,7 +673,7 @@ PyLong_FromUnsignedLongLong:PyObject*::+1: PyLong_FromUnsignedLongLong:unsigned long long:v:: PyLong_FromString:PyObject*::+1: -PyLong_FromString:char*:str:: +PyLong_FromString:const char*:str:: PyLong_FromString:char**:pend:: PyLong_FromString:int:base:: @@ -696,7 +701,7 @@ PyMapping_DelItemString:const char*:key:: PyMapping_GetItemString:PyObject*::+1: PyMapping_GetItemString:PyObject*:o:0: -PyMapping_GetItemString:char*:key:: +PyMapping_GetItemString:const char*:key:: PyMapping_HasKey:int::: PyMapping_HasKey:PyObject*:o:0: @@ -704,7 +709,7 @@ PyMapping_HasKey:PyObject*:key:: PyMapping_HasKeyString:int::: PyMapping_HasKeyString:PyObject*:o:0: -PyMapping_HasKeyString:char*:key:: +PyMapping_HasKeyString:const char*:key:: PyMapping_Items:PyObject*::+1: PyMapping_Items:PyObject*:o:0: @@ -717,7 +722,7 @@ PyMapping_Length:PyObject*:o:0: PyMapping_SetItemString:int::: PyMapping_SetItemString:PyObject*:o:0: -PyMapping_SetItemString:char*:key:: +PyMapping_SetItemString:const char*:key:: PyMapping_SetItemString:PyObject*:v:+1: PyMapping_Values:PyObject*::+1: @@ -730,7 +735,7 @@ PyMarshal_ReadObjectFromFile:PyObject*::+1: PyMarshal_ReadObjectFromFile:FILE*:file:: PyMarshal_ReadObjectFromString:PyObject*::+1: -PyMarshal_ReadObjectFromString:char*:string:: +PyMarshal_ReadObjectFromString:const char*:string:: PyMarshal_ReadObjectFromString:int:len:: PyMarshal_WriteObjectToString:PyObject*::+1: @@ -902,7 +907,7 @@ PyNumber_Xor:PyObject*::+1: PyNumber_Xor:PyObject*:o1:0: PyNumber_Xor:PyObject*:o2:0: -PyObject_AsFileDescriptor:int::: +PyObject_AsFileDescriptor:int::: PyObject_AsFileDescriptor:PyObject*:o:0: PyObject_Call:PyObject*::+1: @@ -912,7 +917,7 @@ PyObject_Call:PyObject*:kw:0: PyObject_CallFunction:PyObject*::+1: PyObject_CallFunction:PyObject*:callable_object:0: -PyObject_CallFunction:char*:format:: +PyObject_CallFunction:const char*:format:: PyObject_CallFunction::...:: PyObject_CallFunctionObjArgs:PyObject*::+1: @@ -921,8 +926,8 @@ PyObject_CallFunctionObjArgs::...:: PyObject_CallMethod:PyObject*::+1: PyObject_CallMethod:PyObject*:o:0: -PyObject_CallMethod:char*:m:: -PyObject_CallMethod:char*:format:: +PyObject_CallMethod:const char*:m:: +PyObject_CallMethod:const char*:format:: PyObject_CallMethod::...:: PyObject_CallMethodObjArgs:PyObject*::+1: diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst new file mode 100644 index 0000000..4bcb633 --- /dev/null +++ b/Doc/distributing/index.rst @@ -0,0 +1,159 @@ +.. _distributing-index: + +############################### + Distributing Python Modules +############################### + +:Email: distutils-sig@python.org + + +As a popular open source development project, Python has an active +supporting community of contributors and users that also make their software +available for other Python developers to use under open source license terms. + +This allows Python users to share and collaborate effectively, benefiting +from the solutions others have already created to common (and sometimes +even rare!) problems, as well as potentially contributing their own +solutions to the common pool. + +This guide covers the distribution part of the process. For a guide to +installing other Python projects, refer to the +:ref:`installation guide <installing-index>`. + +.. note:: + + For corporate and other institutional users, be aware that many + organisations have their own policies around using and contributing to + open source software. Please take such policies into account when making + use of the distribution and installation tools provided with Python. + + +Key terms +========= + +* the `Python Package Index <https://pypi.python.org/pypi>`__ is a public + repository of open source licensed packages made available for use by + other Python users +* the `Python Packaging Authority + <http://packaging.python.org/en/latest/future.html>`__ are the group of + developers and documentation authors responsible for the maintenance and + evolution of the standard packaging tools and the associated metadata and + file format standards. They maintain a variety of tools, documentation + and issue trackers on both `GitHub <https://github.com/pypa>`__ and + `BitBucket <https://bitbucket.org/pypa/>`__. +* ``distutils`` is the original build and distribution system first added to + the Python standard library in 1998. While direct use of ``distutils`` is + being phased out, it still laid the foundation for the current packaging + and distribution infrastructure, and it not only remains part of the + standard library, but its name lives on in other ways (such as the name + of the mailing list used to coordinate Python packaging standards + development). +* ``setuptools`` is a (largely) drop-in replacement for ``distutils`` first + published in 2004. Its most notable addition over the unmodified + ``distutils`` tools was the ability to declare dependencies on other + packages. It is currently recommended as a more regularly updated + alternative to ``distutils`` that offers consistent support for more + recent packaging standards across a wide range of Python versions. +* ``wheel`` (in this context) is a project that adds the ``bdist_wheel`` + command to ``distutils``/``setuptools``. This produces a cross platform + binary packaging format (called "wheels" or "wheel files" and defined in + :pep:`427`) that allows Python libraries, even those including binary + extensions, to be installed on a system without needing to be built + locally. + + +Open source licensing and collaboration +======================================= + +In most parts of the world, software is automatically covered by copyright. +This means that other developers require explicit permission to copy, use, +modify and redistribute the software. + +Open source licensing is a way of explicitly granting such permission in a +relatively consistent way, allowing developers to share and collaborate +efficiently by making common solutions to various problems freely available. +This leaves many developers free to spend more time focusing on the problems +that are relatively unique to their specific situation. + +The distribution tools provided with Python are designed to make it +reasonably straightforward for developers to make their own contributions +back to that common pool of software if they choose to do so. + +The same distribution tools can also be used to distribute software within +an organisation, regardless of whether that software is published as open +source software or not. + + +Installing the tools +==================== + +The standard library does not include build tools that support modern +Python packaging standards, as the core development team has found that it +is important to have standard tools that work consistently, even on older +versions of Python. + +The currently recommended build and distribution tools can be installed +using ``pip``:: + + pip install setuptools wheel twine + +The Python Packaging User Guide includes more details on the `currently +recommended tools`_. + +.. _currently recommended tools: https://packaging.python.org/en/latest/current.html#packaging-tool-recommendations + +Reading the guide +================= + +The Python Packaging User Guide covers the various key steps and elements +involved in creating a project: + +* `Project structure`_ +* `Building and packaging the project`_ +* `Uploading the project to the Python Package Index`_ + +.. _Project structure: \ + http://packaging.python.org/en/latest/tutorial.html#creating-your-own-project +.. _Building and packaging the project: \ + http://packaging.python.org/en/latest/tutorial.html#building-packaging-your-project +.. _Uploading the project to the Python Package Index: \ + http://packaging.python.org/en/latest/tutorial.html#uploading-your-project-to-pypi + + +How do I...? +============ + +These are quick answers or links for some common tasks. + +... choose a name for my project? +--------------------------------- + +This isn't an easy topic, but here are a few tips: + +* check the Python Package Index to see if the name is already in use +* check popular hosting sites like GitHub, BitBucket, etc to see if there + is already a project with that name +* check what comes up in a web search for the name you're considering +* avoid particularly common words, especially ones with multiple meanings, + as they can make it difficult for users to find your software when + searching for it + + +... create and distribute binary extensions? +-------------------------------------------- + +This is actually quite a complex topic, with a variety of alternatives +available depending on exactly what you're aiming to achieve. See the +Python Packaging User Guide for more information and recommendations. + +.. seealso:: + + `Python Packaging User Guide: Binary Extensions + <http://packaging.python.org/en/latest/extensions.html>`__ + +.. other topics: + + Once the Development & Deployment part of PPUG is fleshed out, some of + those sections should be linked from new questions here (most notably, + we should have a question about avoiding depending on PyPI that links to + http://packaging.python.org/en/latest/deployment.html#pypi-mirrors-and-caches) diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst index 9853f02..e1357fa 100644 --- a/Doc/distutils/apiref.rst +++ b/Doc/distutils/apiref.rst @@ -853,17 +853,6 @@ Windows. It also contains the Mingw32CCompiler class which handles the mingw32 port of GCC (same as cygwin in no-cygwin mode). -:mod:`distutils.emxccompiler` --- OS/2 EMX Compiler -=================================================== - -.. module:: distutils.emxccompiler - :synopsis: OS/2 EMX Compiler support - - -This module provides the EMXCCompiler class, a subclass of -:class:`UnixCCompiler` that handles the EMX port of the GNU C compiler to OS/2. - - :mod:`distutils.archive_util` --- Archiving utilities ====================================================== @@ -1004,7 +993,7 @@ directories. Files in *src* that begin with :file:`.nfs` are skipped (more information on these files is available in answer D2 of the `NFS FAQ page - <http://nfs.sourceforge.net/#section_d>`_. + <http://nfs.sourceforge.net/#section_d>`_). .. versionchanged:: 3.3.1 NFS files are ignored. diff --git a/Doc/distutils/builtdist.rst b/Doc/distutils/builtdist.rst index d1ab7db..83c68ae 100644 --- a/Doc/distutils/builtdist.rst +++ b/Doc/distutils/builtdist.rst @@ -239,7 +239,8 @@ tedious and error-prone, so it's usually best to put them in the setup configuration file, :file:`setup.cfg`\ ---see section :ref:`setup-config`. If you distribute or package many Python module distributions, you might want to put options that apply to all of them in your personal Distutils configuration -file (:file:`~/.pydistutils.cfg`). +file (:file:`~/.pydistutils.cfg`). If you want to temporarily disable +this file, you can pass the :option:`--no-user-cfg` option to :file:`setup.py`. There are three steps to building a binary RPM package, all of which are handled automatically by the Distutils: diff --git a/Doc/distutils/examples.rst b/Doc/distutils/examples.rst index 5eb654a..872ecd9 100644 --- a/Doc/distutils/examples.rst +++ b/Doc/distutils/examples.rst @@ -193,9 +193,6 @@ then the corresponding setup script would be :: packages=['foobar', 'foobar.subfoo'], ) -(Again, the empty string in :option:`package_dir` stands for the current -directory.) - .. _single-ext: diff --git a/Doc/distutils/index.rst b/Doc/distutils/index.rst index 1a6f04c..90d1c1a 100644 --- a/Doc/distutils/index.rst +++ b/Doc/distutils/index.rst @@ -1,8 +1,8 @@ .. _distutils-index: -############################### - Distributing Python Modules -############################### +############################################## + Distributing Python Modules (Legacy version) +############################################## :Authors: Greg Ward, Anthony Baxter :Email: distutils-sig@python.org diff --git a/Doc/distutils/sourcedist.rst b/Doc/distutils/sourcedist.rst index 1666436..427b7b1 100644 --- a/Doc/distutils/sourcedist.rst +++ b/Doc/distutils/sourcedist.rst @@ -26,16 +26,16 @@ to create a gzipped tarball and a zip file. The available formats are: +===========+=========================+=========+ | ``zip`` | zip file (:file:`.zip`) | (1),(3) | +-----------+-------------------------+---------+ -| ``gztar`` | gzip'ed tar file | (2),(4) | +| ``gztar`` | gzip'ed tar file | \(2) | | | (:file:`.tar.gz`) | | +-----------+-------------------------+---------+ -| ``bztar`` | bzip2'ed tar file | \(4) | +| ``bztar`` | bzip2'ed tar file | | | | (:file:`.tar.bz2`) | | +-----------+-------------------------+---------+ | ``ztar`` | compressed tar file | \(4) | | | (:file:`.tar.Z`) | | +-----------+-------------------------+---------+ -| ``tar`` | tar file (:file:`.tar`) | \(4) | +| ``tar`` | tar file (:file:`.tar`) | | +-----------+-------------------------+---------+ Notes: @@ -51,8 +51,16 @@ Notes: of the standard Python library since Python 1.6) (4) - requires external utilities: :program:`tar` and possibly one of :program:`gzip`, - :program:`bzip2`, or :program:`compress` + requires the :program:`compress` program. Notice that this format is now + pending for deprecation and will be removed in the future versions of Python. + +When using any ``tar`` format (``gztar``, ``bztar``, ``ztar`` or +``tar``), under Unix you can specify the ``owner`` and ``group`` names +that will be set for each member of the archive. + +For example, if you want all files of the archive to be owned by root:: + + python setup.py sdist --owner=root --group=root .. _manifest: @@ -68,7 +76,7 @@ source distribution: :option:`packages` options * all C source files mentioned in the :option:`ext_modules` or - :option:`libraries` options ( + :option:`libraries` options .. XXX getting C library sources currently broken---no :meth:`get_source_files` method in :file:`build_clib.py`! diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index bd66086..6cb686a 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -285,14 +285,14 @@ be directly useful to you: * ``pythonX.Y-config --cflags`` will give you the recommended flags when compiling:: - $ /opt/bin/python3.3-config --cflags - -I/opt/include/python3.3m -I/opt/include/python3.3m -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes + $ /opt/bin/python3.4-config --cflags + -I/opt/include/python3.4m -I/opt/include/python3.4m -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes * ``pythonX.Y-config --ldflags`` will give you the recommended flags when linking:: - $ /opt/bin/python3.3-config --ldflags - -L/opt/lib/python3.3/config-3.3m -lpthread -ldl -lutil -lm -lpython3.3m -Xlinker -export-dynamic + $ /opt/bin/python3.4-config --ldflags + -L/opt/lib/python3.4/config-3.4m -lpthread -ldl -lutil -lm -lpython3.4m -Xlinker -export-dynamic .. note:: To avoid confusion between several Python installations (and especially diff --git a/Doc/extending/index.rst b/Doc/extending/index.rst index 44a7f92..dd43926 100644 --- a/Doc/extending/index.rst +++ b/Doc/extending/index.rst @@ -21,14 +21,31 @@ Python) that give the language its wide application range. For a detailed description of the whole Python/C API, see the separate :ref:`c-api-index`. -.. note:: - This guide only covers the basic tools for creating extensions provided - as part of this version of CPython. Third party tools may offer simpler - alternatives. Refer to the `binary extensions section - <https://python-packaging-user-guide.readthedocs.org/en/latest/extensions.html>`__ - in the Python Packaging User Guide for more information. +Recommended third party tools +============================= +This guide only covers the basic tools for creating extensions provided +as part of this version of CPython. Third party tools like Cython, +``cffi``, SWIG and Numba offer both simpler and more sophisticated +approaches to creating C and C++ extensions for Python. + +.. seealso:: + + `Python Packaging User Guide: Binary Extensions <https://packaging.python.org/en/latest/extensions.html>`_ + The Python Packaging User Guide not only covers several available + tools that simplify the creation of binary extensions, but also + discusses the various reasons why creating an extension module may be + desirable in the first place. + + +Creating extensions without third party tools +============================================= + +This section of the guide covers creating C and C++ extensions without +assistance from third party tools. It is intended primarily for creators +of those tools, rather than being a recommended way to create your own +C extensions. .. toctree:: :maxdepth: 2 @@ -38,4 +55,17 @@ For a detailed description of the whole Python/C API, see the separate newtypes.rst building.rst windows.rst + +Embedding the CPython runtime in a larger application +===================================================== + +Sometimes, rather than creating an extension that runs inside the Python +interpreter as the main application, it is desirable to instead embed +the CPython runtime inside a larger application. This section covers +some of the details involved in doing that successfully. + +.. toctree:: + :maxdepth: 2 + :numbered: + embedding.rst diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index f484ba4..45b5721 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -157,7 +157,8 @@ to :const:`Py_TPFLAGS_DEFAULT`. :: Py_TPFLAGS_DEFAULT, /* tp_flags */ All types should include this constant in their flags. It enables all of the -members defined by the current version of Python. +members defined until at least Python 3.3. If you need further members, +you will need to OR the corresponding flags. We provide a doc string for the type in :c:member:`~PyTypeObject.tp_doc`. :: @@ -928,8 +929,9 @@ Finalization and De-allocation This function is called when the reference count of the instance of your type is reduced to zero and the Python interpreter wants to reclaim it. If your type -has memory to free or other clean-up to perform, put it here. The object itself -needs to be freed here as well. Here is an example of this function:: +has memory to free or other clean-up to perform, you can put it here. The +object itself needs to be freed here as well. Here is an example of this +function:: static void newdatatype_dealloc(newdatatypeobject * obj) @@ -980,6 +982,22 @@ done. This can be done using the :c:func:`PyErr_Fetch` and Py_TYPE(obj)->tp_free((PyObject*)self); } +.. note:: + There are limitations to what you can safely do in a deallocator function. + First, if your type supports garbage collection (using :c:member:`~PyTypeObject.tp_traverse` + and/or :c:member:`~PyTypeObject.tp_clear`), some of the object's members can have been + cleared or finalized by the time :c:member:`~PyTypeObject.tp_dealloc` is called. Second, in + :c:member:`~PyTypeObject.tp_dealloc`, your object is in an unstable state: its reference + count is equal to zero. Any call to a non-trivial object or API (as in the + example above) might end up calling :c:member:`~PyTypeObject.tp_dealloc` again, causing a + double free and a crash. + + Starting with Python 3.4, it is recommended not to put any complex + finalization code in :c:member:`~PyTypeObject.tp_dealloc`, and instead use the new + :c:member:`~PyTypeObject.tp_finalize` type method. + + .. seealso:: + :pep:`442` explains the new finalization scheme. .. index:: single: string; object representation diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index a9a234b..d196e86 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -95,8 +95,8 @@ To test the type of an object, first make sure it isn't *NULL*, and then use There is also a high-level API to Python objects which is provided by the so-called 'abstract' interface -- read ``Include/abstract.h`` for further details. It allows interfacing with any kind of Python sequence using calls -like :c:func:`PySequence_Length`, :c:func:`PySequence_GetItem`, etc.) as well -as many other useful protocols such as numbers (:c:func:`PyNumber_Index` et. +like :c:func:`PySequence_Length`, :c:func:`PySequence_GetItem`, etc. as well +as many other useful protocols such as numbers (:c:func:`PyNumber_Index` et al.) and mappings in the PyMapping APIs. diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst index 5827f28..3192163 100644 --- a/Doc/faq/gui.rst +++ b/Doc/faq/gui.rst @@ -32,14 +32,14 @@ install (since it comes included with most `binary distributions <http://www.python.org/download/>`_ of Python) and use. For more info about Tk, including pointers to the source, see the `Tcl/Tk home page <http://www.tcl.tk>`_. Tcl/Tk is fully portable to the -MacOS, Windows, and Unix platforms. +Mac OS X, Windows, and Unix platforms. wxWidgets --------- wxWidgets (http://www.wxwidgets.org) is a free, portable GUI class library written in C++ that provides a native look and feel on a -number of platforms, with Windows, MacOS X, GTK, X11, all listed as +number of platforms, with Windows, Mac OS X, GTK, X11, all listed as current stable targets. Language bindings are available for a number of languages including Python, Perl, Ruby, etc. @@ -102,13 +102,9 @@ For OpenGL bindings, see `PyOpenGL <http://pyopengl.sourceforge.net>`_. What platform-specific GUI toolkits exist for Python? ======================================================== -`The Mac port <http://python.org/download/mac>`_ by Jack Jansen has a rich and -ever-growing set of modules that support the native Mac toolbox calls. The port -supports MacOS X's Carbon libraries. - By installing the `PyObjc Objective-C bridge -<http://pyobjc.sourceforge.net>`_, Python programs can use MacOS X's -Cocoa libraries. See the documentation that comes with the Mac port. +<http://pyobjc.sourceforge.net>`_, Python programs can use Mac OS X's +Cocoa libraries. :ref:`Pythonwin <windows-faq>` by Mark Hammond includes an interface to the Microsoft Foundation Classes and a Python programming environment diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index 2566932..5f4ff17 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -211,7 +211,7 @@ using curses, but curses is a fairly large module to learn. try: c = sys.stdin.read(1) print("Got character", repr(c)) - except IOError: + except OSError: pass finally: termios.tcsetattr(fd, termios.TCSAFLUSH, oldterm) @@ -224,7 +224,11 @@ using curses, but curses is a fairly large module to learn. :func:`termios.tcsetattr` turns off stdin's echoing and disables canonical mode. :func:`fcntl.fnctl` is used to obtain stdin's file descriptor flags and modify them for non-blocking mode. Since reading stdin when it is empty - results in an :exc:`IOError`, this error is caught and ignored. + results in an :exc:`OSError`, this error is caught and ignored. + + .. versionchanged:: 3.3 + *sys.stdin.read* used to raise :exc:`IOError`. Starting from Python 3.3 + :exc:`IOError` is alias for :exc:`OSError`. Threads diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 280d5e1..8dcaa3a 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -711,7 +711,7 @@ By default, these interpret the number as decimal, so that ``int('0144') == 144`` and ``int('0x144')`` raises :exc:`ValueError`. ``int(string, base)`` takes the base to convert from as a second optional argument, so ``int('0x144', 16) == 324``. If the base is specified as 0, the number is interpreted using Python's -rules: a leading '0' indicates octal, and '0x' indicates a hex number. +rules: a leading '0o' indicates octal, and '0x' indicates a hex number. Do not use the built-in function :func:`eval` if all you need is to convert strings to numbers. :func:`eval` will be significantly slower and it presents a @@ -732,7 +732,7 @@ To convert, e.g., the number 144 to the string '144', use the built-in type constructor :func:`str`. If you want a hexadecimal or octal representation, use the built-in functions :func:`hex` or :func:`oct`. For fancy formatting, see the :ref:`string-formatting` section, e.g. ``"{:04d}".format(144)`` yields -``'0144'`` and ``"{:.3f}".format(1/3)`` yields ``'0.333'``. +``'0144'`` and ``"{:.3f}".format(1.0/3.0)`` yields ``'0.333'``. How do I modify a string in place? @@ -1751,12 +1751,12 @@ When I edit an imported module and reimport it, the changes don't show up. Why For reasons of efficiency as well as consistency, Python only reads the module file on the first time a module is imported. If it didn't, in a program consisting of many modules where each one imports the same basic module, the -basic module would be parsed and re-parsed many times. To force rereading of a +basic module would be parsed and re-parsed many times. To force re-reading of a changed module, do this:: - import imp + import importlib import modname - imp.reload(modname) + importlib.reload(modname) Warning: this technique is not 100% fool-proof. In particular, modules containing statements like :: @@ -1768,10 +1768,10 @@ module contains class definitions, existing class instances will *not* be updated to use the new class definition. This can result in the following paradoxical behaviour: - >>> import imp + >>> import importlib >>> import cls >>> c = cls.C() # Create an instance of C - >>> imp.reload(cls) + >>> importlib.reload(cls) <module 'cls' from 'cls.py'> >>> isinstance(c, cls.C) # isinstance is false?!? False diff --git a/Doc/glossary.rst b/Doc/glossary.rst index b48eb1e..f71a1f7 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -310,6 +310,15 @@ Glossary >>> sum(i*i for i in range(10)) # sum of squares 0, 1, 4, ... 81 285 + generic function + A function composed of multiple functions implementing the same operation + for different types. Which implementation should be used during a call is + determined by the dispatch algorithm. + + See also the :term:`single dispatch` glossary entry, the + :func:`functools.singledispatch` decorator, and :pep:`443`. + + GIL See :term:`global interpreter lock`. @@ -530,6 +539,10 @@ Glossary See also :term:`package`. + module spec + A namespace containing the import-related information used to load a + module. + MRO See :term:`method resolution order`. @@ -671,20 +684,27 @@ Glossary positional argument See :term:`argument`. - provisional package - A provisional package is one which has been deliberately excluded from + provisional API + A provisional API is one which has been deliberately excluded from the standard library's backwards compatibility guarantees. While major - changes to such packages are not expected, as long as they are marked + changes to such interfaces are not expected, as long as they are marked provisional, backwards incompatible changes (up to and including removal - of the package) may occur if deemed necessary by core developers. Such + of the interface) may occur if deemed necessary by core developers. Such changes will not be made gratuitously -- they will occur only if serious - flaws are uncovered that were missed prior to the inclusion of the - package. + fundamental flaws are uncovered that were missed prior to the inclusion + of the API. + + Even for provisional APIs, backwards incompatible changes are seen as + a "solution of last resort" - every attempt will still be made to find + a backwards compatible resolution to any identified problems. This process allows the standard library to continue to evolve over time, without locking in problematic design errors for extended periods of time. See :pep:`411` for more details. + provisional package + See :term:`provisional API`. + Python 3000 Nickname for the Python 3.x release line (coined long ago when the release of version 3 was something in the distant future.) This is also @@ -771,6 +791,10 @@ Glossary interface can be registered explicitly using :func:`~abc.register`. + single dispatch + A form of :term:`generic function` dispatch where the implementation is + chosen based on the type of a single argument. + slice An object usually containing a portion of a :term:`sequence`. A slice is created using the subscript notation, ``[]`` with colons between numbers diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst new file mode 100644 index 0000000..750ddbe --- /dev/null +++ b/Doc/howto/clinic.rst @@ -0,0 +1,1686 @@ +********************** +Argument Clinic How-To +********************** + +:author: Larry Hastings + + +.. topic:: Abstract + + Argument Clinic is a preprocessor for CPython C files. + Its purpose is to automate all the boilerplate involved + with writing argument parsing code for "builtins". + This document shows you how to convert your first C + function to work with Argument Clinic, and then introduces + some advanced topics on Argument Clinic usage. + + Currently Argument Clinic is considered internal-only + for CPython. Its use is not supported for files outside + CPython, and no guarantees are made regarding backwards + compatibility for future versions. In other words: if you + maintain an external C extension for CPython, you're welcome + to experiment with Argument Clinic in your own code. But the + version of Argument Clinic that ships with CPython 3.5 *could* + be totally incompatible and break all your code. + +The Goals Of Argument Clinic +============================ + +Argument Clinic's primary goal +is to take over responsibility for all argument parsing code +inside CPython. This means that, when you convert a function +to work with Argument Clinic, that function should no longer +do any of its own argument parsing--the code generated by +Argument Clinic should be a "black box" to you, where CPython +calls in at the top, and your code gets called at the bottom, +with ``PyObject *args`` (and maybe ``PyObject *kwargs``) +magically converted into the C variables and types you need. + +In order for Argument Clinic to accomplish its primary goal, +it must be easy to use. Currently, working with CPython's +argument parsing library is a chore, requiring maintaining +redundant information in a surprising number of places. +When you use Argument Clinic, you don't have to repeat yourself. + +Obviously, no one would want to use Argument Clinic unless +it's solving their problem--and without creating new problems of +its own. +So it's paramount that Argument Clinic generate correct code. +It'd be nice if the code was faster, too, but at the very least +it should not introduce a major speed regression. (Eventually Argument +Clinic *should* make a major speedup possible--we could +rewrite its code generator to produce tailor-made argument +parsing code, rather than calling the general-purpose CPython +argument parsing library. That would make for the fastest +argument parsing possible!) + +Additionally, Argument Clinic must be flexible enough to +work with any approach to argument parsing. Python has +some functions with some very strange parsing behaviors; +Argument Clinic's goal is to support all of them. + +Finally, the original motivation for Argument Clinic was +to provide introspection "signatures" for CPython builtins. +It used to be, the introspection query functions would throw +an exception if you passed in a builtin. With Argument +Clinic, that's a thing of the past! + +One idea you should keep in mind, as you work with +Argument Clinic: the more information you give it, the +better job it'll be able to do. +Argument Clinic is admittedly relatively simple right +now. But as it evolves it will get more sophisticated, +and it should be able to do many interesting and smart +things with all the information you give it. + + +Basic Concepts And Usage +======================== + +Argument Clinic ships with CPython; you'll find it in ``Tools/clinic/clinic.py``. +If you run that script, specifying a C file as an argument:: + + % python3 Tools/clinic/clinic.py foo.c + +Argument Clinic will scan over the file looking for lines that +look exactly like this:: + + /*[clinic input] + +When it finds one, it reads everything up to a line that looks +exactly like this:: + + [clinic start generated code]*/ + +Everything in between these two lines is input for Argument Clinic. +All of these lines, including the beginning and ending comment +lines, are collectively called an Argument Clinic "block". + +When Argument Clinic parses one of these blocks, it +generates output. This output is rewritten into the C file +immediately after the block, followed by a comment containing a checksum. +The Argument Clinic block now looks like this:: + + /*[clinic input] + ... clinic input goes here ... + [clinic start generated code]*/ + ... clinic output goes here ... + /*[clinic end generated code: checksum=...]*/ + +If you run Argument Clinic on the same file a second time, Argument Clinic +will discard the old output and write out the new output with a fresh checksum +line. However, if the input hasn't changed, the output won't change either. + +You should never modify the output portion of an Argument Clinic block. Instead, +change the input until it produces the output you want. (That's the purpose of the +checksum--to detect if someone changed the output, as these edits would be lost +the next time Argument Clinic writes out fresh output.) + +For the sake of clarity, here's the terminology we'll use with Argument Clinic: + +* The first line of the comment (``/*[clinic input]``) is the *start line*. +* The last line of the initial comment (``[clinic start generated code]*/``) is the *end line*. +* The last line (``/*[clinic end generated code: checksum=...]*/``) is the *checksum line*. +* In between the start line and the end line is the *input*. +* In between the end line and the checksum line is the *output*. +* All the text collectively, from the start line to the checksum line inclusively, + is the *block*. (A block that hasn't been successfully processed by Argument + Clinic yet doesn't have output or a checksum line, but it's still considered + a block.) + + +Converting Your First Function +============================== + +The best way to get a sense of how Argument Clinic works is to +convert a function to work with it. Here, then, are the bare +minimum steps you'd need to follow to convert a function to +work with Argument Clinic. Note that for code you plan to +check in to CPython, you really should take the conversion farther, +using some of the advanced concepts you'll see later on in +the document (like "return converters" and "self converters"). +But we'll keep it simple for this walkthrough so you can learn. + +Let's dive in! + +0. Make sure you're working with a freshly updated checkout + of the CPython trunk. + +1. Find a Python builtin that calls either :c:func:`PyArg_ParseTuple` + or :c:func:`PyArg_ParseTupleAndKeywords`, and hasn't been converted + to work with Argument Clinic yet. + For my example I'm using ``_pickle.Pickler.dump()``. + +2. If the call to the ``PyArg_Parse`` function uses any of the + following format units:: + + O& + O! + es + es# + et + et# + + or if it has multiple calls to :c:func:`PyArg_ParseTuple`, + you should choose a different function. Argument Clinic *does* + support all of these scenarios. But these are advanced + topics--let's do something simpler for your first function. + + Also, if the function has multiple calls to :c:func:`PyArg_ParseTuple` + or :c:func:`PyArg_ParseTupleAndKeywords` where it supports different + types for the same argument, or if the function uses something besides + PyArg_Parse functions to parse its arguments, it probably + isn't suitable for conversion to Argument Clinic. Argument Clinic + doesn't support generic functions or polymorphic parameters. + +3. Add the following boilerplate above the function, creating our block:: + + /*[clinic input] + [clinic start generated code]*/ + +4. Cut the docstring and paste it in between the ``[clinic]`` lines, + removing all the junk that makes it a properly quoted C string. + When you're done you should have just the text, based at the left + margin, with no line wider than 80 characters. + (Argument Clinic will preserve indents inside the docstring.) + + If the old docstring had a first line that looked like a function + signature, throw that line away. (The docstring doesn't need it + anymore--when you use ``help()`` on your builtin in the future, + the first line will be built automatically based on the function's + signature.) + + Sample:: + + /*[clinic input] + Write a pickled representation of obj to the open file. + [clinic start generated code]*/ + +5. If your docstring doesn't have a "summary" line, Argument Clinic will + complain. So let's make sure it has one. The "summary" line should + be a paragraph consisting of a single 80-column line + at the beginning of the docstring. + + (Our example docstring consists solely of a summary line, so the sample + code doesn't have to change for this step.) + +6. Above the docstring, enter the name of the function, followed + by a blank line. This should be the Python name of the function, + and should be the full dotted path + to the function--it should start with the name of the module, + include any sub-modules, and if the function is a method on + a class it should include the class name too. + + Sample:: + + /*[clinic input] + _pickle.Pickler.dump + + Write a pickled representation of obj to the open file. + [clinic start generated code]*/ + +7. If this is the first time that module or class has been used with Argument + Clinic in this C file, + you must declare the module and/or class. Proper Argument Clinic hygiene + prefers declaring these in a separate block somewhere near the + top of the C file, in the same way that include files and statics go at + the top. (In our sample code we'll just show the two blocks next to + each other.) + + The name of the class and module should be the same as the one + seen by Python. Check the name defined in the :c:type:`PyModuleDef` + or :c:type:`PyTypeObject` as appropriate. + + When you declare a class, you must also specify two aspects of its type + in C: the type declaration you'd use for a pointer to an instance of + this class, and a pointer to the :c:type:`PyTypeObject` for this class. + + Sample:: + + /*[clinic input] + module _pickle + class _pickle.Pickler "PicklerObject *" "&Pickler_Type" + [clinic start generated code]*/ + + /*[clinic input] + _pickle.Pickler.dump + + Write a pickled representation of obj to the open file. + [clinic start generated code]*/ + + + + +8. Declare each of the parameters to the function. Each parameter + should get its own line. All the parameter lines should be + indented from the function name and the docstring. + + The general form of these parameter lines is as follows:: + + name_of_parameter: converter + + If the parameter has a default value, add that after the + converter:: + + name_of_parameter: converter = default_value + + Argument Clinic's support for "default values" is quite sophisticated; + please see :ref:`the section below on default values <default_values>` + for more information. + + Add a blank line below the parameters. + + What's a "converter"? It establishes both the type + of the variable used in C, and the method to convert the Python + value into a C value at runtime. + For now you're going to use what's called a "legacy converter"--a + convenience syntax intended to make porting old code into Argument + Clinic easier. + + For each parameter, copy the "format unit" for that + parameter from the ``PyArg_Parse()`` format argument and + specify *that* as its converter, as a quoted + string. ("format unit" is the formal name for the one-to-three + character substring of the ``format`` parameter that tells + the argument parsing function what the type of the variable + is and how to convert it. For more on format units please + see :ref:`arg-parsing`.) + + For multicharacter format units like ``z#``, use the + entire two-or-three character string. + + Sample:: + + /*[clinic input] + module _pickle + class _pickle.Pickler "PicklerObject *" "&Pickler_Type" + [clinic start generated code]*/ + + /*[clinic input] + _pickle.Pickler.dump + + obj: 'O' + + Write a pickled representation of obj to the open file. + [clinic start generated code]*/ + +9. If your function has ``|`` in the format string, meaning some + parameters have default values, you can ignore it. Argument + Clinic infers which parameters are optional based on whether + or not they have default values. + + If your function has ``$`` in the format string, meaning it + takes keyword-only arguments, specify ``*`` on a line by + itself before the first keyword-only argument, indented the + same as the parameter lines. + + (``_pickle.Pickler.dump`` has neither, so our sample is unchanged.) + + +10. If the existing C function calls :c:func:`PyArg_ParseTuple` + (as opposed to :c:func:`PyArg_ParseTupleAndKeywords`), then all its + arguments are positional-only. + + To mark all parameters as positional-only in Argument Clinic, + add a ``/`` on a line by itself after the last parameter, + indented the same as the parameter lines. + + Currently this is all-or-nothing; either all parameters are + positional-only, or none of them are. (In the future Argument + Clinic may relax this restriction.) + + Sample:: + + /*[clinic input] + module _pickle + class _pickle.Pickler "PicklerObject *" "&Pickler_Type" + [clinic start generated code]*/ + + /*[clinic input] + _pickle.Pickler.dump + + obj: 'O' + / + + Write a pickled representation of obj to the open file. + [clinic start generated code]*/ + +11. It's helpful to write a per-parameter docstring for each parameter. + But per-parameter docstrings are optional; you can skip this step + if you prefer. + + Here's how to add a per-parameter docstring. The first line + of the per-parameter docstring must be indented further than the + parameter definition. The left margin of this first line establishes + the left margin for the whole per-parameter docstring; all the text + you write will be outdented by this amount. You can write as much + text as you like, across multiple lines if you wish. + + Sample:: + + /*[clinic input] + module _pickle + class _pickle.Pickler "PicklerObject *" "&Pickler_Type" + [clinic start generated code]*/ + + /*[clinic input] + _pickle.Pickler.dump + + obj: 'O' + The object to be pickled. + / + + Write a pickled representation of obj to the open file. + [clinic start generated code]*/ + +12. Save and close the file, then run ``Tools/clinic/clinic.py`` on it. + With luck everything worked and your block now has output! Reopen + the file in your text editor to see:: + + /*[clinic input] + module _pickle + class _pickle.Pickler "PicklerObject *" "&Pickler_Type" + [clinic start generated code]*/ + /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/ + + /*[clinic input] + _pickle.Pickler.dump + + obj: 'O' + The object to be pickled. + / + + Write a pickled representation of obj to the open file. + [clinic start generated code]*/ + + PyDoc_STRVAR(_pickle_Pickler_dump__doc__, + "Write a pickled representation of obj to the open file.\n" + "\n" + ... + static PyObject * + _pickle_Pickler_dump_impl(PicklerObject *self, PyObject *obj) + /*[clinic end generated code: checksum=3bd30745bf206a48f8b576a1da3d90f55a0a4187]*/ + + Obviously, if Argument Clinic didn't produce any output, it's because + it found an error in your input. Keep fixing your errors and retrying + until Argument Clinic processes your file without complaint. + +13. Double-check that the argument-parsing code Argument Clinic generated + looks basically the same as the existing code. + + First, ensure both places use the same argument-parsing function. + The existing code must call either + :c:func:`PyArg_ParseTuple` or :c:func:`PyArg_ParseTupleAndKeywords`; + ensure that the code generated by Argument Clinic calls the + *exact* same function. + + Second, the format string passed in to :c:func:`PyArg_ParseTuple` or + :c:func:`PyArg_ParseTupleAndKeywords` should be *exactly* the same + as the hand-written one in the existing function, up to the colon + or semi-colon. + + (Argument Clinic always generates its format strings + with a ``:`` followed by the name of the function. If the + existing code's format string ends with ``;``, to provide + usage help, this change is harmless--don't worry about it.) + + Third, for parameters whose format units require two arguments + (like a length variable, or an encoding string, or a pointer + to a conversion function), ensure that the second argument is + *exactly* the same between the two invocations. + + Fourth, inside the output portion of the block you'll find a preprocessor + macro defining the appropriate static :c:type:`PyMethodDef` structure for + this builtin:: + + #define __PICKLE_PICKLER_DUMP_METHODDEF \ + {"dump", (PyCFunction)__pickle_Pickler_dump, METH_O, __pickle_Pickler_dump__doc__}, + + This static structure should be *exactly* the same as the existing static + :c:type:`PyMethodDef` structure for this builtin. + + If any of these items differ in *any way*, + adjust your Argument Clinic function specification and rerun + ``Tools/clinic/clinic.py`` until they *are* the same. + + +14. Notice that the last line of its output is the declaration + of your "impl" function. This is where the builtin's implementation goes. + Delete the existing prototype of the function you're modifying, but leave + the opening curly brace. Now delete its argument parsing code and the + declarations of all the variables it dumps the arguments into. + Notice how the Python arguments are now arguments to this impl function; + if the implementation used different names for these variables, fix it. + + Let's reiterate, just because it's kind of weird. Your code should now + look like this:: + + static return_type + your_function_impl(...) + /*[clinic end generated code: checksum=...]*/ + { + ... + + Argument Clinic generated the checksum line and the function prototype just + above it. You should write the opening (and closing) curly braces for the + function, and the implementation inside. + + Sample:: + + /*[clinic input] + module _pickle + class _pickle.Pickler "PicklerObject *" "&Pickler_Type" + [clinic start generated code]*/ + /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/ + + /*[clinic input] + _pickle.Pickler.dump + + obj: 'O' + The object to be pickled. + / + + Write a pickled representation of obj to the open file. + [clinic start generated code]*/ + + PyDoc_STRVAR(__pickle_Pickler_dump__doc__, + "Write a pickled representation of obj to the open file.\n" + "\n" + ... + static PyObject * + _pickle_Pickler_dump_impl(PicklerObject *self, PyObject *obj) + /*[clinic end generated code: checksum=3bd30745bf206a48f8b576a1da3d90f55a0a4187]*/ + { + /* Check whether the Pickler was initialized correctly (issue3664). + Developers often forget to call __init__() in their subclasses, which + would trigger a segfault without this check. */ + if (self->write == NULL) { + PyErr_Format(PicklingError, + "Pickler.__init__() was not called by %s.__init__()", + Py_TYPE(self)->tp_name); + return NULL; + } + + if (_Pickler_ClearBuffer(self) < 0) + return NULL; + + ... + +15. Remember the macro with the :c:type:`PyMethodDef` structure for this + function? Find the existing :c:type:`PyMethodDef` structure for this + function and replace it with a reference to the macro. (If the builtin + is at module scope, this will probably be very near the end of the file; + if the builtin is a class method, this will probably be below but relatively + near to the implementation.) + + Note that the body of the macro contains a trailing comma. So when you + replace the existing static :c:type:`PyMethodDef` structure with the macro, + *don't* add a comma to the end. + + Sample:: + + static struct PyMethodDef Pickler_methods[] = { + __PICKLE_PICKLER_DUMP_METHODDEF + __PICKLE_PICKLER_CLEAR_MEMO_METHODDEF + {NULL, NULL} /* sentinel */ + }; + + +16. Compile, then run the relevant portions of the regression-test suite. + This change should not introduce any new compile-time warnings or errors, + and there should be no externally-visible change to Python's behavior. + + Well, except for one difference: ``inspect.signature()`` run on your function + should now provide a valid signature! + + Congratulations, you've ported your first function to work with Argument Clinic! + +Advanced Topics +=============== + +Now that you've had some experience working with Argument Clinic, it's time +for some advanced topics. + + +Symbolic default values +----------------------- + +The default value you provide for a parameter can't be any arbitrary +expression. Currently the following are explicitly supported: + +* Numeric constants (integer and float) +* String constants +* ``True``, ``False``, and ``None`` +* Simple symbolic constants like ``sys.maxsize``, which must + start with the name of the module + +In case you're curious, this is implemented in ``from_builtin()`` +in ``Lib/inspect.py``. + +(In the future, this may need to get even more elaborate, +to allow full expressions like ``CONSTANT - 1``.) + + +Renaming the C functions and variables generated by Argument Clinic +------------------------------------------------------------------- + +Argument Clinic automatically names the functions it generates for you. +Occasionally this may cause a problem, if the generated name collides with +the name of an existing C function. There's an easy solution: override the names +used for the C functions. Just add the keyword ``"as"`` +to your function declaration line, followed by the function name you wish to use. +Argument Clinic will use that function name for the base (generated) function, +then add ``"_impl"`` to the end and use that for the name of the impl function. + +For example, if we wanted to rename the C function names generated for +``pickle.Pickler.dump``, it'd look like this:: + + /*[clinic input] + pickle.Pickler.dump as pickler_dumper + + ... + +The base function would now be named ``pickler_dumper()``, +and the impl function would now be named ``pickler_dumper_impl()``. + + +Similarly, you may have a problem where you want to give a parameter +a specific Python name, but that name may be inconvenient in C. Argument +Clinic allows you to give a parameter different names in Python and in C, +using the same ``"as"`` syntax:: + + /*[clinic input] + pickle.Pickler.dump + + obj: object + file as file_obj: object + protocol: object = NULL + * + fix_imports: bool = True + +Here, the name used in Python (in the signature and the ``keywords`` +array) would be ``file``, but the C variable would be named ``file_obj``. + +You can use this to rename the ``self`` parameter too! + + +Converting functions using PyArg_UnpackTuple +-------------------------------------------- + +To convert a function parsing its arguments with :c:func:`PyArg_UnpackTuple`, +simply write out all the arguments, specifying each as an ``object``. You +may specify the ``type`` argument to cast the type as appropriate. All +arguments should be marked positional-only (add a ``/`` on a line by itself +after the last argument). + +Currently the generated code will use :c:func:`PyArg_ParseTuple`, but this +will change soon. + +Optional Groups +--------------- + +Some legacy functions have a tricky approach to parsing their arguments: +they count the number of positional arguments, then use a ``switch`` statement +to call one of several different :c:func:`PyArg_ParseTuple` calls depending on +how many positional arguments there are. (These functions cannot accept +keyword-only arguments.) This approach was used to simulate optional +arguments back before :c:func:`PyArg_ParseTupleAndKeywords` was created. + +While functions using this approach can often be converted to +use :c:func:`PyArg_ParseTupleAndKeywords`, optional arguments, and default values, +it's not always possible. Some of these legacy functions have +behaviors :c:func:`PyArg_ParseTupleAndKeywords` doesn't directly support. +The most obvious example is the builtin function ``range()``, which has +an optional argument on the *left* side of its required argument! +Another example is ``curses.window.addch()``, which has a group of two +arguments that must always be specified together. (The arguments are +called ``x`` and ``y``; if you call the function passing in ``x``, +you must also pass in ``y``--and if you don't pass in ``x`` you may not +pass in ``y`` either.) + +In any case, the goal of Argument Clinic is to support argument parsing +for all existing CPython builtins without changing their semantics. +Therefore Argument Clinic supports +this alternate approach to parsing, using what are called *optional groups*. +Optional groups are groups of arguments that must all be passed in together. +They can be to the left or the right of the required arguments. They +can *only* be used with positional-only parameters. + +.. note:: Optional groups are *only* intended for use when converting + functions that make multiple calls to :c:func:`PyArg_ParseTuple`! + Functions that use *any* other approach for parsing arguments + should *almost never* be converted to Argument Clinic using + optional groups. Functions using optional groups currently + cannot have accurate sigantures in Python, because Python just + doesn't understand the concept. Please avoid using optional + groups wherever possible. + +To specify an optional group, add a ``[`` on a line by itself before +the parameters you wish to group together, and a ``]`` on a line by itself +after these parameters. As an example, here's how ``curses.window.addch`` +uses optional groups to make the first two parameters and the last +parameter optional:: + + /*[clinic input] + + curses.window.addch + + [ + x: int + X-coordinate. + y: int + Y-coordinate. + ] + + ch: object + Character to add. + + [ + attr: long + Attributes for the character. + ] + / + + ... + + +Notes: + +* For every optional group, one additional parameter will be passed into the + impl function representing the group. The parameter will be an int named + ``group_{direction}_{number}``, + where ``{direction}`` is either ``right`` or ``left`` depending on whether the group + is before or after the required parameters, and ``{number}`` is a monotonically + increasing number (starting at 1) indicating how far away the group is from + the required parameters. When the impl is called, this parameter will be set + to zero if this group was unused, and set to non-zero if this group was used. + (By used or unused, I mean whether or not the parameters received arguments + in this invocation.) + +* If there are no required arguments, the optional groups will behave + as if they're to the right of the required arguments. + +* In the case of ambiguity, the argument parsing code + favors parameters on the left (before the required parameters). + +* Optional groups can only contain positional-only parameters. + +* Optional groups are *only* intended for legacy code. Please do not + use optional groups for new code. + + +Using real Argument Clinic converters, instead of "legacy converters" +--------------------------------------------------------------------- + +To save time, and to minimize how much you need to learn +to achieve your first port to Argument Clinic, the walkthrough above tells +you to use "legacy converters". "Legacy converters" are a convenience, +designed explicitly to make porting existing code to Argument Clinic +easier. And to be clear, their use is acceptable when porting code for +Python 3.4. + +However, in the long term we probably want all our blocks to +use Argument Clinic's real syntax for converters. Why? A couple +reasons: + +* The proper converters are far easier to read and clearer in their intent. +* There are some format units that are unsupported as "legacy converters", + because they require arguments, and the legacy converter syntax doesn't + support specifying arguments. +* In the future we may have a new argument parsing library that isn't + restricted to what :c:func:`PyArg_ParseTuple` supports; this flexibility + won't be available to parameters using legacy converters. + +Therefore, if you don't mind a little extra effort, please use the normal +converters instead of legacy converters. + +In a nutshell, the syntax for Argument Clinic (non-legacy) converters +looks like a Python function call. However, if there are no explicit +arguments to the function (all functions take their default values), +you may omit the parentheses. Thus ``bool`` and ``bool()`` are exactly +the same converters. + +All arguments to Argument Clinic converters are keyword-only. +All Argument Clinic converters accept the following arguments: + + ``c_default`` + The default value for this parameter when defined in C. + Specifically, this will be the initializer for the variable declared + in the "parse function". See :ref:`the section on default values <default_values>` + for how to use this. + Specified as a string. + + ``annotation`` + The annotation value for this parameter. Not currently supported, + because PEP 8 mandates that the Python library may not use + annotations. + +In addition, some converters accept additional arguments. Here is a list +of these arguments, along with their meanings: + + ``bitwise`` + Only supported for unsigned integers. The native integer value of this + Python argument will be written to the parameter without any range checking, + even for negative values. + + ``converter`` + Only supported by the ``object`` converter. Specifies the name of a + :ref:`C "converter function" <o_ampersand>` + to use to convert this object to a native type. + + ``encoding`` + Only supported for strings. Specifies the encoding to use when converting + this string from a Python str (Unicode) value into a C ``char *`` value. + + ``length`` + Only supported for strings. If true, requests that the length of the + string be passed in to the impl function, just after the string parameter, + in a parameter named ``<parameter_name>_length``. + + ``nullable`` + Only supported for strings. If true, this parameter may also be set to + ``None``, in which case the C parameter will be set to ``NULL``. + + ``subclass_of`` + Only supported for the ``object`` converter. Requires that the Python + value be a subclass of a Python type, as expressed in C. + + ``types`` + Only supported for the ``object`` (and ``self``) converter. Specifies + the C type that will be used to declare the variable. Default value is + ``"PyObject *"``. + + ``types`` + A string containing a list of Python types (and possibly pseudo-types); + this restricts the allowable Python argument to values of these types. + (This is not a general-purpose facility; as a rule it only supports + specific lists of types as shown in the legacy converter table.) + + ``zeroes`` + Only supported for strings. If true, embedded NUL bytes (``'\\0'``) are + permitted inside the value. + +Please note, not every possible combination of arguments will work. +Often these arguments are implemented internally by specific ``PyArg_ParseTuple`` +*format units*, with specific behavior. For example, currently you cannot +call ``str`` and pass in ``zeroes=True`` without also specifying an ``encoding``; +although it's perfectly reasonable to think this would work, these semantics don't +map to any existing format unit. So Argument Clinic doesn't support it. (Or, at +least, not yet.) + +Below is a table showing the mapping of legacy converters into real +Argument Clinic converters. On the left is the legacy converter, +on the right is the text you'd replace it with. + +========= ================================================================================= +``'B'`` ``unsigned_char(bitwise=True)`` +``'b'`` ``unsigned_char`` +``'c'`` ``char`` +``'C'`` ``int(types='str')`` +``'d'`` ``double`` +``'D'`` ``Py_complex`` +``'es#'`` ``str(encoding='name_of_encoding', length=True, zeroes=True)`` +``'es'`` ``str(encoding='name_of_encoding')`` +``'et#'`` ``str(encoding='name_of_encoding', types='bytes bytearray str', length=True)`` +``'et'`` ``str(encoding='name_of_encoding', types='bytes bytearray str')`` +``'f'`` ``float`` +``'h'`` ``short`` +``'H'`` ``unsigned_short(bitwise=True)`` +``'i'`` ``int`` +``'I'`` ``unsigned_int(bitwise=True)`` +``'k'`` ``unsigned_long(bitwise=True)`` +``'K'`` ``unsigned_PY_LONG_LONG(bitwise=True)`` +``'L'`` ``PY_LONG_LONG`` +``'n'`` ``Py_ssize_t`` +``'O!'`` ``object(subclass_of='&PySomething_Type')`` +``'O&'`` ``object(converter='name_of_c_function')`` +``'O'`` ``object`` +``'p'`` ``bool`` +``'s#'`` ``str(length=True)`` +``'S'`` ``PyBytesObject`` +``'s'`` ``str`` +``'s*'`` ``Py_buffer(types='str bytes bytearray buffer')`` +``'u#'`` ``Py_UNICODE(length=True)`` +``'u'`` ``Py_UNICODE`` +``'U'`` ``unicode`` +``'w*'`` ``Py_buffer(types='bytearray rwbuffer')`` +``'y#'`` ``str(types='bytes', length=True)`` +``'Y'`` ``PyByteArrayObject`` +``'y'`` ``str(types='bytes')`` +``'y*'`` ``Py_buffer`` +``'Z#'`` ``Py_UNICODE(nullable=True, length=True)`` +``'z#'`` ``str(nullable=True, length=True)`` +``'Z'`` ``Py_UNICODE(nullable=True)`` +``'z'`` ``str(nullable=True)`` +``'z*'`` ``Py_buffer(types='str bytes bytearray buffer', nullable=True)`` +========= ================================================================================= + +As an example, here's our sample ``pickle.Pickler.dump`` using the proper +converter:: + + /*[clinic input] + pickle.Pickler.dump + + obj: object + The object to be pickled. + / + + Write a pickled representation of obj to the open file. + [clinic start generated code]*/ + +Argument Clinic will show you all the converters it has +available. For each converter it'll show you all the parameters +it accepts, along with the default value for each parameter. +Just run ``Tools/clinic/clinic.py --converters`` to see the full list. + +Py_buffer +--------- + +When using the ``Py_buffer`` converter +(or the ``'s*'``, ``'w*'``, ``'*y'``, or ``'z*'`` legacy converters), +you *must* not call :c:func:`PyBuffer_Release` on the provided buffer. +Argument Clinic generates code that does it for you (in the parsing function). + + + +Advanced converters +------------------- + +Remeber those format units you skipped for your first +time because they were advanced? Here's how to handle those too. + +The trick is, all those format units take arguments--either +conversion functions, or types, or strings specifying an encoding. +(But "legacy converters" don't support arguments. That's why we +skipped them for your first function.) The argument you specified +to the format unit is now an argument to the converter; this +argument is either ``converter`` (for ``O&``), ``subclass_of`` (for ``O!``), +or ``encoding`` (for all the format units that start with ``e``). + +When using ``subclass_of``, you may also want to use the other +custom argument for ``object()``: ``type``, which lets you set the type +actually used for the parameter. For example, if you want to ensure +that the object is a subclass of ``PyUnicode_Type``, you probably want +to use the converter ``object(type='PyUnicodeObject *', subclass_of='&PyUnicode_Type')``. + +One possible problem with using Argument Clinic: it takes away some possible +flexibility for the format units starting with ``e``. When writing a +``PyArg_Parse`` call by hand, you could theoretically decide at runtime what +encoding string to pass in to :c:func:`PyArg_ParseTuple`. But now this string must +be hard-coded at Argument-Clinic-preprocessing-time. This limitation is deliberate; +it made supporting this format unit much easier, and may allow for future optimizations. +This restriction doesn't seem unreasonable; CPython itself always passes in static +hard-coded encoding strings for parameters whose format units start with ``e``. + + +.. _default_values: + +Parameter default values +------------------------ + +Default values for parameters can be any of a number of values. +At their simplest, they can be string, int, or float literals:: + + foo: str = "abc" + bar: int = 123 + bat: float = 45.6 + +They can also use any of Python's built-in constants:: + + yep: bool = True + nope: bool = False + nada: object = None + +There's also special support for a default value of ``NULL``, and +for simple expressions, documented in the following sections. + + +The ``NULL`` default value +-------------------------- + +For string and object parameters, you can set them to ``None`` to indicate +that there's no default. However, that means the C variable will be +initialized to ``Py_None``. For convenience's sakes, there's a special +value called ``NULL`` for just this reason: from Python's perspective it +behaves like a default value of ``None``, but the C variable is initialized +with ``NULL``. + +Expressions specified as default values +--------------------------------------- + +The default value for a parameter can be more than just a literal value. +It can be an entire expression, using math operators and looking up attributes +on objects. However, this support isn't exactly simple, because of some +non-obvious semantics. + +Consider the following example:: + + foo: Py_ssize_t = sys.maxsize - 1 + +``sys.maxsize`` can have different values on different platforms. Therefore +Argument Clinic can't simply evaluate that expression locally and hard-code it +in C. So it stores the default in such a way that it will get evaluated at +runtime, when the user asks for the function's signature. + +What namespace is available when the expression is evaluated? It's evaluated +in the context of the module the builtin came from. So, if your module has an +attribute called "``max_widgets``", you may simply use it:: + + foo: Py_ssize_t = max_widgets + +If the symbol isn't found in the current module, it fails over to looking in +``sys.modules``. That's how it can find ``sys.maxsize`` for example. (Since you +don't know in advance what modules the user will load into their interpreter, +it's best to restrict yourself to modules that are preloaded by Python itself.) + +Evaluating default values only at runtime means Argument Clinic can't compute +the correct equivalent C default value. So you need to tell it explicitly. +When you use an expression, you must also specify the equivalent expression +in C, using the ``c_default`` parameter to the converter:: + + foo: Py_ssize_t(c_default="PY_SSIZE_T_MAX - 1") = sys.maxsize - 1 + +Another complication: Argument Clinic can't know in advance whether or not the +expression you supply is valid. It parses it to make sure it looks legal, but +it can't *actually* know. You must be very careful when using expressions to +specify values that are guaranteed to be valid at runtime! + +Finally, because expressions must be representable as static C values, there +are many restrictions on legal expressions. Here's a list of Python features +you're not permitted to use: + +* Function calls. +* Inline if statements (``3 if foo else 5``). +* Automatic sequence unpacking (``*[1, 2, 3]``). +* List/set/dict comprehensions and generator expressions. +* Tuple/list/set/dict literals. + + + +Using a return converter +------------------------ + +By default the impl function Argument Clinic generates for you returns ``PyObject *``. +But your C function often computes some C type, then converts it into the ``PyObject *`` +at the last moment. Argument Clinic handles converting your inputs from Python types +into native C types--why not have it convert your return value from a native C type +into a Python type too? + +That's what a "return converter" does. It changes your impl function to return +some C type, then adds code to the generated (non-impl) function to handle converting +that value into the appropriate ``PyObject *``. + +The syntax for return converters is similar to that of parameter converters. +You specify the return converter like it was a return annotation on the +function itself. Return converters behave much the same as parameter converters; +they take arguments, the arguments are all keyword-only, and if you're not changing +any of the default arguments you can omit the parentheses. + +(If you use both ``"as"`` *and* a return converter for your function, +the ``"as"`` should come before the return converter.) + +There's one additional complication when using return converters: how do you +indicate an error has occured? Normally, a function returns a valid (non-``NULL``) +pointer for success, and ``NULL`` for failure. But if you use an integer return converter, +all integers are valid. How can Argument Clinic detect an error? Its solution: each return +converter implicitly looks for a special value that indicates an error. If you return +that value, and an error has been set (``PyErr_Occurred()`` returns a true +value), then the generated code will propogate the error. Otherwise it will +encode the value you return like normal. + +Currently Argument Clinic supports only a few return converters:: + + bool + int + unsigned int + long + unsigned int + size_t + Py_ssize_t + float + double + DecodeFSDefault + +None of these take parameters. For the first three, return -1 to indicate +error. For ``DecodeFSDefault``, the return type is ``char *``; return a NULL +pointer to indicate an error. + +(There's also an experimental ``NoneType`` converter, which lets you +return ``Py_None`` on success or ``NULL`` on failure, without having +to increment the reference count on ``Py_None``. I'm not sure it adds +enough clarity to be worth using.) + +To see all the return converters Argument Clinic supports, along with +their parameters (if any), +just run ``Tools/clinic/clinic.py --converters`` for the full list. + + +Cloning existing functions +-------------------------- + +If you have a number of functions that look similar, you may be able to +use Clinic's "clone" feature. When you clone an existing function, +you reuse: + +* its parameters, including + + * their names, + + * their converters, with all parameters, + + * their default values, + + * their per-parameter docstrings, + + * their *kind* (whether they're positional only, + positional or keyword, or keyword only), and + +* its return converter. + +The only thing not copied from the original function is its docstring; +the syntax allows you to specify a new docstring. + +Here's the syntax for cloning a function:: + + /*[clinic input] + module.class.new_function [as c_basename] = module.class.existing_function + + Docstring for new_function goes here. + [clinic start generated code]*/ + +(The functions can be in different modules or classes. I wrote +``module.class`` in the sample just to illustrate that you must +use the full path to *both* functions.) + +Sorry, there's no syntax for partially-cloning a function, or cloning a function +then modifying it. Cloning is an all-or nothing proposition. + +Also, the function you are cloning from must have been previously defined +in the current file. + +Calling Python code +------------------- + +The rest of the advanced topics require you to write Python code +which lives inside your C file and modifies Argument Clinic's +runtime state. This is simple: you simply define a Python block. + +A Python block uses different delimiter lines than an Argument +Clinic function block. It looks like this:: + + /*[python input] + # python code goes here + [python start generated code]*/ + +All the code inside the Python block is executed at the +time it's parsed. All text written to stdout inside the block +is redirected into the "output" after the block. + +As an example, here's a Python block that adds a static integer +variable to the C code:: + + /*[python input] + print('static int __ignored_unused_variable__ = 0;') + [python start generated code]*/ + static int __ignored_unused_variable__ = 0; + /*[python checksum:...]*/ + + +Using a "self converter" +------------------------ + +Argument Clinic automatically adds a "self" parameter for you +using a default converter. It automatically sets the ``type`` +of this parameter to the "pointer to an instance" you specified +when you declared the type. However, you can override +Argument Clinic's converter and specify one yourself. +Just add your own ``self`` parameter as the first parameter in a +block, and ensure that its converter is an instance of +``self_converter`` or a subclass thereof. + +What's the point? This lets you override the type of ``self``, +or give it a different default name. + +How do you specify the custom type you want to cast ``self`` to? +If you only have one or two functions with the same type for ``self``, +you can directly use Argument Clinic's existing ``self`` converter, +passing in the type you want to use as the ``type`` parameter:: + + /*[clinic input] + + _pickle.Pickler.dump + + self: self(type="PicklerObject *") + obj: object + / + + Write a pickled representation of the given object to the open file. + [clinic start generated code]*/ + +On the other hand, if you have a lot of functions that will use the same +type for ``self``, it's best to create your own converter, subclassing +``self_converter`` but overwriting the ``type`` member:: + + /*[python input] + class PicklerObject_converter(self_converter): + type = "PicklerObject *" + [python start generated code]*/ + + /*[clinic input] + + _pickle.Pickler.dump + + self: PicklerObject + obj: object + / + + Write a pickled representation of the given object to the open file. + [clinic start generated code]*/ + + + +Writing a custom converter +-------------------------- + +As we hinted at in the previous section... you can write your own converters! +A converter is simply a Python class that inherits from ``CConverter``. +The main purpose of a custom converter is if you have a parameter using +the ``O&`` format unit--parsing this parameter means calling +a :c:func:`PyArg_ParseTuple` "converter function". + +Your converter class should be named ``*something*_converter``. +If the name follows this convention, then your converter class +will be automatically registered with Argument Clinic; its name +will be the name of your class with the ``_converter`` suffix +stripped off. (This is accomplished with a metaclass.) + +You shouldn't subclass ``CConverter.__init__``. Instead, you should +write a ``converter_init()`` function. ``converter_init()`` +always accepts a ``self`` parameter; after that, all additional +parameters *must* be keyword-only. Any arguments passed in to +the converter in Argument Clinic will be passed along to your +``converter_init()``. + +There are some additional members of ``CConverter`` you may wish +to specify in your subclass. Here's the current list: + +``type`` + The C type to use for this variable. + ``type`` should be a Python string specifying the type, e.g. ``int``. + If this is a pointer type, the type string should end with ``' *'``. + +``default`` + The Python default value for this parameter, as a Python value. + Or the magic value ``unspecified`` if there is no default. + +``py_default`` + ``default`` as it should appear in Python code, + as a string. + Or ``None`` if there is no default. + +``c_default`` + ``default`` as it should appear in C code, + as a string. + Or ``None`` if there is no default. + +``c_ignored_default`` + The default value used to initialize the C variable when + there is no default, but not specifying a default may + result in an "uninitialized variable" warning. This can + easily happen when using option groups--although + properly-written code will never actually use this value, + the variable does get passed in to the impl, and the + C compiler will complain about the "use" of the + uninitialized value. This value should always be a + non-empty string. + +``converter`` + The name of the C converter function, as a string. + +``impl_by_reference`` + A boolean value. If true, + Argument Clinic will add a ``&`` in front of the name of + the variable when passing it into the impl function. + +``parse_by_reference`` + A boolean value. If true, + Argument Clinic will add a ``&`` in front of the name of + the variable when passing it into :c:func:`PyArg_ParseTuple`. + + +Here's the simplest example of a custom converter, from ``Modules/zlibmodule.c``:: + + /*[python input] + + class uint_converter(CConverter): + type = 'unsigned int' + converter = 'uint_converter' + + [python start generated code]*/ + /*[python end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/ + +This block adds a converter to Argument Clinic named ``uint``. Parameters +declared as ``uint`` will be declared as type ``unsigned int``, and will +be parsed by the ``'O&'`` format unit, which will call the ``uint_converter`` +converter function. +``uint`` variables automatically support default values. + +More sophisticated custom converters can insert custom C code to +handle initialization and cleanup. +You can see more examples of custom converters in the CPython +source tree; grep the C files for the string ``CConverter``. + +Writing a custom return converter +--------------------------------- + +Writing a custom return converter is much like writing +a custom converter. Except it's somewhat simpler, because return +converters are themselves much simpler. + +Return converters must subclass ``CReturnConverter``. +There are no examples yet of custom return converters, +because they are not widely used yet. If you wish to +write your own return converter, please read ``Tools/clinic/clinic.py``, +specifically the implementation of ``CReturnConverter`` and +all its subclasses. + +METH_O and METH_NOARGS +---------------------------------------------- + +To convert a function using ``METH_O``, make sure the function's +single argument is using the ``object`` converter, and mark the +arguments as positional-only:: + + /*[clinic input] + meth_o_sample + + argument: object + / + [clinic start generated code]*/ + + +To convert a function using ``METH_NOARGS``, just don't specify +any arguments. + +You can still use a self converter, a return converter, and specify +a ``type`` argument to the object converter for ``METH_O``. + +tp_new and tp_init functions +---------------------------------------------- + +You can convert ``tp_new`` and ``tp_init`` functions. Just name +them ``__new__`` or ``__init__`` as appropriate. Notes: + +* The function name generated for ``__new__`` doesn't end in ``__new__`` + like it would by default. It's just the name of the class, converted + into a valid C identifier. + +* No ``PyMethodDef`` ``#define`` is generated for these functions. + +* ``__init__`` functions return ``int``, not ``PyObject *``. + +* Use the docstring as the class docstring. + +* Although ``__new__`` and ``__init__`` functions must always + accept both the ``args`` and ``kwargs`` objects, when converting + you may specify any signature for these functions that you like. + (If your function doesn't support keywords, the parsing function + generated will throw an exception if it receives any.) + +Changing and redirecting Clinic's output +---------------------------------------- + +It can be inconvenient to have Clinic's output interspersed with +your conventional hand-edited C code. Luckily, Clinic is configurable: +you can buffer up its output for printing later (or earlier!), or write +its output to a separate file. You can also add a prefix or suffix to +every line of Clinic's generated output. + +While changing Clinic's output in this manner can be a boon to readability, +it may result in Clinic code using types before they are defined, or +your code attempting to use Clinic-generated code befire it is defined. +These problems can be easily solved by rearranging the declarations in your file, +or moving where Clinic's generated code goes. (This is why the default behavior +of Clinic is to output everything into the current block; while many people +consider this hampers readability, it will never require rearranging your +code to fix definition-before-use problems.) + +Let's start with defining some terminology: + +*field* + A field, in this context, is a subsection of Clinic's output. + For example, the ``#define`` for the ``PyMethodDef`` structure + is a field, called ``methoddef_define``. Clinic has seven + different fields it can output per function definition:: + + docstring_prototype + docstring_definition + methoddef_define + impl_prototype + parser_prototype + parser_definition + impl_definition + + All the names are of the form ``"<a>_<b>"``, + where ``"<a>"`` is the semantic object represented (the parsing function, + the impl function, the docstring, or the methoddef structure) and ``"<b>"`` + represents what kind of statement the field is. Field names that end in + ``"_prototype"`` + represent forward declarations of that thing, without the actual body/data + of the thing; field names that end in ``"_definition"`` represent the actual + definition of the thing, with the body/data of the thing. (``"methoddef"`` + is special, it's the only one that ends with ``"_define"``, representing that + it's a preprocessor #define.) + +*destination* + A destination is a place Clinic can write output to. There are + five built-in destinations: + + ``block`` + The default destination: printed in the output section of + the current Clinic block. + + ``buffer`` + A text buffer where you can save text for later. Text sent + here is appended to the end of any exsiting text. It's an + error to have any text left in the buffer when Clinic finishes + processing a file. + + ``file`` + A separate "clinic file" that will be created automatically by Clinic. + The filename chosen for the file is ``{basename}.clinic{extension}``, + where ``basename`` and ``extension`` were assigned the output + from ``os.path.splitext()`` run on the current file. (Example: + the ``file`` destination for ``_pickle.c`` would be written to + ``_pickle.clinic.c``.) + + **Important: When using a** ``file`` **destination, you** + *must check in* **the generated file!** + + ``two-pass`` + A buffer like ``buffer``. However, a two-pass buffer can only + be written once, and it prints out all text sent to it during + all of processing, even from Clinic blocks *after* the + + ``suppress`` + The text is suppressed--thrown away. + + +Clinic defines five new directives that let you reconfigure its output. + +The first new directive is ``dump``:: + + dump <destination> + +This dumps the current contents of the named destination into the output of +the current block, and empties it. This only works with ``buffer`` and +``two-pass`` destinations. + +The second new directive is ``output``. The most basic form of ``output`` +is like this:: + + output <field> <destination> + +This tells Clinic to output *field* to *destination*. ``output`` also +supports a special meta-destination, called ``everything``, which tells +Clinic to output *all* fields to that *destination*. + +``output`` has a number of other functions:: + + output push + output pop + output preset <preset> + + +``output push`` and ``output pop`` allow you to push and pop +configurations on an internal configuration stack, so that you +can temporarily modify the output configuration, then easily restore +the previous configuration. Simply push before your change to save +the current configuration, then pop when you wish to restore the +previous configuration. + +``output preset`` sets Clinic's output to one of several built-in +preset configurations, as follows: + + ``block`` + Clinic's original starting configuration. Writes everything + immediately after the input block. + + Suppress the ``parser_prototype`` + and ``docstring_prototype``, write everything else to ``block``. + + ``file`` + Designed to write everything to the "clinic file" that it can. + You then ``#include`` this file near the top of your file. + You may need to rearrange your file to make this work, though + usually this just means creating forward declarations for various + ``typedef`` and ``PyTypeObject`` definitions. + + Suppress the ``parser_prototype`` + and ``docstring_prototype``, write the ``impl_definition`` to + ``block``, and write everything else to ``file``. + + The default filename is ``"{dirname}/clinic/{basename}.h"``. + + ``buffer`` + Save up all most of the output from Clinic, to be written into + your file near the end. For Python files implementing modules + or builtin types, it's recommended that you dump the buffer + just above the static structures for your module or + builtin type; these are normally very near the end. Using + ``buffer`` may require even more editing than ``file``, if + your file has static ``PyMethodDef`` arrays defined in the + middle of the file. + + Suppress the ``parser_prototype``, ``impl_prototype``, + and ``docstring_prototype``, write the ``impl_definition`` to + ``block``, and write everything else to ``file``. + + ``two-pass`` + Similar to the ``buffer`` preset, but writes forward declarations to + the ``two-pass`` buffer, and definitions to the ``buffer``. + This is similar to the ``buffer`` preset, but may require + less editing than ``buffer``. Dump the ``two-pass`` buffer + near the top of your file, and dump the ``buffer`` near + the end just like you would when using the ``buffer`` preset. + + Suppresses the ``impl_prototype``, write the ``impl_definition`` + to ``block``, write ``docstring_prototype``, ``methoddef_define``, + and ``parser_prototype`` to ``two-pass``, write everything else + to ``buffer``. + + ``partial-buffer`` + Similar to the ``buffer`` preset, but writes more things to ``block``, + only writing the really big chunks of generated code to ``buffer``. + This avoids the definition-before-use problem of ``buffer`` completely, + at the small cost of having slightly more stuff in the block's output. + Dump the ``buffer`` near the end, just like you would when using + the ``buffer`` preset. + + Suppresses the ``impl_prototype``, write the ``docstring_definition`` + and ``parser_defintion`` to ``buffer``, write everything else to ``block``. + +The third new directive is ``destination``:: + + destination <name> <command> [...] + +This performs an operation on the destination named ``name``. + +There are two defined subcommands: ``new`` and ``clear``. + +The ``new`` subcommand works like this:: + + destination <name> new <type> + +This creates a new destination with name ``<name>`` and type ``<type>``. + +There are five destination types: + + ``suppress`` + Throws the text away. + + ``block`` + Writes the text to the current block. This is what Clinic + originally did. + + ``buffer`` + A simple text buffer, like the "buffer" builtin destination above. + + ``file`` + A text file. The file destination takes an extra argument, + a template to use for building the filename, like so: + + destination <name> new <type> <file_template> + + The template can use three strings internally that will be replaced + by bits of the filename: + + {path} + The full path to the file, including directory and full filename. + {dirname} + The name of the directory the file is in. + {basename} + Just the name of the file, not including the directory. + {basename_root} + Basename with the extension clipped off + (everything up to but not including the last '.'). + {basename_extension} + The last '.' and everything after it. If the basename + does not contain a period, this will be the empty string. + + If there are no periods in the filename, {basename} and {filename} + are the same, and {extension} is empty. "{basename}{extension}" + is always exactly the same as "{filename}"." + + ``two-pass`` + A two-pass buffer, like the "two-pass" builtin destination above. + + +The ``clear`` subcommand works like this:: + + destination <name> clear + +It removes all the accumulated text up to this point in the destination. +(I don't know what you'd need this for, but I thought maybe it'd be +useful while someone's experimenting.) + +The fourth new directive is ``set``:: + + set line_prefix "string" + set line_suffix "string" + +``set`` lets you set two internal variables in Clinic. +``line_prefix`` is a string that will be prepended to every line of Clinic's output; +``line_suffix`` is a string that will be appended to every line of Clinic's output. + +Both of these suport two format strings: + + ``{block comment start}`` + Turns into the string ``/*``, the start-comment text sequence for C files. + + ``{block comment end}`` + Turns into the string ``*/``, the end-comment text sequence for C files. + +The final new directive is one you shouldn't need to use directly, +called ``preserve``:: + + preserve + +This tells Clinic that the current contents of the output should be kept, unmodifed. +This is used internally by Clinic when dumping output into ``file`` files; wrapping +it in a Clinic block lets Clinic use its existing checksum functionality to ensure +the file was not modified by hand before it gets overwritten. + + +The #ifdef trick +---------------------------------------------- + +If you're converting a function that isn't available on all platforms, +there's a trick you can use to make life a little easier. The existing +code probably looks like this:: + + #ifdef HAVE_FUNCTIONNAME + static module_functionname(...) + { + ... + } + #endif /* HAVE_FUNCTIONNAME */ + +And then in the ``PyMethodDef`` structure at the bottom the existing code +will have:: + + #ifdef HAVE_FUNCTIONNAME + {'functionname', ... }, + #endif /* HAVE_FUNCTIONNAME */ + +In this scenario, you should enclose the body of your impl function inside the ``#ifdef``, +like so:: + + #ifdef HAVE_FUNCTIONNAME + /*[clinic input] + module.functionname + ... + [clinic start generated code]*/ + static module_functionname(...) + { + ... + } + #endif /* HAVE_FUNCTIONNAME */ + +Then, remove those three lines from the ``PyMethodDef`` structure, +replacing them with the macro Argument Clinic generated:: + + MODULE_FUNCTIONNAME_METHODDEF + +(You can find the real name for this macro inside the generated code. +Or you can calculate it yourself: it's the name of your function as defined +on the first line of your block, but with periods changed to underscores, +uppercased, and ``"_METHODDEF"`` added to the end.) + +Perhaps you're wondering: what if ``HAVE_FUNCTIONNAME`` isn't defined? +The ``MODULE_FUNCTIONNAME_METHODDEF`` macro won't be defined either! + +Here's where Argument Clinic gets very clever. It actually detects that the +Argument Clinic block might be deactivated by the ``#ifdef``. When that +happens, it generates a little extra code that looks like this:: + + #ifndef MODULE_FUNCTIONNAME_METHODDEF + #define MODULE_FUNCTIONNAME_METHODDEF + #endif /* !defined(MODULE_FUNCTIONNAME_METHODDEF) */ + +That means the macro always works. If the function is defined, this turns +into the correct structure, including the trailing comma. If the function is +undefined, this turns into nothing. + +However, this causes one ticklish problem: where should Argument Clinic put this +extra code when using the "block" output preset? It can't go in the output block, +because that could be decativated by the ``#ifdef``. (That's the whole point!) + +In this situation, Argument Clinic writes the extra code to the "buffer" destination. +This may mean that you get a complaint from Argument Clinic:: + + Warning in file "Modules/posixmodule.c" on line 12357: + Destination buffer 'buffer' not empty at end of file, emptying. + +When this happens, just open your file, find the ``dump buffer`` block that +Argument Clinic added to your file (it'll be at the very bottom), then +move it above the ``PyMethodDef`` structure where that macro is used. + + + +Using Argument Clinic in Python files +------------------------------------- + +It's actually possible to use Argument Clinic to preprocess Python files. +There's no point to using Argument Clinic blocks, of course, as the output +wouldn't make any sense to the Python interpreter. But using Argument Clinic +to run Python blocks lets you use Python as a Python preprocessor! + +Since Python comments are different from C comments, Argument Clinic +blocks embedded in Python files look slightly different. They look like this:: + + #/*[python input] + #print("def foo(): pass") + #[python start generated code]*/ + def foo(): pass + #/*[python checksum:...]*/ diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 7189c65..97c90c7 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -3,7 +3,7 @@ ******************************** :Author: A. M. Kuchling -:Release: 0.31 +:Release: 0.32 In this document, we'll take a tour of Python's features suitable for implementing programs in a functional style. After an introduction to the @@ -15,9 +15,9 @@ concepts of functional programming, we'll look at language features such as Introduction ============ -This section explains the basic concept of functional programming; if you're -just interested in learning about Python language features, skip to the next -section. +This section explains the basic concept of functional programming; if +you're just interested in learning about Python language features, +skip to the next section on :ref:`functional-howto-iterators`. Programming languages support decomposing problems in several different ways: @@ -173,6 +173,8 @@ new programs by arranging existing functions in a new configuration and writing a few functions specialized for the current task. +.. _functional-howto-iterators: + Iterators ========= @@ -670,7 +672,7 @@ indexes at which certain conditions are met:: :func:`sorted(iterable, key=None, reverse=False) <sorted>` collects all the elements of the iterable into a list, sorts the list, and returns the sorted -result. The *key*, and *reverse* arguments are passed through to the +result. The *key* and *reverse* arguments are passed through to the constructed list's :meth:`~list.sort` method. :: >>> import random @@ -836,7 +838,8 @@ Another group of functions chooses a subset of an iterator's elements based on a predicate. :func:`itertools.filterfalse(predicate, iter) <itertools.filterfalse>` is the -opposite, returning all elements for which the predicate returns false:: +opposite of :func:`filter`, returning all elements for which the predicate +returns false:: itertools.filterfalse(is_even, itertools.count()) => 1, 3, 5, 7, 9, 11, 13, 15, ... @@ -864,6 +867,77 @@ iterable's results. :: itertools.dropwhile(is_even, itertools.count()) => 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, ... +:func:`itertools.compress(data, selectors) <itertools.compress>` takes two +iterators and returns only those elements of *data* for which the corresponding +element of *selectors* is true, stopping whenever either one is exhausted:: + + itertools.compress([1,2,3,4,5], [True, True, False, False, True]) => + 1, 2, 5 + + +Combinatoric functions +---------------------- + +The :func:`itertools.combinations(iterable, r) <itertools.combinations>` +returns an iterator giving all possible *r*-tuple combinations of the +elements contained in *iterable*. :: + + itertools.combinations([1, 2, 3, 4, 5], 2) => + (1, 2), (1, 3), (1, 4), (1, 5), + (2, 3), (2, 4), (2, 5), + (3, 4), (3, 5), + (4, 5) + + itertools.combinations([1, 2, 3, 4, 5], 3) => + (1, 2, 3), (1, 2, 4), (1, 2, 5), (1, 3, 4), (1, 3, 5), (1, 4, 5), + (2, 3, 4), (2, 3, 5), (2, 4, 5), + (3, 4, 5) + +The elements within each tuple remain in the same order as +*iterable* returned them. For example, the number 1 is always before +2, 3, 4, or 5 in the examples above. A similar function, +:func:`itertools.permutations(iterable, r=None) <itertools.permutations>`, +removes this constraint on the order, returning all possible +arrangements of length *r*:: + + itertools.permutations([1, 2, 3, 4, 5], 2) => + (1, 2), (1, 3), (1, 4), (1, 5), + (2, 1), (2, 3), (2, 4), (2, 5), + (3, 1), (3, 2), (3, 4), (3, 5), + (4, 1), (4, 2), (4, 3), (4, 5), + (5, 1), (5, 2), (5, 3), (5, 4) + + itertools.permutations([1, 2, 3, 4, 5]) => + (1, 2, 3, 4, 5), (1, 2, 3, 5, 4), (1, 2, 4, 3, 5), + ... + (5, 4, 3, 2, 1) + +If you don't supply a value for *r* the length of the iterable is used, +meaning that all the elements are permuted. + +Note that these functions produce all of the possible combinations by +position and don't require that the contents of *iterable* are unique:: + + itertools.permutations('aba', 3) => + ('a', 'b', 'a'), ('a', 'a', 'b'), ('b', 'a', 'a'), + ('b', 'a', 'a'), ('a', 'a', 'b'), ('a', 'b', 'a') + +The identical tuple ``('a', 'a', 'b')`` occurs twice, but the two 'a' +strings came from different positions. + +The :func:`itertools.combinations_with_replacement(iterable, r) <itertools.combinations_with_replacement>` +function relaxes a different constraint: elements can be repeated +within a single tuple. Conceptually an element is selected for the +first position of each tuple and then is replaced before the second +element is selected. :: + + itertools.combinations_with_replacement([1, 2, 3, 4, 5], 2) => + (1, 1), (1, 2), (1, 3), (1, 4), (1, 5), + (2, 2), (2, 3), (2, 4), (2, 5), + (3, 3), (3, 4), (3, 5), + (4, 4), (4, 5), + (5, 5) + Grouping elements ----------------- @@ -986,6 +1060,17 @@ write the obvious :keyword:`for` loop:: for i in [1,2,3]: product *= i +A related function is `itertools.accumulate(iterable, func=operator.add) <itertools.accumulate`. +It performs the same calculation, but instead of returning only the +final result, :func:`accumulate` returns an iterator that also yields +each partial result:: + + itertools.accumulate([1,2,3,4,5]) => + 1, 3, 6, 10, 15 + + itertools.accumulate([1,2,3,4,5], operator.mul) => + 1, 2, 6, 24, 120 + The operator module ------------------- @@ -1159,51 +1244,6 @@ features in Python 2.5. .. comment - Topics to place - ----------------------------- - - XXX os.walk() - - XXX Need a large example. - - But will an example add much? I'll post a first draft and see - what the comments say. - -.. comment - - Original outline: - Introduction - Idea of FP - Programs built out of functions - Functions are strictly input-output, no internal state - Opposed to OO programming, where objects have state - - Why FP? - Formal provability - Assignment is difficult to reason about - Not very relevant to Python - Modularity - Small functions that do one thing - Debuggability: - Easy to test due to lack of state - Easy to verify output from intermediate steps - Composability - You assemble a toolbox of functions that can be mixed - - Tackling a problem - Need a significant example - - Iterators - Generators - The itertools module - List comprehensions - Small functions and the lambda statement - Built-in functions - map - filter - -.. comment - Handy little function for printing part of an iterator -- used while writing this document. @@ -1214,5 +1254,3 @@ features in Python 2.5. sys.stdout.write(str(elem)) sys.stdout.write(', ') print(elem[-1]) - - diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index 81a4f8b..2c9d699 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -28,4 +28,5 @@ Currently, the HOWTOs are: webservers.rst argparse.rst ipaddress.rst + clinic.rst diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 563da9d..82c8f51 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -416,7 +416,7 @@ module. Here is a basic working example:: Simple TCP socket-based logging receiver suitable for testing. """ - allow_reuse_address = 1 + allow_reuse_address = True def __init__(self, host='localhost', port=logging.handlers.DEFAULT_TCP_LOGGING_PORT, @@ -704,9 +704,7 @@ the basis for code meeting your own specific requirements:: break logger = logging.getLogger(record.name) logger.handle(record) # No level or filter logic applied - just do it! - except (KeyboardInterrupt, SystemExit): - raise - except: + except Exception: import sys, traceback print('Whoops! Problem:', file=sys.stderr) traceback.print_exc(file=sys.stderr) @@ -1310,7 +1308,7 @@ This dictionary is passed to :func:`~config.dictConfig` to put the configuration } For more information about this configuration, you can see the `relevant -section <https://docs.djangoproject.com/en/1.3/topics/logging/#configuring-logging>`_ +section <https://docs.djangoproject.com/en/1.6/topics/logging/#configuring-logging>`_ of the Django documentation. .. _cookbook-rotator-namer: diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 55b1c86..0387bea 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -901,10 +901,10 @@ provided: disk files, rotating the log file at certain timed intervals. #. :class:`~handlers.SocketHandler` instances send messages to TCP/IP - sockets. + sockets. Since 3.4, Unix domain sockets are also supported. #. :class:`~handlers.DatagramHandler` instances send messages to UDP - sockets. + sockets. Since 3.4, Unix domain sockets are also supported. #. :class:`~handlers.SMTPHandler` instances send messages to a designated email address. @@ -1027,6 +1027,15 @@ You can write code like this:: so that if the logger's threshold is set above ``DEBUG``, the calls to :func:`expensive_func1` and :func:`expensive_func2` are never made. +.. note:: In some cases, :meth:`~Logger.isEnabledFor` can iself be more + expensive than you'd like (e.g. for deeply nested loggers where an explicit + level is only set high up in the logger hierarchy). In such cases (or if you + want to avoid calling a method in tight loops), you can cache the result of a + call to :meth:`~Logger.isEnabledFor` in a local or instance variable, and use + that instead of calling the method each time. Such a cached value would only + need to be recomputed when the logging configuration changes dynamically + while the application is running (which is not all that common). + There are other optimizations which can be made for specific applications which need more precise control over what logging information is collected. Here's a list of things you can do to avoid processing during logging which you don't @@ -1036,6 +1045,12 @@ need: | What you don't want to collect | How to avoid collecting it | +===============================================+========================================+ | Information about where calls were made from. | Set ``logging._srcfile`` to ``None``. | +| | This avoids calling | +| | :func:`sys._getframe`, which may help | +| | to speed up your code in environments | +| | like PyPy (which can't speed up code | +| | that uses :func:`sys._getframe`), if | +| | and when PyPy supports Python 3.x. | +-----------------------------------------------+----------------------------------------+ | Threading information. | Set ``logging.logThreads`` to ``0``. | +-----------------------------------------------+----------------------------------------+ diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst index 5203e53..fbe763b 100644 --- a/Doc/howto/regex.rst +++ b/Doc/howto/regex.rst @@ -271,8 +271,8 @@ performing string substitutions. :: >>> import re >>> p = re.compile('ab*') - >>> p #doctest: +ELLIPSIS - <_sre.SRE_Pattern object at 0x...> + >>> p + re.compile('ab*') :func:`re.compile` also accepts an optional *flags* argument, used to enable various special features and syntax variations. We'll go over the available @@ -383,8 +383,8 @@ Python interpreter, import the :mod:`re` module, and compile a RE:: >>> import re >>> p = re.compile('[a-z]+') - >>> p #doctest: +ELLIPSIS - <_sre.SRE_Pattern object at 0x...> + >>> p + re.compile('[a-z]+') Now, you can try matching various strings against the RE ``[a-z]+``. An empty string shouldn't match at all, since ``+`` means 'one or more repetitions'. @@ -402,7 +402,7 @@ should store the result in a variable for later use. :: >>> m = p.match('tempo') >>> m #doctest: +ELLIPSIS - <_sre.SRE_Match object at 0x...> + <_sre.SRE_Match object; span=(0, 5), match='tempo'> Now you can query the :ref:`match object <match-objects>` for information about the matching string. :ref:`match object <match-objects>` instances @@ -441,7 +441,7 @@ case. :: >>> print(p.match('::: message')) None >>> m = p.search('::: message'); print(m) #doctest: +ELLIPSIS - <_sre.SRE_Match object at 0x...> + <_sre.SRE_Match object; span=(4, 11), match='message'> >>> m.group() 'message' >>> m.span() @@ -493,7 +493,7 @@ the RE string added as the first argument, and still return either ``None`` or a >>> print(re.match(r'From\s+', 'Fromage amk')) None >>> re.match(r'From\s+', 'From amk Thu May 14 19:12:10 1998') #doctest: +ELLIPSIS - <_sre.SRE_Match object at 0x...> + <_sre.SRE_Match object; span=(0, 5), match='From '> Under the hood, these functions simply create a pattern object for you and call the appropriate method on it. They also store the compiled @@ -685,7 +685,7 @@ given location, they can obviously be matched an infinite number of times. line, the RE to use is ``^From``. :: >>> print(re.search('^From', 'From Here to Eternity')) #doctest: +ELLIPSIS - <_sre.SRE_Match object at 0x...> + <_sre.SRE_Match object; span=(0, 4), match='From'> >>> print(re.search('^From', 'Reciting From Memory')) None @@ -697,11 +697,11 @@ given location, they can obviously be matched an infinite number of times. or any location followed by a newline character. :: >>> print(re.search('}$', '{block}')) #doctest: +ELLIPSIS - <_sre.SRE_Match object at 0x...> + <_sre.SRE_Match object; span=(6, 7), match='}'> >>> print(re.search('}$', '{block} ')) None >>> print(re.search('}$', '{block}\n')) #doctest: +ELLIPSIS - <_sre.SRE_Match object at 0x...> + <_sre.SRE_Match object; span=(6, 7), match='}'> To match a literal ``'$'``, use ``\$`` or enclose it inside a character class, as in ``[$]``. @@ -726,7 +726,7 @@ given location, they can obviously be matched an infinite number of times. >>> p = re.compile(r'\bclass\b') >>> print(p.search('no class at all')) #doctest: +ELLIPSIS - <_sre.SRE_Match object at 0x...> + <_sre.SRE_Match object; span=(3, 8), match='class'> >>> print(p.search('the declassified algorithm')) None >>> print(p.search('one subclass is')) @@ -744,7 +744,7 @@ given location, they can obviously be matched an infinite number of times. >>> print(p.search('no class at all')) None >>> print(p.search('\b' + 'class' + '\b')) #doctest: +ELLIPSIS - <_sre.SRE_Match object at 0x...> + <_sre.SRE_Match object; span=(0, 7), match='\x08class\x08'> Second, inside a character class, where there's no use for this assertion, ``\b`` represents the backspace character, for compatibility with Python's diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst index 7a9b0ed..c04fd55 100644 --- a/Doc/howto/sockets.rst +++ b/Doc/howto/sockets.rst @@ -204,13 +204,15 @@ length message:: totalsent = totalsent + sent def myreceive(self): - msg = b'' - while len(msg) < MSGLEN: - chunk = self.sock.recv(MSGLEN-len(msg)) + chunks = [] + bytes_recd = 0 + while bytes_recd < MSGLEN: + chunk = self.sock.recv(min(MSGLEN - bytes_recd, 2048)) if chunk == b'': raise RuntimeError("socket connection broken") - msg = msg + chunk - return msg + chunks.append(chunk) + bytes_recd = bytes_recd + len(chunk) + return b''.join(chunks) The sending code here is usable for almost any messaging scheme - in Python you send strings, and you can use ``len()`` to determine its length (even if it has diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index 9d48a78..632e525 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -49,7 +49,7 @@ another and managed to catch on. 255 characters aren't very many. For example, you can't fit both the accented characters used in Western Europe and the Cyrillic alphabet used for Russian -into the 128--255 range because there are more than 127 such characters. +into the 128--255 range because there are more than 128 such characters. You could write files using different codes (all your Russian files in a coding system called KOI8, all your French files in a different coding system called @@ -246,7 +246,7 @@ include a Unicode character in a string literal:: try: with open('/tmp/input.txt', 'r') as f: ... - except IOError: + except OSError: # 'File not found' error message. print("Fichier non trouvé") diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 5a22660..3d5de32 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -507,7 +507,7 @@ than the URL you pass to .add_password() will also match. :: -- ``ProxyHandler`` (if a proxy setting such as an :envvar:`http_proxy` environment variable is set), ``UnknownHandler``, ``HTTPHandler``, ``HTTPDefaultErrorHandler``, ``HTTPRedirectHandler``, ``FTPHandler``, - ``FileHandler``, ``HTTPErrorProcessor``. + ``FileHandler``, ``DataHandler``, ``HTTPErrorProcessor``. ``top_level_url`` is in fact *either* a full URL (including the 'http:' scheme component and the hostname and optionally the port number) diff --git a/Doc/howto/webservers.rst b/Doc/howto/webservers.rst index 72ccd1f..6fa51da 100644 --- a/Doc/howto/webservers.rst +++ b/Doc/howto/webservers.rst @@ -687,7 +687,7 @@ published, which is a good starting point. The newest version of TurboGears, version 2.0, moves even further in direction of WSGI support and a component-based architecture. TurboGears 2 is based on the WSGI stack of another popular component-based web framework, `Pylons -<http://pylonshq.com/>`_. +<http://www.pylonsproject.org/>`_. Zope diff --git a/Doc/includes/email-alternative-new-api.py b/Doc/includes/email-alternative-new-api.py new file mode 100644 index 0000000..c1255a6 --- /dev/null +++ b/Doc/includes/email-alternative-new-api.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +import smtplib + +from email.message import EmailMessage +from email.headerregistry import Address +from email.utils import make_msgid + +# Create the base text message. +msg = EmailMessage() +msg['Subject'] = "Ayons asperges pour le déjeuner" +msg['From'] = Address("Pepé Le Pew", "pepe@example.com") +msg['To'] = (Address("Penelope Pussycat", "penelope@example.com"), + Address("Fabrette Pussycat", "fabrette@example.com")) +msg.set_content("""\ +Salut! + +Cela ressemble à un excellent recipie[1] déjeuner. + +[1] http://www.yummly.com/recipe/Roasted-Asparagus-Epicurious-203718 + +--Pepé +""") + +# Add the html version. This converts the message into a multipart/alternative +# container, with the original text message as the first part and the new html +# message as the second part. +asparagus_cid = make_msgid() +msg.add_alternative("""\ +<html> + <head></head> + <body> + <p>Salut!<\p> + <p>Cela ressemble à un excellent + <a href="http://www.yummly.com/recipe/Roasted-Asparagus-Epicurious-203718> + recipie + </a> déjeuner. + </p> + <img src="cid:{asparagus_cid}" \> + </body> +</html> +""".format(asparagus_cid=asparagus_cid[1:-1]), subtype='html') +# note that we needed to peel the <> off the msgid for use in the html. + +# Now add the related image to the html part. +with open("roasted-asparagus.jpg", 'rb') as img: + msg.get_payload()[1].add_related(img.read(), 'image', 'jpeg', + cid=asparagus_cid) + +# Make a local copy of what we are going to send. +with open('outgoing.msg', 'wb') as f: + f.write(bytes(msg)) + +# Send the message via local SMTP server. +with smtplib.SMTP('localhost') as s: + s.send_message(msg) diff --git a/Doc/includes/email-dir.py b/Doc/includes/email-dir.py index cc5529e..3c7c770 100644 --- a/Doc/includes/email-dir.py +++ b/Doc/includes/email-dir.py @@ -8,7 +8,7 @@ import smtplib # For guessing MIME type based on file name extension import mimetypes -from optparse import OptionParser +from argparse import ArgumentParser from email import encoders from email.message import Message @@ -22,44 +22,36 @@ COMMASPACE = ', ' def main(): - parser = OptionParser(usage="""\ + parser = ArgumentParser(description="""\ Send the contents of a directory as a MIME message. - -Usage: %prog [options] - Unless the -o option is given, the email is sent by forwarding to your local SMTP server, which then does the normal delivery process. Your local machine must be running an SMTP server. """) - parser.add_option('-d', '--directory', - type='string', action='store', - help="""Mail the contents of the specified directory, - otherwise use the current directory. Only the regular - files in the directory are sent, and we don't recurse to - subdirectories.""") - parser.add_option('-o', '--output', - type='string', action='store', metavar='FILE', - help="""Print the composed message to FILE instead of - sending the message to the SMTP server.""") - parser.add_option('-s', '--sender', - type='string', action='store', metavar='SENDER', - help='The value of the From: header (required)') - parser.add_option('-r', '--recipient', - type='string', action='append', metavar='RECIPIENT', - default=[], dest='recipients', - help='A To: header value (at least one required)') - opts, args = parser.parse_args() - if not opts.sender or not opts.recipients: - parser.print_help() - sys.exit(1) - directory = opts.directory + parser.add_argument('-d', '--directory', + help="""Mail the contents of the specified directory, + otherwise use the current directory. Only the regular + files in the directory are sent, and we don't recurse to + subdirectories.""") + parser.add_argument('-o', '--output', + metavar='FILE', + help="""Print the composed message to FILE instead of + sending the message to the SMTP server.""") + parser.add_argument('-s', '--sender', required=True, + help='The value of the From: header (required)') + parser.add_argument('-r', '--recipient', required=True, + action='append', metavar='RECIPIENT', + default=[], dest='recipients', + help='A To: header value (at least one required)') + args = parser.parse_args() + directory = args.directory if not directory: directory = '.' # Create the enclosing (outer) message outer = MIMEMultipart() outer['Subject'] = 'Contents of directory %s' % os.path.abspath(directory) - outer['To'] = COMMASPACE.join(opts.recipients) - outer['From'] = opts.sender + outer['To'] = COMMASPACE.join(args.recipients) + outer['From'] = args.sender outer.preamble = 'You will not see this in a MIME-aware mail reader.\n' for filename in os.listdir(directory): @@ -76,23 +68,19 @@ must be running an SMTP server. ctype = 'application/octet-stream' maintype, subtype = ctype.split('/', 1) if maintype == 'text': - fp = open(path) - # Note: we should handle calculating the charset - msg = MIMEText(fp.read(), _subtype=subtype) - fp.close() + with open(path) as fp: + # Note: we should handle calculating the charset + msg = MIMEText(fp.read(), _subtype=subtype) elif maintype == 'image': - fp = open(path, 'rb') - msg = MIMEImage(fp.read(), _subtype=subtype) - fp.close() + with open(path, 'rb') as fp: + msg = MIMEImage(fp.read(), _subtype=subtype) elif maintype == 'audio': - fp = open(path, 'rb') - msg = MIMEAudio(fp.read(), _subtype=subtype) - fp.close() + with open(path, 'rb') as fp: + msg = MIMEAudio(fp.read(), _subtype=subtype) else: - fp = open(path, 'rb') - msg = MIMEBase(maintype, subtype) - msg.set_payload(fp.read()) - fp.close() + with open(path, 'rb') as fp: + msg = MIMEBase(maintype, subtype) + msg.set_payload(fp.read()) # Encode the payload using Base64 encoders.encode_base64(msg) # Set the filename parameter @@ -100,14 +88,12 @@ must be running an SMTP server. outer.attach(msg) # Now send or store the message composed = outer.as_string() - if opts.output: - fp = open(opts.output, 'w') - fp.write(composed) - fp.close() + if args.output: + with open(args.output, 'w') as fp: + fp.write(composed) else: - s = smtplib.SMTP('localhost') - s.sendmail(opts.sender, opts.recipients, composed) - s.quit() + with smtplib.SMTP('localhost') as s: + s.sendmail(args.sender, args.recipients, composed) if __name__ == '__main__': diff --git a/Doc/includes/email-read-alternative-new-api.py b/Doc/includes/email-read-alternative-new-api.py new file mode 100644 index 0000000..8ab4e9f --- /dev/null +++ b/Doc/includes/email-read-alternative-new-api.py @@ -0,0 +1,74 @@ +import os +import sys +import tempfile +import mimetypes +import webbrowser + +# Import the email modules we'll need +from email import policy +from email.parser import BytesParser + +# An imaginary module that would make this work and be safe. +from imaginary import magic_html_parser + +# In a real program you'd get the filename from the arguments. +msg = BytesParser(policy=policy.default).parse(open('outgoing.msg', 'rb')) + +# Now the header items can be accessed as a dictionary, and any non-ASCII will +# be converted to unicode: +print('To:', msg['to']) +print('From:', msg['from']) +print('Subject:', msg['subject']) + +# If we want to print a priview of the message content, we can extract whatever +# the least formatted payload is and print the first three lines. Of course, +# if the message has no plain text part printing the first three lines of html +# is probably useless, but this is just a conceptual example. +simplest = msg.get_body(preferencelist=('plain', 'html')) +print() +print(''.join(simplest.get_content().splitlines(keepends=True)[:3])) + +ans = input("View full message?") +if ans.lower()[0] == 'n': + sys.exit() + +# We can extract the richest alternative in order to display it: +richest = msg.get_body() +partfiles = {} +if richest['content-type'].maintype == 'text': + if richest['content-type'].subtype == 'plain': + for line in richest.get_content().splitlines(): + print(line) + sys.exit() + elif richest['content-type'].subtype == 'html': + body = richest + else: + print("Don't know how to display {}".format(richest.get_content_type())) + sys.exit() +elif richest['content-type'].content_type == 'multipart/related': + body = richest.get_body(preferencelist=('html')) + for part in richest.iter_attachments(): + fn = part.get_filename() + if fn: + extension = os.path.splitext(part.get_filename())[1] + else: + extension = mimetypes.guess_extension(part.get_content_type()) + with tempfile.NamedTemporaryFile(suffix=extension, delete=False) as f: + f.write(part.get_content()) + # again strip the <> to go from email form of cid to html form. + partfiles[part['content-id'][1:-1]] = f.name +else: + print("Don't know how to display {}".format(richest.get_content_type())) + sys.exit() +with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: + # The magic_html_parser has to rewrite the href="cid:...." attributes to + # point to the filenames in partfiles. It also has to do a safety-sanitize + # of the html. It could be written using html.parser. + f.write(magic_html_parser(body.get_content(), partfiles)) +webbrowser.open(f.name) +os.remove(f.name) +for fn in partfiles.values(): + os.remove(fn) + +# Of course, there are lots of email messages that could break this simple +# minded program, but it will handle the most common ones. diff --git a/Doc/includes/email-unpack.py b/Doc/includes/email-unpack.py index 3653543..574a0b6 100644 --- a/Doc/includes/email-unpack.py +++ b/Doc/includes/email-unpack.py @@ -8,41 +8,27 @@ import email import errno import mimetypes -from optparse import OptionParser +from argparse import ArgumentParser def main(): - parser = OptionParser(usage="""\ + parser = ArgumentParser(description="""\ Unpack a MIME message into a directory of files. - -Usage: %prog [options] msgfile """) - parser.add_option('-d', '--directory', - type='string', action='store', - help="""Unpack the MIME message into the named - directory, which will be created if it doesn't already - exist.""") - opts, args = parser.parse_args() - if not opts.directory: - parser.print_help() - sys.exit(1) + parser.add_argument('-d', '--directory', required=True, + help="""Unpack the MIME message into the named + directory, which will be created if it doesn't already + exist.""") + parser.add_argument('msgfile') + args = parser.parse_args() - try: - msgfile = args[0] - except IndexError: - parser.print_help() - sys.exit(1) + with open(args.msgfile) as fp: + msg = email.message_from_file(fp) try: - os.mkdir(opts.directory) - except OSError as e: - # Ignore directory exists error - if e.errno != errno.EEXIST: - raise - - fp = open(msgfile) - msg = email.message_from_file(fp) - fp.close() + os.mkdir(args.directory) + except FileExistsError: + pass counter = 1 for part in msg.walk(): @@ -59,9 +45,8 @@ Usage: %prog [options] msgfile ext = '.bin' filename = 'part-%03d%s' % (counter, ext) counter += 1 - fp = open(os.path.join(opts.directory, filename), 'wb') - fp.write(part.get_payload(decode=True)) - fp.close() + with open(os.path.join(args.directory, filename), 'wb') as fp: + fp.write(part.get_payload(decode=True)) if __name__ == '__main__': diff --git a/Doc/includes/mp_benchmarks.py b/Doc/includes/mp_benchmarks.py deleted file mode 100644 index 3763ea9..0000000 --- a/Doc/includes/mp_benchmarks.py +++ /dev/null @@ -1,239 +0,0 @@ -# -# Simple benchmarks for the multiprocessing package -# -# Copyright (c) 2006-2008, R Oudkerk -# All rights reserved. -# - -import time -import multiprocessing -import threading -import queue -import gc - -_timer = time.perf_counter - -delta = 1 - - -#### TEST_QUEUESPEED - -def queuespeed_func(q, c, iterations): - a = '0' * 256 - c.acquire() - c.notify() - c.release() - - for i in range(iterations): - q.put(a) - - q.put('STOP') - -def test_queuespeed(Process, q, c): - elapsed = 0 - iterations = 1 - - while elapsed < delta: - iterations *= 2 - - p = Process(target=queuespeed_func, args=(q, c, iterations)) - c.acquire() - p.start() - c.wait() - c.release() - - result = None - t = _timer() - - while result != 'STOP': - result = q.get() - - elapsed = _timer() - t - - p.join() - - print(iterations, 'objects passed through the queue in', elapsed, 'seconds') - print('average number/sec:', iterations/elapsed) - - -#### TEST_PIPESPEED - -def pipe_func(c, cond, iterations): - a = '0' * 256 - cond.acquire() - cond.notify() - cond.release() - - for i in range(iterations): - c.send(a) - - c.send('STOP') - -def test_pipespeed(): - c, d = multiprocessing.Pipe() - cond = multiprocessing.Condition() - elapsed = 0 - iterations = 1 - - while elapsed < delta: - iterations *= 2 - - p = multiprocessing.Process(target=pipe_func, - args=(d, cond, iterations)) - cond.acquire() - p.start() - cond.wait() - cond.release() - - result = None - t = _timer() - - while result != 'STOP': - result = c.recv() - - elapsed = _timer() - t - p.join() - - print(iterations, 'objects passed through connection in',elapsed,'seconds') - print('average number/sec:', iterations/elapsed) - - -#### TEST_SEQSPEED - -def test_seqspeed(seq): - elapsed = 0 - iterations = 1 - - while elapsed < delta: - iterations *= 2 - - t = _timer() - - for i in range(iterations): - a = seq[5] - - elapsed = _timer() - t - - print(iterations, 'iterations in', elapsed, 'seconds') - print('average number/sec:', iterations/elapsed) - - -#### TEST_LOCK - -def test_lockspeed(l): - elapsed = 0 - iterations = 1 - - while elapsed < delta: - iterations *= 2 - - t = _timer() - - for i in range(iterations): - l.acquire() - l.release() - - elapsed = _timer() - t - - print(iterations, 'iterations in', elapsed, 'seconds') - print('average number/sec:', iterations/elapsed) - - -#### TEST_CONDITION - -def conditionspeed_func(c, N): - c.acquire() - c.notify() - - for i in range(N): - c.wait() - c.notify() - - c.release() - -def test_conditionspeed(Process, c): - elapsed = 0 - iterations = 1 - - while elapsed < delta: - iterations *= 2 - - c.acquire() - p = Process(target=conditionspeed_func, args=(c, iterations)) - p.start() - - c.wait() - - t = _timer() - - for i in range(iterations): - c.notify() - c.wait() - - elapsed = _timer() - t - - c.release() - p.join() - - print(iterations * 2, 'waits in', elapsed, 'seconds') - print('average number/sec:', iterations * 2 / elapsed) - -#### - -def test(): - manager = multiprocessing.Manager() - - gc.disable() - - print('\n\t######## testing Queue.Queue\n') - test_queuespeed(threading.Thread, queue.Queue(), - threading.Condition()) - print('\n\t######## testing multiprocessing.Queue\n') - test_queuespeed(multiprocessing.Process, multiprocessing.Queue(), - multiprocessing.Condition()) - print('\n\t######## testing Queue managed by server process\n') - test_queuespeed(multiprocessing.Process, manager.Queue(), - manager.Condition()) - print('\n\t######## testing multiprocessing.Pipe\n') - test_pipespeed() - - print() - - print('\n\t######## testing list\n') - test_seqspeed(list(range(10))) - print('\n\t######## testing list managed by server process\n') - test_seqspeed(manager.list(list(range(10)))) - print('\n\t######## testing Array("i", ..., lock=False)\n') - test_seqspeed(multiprocessing.Array('i', list(range(10)), lock=False)) - print('\n\t######## testing Array("i", ..., lock=True)\n') - test_seqspeed(multiprocessing.Array('i', list(range(10)), lock=True)) - - print() - - print('\n\t######## testing threading.Lock\n') - test_lockspeed(threading.Lock()) - print('\n\t######## testing threading.RLock\n') - test_lockspeed(threading.RLock()) - print('\n\t######## testing multiprocessing.Lock\n') - test_lockspeed(multiprocessing.Lock()) - print('\n\t######## testing multiprocessing.RLock\n') - test_lockspeed(multiprocessing.RLock()) - print('\n\t######## testing lock managed by server process\n') - test_lockspeed(manager.Lock()) - print('\n\t######## testing rlock managed by server process\n') - test_lockspeed(manager.RLock()) - - print() - - print('\n\t######## testing threading.Condition\n') - test_conditionspeed(threading.Thread, threading.Condition()) - print('\n\t######## testing multiprocessing.Condition\n') - test_conditionspeed(multiprocessing.Process, multiprocessing.Condition()) - print('\n\t######## testing condition managed by a server process\n') - test_conditionspeed(multiprocessing.Process, manager.Condition()) - - gc.enable() - -if __name__ == '__main__': - multiprocessing.freeze_support() - test() diff --git a/Doc/includes/mp_newtype.py b/Doc/includes/mp_newtype.py index 7291743..1c796a5 100644 --- a/Doc/includes/mp_newtype.py +++ b/Doc/includes/mp_newtype.py @@ -1,11 +1,3 @@ -# -# This module shows how to use arbitrary callables with a subclass of -# `BaseManager`. -# -# Copyright (c) 2006-2008, R Oudkerk -# All rights reserved. -# - from multiprocessing import freeze_support from multiprocessing.managers import BaseManager, BaseProxy import operator @@ -27,12 +19,10 @@ def baz(): # Proxy type for generator objects class GeneratorProxy(BaseProxy): - _exposed_ = ('next', '__next__') + _exposed_ = ['__next__'] def __iter__(self): return self def __next__(self): - return self._callmethod('next') - def __next__(self): return self._callmethod('__next__') # Function to return the operator module @@ -90,8 +80,6 @@ def test(): op = manager.operator() print('op.add(23, 45) =', op.add(23, 45)) print('op.pow(2, 94) =', op.pow(2, 94)) - print('op.getslice(range(10), 2, 6) =', op.getslice(list(range(10)), 2, 6)) - print('op.repeat(range(5), 3) =', op.repeat(list(range(5)), 3)) print('op._exposed_ =', op._exposed_) ## diff --git a/Doc/includes/mp_pool.py b/Doc/includes/mp_pool.py index 1578498..11101e1 100644 --- a/Doc/includes/mp_pool.py +++ b/Doc/includes/mp_pool.py @@ -1,10 +1,3 @@ -# -# A test of `multiprocessing.Pool` class -# -# Copyright (c) 2006-2008, R Oudkerk -# All rights reserved. -# - import multiprocessing import time import random @@ -46,269 +39,115 @@ def noop(x): # def test(): - print('cpu_count() = %d\n' % multiprocessing.cpu_count()) - - # - # Create pool - # - PROCESSES = 4 print('Creating pool with %d processes\n' % PROCESSES) - pool = multiprocessing.Pool(PROCESSES) - print('pool = %s' % pool) - print() - - # - # Tests - # - - TASKS = [(mul, (i, 7)) for i in range(10)] + \ - [(plus, (i, 8)) for i in range(10)] - - results = [pool.apply_async(calculate, t) for t in TASKS] - imap_it = pool.imap(calculatestar, TASKS) - imap_unordered_it = pool.imap_unordered(calculatestar, TASKS) - - print('Ordered results using pool.apply_async():') - for r in results: - print('\t', r.get()) - print() - - print('Ordered results using pool.imap():') - for x in imap_it: - print('\t', x) - print() - - print('Unordered results using pool.imap_unordered():') - for x in imap_unordered_it: - print('\t', x) - print() - - print('Ordered results using pool.map() --- will block till complete:') - for x in pool.map(calculatestar, TASKS): - print('\t', x) - print() - - # - # Simple benchmarks - # - - N = 100000 - print('def pow3(x): return x**3') - t = time.time() - A = list(map(pow3, range(N))) - print('\tmap(pow3, range(%d)):\n\t\t%s seconds' % \ - (N, time.time() - t)) + with multiprocessing.Pool(PROCESSES) as pool: + # + # Tests + # - t = time.time() - B = pool.map(pow3, range(N)) - print('\tpool.map(pow3, range(%d)):\n\t\t%s seconds' % \ - (N, time.time() - t)) + TASKS = [(mul, (i, 7)) for i in range(10)] + \ + [(plus, (i, 8)) for i in range(10)] - t = time.time() - C = list(pool.imap(pow3, range(N), chunksize=N//8)) - print('\tlist(pool.imap(pow3, range(%d), chunksize=%d)):\n\t\t%s' \ - ' seconds' % (N, N//8, time.time() - t)) + results = [pool.apply_async(calculate, t) for t in TASKS] + imap_it = pool.imap(calculatestar, TASKS) + imap_unordered_it = pool.imap_unordered(calculatestar, TASKS) - assert A == B == C, (len(A), len(B), len(C)) - print() + print('Ordered results using pool.apply_async():') + for r in results: + print('\t', r.get()) + print() - L = [None] * 1000000 - print('def noop(x): pass') - print('L = [None] * 1000000') + print('Ordered results using pool.imap():') + for x in imap_it: + print('\t', x) + print() - t = time.time() - A = list(map(noop, L)) - print('\tmap(noop, L):\n\t\t%s seconds' % \ - (time.time() - t)) + print('Unordered results using pool.imap_unordered():') + for x in imap_unordered_it: + print('\t', x) + print() - t = time.time() - B = pool.map(noop, L) - print('\tpool.map(noop, L):\n\t\t%s seconds' % \ - (time.time() - t)) + print('Ordered results using pool.map() --- will block till complete:') + for x in pool.map(calculatestar, TASKS): + print('\t', x) + print() - t = time.time() - C = list(pool.imap(noop, L, chunksize=len(L)//8)) - print('\tlist(pool.imap(noop, L, chunksize=%d)):\n\t\t%s seconds' % \ - (len(L)//8, time.time() - t)) + # + # Test error handling + # - assert A == B == C, (len(A), len(B), len(C)) - print() + print('Testing error handling:') - del A, B, C, L - - # - # Test error handling - # - - print('Testing error handling:') - - try: - print(pool.apply(f, (5,))) - except ZeroDivisionError: - print('\tGot ZeroDivisionError as expected from pool.apply()') - else: - raise AssertionError('expected ZeroDivisionError') - - try: - print(pool.map(f, list(range(10)))) - except ZeroDivisionError: - print('\tGot ZeroDivisionError as expected from pool.map()') - else: - raise AssertionError('expected ZeroDivisionError') - - try: - print(list(pool.imap(f, list(range(10))))) - except ZeroDivisionError: - print('\tGot ZeroDivisionError as expected from list(pool.imap())') - else: - raise AssertionError('expected ZeroDivisionError') - - it = pool.imap(f, list(range(10))) - for i in range(10): try: - x = next(it) + print(pool.apply(f, (5,))) except ZeroDivisionError: - if i == 5: - pass - except StopIteration: - break + print('\tGot ZeroDivisionError as expected from pool.apply()') else: - if i == 5: - raise AssertionError('expected ZeroDivisionError') - - assert i == 9 - print('\tGot ZeroDivisionError as expected from IMapIterator.next()') - print() + raise AssertionError('expected ZeroDivisionError') - # - # Testing timeouts - # - - print('Testing ApplyResult.get() with timeout:', end=' ') - res = pool.apply_async(calculate, TASKS[0]) - while 1: - sys.stdout.flush() try: - sys.stdout.write('\n\t%s' % res.get(0.02)) - break - except multiprocessing.TimeoutError: - sys.stdout.write('.') - print() - print() + print(pool.map(f, list(range(10)))) + except ZeroDivisionError: + print('\tGot ZeroDivisionError as expected from pool.map()') + else: + raise AssertionError('expected ZeroDivisionError') - print('Testing IMapIterator.next() with timeout:', end=' ') - it = pool.imap(calculatestar, TASKS) - while 1: - sys.stdout.flush() try: - sys.stdout.write('\n\t%s' % it.next(0.02)) - except StopIteration: - break - except multiprocessing.TimeoutError: - sys.stdout.write('.') - print() - print() - - # - # Testing callback - # - - print('Testing callback:') - - A = [] - B = [56, 0, 1, 8, 27, 64, 125, 216, 343, 512, 729] - - r = pool.apply_async(mul, (7, 8), callback=A.append) - r.wait() - - r = pool.map_async(pow3, list(range(10)), callback=A.extend) - r.wait() - - if A == B: - print('\tcallbacks succeeded\n') - else: - print('\t*** callbacks failed\n\t\t%s != %s\n' % (A, B)) - - # - # Check there are no outstanding tasks - # - - assert not pool._cache, 'cache = %r' % pool._cache - - # - # Check close() methods - # - - print('Testing close():') - - for worker in pool._pool: - assert worker.is_alive() - - result = pool.apply_async(time.sleep, [0.5]) - pool.close() - pool.join() - - assert result.get() is None - - for worker in pool._pool: - assert not worker.is_alive() - - print('\tclose() succeeded\n') - - # - # Check terminate() method - # - - print('Testing terminate():') - - pool = multiprocessing.Pool(2) - DELTA = 0.1 - ignore = pool.apply(pow3, [2]) - results = [pool.apply_async(time.sleep, [DELTA]) for i in range(100)] - pool.terminate() - pool.join() - - for worker in pool._pool: - assert not worker.is_alive() - - print('\tterminate() succeeded\n') - - # - # Check garbage collection - # - - print('Testing garbage collection:') - - pool = multiprocessing.Pool(2) - DELTA = 0.1 - processes = pool._pool - ignore = pool.apply(pow3, [2]) - results = [pool.apply_async(time.sleep, [DELTA]) for i in range(100)] - - results = pool = None - - time.sleep(DELTA * 2) - - for worker in processes: - assert not worker.is_alive() - - print('\tgarbage collection succeeded\n') + print(list(pool.imap(f, list(range(10))))) + except ZeroDivisionError: + print('\tGot ZeroDivisionError as expected from list(pool.imap())') + else: + raise AssertionError('expected ZeroDivisionError') + + it = pool.imap(f, list(range(10))) + for i in range(10): + try: + x = next(it) + except ZeroDivisionError: + if i == 5: + pass + except StopIteration: + break + else: + if i == 5: + raise AssertionError('expected ZeroDivisionError') + + assert i == 9 + print('\tGot ZeroDivisionError as expected from IMapIterator.next()') + print() + + # + # Testing timeouts + # + + print('Testing ApplyResult.get() with timeout:', end=' ') + res = pool.apply_async(calculate, TASKS[0]) + while 1: + sys.stdout.flush() + try: + sys.stdout.write('\n\t%s' % res.get(0.02)) + break + except multiprocessing.TimeoutError: + sys.stdout.write('.') + print() + print() + + print('Testing IMapIterator.next() with timeout:', end=' ') + it = pool.imap(calculatestar, TASKS) + while 1: + sys.stdout.flush() + try: + sys.stdout.write('\n\t%s' % it.next(0.02)) + except StopIteration: + break + except multiprocessing.TimeoutError: + sys.stdout.write('.') + print() + print() if __name__ == '__main__': multiprocessing.freeze_support() - - assert len(sys.argv) in (1, 2) - - if len(sys.argv) == 1 or sys.argv[1] == 'processes': - print(' Using processes '.center(79, '-')) - elif sys.argv[1] == 'threads': - print(' Using threads '.center(79, '-')) - import multiprocessing.dummy as multiprocessing - else: - print('Usage:\n\t%s [processes | threads]' % sys.argv[0]) - raise SystemExit(2) - test() diff --git a/Doc/includes/mp_synchronize.py b/Doc/includes/mp_synchronize.py deleted file mode 100644 index 81dbc38..0000000 --- a/Doc/includes/mp_synchronize.py +++ /dev/null @@ -1,278 +0,0 @@ -# -# A test file for the `multiprocessing` package -# -# Copyright (c) 2006-2008, R Oudkerk -# All rights reserved. -# - -import time -import sys -import random -from queue import Empty - -import multiprocessing # may get overwritten - - -#### TEST_VALUE - -def value_func(running, mutex): - random.seed() - time.sleep(random.random()*4) - - mutex.acquire() - print('\n\t\t\t' + str(multiprocessing.current_process()) + ' has finished') - running.value -= 1 - mutex.release() - -def test_value(): - TASKS = 10 - running = multiprocessing.Value('i', TASKS) - mutex = multiprocessing.Lock() - - for i in range(TASKS): - p = multiprocessing.Process(target=value_func, args=(running, mutex)) - p.start() - - while running.value > 0: - time.sleep(0.08) - mutex.acquire() - print(running.value, end=' ') - sys.stdout.flush() - mutex.release() - - print() - print('No more running processes') - - -#### TEST_QUEUE - -def queue_func(queue): - for i in range(30): - time.sleep(0.5 * random.random()) - queue.put(i*i) - queue.put('STOP') - -def test_queue(): - q = multiprocessing.Queue() - - p = multiprocessing.Process(target=queue_func, args=(q,)) - p.start() - - o = None - while o != 'STOP': - try: - o = q.get(timeout=0.3) - print(o, end=' ') - sys.stdout.flush() - except Empty: - print('TIMEOUT') - - print() - - -#### TEST_CONDITION - -def condition_func(cond): - cond.acquire() - print('\t' + str(cond)) - time.sleep(2) - print('\tchild is notifying') - print('\t' + str(cond)) - cond.notify() - cond.release() - -def test_condition(): - cond = multiprocessing.Condition() - - p = multiprocessing.Process(target=condition_func, args=(cond,)) - print(cond) - - cond.acquire() - print(cond) - cond.acquire() - print(cond) - - p.start() - - print('main is waiting') - cond.wait() - print('main has woken up') - - print(cond) - cond.release() - print(cond) - cond.release() - - p.join() - print(cond) - - -#### TEST_SEMAPHORE - -def semaphore_func(sema, mutex, running): - sema.acquire() - - mutex.acquire() - running.value += 1 - print(running.value, 'tasks are running') - mutex.release() - - random.seed() - time.sleep(random.random()*2) - - mutex.acquire() - running.value -= 1 - print('%s has finished' % multiprocessing.current_process()) - mutex.release() - - sema.release() - -def test_semaphore(): - sema = multiprocessing.Semaphore(3) - mutex = multiprocessing.RLock() - running = multiprocessing.Value('i', 0) - - processes = [ - multiprocessing.Process(target=semaphore_func, - args=(sema, mutex, running)) - for i in range(10) - ] - - for p in processes: - p.start() - - for p in processes: - p.join() - - -#### TEST_JOIN_TIMEOUT - -def join_timeout_func(): - print('\tchild sleeping') - time.sleep(5.5) - print('\n\tchild terminating') - -def test_join_timeout(): - p = multiprocessing.Process(target=join_timeout_func) - p.start() - - print('waiting for process to finish') - - while 1: - p.join(timeout=1) - if not p.is_alive(): - break - print('.', end=' ') - sys.stdout.flush() - - -#### TEST_EVENT - -def event_func(event): - print('\t%r is waiting' % multiprocessing.current_process()) - event.wait() - print('\t%r has woken up' % multiprocessing.current_process()) - -def test_event(): - event = multiprocessing.Event() - - processes = [multiprocessing.Process(target=event_func, args=(event,)) - for i in range(5)] - - for p in processes: - p.start() - - print('main is sleeping') - time.sleep(2) - - print('main is setting event') - event.set() - - for p in processes: - p.join() - - -#### TEST_SHAREDVALUES - -def sharedvalues_func(values, arrays, shared_values, shared_arrays): - for i in range(len(values)): - v = values[i][1] - sv = shared_values[i].value - assert v == sv - - for i in range(len(values)): - a = arrays[i][1] - sa = list(shared_arrays[i][:]) - assert a == sa - - print('Tests passed') - -def test_sharedvalues(): - values = [ - ('i', 10), - ('h', -2), - ('d', 1.25) - ] - arrays = [ - ('i', list(range(100))), - ('d', [0.25 * i for i in range(100)]), - ('H', list(range(1000))) - ] - - shared_values = [multiprocessing.Value(id, v) for id, v in values] - shared_arrays = [multiprocessing.Array(id, a) for id, a in arrays] - - p = multiprocessing.Process( - target=sharedvalues_func, - args=(values, arrays, shared_values, shared_arrays) - ) - p.start() - p.join() - - assert p.exitcode == 0 - - -#### - -def test(namespace=multiprocessing): - global multiprocessing - - multiprocessing = namespace - - for func in [test_value, test_queue, test_condition, - test_semaphore, test_join_timeout, test_event, - test_sharedvalues]: - - print('\n\t######## %s\n' % func.__name__) - func() - - ignore = multiprocessing.active_children() # cleanup any old processes - if hasattr(multiprocessing, '_debug_info'): - info = multiprocessing._debug_info() - if info: - print(info) - raise ValueError('there should be no positive refcounts left') - - -if __name__ == '__main__': - multiprocessing.freeze_support() - - assert len(sys.argv) in (1, 2) - - if len(sys.argv) == 1 or sys.argv[1] == 'processes': - print(' Using processes '.center(79, '-')) - namespace = multiprocessing - elif sys.argv[1] == 'manager': - print(' Using processes and a manager '.center(79, '-')) - namespace = multiprocessing.Manager() - namespace.Process = multiprocessing.Process - namespace.current_process = multiprocessing.current_process - namespace.active_children = multiprocessing.active_children - elif sys.argv[1] == 'threads': - print(' Using threads '.center(79, '-')) - import multiprocessing.dummy as namespace - else: - print('Usage:\n\t%s [processes | manager | threads]' % sys.argv[0]) - raise SystemExit(2) - - test(namespace) diff --git a/Doc/includes/mp_webserver.py b/Doc/includes/mp_webserver.py deleted file mode 100644 index 651024d..0000000 --- a/Doc/includes/mp_webserver.py +++ /dev/null @@ -1,70 +0,0 @@ -# -# Example where a pool of http servers share a single listening socket -# -# On Windows this module depends on the ability to pickle a socket -# object so that the worker processes can inherit a copy of the server -# object. (We import `multiprocessing.reduction` to enable this pickling.) -# -# Not sure if we should synchronize access to `socket.accept()` method by -# using a process-shared lock -- does not seem to be necessary. -# -# Copyright (c) 2006-2008, R Oudkerk -# All rights reserved. -# - -import os -import sys - -from multiprocessing import Process, current_process, freeze_support -from http.server import HTTPServer -from http.server import SimpleHTTPRequestHandler - -if sys.platform == 'win32': - import multiprocessing.reduction # make sockets pickable/inheritable - - -def note(format, *args): - sys.stderr.write('[%s]\t%s\n' % (current_process().name, format % args)) - - -class RequestHandler(SimpleHTTPRequestHandler): - # we override log_message() to show which process is handling the request - def log_message(self, format, *args): - note(format, *args) - -def serve_forever(server): - note('starting server') - try: - server.serve_forever() - except KeyboardInterrupt: - pass - - -def runpool(address, number_of_processes): - # create a single server object -- children will each inherit a copy - server = HTTPServer(address, RequestHandler) - - # create child processes to act as workers - for i in range(number_of_processes - 1): - Process(target=serve_forever, args=(server,)).start() - - # main process also acts as a worker - serve_forever(server) - - -def test(): - DIR = os.path.join(os.path.dirname(__file__), '..') - ADDRESS = ('localhost', 8000) - NUMBER_OF_PROCESSES = 4 - - print('Serving at http://%s:%d using %d worker processes' % \ - (ADDRESS[0], ADDRESS[1], NUMBER_OF_PROCESSES)) - print('To exit press Ctrl-' + ['C', 'Break'][sys.platform=='win32']) - - os.chdir(DIR) - runpool(ADDRESS, NUMBER_OF_PROCESSES) - - -if __name__ == '__main__': - freeze_support() - test() diff --git a/Doc/includes/mp_workers.py b/Doc/includes/mp_workers.py index e66d97b..3b92269 100644 --- a/Doc/includes/mp_workers.py +++ b/Doc/includes/mp_workers.py @@ -1,16 +1,3 @@ -# -# Simple example which uses a pool of workers to carry out some tasks. -# -# Notice that the results will probably not come out of the output -# queue in the same in the same order as the corresponding tasks were -# put on the input queue. If it is important to get the results back -# in the original order then consider using `Pool.map()` or -# `Pool.imap()` (which will save on the amount of code needed anyway). -# -# Copyright (c) 2006-2008, R Oudkerk -# All rights reserved. -# - import time import random diff --git a/Doc/includes/typestruct.h b/Doc/includes/typestruct.h index 32647c0..fcb846a 100644 --- a/Doc/includes/typestruct.h +++ b/Doc/includes/typestruct.h @@ -70,4 +70,11 @@ typedef struct _typeobject { PyObject *tp_subclasses; PyObject *tp_weaklist; + destructor tp_del; + + /* Type attribute cache version tag. Added in version 2.6 */ + unsigned int tp_version_tag; + + destructor tp_finalize; + } PyTypeObject; diff --git a/Doc/install/index.rst b/Doc/install/index.rst index 0c9a9f2..d0fea3d 100644 --- a/Doc/install/index.rst +++ b/Doc/install/index.rst @@ -2,9 +2,9 @@ .. _install-index: -***************************** - Installing Python Modules -***************************** +******************************************** + Installing Python Modules (Legacy version) +******************************************** :Author: Greg Ward diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst new file mode 100644 index 0000000..1284613 --- /dev/null +++ b/Doc/installing/index.rst @@ -0,0 +1,210 @@ +.. highlightlang:: none + +.. _installing-index: + +***************************** + Installing Python Modules +***************************** + +:Email: distutils-sig@python.org + +As a popular open source development project, Python has an active +supporting community of contributors and users that also make their software +available for other Python developers to use under open source license terms. + +This allows Python users to share and collaborate effectively, benefiting +from the solutions others have already created to common (and sometimes +even rare!) problems, as well as potentially contributing their own +solutions to the common pool. + +This guide covers the installation part of the process. For a guide to +creating and sharing your own Python projects, refer to the +:ref:`distribution guide <distributing-index>`. + +.. note:: + + For corporate and other institutional users, be aware that many + organisations have their own policies around using and contributing to + open source software. Please take such policies into account when making + use of the distribution and installation tools provided with Python. + + +Key terms +========= + +* ``pip`` is the preferred installer program. Starting with Python 3.4, it + is included by default with the Python binary installers. +* a virtual environment is a semi-isolated Python environment that allows + packages to be installed for use by a particular application, rather than + being installed system wide +* ``pyvenv`` is the standard tool for creating virtual environments, and has + been part of Python since Python 3.3. Starting with Python 3.4, it + defaults to installing ``pip`` into all created virtual environments +* the `Python Package Index <https://pypi.python.org/pypi>`__ is a public + repository of open source licensed packages made available for use by + other Python users +* the `Python Packaging Authority + <http://packaging.python.org/en/latest/future.html>`__ are the group of + developers and documentation authors responsible for the maintenance and + evolution of the standard packaging tools and the associated metadata and + file format standards. They maintain a variety of tools, documentation + and issue trackers on both `GitHub <https://github.com/pypa>`__ and + `BitBucket <https://bitbucket.org/pypa/>`__. +* ``distutils`` is the original build and distribution system first added to + the Python standard library in 1998. While direct use of ``distutils`` is + being phased out, it still laid the foundation for the current packaging + and distribution infrastructure, and it not only remains part of the + standard library, but its name lives on in other ways (such as the name + of the mailing list used to coordinate Python packaging standards + development). + + +Basic usage +=========== + +The standard packaging tools are all designed to be used from the command +line. For Windows users, the examples below assume that the option to +adjust the system PATH environment variable was selected when installing +Python. For Linux users, the command to install into the system version of +Python 3 is likely to be ``pip3`` rather than ``pip``. + +The following command will install the latest version of a module and its +dependencies from the Python Package Index:: + + pip install SomePackage + +It's also possible to specify an exact or minimum version directly on the +command line:: + + pip install SomePackage==1.0.4 # specific version + pip install 'SomePackage>=1.0.4' # minimum version + +Normally, if a suitable module is already installed, attempting to install +it again will have no effect. Upgrading existing modules must be requested +explicitly:: + + pip install --upgrade SomePackage + +More information and resources regarding ``pip`` and its capabilities can be +found in the `Python Packaging User Guide <http://packaging.python.org>`__. + +``pyvenv`` has its own documentation at :ref:`scripts-pyvenv`. Installing +into an active virtual environment uses the commands shown above. + +.. seealso:: + + `Python Packaging User Guide: Installing Python packages + <http://packaging.python.org/en/latest/tutorial.html#installing-python-packages>`__ + + +How do I ...? +============= + +These are quick answers or links for some common tasks. + +... install ``pip`` in versions of Python prior to Python 3.4? +-------------------------------------------------------------- + +Python only started bundling ``pip`` with Python 3.4. For earlier versions, +``pip`` needs to be "bootstrapped" as described in the Python Packaging +User Guide. + +.. seealso:: + + `Python Packaging User Guide: Installing the Tools + <http://packaging.python.org/en/latest/tutorial.html#installing-the-tools>`__ + + +.. installing-per-user-installation: + +... install packages just for the current user? +----------------------------------------------- + +Passing the ``--user`` option to ``pip install`` will install a package +just for the current user, rather than for all users of the system. + + +... install scientific Python packages? +--------------------------------------- + +A number of scientific Python packages have complex binary dependencies, and +aren't currently easy to install using ``pip`` directly. At this point in +time, it will often be easier for users to install these packages by +`other means +<http://packaging.python.org/en/latest/platforms.html#installing-scientific-packages>`__ +rather than attempting to install them with ``pip``. + +.. seealso:: + + `Python Packaging User Guide: Installing Scientific Packages + <http://packaging.python.org/en/latest/platforms.html#installing-scientific-packages>`__ + + +... work with multiple versions of Python installed in parallel? +---------------------------------------------------------------- + +On Linux, Mac OS X and other POSIX systems, use the versioned Python commands +in combination with the ``-m`` switch to run the appropriate copy of +``pip``:: + + python2 -m pip install SomePackage # default Python 2 + python2.7 -m pip install SomePackage # specifically Python 2.7 + python3 -m pip install SomePackage # default Python 3 + python3.4 -m pip install SomePackage # specifically Python 3.4 + +(appropriately versioned ``pip`` commands may also be available) + +On Windows, use the ``py`` Python launcher in combination with the ``-m`` +switch:: + + py -2 -m pip install SomePackage # default Python 2 + py -2.7 -m pip install SomePackage # specifically Python 2.7 + py -3 -m pip install SomePackage # default Python 3 + py -3.4 -m pip install SomePackage # specifically Python 3.4 + +.. other questions: + + Once the Development & Deployment part of PPUG is fleshed out, some of + those sections should be linked from new questions here (most notably, + we should have a question about avoiding depending on PyPI that links to + http://packaging.python.org/en/latest/deployment.html#pypi-mirrors-and-caches) + + +Common installation issues +========================== + +Installing into the system Python on Linux +------------------------------------------ + +On Linux systems, a Python installation will typically be included as part +of the distribution. Installing into this Python installation requires +root access to the system, and may interfere with the operation of the +system package manager and other components of the system if a component +is unexpectedly upgraded using ``pip``. + +On such systems, it is often better to use a virtual environment or a +per-user installation when installing packages with ``pip``. + + +Installing binary extensions +---------------------------- + +Python has typically relied heavily on source based distribution, with end +users being expected to compile extension modules from source as part of +the installation process. + +With the introduction of support for the binary ``wheel`` format, and the +ability to publish wheels for at least Windows and Mac OS X through the +Python Package Index, this problem is expected to diminish over time, +as users are more regularly able to install pre-built extensions rather +than needing to build them themselves. + +Some of the solutions for installing `scientific software +<http://packaging.python.org/en/latest/platforms.html#installing-scientific-packages>`__ +that is not yet available as pre-built ``wheel`` files may also help with +obtaining other binary extensions without needing to build them locally. + +.. seealso:: + + `Python Packaging User Guide: Binary Extensions + <http://packaging.python.org/en/latest/extensions.html>`__ diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst index 1e5f42d..6473861 100644 --- a/Doc/library/2to3.rst +++ b/Doc/library/2to3.rst @@ -142,6 +142,39 @@ and off individually. They are described here in more detail. Removes usage of :func:`apply`. For example ``apply(function, *args, **kwargs)`` is converted to ``function(*args, **kwargs)``. +.. 2to3fixer:: asserts + + Replaces deprecated :mod:`unittest` method names with the correct ones. + + ================================ ========================================== + From To + ================================ ========================================== + ``failUnlessEqual(a, b)`` :meth:`assertEqual(a, b) + <unittest.TestCase.assertEqual>` + ``assertEquals(a, b)`` :meth:`assertEqual(a, b) + <unittest.TestCase.assertEqual>` + ``failIfEqual(a, b)`` :meth:`assertNotEqual(a, b) + <unittest.TestCase.assertNotEqual>` + ``assertNotEquals(a, b)`` :meth:`assertNotEqual(a, b) + <unittest.TestCase.assertNotEqual>` + ``failUnless(a)`` :meth:`assertTrue(a) + <unittest.TestCase.assertTrue>` + ``assert_(a)`` :meth:`assertTrue(a) + <unittest.TestCase.assertTrue>` + ``failIf(a)`` :meth:`assertFalse(a) + <unittest.TestCase.assertFalse>` + ``failUnlessRaises(exc, cal)`` :meth:`assertRaises(exc, cal) + <unittest.TestCase.assertRaises>` + ``failUnlessAlmostEqual(a, b)`` :meth:`assertAlmostEqual(a, b) + <unittest.TestCase.assertAlmostEqual>` + ``assertAlmostEquals(a, b)`` :meth:`assertAlmostEqual(a, b) + <unittest.TestCase.assertAlmostEqual>` + ``failIfAlmostEqual(a, b)`` :meth:`assertNotAlmostEqual(a, b) + <unittest.TestCase.assertNotAlmostEqual>` + ``assertNotAlmostEquals(a, b)`` :meth:`assertNotAlmostEqual(a, b) + <unittest.TestCase.assertNotAlmostEqual>` + ================================ ========================================== + .. 2to3fixer:: basestring Converts :class:`basestring` to :class:`str`. @@ -342,6 +375,10 @@ and off individually. They are described here in more detail. Handles the move of :func:`reduce` to :func:`functools.reduce`. +.. 2to3fixer:: reload + + Converts :func:`reload` to :func:`imp.reload`. + .. 2to3fixer:: renames Changes :data:`sys.maxint` to :data:`sys.maxsize`. @@ -355,7 +392,7 @@ and off individually. They are described here in more detail. Replaces use of the :class:`set` constructor with set literals. This fixer is optional. -.. 2to3fixer:: standard_error +.. 2to3fixer:: standarderror Renames :exc:`StandardError` to :exc:`Exception`. diff --git a/Doc/library/__main__.rst b/Doc/library/__main__.rst index a1d3c24..fcbe53d 100644 --- a/Doc/library/__main__.rst +++ b/Doc/library/__main__.rst @@ -5,13 +5,19 @@ .. module:: __main__ :synopsis: The environment where the top-level script is run. +``'__main__'`` is the name of the scope in which top-level code executes. +A module's __name__ is set equal to ``'__main__'`` when read from +standard input, a script, or from an interactive prompt. -This module represents the (otherwise anonymous) scope in which the -interpreter's main program executes --- commands read either from standard -input, from a script file, or from an interactive prompt. It is this -environment in which the idiomatic "conditional script" stanza causes a script -to run:: +A module can discover whether or not it is running in the main scope by +checking its own ``__name__``, which allows a common idiom for conditionally +executing code in a module when it is run as a script or with ``python +-m`` but not when it is imported: if __name__ == "__main__": + # execute only if run as a script main() +For a package, the same effect can be achieved by including a +``__main__.py`` module, the contents of which will be executed when the +module is run with ``-m``. diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index 9299124..7a73704 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -12,9 +12,9 @@ -------------- This module provides the infrastructure for defining :term:`abstract base -classes <abstract base class>` (ABCs) in Python, as outlined in :pep:`3119`; see the PEP for why this -was added to Python. (See also :pep:`3141` and the :mod:`numbers` module -regarding a type hierarchy for numbers based on ABCs.) +classes <abstract base class>` (ABCs) in Python, as outlined in :pep:`3119`; +see the PEP for why this was added to Python. (See also :pep:`3141` and the +:mod:`numbers` module regarding a type hierarchy for numbers based on ABCs.) The :mod:`collections` module has some concrete classes that derive from ABCs; these can, of course, be further derived. In addition the @@ -23,7 +23,7 @@ a class or instance provides a particular interface, for example, is it hashable or a mapping. -This module provides the following class: +This module provides the following classes: .. class:: ABCMeta @@ -58,6 +58,10 @@ This module provides the following class: .. versionchanged:: 3.3 Returns the registered subclass, to allow usage as a class decorator. + .. versionchanged:: 3.4 + To detect calls to :meth:`register`, you can use the + :func:`get_cache_token` function. + You can also override this method in an abstract base class: .. method:: __subclasshook__(subclass) @@ -127,6 +131,19 @@ This module provides the following class: available as a method of ``Foo``, so it is provided separately. +.. class:: ABC + + A helper class that has :class:`ABCMeta` as its metaclass. With this class, + an abstract base class can be created by simply deriving from :class:`ABC`, + avoiding sometimes confusing metaclass usage. + + Note that the type of :class:`ABC` is still :class:`ABCMeta`, therefore + inheriting from :class:`ABC` requires the usual precautions regarding metaclass + usage, as multiple inheritance may lead to metaclass conflicts. + + .. versionadded:: 3.4 + + The :mod:`abc` module also provides the following decorators: .. decorator:: abstractmethod @@ -295,6 +312,19 @@ The :mod:`abc` module also provides the following decorators: :func:`abstractmethod`, making this decorator redundant. +The :mod:`abc` module also provides the following functions: + +.. function:: get_cache_token() + + Returns the current abstract base class cache token. + + The token is an opaque object (that supports equality testing) identifying + the current version of the abstract base class cache for virtual subclasses. + The token changes with every call to :meth:`ABCMeta.register` on any ABC. + + .. versionadded:: 3.4 + + .. rubric:: Footnotes .. [#] C++ programmers should note that Python's virtual base class diff --git a/Doc/library/aifc.rst b/Doc/library/aifc.rst index 8a3541b..6fbcf28 100644 --- a/Doc/library/aifc.rst +++ b/Doc/library/aifc.rst @@ -30,8 +30,8 @@ sampling rate or frame rate is the number of times per second the sound is sampled. The number of channels indicate if the audio is mono, stereo, or quadro. Each frame consists of one sample per channel. The sample size is the size in bytes of each sample. Thus a frame consists of -*nchannels*\*\ *samplesize* bytes, and a second's worth of audio consists of -*nchannels*\*\ *samplesize*\*\ *framerate* bytes. +``nchannels * samplesize`` bytes, and a second's worth of audio consists of +``nchannels * samplesize * framerate`` bytes. For example, CD quality audio has a sample size of two bytes (16 bits), uses two channels (stereo) and has a frame rate of 44,100 frames/second. This gives a @@ -51,6 +51,11 @@ Module :mod:`aifc` defines the following function: used for writing, the file object should be seekable, unless you know ahead of time how many samples you are going to write in total and use :meth:`writeframesraw` and :meth:`setnframes`. + The :func:`.open` function may be used in a :keyword:`with` statement. When + the :keyword:`with` block completes, the :meth:`~aifc.close` method is called. + + .. versionchanged:: 3.4 + Support for the :keyword:`with` statement was added. Objects returned by :func:`.open` when a file is opened for reading have the following methods: @@ -92,7 +97,9 @@ following methods: .. method:: aifc.getparams() - Return a tuple consisting of all of the above values in the above order. + Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth, + framerate, nframes, comptype, compname)``, equivalent to output of the + :meth:`get\*` methods. .. method:: aifc.getmarkers() @@ -218,12 +225,18 @@ number of frames must be filled in. Write data to the output file. This method can only be called after the audio file parameters have been set. + .. versionchanged:: 3.4 + Any :term:`bytes-like object` is now accepted. + .. method:: aifc.writeframesraw(data) Like :meth:`writeframes`, except that the header of the audio file is not updated. + .. versionchanged:: 3.4 + Any :term:`bytes-like object` is now accepted. + .. method:: aifc.close() diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index b686c44..b1b5135 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -977,9 +977,9 @@ See the section on the default_ keyword argument for information on when the ``type`` argument is applied to default arguments. To ease the use of various types of files, the argparse module provides the -factory FileType which takes the ``mode=`` and ``bufsize=`` arguments of the -:func:`open` function. For example, ``FileType('w')`` can be used to create a -writable file:: +factory FileType which takes the ``mode=``, ``bufsize=``, ``encoding=`` and +``errors=`` arguments of the :func:`open` function. For example, +``FileType('w')`` can be used to create a writable file:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('bar', type=argparse.FileType('w')) @@ -1648,17 +1648,19 @@ Sub-commands FileType objects ^^^^^^^^^^^^^^^^ -.. class:: FileType(mode='r', bufsize=None) +.. class:: FileType(mode='r', bufsize=-1, encoding=None, errors=None) The :class:`FileType` factory creates objects that can be passed to the type argument of :meth:`ArgumentParser.add_argument`. Arguments that have - :class:`FileType` objects as their type will open command-line arguments as files - with the requested modes and buffer sizes:: + :class:`FileType` objects as their type will open command-line arguments as + files with the requested modes, buffer sizes, encodings and error handling + (see the :func:`open` function for more details):: >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--output', type=argparse.FileType('wb', 0)) - >>> parser.parse_args(['--output', 'out']) - Namespace(output=<_io.BufferedWriter name='out'>) + >>> parser.add_argument('--raw', type=argparse.FileType('wb', 0)) + >>> parser.add_argument('out', type=argparse.FileType('w', encoding='UTF-8')) + >>> parser.parse_args(['--raw', 'raw.dat', 'file.txt']) + Namespace(out=<_io.TextIOWrapper name='file.txt' mode='w' encoding='UTF-8'>, raw=<_io.FileIO name='raw.dat' mode='wb'>) FileType objects understand the pseudo-argument ``'-'`` and automatically convert this into ``sys.stdin`` for readable :class:`FileType` objects and @@ -1669,6 +1671,9 @@ FileType objects >>> parser.parse_args(['-']) Namespace(infile=<_io.TextIOWrapper name='<stdin>' encoding='UTF-8'>) + .. versionadded:: 3.4 + The *encodings* and *errors* keyword arguments. + Argument groups ^^^^^^^^^^^^^^^ diff --git a/Doc/library/asynchat.rst b/Doc/library/asynchat.rst index adbd3be..c6fa061 100644 --- a/Doc/library/asynchat.rst +++ b/Doc/library/asynchat.rst @@ -10,6 +10,11 @@ -------------- +.. note:: + + This module exists for backwards compatibility only. For new code we + recommend using :mod:`asyncio`. + This module builds on the :mod:`asyncore` infrastructure, simplifying asynchronous clients and servers and making it easier to handle protocols whose elements are terminated by arbitrary strings, or are of variable length. diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst new file mode 100644 index 0000000..9d6f054 --- /dev/null +++ b/Doc/library/asyncio-dev.rst @@ -0,0 +1,251 @@ +.. currentmodule:: asyncio + +.. _asyncio-dev: + +Develop with asyncio +==================== + +Asynchronous programming is different than classical "sequential" programming. +This page lists common traps and explains how to avoid them. + + +.. _asyncio-multithreading: + +Concurrency and multithreading +------------------------------ + +An event loop runs in a thread and executes all callbacks and tasks in the same +thread. While a task is running in the event loop, no other task is running in +the same thread. But when the task uses ``yield from``, the task is suspended +and the event loop executes the next task. + +To schedule a callback from a different thread, the +:meth:`BaseEventLoop.call_soon_threadsafe` method should be used. Example to +schedule a coroutine from a different thread:: + + loop.call_soon_threadsafe(asyncio.async, coro_func()) + +Most asyncio objects are not thread safe. You should only worry if you access +objects outside the event loop. For example, to cancel a future, don't call +directly its :meth:`Future.cancel` method, but:: + + loop.call_soon_threadsafe(fut.cancel) + +To handle signals and to execute subprocesses, the event loop must be run in +the main thread. + +The :meth:`BaseEventLoop.run_in_executor` method can be used with a thread pool +executor to execute a callback in different thread to not block the thread of +the event loop. + +.. seealso:: + + See the :ref:`Synchronization primitives <asyncio-sync>` section to + synchronize tasks. + + +.. _asyncio-handle-blocking: + +Handle blocking functions correctly +----------------------------------- + +Blocking functions should not be called directly. For example, if a function +blocks for 1 second, other tasks are delayed by 1 second which can have an +important impact on reactivity. + +For networking and subprocesses, the :mod:`asyncio` module provides high-level +APIs like :ref:`protocols <asyncio-protocol>`. + +An executor can be used to run a task in a different thread or even in a +different process, to not block the thread of the event loop. See the +:meth:`BaseEventLoop.run_in_executor` method. + +.. seealso:: + + The :ref:`Delayed calls <asyncio-delayed-calls>` section details how the + event loop handles time. + + +.. _asyncio-logger: + +Logging +------- + +The :mod:`asyncio` module logs information with the :mod:`logging` module in +the logger ``'asyncio'``. + + +.. _asyncio-coroutine-not-scheduled: + +Detect coroutine objects never scheduled +---------------------------------------- + +When a coroutine function is called but not passed to :func:`async` or to the +:class:`Task` constructor, it is not scheduled and it is probably a bug. + +To detect such bug, set the environment variable :envvar:`PYTHONASYNCIODEBUG` +to ``1``. When the coroutine object is destroyed by the garbage collector, a +log will be emitted with the traceback where the coroutine function was called. +See the :ref:`asyncio logger <asyncio-logger>`. + +The debug flag changes the behaviour of the :func:`coroutine` decorator. The +debug flag value is only used when then coroutine function is defined, not when +it is called. Coroutine functions defined before the debug flag is set to +``True`` will not be tracked. For example, it is not possible to debug +coroutines defined in the :mod:`asyncio` module, because the module must be +imported before the flag value can be changed. + +Example with the bug:: + + import asyncio + + @asyncio.coroutine + def test(): + print("never scheduled") + + test() + +Output in debug mode:: + + Coroutine 'test' defined at test.py:4 was never yielded from + +The fix is to call the :func:`async` function or create a :class:`Task` object +with this coroutine object. + + +Detect exceptions not consumed +------------------------------ + +Python usually calls :func:`sys.displayhook` on unhandled exceptions. If +:meth:`Future.set_exception` is called, but the exception is not consumed, +:func:`sys.displayhook` is not called. Instead, a log is emitted when the +future is deleted by the garbage collector, with the traceback where the +exception was raised. See the :ref:`asyncio logger <asyncio-logger>`. + +Example of unhandled exception:: + + import asyncio + + @asyncio.coroutine + def bug(): + raise Exception("not consumed") + + loop = asyncio.get_event_loop() + asyncio.async(bug()) + loop.run_forever() + +Output:: + + Future/Task exception was never retrieved: + Traceback (most recent call last): + File "/usr/lib/python3.4/asyncio/tasks.py", line 279, in _step + result = next(coro) + File "/usr/lib/python3.4/asyncio/tasks.py", line 80, in coro + res = func(*args, **kw) + File "test.py", line 5, in bug + raise Exception("not consumed") + Exception: not consumed + +There are different options to fix this issue. The first option is to chain to +coroutine in another coroutine and use classic try/except:: + + @asyncio.coroutine + def handle_exception(): + try: + yield from bug() + except Exception: + print("exception consumed") + + loop = asyncio.get_event_loop() + asyncio.async(handle_exception()) + loop.run_forever() + +Another option is to use the :meth:`BaseEventLoop.run_until_complete` +function:: + + task = asyncio.async(bug()) + try: + loop.run_until_complete(task) + except Exception: + print("exception consumed") + +See also the :meth:`Future.exception` method. + + +Chain coroutines correctly +-------------------------- + +When a coroutine function calls other coroutine functions and tasks, they +should be chained explicitly with ``yield from``. Otherwise, the execution is +not guaranteed to be sequential. + +Example with different bugs using :func:`asyncio.sleep` to simulate slow +operations:: + + import asyncio + + @asyncio.coroutine + def create(): + yield from asyncio.sleep(3.0) + print("(1) create file") + + @asyncio.coroutine + def write(): + yield from asyncio.sleep(1.0) + print("(2) write into file") + + @asyncio.coroutine + def close(): + print("(3) close file") + + @asyncio.coroutine + def test(): + asyncio.async(create()) + asyncio.async(write()) + asyncio.async(close()) + yield from asyncio.sleep(2.0) + loop.stop() + + loop = asyncio.get_event_loop() + asyncio.async(test()) + loop.run_forever() + print("Pending tasks at exit: %s" % asyncio.Task.all_tasks(loop)) + loop.close() + +Expected output:: + + (1) create file + (2) write into file + (3) close file + Pending tasks at exit: set() + +Actual output:: + + (3) close file + (2) write into file + Pending tasks at exit: {Task(<create>)<PENDING>} + +The loop stopped before the ``create()`` finished, ``close()`` has been called +before ``write()``, whereas coroutine functions were called in this order: +``create()``, ``write()``, ``close()``. + +To fix the example, tasks must be marked with ``yield from``:: + + @asyncio.coroutine + def test(): + yield from asyncio.async(create()) + yield from asyncio.async(write()) + yield from asyncio.async(close()) + yield from asyncio.sleep(2.0) + loop.stop() + +Or without ``asyncio.async()``:: + + @asyncio.coroutine + def test(): + yield from create() + yield from write() + yield from close() + yield from asyncio.sleep(2.0) + loop.stop() + diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst new file mode 100644 index 0000000..7717b7a --- /dev/null +++ b/Doc/library/asyncio-eventloop.rst @@ -0,0 +1,674 @@ +.. currentmodule:: asyncio + +.. _asyncio-event-loop: + +Event loops +=========== + +The event loop is the central execution device provided by :mod:`asyncio`. +It provides multiple facilities, amongst which: + +* Registering, executing and cancelling delayed calls (timeouts). + +* Creating client and server :ref:`transports <asyncio-transport>` for various + kinds of communication. + +* Launching subprocesses and the associated :ref:`transports + <asyncio-transport>` for communication with an external program. + +* Delegating costly function calls to a pool of threads. + +Event loop policies and the default policy +------------------------------------------ + +Event loop management is abstracted with a *policy* pattern, to provide maximal +flexibility for custom platforms and frameworks. Throughout the execution of a +process, a single global policy object manages the event loops available to the +process based on the calling context. A policy is an object implementing the +:class:`AbstractEventLoopPolicy` interface. + +For most users of :mod:`asyncio`, policies never have to be dealt with +explicitly, since the default global policy is sufficient. + +The default policy defines context as the current thread, and manages an event +loop per thread that interacts with :mod:`asyncio`. The module-level functions +:func:`get_event_loop` and :func:`set_event_loop` provide convenient access to +event loops managed by the default policy. + +Event loop functions +-------------------- + +The following functions are convenient shortcuts to accessing the methods of the +global policy. Note that this provides access to the default policy, unless an +alternative policy was set by calling :func:`set_event_loop_policy` earlier in +the execution of the process. + +.. function:: get_event_loop() + + Equivalent to calling ``get_event_loop_policy().get_event_loop()``. + +.. function:: set_event_loop(loop) + + Equivalent to calling ``get_event_loop_policy().set_event_loop(loop)``. + +.. function:: new_event_loop() + + Equivalent to calling ``get_event_loop_policy().new_event_loop()``. + +Event loop policy interface +--------------------------- + +An event loop policy must implement the following interface: + +.. class:: AbstractEventLoopPolicy + + .. method:: get_event_loop() + + Get the event loop for the current context. Returns an event loop object + implementing the :class:`BaseEventLoop` interface, or raises an exception in case + no event loop has been set for the current context and the current policy + does not specify to create one. It should never return ``None``. + + .. method:: set_event_loop(loop) + + Set the event loop for the current context to *loop*. + + .. method:: new_event_loop() + + Create and return a new event loop object according to this policy's rules. + If there's need to set this loop as the event loop for the current context, + :meth:`set_event_loop` must be called explicitly. + +Access to the global loop policy +-------------------------------- + +.. function:: get_event_loop_policy() + + Get the current event loop policy. + +.. function:: set_event_loop_policy(policy) + + Set the current event loop policy. If *policy* is ``None``, the default + policy is restored. + +Run an event loop +----------------- + +.. method:: BaseEventLoop.run_forever() + + Run until :meth:`stop` is called. + +.. method:: BaseEventLoop.run_until_complete(future) + + Run until the :class:`Future` is done. + + If the argument is a :ref:`coroutine <coroutine>`, it is wrapped + in a :class:`Task`. + + Return the Future's result, or raise its exception. + +.. method:: BaseEventLoop.is_running() + + Returns running status of event loop. + +.. method:: BaseEventLoop.stop() + + Stop running the event loop. + + Every callback scheduled before :meth:`stop` is called will run. + Callback scheduled after :meth:`stop` is called won't. However, those + callbacks will run if :meth:`run_forever` is called again later. + +.. method:: BaseEventLoop.is_closed() + + Returns ``True`` if the event loop was closed. + + .. versionadded:: 3.4.2 + +.. method:: BaseEventLoop.close() + + Close the event loop. The loop should not be running. + + This clears the queues and shuts down the executor, but does not wait for + the executor to finish. + + This is idempotent and irreversible. No other methods should be called after + this one. + + +Calls +----- + +.. method:: BaseEventLoop.call_soon(callback, \*args) + + Arrange for a callback to be called as soon as possible. + + This operates as a FIFO queue, callbacks are called in the order in + which they are registered. Each callback will be called exactly once. + + Any positional arguments after the callback will be passed to the + callback when it is called. + + An instance of :class:`asyncio.Handle` is returned. + +.. method:: BaseEventLoop.call_soon_threadsafe(callback, \*args) + + Like :meth:`call_soon`, but thread safe. + + +.. _asyncio-delayed-calls: + +Delayed calls +------------- + +The event loop has its own internal clock for computing timeouts. +Which clock is used depends on the (platform-specific) event loop +implementation; ideally it is a monotonic clock. This will generally be +a different clock than :func:`time.time`. + +.. note:: + + Timeouts (relative *delay* or absolute *when*) should not exceed one day. + + +.. method:: BaseEventLoop.call_later(delay, callback, *args) + + Arrange for the *callback* to be called after the given *delay* + seconds (either an int or float). + + An instance of :class:`asyncio.Handle` is returned. + + *callback* will be called exactly once per call to :meth:`call_later`. + If two callbacks are scheduled for exactly the same time, it is + undefined which will be called first. + + The optional positional *args* will be passed to the callback when it + is called. If you want the callback to be called with some named + arguments, use a closure or :func:`functools.partial`. + +.. method:: BaseEventLoop.call_at(when, callback, *args) + + Arrange for the *callback* to be called at the given absolute timestamp + *when* (an int or float), using the same time reference as :meth:`time`. + + This method's behavior is the same as :meth:`call_later`. + +.. method:: BaseEventLoop.time() + + Return the current time, as a :class:`float` value, according to the + event loop's internal clock. + +.. seealso:: + + The :func:`asyncio.sleep` function. + + +Creating connections +-------------------- + +.. method:: BaseEventLoop.create_connection(protocol_factory, host=None, port=None, \*, ssl=None, family=0, proto=0, flags=0, sock=None, local_addr=None, server_hostname=None) + + Create a streaming transport connection to a given Internet *host* and + *port*: socket family :py:data:`~socket.AF_INET` or + :py:data:`~socket.AF_INET6` depending on *host* (or *family* if specified), + socket type :py:data:`~socket.SOCK_STREAM`. *protocol_factory* must be a + callable returning a :ref:`protocol <asyncio-protocol>` instance. + + This method is a :ref:`coroutine <coroutine>` which will try to + establish the connection in the background. When successful, the + coroutine returns a ``(transport, protocol)`` pair. + + The chronological synopsis of the underlying operation is as follows: + + #. The connection is established, and a :ref:`transport <asyncio-transport>` + is created to represent it. + + #. *protocol_factory* is called without arguments and must return a + :ref:`protocol <asyncio-protocol>` instance. + + #. The protocol instance is tied to the transport, and its + :meth:`connection_made` method is called. + + #. The coroutine returns successfully with the ``(transport, protocol)`` + pair. + + The created transport is an implementation-dependent bidirectional stream. + + .. note:: + *protocol_factory* can be any kind of callable, not necessarily + a class. For example, if you want to use a pre-created + protocol instance, you can pass ``lambda: my_protocol``. + + Options allowing to change how the connection is created: + + * *ssl*: if given and not false, a SSL/TLS transport is created + (by default a plain TCP transport is created). If *ssl* is + a :class:`ssl.SSLContext` object, this context is used to create + the transport; if *ssl* is :const:`True`, a context with some + unspecified default settings is used. + + .. seealso:: :ref:`SSL/TLS security considerations <ssl-security>` + + * *server_hostname*, is only for use together with *ssl*, + and sets or overrides the hostname that the target server's certificate + will be matched against. By default the value of the *host* argument + is used. If *host* is empty, there is no default and you must pass a + value for *server_hostname*. If *server_hostname* is an empty + string, hostname matching is disabled (which is a serious security + risk, allowing for man-in-the-middle-attacks). + + * *family*, *proto*, *flags* are the optional address family, protocol + and flags to be passed through to getaddrinfo() for *host* resolution. + If given, these should all be integers from the corresponding + :mod:`socket` module constants. + + * *sock*, if given, should be an existing, already connected + :class:`socket.socket` object to be used by the transport. + If *sock* is given, none of *host*, *port*, *family*, *proto*, *flags* + and *local_addr* should be specified. + + * *local_addr*, if given, is a ``(local_host, local_port)`` tuple used + to bind the socket to locally. The *local_host* and *local_port* + are looked up using getaddrinfo(), similarly to *host* and *port*. + + .. seealso:: + + The :func:`open_connection` function can be used to get a pair of + (:class:`StreamReader`, :class:`StreamWriter`) instead of a protocol. + + +.. method:: BaseEventLoop.create_datagram_endpoint(protocol_factory, local_addr=None, remote_addr=None, \*, family=0, proto=0, flags=0) + + Create datagram connection: socket family :py:data:`~socket.AF_INET` or + :py:data:`~socket.AF_INET6` depending on *host* (or *family* if specified), + socket type :py:data:`~socket.SOCK_DGRAM`. + + This method is a :ref:`coroutine <coroutine>` which will try to + establish the connection in the background. When successful, the + coroutine returns a ``(transport, protocol)`` pair. + + See the :meth:`BaseEventLoop.create_connection` method for parameters. + + +.. method:: BaseEventLoop.create_unix_connection(protocol_factory, path, \*, ssl=None, sock=None, server_hostname=None) + + Create UNIX connection: socket family :py:data:`~socket.AF_UNIX`, socket + type :py:data:`~socket.SOCK_STREAM`. The :py:data:`~socket.AF_UNIX` socket + family is used to communicate between processes on the same machine + efficiently. + + This method is a :ref:`coroutine <coroutine>` which will try to + establish the connection in the background. When successful, the + coroutine returns a ``(transport, protocol)`` pair. + + See the :meth:`BaseEventLoop.create_connection` method for parameters. + + Availability: UNIX. + + +Creating listening connections +------------------------------ + +.. method:: BaseEventLoop.create_server(protocol_factory, host=None, port=None, \*, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None) + + A :ref:`coroutine <coroutine>` method which creates a TCP server bound to + host and port. + + The return value is a :class:`AbstractServer` object which can be used to stop + the service. + + If *host* is an empty string or None all interfaces are assumed + and a list of multiple sockets will be returned (most likely + one for IPv4 and another one for IPv6). + + *family* can be set to either :data:`~socket.AF_INET` or + :data:`~socket.AF_INET6` to force the socket to use IPv4 or IPv6. If not set + it will be determined from host (defaults to :data:`~socket.AF_UNSPEC`). + + *flags* is a bitmask for :meth:`getaddrinfo`. + + *sock* can optionally be specified in order to use a preexisting + socket object. + + *backlog* is the maximum number of queued connections passed to + :meth:`~socket.socket.listen` (defaults to 100). + + ssl can be set to an :class:`~ssl.SSLContext` to enable SSL over the + accepted connections. + + *reuse_address* tells the kernel to reuse a local socket in + TIME_WAIT state, without waiting for its natural timeout to + expire. If not specified will automatically be set to True on + UNIX. + + .. seealso:: + + The function :func:`start_server` creates a (:class:`StreamReader`, + :class:`StreamWriter`) pair and calls back a function with this pair. + + +.. method:: BaseEventLoop.create_unix_server(protocol_factory, path=None, \*, sock=None, backlog=100, ssl=None) + + Similar to :meth:`BaseEventLoop.create_server`, but specific to the + socket family :py:data:`~socket.AF_UNIX`. + + Availability: UNIX. + + + +Watch file descriptors +---------------------- + +.. method:: BaseEventLoop.add_reader(fd, callback, \*args) + + Start watching the file descriptor for read availability and then call the + *callback* with specified arguments. + +.. method:: BaseEventLoop.remove_reader(fd) + + Stop watching the file descriptor for read availability. + +.. method:: BaseEventLoop.add_writer(fd, callback, \*args) + + Start watching the file descriptor for write availability and then call the + *callback* with specified arguments. + +.. method:: BaseEventLoop.remove_writer(fd) + + Stop watching the file descriptor for write availability. + + +Low-level socket operations +--------------------------- + +.. method:: BaseEventLoop.sock_recv(sock, nbytes) + + Receive data from the socket. The return value is a bytes object + representing the data received. The maximum amount of data to be received + at once is specified by *nbytes*. + + This method is a :ref:`coroutine <coroutine>`. + + .. seealso:: + + The :meth:`socket.socket.recv` method. + +.. method:: BaseEventLoop.sock_sendall(sock, data) + + Send data to the socket. The socket must be connected to a remote socket. + This method continues to send data from *data* until either all data has + been sent or an error occurs. ``None`` is returned on success. On error, + an exception is raised, and there is no way to determine how much data, if + any, was successfully processed by the receiving end of the connection. + + This method is a :ref:`coroutine <coroutine>`. + + .. seealso:: + + The :meth:`socket.socket.sendall` method. + +.. method:: BaseEventLoop.sock_connect(sock, address) + + Connect to a remote socket at *address*. + + The *address* must be already resolved to avoid the trap of hanging the + entire event loop when the address requires doing a DNS lookup. For + example, it must be an IP address, not an hostname, for + :py:data:`~socket.AF_INET` and :py:data:`~socket.AF_INET6` address families. + Use :meth:`getaddrinfo` to resolve the hostname asynchronously. + + This method is a :ref:`coroutine <coroutine>`. + + .. seealso:: + + The :meth:`BaseEventLoop.create_connection` method, the + :func:`open_connection` function and the :meth:`socket.socket.connect` + method. + + +.. method:: BaseEventLoop.sock_accept(sock) + + Accept a connection. The socket must be bound to an address and listening + for connections. The return value is a pair ``(conn, address)`` where *conn* + is a *new* socket object usable to send and receive data on the connection, + and *address* is the address bound to the socket on the other end of the + connection. + + This method is a :ref:`coroutine <coroutine>`. + + .. seealso:: + + The :meth:`BaseEventLoop.create_server` method, the :func:`start_server` + function and the :meth:`socket.socket.accept` method. + + +Resolve host name +----------------- + +.. method:: BaseEventLoop.getaddrinfo(host, port, \*, family=0, type=0, proto=0, flags=0) + + This method is a :ref:`coroutine <coroutine>`, similar to + :meth:`socket.getaddrinfo` function but non-blocking. + +.. method:: BaseEventLoop.getnameinfo(sockaddr, flags=0) + + This method is a :ref:`coroutine <coroutine>`, similar to + :meth:`socket.getnameinfo` function but non-blocking. + + +Connect pipes +------------- + +.. method:: BaseEventLoop.connect_read_pipe(protocol_factory, pipe) + + Register read pipe in eventloop. Set the *pipe* to non-blocking mode. + + *protocol_factory* should instantiate object with :class:`Protocol` + interface. *pipe* is a :term:`file-like object <file object>`. + Return pair ``(transport, protocol)``, where *transport* supports the + :class:`ReadTransport` interface. + + This method is a :ref:`coroutine <coroutine>`. + +.. method:: BaseEventLoop.connect_write_pipe(protocol_factory, pipe) + + Register write pipe in eventloop. + + *protocol_factory* should instantiate object with :class:`BaseProtocol` + interface. Pipe is file-like object already switched to nonblocking. + Return pair (transport, protocol), where transport support + :class:`WriteTransport` interface. + + This method is a :ref:`coroutine <coroutine>`. + +.. seealso:: + + The :meth:`BaseEventLoop.subprocess_exec` and + :meth:`BaseEventLoop.subprocess_shell` methods. + + +UNIX signals +------------ + +Availability: UNIX only. + +.. method:: BaseEventLoop.add_signal_handler(signum, callback, \*args) + + Add a handler for a signal. + + Raise :exc:`ValueError` if the signal number is invalid or uncatchable. + Raise :exc:`RuntimeError` if there is a problem setting up the handler. + +.. method:: BaseEventLoop.remove_signal_handler(sig) + + Remove a handler for a signal. + + Return ``True`` if a signal handler was removed, ``False`` if not. + +.. seealso:: + + The :mod:`signal` module. + + +Executor +-------- + +Call a function in an :class:`~concurrent.futures.Executor` (pool of threads or +pool of processes). By default, an event loop uses a thread pool executor +(:class:`~concurrent.futures.ThreadPoolExecutor`). + +.. method:: BaseEventLoop.run_in_executor(executor, callback, \*args) + + Arrange for a callback to be called in the specified executor. + + The *executor* argument should be an :class:`~concurrent.futures.Executor` + instance. The default executor is used if *executor* is ``None``. + + This method is a :ref:`coroutine <coroutine>`. + +.. method:: BaseEventLoop.set_default_executor(executor) + + Set the default executor used by :meth:`run_in_executor`. + + +Error Handling API +------------------ + +Allows to customize how exceptions are handled in the event loop. + +.. method:: BaseEventLoop.set_exception_handler(handler) + + Set *handler* as the new event loop exception handler. + + If *handler* is ``None``, the default exception handler will + be set. + + If *handler* is a callable object, it should have a + matching signature to ``(loop, context)``, where ``loop`` + will be a reference to the active event loop, ``context`` + will be a ``dict`` object (see :meth:`call_exception_handler` + documentation for details about context). + +.. method:: BaseEventLoop.default_exception_handler(context) + + Default exception handler. + + This is called when an exception occurs and no exception + handler is set, and can be called by a custom exception + handler that wants to defer to the default behavior. + + *context* parameter has the same meaning as in + :meth:`call_exception_handler`. + +.. method:: BaseEventLoop.call_exception_handler(context) + + Call the current event loop exception handler. + + *context* is a ``dict`` object containing the following keys + (new keys may be introduced later): + + * 'message': Error message; + * 'exception' (optional): Exception object; + * 'future' (optional): :class:`asyncio.Future` instance; + * 'handle' (optional): :class:`asyncio.Handle` instance; + * 'protocol' (optional): :ref:`Protocol <asyncio-protocol>` instance; + * 'transport' (optional): :ref:`Transport <asyncio-transport>` instance; + * 'socket' (optional): :class:`socket.socket` instance. + + .. note:: + + Note: this method should not be overloaded in subclassed + event loops. For any custom exception handling, use + :meth:`set_exception_handler()` method. + +Debug mode +---------- + +.. method:: BaseEventLoop.get_debug() + + Get the debug mode (:class:`bool`) of the event loop, ``False`` by default. + +.. method:: BaseEventLoop.set_debug(enabled: bool) + + Set the debug mode of the event loop. + +.. seealso:: + + The :ref:`Develop with asyncio <asyncio-dev>` section. + + +Server +------ + +.. class:: AbstractServer + + Abstract server returned by :func:`BaseEventLoop.create_server`. + + .. method:: close() + + Stop serving. This leaves existing connections open. + + .. method:: wait_closed() + + A :ref:`coroutine <coroutine>` to wait until service is closed. + + +Handle +------ + +.. class:: Handle + + A callback wrapper object returned by :func:`BaseEventLoop.call_soon`, + :func:`BaseEventLoop.call_soon_threadsafe`, :func:`BaseEventLoop.call_later`, + and :func:`BaseEventLoop.call_at`. + + .. method:: cancel() + + Cancel the call. + + +.. _asyncio-hello-world-callback: + +Example: Hello World (callback) +------------------------------- + +Print ``Hello World`` every two seconds, using a callback:: + + import asyncio + + def print_and_repeat(loop): + print('Hello World') + loop.call_later(2, print_and_repeat, loop) + + loop = asyncio.get_event_loop() + loop.call_soon(print_and_repeat, loop) + loop.run_forever() + +.. seealso:: + + :ref:`Hello World example using a coroutine <asyncio-hello-world-coroutine>`. + + +Example: Set signal handlers for SIGINT and SIGTERM +--------------------------------------------------- + +Register handlers for signals :py:data:`SIGINT` and :py:data:`SIGTERM`:: + + import asyncio + import functools + import os + import signal + + def ask_exit(signame): + print("got signal %s: exit" % signame) + loop.stop() + + loop = asyncio.get_event_loop() + for signame in ('SIGINT', 'SIGTERM'): + loop.add_signal_handler(getattr(signal, signame), + functools.partial(ask_exit, signame)) + + print("Event loop running forever, press CTRL+c to interrupt.") + print("pid %s: send SIGINT or SIGTERM to exit." % os.getpid()) + loop.run_forever() + diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst new file mode 100644 index 0000000..9c52b38 --- /dev/null +++ b/Doc/library/asyncio-protocol.rst @@ -0,0 +1,502 @@ +.. currentmodule:: asyncio + ++++++++++++++++++++++++++++++++++++++++++ +Transports and protocols (low-level API) ++++++++++++++++++++++++++++++++++++++++++ + +.. _asyncio-transport: + +Transports +========== + +Transports are classes provided by :mod:`asyncio` in order to abstract +various kinds of communication channels. You generally won't instantiate +a transport yourself; instead, you will call a :class:`BaseEventLoop` method +which will create the transport and try to initiate the underlying +communication channel, calling you back when it succeeds. + +Once the communication channel is established, a transport is always +paired with a :ref:`protocol <asyncio-protocol>` instance. The protocol can +then call the transport's methods for various purposes. + +:mod:`asyncio` currently implements transports for TCP, UDP, SSL, and +subprocess pipes. The methods available on a transport depend on +the transport's kind. + + +BaseTransport +------------- + +.. class:: BaseTransport + + Base class for transports. + + .. method:: close(self) + + Close the transport. If the transport has a buffer for outgoing + data, buffered data will be flushed asynchronously. No more data + will be received. After all buffered data is flushed, the + protocol's :meth:`connection_lost` method will be called with + :const:`None` as its argument. + + + .. method:: get_extra_info(name, default=None) + + Return optional transport information. *name* is a string representing + the piece of transport-specific information to get, *default* is the + value to return if the information doesn't exist. + + This method allows transport implementations to easily expose + channel-specific information. + + * socket: + + - ``'peername'``: the remote address to which the socket is connected, + result of :meth:`socket.socket.getpeername` (``None`` on error) + - ``'socket'``: :class:`socket.socket` instance + - ``'sockname'``: the socket's own address, + result of :meth:`socket.socket.getsockname` + + * SSL socket: + + - ``'compression'``: the compression algorithm being used as a string, + or ``None`` if the connection isn't compressed; result of + :meth:`ssl.SSLSocket.compression` + - ``'cipher'``: a three-value tuple containing the name of the cipher + being used, the version of the SSL protocol that defines its use, and + the number of secret bits being used; result of + :meth:`ssl.SSLSocket.cipher` + - ``'peercert'``: peer certificate; result of + :meth:`ssl.SSLSocket.getpeercert` + - ``'sslcontext'``: :class:`ssl.SSLContext` instance + + * pipe: + + - ``'pipe'``: pipe object + + * subprocess: + + - ``'subprocess'``: :class:`subprocess.Popen` instance + + +ReadTransport +------------- + +.. class:: ReadTransport + + Interface for read-only transports. + + .. method:: pause_reading() + + Pause the receiving end of the transport. No data will be passed to + the protocol's :meth:`data_received` method until :meth:`resume_reading` + is called. + + .. method:: resume_reading() + + Resume the receiving end. The protocol's :meth:`data_received` method + will be called once again if some data is available for reading. + + +WriteTransport +-------------- + +.. class:: WriteTransport + + Interface for write-only transports. + + .. method:: abort() + + Close the transport immediately, without waiting for pending operations + to complete. Buffered data will be lost. No more data will be received. + The protocol's :meth:`connection_lost` method will eventually be + called with :const:`None` as its argument. + + .. method:: can_write_eof() + + Return :const:`True` if the transport supports :meth:`write_eof`, + :const:`False` if not. + + .. method:: get_write_buffer_size() + + Return the current size of the output buffer used by the transport. + + .. method:: set_write_buffer_limits(high=None, low=None) + + Set the *high*- and *low*-water limits for write flow control. + + These two values control when call the protocol's + :meth:`pause_writing` and :meth:`resume_writing` methods are called. + If specified, the low-water limit must be less than or equal to the + high-water limit. Neither *high* nor *low* can be negative. + + The defaults are implementation-specific. If only the + high-water limit is given, the low-water limit defaults to a + implementation-specific value less than or equal to the + high-water limit. Setting *high* to zero forces *low* to zero as + well, and causes :meth:`pause_writing` to be called whenever the + buffer becomes non-empty. Setting *low* to zero causes + :meth:`resume_writing` to be called only once the buffer is empty. + Use of zero for either limit is generally sub-optimal as it + reduces opportunities for doing I/O and computation + concurrently. + + .. method:: write(data) + + Write some *data* bytes to the transport. + + This method does not block; it buffers the data and arranges for it + to be sent out asynchronously. + + .. method:: writelines(list_of_data) + + Write a list (or any iterable) of data bytes to the transport. + This is functionally equivalent to calling :meth:`write` on each + element yielded by the iterable, but may be implemented more efficiently. + + .. method:: write_eof() + + Close the write end of the transport after flushing buffered data. + Data may still be received. + + This method can raise :exc:`NotImplementedError` if the transport + (e.g. SSL) doesn't support half-closes. + + +DatagramTransport +----------------- + +.. method:: DatagramTransport.sendto(data, addr=None) + + Send the *data* bytes to the remote peer given by *addr* (a + transport-dependent target address). If *addr* is :const:`None`, the + data is sent to the target address given on transport creation. + + This method does not block; it buffers the data and arranges for it + to be sent out asynchronously. + +.. method:: DatagramTransport.abort() + + Close the transport immediately, without waiting for pending operations + to complete. Buffered data will be lost. No more data will be received. + The protocol's :meth:`connection_lost` method will eventually be + called with :const:`None` as its argument. + + +BaseSubprocessTransport +----------------------- + +.. class:: BaseSubprocessTransport + + .. method:: get_pid() + + Return the subprocess process id as an integer. + + .. method:: get_pipe_transport(fd) + + Return the transport for the communication pipe corresponding to the + integer file descriptor *fd*. The return value can be a readable or + writable streaming transport, depending on the *fd*. If *fd* doesn't + correspond to a pipe belonging to this transport, :const:`None` is + returned. + + .. method:: get_returncode() + + Return the subprocess returncode as an integer or :const:`None` + if it hasn't returned, similarly to the + :attr:`subprocess.Popen.returncode` attribute. + + .. method:: kill(self) + + Kill the subprocess, as in :meth:`subprocess.Popen.kill` + + On POSIX systems, the function sends SIGKILL to the subprocess. + On Windows, this method is an alias for :meth:`terminate`. + + .. method:: send_signal(signal) + + Send the *signal* number to the subprocess, as in + :meth:`subprocess.Popen.send_signal`. + + .. method:: terminate() + + Ask the subprocess to stop, as in :meth:`subprocess.Popen.terminate`. + This method is an alias for the :meth:`close` method. + + On POSIX systems, this method sends SIGTERM to the subprocess. + On Windows, the Windows API function TerminateProcess() is called to + stop the subprocess. + + +.. _asyncio-protocol: + +Protocols +========= + +:mod:`asyncio` provides base classes that you can subclass to implement +your network protocols. Those classes are used in conjunction with +:ref:`transports <asyncio-transport>` (see below): the protocol parses incoming +data and asks for the writing of outgoing data, while the transport is +responsible for the actual I/O and buffering. + +When subclassing a protocol class, it is recommended you override certain +methods. Those methods are callbacks: they will be called by the transport +on certain events (for example when some data is received); you shouldn't +call them yourself, unless you are implementing a transport. + +.. note:: + All callbacks have default implementations, which are empty. Therefore, + you only need to implement the callbacks for the events in which you + are interested. + + +Protocol classes +---------------- + +.. class:: Protocol + + The base class for implementing streaming protocols (for use with + e.g. TCP and SSL transports). + +.. class:: DatagramProtocol + + The base class for implementing datagram protocols (for use with + e.g. UDP transports). + +.. class:: SubprocessProtocol + + The base class for implementing protocols communicating with child + processes (through a set of unidirectional pipes). + + +Connection callbacks +-------------------- + +These callbacks may be called on :class:`Protocol`, :class:`DatagramProtocol` +and :class:`SubprocessProtocol` instances: + +.. method:: BaseProtocol.connection_made(transport) + + Called when a connection is made. + + The *transport* argument is the transport representing the + connection. You are responsible for storing it somewhere + (e.g. as an attribute) if you need to. + +.. method:: BaseProtocol.connection_lost(exc) + + Called when the connection is lost or closed. + + The argument is either an exception object or :const:`None`. + The latter means a regular EOF is received, or the connection was + aborted or closed by this side of the connection. + +:meth:`~BaseProtocol.connection_made` and :meth:`~BaseProtocol.connection_lost` +are called exactly once per successful connection. All other callbacks will be +called between those two methods, which allows for easier resource management +in your protocol implementation. + +The following callbacks may be called only on :class:`SubprocessProtocol` +instances: + +.. method:: SubprocessProtocol.pipe_data_received(fd, data) + + Called when the child process writes data into its stdout or stderr pipe. + *fd* is the integer file descriptor of the pipe. *data* is a non-empty + bytes object containing the data. + +.. method:: SubprocessProtocol.pipe_connection_lost(fd, exc) + + Called when one of the pipes communicating with the child process + is closed. *fd* is the integer file descriptor that was closed. + +.. method:: SubprocessProtocol.process_exited() + + Called when the child process has exited. + + +Streaming protocols +------------------- + +The following callbacks are called on :class:`Protocol` instances: + +.. method:: Protocol.data_received(data) + + Called when some data is received. *data* is a non-empty bytes object + containing the incoming data. + + .. note:: + Whether the data is buffered, chunked or reassembled depends on + the transport. In general, you shouldn't rely on specific semantics + and instead make your parsing generic and flexible enough. However, + data is always received in the correct order. + +.. method:: Protocol.eof_received() + + Calls when the other end signals it won't send any more data + (for example by calling :meth:`write_eof`, if the other end also uses + asyncio). + + This method may return a false value (including None), in which case + the transport will close itself. Conversely, if this method returns a + true value, closing the transport is up to the protocol. Since the + default implementation returns None, it implicitly closes the connection. + + .. note:: + Some transports such as SSL don't support half-closed connections, + in which case returning true from this method will not prevent closing + the connection. + +:meth:`data_received` can be called an arbitrary number of times during +a connection. However, :meth:`eof_received` is called at most once +and, if called, :meth:`data_received` won't be called after it. + +Datagram protocols +------------------ + +The following callbacks are called on :class:`DatagramProtocol` instances. + +.. method:: DatagramProtocol.datagram_received(data, addr) + + Called when a datagram is received. *data* is a bytes object containing + the incoming data. *addr* is the address of the peer sending the data; + the exact format depends on the transport. + +.. method:: DatagramProtocol.error_received(exc) + + Called when a previous send or receive operation raises an + :class:`OSError`. *exc* is the :class:`OSError` instance. + + This method is called in rare conditions, when the transport (e.g. UDP) + detects that a datagram couldn't be delivered to its recipient. + In many conditions though, undeliverable datagrams will be silently + dropped. + + +Flow control callbacks +---------------------- + +These callbacks may be called on :class:`Protocol`, +:class:`DatagramProtocol` and :class:`SubprocessProtocol` instances: + +.. method:: BaseProtocol.pause_writing() + + Called when the transport's buffer goes over the high-water mark. + +.. method:: BaseProtocol.resume_writing() + + Called when the transport's buffer drains below the low-water mark. + + +:meth:`pause_writing` and :meth:`resume_writing` calls are paired -- +:meth:`pause_writing` is called once when the buffer goes strictly over +the high-water mark (even if subsequent writes increases the buffer size +even more), and eventually :meth:`resume_writing` is called once when the +buffer size reaches the low-water mark. + +.. note:: + If the buffer size equals the high-water mark, + :meth:`pause_writing` is not called -- it must go strictly over. + Conversely, :meth:`resume_writing` is called when the buffer size is + equal or lower than the low-water mark. These end conditions + are important to ensure that things go as expected when either + mark is zero. + +.. note:: + On BSD systems (OS X, FreeBSD, etc.) flow control is not supported + for :class:`DatagramProtocol`, because send failures caused by + writing too many packets cannot be detected easily. The socket + always appears 'ready' and excess packets are dropped; an + :class:`OSError` with errno set to :const:`errno.ENOBUFS` may or + may not be raised; if it is raised, it will be reported to + :meth:`DatagramProtocol.error_received` but otherwise ignored. + + +Coroutines and protocols +------------------------ + +Coroutines can be scheduled in a protocol method using :func:`async`, but there +is not guarantee on the execution order. Protocols are not aware of coroutines +created in protocol methods and so will not wait for them. + +To have a reliable execution order, use :ref:`stream objects <asyncio-streams>` in a +coroutine with ``yield from``. For example, the :meth:`StreamWriter.drain` +coroutine can be used to wait until the write buffer is flushed. + + +Protocol example: TCP echo server and client +============================================ + +Echo client +----------- + +TCP echo client example, send data and wait until the connection is closed:: + + import asyncio + + class EchoClient(asyncio.Protocol): + message = 'This is the message. It will be echoed.' + + def connection_made(self, transport): + transport.write(self.message.encode()) + print('data sent: {}'.format(self.message)) + + def data_received(self, data): + print('data received: {}'.format(data.decode())) + + def connection_lost(self, exc): + print('server closed the connection') + asyncio.get_event_loop().stop() + + loop = asyncio.get_event_loop() + coro = loop.create_connection(EchoClient, '127.0.0.1', 8888) + loop.run_until_complete(coro) + loop.run_forever() + loop.close() + +The event loop is running twice. The +:meth:`~BaseEventLoop.run_until_complete` method is preferred in this short +example to raise an exception if the server is not listening, instead of +having to write a short coroutine to handle the exception and stop the +running loop. At :meth:`~BaseEventLoop.run_until_complete` exit, the loop is +no more running, so there is no need to stop the loop in case of an error. + +Echo server +----------- + +TCP echo server example, send back received data and close the connection:: + + import asyncio + + class EchoServer(asyncio.Protocol): + def connection_made(self, transport): + peername = transport.get_extra_info('peername') + print('connection from {}'.format(peername)) + self.transport = transport + + def data_received(self, data): + print('data received: {}'.format(data.decode())) + self.transport.write(data) + + # close the socket + self.transport.close() + + loop = asyncio.get_event_loop() + coro = loop.create_server(EchoServer, '127.0.0.1', 8888) + server = loop.run_until_complete(coro) + print('serving on {}'.format(server.sockets[0].getsockname())) + + try: + loop.run_forever() + except KeyboardInterrupt: + print("exit") + finally: + server.close() + loop.close() + +:meth:`Transport.close` can be called immediately after +:meth:`WriteTransport.write` even if data are not sent yet on the socket: both +methods are asynchronous. ``yield from`` is not needed because these transport +methods are not coroutines. + + diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst new file mode 100644 index 0000000..4543af4 --- /dev/null +++ b/Doc/library/asyncio-stream.rst @@ -0,0 +1,278 @@ +.. currentmodule:: asyncio + +.. _asyncio-streams: + +++++++++++++++++++++++++ +Streams (high-level API) +++++++++++++++++++++++++ + +Stream functions +================ + +.. function:: open_connection(host=None, port=None, \*, loop=None, limit=None, **kwds) + + A wrapper for :meth:`~BaseEventLoop.create_connection()` returning a (reader, + writer) pair. + + The reader returned is a :class:`StreamReader` instance; the writer is + a :class:`StreamWriter` instance. + + The arguments are all the usual arguments to + :meth:`BaseEventLoop.create_connection` except *protocol_factory*; most + common are positional host and port, with various optional keyword arguments + following. + + Additional optional keyword arguments are *loop* (to set the event loop + instance to use) and *limit* (to set the buffer limit passed to the + :class:`StreamReader`). + + (If you want to customize the :class:`StreamReader` and/or + :class:`StreamReaderProtocol` classes, just copy the code -- there's really + nothing special here except some convenience.) + + This function is a :ref:`coroutine <coroutine>`. + +.. function:: start_server(client_connected_cb, host=None, port=None, \*, loop=None, limit=None, **kwds) + + Start a socket server, with a callback for each client connected. + + The first parameter, *client_connected_cb*, takes two parameters: + *client_reader*, *client_writer*. *client_reader* is a + :class:`StreamReader` object, while *client_writer* is a + :class:`StreamWriter` object. This parameter can either be a plain callback + function or a :ref:`coroutine function <coroutine>`; if it is a coroutine + function, it will be automatically converted into a :class:`Task`. + + The rest of the arguments are all the usual arguments to + :meth:`~BaseEventLoop.create_server()` except *protocol_factory*; most + common are positional host and port, with various optional keyword arguments + following. The return value is the same as + :meth:`~BaseEventLoop.create_server()`. + + Additional optional keyword arguments are *loop* (to set the event loop + instance to use) and *limit* (to set the buffer limit passed to the + :class:`StreamReader`). + + The return value is the same as :meth:`~BaseEventLoop.create_server()`, i.e. + a :class:`AbstractServer` object which can be used to stop the service. + + This function is a :ref:`coroutine <coroutine>`. + +.. function:: open_unix_connection(path=None, \*, loop=None, limit=None, **kwds) + + A wrapper for :meth:`~BaseEventLoop.create_unix_connection()` returning + a (reader, writer) pair. + + See :func:`open_connection` for information about return value and other + details. + + This function is a :ref:`coroutine <coroutine>`. + + Availability: UNIX. + +.. function:: start_unix_server(client_connected_cb, path=None, \*, loop=None, limit=None, **kwds) + + Start a UNIX Domain Socket server, with a callback for each client connected. + + See :func:`start_server` for information about return value and other + details. + + This function is a :ref:`coroutine <coroutine>`. + + Availability: UNIX. + + +StreamReader +============ + +.. class:: StreamReader(limit=None, loop=None) + + .. method:: exception() + + Get the exception. + + .. method:: feed_eof() + + Acknowledge the EOF. + + .. method:: feed_data(data) + + Feed *data* bytes in the internal buffer. Any operations waiting + for the data will be resumed. + + .. method:: set_exception(exc) + + Set the exception. + + .. method:: set_transport(transport) + + Set the transport. + + .. method:: read(n=-1) + + Read up to *n* bytes. If *n* is not provided, or set to ``-1``, + read until EOF and return all read bytes. + + If the EOF was received and the internal buffer is empty, + return an empty ``bytes`` object. + + This method is a :ref:`coroutine <coroutine>`. + + .. method:: readline() + + Read one line, where "line" is a sequence of bytes ending with ``\n``. + + If EOF is received, and ``\n`` was not found, the method will + return the partial read bytes. + + If the EOF was received and the internal buffer is empty, + return an empty ``bytes`` object. + + This method is a :ref:`coroutine <coroutine>`. + + .. method:: readexactly(n) + + Read exactly *n* bytes. Raise an :exc:`IncompleteReadError` if the end of + the stream is reached before *n* can be read, the + :attr:`IncompleteReadError.partial` attribute of the exception contains + the partial read bytes. + + This method is a :ref:`coroutine <coroutine>`. + + .. method:: at_eof() + + Return ``True`` if the buffer is empty and :meth:`feed_eof` was called. + + +StreamWriter +============ + +.. class:: StreamWriter(transport, protocol, reader, loop) + + Wraps a Transport. + + This exposes :meth:`write`, :meth:`writelines`, :meth:`can_write_eof()`, + :meth:`write_eof`, :meth:`get_extra_info` and :meth:`close`. It adds + :meth:`drain` which returns an optional :class:`Future` on which you can + wait for flow control. It also adds a transport attribute which references + the :class:`Transport` directly. + + .. attribute:: transport + + Transport. + + .. method:: can_write_eof() + + Return :const:`True` if the transport supports :meth:`write_eof`, + :const:`False` if not. See :meth:`WriteTransport.can_write_eof`. + + .. method:: close() + + Close the transport: see :meth:`BaseTransport.close`. + + .. method:: drain() + + Wait until the write buffer of the underlying transport is flushed. + + This method has an unusual return value. The intended use is to write:: + + w.write(data) + yield from w.drain() + + When there's nothing to wait for, :meth:`drain()` returns ``()``, and the + yield-from continues immediately. When the transport buffer is full (the + protocol is paused), :meth:`drain` creates and returns a + :class:`Future` and the yield-from will block until + that Future is completed, which will happen when the buffer is + (partially) drained and the protocol is resumed. + + .. method:: get_extra_info(name, default=None) + + Return optional transport information: see + :meth:`BaseTransport.get_extra_info`. + + .. method:: write(data) + + Write some *data* bytes to the transport: see + :meth:`WriteTransport.write`. + + .. method:: writelines(data) + + Write a list (or any iterable) of data bytes to the transport: + see :meth:`WriteTransport.writelines`. + + .. method:: write_eof() + + Close the write end of the transport after flushing buffered data: + see :meth:`WriteTransport.write_eof`. + + +StreamReaderProtocol +==================== + +.. class:: StreamReaderProtocol(stream_reader, client_connected_cb=None, loop=None) + + Trivial helper class to adapt between :class:`Protocol` and + :class:`StreamReader`. Sublclass of :class:`Protocol`. + + *stream_reader* is a :class:`StreamReader` instance, *client_connected_cb* + is an optional function called with (stream_reader, stream_writer) when a + connection is made, *loop* is the event loop instance to use. + + (This is a helper class instead of making :class:`StreamReader` itself a + :class:`Protocol` subclass, because the :class:`StreamReader` has other + potential uses, and to prevent the user of the :class:`StreamReader` to + accidentally call inappropriate methods of the protocol.) + + +IncompleteReadError +=================== + +.. exception:: IncompleteReadError + + Incomplete read error, subclass of :exc:`EOFError`. + + .. attribute:: expected + + Total number of expected bytes (:class:`int`). + + .. attribute:: partial + + Read bytes string before the end of stream was reached (:class:`bytes`). + + +Example +======= + +Simple example querying HTTP headers of the URL passed on the command line:: + + import asyncio + import urllib.parse + import sys + + @asyncio.coroutine + def print_http_headers(url): + url = urllib.parse.urlsplit(url) + reader, writer = yield from asyncio.open_connection(url.hostname, 80) + query = ('HEAD {url.path} HTTP/1.0\r\n' + 'Host: {url.hostname}\r\n' + '\r\n').format(url=url) + writer.write(query.encode('latin-1')) + while True: + line = yield from reader.readline() + if not line: + break + line = line.decode('latin1').rstrip() + if line: + print('HTTP header> %s' % line) + + url = sys.argv[1] + loop = asyncio.get_event_loop() + task = asyncio.async(print_http_headers(url)) + loop.run_until_complete(task) + loop.close() + +Usage:: + + python example.py http://example.com/path/page.html + diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst new file mode 100644 index 0000000..dc48eb5 --- /dev/null +++ b/Doc/library/asyncio-subprocess.rst @@ -0,0 +1,268 @@ +.. currentmodule:: asyncio + +Subprocess +========== + +Operating system support +------------------------ + +On Windows, the default event loop uses :class:`selectors.SelectSelector` +which only supports sockets. The :class:`ProactorEventLoop` should be used to +support subprocesses. However, the latter does not support SSL. + +On Mac OS X older than 10.9 (Mavericks), :class:`selectors.KqueueSelector` +does not support character devices like PTY, whereas it is used by the +default event loop. The :class:`SelectorEventLoop` can be used with +:class:`SelectSelector` or :class:`PollSelector` to handle character +devices on Mac OS X 10.6 (Snow Leopard) and later. + + +Create a subprocess: high-level API using Process +------------------------------------------------- + +.. function:: create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None, loop=None, limit=None, \*\*kwds) + + Run the shell command *cmd* given as a string. Return a :class:`~asyncio.subprocess.Process` + instance. + + The optional *limit* parameter sets the buffer limit passed to the + :class:`StreamReader`. + + This function is a :ref:`coroutine <coroutine>`. + +.. function:: create_subprocess_exec(\*args, stdin=None, stdout=None, stderr=None, loop=None, limit=None, \*\*kwds) + + Create a subprocess. Return a :class:`~asyncio.subprocess.Process` instance. + + The optional *limit* parameter sets the buffer limit passed to the + :class:`StreamReader`. + + This function is a :ref:`coroutine <coroutine>`. + +Use the :meth:`BaseEventLoop.connect_read_pipe` and +:meth:`BaseEventLoop.connect_write_pipe` methods to connect pipes. + + +Create a subprocess: low-level API using subprocess.Popen +--------------------------------------------------------- + +Run subprocesses asynchronously using the :mod:`subprocess` module. + +.. method:: BaseEventLoop.subprocess_exec(protocol_factory, \*args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, \*\*kwargs) + + Create a subprocess from one or more string arguments, where the first string + specifies the program to execute, and the remaining strings specify the + program's arguments. (Thus, together the string arguments form the + ``sys.argv`` value of the program, assuming it is a Python script.) This is + similar to the standard library :class:`subprocess.Popen` class called with + shell=False and the list of strings passed as the first argument; + however, where :class:`~subprocess.Popen` takes a single argument which is + list of strings, :func:`subprocess_exec` takes multiple string arguments. + + Other parameters: + + * *stdin*: Either a file-like object representing the pipe to be connected + to the subprocess's standard input stream using + :meth:`~BaseEventLoop.connect_write_pipe`, or the constant + :const:`subprocess.PIPE` (the default). By default a new pipe will be + created and connected. + + * *stdout*: Either a file-like object representing the pipe to be connected + to the subprocess's standard output stream using + :meth:`~BaseEventLoop.connect_read_pipe`, or the constant + :const:`subprocess.PIPE` (the default). By default a new pipe will be + created and connected. + + * *stderr*: Either a file-like object representing the pipe to be connected + to the subprocess's standard error stream using + :meth:`~BaseEventLoop.connect_read_pipe`, or one of the constants + :const:`subprocess.PIPE` (the default) or :const:`subprocess.STDOUT`. + By default a new pipe will be created and connected. When + :const:`subprocess.STDOUT` is specified, the subprocess's standard error + stream will be connected to the same pipe as the standard output stream. + + * All other keyword arguments are passed to :class:`subprocess.Popen` + without interpretation, except for *bufsize*, *universal_newlines* and + *shell*, which should not be specified at all. + + Returns a pair of ``(transport, protocol)``, where *transport* is an + instance of :class:`BaseSubprocessTransport`. + + This method is a :ref:`coroutine <coroutine>`. + + See the constructor of the :class:`subprocess.Popen` class for parameters. + +.. method:: BaseEventLoop.subprocess_shell(protocol_factory, cmd, \*, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, \*\*kwargs) + + Create a subprocess from *cmd*, which is a string using the platform's + "shell" syntax. This is similar to the standard library + :class:`subprocess.Popen` class called with ``shell=True``. + + See :meth:`~BaseEventLoop.subprocess_exec` for more details about + the remaining arguments. + + Returns a pair of ``(transport, protocol)``, where *transport* is an + instance of :class:`BaseSubprocessTransport`. + + This method is a :ref:`coroutine <coroutine>`. + + See the constructor of the :class:`subprocess.Popen` class for parameters. + +.. seealso:: + + The :meth:`BaseEventLoop.connect_read_pipe` and + :meth:`BaseEventLoop.connect_write_pipe` methods. + + +Constants +--------- + +.. data:: asyncio.subprocess.PIPE + + Special value that can be used as the *stdin*, *stdout* or *stderr* argument + to :func:`create_subprocess_shell` and :func:`create_subprocess_exec` and + indicates that a pipe to the standard stream should be opened. + +.. data:: asyncio.subprocess.STDOUT + + Special value that can be used as the *stderr* argument to + :func:`create_subprocess_shell` and :func:`create_subprocess_exec` and + indicates that standard error should go into the same handle as standard + output. + +.. data:: asyncio.subprocess.DEVNULL + + Special value that can be used as the *stdin*, *stdout* or *stderr* argument + to :func:`create_subprocess_shell` and :func:`create_subprocess_exec` and + indicates that the special file :data:`os.devnull` will be used. + + +Process +------- + +.. class:: asyncio.subprocess.Process + + .. attribute:: pid + + The identifier of the process. + + Note that if you set the *shell* argument to ``True``, this is the + process identifier of the spawned shell. + + .. attribute:: returncode + + Return code of the process when it exited. A ``None`` value indicates + that the process has not terminated yet. + + A negative value ``-N`` indicates that the child was terminated by signal + ``N`` (Unix only). + + .. attribute:: stdin + + Standard input stream (write), ``None`` if the process was created with + ``stdin=None``. + + .. attribute:: stdout + + Standard output stream (read), ``None`` if the process was created with + ``stdout=None``. + + .. attribute:: stderr + + Standard error stream (read), ``None`` if the process was created with + ``stderr=None``. + + .. method:: communicate(input=None) + + Interact with process: Send data to stdin. Read data from stdout and + stderr, until end-of-file is reached. Wait for process to terminate. + The optional *input* argument should be data to be sent to the child + process, or ``None``, if no data should be sent to the child. The type + of *input* must be bytes. + + :meth:`communicate` returns a tuple ``(stdoutdata, stderrdata)``. + + Note that if you want to send data to the process's stdin, you need to + create the Process object with ``stdin=PIPE``. Similarly, to get anything + other than ``None`` in the result tuple, you need to give ``stdout=PIPE`` + and/or ``stderr=PIPE`` too. + + .. note:: + + The data read is buffered in memory, so do not use this method if the + data size is large or unlimited. + + This method is a :ref:`coroutine <coroutine>`. + + .. method:: kill() + + Kills the child. On Posix OSs the function sends :py:data:`SIGKILL` to + the child. On Windows :meth:`kill` is an alias for :meth:`terminate`. + + .. method:: send_signal(signal) + + Sends the signal *signal* to the child process. + + .. note:: + + On Windows, :py:data:`SIGTERM` is an alias for :meth:`terminate`. + ``CTRL_C_EVENT`` and ``CTRL_BREAK_EVENT`` can be sent to processes + started with a *creationflags* parameter which includes + ``CREATE_NEW_PROCESS_GROUP``. + + .. method:: terminate() + + Stop the child. On Posix OSs the method sends :py:data:`signal.SIGTERM` + to the child. On Windows the Win32 API function + :c:func:`TerminateProcess` is called to stop the child. + + .. method:: wait(): + + Wait for child process to terminate. Set and return :attr:`returncode` + attribute. + + This method is a :ref:`coroutine <coroutine>`. + + +Example +------- + +Implement a function similar to :func:`subprocess.getstatusoutput`, except that +it does not use a shell. Get the output of the "python -m platform" command and +display the output:: + + import asyncio + import os + import sys + from asyncio import subprocess + + @asyncio.coroutine + def getstatusoutput(*args): + proc = yield from asyncio.create_subprocess_exec( + *args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + try: + stdout, _ = yield from proc.communicate() + except: + proc.kill() + yield from proc.wait() + raise + exitcode = yield from proc.wait() + return (exitcode, stdout) + + if os.name == 'nt': + loop = asyncio.ProactorEventLoop() + asyncio.set_event_loop(loop) + else: + loop = asyncio.get_event_loop() + coro = getstatusoutput(sys.executable, '-m', 'platform') + exitcode, stdout = loop.run_until_complete(coro) + if not exitcode: + stdout = stdout.decode('ascii').rstrip() + print("Platform: %s" % stdout) + else: + print("Python failed with exit code %s:" % exitcode, flush=True) + sys.stdout.buffer.write(stdout) + sys.stdout.buffer.flush() + loop.close() diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst new file mode 100644 index 0000000..a299f09 --- /dev/null +++ b/Doc/library/asyncio-sync.rst @@ -0,0 +1,410 @@ +.. currentmodule:: asyncio +.. _asyncio-sync: + +Synchronization primitives +========================== + +Locks +----- + +Lock +^^^^ + +.. class:: Lock(\*, loop=None) + + Primitive lock objects. + + A primitive lock is a synchronization primitive that is not owned by a + particular coroutine when locked. A primitive lock is in one of two states, + 'locked' or 'unlocked'. + + It is created in the unlocked state. It has two basic methods, :meth:`acquire` + and :meth:`release`. When the state is unlocked, acquire() changes the state to + locked and returns immediately. When the state is locked, acquire() blocks + until a call to release() in another coroutine changes it to unlocked, then + the acquire() call resets it to locked and returns. The release() method + should only be called in the locked state; it changes the state to unlocked + and returns immediately. If an attempt is made to release an unlocked lock, + a :exc:`RuntimeError` will be raised. + + When more than one coroutine is blocked in acquire() waiting for the state + to turn to unlocked, only one coroutine proceeds when a release() call + resets the state to unlocked; first coroutine which is blocked in acquire() + is being processed. + + :meth:`acquire` is a coroutine and should be called with ``yield from``. + + Locks also support the context manager protocol. ``(yield from lock)`` + should be used as context manager expression. + + Usage:: + + lock = Lock() + ... + yield from lock + try: + ... + finally: + lock.release() + + Context manager usage:: + + lock = Lock() + ... + with (yield from lock): + ... + + Lock objects can be tested for locking state:: + + if not lock.locked(): + yield from lock + else: + # lock is acquired + ... + + .. method:: locked() + + Return ``True`` if the lock is acquired. + + .. method:: acquire() + + Acquire a lock. + + This method blocks until the lock is unlocked, then sets it to locked and + returns ``True``. + + This method is a :ref:`coroutine <coroutine>`. + + .. method:: release() + + Release a lock. + + When the lock is locked, reset it to unlocked, and return. If any other + coroutines are blocked waiting for the lock to become unlocked, allow + exactly one of them to proceed. + + When invoked on an unlocked lock, a :exc:`RuntimeError` is raised. + + There is no return value. + + +Event +^^^^^ + +.. class:: Event(\*, loop=None) + + An Event implementation, asynchronous equivalent to :class:`threading.Event`. + + Class implementing event objects. An event manages a flag that can be set to + true with the :meth:`set` method and reset to false with the :meth:`clear` + method. The :meth:`wait` method blocks until the flag is true. The flag is + initially false. + + .. method:: clear() + + Reset the internal flag to false. Subsequently, coroutines calling + :meth:`wait` will block until :meth:`set` is called to set the internal + flag to true again. + + .. method:: is_set() + + Return ``True`` if and only if the internal flag is true. + + .. method:: set() + + Set the internal flag to true. All coroutines waiting for it to become + true are awakened. Coroutine that call :meth:`wait` once the flag is true + will not block at all. + + .. method:: wait() + + Block until the internal flag is true. + + If the internal flag is true on entry, return ``True`` immediately. + Otherwise, block until another coroutine calls :meth:`set` to set the + flag to true, then return ``True``. + + This method is a :ref:`coroutine <coroutine>`. + + +Condition +^^^^^^^^^ + +.. class:: Condition(\*, loop=None) + + A Condition implementation, asynchronous equivalent to + :class:`threading.Condition`. + + This class implements condition variable objects. A condition variable + allows one or more coroutines to wait until they are notified by another + coroutine. + + A new :class:`Lock` object is created and used as the underlying lock. + + .. method:: acquire() + + Acquire the underlying lock. + + This method blocks until the lock is unlocked, then sets it to locked and + returns ``True``. + + This method is a :ref:`coroutine <coroutine>`. + + .. method:: notify(n=1) + + By default, wake up one coroutine waiting on this condition, if any. + If the calling coroutine has not acquired the lock when this method is + called, a :exc:`RuntimeError` is raised. + + This method wakes up at most *n* of the coroutines waiting for the + condition variable; it is a no-op if no coroutines are waiting. + + .. note:: + + An awakened coroutine does not actually return from its :meth:`wait` + call until it can reacquire the lock. Since :meth:`notify` does not + release the lock, its caller should. + + .. method:: locked() + + Return ``True`` if the underlying lock is acquired. + + .. method:: notify_all() + + Wake up all threads waiting on this condition. This method acts like + :meth:`notify`, but wakes up all waiting threads instead of one. If the + calling thread has not acquired the lock when this method is called, a + :exc:`RuntimeError` is raised. + + .. method:: release() + + Release the underlying lock. + + When the lock is locked, reset it to unlocked, and return. If any other + coroutines are blocked waiting for the lock to become unlocked, allow + exactly one of them to proceed. + + When invoked on an unlocked lock, a :exc:`RuntimeError` is raised. + + There is no return value. + + .. method:: wait() + + Wait until notified. + + If the calling coroutine has not acquired the lock when this method is + called, a :exc:`RuntimeError` is raised. + + This method releases the underlying lock, and then blocks until it is + awakened by a :meth:`notify` or :meth:`notify_all` call for the same + condition variable in another coroutine. Once awakened, it re-acquires + the lock and returns ``True``. + + This method is a :ref:`coroutine <coroutine>`. + + .. method:: wait_for(predicate) + + Wait until a predicate becomes true. + + The predicate should be a callable which result will be interpreted as a + boolean value. The final predicate value is the return value. + + This method is a :ref:`coroutine <coroutine>`. + + +Semaphores +---------- + +Semaphore +^^^^^^^^^ + +.. class:: Semaphore(value=1, \*, loop=None) + + A Semaphore implementation. + + A semaphore manages an internal counter which is decremented by each + :meth:`acquire` call and incremented by each :meth:`release` call. The + counter can never go below zero; when :meth:`acquire` finds that it is zero, + it blocks, waiting until some other thread calls :meth:`release`. + + Semaphores also support the context manager protocol. + + The optional argument gives the initial value for the internal counter; it + defaults to ``1``. If the value given is less than ``0``, :exc:`ValueError` + is raised. + + .. method:: acquire() + + Acquire a semaphore. + + If the internal counter is larger than zero on entry, decrement it by one + and return ``True`` immediately. If it is zero on entry, block, waiting + until some other coroutine has called :meth:`release` to make it larger + than ``0``, and then return ``True``. + + This method is a :ref:`coroutine <coroutine>`. + + .. method:: locked() + + Returns ``True`` if semaphore can not be acquired immediately. + + .. method:: release() + + Release a semaphore, incrementing the internal counter by one. When it + was zero on entry and another coroutine is waiting for it to become + larger than zero again, wake up that coroutine. + + +BoundedSemaphore +^^^^^^^^^^^^^^^^ + +.. class:: BoundedSemaphore(value=1, \*, loop=None) + + A bounded semaphore implementation. Inherit from :class:`Semaphore`. + + This raises :exc:`ValueError` in :meth:`~Semaphore.release` if it would + increase the value above the initial value. + + +Queues +------ + +Queue +^^^^^ + +.. class:: Queue(maxsize=0, \*, loop=None) + + A queue, useful for coordinating producer and consumer coroutines. + + If *maxsize* is less than or equal to zero, the queue size is infinite. If + it is an integer greater than ``0``, then ``yield from put()`` will block + when the queue reaches *maxsize*, until an item is removed by :meth:`get`. + + Unlike the standard library :mod:`queue`, you can reliably know this Queue's + size with :meth:`qsize`, since your single-threaded asyncio application won't + be interrupted between calling :meth:`qsize` and doing an operation on the + Queue. + + .. method:: empty() + + Return ``True`` if the queue is empty, ``False`` otherwise. + + .. method:: full() + + Return ``True`` if there are maxsize items in the queue. + + .. note:: + + If the Queue was initialized with ``maxsize=0`` (the default), then + :meth:`full()` is never ``True``. + + .. method:: get() + + Remove and return an item from the queue. + + If you yield from :meth:`get()`, wait until a item is available. + + This method is a :ref:`coroutine <coroutine>`. + + .. method:: get_nowait() + + Remove and return an item from the queue. + + Return an item if one is immediately available, else raise + :exc:`QueueEmpty`. + + .. method:: put(item) + + Put an item into the queue. + + If you yield from ``put()``, wait until a free slot is available before + adding item. + + This method is a :ref:`coroutine <coroutine>`. + + .. method:: put_nowait(item) + + Put an item into the queue without blocking. + + If no free slot is immediately available, raise :exc:`QueueFull`. + + .. method:: qsize() + + Number of items in the queue. + + .. attribute:: maxsize + + Number of items allowed in the queue. + + +PriorityQueue +^^^^^^^^^^^^^ + +.. class:: PriorityQueue + + A subclass of :class:`Queue`; retrieves entries in priority order (lowest + first). + + Entries are typically tuples of the form: (priority number, data). + + +LifoQueue +^^^^^^^^^ + +.. class:: LifoQueue + + A subclass of :class:`Queue` that retrieves most recently added entries + first. + + +JoinableQueue +^^^^^^^^^^^^^ + +.. class:: JoinableQueue + + A subclass of :class:`Queue` with :meth:`task_done` and :meth:`join` + methods. + + .. method:: join() + + Block until all items in the queue have been gotten and processed. + + The count of unfinished tasks goes up whenever an item is added to the + queue. The count goes down whenever a consumer thread calls + :meth:`task_done` to indicate that the item was retrieved and all work on + it is complete. When the count of unfinished tasks drops to zero, + :meth:`join` unblocks. + + This method is a :ref:`coroutine <coroutine>`. + + .. method:: task_done() + + Indicate that a formerly enqueued task is complete. + + Used by queue consumers. For each :meth:`~Queue.get` used to fetch a task, a + subsequent call to :meth:`task_done` tells the queue that the processing + on the task is complete. + + If a :meth:`join` is currently blocking, it will resume when all items + have been processed (meaning that a :meth:`task_done` call was received + for every item that had been :meth:`~Queue.put` into the queue). + + Raises :exc:`ValueError` if called more times than there were items + placed in the queue. + + +Exceptions +^^^^^^^^^^ + +.. exception:: QueueEmpty + + Exception raised when non-blocking :meth:`~Queue.get` (or + :meth:`~Queue.get_nowait`) is called + on a :class:`Queue` object which is empty. + + +.. exception:: QueueFull + + Exception raised when non-blocking :meth:`~Queue.put` (or + :meth:`~Queue.put_nowait`) is called + on a :class:`Queue` object which is full. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst new file mode 100644 index 0000000..f1894da --- /dev/null +++ b/Doc/library/asyncio-task.rst @@ -0,0 +1,580 @@ +.. currentmodule:: asyncio + +Tasks and coroutines +==================== + +.. _coroutine: + +Coroutines +---------- + +A coroutine is a generator that follows certain conventions. For +documentation purposes, all coroutines should be decorated with +``@asyncio.coroutine``, but this cannot be strictly enforced. + +Coroutines use the ``yield from`` syntax introduced in :pep:`380`, +instead of the original ``yield`` syntax. + +The word "coroutine", like the word "generator", is used for two +different (though related) concepts: + +- The function that defines a coroutine (a function definition + decorated with ``@asyncio.coroutine``). If disambiguation is needed + we will call this a *coroutine function* (:func:`iscoroutinefunction` + returns ``True``). + +- The object obtained by calling a coroutine function. This object + represents a computation or an I/O operation (usually a combination) + that will complete eventually. If disambiguation is needed we will + call it a *coroutine object* (:func:`iscoroutine` returns ``True``). + +Things a coroutine can do: + +- ``result = yield from future`` -- suspends the coroutine until the + future is done, then returns the future's result, or raises an + exception, which will be propagated. (If the future is cancelled, + it will raise a ``CancelledError`` exception.) Note that tasks are + futures, and everything said about futures also applies to tasks. + +- ``result = yield from coroutine`` -- wait for another coroutine to + produce a result (or raise an exception, which will be propagated). + The ``coroutine`` expression must be a *call* to another coroutine. + +- ``return expression`` -- produce a result to the coroutine that is + waiting for this one using ``yield from``. + +- ``raise exception`` -- raise an exception in the coroutine that is + waiting for this one using ``yield from``. + +Calling a coroutine does not start its code running -- it is just a +generator, and the coroutine object returned by the call is really a +generator object, which doesn't do anything until you iterate over it. +In the case of a coroutine object, there are two basic ways to start +it running: call ``yield from coroutine`` from another coroutine +(assuming the other coroutine is already running!), or convert it to a +:class:`Task`. + +Coroutines (and tasks) can only run when the event loop is running. + +.. decorator:: coroutine + + Decorator to mark coroutines. + + If the coroutine is not yielded from before it is destroyed, an error + message is logged. See :ref:`Detect coroutines never scheduled + <asyncio-coroutine-not-scheduled>`. + +.. note:: + + In this documentation, some methods are documented as coroutines, + even if they are plain Python functions returning a :class:`Future`. + This is intentional to have a freedom of tweaking the implementation + of these functions in the future. If such a function is needed to be + used in a callback-style code, wrap its result with :func:`async`. + + +.. _asyncio-hello-world-coroutine: + +Example: "Hello World" coroutine +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Print ``"Hello World"`` every two seconds using a coroutine:: + + import asyncio + + @asyncio.coroutine + def greet_every_two_seconds(): + while True: + print('Hello World') + yield from asyncio.sleep(2) + + loop = asyncio.get_event_loop() + loop.run_until_complete(greet_every_two_seconds()) + +.. seealso:: + + :ref:`Hello World example using a callback <asyncio-hello-world-callback>`. + + +Example: Chain coroutines +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example chaining coroutines:: + + import asyncio + + @asyncio.coroutine + def compute(x, y): + print("Compute %s + %s ..." % (x, y)) + yield from asyncio.sleep(1.0) + return x + y + + @asyncio.coroutine + def print_sum(x, y): + result = yield from compute(x, y) + print("%s + %s = %s" % (x, y, result)) + + loop = asyncio.get_event_loop() + loop.run_until_complete(print_sum(1, 2)) + loop.close() + +``compute()`` is chained to ``print_sum()``: ``print_sum()`` coroutine waits +until ``compute()`` is completed before returning its result. + +Sequence diagram of the example: + +.. image:: tulip_coro.png + :align: center + +The "Task" is created by the :meth:`BaseEventLoop.run_until_complete` method +when it gets a coroutine object instead of a task. + +The diagram shows the control flow, it does not describe exactly how things +work internally. For example, the sleep coroutine creates an internal future +which uses :meth:`BaseEventLoop.call_later` to wake up the task in 1 second. + + +InvalidStateError +----------------- + +.. exception:: InvalidStateError + + The operation is not allowed in this state. + + +TimeoutError +------------ + +.. exception:: TimeoutError + + The operation exceeded the given deadline. + +.. note:: + + This exception is different from the builtin :exc:`TimeoutError` exception! + + +Future +------ + +.. class:: Future(\*, loop=None) + + This class is *almost* compatible with :class:`concurrent.futures.Future`. + + Differences: + + - :meth:`result` and :meth:`exception` do not take a timeout argument and + raise an exception when the future isn't done yet. + + - Callbacks registered with :meth:`add_done_callback` are always called + via the event loop's :meth:`~BaseEventLoop.call_soon_threadsafe`. + + - This class is not compatible with the :func:`~concurrent.futures.wait` and + :func:`~concurrent.futures.as_completed` functions in the + :mod:`concurrent.futures` package. + + .. method:: cancel() + + Cancel the future and schedule callbacks. + + If the future is already done or cancelled, return ``False``. Otherwise, + change the future's state to cancelled, schedule the callbacks and return + ``True``. + + .. method:: cancelled() + + Return ``True`` if the future was cancelled. + + .. method:: done() + + Return True if the future is done. + + Done means either that a result / exception are available, or that the + future was cancelled. + + .. method:: result() + + Return the result this future represents. + + If the future has been cancelled, raises :exc:`CancelledError`. If the + future's result isn't yet available, raises :exc:`InvalidStateError`. If + the future is done and has an exception set, this exception is raised. + + .. method:: exception() + + Return the exception that was set on this future. + + The exception (or ``None`` if no exception was set) is returned only if + the future is done. If the future has been cancelled, raises + :exc:`CancelledError`. If the future isn't done yet, raises + :exc:`InvalidStateError`. + + .. method:: add_done_callback(fn) + + Add a callback to be run when the future becomes done. + + The callback is called with a single argument - the future object. If the + future is already done when this is called, the callback is scheduled + with :meth:`~BaseEventLoop.call_soon`. + + .. method:: remove_done_callback(fn) + + Remove all instances of a callback from the "call when done" list. + + Returns the number of callbacks removed. + + .. method:: set_result(result) + + Mark the future done and set its result. + + If the future is already done when this method is called, raises + :exc:`InvalidStateError`. + + .. method:: set_exception(exception) + + Mark the future done and set an exception. + + If the future is already done when this method is called, raises + :exc:`InvalidStateError`. + + +Example: Future with run_until_complete() +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example combining a :class:`Future` and a :ref:`coroutine function +<coroutine>`:: + + import asyncio + + @asyncio.coroutine + def slow_operation(future): + yield from asyncio.sleep(1) + future.set_result('Future is done!') + + loop = asyncio.get_event_loop() + future = asyncio.Future() + asyncio.Task(slow_operation(future)) + loop.run_until_complete(future) + print(future.result()) + loop.close() + +The coroutine function is responsible of the computation (which takes 1 second) +and it stores the result into the future. The +:meth:`~BaseEventLoop.run_until_complete` method waits for the completion of +the future. + +.. note:: + The :meth:`~BaseEventLoop.run_until_complete` method uses internally the + :meth:`~Future.add_done_callback` method to be notified when the future is + done. + + +Example: Future with run_forever() +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The previous example can be written differently using the +:meth:`Future.add_done_callback` method to describe explicitly the control +flow:: + + import asyncio + + @asyncio.coroutine + def slow_operation(future): + yield from asyncio.sleep(1) + future.set_result('Future is done!') + + def got_result(future): + print(future.result()) + loop.stop() + + loop = asyncio.get_event_loop() + future = asyncio.Future() + asyncio.Task(slow_operation(future)) + future.add_done_callback(got_result) + try: + loop.run_forever() + finally: + loop.close() + +In this example, the future is responsible to display the result and to stop +the loop. + +.. note:: + The "slow_operation" coroutine object is only executed when the event loop + starts running, so it is possible to add a "done callback" to the future + after creating the task scheduling the coroutine object. + + + +Task +---- + +.. class:: Task(coro, \*, loop=None) + + A coroutine object wrapped in a :class:`Future`. Subclass of :class:`Future`. + + .. classmethod:: all_tasks(loop=None) + + Return a set of all tasks for an event loop. + + By default all tasks for the current event loop are returned. + + .. classmethod:: current_task(loop=None) + + Return the currently running task in an event loop or ``None``. + + By default the current task for the current event loop is returned. + + ``None`` is returned when called not in the context of a :class:`Task`. + + .. method:: cancel() + + Request this task to cancel itself. + + This arranges for a :exc:`~concurrent.futures.CancelledError` to be + thrown into the wrapped coroutine on the next cycle through the event + loop. The coroutine then has a chance to clean up or even deny the + request using try/except/finally. + + Contrary to :meth:`Future.cancel`, this does not guarantee that the task + will be cancelled: the exception might be caught and acted upon, delaying + cancellation of the task or preventing it completely. The task may also + return a value or raise a different exception. + + Immediately after this method is called, :meth:`~Future.cancelled` will + not return ``True`` (unless the task was already cancelled). A task will + be marked as cancelled when the wrapped coroutine terminates with a + :exc:`~concurrent.futures.CancelledError` exception (even if + :meth:`cancel` was not called). + + .. method:: get_stack(\*, limit=None) + + Return the list of stack frames for this task's coroutine. + + If the coroutine is active, this returns the stack where it is suspended. + If the coroutine has completed successfully or was cancelled, this + returns an empty list. If the coroutine was terminated by an exception, + this returns the list of traceback frames. + + The frames are always ordered from oldest to newest. + + The optional limit gives the maximum number of frames to return; by + default all available frames are returned. Its meaning differs depending + on whether a stack or a traceback is returned: the newest frames of a + stack are returned, but the oldest frames of a traceback are returned. + (This matches the behavior of the traceback module.) + + For reasons beyond our control, only one stack frame is returned for a + suspended coroutine. + + .. method:: print_stack(\*, limit=None, file=None) + + Print the stack or traceback for this task's coroutine. + + This produces output similar to that of the traceback module, for the + frames retrieved by get_stack(). The limit argument is passed to + get_stack(). The file argument is an I/O stream to which the output + goes; by default it goes to sys.stderr. + + +Example: Parallel execution of tasks +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example executing 3 tasks (A, B, C) in parallel:: + + import asyncio + + @asyncio.coroutine + def factorial(name, number): + f = 1 + for i in range(2, number+1): + print("Task %s: Compute factorial(%s)..." % (name, i)) + yield from asyncio.sleep(1) + f *= i + print("Task %s: factorial(%s) = %s" % (name, number, f)) + + tasks = [ + asyncio.Task(factorial("A", 2)), + asyncio.Task(factorial("B", 3)), + asyncio.Task(factorial("C", 4))] + + loop = asyncio.get_event_loop() + loop.run_until_complete(asyncio.wait(tasks)) + loop.close() + +Output:: + + Task A: Compute factorial(2)... + Task B: Compute factorial(2)... + Task C: Compute factorial(2)... + Task A: factorial(2) = 2 + Task B: Compute factorial(3)... + Task C: Compute factorial(3)... + Task B: factorial(3) = 6 + Task C: Compute factorial(4)... + Task C: factorial(4) = 24 + +A task is automatically scheduled for execution when it is created. The event +loop stops when all tasks are done. + + +Task functions +-------------- + +.. note:: + + In the functions below, the optional *loop* argument allows to explicitly set + the event loop object used by the underlying task or coroutine. If it's + not provided, the default event loop is used. + +.. function:: as_completed(fs, \*, loop=None, timeout=None) + + Return an iterator whose values, when waited for, are :class:`Future` + instances. + + Raises :exc:`asyncio.TimeoutError` if the timeout occurs before all Futures + are done. + + Example:: + + for f in as_completed(fs): + result = yield from f # The 'yield from' may raise + # Use result + + .. note:: + + The futures ``f`` are not necessarily members of fs. + +.. function:: async(coro_or_future, \*, loop=None) + + Wrap a :ref:`coroutine object <coroutine>` in a future. + + If the argument is a :class:`Future`, it is returned directly. + +.. function:: gather(\*coros_or_futures, loop=None, return_exceptions=False) + + Return a future aggregating results from the given coroutine objects or + futures. + + All futures must share the same event loop. If all the tasks are done + successfully, the returned future's result is the list of results (in the + order of the original sequence, not necessarily the order of results + arrival). If *return_exceptions* is True, exceptions in the tasks are + treated the same as successful results, and gathered in the result list; + otherwise, the first raised exception will be immediately propagated to the + returned future. + + Cancellation: if the outer Future is cancelled, all children (that have not + completed yet) are also cancelled. If any child is cancelled, this is + treated as if it raised :exc:`~concurrent.futures.CancelledError` -- the + outer Future is *not* cancelled in this case. (This is to prevent the + cancellation of one child to cause other children to be cancelled.) + +.. function:: iscoroutine(obj) + + Return ``True`` if *obj* is a :ref:`coroutine object <coroutine>`. + +.. function:: iscoroutinefunction(obj) + + Return ``True`` if *func* is a decorated :ref:`coroutine function + <coroutine>`. + +.. function:: sleep(delay, result=None, \*, loop=None) + + Create a :ref:`coroutine <coroutine>` that completes after a given + time (in seconds). If *result* is provided, it is produced to the caller + when the coroutine completes. + + The resolution of the sleep depends on the :ref:`granularity of the event + loop <asyncio-delayed-calls>`. + +.. function:: shield(arg, \*, loop=None) + + Wait for a future, shielding it from cancellation. + + The statement:: + + res = yield from shield(something()) + + is exactly equivalent to the statement:: + + res = yield from something() + + *except* that if the coroutine containing it is cancelled, the task running + in ``something()`` is not cancelled. From the point of view of + ``something()``, the cancellation did not happen. But its caller is still + cancelled, so the yield-from expression still raises + :exc:`~concurrent.futures.CancelledError`. Note: If ``something()`` is + cancelled by other means this will still cancel ``shield()``. + + If you want to completely ignore cancellation (not recommended) you can + combine ``shield()`` with a try/except clause, as follows:: + + try: + res = yield from shield(something()) + except CancelledError: + res = None + +.. function:: wait(futures, \*, loop=None, timeout=None, return_when=ALL_COMPLETED) + + Wait for the Futures and coroutine objects given by the sequence *futures* + to complete. Coroutines will be wrapped in Tasks. Returns two sets of + :class:`Future`: (done, pending). + + The sequence *futures* must not be empty. + + *timeout* can be used to control the maximum number of seconds to wait before + returning. *timeout* can be an int or float. If *timeout* is not specified + or ``None``, there is no limit to the wait time. + + *return_when* indicates when this function should return. It must be one of + the following constants of the :mod:`concurrent.futures` module: + + .. tabularcolumns:: |l|L| + + +-----------------------------+----------------------------------------+ + | Constant | Description | + +=============================+========================================+ + | :const:`FIRST_COMPLETED` | The function will return when any | + | | future finishes or is cancelled. | + +-----------------------------+----------------------------------------+ + | :const:`FIRST_EXCEPTION` | The function will return when any | + | | future finishes by raising an | + | | exception. If no future raises an | + | | exception then it is equivalent to | + | | :const:`ALL_COMPLETED`. | + +-----------------------------+----------------------------------------+ + | :const:`ALL_COMPLETED` | The function will return when all | + | | futures finish or are cancelled. | + +-----------------------------+----------------------------------------+ + + This function is a :ref:`coroutine <coroutine>`. + + Usage:: + + done, pending = yield from asyncio.wait(fs) + + .. note:: + + This does not raise :exc:`asyncio.TimeoutError`! Futures that aren't done + when the timeout occurs are returned in the second set. + + +.. function:: wait_for(fut, timeout, \*, loop=None) + + Wait for the single :class:`Future` or :ref:`coroutine object <coroutine>` + to complete, with timeout. If *timeout* is ``None``, block until the future + completes. + + Coroutine will be wrapped in :class:`Task`. + + Returns result of the Future or coroutine. When a timeout occurs, it + cancels the task and raises :exc:`asyncio.TimeoutError`. To avoid the task + cancellation, wrap it in :func:`shield`. + + This function is a :ref:`coroutine <coroutine>`. + + Usage:: + + result = yield from asyncio.wait_for(fut, 60.0) + diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst new file mode 100644 index 0000000..23731b1 --- /dev/null +++ b/Doc/library/asyncio.rst @@ -0,0 +1,59 @@ +:mod:`asyncio` -- Asynchronous I/O, event loop, coroutines and tasks +==================================================================== + +.. module:: asyncio + :synopsis: Asynchronous I/O, event loop, coroutines and tasks. + +.. versionadded:: 3.4 + +**Source code:** :source:`Lib/asyncio/` + +-------------- + +This module provides infrastructure for writing single-threaded concurrent +code using coroutines, multiplexing I/O access over sockets and other +resources, running network clients and servers, and other related primitives. +Here is a more detailed list of the package contents: + +* a pluggable :ref:`event loop <asyncio-event-loop>` with various system-specific + implementations; + +* :ref:`transport <asyncio-transport>` and :ref:`protocol <asyncio-protocol>` abstractions + (similar to those in `Twisted <http://twistedmatrix.com/>`_); + +* concrete support for TCP, UDP, SSL, subprocess pipes, delayed calls, and + others (some may be system-dependent); + +* a :class:`Future` class that mimics the one in the :mod:`concurrent.futures` + module, but adapted for use with the event loop; + +* coroutines and tasks based on ``yield from`` (:PEP:`380`), to help write + concurrent code in a sequential fashion; + +* cancellation support for :class:`Future`\s and coroutines; + +* :ref:`synchronization primitives <asyncio-sync>` for use between coroutines in + a single thread, mimicking those in the :mod:`threading` module; + +* an interface for passing work off to a threadpool, for times when + you absolutely, positively have to use a library that makes blocking + I/O calls. + +Table of content: + +.. toctree:: + :maxdepth: 3 + + asyncio-eventloop.rst + asyncio-task.rst + asyncio-protocol.rst + asyncio-stream.rst + asyncio-subprocess.rst + asyncio-sync.rst + asyncio-dev.rst + +.. seealso:: + + The :mod:`asyncio` module was designed in the :PEP:`3156`. For a + motivational primer on transports and protocols, see :PEP:`3153`. + diff --git a/Doc/library/asyncore.rst b/Doc/library/asyncore.rst index 1521e72..0adf8d9 100644 --- a/Doc/library/asyncore.rst +++ b/Doc/library/asyncore.rst @@ -13,6 +13,11 @@ -------------- +.. note:: + + This module exists for backwards compatibility only. For new code we + recommend using :mod:`asyncio`. + This module provides the basic infrastructure for writing asynchronous socket service clients and servers. diff --git a/Doc/library/audioop.rst b/Doc/library/audioop.rst index edb3870..ce127aa 100644 --- a/Doc/library/audioop.rst +++ b/Doc/library/audioop.rst @@ -6,9 +6,14 @@ The :mod:`audioop` module contains some useful operations on sound fragments. -It operates on sound fragments consisting of signed integer samples 8, 16 or 32 -bits wide, stored in bytes objects. All scalar items are integers, unless -specified otherwise. +It operates on sound fragments consisting of signed integer samples 8, 16, 24 +or 32 bits wide, stored in :term:`bytes-like object`\ s. All scalar items are +integers, unless specified otherwise. + +.. versionchanged:: 3.4 + Support for 24-bit samples was added. + All functions now accept any :term:`bytes-like object`. + String input now results in an immediate error. .. index:: single: Intel/DVI ADPCM @@ -35,7 +40,7 @@ The module defines the following variables and functions: .. function:: add(fragment1, fragment2, width) Return a fragment which is the addition of the two samples passed as parameters. - *width* is the sample width in bytes, either ``1``, ``2`` or ``4``. Both + *width* is the sample width in bytes, either ``1``, ``2``, ``3`` or ``4``. Both fragments should have the same length. Samples are truncated in case of overflow. @@ -70,6 +75,14 @@ The module defines the following variables and functions: sample. Samples wrap around in case of overflow. +.. function:: byteswap(fragment, width) + + "Byteswap" all samples in a fragment and returns the modified fragment. + Converts big-endian samples to little-endian and vice versa. + + .. versionadded:: 3.4 + + .. function:: cross(fragment, width) Return the number of zero crossings in the fragment passed as an argument. @@ -133,19 +146,19 @@ The module defines the following variables and functions: .. function:: lin2lin(fragment, width, newwidth) - Convert samples between 1-, 2- and 4-byte formats. + Convert samples between 1-, 2-, 3- and 4-byte formats. .. note:: - In some audio formats, such as .WAV files, 16 and 32 bit samples are + In some audio formats, such as .WAV files, 16, 24 and 32 bit samples are signed, but 8 bit samples are unsigned. So when converting to 8 bit wide samples for these formats, you need to also add 128 to the result:: new_frames = audioop.lin2lin(frames, old_width, 1) new_frames = audioop.bias(new_frames, 1, 128) - The same, in reverse, has to be applied when converting from 8 to 16 or 32 - bit width samples. + The same, in reverse, has to be applied when converting from 8 to 16, 24 + or 32 bit width samples. .. function:: lin2ulaw(fragment, width) diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index f0d11b0..02b4d7b 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -1,32 +1,42 @@ -:mod:`base64` --- RFC 3548: Base16, Base32, Base64 Data Encodings -================================================================= +:mod:`base64` --- Base16, Base32, Base64, Base85 Data Encodings +=============================================================== .. module:: base64 - :synopsis: RFC 3548: Base16, Base32, Base64 Data Encodings + :synopsis: RFC 3548: Base16, Base32, Base64 Data Encodings; + Base85 and Ascii85 .. index:: pair: base64; encoding single: MIME; base64 encoding -This module provides data encoding and decoding as specified in :rfc:`3548`. -This standard defines the Base16, Base32, and Base64 algorithms for encoding -and decoding arbitrary binary strings into ASCII-only byte strings that can be +This module provides functions for encoding binary data to printable +ASCII characters and decoding such encodings back to binary data. +It provides encoding and decoding functions for the encodings specified in +in :rfc:`3548`, which defines the Base16, Base32, and Base64 algorithms, +and for the de-facto standard Ascii85 and Base85 encodings. + +The :rfc:`3548` encodings are suitable for encoding binary data so that it can safely sent by email, used as parts of URLs, or included as part of an HTTP POST request. The encoding algorithm is not the same as the :program:`uuencode` program. -There are two interfaces provided by this module. The modern interface -supports encoding and decoding ASCII byte string objects using all three -alphabets. Additionally, the decoding functions of the modern interface also -accept Unicode strings containing only ASCII characters. The legacy interface -provides for encoding and decoding to and from file-like objects as well as -byte strings, but only using the Base64 standard alphabet. +There are two :rfc:`3548` interfaces provided by this module. The modern +interface supports encoding and decoding ASCII byte string objects using all +three :rfc:`3548` defined alphabets (normal, URL-safe, and filesystem-safe). +Additionally, the decoding functions of the modern interface also accept +Unicode strings containing only ASCII characters. The legacy interface provides +for encoding and decoding to and from file-like objects as well as byte +strings, but only using the Base64 standard alphabet. .. versionchanged:: 3.3 ASCII-only Unicode strings are now accepted by the decoding functions of the modern interface. +.. versionchanged:: 3.4 + Any :term:`bytes-like object`\ s are now accepted by all + encoding and decoding functions in this module. Ascii85/Base85 support added. + The modern interface provides: .. function:: b64encode(s, altchars=None) @@ -128,6 +138,76 @@ The modern interface provides: string. +.. function:: a85encode(s, *, foldspaces=False, wrapcol=0, pad=False, adobe=False) + + Encode a byte string using Ascii85. + + *s* is the string to encode. The encoded byte string is returned. + + *foldspaces* is an optional flag that uses the special short sequence 'y' + instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This + feature is not supported by the "standard" Ascii85 encoding. + + *wrapcol* controls whether the output should have newline ('\n') + characters added to it. If this is non-zero, each output line will be + at most this many characters long. + + *pad* controls whether the input string is padded to a multiple of 4 + before encoding. Note that the ``btoa`` implementation always pads. + + *adobe* controls whether the encoded byte sequence is framed with ``<~`` + and ``~>``, which is used by the Adobe implementation. + + .. versionadded:: 3.4 + + +.. function:: a85decode(s, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v') + + Decode an Ascii85 encoded byte string. + + *s* is the byte string to decode. + + *foldspaces* is a flag that specifies whether the 'y' short sequence + should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). + This feature is not supported by the "standard" Ascii85 encoding. + + *adobe* controls whether the input sequence is in Adobe Ascii85 format + (i.e. is framed with <~ and ~>). + + *ignorechars* should be a byte string containing characters to ignore + from the input. This should only contain whitespace characters, and by + default contains all whitespace characters in ASCII. + + .. versionadded:: 3.4 + + +.. function:: b85encode(s, pad=False) + + Encode a byte string using base85, as used in e.g. git-style binary + diffs. + + If *pad* is true, the input is padded with "\\0" so its length is a + multiple of 4 characters before encoding. + + .. versionadded:: 3.4 + + +.. function:: b85decode(b) + + Decode base85-encoded byte string. Padding is implicitly removed, if + necessary. + + .. versionadded:: 3.4 + + +.. note:: + Both Base85 and Ascii85 have an expansion factor of 5 to 4 (5 Base85 or + Ascii85 characters can encode 4 binary bytes), while the better-known + Base64 has an expansion factor of 6 to 4. They are therefore more + efficient when space expensive. They differ by details such as the + character map used for encoding. + + The legacy interface: .. function:: decode(input, output) diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index b79bccd..488cda5 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -37,8 +37,8 @@ All of the classes in this module may safely be accessed from multiple threads. file object to read from or write to. The *mode* argument can be any of ``'r'``, ``'rb'``, ``'w'``, ``'wb'``, - ``'a'``, or ``'ab'`` for binary mode, or ``'rt'``, ``'wt'``, or ``'at'`` for - text mode. The default is ``'rb'``. + ``'x'``, ``'xb'``, ``'a'`` or ``'ab'`` for binary mode, or ``'rt'``, + ``'wt'``, ``'xt'``, or ``'at'`` for text mode. The default is ``'rb'``. The *compresslevel* argument is an integer from 1 to 9, as for the :class:`BZ2File` constructor. @@ -54,6 +54,9 @@ All of the classes in this module may safely be accessed from multiple threads. .. versionadded:: 3.3 + .. versionchanged:: 3.4 + The ``'x'`` (exclusive creation) mode was added. + .. class:: BZ2File(filename, mode='r', buffering=None, compresslevel=9) @@ -64,8 +67,9 @@ All of the classes in this module may safely be accessed from multiple threads. be used to read or write the compressed data. The *mode* argument can be either ``'r'`` for reading (default), ``'w'`` for - overwriting, or ``'a'`` for appending. These can equivalently be given as - ``'rb'``, ``'wb'``, and ``'ab'`` respectively. + overwriting, ``'x'`` for exclusive creation, or ``'a'`` for appending. These + can equivalently be given as ``'rb'``, ``'wb'``, ``'xb'`` and ``'ab'`` + respectively. If *filename* is a file object (rather than an actual file name), a mode of ``'w'`` does not truncate the file, and is instead equivalent to ``'a'``. @@ -113,6 +117,9 @@ All of the classes in this module may safely be accessed from multiple threads. The ``'a'`` (append) mode was added, along with support for reading multi-stream files. + .. versionchanged:: 3.4 + The ``'x'`` (exclusive creation) mode was added. + Incremental (de)compression --------------------------- diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst index c4e7c60..fa13145 100644 --- a/Doc/library/cgi.rst +++ b/Doc/library/cgi.rst @@ -142,9 +142,11 @@ If a field represents an uploaded file, accessing the value via the method reads the entire file in memory as bytes. This may not be what you want. You can test for an uploaded file by testing either the :attr:`~FieldStorage.filename` attribute or the :attr:`~FieldStorage.file` -attribute. You can then read the data at leisure from the :attr:`!file` -attribute (the :func:`~io.RawIOBase.read` and :func:`~io.IOBase.readline` -methods will return bytes):: +attribute. You can then read the data from the :attr:`!file` +attribute before it is automatically closed as part of the garbage collection of +the :class:`FieldStorage` instance +(the :func:`~io.RawIOBase.read` and :func:`~io.IOBase.readline` methods will +return bytes):: fileitem = form["userfile"] if fileitem.file: @@ -176,6 +178,11 @@ actually be instances of the class :class:`MiniFieldStorage`. In this case, the A form submitted via POST that also has a query string will contain both :class:`FieldStorage` and :class:`MiniFieldStorage` items. +.. versionchanged:: 3.4 + The :attr:`~FieldStorage.file` attribute is automatically closed upon the + garbage collection of the creating :class:`FieldStorage` instance. + + Higher Level Interface ---------------------- diff --git a/Doc/library/code.rst b/Doc/library/code.rst index e869004..5b5d7cc 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -132,12 +132,15 @@ interpreter objects as well as the following additions. .. method:: InteractiveConsole.interact(banner=None) - Closely emulate the interactive Python console. The optional banner argument + Closely emulate the interactive Python console. The optional *banner* argument specify the banner to print before the first interaction; by default it prints a banner similar to the one printed by the standard Python interpreter, followed by the class name of the console object in parentheses (so as not to confuse this with the real interpreter -- since it's so close!). + .. versionchanged:: 3.4 + To suppress printing any banner, pass an empty string. + .. method:: InteractiveConsole.push(line) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 009ae26..fb3af3b 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -22,9 +22,10 @@ manages the codec and error handling lookup process. It defines the following functions: -.. function:: encode(obj, encoding='utf-8', errors='strict') +.. function:: encode(obj, [encoding[, errors]]) - Encodes *obj* using the codec registered for *encoding*. + Encodes *obj* using the codec registered for *encoding*. The default + encoding is ``utf-8``. *Errors* may be given to set the desired error handling scheme. The default error handler is ``strict`` meaning that encoding errors raise @@ -32,9 +33,10 @@ It defines the following functions: :exc:`UnicodeEncodeError`). Refer to :ref:`codec-base-classes` for more information on codec error handling. -.. function:: decode(obj, encoding='utf-8', errors='strict') +.. function:: decode(obj, [encoding[, errors]]) - Decodes *obj* using the codec registered for *encoding*. + Decodes *obj* using the codec registered for *encoding*. The default + encoding is ``utf-8``. *Errors* may be given to set the desired error handling scheme. The default error handler is ``strict`` meaning that decoding errors raise @@ -365,18 +367,23 @@ and implemented by all standard Python codecs: | | in :pep:`383`. | +-------------------------+-----------------------------------------------+ -In addition, the following error handlers are specific to a single codec: +In addition, the following error handlers are specific to Unicode encoding +schemes: -+-------------------+---------+-------------------------------------------+ -| Value | Codec | Meaning | -+===================+=========+===========================================+ -|``'surrogatepass'``| utf-8 | Allow encoding and decoding of surrogate | -| | | codes in UTF-8. | -+-------------------+---------+-------------------------------------------+ ++-------------------+------------------------+-------------------------------------------+ +| Value | Codec | Meaning | ++===================+========================+===========================================+ +|``'surrogatepass'``| utf-8, utf-16, utf-32, | Allow encoding and decoding of surrogate | +| | utf-16-be, utf-16-le, | codes in all the Unicode encoding schemes.| +| | utf-32-be, utf-32-le | | ++-------------------+------------------------+-------------------------------------------+ .. versionadded:: 3.1 The ``'surrogateescape'`` and ``'surrogatepass'`` error handlers. +.. versionchanged:: 3.4 + The ``'surrogatepass'`` error handlers now works with utf-16\* and utf-32\* codecs. + The set of allowed values can be extended via :meth:`register_error`. @@ -966,6 +973,10 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | cp037 | IBM037, IBM039 | English | +-----------------+--------------------------------+--------------------------------+ +| cp273 | 273, IBM273, csIBM273 | German | +| | | | +| | | .. versionadded:: 3.4 | ++-----------------+--------------------------------+--------------------------------+ | cp424 | EBCDIC-CP-HE, IBM424 | Hebrew | +-----------------+--------------------------------+--------------------------------+ | cp437 | 437, IBM437 | English | @@ -1022,6 +1033,10 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | cp1026 | ibm1026 | Turkish | +-----------------+--------------------------------+--------------------------------+ +| cp1125 | 1125, ibm1125, cp866u, ruscii | Ukrainian | +| | | | +| | | .. versionadded:: 3.4 | ++-----------------+--------------------------------+--------------------------------+ | cp1140 | ibm1140 | Western Europe | +-----------------+--------------------------------+--------------------------------+ | cp1250 | windows-1250 | Central and Eastern Europe | @@ -1167,6 +1182,12 @@ particular, the following variants typically exist: | utf_8_sig | | all languages | +-----------------+--------------------------------+--------------------------------+ +.. versionchanged:: 3.4 + The utf-16\* and utf-32\* encoders no longer allow surrogate code points + (U+D800--U+DFFF) to be encoded. The utf-32\* decoders no longer decode + byte sequences that correspond to surrogate code points. + + Python Specific Encodings ------------------------- @@ -1177,6 +1198,9 @@ common use case for codecs, the underlying codec infrastructure supports arbitrary data transforms rather than just text encodings). For asymmetric codecs, the stated purpose describes the encoding direction. +Text Encodings +^^^^^^^^^^^^^^ + The following codecs provide :class:`str` to :class:`bytes` encoding and :term:`bytes-like object` to :class:`str` decoding, similar to the Unicode text encodings. @@ -1223,57 +1247,83 @@ encodings. | | | .. deprecated:: 3.3 | +--------------------+---------+---------------------------+ -The following codecs provide :term:`bytes-like object` to :class:`bytes` -mappings. - - -.. tabularcolumns:: |l|L|L| - -+----------------------+---------------------------+------------------------------+ -| Codec | Purpose | Encoder/decoder | -+======================+===========================+==============================+ -| base64_codec [#b64]_ | Convert operand to MIME | :meth:`base64.b64encode`, | -| | base64 (the result always | :meth:`base64.b64decode` | -| | includes a trailing | | -| | ``'\n'``) | | -+----------------------+---------------------------+------------------------------+ -| bz2_codec | Compress the operand | :meth:`bz2.compress`, | -| | using bz2 | :meth:`bz2.decompress` | -+----------------------+---------------------------+------------------------------+ -| hex_codec | Convert operand to | :meth:`base64.b16encode`, | -| | hexadecimal | :meth:`base64.b16decode` | -| | representation, with two | | -| | digits per byte | | -+----------------------+---------------------------+------------------------------+ -| quopri_codec | Convert operand to MIME | :meth:`quopri.encodestring`, | -| | quoted printable | :meth:`quopri.decodestring` | -+----------------------+---------------------------+------------------------------+ -| uu_codec | Convert the operand using | :meth:`uu.encode`, | -| | uuencode | :meth:`uu.decode` | -+----------------------+---------------------------+------------------------------+ -| zlib_codec | Compress the operand | :meth:`zlib.compress`, | -| | using gzip | :meth:`zlib.decompress` | -+----------------------+---------------------------+------------------------------+ - -.. [#b64] Rather than accepting any :term:`bytes-like object`, - ``'base64_codec'`` accepts only :class:`bytes` and :class:`bytearray` for - encoding and only :class:`bytes`, :class:`bytearray`, and ASCII-only - instances of :class:`str` for decoding - - -The following codecs provide :class:`str` to :class:`str` mappings. +.. _binary-transforms: + +Binary Transforms +^^^^^^^^^^^^^^^^^ + +The following codecs provide binary transforms: :term:`bytes-like object` +to :class:`bytes` mappings. + + +.. tabularcolumns:: |l|L|L|L| + ++----------------------+------------------+------------------------------+------------------------------+ +| Codec | Aliases | Purpose | Encoder / decoder | ++======================+==================+==============================+==============================+ +| base64_codec [#b64]_ | base64, base_64 | Convert operand to MIME | :meth:`base64.b64encode` / | +| | | base64 (the result always | :meth:`base64.b64decode` | +| | | includes a trailing | | +| | | ``'\n'``) | | +| | | | | +| | | .. versionchanged:: 3.4 | | +| | | accepts any | | +| | | :term:`bytes-like object` | | +| | | as input for encoding and | | +| | | decoding | | ++----------------------+------------------+------------------------------+------------------------------+ +| bz2_codec | bz2 | Compress the operand | :meth:`bz2.compress` / | +| | | using bz2 | :meth:`bz2.decompress` | ++----------------------+------------------+------------------------------+------------------------------+ +| hex_codec | hex | Convert operand to | :meth:`base64.b16encode` / | +| | | hexadecimal | :meth:`base64.b16decode` | +| | | representation, with two | | +| | | digits per byte | | ++----------------------+------------------+------------------------------+------------------------------+ +| quopri_codec | quopri, | Convert operand to MIME | :meth:`quopri.encodestring` /| +| | quotedprintable, | quoted printable | :meth:`quopri.decodestring` | +| | quoted_printable | | | ++----------------------+------------------+------------------------------+------------------------------+ +| uu_codec | uu | Convert the operand using | :meth:`uu.encode` / | +| | | uuencode | :meth:`uu.decode` | ++----------------------+------------------+------------------------------+------------------------------+ +| zlib_codec | zip, zlib | Compress the operand | :meth:`zlib.compress` / | +| | | using gzip | :meth:`zlib.decompress` | ++----------------------+------------------+------------------------------+------------------------------+ + +.. [#b64] In addition to :term:`bytes-like objects <bytes-like object>`, + ``'base64_codec'`` also accepts ASCII-only instances of :class:`str` for + decoding -.. tabularcolumns:: |l|L| +.. versionadded:: 3.2 + Restoration of the binary transforms. + +.. versionchanged:: 3.4 + Restoration of the aliases for the binary transforms. -+--------------------+---------------------------+ -| Codec | Purpose | -+====================+===========================+ -| rot_13 | Returns the Caesar-cypher | -| | encryption of the operand | -+--------------------+---------------------------+ + +.. _text-transforms: + +Text Transforms +^^^^^^^^^^^^^^^ + +The following codec provides a text transform: a :class:`str` to :class:`str` +mapping. + +.. tabularcolumns:: |l|l|L| + ++--------------------+---------+---------------------------+ +| Codec | Aliases | Purpose | ++====================+=========+===========================+ +| rot_13 | rot13 | Returns the Caesar-cypher | +| | | encryption of the operand | ++--------------------+---------+---------------------------+ .. versionadded:: 3.2 - bytes-to-bytes and str-to-str codecs. + Restoration of the ``rot_13`` text transform. + +.. versionchanged:: 3.4 + Restoration of the ``rot13`` alias. :mod:`encodings.idna` --- Internationalized Domain Names in Applications diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 4b21932..356f473 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -15,7 +15,7 @@ import itertools __name__ = '<doctest>' -**Source code:** :source:`Lib/collections/abc.py` +**Source code:** :source:`Lib/_collections_abc.py` -------------- diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 4def1dc..f5fe12a 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -76,14 +76,19 @@ The class can be used to simulate nested scopes and is useful in templating. be modified to change which mappings are searched. The list should always contain at least one mapping. - .. method:: new_child() + .. method:: new_child(m=None) - Returns a new :class:`ChainMap` containing a new :class:`dict` followed by - all of the maps in the current instance. A call to ``d.new_child()`` is - equivalent to: ``ChainMap({}, *d.maps)``. This method is used for + Returns a new :class:`ChainMap` containing a new map followed by + all of the maps in the current instance. If ``m`` is specified, + it becomes the new map at the front of the list of mappings; if not + specified, an empty dict is used, so that a call to ``d.new_child()`` + is equivalent to: ``ChainMap({}, *d.maps)``. This method is used for creating subcontexts that can be updated without altering values in any of the parent mappings. + .. versionchanged:: 3.4 + The optional ``m`` parameter was added. + .. attribute:: parents Property returning a new :class:`ChainMap` containing all of the maps in @@ -369,10 +374,6 @@ or subtracting from an empty counter. .. seealso:: - * `Counter class <http://code.activestate.com/recipes/576611/>`_ - adapted for Python 2.5 and an early `Bag recipe - <http://code.activestate.com/recipes/259174/>`_ for Python 2.4. - * `Bag class <http://www.gnu.org/software/smalltalk/manual-base/html_node/Bag.html>`_ in Smalltalk. @@ -915,11 +916,6 @@ and more efficient to use a simple class declaration: >>> class Status: open, pending, closed = range(3) -.. seealso:: - - * `Named tuple recipe <http://code.activestate.com/recipes/500261/>`_ - adapted for Python 2.4. - * `Recipe for named tuple abstract base class with a metaclass mix-in <http://code.activestate.com/recipes/577629-namedtupleabc-abstract-base-class-mix-in-for-named/>`_ by Jan Kaliszewski. Besides providing an :term:`abstract base class` for @@ -982,10 +978,6 @@ The :class:`OrderedDict` constructor and :meth:`update` method both accept keyword arguments, but their order is lost because Python's function call semantics pass-in keyword arguments using a regular unordered dictionary. -.. seealso:: - - `Equivalent OrderedDict recipe <http://code.activestate.com/recipes/576693/>`_ - that runs on Python 2.4 or later. :class:`OrderedDict` Examples and Recipes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index b12c217..41e9e1b 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -20,7 +20,8 @@ compile Python sources. .. program:: compileall -.. cmdoption:: [directory|file]... +.. cmdoption:: directory ... + file ... Positional arguments are files to compile or directories that contain source files, traversed recursively. If no argument is given, behave as if diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst index f2a512f..0de281b 100644 --- a/Doc/library/concurrency.rst +++ b/Doc/library/concurrency.rst @@ -20,7 +20,6 @@ multitasking). Here's an overview: subprocess.rst sched.rst queue.rst - select.rst The following are support modules for some of the above services: diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 575b1ea..0495737 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -164,6 +164,9 @@ uses a pool of processes to execute calls asynchronously. allows it to side-step the :term:`Global Interpreter Lock` but also means that only picklable objects can be executed and returned. +The ``__main__`` module must be importable by worker subprocesses. This means +that :class:`ProcessPoolExecutor` will not work in the interactive interpreter. + Calling :class:`Executor` or :class:`Future` methods from a callable submitted to a :class:`ProcessPoolExecutor` will result in deadlock. diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index fba48f4..9f174d7 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -95,6 +95,83 @@ Functions and classes provided: ``page.close()`` will be called when the :keyword:`with` block is exited. +.. function:: suppress(*exceptions) + + Return a context manager that suppresses any of the specified exceptions + if they occur in the body of a with statement and then resumes execution + with the first statement following the end of the with statement. + + As with any other mechanism that completely suppresses exceptions, this + context manager should be used only to cover very specific errors where + silently continuing with program execution is known to be the right + thing to do. + + For example:: + + from contextlib import suppress + + with suppress(FileNotFoundError): + os.remove('somefile.tmp') + + with suppress(FileNotFoundError): + os.remove('someotherfile.tmp') + + This code is equivalent to:: + + try: + os.remove('somefile.tmp') + except FileNotFoundError: + pass + + try: + os.remove('someotherfile.tmp') + except FileNotFoundError: + pass + + This context manager is :ref:`reentrant <reentrant-cms>`. + + .. versionadded:: 3.4 + + +.. function:: redirect_stdout(new_target) + + Context manager for temporarily redirecting :data:`sys.stdout` to + another file or file-like object. + + This tool adds flexibility to existing functions or classes whose output + is hardwired to stdout. + + For example, the output of :func:`help` normally is sent to *sys.stdout*. + You can capture that output in a string by redirecting the output to a + :class:`io.StringIO` object:: + + f = io.StringIO() + with redirect_stdout(f): + help(pow) + s = f.getvalue() + + To send the output of :func:`help` to a file on disk, redirect the output + to a regular file:: + + with open('help.txt', 'w') as f: + with redirect_stdout(f): + help(pow) + + To send the output of :func:`help` to *sys.stderr*:: + + with redirect_stdout(sys.stderr): + help(pow) + + Note that the global side effect on :data:`sys.stdout` means that this + context manager is not suitable for use in library code and most threaded + applications. It also has no effect on the output of subprocesses. + However, it is still a useful approach for many utility scripts. + + This context manager is :ref:`reusable but not reentrant <reusable-cms>`. + + .. versionadded:: 3.4 + + .. class:: ContextDecorator() A base class that enables a context manager to also be used as a decorator. @@ -294,7 +371,7 @@ some of the context managers being optional:: with ExitStack() as stack: for resource in resources: stack.enter_context(resource) - if need_special resource: + if need_special_resource(): special = acquire_special_resource() stack.callback(release_special_resource, special) # Perform operations that use the acquired resources @@ -520,3 +597,153 @@ an explicit ``with`` statement. The specification, background, and examples for the Python :keyword:`with` statement. +.. _single-use-reusable-and-reentrant-cms: + +Single use, reusable and reentrant context managers +--------------------------------------------------- + +Most context managers are written in a way that means they can only be +used effectively in a :keyword:`with` statement once. These single use +context managers must be created afresh each time they're used - +attempting to use them a second time will trigger an exception or +otherwise not work correctly. + +This common limitation means that it is generally advisable to create +context managers directly in the header of the :keyword:`with` statement +where they are used (as shown in all of the usage examples above). + +Files are an example of effectively single use context managers, since +the first :keyword:`with` statement will close the file, preventing any +further IO operations using that file object. + +Context managers created using :func:`contextmanager` are also single use +context managers, and will complain about the underlying generator failing +to yield if an attempt is made to use them a second time:: + + >>> from contextlib import contextmanager + >>> @contextmanager + ... def singleuse(): + ... print("Before") + ... yield + ... print("After") + ... + >>> cm = singleuse() + >>> with cm: + ... pass + ... + Before + After + >>> with cm: + ... pass + ... + Traceback (most recent call last): + ... + RuntimeError: generator didn't yield + + +.. _reentrant-cms: + +Reentrant context managers +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +More sophisticated context managers may be "reentrant". These context +managers can not only be used in multiple :keyword:`with` statements, +but may also be used *inside* a :keyword:`with` statement that is already +using the same context manager. + +:class:`threading.RLock` is an example of a reentrant context manager, as are +:func:`suppress` and :func:`redirect_stdout`. Here's a very simple example of +reentrant use:: + + >>> from contextlib import redirect_stdout + >>> from io import StringIO + >>> stream = StringIO() + >>> write_to_stream = redirect_stdout(stream) + >>> with write_to_stream: + ... print("This is written to the stream rather than stdout") + ... with write_to_stream: + ... print("This is also written to the stream") + ... + >>> print("This is written directly to stdout") + This is written directly to stdout + >>> print(stream.getvalue()) + This is written to the stream rather than stdout + This is also written to the stream + +Real world examples of reentrancy are more likely to involve multiple +functions calling each other and hence be far more complicated than this +example. + +Note also that being reentrant is *not* the same thing as being thread safe. +:func:`redirect_stdout`, for example, is definitely not thread safe, as it +makes a global modification to the system state by binding :data:`sys.stdout` +to a different stream. + + +.. _reusable-cms: + +Reusable context managers +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Distinct from both single use and reentrant context managers are "reusable" +context managers (or, to be completely explicit, "reusable, but not +reentrant" context managers, since reentrant context managers are also +reusable). These context managers support being used multiple times, but +will fail (or otherwise not work correctly) if the specific context manager +instance has already been used in a containing with statement. + +:class:`threading.Lock` is an example of a reusable, but not reentrant, +context manager (for a reentrant lock, it is necessary to use +:class:`threading.RLock` instead). + +Another example of a reusable, but not reentrant, context manager is +:class:`ExitStack`, as it invokes *all* currently registered callbacks +when leaving any with statement, regardless of where those callbacks +were added:: + + >>> from contextlib import ExitStack + >>> stack = ExitStack() + >>> with stack: + ... stack.callback(print, "Callback: from first context") + ... print("Leaving first context") + ... + Leaving first context + Callback: from first context + >>> with stack: + ... stack.callback(print, "Callback: from second context") + ... print("Leaving second context") + ... + Leaving second context + Callback: from second context + >>> with stack: + ... stack.callback(print, "Callback: from outer context") + ... with stack: + ... stack.callback(print, "Callback: from inner context") + ... print("Leaving inner context") + ... print("Leaving outer context") + ... + Leaving inner context + Callback: from inner context + Callback: from outer context + Leaving outer context + +As the output from the example shows, reusing a single stack object across +multiple with statements works correctly, but attempting to nest them +will cause the stack to be cleared at the end of the innermost with +statement, which is unlikely to be desirable behaviour. + +Using separate :class:`ExitStack` instances instead of reusing a single +instance avoids that problem:: + + >>> from contextlib import ExitStack + >>> with ExitStack() as outer_stack: + ... outer_stack.callback(print, "Callback: from outer context") + ... with ExitStack() as inner_stack: + ... inner_stack.callback(print, "Callback: from inner context") + ... print("Leaving inner context") + ... print("Leaving outer context") + ... + Leaving inner context + Callback: from inner context + Leaving outer context + Callback: from outer context diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 314636e..f3e60b4 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -12,7 +12,7 @@ The :mod:`curses` module provides an interface to the curses library, the de-facto standard for portable advanced terminal handling. While curses is most widely used in the Unix environment, versions are available -for DOS, OS/2, and possibly other systems as well. This extension module is +for Windows, DOS, and possibly other systems as well. This extension module is designed to match the API of ncurses, an open-source curses library hosted on Linux and the BSD variants of Unix. diff --git a/Doc/library/datatypes.rst b/Doc/library/datatypes.rst index d0382e0..48af082 100644 --- a/Doc/library/datatypes.rst +++ b/Doc/library/datatypes.rst @@ -30,3 +30,4 @@ The following modules are documented in this chapter: copy.rst pprint.rst reprlib.rst + enum.rst diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 065e850..e4f1eb2 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -170,10 +170,12 @@ dates or times. * ``0 <= seconds < 3600*24`` (the number of seconds in one day) * ``-999999999 <= days <= 999999999`` - If any argument is a float and there are fractional microseconds, the fractional - microseconds left over from all arguments are combined and their sum is rounded - to the nearest microsecond. If no argument is a float, the conversion and - normalization processes are exact (no information is lost). + If any argument is a float and there are fractional microseconds, + the fractional microseconds left over from all arguments are + combined and their sum is rounded to the nearest microsecond using + round-half-to-even tiebreaker. If no argument is a float, the + conversion and normalization processes are exact (no information is + lost). If the normalized value of days lies outside the indicated range, :exc:`OverflowError` is raised. diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 81a05c7..f5496d5 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -73,33 +73,39 @@ Key and values are always stored as bytes. This means that when strings are used they are implicitly converted to the default encoding before being stored. +These objects also support being used in a :keyword:`with` statement, which +will automatically close them when done. + +.. versionchanged:: 3.4 + Added native support for the context management protocol to the objects + returned by :func:`.open`. + The following example records some hostnames and a corresponding title, and then prints out the contents of the database:: import dbm # Open database, creating it if necessary. - db = dbm.open('cache', 'c') + with dbm.open('cache', 'c') as db: - # Record some values - db[b'hello'] = b'there' - db['www.python.org'] = 'Python Website' - db['www.cnn.com'] = 'Cable News Network' + # Record some values + db[b'hello'] = b'there' + db['www.python.org'] = 'Python Website' + db['www.cnn.com'] = 'Cable News Network' - # Note that the keys are considered bytes now. - assert db[b'www.python.org'] == b'Python Website' - # Notice how the value is now in bytes. - assert db['www.cnn.com'] == b'Cable News Network' + # Note that the keys are considered bytes now. + assert db[b'www.python.org'] == b'Python Website' + # Notice how the value is now in bytes. + assert db['www.cnn.com'] == b'Cable News Network' - # Often-used methods of the dict interface work too. - print(db.get('python.org', b'not present')) + # Often-used methods of the dict interface work too. + print(db.get('python.org', b'not present')) - # Storing a non-string key or value will raise an exception (most - # likely a TypeError). - db['www.yahoo.com'] = 4 + # Storing a non-string key or value will raise an exception (most + # likely a TypeError). + db['www.yahoo.com'] = 4 - # Close when done. - db.close() + # db is automatically closed when leaving the with statement. .. seealso:: diff --git a/Doc/library/debug.rst b/Doc/library/debug.rst index c69fb1c..88a2fa6 100644 --- a/Doc/library/debug.rst +++ b/Doc/library/debug.rst @@ -15,3 +15,4 @@ allowing you to identify bottlenecks in your programs. profile.rst timeit.rst trace.rst + tracemalloc.rst diff --git a/Doc/library/development.rst b/Doc/library/development.rst index 2368769..06e7048 100644 --- a/Doc/library/development.rst +++ b/Doc/library/development.rst @@ -23,4 +23,3 @@ The list of modules described in this chapter is: unittest.mock-examples.rst 2to3.rst test.rst - venv.rst diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index 81dc0f1..2a75d2c 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -226,8 +226,8 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. :file:`Tools/scripts/ndiff.py` is a command-line front-end to this function. - >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(1), - ... 'ore\ntree\nemu\n'.splitlines(1)) + >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(keepends=True), + ... 'ore\ntree\nemu\n'.splitlines(keepends=True)) >>> print(''.join(diff), end="") - one ? ^ @@ -250,8 +250,8 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. Example: - >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(1), - ... 'ore\ntree\nemu\n'.splitlines(1)) + >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(keepends=True), + ... 'ore\ntree\nemu\n'.splitlines(keepends=True)) >>> diff = list(diff) # materialize the generated delta into a list >>> print(''.join(restore(diff, 1)), end="") one @@ -650,7 +650,7 @@ obtained from the :meth:`~io.BaseIO.readlines` method of file-like objects): ... 2. Explicit is better than implicit. ... 3. Simple is better than complex. ... 4. Complex is better than complicated. - ... '''.splitlines(1) + ... '''.splitlines(keepends=True) >>> len(text1) 4 >>> text1[0][-1] @@ -659,7 +659,7 @@ obtained from the :meth:`~io.BaseIO.readlines` method of file-like objects): ... 3. Simple is better than complex. ... 4. Complicated is better than complex. ... 5. Flat is better than nested. - ... '''.splitlines(1) + ... '''.splitlines(keepends=True) Next we instantiate a Differ object: diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index ec7112d..d86550f 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -26,7 +26,8 @@ Example: Given the function :func:`myfunc`:: def myfunc(alist): return len(alist) -the following command can be used to get the disassembly of :func:`myfunc`:: +the following command can be used to display the disassembly of +:func:`myfunc`:: >>> dis.dis(myfunc) 2 0 LOAD_GLOBAL 0 (len) @@ -36,8 +37,77 @@ the following command can be used to get the disassembly of :func:`myfunc`:: (The "2" is a line number). -The :mod:`dis` module defines the following functions and constants: +Bytecode analysis +----------------- +.. versionadded:: 3.4 + +The bytecode analysis API allows pieces of Python code to be wrapped in a +:class:`Bytecode` object that provides easy access to details of the +compiled code. + +.. class:: Bytecode(x, *, first_line=None, current_offset=None) + + Analyse the bytecode corresponding to a function, method, string of + source code, or a code object (as returned by :func:`compile`). + + This is a convenience wrapper around many of the functions listed below, + most notably :func:`get_instructions`, as iterating over a + :class:`Bytecode` instance yields the bytecode operations as + :class:`Instruction` instances. + + If *first_line* is not None, it indicates the line number that should + be reported for the first source line in the disassembled code. + Otherwise, the source line information (if any) is taken directly from + the disassembled code object. + + If *current_offset* is not None, it refers to an instruction offset + in the disassembled code. Setting this means :meth:`dis` will display + a "current instruction" marker against the specified opcode. + + .. classmethod:: from_traceback(tb) + + Construct a :class:`Bytecode` instance from the given traceback, + setting *current_offset* to the instruction responsible for the + exception. + + .. data:: codeobj + + The compiled code object. + + .. data:: first_line + + The first source line of the code object (if available) + + .. method:: dis() + + Return a formatted view of the bytecode operations (the same as + printed by :func:`dis`, but returned as a multi-line string). + + .. method:: info() + + Return a formatted multi-line string with detailed information about the + code object, like :func:`code_info`. + +Example:: + + >>> bytecode = dis.Bytecode(myfunc) + >>> for instr in bytecode: + ... print(instr.opname) + ... + LOAD_GLOBAL + LOAD_FAST + CALL_FUNCTION + RETURN_VALUE + + +Analysis functions +------------------ + +The :mod:`dis` module also defines the following analysis functions that +convert the input directly to the desired output. They can be useful if +only a single operation is being performed, so the intermediate analysis +object isn't useful: .. function:: code_info(x) @@ -51,17 +121,22 @@ The :mod:`dis` module defines the following functions and constants: .. versionadded:: 3.2 -.. function:: show_code(x) +.. function:: show_code(x, *, file=None) Print detailed code object information for the supplied function, method, - source code string or code object to stdout. + source code string or code object to *file* (or ``sys.stdout`` if *file* + is not specified). - This is a convenient shorthand for ``print(code_info(x))``, intended for - interactive exploration at the interpreter prompt. + This is a convenient shorthand for ``print(code_info(x), file=file)``, + intended for interactive exploration at the interpreter prompt. .. versionadded:: 3.2 -.. function:: dis(x=None) + .. versionchanged:: 3.4 + Added ``file`` parameter + + +.. function:: dis(x=None, *, file=None) Disassemble the *x* object. *x* can denote either a module, a class, a method, a function, a code object, a string of source code or a byte sequence @@ -72,16 +147,28 @@ The :mod:`dis` module defines the following functions and constants: disassembled. If no object is provided, this function disassembles the last traceback. + The disassembly is written as text to the supplied ``file`` argument if + provided and to ``sys.stdout`` otherwise. -.. function:: distb(tb=None) + .. versionchanged:: 3.4 + Added ``file`` parameter + + +.. function:: distb(tb=None, *, file=None) Disassemble the top-of-stack function of a traceback, using the last traceback if none was passed. The instruction causing the exception is indicated. + The disassembly is written as text to the supplied ``file`` argument if + provided and to ``sys.stdout`` otherwise. + + .. versionchanged:: 3.4 + Added ``file`` parameter + -.. function:: disassemble(code, lasti=-1) - disco(code, lasti=-1) +.. function:: disassemble(code, lasti=-1, *, file=None) + disco(code, lasti=-1, *, file=None) Disassemble a code object, indicating the last instruction if *lasti* was provided. The output is divided in the following columns: @@ -97,6 +184,28 @@ The :mod:`dis` module defines the following functions and constants: The parameter interpretation recognizes local and global variable names, constant values, branch targets, and compare operators. + The disassembly is written as text to the supplied ``file`` argument if + provided and to ``sys.stdout`` otherwise. + + .. versionchanged:: 3.4 + Added ``file`` parameter + + +.. function:: get_instructions(x, *, first_line=None) + + Return an iterator over the instructions in the supplied function, method, + source code string or code object. + + The iterator generates a series of :class:`Instruction` named tuples + giving the details of each operation in the supplied code. + + If *first_line* is not None, it indicates the line number that should + be reported for the first source line in the disassembled code. + Otherwise, the source line information (if any) is taken directly from + the disassembled code object. + + .. versionadded:: 3.4 + .. function:: findlinestarts(code) @@ -111,60 +220,66 @@ The :mod:`dis` module defines the following functions and constants: return a list of these offsets. -.. data:: opname +.. function:: stack_effect(opcode, [oparg]) - Sequence of operation names, indexable using the bytecode. + Compute the stack effect of *opcode* with argument *oparg*. + .. versionadded:: 3.4 -.. data:: opmap +.. _bytecodes: - Dictionary mapping operation names to bytecodes. +Python Bytecode Instructions +---------------------------- +The :func:`get_instructions` function and :class:`Bytecode` class provide +details of bytecode instructions as :class:`Instruction` instances: -.. data:: cmp_op +.. class:: Instruction - Sequence of all compare operation names. + Details for a bytecode operation + .. data:: opcode -.. data:: hasconst + numeric code for operation, corresponding to the opcode values listed + below and the bytecode values in the :ref:`opcode_collections`. - Sequence of bytecodes that have a constant parameter. + .. data:: opname -.. data:: hasfree + human readable name for operation - Sequence of bytecodes that access a free variable. + .. data:: arg -.. data:: hasname + numeric argument to operation (if any), otherwise None - Sequence of bytecodes that access an attribute by name. + .. data:: argval -.. data:: hasjrel + resolved arg value (if known), otherwise same as arg - Sequence of bytecodes that have a relative jump target. + .. data:: argrepr -.. data:: hasjabs + human readable description of operation argument - Sequence of bytecodes that have an absolute jump target. + .. data:: offset -.. data:: haslocal + start index of operation within bytecode sequence - Sequence of bytecodes that access a local variable. + .. data:: starts_line -.. data:: hascompare + line started by this opcode (if any), otherwise None - Sequence of bytecodes of Boolean operations. + .. data:: is_jump_target -.. _bytecodes: + ``True`` if other code jumps to here, otherwise ``False`` + + .. versionadded:: 3.4 -Python Bytecode Instructions ----------------------------- The Python compiler currently generates the following bytecode instructions. @@ -506,12 +621,6 @@ the stack so that it is available for further iterations of the loop. .. XXX explain the WHY stuff! -.. opcode:: STORE_LOCALS - - Pops TOS from the stack and stores it as the current frame's ``f_locals``. - This is used in class construction. - - All of the following opcodes expect arguments. An argument is two bytes, with the more significant byte last. @@ -722,6 +831,13 @@ the more significant byte last. Pushes a reference to the object the cell contains on the stack. +.. opcode:: LOAD_CLASSDEREF (i) + + Much like :opcode:`LOAD_DEREF` but first checks the locals dictionary before + consulting the cell. This is used for loading free variables in class + bodies. + + .. opcode:: STORE_DEREF (i) Stores TOS into the cell contained in slot *i* of the cell and free variable @@ -820,3 +936,62 @@ the more significant byte last. which don't take arguments ``< HAVE_ARGUMENT`` and those which do ``>= HAVE_ARGUMENT``. +.. _opcode_collections: + +Opcode collections +------------------ + +These collections are provided for automatic introspection of bytecode +instructions: + +.. data:: opname + + Sequence of operation names, indexable using the bytecode. + + +.. data:: opmap + + Dictionary mapping operation names to bytecodes. + + +.. data:: cmp_op + + Sequence of all compare operation names. + + +.. data:: hasconst + + Sequence of bytecodes that have a constant parameter. + + +.. data:: hasfree + + Sequence of bytecodes that access a free variable (note that 'free' in + this context refers to names in the current scope that are referenced by + inner scopes or names in outer scopes that are referenced from this scope. + It does *not* include references to global or builtin scopes). + + +.. data:: hasname + + Sequence of bytecodes that access an attribute by name. + + +.. data:: hasjrel + + Sequence of bytecodes that have a relative jump target. + + +.. data:: hasjabs + + Sequence of bytecodes that have an absolute jump target. + + +.. data:: haslocal + + Sequence of bytecodes that access a local variable. + + +.. data:: hascompare + + Sequence of bytecodes of Boolean operations. diff --git a/Doc/library/distribution.rst b/Doc/library/distribution.rst new file mode 100644 index 0000000..fb3f5df --- /dev/null +++ b/Doc/library/distribution.rst @@ -0,0 +1,14 @@ +*********************************** +Software Packaging and Distribution +*********************************** + +These libraries help you with publishing and installing Python software. +While these modules are designed to work in conjunction with the +`Python Package Index <https://pypi.python.org>`__, they can also be used +with a local index server, or without any index server at all. + +.. toctree:: + + distutils.rst + ensurepip.rst + venv.rst diff --git a/Doc/library/distutils.rst b/Doc/library/distutils.rst index 6666a9b..8e3732b 100644 --- a/Doc/library/distutils.rst +++ b/Doc/library/distutils.rst @@ -12,14 +12,15 @@ additional modules into a Python installation. The new modules may be either 100%-pure Python, or may be extension modules written in C, or may be collections of Python packages which include modules coded in both Python and C. +Most Python users will *not* want to use this module directly, but instead +use the cross-version tools maintained by the Python Packaging Authority. +Refer to the `Python Packaging User Guide <http://packaging.python.org>`_ +for more information. -User documentation and API reference are provided in another document: +For the benefits of packaging tool authors and users seeking a deeper +understanding of the details of the current packaging and distribution +system, the legacy :mod:`distutils` based user documentation and API +reference remain available: -.. seealso:: - - :ref:`distutils-index` - The manual for developers and packagers of Python modules. This describes - how to prepare :mod:`distutils`\ -based packages so that they may be - easily installed into an existing Python installation. It also contains - instructions for end-users wanting to install a distutils-based package, - :ref:`install-index`. +* :ref:`install-index` +* :ref:`distutils-index` diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 222c719..50626e9 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -278,6 +278,10 @@ strings are treated as if they were docstrings. In output, a key ``K`` in Any classes found are recursively searched similarly, to test docstrings in their contained methods and nested classes. +.. impl-detail:: + Prior to version 3.4, extension modules written in C were not fully + searched by doctest. + .. _doctest-finding-examples: @@ -495,7 +499,11 @@ Option Flags A number of option flags control various aspects of doctest's behavior. Symbolic names for the flags are supplied as module constants, which can be or'ed together and passed to various functions. The names can also be used in -:ref:`doctest directives <doctest-directives>`. +:ref:`doctest directives <doctest-directives>`, and may be passed to the +doctest command line interface via the ``-o`` option. + +.. versionadded:: 3.4 + The ``-o`` command line option. The first group of options define test semantics, controlling aspects of how doctest decides whether actual output matches an example's expected output: @@ -633,6 +641,19 @@ The second group of options controls how test failures are reported: the output is suppressed. +.. data:: FAIL_FAST + + When specified, exit after the first failing example and don't attempt to run + the remaining examples. Thus, the number of failures reported will be at most + 1. This flag may be useful during debugging, since examples after the first + failure won't even produce debugging output. + + The doctest command line accepts the option ``-f`` as a shorthand for ``-o + FAIL_FAST``. + + .. versionadded:: 3.4 + + .. data:: REPORTING_FLAGS A bitmask or'ing together all the reporting flags above. @@ -1269,9 +1290,8 @@ DocTestFinder objects A processing class used to extract the :class:`DocTest`\ s that are relevant to a given object, from its docstring and the docstrings of its contained objects. - :class:`DocTest`\ s can currently be extracted from the following object types: - modules, functions, classes, methods, staticmethods, classmethods, and - properties. + :class:`DocTest`\ s can be extracted from modules, classes, functions, + methods, staticmethods, classmethods, and properties. The optional argument *verbose* can be used to display the objects searched by the finder. It defaults to ``False`` (no output). diff --git a/Doc/library/email-examples.rst b/Doc/library/email-examples.rst index 32cecf3..cbbcb78 100644 --- a/Doc/library/email-examples.rst +++ b/Doc/library/email-examples.rst @@ -40,6 +40,36 @@ text version: [2]_ .. literalinclude:: ../includes/email-alternative.py +.. _email-contentmanager-api-examples: + +Examples using the Provisional API +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Here is a reworking of the last example using the provisional API. To make +things a bit more interesting, we include a related image in the html part, and +we save a copy of what we are going to send to disk, as well as sending it. + +This example also shows how easy it is to include non-ASCII, and simplifies the +sending of the message using the :meth:`.send_message` method of the +:mod:`smtplib` module. + +.. literalinclude:: ../includes/email-alternative-new-api.py + +If we were instead sent the message from the last example, here is one +way we could process it: + +.. literalinclude:: ../includes/email-read-alternative-new-api.py + +Up to the prompt, the output from the above is:: + + To: Penelope Pussycat <"penelope@example.com">, Fabrette Pussycat <"fabrette@example.com"> + From: Pepé Le Pew <pepe@example.com> + Subject: Ayons asperges pour le déjeuner + + Salut! + + Cela ressemble à un excellent recipie[1] déjeuner. + + .. rubric:: Footnotes .. [1] Thanks to Matthew Dixon Cowles for the original inspiration and examples. diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst new file mode 100644 index 0000000..8f0bfdb --- /dev/null +++ b/Doc/library/email.contentmanager.rst @@ -0,0 +1,430 @@ +:mod:`email.contentmanager`: Managing MIME Content +-------------------------------------------------- + +.. module:: email.contentmanager + :synopsis: Storing and Retrieving Content from MIME Parts + +.. moduleauthor:: R. David Murray <rdmurray@bitdance.com> +.. sectionauthor:: R. David Murray <rdmurray@bitdance.com> + + +.. note:: + + The contentmanager module has been included in the standard library on a + :term:`provisional basis <provisional package>`. Backwards incompatible + changes (up to and including removal of the module) may occur if deemed + necessary by the core developers. + +.. versionadded:: 3.4 + as a :term:`provisional module <provisional package>`. + +The :mod:`~email.message` module provides a class that can represent an +arbitrary email message. That basic message model has a useful and flexible +API, but it provides only a lower-level API for interacting with the generic +parts of a message (the headers, generic header parameters, and the payload, +which may be a list of sub-parts). This module provides classes and tools +that provide an enhanced and extensible API for dealing with various specific +types of content, including the ability to retrieve the content of the message +as a specialized object type rather than as a simple bytes object. The module +automatically takes care of the RFC-specified MIME details (required headers +and parameters, etc.) for the certain common content types content properties, +and support for additional types can be added by an application using the +extension mechanisms. + +This module defines the eponymous "Content Manager" classes. The base +:class:`.ContentManager` class defines an API for registering content +management functions which extract data from ``Message`` objects or insert data +and headers into ``Message`` objects, thus providing a way of converting +between ``Message`` objects containing data and other representations of that +data (Python data types, specialized Python objects, external files, etc). The +module also defines one concrete content manager: :data:`raw_data_manager` +converts between MIME content types and ``str`` or ``bytes`` data. It also +provides a convenient API for managing the MIME parameters when inserting +content into ``Message``\ s. It also handles inserting and extracting +``Message`` objects when dealing with the ``message/rfc822`` content type. + +Another part of the enhanced interface is subclasses of +:class:`~email.message.Message` that provide new convenience API functions, +including convenience methods for calling the Content Managers derived from +this module. + +.. note:: + + Although :class:`.EmailMessage` and :class:`.MIMEPart` are currently + documented in this module because of the provisional nature of the code, the + implementation lives in the :mod:`email.message` module. + +.. currentmodule:: email.message + +.. class:: EmailMessage(policy=default) + + If *policy* is specified (it must be an instance of a :mod:`~email.policy` + class) use the rules it specifies to udpate and serialize the representation + of the message. If *policy* is not set, use the + :class:`~email.policy.default` policy, which follows the rules of the email + RFCs except for line endings (instead of the RFC mandated ``\r\n``, it uses + the Python standard ``\n`` line endings). For more information see the + :mod:`~email.policy` documentation. + + This class is a subclass of :class:`~email.message.Message`. It adds + the following methods: + + + .. attribute:: is_attachment + + Set to ``True`` if there is a :mailheader:`Content-Disposition` header + and its (case insensitive) value is ``attachment``, ``False`` otherwise. + + + .. method:: get_body(preferencelist=('related', 'html', 'plain')) + + Return the MIME part that is the best candidate to be the "body" of the + message. + + *preferencelist* must be a sequence of strings from the set ``related``, + ``html``, and ``plain``, and indicates the order of preference for the + content type of the part returned. + + Start looking for candidate matches with the object on which the + ``get_body`` method is called. + + If ``related`` is not included in *preferencelist*, consider the root + part (or subpart of the root part) of any related encountered as a + candidate if the (sub-)part matches a preference. + + When encountering a ``multipart/related``, check the ``start`` parameter + and if a part with a matching :mailheader:`Content-ID` is found, consider + only it when looking for candidate matches. Otherwise consider only the + first (default root) part of the ``multipart/related``. + + If a part has a :mailheader:`Content-Disposition` header, only consider + the part a candidate match if the value of the header is ``inline``. + + If none of the candidates matches any of the preferences in + *preferneclist*, return ``None``. + + Notes: (1) For most applications the only *preferencelist* combinations + that really make sense are ``('plain',)``, ``('html', 'plain')``, and the + default, ``('related', 'html', 'plain')``. (2) Because matching starts + with the object on which ``get_body`` is called, calling ``get_body`` on + a ``multipart/related`` will return the object itself unless + *preferencelist* has a non-default value. (3) Messages (or message parts) + that do not specify a :mailheader:`Content-Type` or whose + :mailheader:`Content-Type` header is invalid will be treated as if they + are of type ``text/plain``, which may occasionally cause ``get_body`` to + return unexpected results. + + + .. method:: iter_attachments() + + Return an iterator over all of the parts of the message that are not + candidate "body" parts. That is, skip the first occurrence of each of + ``text/plain``, ``text/html``, ``multipart/related``, or + ``multipart/alternative`` (unless they are explicitly marked as + attachments via :mailheader:`Content-Disposition: attachment`), and + return all remaining parts. When applied directly to a + ``multipart/related``, return an iterator over the all the related parts + except the root part (ie: the part pointed to by the ``start`` parameter, + or the first part if there is no ``start`` parameter or the ``start`` + parameter doesn't match the :mailheader:`Content-ID` of any of the + parts). When applied directly to a ``multipart/alternative`` or a + non-``multipart``, return an empty iterator. + + + .. method:: iter_parts() + + Return an iterator over all of the immediate sub-parts of the message, + which will be empty for a non-``multipart``. (See also + :meth:`~email.message.walk`.) + + + .. method:: get_content(*args, content_manager=None, **kw) + + Call the ``get_content`` method of the *content_manager*, passing self + as the message object, and passing along any other arguments or keywords + as additional arguments. If *content_manager* is not specified, use + the ``content_manager`` specified by the current :mod:`~email.policy`. + + + .. method:: set_content(*args, content_manager=None, **kw) + + Call the ``set_content`` method of the *content_manager*, passing self + as the message object, and passing along any other arguments or keywords + as additional arguments. If *content_manager* is not specified, use + the ``content_manager`` specified by the current :mod:`~email.policy`. + + + .. method:: make_related(boundary=None) + + Convert a non-``multipart`` message into a ``multipart/related`` message, + moving any existing :mailheader:`Content-` headers and payload into a + (new) first part of the ``multipart``. If *boundary* is specified, use + it as the boundary string in the multipart, otherwise leave the boundary + to be automatically created when it is needed (for example, when the + message is serialized). + + + .. method:: make_alternative(boundary=None) + + Convert a non-``multipart`` or a ``multipart/related`` into a + ``multipart/alternative``, moving any existing :mailheader:`Content-` + headers and payload into a (new) first part of the ``multipart``. If + *boundary* is specified, use it as the boundary string in the multipart, + otherwise leave the boundary to be automatically created when it is + needed (for example, when the message is serialized). + + + .. method:: make_mixed(boundary=None) + + Convert a non-``multipart``, a ``multipart/related``, or a + ``multipart-alternative`` into a ``multipart/mixed``, moving any existing + :mailheader:`Content-` headers and payload into a (new) first part of the + ``multipart``. If *boundary* is specified, use it as the boundary string + in the multipart, otherwise leave the boundary to be automatically + created when it is needed (for example, when the message is serialized). + + + .. method:: add_related(*args, content_manager=None, **kw) + + If the message is a ``multipart/related``, create a new message + object, pass all of the arguments to its :meth:`set_content` method, + and :meth:`~email.message.Message.attach` it to the ``multipart``. If + the message is a non-``multipart``, call :meth:`make_related` and then + proceed as above. If the message is any other type of ``multipart``, + raise a :exc:`TypeError`. If *content_manager* is not specified, use + the ``content_manager`` specified by the current :mod:`~email.policy`. + If the added part has no :mailheader:`Content-Disposition` header, + add one with the value ``inline``. + + + .. method:: add_alternative(*args, content_manager=None, **kw) + + If the message is a ``multipart/alternative``, create a new message + object, pass all of the arguments to its :meth:`set_content` method, and + :meth:`~email.message.Message.attach` it to the ``multipart``. If the + message is a non-``multipart`` or ``multipart/related``, call + :meth:`make_alternative` and then proceed as above. If the message is + any other type of ``multipart``, raise a :exc:`TypeError`. If + *content_manager* is not specified, use the ``content_manager`` specified + by the current :mod:`~email.policy`. + + + .. method:: add_attachment(*args, content_manager=None, **kw) + + If the message is a ``multipart/mixed``, create a new message object, + pass all of the arguments to its :meth:`set_content` method, and + :meth:`~email.message.Message.attach` it to the ``multipart``. If the + message is a non-``multipart``, ``multipart/related``, or + ``multipart/alternative``, call :meth:`make_mixed` and then proceed as + above. If *content_manager* is not specified, use the ``content_manager`` + specified by the current :mod:`~email.policy`. If the added part + has no :mailheader:`Content-Disposition` header, add one with the value + ``attachment``. This method can be used both for explicit attachments + (:mailheader:`Content-Disposition: attachment` and ``inline`` attachments + (:mailheader:`Content-Disposition: inline`), by passing appropriate + options to the ``content_manager``. + + + .. method:: clear() + + Remove the payload and all of the headers. + + + .. method:: clear_content() + + Remove the payload and all of the :exc:`Content-` headers, leaving + all other headers intact and in their original order. + + +.. class:: MIMEPart(policy=default) + + This class represents a subpart of a MIME message. It is identical to + :class:`EmailMessage`, except that no :mailheader:`MIME-Version` headers are + added when :meth:`~EmailMessage.set_content` is called, since sub-parts do + not need their own :mailheader:`MIME-Version` headers. + + +.. currentmodule:: email.contentmanager + +.. class:: ContentManager() + + Base class for content managers. Provides the standard registry mechanisms + to register converters between MIME content and other representations, as + well as the ``get_content`` and ``set_content`` dispatch methods. + + + .. method:: get_content(msg, *args, **kw) + + Look up a handler function based on the ``mimetype`` of *msg* (see next + paragraph), call it, passing through all arguments, and return the result + of the call. The expectation is that the handler will extract the + payload from *msg* and return an object that encodes information about + the extracted data. + + To find the handler, look for the following keys in the registry, + stopping with the first one found: + + * the string representing the full MIME type (``maintype/subtype``) + * the string representing the ``maintype`` + * the empty string + + If none of these keys produce a handler, raise a :exc:`KeyError` for the + full MIME type. + + + .. method:: set_content(msg, obj, *args, **kw) + + If the ``maintype`` is ``multipart``, raise a :exc:`TypeError`; otherwise + look up a handler function based on the type of *obj* (see next + paragraph), call :meth:`~email.message.EmailMessage.clear_content` on the + *msg*, and call the handler function, passing through all arguments. The + expectation is that the handler will transform and store *obj* into + *msg*, possibly making other changes to *msg* as well, such as adding + various MIME headers to encode information needed to interpret the stored + data. + + To find the handler, obtain the type of *obj* (``typ = type(obj)``), and + look for the following keys in the registry, stopping with the first one + found: + + * the type itself (``typ``) + * the type's fully qualified name (``typ.__module__ + '.' + + typ.__qualname__``). + * the type's qualname (``typ.__qualname__``) + * the type's name (``typ.__name__``). + + If none of the above match, repeat all of the checks above for each of + the types in the :term:`MRO` (``typ.__mro__``). Finally, if no other key + yields a handler, check for a handler for the key ``None``. If there is + no handler for ``None``, raise a :exc:`KeyError` for the fully + qualified name of the type. + + Also add a :mailheader:`MIME-Version` header if one is not present (see + also :class:`.MIMEPart`). + + + .. method:: add_get_handler(key, handler) + + Record the function *handler* as the handler for *key*. For the possible + values of *key*, see :meth:`get_content`. + + + .. method:: add_set_handler(typekey, handler) + + Record *handler* as the function to call when an object of a type + matching *typekey* is passed to :meth:`set_content`. For the possible + values of *typekey*, see :meth:`set_content`. + + +Content Manager Instances +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Currently the email package provides only one concrete content manager, +:data:`raw_data_manager`, although more may be added in the future. +:data:`raw_data_manager` is the +:attr:`~email.policy.EmailPolicy.content_manager` provided by +:attr:`~email.policy.EmailPolicy` and its derivatives. + + +.. data:: raw_data_manager + + This content manager provides only a minimum interface beyond that provided + by :class:`~email.message.Message` itself: it deals only with text, raw + byte strings, and :class:`~email.message.Message` objects. Nevertheless, it + provides significant advantages compared to the base API: ``get_content`` on + a text part will return a unicode string without the application needing to + manually decode it, ``set_content`` provides a rich set of options for + controlling the headers added to a part and controlling the content transfer + encoding, and it enables the use of the various ``add_`` methods, thereby + simplifying the creation of multipart messages. + + .. method:: get_content(msg, errors='replace') + + Return the payload of the part as either a string (for ``text`` parts), a + :class:`~email.message.EmailMessage` object (for ``message/rfc822`` + parts), or a ``bytes`` object (for all other non-multipart types). Raise + a :exc:`KeyError` if called on a ``multipart``. If the part is a + ``text`` part and *errors* is specified, use it as the error handler when + decoding the payload to unicode. The default error handler is + ``replace``. + + .. method:: set_content(msg, <'str'>, subtype="plain", charset='utf-8' \ + cte=None, \ + disposition=None, filename=None, cid=None, \ + params=None, headers=None) + set_content(msg, <'bytes'>, maintype, subtype, cte="base64", \ + disposition=None, filename=None, cid=None, \ + params=None, headers=None) + set_content(msg, <'Message'>, cte=None, \ + disposition=None, filename=None, cid=None, \ + params=None, headers=None) + set_content(msg, <'list'>, subtype='mixed', \ + disposition=None, filename=None, cid=None, \ + params=None, headers=None) + + Add headers and payload to *msg*: + + Add a :mailheader:`Content-Type` header with a ``maintype/subtype`` + value. + + * For ``str``, set the MIME ``maintype`` to ``text``, and set the + subtype to *subtype* if it is specified, or ``plain`` if it is not. + * For ``bytes``, use the specified *maintype* and *subtype*, or + raise a :exc:`TypeError` if they are not specified. + * For :class:`~email.message.Message` objects, set the maintype to + ``message``, and set the subtype to *subtype* if it is specified + or ``rfc822`` if it is not. If *subtype* is ``partial``, raise an + error (``bytes`` objects must be used to construct + ``message/partial`` parts). + * For *<'list'>*, which should be a list of + :class:`~email.message.Message` objects, set the ``maintype`` to + ``multipart``, and the ``subtype`` to *subtype* if it is + specified, and ``mixed`` if it is not. If the message parts in + the *<'list'>* have :mailheader:`MIME-Version` headers, remove + them. + + If *charset* is provided (which is valid only for ``str``), encode the + string to bytes using the specified character set. The default is + ``utf-8``. If the specified *charset* is a known alias for a standard + MIME charset name, use the standard charset instead. + + If *cte* is set, encode the payload using the specified content transfer + encoding, and set the :mailheader:`Content-Transfer-Endcoding` header to + that value. For ``str`` objects, if it is not set use heuristics to + determine the most compact encoding. Possible values for *cte* are + ``quoted-printable``, ``base64``, ``7bit``, ``8bit``, and ``binary``. + If the input cannot be encoded in the specified encoding (eg: ``7bit``), + raise a :exc:`ValueError`. For :class:`~email.message.Message`, per + :rfc:`2046`, raise an error if a *cte* of ``quoted-printable`` or + ``base64`` is requested for *subtype* ``rfc822``, and for any *cte* + other than ``7bit`` for *subtype* ``external-body``. For + ``message/rfc822``, use ``8bit`` if *cte* is not specified. For all + other values of *subtype*, use ``7bit``. + + .. note:: A *cte* of ``binary`` does not actually work correctly yet. + The ``Message`` object as modified by ``set_content`` is correct, but + :class:`~email.generator.BytesGenerator` does not serialize it + correctly. + + If *disposition* is set, use it as the value of the + :mailheader:`Content-Disposition` header. If not specified, and + *filename* is specified, add the header with the value ``attachment``. + If it is not specified and *filename* is also not specified, do not add + the header. The only valid values for *disposition* are ``attachment`` + and ``inline``. + + If *filename* is specified, use it as the value of the ``filename`` + parameter of the :mailheader:`Content-Disposition` header. There is no + default. + + If *cid* is specified, add a :mailheader:`Content-ID` header with + *cid* as its value. + + If *params* is specified, iterate its ``items`` method and use the + resulting ``(key, value)`` pairs to set additional paramters on the + :mailheader:`Content-Type` header. + + If *headers* is specified and is a list of strings of the form + ``headername: headervalue`` or a list of ``header`` objects + (distinguised from strings by having a ``name`` attribute), add the + headers to *msg*. diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst index c172acb..48d41e1 100644 --- a/Doc/library/email.generator.rst +++ b/Doc/library/email.generator.rst @@ -112,7 +112,7 @@ formatted string representation of a message object. For more detail, see :mod:`email.message`. .. class:: BytesGenerator(outfp, mangle_from_=True, maxheaderlen=78, *, \ - policy=policy.default) + policy=None) The constructor for the :class:`BytesGenerator` class takes a binary :term:`file-like object` called *outfp* for an argument. *outfp* must @@ -134,9 +134,11 @@ formatted string representation of a message object. For more detail, see wrapping. The default is 78, as recommended (but not required) by :rfc:`2822`. + The *policy* keyword specifies a :mod:`~email.policy` object that controls a - number of aspects of the generator's operation. The default policy - maintains backward compatibility. + number of aspects of the generator's operation. If no *policy* is specified, + then the *policy* attached to the message object passed to :attr:`flatten` + is used. .. versionchanged:: 3.3 Added the *policy* keyword. @@ -174,7 +176,7 @@ formatted string representation of a message object. For more detail, see Optional *linesep* specifies the line separator character used to terminate lines in the output. If specified it overrides the value - specified by the ``Generator``\ 's ``policy``. + specified by the ``Generator``\ or *msg*\ 's ``policy``. .. method:: clone(fp) diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index 3075e86..8573e85 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -33,23 +33,31 @@ Here are the methods of the :class:`Message` class: .. class:: Message(policy=compat32) - The *policy* argument determiens the :mod:`~email.policy` that will be used - to update the message model. The default value, :class:`compat32 - <email.policy.Compat32>` maintains backward compatibility with the - Python 3.2 version of the email package. For more information see the + If *policy* is specified (it must be an instance of a :mod:`~email.policy` + class) use the rules it specifies to update and serialize the representation + of the message. If *policy* is not set, use the :class:`compat32 + <email.policy.Compat32>` policy, which maintains backward compatibility with + the Python 3.2 version of the email package. For more information see the :mod:`~email.policy` documentation. .. versionchanged:: 3.3 The *policy* keyword argument was added. - .. method:: as_string(unixfrom=False, maxheaderlen=0) + .. method:: as_string(unixfrom=False, maxheaderlen=0, policy=None) Return the entire message flattened as a string. When optional *unixfrom* - is ``True``, the envelope header is included in the returned string. - *unixfrom* defaults to ``False``. Flattening the message may trigger - changes to the :class:`Message` if defaults need to be filled in to - complete the transformation to a string (for example, MIME boundaries may - be generated or modified). + is true, the envelope header is included in the returned string. + *unixfrom* defaults to ``False``. For backward compabitility reasons, + *maxheaderlen* defaults to ``0``, so if you want a different value you + must override it explicitly (the value specified for *max_line_length* in + the policy will be ignored by this method). The *policy* argument may be + used to override the default policy obtained from the message instance. + This can be used to control some of the formatting produced by the + method, since the specified *policy* will be passed to the ``Generator``. + + Flattening the message may trigger changes to the :class:`Message` if + defaults need to be filled in to complete the transformation to a string + (for example, MIME boundaries may be generated or modified). Note that this method is provided as a convenience and may not always format the message the way you want. For example, by default it does @@ -65,10 +73,58 @@ Here are the methods of the :class:`Message` class: g.flatten(msg) text = fp.getvalue() + If the message object contains binary data that is not encoded according + to RFC standards, the non-compliant data will be replaced by unicode + "unknown character" code points. (See also :meth:`.as_bytes` and + :class:`~email.generator.BytesGenerator`.) + + .. versionchanged:: 3.4 the *policy* keyword argument was added. + .. method:: __str__() - Equivalent to ``as_string(unixfrom=True)``. + Equivalent to :meth:`.as_string()`. Allows ``str(msg)`` to produce a + string containing the formatted message. + + + .. method:: as_bytes(unixfrom=False, policy=None) + + Return the entire message flattened as a bytes object. When optional + *unixfrom* is true, the envelope header is included in the returned + string. *unixfrom* defaults to ``False``. The *policy* argument may be + used to override the default policy obtained from the message instance. + This can be used to control some of the formatting produced by the + method, since the specified *policy* will be passed to the + ``BytesGenerator``. + + Flattening the message may trigger changes to the :class:`Message` if + defaults need to be filled in to complete the transformation to a string + (for example, MIME boundaries may be generated or modified). + + Note that this method is provided as a convenience and may not always + format the message the way you want. For example, by default it does + not do the mangling of lines that begin with ``From`` that is + required by the unix mbox format. For more flexibility, instantiate a + :class:`~email.generator.BytesGenerator` instance and use its + :meth:`~email.generator.BytesGenerator.flatten` method directly. + For example:: + + from io import BytesIO + from email.generator import BytesGenerator + fp = BytesIO() + g = BytesGenerator(fp, mangle_from_=True, maxheaderlen=60) + g.flatten(msg) + text = fp.getvalue() + + .. versionadded:: 3.4 + + + .. method:: __bytes__() + + Equivalent to :meth:`.as_bytes()`. Allows ``bytes(msg)`` to produce a + bytes object containing the formatted message. + + .. versionadded:: 3.4 .. method:: is_multipart() @@ -410,7 +466,8 @@ Here are the methods of the :class:`Message` class: to ``False``. - .. method:: set_param(param, value, header='Content-Type', requote=True, charset=None, language='') + .. method:: set_param(param, value, header='Content-Type', requote=True, \ + charset=None, language='', replace=False) Set a parameter in the :mailheader:`Content-Type` header. If the parameter already exists in the header, its value will be replaced with @@ -427,6 +484,12 @@ Here are the methods of the :class:`Message` class: language, defaulting to the empty string. Both *charset* and *language* should be strings. + If *replace* is ``False`` (the default) the header is moved to the + end of the list of headers. If *replace* is ``True``, the header + will be updated in place. + + .. versionchanged:: 3.4 ``replace`` keyword was added. + .. method:: del_param(param, header='content-type', requote=True) diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index ee6af3f..9f137cd 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -60,15 +60,18 @@ list of defects that it can find. Here is the API for the :class:`FeedParser`: -.. class:: FeedParser(_factory=email.message.Message, *, policy=policy.default) +.. class:: FeedParser(_factory=email.message.Message, *, policy=policy.compat32) Create a :class:`FeedParser` instance. Optional *_factory* is a no-argument callable that will be called whenever a new message object is needed. It defaults to the :class:`email.message.Message` class. - The *policy* keyword specifies a :mod:`~email.policy` object that controls a - number of aspects of the parser's operation. The default policy maintains - backward compatibility. + If *policy* is specified (it must be an instance of a :mod:`~email.policy` + class) use the rules it specifies to update the representation of the + message. If *policy* is not set, use the :class:`compat32 + <email.policy.Compat32>` policy, which maintains backward compatibility with + the Python 3.2 version of the email package. For more information see the + :mod:`~email.policy` documentation. .. versionchanged:: 3.3 Added the *policy* keyword. @@ -113,7 +116,7 @@ have the same API as the :class:`Parser` and :class:`BytesParser` classes. The BytesHeaderParser class. -.. class:: Parser(_class=email.message.Message, *, policy=policy.default) +.. class:: Parser(_class=email.message.Message, *, policy=policy.compat32) The constructor for the :class:`Parser` class takes an optional argument *_class*. This must be a callable factory (such as a function or a class), and @@ -121,9 +124,12 @@ have the same API as the :class:`Parser` and :class:`BytesParser` classes. :class:`~email.message.Message` (see :mod:`email.message`). The factory will be called without arguments. - The *policy* keyword specifies a :mod:`~email.policy` object that controls a - number of aspects of the parser's operation. The default policy maintains - backward compatibility. + If *policy* is specified (it must be an instance of a :mod:`~email.policy` + class) use the rules it specifies to update the representation of the + message. If *policy* is not set, use the :class:`compat32 + <email.policy.Compat32>` policy, which maintains backward compatibility with + the Python 3.2 version of the email package. For more information see the + :mod:`~email.policy` documentation. .. versionchanged:: 3.3 Removed the *strict* argument that was deprecated in 2.4. Added the @@ -159,15 +165,18 @@ have the same API as the :class:`Parser` and :class:`BytesParser` classes. Optional *headersonly* is as with the :meth:`parse` method. -.. class:: BytesParser(_class=email.message.Message, *, policy=policy.default) +.. class:: BytesParser(_class=email.message.Message, *, policy=policy.compat32) This class is exactly parallel to :class:`Parser`, but handles bytes input. The *_class* and *strict* arguments are interpreted in the same way as for the :class:`Parser` constructor. - The *policy* keyword specifies a :mod:`~email.policy` object that - controls a number of aspects of the parser's operation. The default - policy maintains backward compatibility. + If *policy* is specified (it must be an instance of a :mod:`~email.policy` + class) use the rules it specifies to update the representation of the + message. If *policy* is not set, use the :class:`compat32 + <email.policy.Compat32>` policy, which maintains backward compatibility with + the Python 3.2 version of the email package. For more information see the + :mod:`~email.policy` documentation. .. versionchanged:: 3.3 Removed the *strict* argument. Added the *policy* keyword. @@ -209,7 +218,7 @@ in the top-level :mod:`email` package namespace. .. currentmodule:: email .. function:: message_from_string(s, _class=email.message.Message, *, \ - policy=policy.default) + policy=policy.compat32) Return a message object structure from a string. This is exactly equivalent to ``Parser().parsestr(s)``. *_class* and *policy* are interpreted as @@ -219,7 +228,7 @@ in the top-level :mod:`email` package namespace. Removed the *strict* argument. Added the *policy* keyword. .. function:: message_from_bytes(s, _class=email.message.Message, *, \ - policy=policy.default) + policy=policy.compat32) Return a message object structure from a byte string. This is exactly equivalent to ``BytesParser().parsebytes(s)``. Optional *_class* and @@ -231,7 +240,7 @@ in the top-level :mod:`email` package namespace. Removed the *strict* argument. Added the *policy* keyword. .. function:: message_from_file(fp, _class=email.message.Message, *, \ - policy=policy.default) + policy=policy.compat32) Return a message object structure tree from an open :term:`file object`. This is exactly equivalent to ``Parser().parse(fp)``. *_class* @@ -242,7 +251,7 @@ in the top-level :mod:`email` package namespace. Removed the *strict* argument. Added the *policy* keyword. .. function:: message_from_binary_file(fp, _class=email.message.Message, *, \ - policy=policy.default) + policy=policy.compat32) Return a message object structure tree from an open binary :term:`file object`. This is exactly equivalent to ``BytesParser().parse(fp)``. diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index cb2023c..97358f0 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -97,6 +97,17 @@ correct line separator characters when creating the binary string to feed into ``sendmail's`` ``stdin``, where the default policy would use ``\n`` line separators. +Some email package methods accept a *policy* keyword argument, allowing the +policy to be overridden for that method. For example, the following code uses +the :meth:`~email.message.Message.as_string` method of the *msg* object from +the previous example and writes the message to a file using the native line +separators for the platform on which it is running:: + + >>> import os + >>> with open('converted.txt', 'wb') as f: + ... f.write(msg.as_bytes(policy=msg.policy.clone(linesep=os.linesep))) + 17 + Policy objects can also be combined using the addition operator, producing a policy object whose settings are a combination of the non-default values of the summed objects:: @@ -360,7 +371,7 @@ added matters. To illustrate:: to) :rfc:`5322`, :rfc:`2047`, and the current MIME RFCs. This policy adds new header parsing and folding algorithms. Instead of - simple strings, headers are custom objects with custom attributes depending + simple strings, headers are ``str`` subclasses with attributes that depend on the type of the field. The parsing and folding algorithm fully implement :rfc:`2047` and :rfc:`5322`. @@ -397,6 +408,20 @@ added matters. To illustrate:: fields are treated as unstructured. This list will be completed before the extension is marked stable.) + .. attribute:: content_manager + + An object with at least two methods: get_content and set_content. When + the :meth:`~email.message.Message.get_content` or + :meth:`~email.message.Message.set_content` method of a + :class:`~email.message.Message` object is called, it calls the + corresponding method of this object, passing it the message object as its + first argument, and any arguments or keywords that were passed to it as + additional arguments. By default ``content_manager`` is set to + :data:`~email.contentmanager.raw_data_manager`. + + .. versionadded:: 3.4 + + The class provides the following concrete implementations of the abstract methods of :class:`Policy`: @@ -416,7 +441,7 @@ added matters. To illustrate:: The name is returned unchanged. If the input value has a ``name`` attribute and it matches *name* ignoring case, the value is returned unchanged. Otherwise the *name* and *value* are passed to - ``header_factory``, and the resulting custom header object is returned as + ``header_factory``, and the resulting header object is returned as the value. In this case a ``ValueError`` is raised if the input value contains CR or LF characters. @@ -424,7 +449,7 @@ added matters. To illustrate:: If the value has a ``name`` attribute, it is returned to unmodified. Otherwise the *name*, and the *value* with any CR or LF characters - removed, are passed to the ``header_factory``, and the resulting custom + removed, are passed to the ``header_factory``, and the resulting header object is returned. Any surrogateescaped bytes get turned into the unicode unknown-character glyph. @@ -434,9 +459,9 @@ added matters. To illustrate:: A value is considered to be a 'source value' if and only if it does not have a ``name`` attribute (having a ``name`` attribute means it is a header object of some sort). If a source value needs to be refolded - according to the policy, it is converted into a custom header object by + according to the policy, it is converted into a header object by passing the *name* and the *value* with any CR and LF characters removed - to the ``header_factory``. Folding of a custom header object is done by + to the ``header_factory``. Folding of a header object is done by calling its ``fold`` method with the current policy. Source values are split into lines using :meth:`~str.splitlines`. If @@ -491,23 +516,23 @@ With all of these :class:`EmailPolicies <.EmailPolicy>`, the effective API of the email package is changed from the Python 3.2 API in the following ways: * Setting a header on a :class:`~email.message.Message` results in that - header being parsed and a custom header object created. + header being parsed and a header object created. * Fetching a header value from a :class:`~email.message.Message` results - in that header being parsed and a custom header object created and + in that header being parsed and a header object created and returned. - * Any custom header object, or any header that is refolded due to the + * Any header object, or any header that is refolded due to the policy settings, is folded using an algorithm that fully implements the RFC folding algorithms, including knowing where encoded words are required and allowed. From the application view, this means that any header obtained through the -:class:`~email.message.Message` is a custom header object with custom +:class:`~email.message.Message` is a header object with extra attributes, whose string value is the fully decoded unicode value of the header. Likewise, a header may be assigned a new value, or a new header created, using a unicode string, and the policy will take care of converting the unicode string into the correct RFC encoded form. -The custom header objects and their attributes are described in +The header objects and their attributes are described in :mod:`~email.headerregistry`. diff --git a/Doc/library/email.rst b/Doc/library/email.rst index a6cbbce..331d2ef 100644 --- a/Doc/library/email.rst +++ b/Doc/library/email.rst @@ -53,6 +53,7 @@ Contents of the :mod:`email` package documentation: email.generator.rst email.policy.rst email.headerregistry.rst + email.contentmanager.rst email.mime.rst email.header.rst email.charset.rst diff --git a/Doc/library/email.util.rst b/Doc/library/email.util.rst index f75975e..219e284 100644 --- a/Doc/library/email.util.rst +++ b/Doc/library/email.util.rst @@ -98,12 +98,9 @@ There are several useful utilities provided in the :mod:`email.utils` module: .. function:: mktime_tz(tuple) - Turn a 10-tuple as returned by :func:`parsedate_tz` into a UTC timestamp. It - the timezone item in the tuple is ``None``, assume local time. Minor - deficiency: :func:`mktime_tz` interprets the first 8 elements of *tuple* as a - local time and then compensates for the timezone difference. This may yield a - slight error around changes in daylight savings time, though not worth worrying - about for common use. + Turn a 10-tuple as returned by :func:`parsedate_tz` into a UTC + timestamp (seconds since the Epoch). If the timezone item in the + tuple is ``None``, assume local time. .. function:: formatdate(timeval=None, localtime=False, usegmt=False) @@ -210,4 +207,3 @@ There are several useful utilities provided in the :mod:`email.utils` module: .. [#] Note that the sign of the timezone offset is the opposite of the sign of the ``time.timezone`` variable for the same timezone; the latter variable follows the POSIX standard while this module follows :rfc:`2822`. - diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst new file mode 100644 index 0000000..8012f51 --- /dev/null +++ b/Doc/library/ensurepip.rst @@ -0,0 +1,131 @@ +:mod:`ensurepip` --- Bootstrapping the ``pip`` installer +======================================================== + +.. module:: ensurepip + :synopsis: Bootstrapping the ``pip`` installer into an existing Python + installation or virtual environment. + +.. versionadded:: 3.4 + +The :mod:`ensurepip` package provides support for bootstrapping the ``pip`` +installer into an existing Python installation or virtual environment. This +bootstrapping approach reflects the fact that ``pip`` is an independent +project with its own release cycle, and the latest available stable version +is bundled with maintenance and feature releases of the CPython reference +interpreter. + +In most cases, end users of Python shouldn't need to invoke this module +directly (as ``pip`` should be bootstrapped by default), but it may be +needed if installing ``pip`` was skipped when installing Python (or +when creating a virtual environment) or after explicitly uninstalling +``pip``. + +.. note:: + + This module *does not* access the internet. All of the components + needed to bootstrap ``pip`` are included as internal parts of the + package. + +.. seealso:: + + :ref:`installing-index` + The end user guide for installing Python packages + + :pep:`453`: Explicit bootstrapping of pip in Python installations + The original rationale and specification for this module. + + +Command line interface +---------------------- + +The command line interface is invoked using the interpreter's ``-m`` switch. + +The simplest possible invocation is:: + + python -m ensurepip + +This invocation will install ``pip`` if it is not already installed, +but otherwise does nothing. To ensure the installed version of ``pip`` +is at least as recent as the one bundled with ``ensurepip``, pass the +``--upgrade`` option:: + + python -m ensurepip --upgrade + +By default, ``pip`` is installed into the current virtual environment +(if one is active) or into the system site packages (if there is no +active virtual environment). The installation location can be controlled +through two additional command line options: + +* ``--root <dir>``: Installs ``pip`` relative to the given root directory + rather than the root of the currently active virtual environment (if any) + or the default root for the current Python installation. +* ``--user``: Installs ``pip`` into the user site packages directory rather + than globally for the current Python installation (this option is not + permitted inside an active virtual environment). + +By default, the scripts ``pipX`` and ``pipX.Y`` will be installed (where +X.Y stands for the version of Python used to invoke ``ensurepip``). The +scripts installed can be controlled through two additional command line +options: + +* ``--altinstall``: if an alternate installation is requested, the ``pipX`` + script will *not* be installed. + +* ``--default-pip``: if a "default pip" installation is requested, the + ``pip`` script will be installed in addition to the two regular scripts. + +Providing both of the script selection options will trigger an exception. + + +Module API +---------- + +:mod:`ensurepip` exposes two functions for programmatic use: + +.. function:: version() + + Returns a string specifying the bundled version of pip that will be + installed when bootstrapping an environment. + +.. function:: bootstrap(root=None, upgrade=False, user=False, \ + altinstall=False, default_pip=False, \ + verbosity=0) + + Bootstraps ``pip`` into the current or designated environment. + + *root* specifies an alternative root directory to install relative to. + If *root* is None, then installation uses the default install location + for the current environment. + + *upgrade* indicates whether or not to upgrade an existing installation + of an earlier version of ``pip`` to the bundled version. + + *user* indicates whether to use the user scheme rather than installing + globally. + + By default, the scripts ``pipX`` and ``pipX.Y`` will be installed (where + X.Y stands for the current version of Python). + + If *altinstall* is set, then ``pipX`` will *not* be installed. + + If *default_pip* is set, then ``pip`` will be installed in addition to + the two regular scripts. + + Setting both *altinstall* and *default_pip* will trigger + :exc:`ValueError`. + + *verbosity* controls the level of output to :data:`sys.stdout` from the + bootstrapping operation. + + .. note:: + + The bootstrapping process has side effects on both ``sys.path`` and + ``os.environ``. Invoking the command line interface in a subprocess + instead allows these side effects to be avoided. + + .. note:: + + The bootstrapping process may install additional modules required by + ``pip``, but other software should not assume those dependencies will + always be present by default (as the dependencies may be removed in a + future version of ``pip``). diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst new file mode 100644 index 0000000..acdcf7f --- /dev/null +++ b/Doc/library/enum.rst @@ -0,0 +1,749 @@ +:mod:`enum` --- Support for enumerations +======================================== + +.. module:: enum + :synopsis: Implementation of an enumeration class. + +.. :moduleauthor:: Ethan Furman <ethan@stoneleaf.us> +.. :sectionauthor:: Barry Warsaw <barry@python.org>, +.. :sectionauthor:: Eli Bendersky <eliben@gmail.com>, +.. :sectionauthor:: Ethan Furman <ethan@stoneleaf.us> + +.. versionadded:: 3.4 + +**Source code:** :source:`Lib/enum.py` + +---------------- + +An enumeration is a set of symbolic names (members) bound to unique, +constant values. Within an enumeration, the members can be compared +by identity, and the enumeration itself can be iterated over. + + +Module Contents +--------------- + +This module defines two enumeration classes that can be used to define unique +sets of names and values: :class:`Enum` and :class:`IntEnum`. It also defines +one decorator, :func:`unique`. + +.. class:: Enum + + Base class for creating enumerated constants. See section + `Functional API`_ for an alternate construction syntax. + +.. class:: IntEnum + + Base class for creating enumerated constants that are also + subclasses of :class:`int`. + +.. function:: unique + + Enum class decorator that ensures only one name is bound to any one value. + + +Creating an Enum +---------------- + +Enumerations are created using the :keyword:`class` syntax, which makes them +easy to read and write. An alternative creation method is described in +`Functional API`_. To define an enumeration, subclass :class:`Enum` as +follows:: + + >>> from enum import Enum + >>> class Color(Enum): + ... red = 1 + ... green = 2 + ... blue = 3 + ... + +.. note:: Nomenclature + + - The class :class:`Color` is an *enumeration* (or *enum*) + - The attributes :attr:`Color.red`, :attr:`Color.green`, etc., are + *enumeration members* (or *enum members*). + - The enum members have *names* and *values* (the name of + :attr:`Color.red` is ``red``, the value of :attr:`Color.blue` is + ``3``, etc.) + +.. note:: + + Even though we use the :keyword:`class` syntax to create Enums, Enums + are not normal Python classes. See `How are Enums different?`_ for + more details. + +Enumeration members have human readable string representations:: + + >>> print(Color.red) + Color.red + +...while their ``repr`` has more information:: + + >>> print(repr(Color.red)) + <Color.red: 1> + +The *type* of an enumeration member is the enumeration it belongs to:: + + >>> type(Color.red) + <enum 'Color'> + >>> isinstance(Color.green, Color) + True + >>> + +Enum members also have a property that contains just their item name:: + + >>> print(Color.red.name) + red + +Enumerations support iteration, in definition order:: + + >>> class Shake(Enum): + ... vanilla = 7 + ... chocolate = 4 + ... cookies = 9 + ... mint = 3 + ... + >>> for shake in Shake: + ... print(shake) + ... + Shake.vanilla + Shake.chocolate + Shake.cookies + Shake.mint + +Enumeration members are hashable, so they can be used in dictionaries and sets:: + + >>> apples = {} + >>> apples[Color.red] = 'red delicious' + >>> apples[Color.green] = 'granny smith' + >>> apples == {Color.red: 'red delicious', Color.green: 'granny smith'} + True + + +Programmatic access to enumeration members and their attributes +--------------------------------------------------------------- + +Sometimes it's useful to access members in enumerations programmatically (i.e. +situations where ``Color.red`` won't do because the exact color is not known +at program-writing time). ``Enum`` allows such access:: + + >>> Color(1) + <Color.red: 1> + >>> Color(3) + <Color.blue: 3> + +If you want to access enum members by *name*, use item access:: + + >>> Color['red'] + <Color.red: 1> + >>> Color['green'] + <Color.green: 2> + +If you have an enum member and need its :attr:`name` or :attr:`value`:: + + >>> member = Color.red + >>> member.name + 'red' + >>> member.value + 1 + + +Duplicating enum members and values +----------------------------------- + +Having two enum members with the same name is invalid:: + + >>> class Shape(Enum): + ... square = 2 + ... square = 3 + ... + Traceback (most recent call last): + ... + TypeError: Attempted to reuse key: 'square' + +However, two enum members are allowed to have the same value. Given two members +A and B with the same value (and A defined first), B is an alias to A. By-value +lookup of the value of A and B will return A. By-name lookup of B will also +return A:: + + >>> class Shape(Enum): + ... square = 2 + ... diamond = 1 + ... circle = 3 + ... alias_for_square = 2 + ... + >>> Shape.square + <Shape.square: 2> + >>> Shape.alias_for_square + <Shape.square: 2> + >>> Shape(2) + <Shape.square: 2> + +.. note:: + + Attempting to create a member with the same name as an already + defined attribute (another member, a method, etc.) or attempting to create + an attribute with the same name as a member is not allowed. + + +Ensuring unique enumeration values +---------------------------------- + +By default, enumerations allow multiple names as aliases for the same value. +When this behavior isn't desired, the following decorator can be used to +ensure each value is used only once in the enumeration: + +.. decorator:: unique + +A :keyword:`class` decorator specifically for enumerations. It searches an +enumeration's :attr:`__members__` gathering any aliases it finds; if any are +found :exc:`ValueError` is raised with the details:: + + >>> from enum import Enum, unique + >>> @unique + ... class Mistake(Enum): + ... one = 1 + ... two = 2 + ... three = 3 + ... four = 3 + ... + Traceback (most recent call last): + ... + ValueError: duplicate values found in <enum 'Mistake'>: four -> three + + +Iteration +--------- + +Iterating over the members of an enum does not provide the aliases:: + + >>> list(Shape) + [<Shape.square: 2>, <Shape.diamond: 1>, <Shape.circle: 3>] + +The special attribute ``__members__`` is an ordered dictionary mapping names +to members. It includes all names defined in the enumeration, including the +aliases:: + + >>> for name, member in Shape.__members__.items(): + ... name, member + ... + ('square', <Shape.square: 2>) + ('diamond', <Shape.diamond: 1>) + ('circle', <Shape.circle: 3>) + ('alias_for_square', <Shape.square: 2>) + +The ``__members__`` attribute can be used for detailed programmatic access to +the enumeration members. For example, finding all the aliases:: + + >>> [name for name, member in Shape.__members__.items() if member.name != name] + ['alias_for_square'] + + +Comparisons +----------- + +Enumeration members are compared by identity:: + + >>> Color.red is Color.red + True + >>> Color.red is Color.blue + False + >>> Color.red is not Color.blue + True + +Ordered comparisons between enumeration values are *not* supported. Enum +members are not integers (but see `IntEnum`_ below):: + + >>> Color.red < Color.blue + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + TypeError: unorderable types: Color() < Color() + +Equality comparisons are defined though:: + + >>> Color.blue == Color.red + False + >>> Color.blue != Color.red + True + >>> Color.blue == Color.blue + True + +Comparisons against non-enumeration values will always compare not equal +(again, :class:`IntEnum` was explicitly designed to behave differently, see +below):: + + >>> Color.blue == 2 + False + + +Allowed members and attributes of enumerations +---------------------------------------------- + +The examples above use integers for enumeration values. Using integers is +short and handy (and provided by default by the `Functional API`_), but not +strictly enforced. In the vast majority of use-cases, one doesn't care what +the actual value of an enumeration is. But if the value *is* important, +enumerations can have arbitrary values. + +Enumerations are Python classes, and can have methods and special methods as +usual. If we have this enumeration:: + + >>> class Mood(Enum): + ... funky = 1 + ... happy = 3 + ... + ... def describe(self): + ... # self is the member here + ... return self.name, self.value + ... + ... def __str__(self): + ... return 'my custom str! {0}'.format(self.value) + ... + ... @classmethod + ... def favorite_mood(cls): + ... # cls here is the enumeration + ... return cls.happy + ... + +Then:: + + >>> Mood.favorite_mood() + <Mood.happy: 3> + >>> Mood.happy.describe() + ('happy', 3) + >>> str(Mood.funky) + 'my custom str! 1' + +The rules for what is allowed are as follows: _sunder_ names (starting and +ending with a single underscore) are reserved by enum and cannot be used; +all other attributes defined within an enumeration will become members of this +enumeration, with the exception of *__dunder__* names and descriptors (methods +are also descriptors). + +Note: if your enumeration defines :meth:`__new__` and/or :meth:`__init__` then +whatever value(s) were given to the enum member will be passed into those +methods. See `Planet`_ for an example. + + +Restricted subclassing of enumerations +-------------------------------------- + +Subclassing an enumeration is allowed only if the enumeration does not define +any members. So this is forbidden:: + + >>> class MoreColor(Color): + ... pink = 17 + ... + Traceback (most recent call last): + ... + TypeError: Cannot extend enumerations + +But this is allowed:: + + >>> class Foo(Enum): + ... def some_behavior(self): + ... pass + ... + >>> class Bar(Foo): + ... happy = 1 + ... sad = 2 + ... + +Allowing subclassing of enums that define members would lead to a violation of +some important invariants of types and instances. On the other hand, it makes +sense to allow sharing some common behavior between a group of enumerations. +(See `OrderedEnum`_ for an example.) + + +Pickling +-------- + +Enumerations can be pickled and unpickled:: + + >>> from test.test_enum import Fruit + >>> from pickle import dumps, loads + >>> Fruit.tomato is loads(dumps(Fruit.tomato)) + True + +The usual restrictions for pickling apply: picklable enums must be defined in +the top level of a module, since unpickling requires them to be importable +from that module. + +.. note:: + + With pickle protocol version 4 it is possible to easily pickle enums + nested in other classes. + +It is possible to modify how Enum members are pickled/unpickled by defining +:meth:`__reduce_ex__` in the enumeration class. + + +Functional API +-------------- + +The :class:`Enum` class is callable, providing the following functional API:: + + >>> Animal = Enum('Animal', 'ant bee cat dog') + >>> Animal + <enum 'Animal'> + >>> Animal.ant + <Animal.ant: 1> + >>> Animal.ant.value + 1 + >>> list(Animal) + [<Animal.ant: 1>, <Animal.bee: 2>, <Animal.cat: 3>, <Animal.dog: 4>] + +The semantics of this API resemble :class:`~collections.namedtuple`. The first +argument of the call to :class:`Enum` is the name of the enumeration. + +The second argument is the *source* of enumeration member names. It can be a +whitespace-separated string of names, a sequence of names, a sequence of +2-tuples with key/value pairs, or a mapping (e.g. dictionary) of names to +values. The last two options enable assigning arbitrary values to +enumerations; the others auto-assign increasing integers starting with 1. A +new class derived from :class:`Enum` is returned. In other words, the above +assignment to :class:`Animal` is equivalent to:: + + >>> class Animals(Enum): + ... ant = 1 + ... bee = 2 + ... cat = 3 + ... dog = 4 + ... + +The reason for defaulting to ``1`` as the starting number and not ``0`` is +that ``0`` is ``False`` in a boolean sense, but enum members all evaluate +to ``True``. + +Pickling enums created with the functional API can be tricky as frame stack +implementation details are used to try and figure out which module the +enumeration is being created in (e.g. it will fail if you use a utility +function in separate module, and also may not work on IronPython or Jython). +The solution is to specify the module name explicitly as follows:: + + >>> Animals = Enum('Animals', 'ant bee cat dog', module=__name__) + +.. warning:: + + If ``module`` is not supplied, and Enum cannot determine what it is, + the new Enum members will not be unpicklable; to keep errors closer to + the source, pickling will be disabled. + +The new pickle protocol 4 also, in some circumstances, relies on +:attr:`__qualname__` being set to the location where pickle will be able +to find the class. For example, if the class was made available in class +SomeData in the global scope:: + + >>> Animals = Enum('Animals', 'ant bee cat dog', qualname='SomeData.Animals') + +The complete signature is:: + + Enum(value='NewEnumName', names=<...>, *, module='...', qualname='...', type=<mixed-in class>) + +:value: What the new Enum class will record as its name. + +:names: The Enum members. This can be a whitespace or comma separated string + (values will start at 1):: + + 'red green blue' | 'red,green,blue' | 'red, green, blue' + + or an iterator of (name, value) pairs:: + + [('cyan', 4), ('magenta', 5), ('yellow', 6)] + + or a mapping:: + + {'chartreuse': 7, 'sea_green': 11, 'rosemary': 42} + +:module: name of module where new Enum class can be found. + +:qualname: where in module new Enum class can be found. + +:type: type to mix in to new Enum class. + + +Derived Enumerations +-------------------- + +IntEnum +^^^^^^^ + +A variation of :class:`Enum` is provided which is also a subclass of +:class:`int`. Members of an :class:`IntEnum` can be compared to integers; +by extension, integer enumerations of different types can also be compared +to each other:: + + >>> from enum import IntEnum + >>> class Shape(IntEnum): + ... circle = 1 + ... square = 2 + ... + >>> class Request(IntEnum): + ... post = 1 + ... get = 2 + ... + >>> Shape == 1 + False + >>> Shape.circle == 1 + True + >>> Shape.circle == Request.post + True + +However, they still can't be compared to standard :class:`Enum` enumerations:: + + >>> class Shape(IntEnum): + ... circle = 1 + ... square = 2 + ... + >>> class Color(Enum): + ... red = 1 + ... green = 2 + ... + >>> Shape.circle == Color.red + False + +:class:`IntEnum` values behave like integers in other ways you'd expect:: + + >>> int(Shape.circle) + 1 + >>> ['a', 'b', 'c'][Shape.circle] + 'b' + >>> [i for i in range(Shape.square)] + [0, 1] + +For the vast majority of code, :class:`Enum` is strongly recommended, +since :class:`IntEnum` breaks some semantic promises of an enumeration (by +being comparable to integers, and thus by transitivity to other +unrelated enumerations). It should be used only in special cases where +there's no other choice; for example, when integer constants are +replaced with enumerations and backwards compatibility is required with code +that still expects integers. + + +Others +^^^^^^ + +While :class:`IntEnum` is part of the :mod:`enum` module, it would be very +simple to implement independently:: + + class IntEnum(int, Enum): + pass + +This demonstrates how similar derived enumerations can be defined; for example +a :class:`StrEnum` that mixes in :class:`str` instead of :class:`int`. + +Some rules: + +1. When subclassing :class:`Enum`, mix-in types must appear before + :class:`Enum` itself in the sequence of bases, as in the :class:`IntEnum` + example above. +2. While :class:`Enum` can have members of any type, once you mix in an + additional type, all the members must have values of that type, e.g. + :class:`int` above. This restriction does not apply to mix-ins which only + add methods and don't specify another data type such as :class:`int` or + :class:`str`. +3. When another data type is mixed in, the :attr:`value` attribute is *not the + same* as the enum member itself, although it is equivalent and will compare + equal. +4. %-style formatting: `%s` and `%r` call :class:`Enum`'s :meth:`__str__` and + :meth:`__repr__` respectively; other codes (such as `%i` or `%h` for + IntEnum) treat the enum member as its mixed-in type. +5. :meth:`str.__format__` (or :func:`format`) will use the mixed-in + type's :meth:`__format__`. If the :class:`Enum`'s :func:`str` or + :func:`repr` is desired use the `!s` or `!r` :class:`str` format codes. + + +Interesting examples +-------------------- + +While :class:`Enum` and :class:`IntEnum` are expected to cover the majority of +use-cases, they cannot cover them all. Here are recipes for some different +types of enumerations that can be used directly, or as examples for creating +one's own. + + +AutoNumber +^^^^^^^^^^ + +Avoids having to specify the value for each enumeration member:: + + >>> class AutoNumber(Enum): + ... def __new__(cls): + ... value = len(cls.__members__) + 1 + ... obj = object.__new__(cls) + ... obj._value_ = value + ... return obj + ... + >>> class Color(AutoNumber): + ... red = () + ... green = () + ... blue = () + ... + >>> Color.green.value == 2 + True + +.. note:: + + The :meth:`__new__` method, if defined, is used during creation of the Enum + members; it is then replaced by Enum's :meth:`__new__` which is used after + class creation for lookup of existing members. Due to the way Enums are + supposed to behave, there is no way to customize Enum's :meth:`__new__`. + + +OrderedEnum +^^^^^^^^^^^ + +An ordered enumeration that is not based on :class:`IntEnum` and so maintains +the normal :class:`Enum` invariants (such as not being comparable to other +enumerations):: + + >>> class OrderedEnum(Enum): + ... def __ge__(self, other): + ... if self.__class__ is other.__class__: + ... return self.value >= other.value + ... return NotImplemented + ... def __gt__(self, other): + ... if self.__class__ is other.__class__: + ... return self.value > other.value + ... return NotImplemented + ... def __le__(self, other): + ... if self.__class__ is other.__class__: + ... return self.value <= other.value + ... return NotImplemented + ... def __lt__(self, other): + ... if self.__class__ is other.__class__: + ... return self.value < other.value + ... return NotImplemented + ... + >>> class Grade(OrderedEnum): + ... A = 5 + ... B = 4 + ... C = 3 + ... D = 2 + ... F = 1 + ... + >>> Grade.C < Grade.A + True + + +DuplicateFreeEnum +^^^^^^^^^^^^^^^^^ + +Raises an error if a duplicate member name is found instead of creating an +alias:: + + >>> class DuplicateFreeEnum(Enum): + ... def __init__(self, *args): + ... cls = self.__class__ + ... if any(self.value == e.value for e in cls): + ... a = self.name + ... e = cls(self.value).name + ... raise ValueError( + ... "aliases not allowed in DuplicateFreeEnum: %r --> %r" + ... % (a, e)) + ... + >>> class Color(DuplicateFreeEnum): + ... red = 1 + ... green = 2 + ... blue = 3 + ... grene = 2 + ... + Traceback (most recent call last): + ... + ValueError: aliases not allowed in DuplicateFreeEnum: 'grene' --> 'green' + +.. note:: + + This is a useful example for subclassing Enum to add or change other + behaviors as well as disallowing aliases. If the only desired change is + disallowing aliases, the :func:`unique` decorator can be used instead. + + +Planet +^^^^^^ + +If :meth:`__new__` or :meth:`__init__` is defined the value of the enum member +will be passed to those methods:: + + >>> class Planet(Enum): + ... MERCURY = (3.303e+23, 2.4397e6) + ... VENUS = (4.869e+24, 6.0518e6) + ... EARTH = (5.976e+24, 6.37814e6) + ... MARS = (6.421e+23, 3.3972e6) + ... JUPITER = (1.9e+27, 7.1492e7) + ... SATURN = (5.688e+26, 6.0268e7) + ... URANUS = (8.686e+25, 2.5559e7) + ... NEPTUNE = (1.024e+26, 2.4746e7) + ... def __init__(self, mass, radius): + ... self.mass = mass # in kilograms + ... self.radius = radius # in meters + ... @property + ... def surface_gravity(self): + ... # universal gravitational constant (m3 kg-1 s-2) + ... G = 6.67300E-11 + ... return G * self.mass / (self.radius * self.radius) + ... + >>> Planet.EARTH.value + (5.976e+24, 6378140.0) + >>> Planet.EARTH.surface_gravity + 9.802652743337129 + + +How are Enums different? +------------------------ + +Enums have a custom metaclass that affects many aspects of both derived Enum +classes and their instances (members). + + +Enum Classes +^^^^^^^^^^^^ + +The :class:`EnumMeta` metaclass is responsible for providing the +:meth:`__contains__`, :meth:`__dir__`, :meth:`__iter__` and other methods that +allow one to do things with an :class:`Enum` class that fail on a typical +class, such as `list(Color)` or `some_var in Color`. :class:`EnumMeta` is +responsible for ensuring that various other methods on the final :class:`Enum` +class are correct (such as :meth:`__new__`, :meth:`__getnewargs__`, +:meth:`__str__` and :meth:`__repr__`) + + +Enum Members (aka instances) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The most interesting thing about Enum members is that they are singletons. +:class:`EnumMeta` creates them all while it is creating the :class:`Enum` +class itself, and then puts a custom :meth:`__new__` in place to ensure +that no new ones are ever instantiated by returning only the existing +member instances. + + +Finer Points +^^^^^^^^^^^^ + +Enum members are instances of an Enum class, and even though they are +accessible as `EnumClass.member`, they are not accessible directly from +the member:: + + >>> Color.red + <Color.red: 1> + >>> Color.red.blue + Traceback (most recent call last): + ... + AttributeError: 'Color' object has no attribute 'blue' + +Likewise, the :attr:`__members__` is only available on the class. + +If you give your :class:`Enum` subclass extra methods, like the `Planet`_ +class above, those methods will show up in a :func:`dir` of the member, +but not of the class:: + + >>> dir(Planet) + ['EARTH', 'JUPITER', 'MARS', 'MERCURY', 'NEPTUNE', 'SATURN', 'URANUS', 'VENUS', '__class__', '__doc__', '__members__', '__module__'] + >>> dir(Planet.EARTH) + ['__class__', '__doc__', '__module__', 'name', 'surface_gravity', 'value'] + +A :meth:`__new__` method will only be used for the creation of the +:class:`Enum` members -- after that it is replaced. This means if you wish to +change how :class:`Enum` members are looked up you either have to write a +helper function or a :func:`classmethod`. diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 6e4559c..d2c2ef4 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -28,10 +28,10 @@ handler or to report an error condition "just like" the situation in which the interpreter raises the same exception; but beware that there is nothing to prevent user code from raising an inappropriate error. -The built-in exception classes can be sub-classed to define new exceptions; -programmers are encouraged to at least derive new exceptions from the -:exc:`Exception` class and not :exc:`BaseException`. More information on -defining exceptions is available in the Python Tutorial under +The built-in exception classes can be subclassed to define new exceptions; +programmers are encouraged to derive new exceptions from the :exc:`Exception` +class or one of its subclasses, and not from :exc:`BaseException`. More +information on defining exceptions is available in the Python Tutorial under :ref:`tut-userexceptions`. When raising (or re-raising) an exception in an :keyword:`except` clause @@ -82,7 +82,7 @@ The following exceptions are used mostly as base classes for other exceptions. .. attribute:: args The tuple of arguments given to the exception constructor. Some built-in - exceptions (like :exc:`IOError`) expect a certain number of arguments and + exceptions (like :exc:`OSError`) expect a certain number of arguments and assign a special meaning to the elements of this tuple, while others are usually called only with a single string giving an error message. @@ -161,7 +161,7 @@ The following exceptions are the exceptions that are usually raised. .. exception:: GeneratorExit - Raise when a :term:`generator`\'s :meth:`close` method is called. It + Raised when a :term:`generator`\'s :meth:`close` method is called. It directly inherits from :exc:`BaseException` instead of :exc:`Exception` since it is technically not an error. @@ -253,12 +253,22 @@ The following exceptions are the exceptions that are usually raised. For exceptions that involve a file system path (such as :func:`open` or :func:`os.unlink`), the exception instance will contain an additional attribute, :attr:`filename`, which is the file name passed to the function. + For functions that involve two file system paths (such as + :func:`os.rename`), the exception instance will contain a second + :attr:`filename2` attribute corresponding to the second file name passed + to the function. + .. versionchanged:: 3.3 :exc:`EnvironmentError`, :exc:`IOError`, :exc:`WindowsError`, :exc:`VMSError`, :exc:`socket.error`, :exc:`select.error` and :exc:`mmap.error` have been merged into :exc:`OSError`. + .. versionchanged:: 3.4 + The :attr:`filename` attribute is now the original file name passed to + the function, instead of the name encoded to or decoded from the + filesystem encoding. Also, the :attr:`filename2` attribute was added. + .. exception:: OverflowError @@ -447,10 +457,6 @@ starting from Python 3.3, they are aliases of :exc:`OSError`. .. exception:: IOError -.. exception:: VMSError - - Only available on VMS. - .. exception:: WindowsError Only available on Windows. diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index 3c33621..eb2016a 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -4,12 +4,14 @@ .. module:: faulthandler :synopsis: Dump the Python traceback. +.. versionadded:: 3.3 + This module contains functions to dump Python tracebacks explicitly, on a fault, after a timeout, or on a user signal. Call :func:`faulthandler.enable` to install fault handlers for the :const:`SIGSEGV`, :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS`, and :const:`SIGILL` signals. You can also enable them at startup by setting the :envvar:`PYTHONFAULTHANDLER` environment -variable or by using :option:`-X` ``faulthandler`` command line option. +variable or by using the :option:`-X` ``faulthandler`` command line option. The fault handler is compatible with system fault handlers like Apport or the Windows fault handler. The module uses an alternative stack for signal handlers @@ -27,6 +29,7 @@ tracebacks: * Only the filename, the function name and the line number are displayed. (no source code) * It is limited to 100 frames and 100 threads. +* The order is reversed: the most recent call is shown first. By default, the Python traceback is written to :data:`sys.stderr`. To see tracebacks, applications must be run in the terminal. A log file can @@ -35,11 +38,9 @@ alternatively be passed to :func:`faulthandler.enable`. The module is implemented in C, so tracebacks can be dumped on a crash or when Python is deadlocked. -.. versionadded:: 3.3 - -Dump the traceback ------------------- +Dumping the traceback +--------------------- .. function:: dump_traceback(file=sys.stderr, all_threads=True) @@ -68,8 +69,8 @@ Fault handler state Check if the fault handler is enabled. -Dump the tracebacks after a timeout ------------------------------------ +Dumping the tracebacks after a timeout +-------------------------------------- .. function:: dump_traceback_later(timeout, repeat=False, file=sys.stderr, exit=False) @@ -89,8 +90,8 @@ Dump the tracebacks after a timeout Cancel the last call to :func:`dump_traceback_later`. -Dump the traceback on a user signal ------------------------------------ +Dumping the traceback on a user signal +-------------------------------------- .. function:: register(signum, file=sys.stderr, all_threads=True, chain=False) @@ -109,8 +110,8 @@ Dump the traceback on a user signal Not available on Windows. -File descriptor issue ---------------------- +Issue with file descriptors +--------------------------- :func:`enable`, :func:`dump_traceback_later` and :func:`register` keep the file descriptor of their *file* argument. If the file is closed and its file @@ -122,14 +123,20 @@ these functions again each time that the file is replaced. Example ------- -Example of a segmentation fault on Linux: :: +.. highlight:: sh + +Example of a segmentation fault on Linux with and without enabling the fault +handler:: + + $ python3 -c "import ctypes; ctypes.string_at(0)" + Segmentation fault - $ python -q -X faulthandler + $ python3 -q -X faulthandler >>> import ctypes >>> ctypes.string_at(0) Fatal Python error: Segmentation fault - Current thread 0x00007fb899f39700: + Current thread 0x00007fb899f39700 (most recent call first): File "/home/python/cpython/Lib/ctypes/__init__.py", line 486 in string_at File "<stdin>", line 1 in <module> Segmentation fault diff --git a/Doc/library/filecmp.rst b/Doc/library/filecmp.rst index 8a88f8c..06d3f21 100644 --- a/Doc/library/filecmp.rst +++ b/Doc/library/filecmp.rst @@ -27,6 +27,10 @@ The :mod:`filecmp` module defines the following functions: Note that no external programs are called from this function, giving it portability and efficiency. + This function uses a cache for past comparisons and the results, + with cache entries invalidated if the :func:`os.stat` information for the + file changes. The entire cache may be cleared using :func:`clear_cache`. + .. function:: cmpfiles(dir1, dir2, common, shallow=True) @@ -48,6 +52,15 @@ The :mod:`filecmp` module defines the following functions: one of the three returned lists. +.. function:: clear_cache() + + Clear the filecmp cache. This may be useful if a file is compared so quickly + after it is modified that it is within the mtime resolution of + the underlying filesystem. + + .. versionadded:: 3.4 + + .. _dircmp-objects: The :class:`dircmp` class @@ -55,28 +68,25 @@ The :class:`dircmp` class .. class:: dircmp(a, b, ignore=None, hide=None) - Construct a new directory comparison object, to compare the directories *a* and - *b*. *ignore* is a list of names to ignore, and defaults to ``['RCS', 'CVS', - 'tags']``. *hide* is a list of names to hide, and defaults to ``[os.curdir, - os.pardir]``. + Construct a new directory comparison object, to compare the directories *a* + and *b*. *ignore* is a list of names to ignore, and defaults to + :attr:`filecmp.DEFAULT_IGNORES`. *hide* is a list of names to hide, and + defaults to ``[os.curdir, os.pardir]``. The :class:`dircmp` class compares files by doing *shallow* comparisons as described for :func:`filecmp.cmp`. The :class:`dircmp` class provides the following methods: - .. method:: report() Print (to :data:`sys.stdout`) a comparison between *a* and *b*. - .. method:: report_partial_closure() Print a comparison between *a* and *b* and common immediate subdirectories. - .. method:: report_full_closure() Print a comparison between *a* and *b* and common subdirectories @@ -133,7 +143,7 @@ The :class:`dircmp` class .. attribute:: common_files - Files in both *a* and *b* + Files in both *a* and *b*. .. attribute:: common_funny @@ -164,6 +174,12 @@ The :class:`dircmp` class A dictionary mapping names in :attr:`common_dirs` to :class:`dircmp` objects. +.. attribute:: DEFAULT_IGNORES + + .. versionadded:: 3.4 + + List of directories ignored by :class:`dircmp` by default. + Here is a simplified example of using the ``subdirs`` attribute to search recursively through two directories to show common different files:: diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index d5a4875..ee06830 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -160,6 +160,9 @@ available for subclassing as well: .. versionchanged:: 3.2 Can be used as a context manager. + .. deprecated:: 3.4 + The ``'rU'`` and ``'U'`` modes. + **Optional in-place filtering:** if the keyword argument ``inplace=True`` is passed to :func:`fileinput.input` or to the :class:`FileInput` constructor, the diff --git a/Doc/library/filesys.rst b/Doc/library/filesys.rst index 58998a8..03e085b 100644 --- a/Doc/library/filesys.rst +++ b/Doc/library/filesys.rst @@ -12,6 +12,7 @@ in this chapter is: .. toctree:: + pathlib.rst os.path.rst fileinput.rst stat.rst diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index e0434b0..ef93f05 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -86,7 +86,7 @@ patterns. '.*\\.txt$' >>> reobj = re.compile(regex) >>> reobj.match('foobar.txt') - <_sre.SRE_Match object at 0x...> + <_sre.SRE_Match object; span=(0, 10), match='foobar.txt'> .. seealso:: diff --git a/Doc/library/formatter.rst b/Doc/library/formatter.rst index 88be11c..1847a80 100644 --- a/Doc/library/formatter.rst +++ b/Doc/library/formatter.rst @@ -3,6 +3,11 @@ .. module:: formatter :synopsis: Generic output formatter and device interface. + :deprecated: + +.. deprecated:: 3.4 + Due to lack of usage, the formatter module has been deprecated and is slated + for removal in Python 3.6. This module supports two interface definitions, each with multiple diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index dcb2ac4..bfec9b0 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -80,40 +80,34 @@ The module defines the following items: :rfc:`4217`. Connect as usual to port 21 implicitly securing the FTP control connection before authenticating. Securing the data connection requires the user to - explicitly ask for it by calling the :meth:`prot_p` method. - *keyfile* and *certfile* are optional -- they can contain a PEM formatted - private key and certificate chain file name for the SSL connection. - *context* parameter is a :class:`ssl.SSLContext` object which allows - bundling SSL configuration options, certificates and private keys into a - single (potentially long-lived) structure. *source_address* is a 2-tuple - ``(host, port)`` for the socket to bind to as its source address before - connecting. + explicitly ask for it by calling the :meth:`prot_p` method. *context* + is a :class:`ssl.SSLContext` object which allows bundling SSL configuration + options, certificates and private keys into a single (potentially + long-lived) structure. Please read :ref:`ssl-security` for best practices. + + *keyfile* and *certfile* are a legacy alternative to *context* -- they + can point to PEM-formatted private key and certificate chain files + (respectively) for the SSL connection. .. versionadded:: 3.2 .. versionchanged:: 3.3 *source_address* parameter was added. - Here's a sample session using the :class:`FTP_TLS` class: - - >>> from ftplib import FTP_TLS - >>> ftps = FTP_TLS('ftp.python.org') - >>> ftps.login() # login anonymously before securing control channel - >>> ftps.prot_p() # switch to secure data connection - >>> ftps.retrlines('LIST') # list directory content securely - total 9 - drwxr-xr-x 8 root wheel 1024 Jan 3 1994 . - drwxr-xr-x 8 root wheel 1024 Jan 3 1994 .. - drwxr-xr-x 2 root wheel 1024 Jan 3 1994 bin - drwxr-xr-x 2 root wheel 1024 Jan 3 1994 etc - d-wxrwxr-x 2 ftp wheel 1024 Sep 5 13:43 incoming - drwxr-xr-x 2 root wheel 1024 Nov 17 1993 lib - drwxr-xr-x 6 1094 wheel 1024 Sep 13 19:07 pub - drwxr-xr-x 3 root wheel 1024 Jan 3 1994 usr - -rw-r--r-- 1 root root 312 Aug 1 1994 welcome.msg - '226 Transfer complete.' - >>> ftps.quit() - >>> + .. versionchanged:: 3.4 + The class now supports hostname check with + :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see + :data:`ssl.HAS_SNI`). + + Here's a sample session using the :class:`FTP_TLS` class:: + + >>> ftps = FTP_TLS('ftp.pureftpd.org') + >>> ftps.login() + '230 Anonymous user logged in' + >>> ftps.prot_p() + '200 Data protection level set to "private"' + >>> ftps.nlst() + ['6jack', 'OpenBSD', 'antilink', 'blogbench', 'bsdcam', 'clockspeed', 'djbdns-jedi', 'docs', 'eaccelerator-jedi', 'favicon.ico', 'francotone', 'fugu', 'ignore', 'libpuzzle', 'metalog', 'minidentd', 'misc', 'mysql-udf-global-user-variables', 'php-jenkins-hash', 'php-skein-hash', 'php-webdav', 'phpaudit', 'phpbench', 'pincaster', 'ping', 'posto', 'pub', 'public', 'public_keys', 'pure-ftpd', 'qscan', 'qtc', 'sharedance', 'skycache', 'sound', 'tmp', 'ucarp'] .. exception:: error_reply @@ -427,6 +421,11 @@ FTP_TLS Objects Set up secure control connection by using TLS or SSL, depending on what specified in :meth:`ssl_version` attribute. + .. versionchanged:: 3.4 + The method now supports hostname check with + :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see + :data:`ssl.HAS_SNI`). + .. method:: FTP_TLS.ccc() Revert control channel back to plaintext. This can be useful to take diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 7ed25c1..02e6374 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -540,8 +540,13 @@ are always available. They are listed here in alphabetical order. A call to ``format(value, format_spec)`` is translated to ``type(value).__format__(format_spec)`` which bypasses the instance dictionary when searching for the value's :meth:`__format__` method. A - :exc:`TypeError` exception is raised if the method is not found or if either - the *format_spec* or the return value are not strings. + :exc:`TypeError` exception is raised if the method search reaches + :mod:`object` and the *format_spec* is non-empty, or if either the + *format_spec* or the return value are not strings. + + .. versionchanged:: 3.4 + ``object().__format__(format_spec)`` raises :exc:`TypeError` + if *format_spec* is not an empty string. .. _func-frozenset: @@ -605,6 +610,10 @@ are always available. They are listed here in alphabetical order. This function is added to the built-in namespace by the :mod:`site` module. + .. versionchanged:: 3.4 + Changes to :mod:`pydoc` and :mod:`inspect` mean that the reported + signatures for callables are now more comprehensive and consistent. + .. function:: hex(x) @@ -676,6 +685,12 @@ are always available. They are listed here in alphabetical order. The integer type is described in :ref:`typesnumeric`. + .. versionchanged:: 3.4 + If *base* is not an instance of :class:`int` and the *base* object has a + :meth:`base.__index__ <object.__index__>` method, that method is called + to obtain an integer for the base. Previous versions used + :meth:`base.__int__ <object.__int__>` instead of :meth:`base.__index__ + <object.__index__>`. .. function:: isinstance(object, classinfo) @@ -758,25 +773,31 @@ are always available. They are listed here in alphabetical order. already arranged into argument tuples, see :func:`itertools.starmap`\. -.. function:: max(iterable, *[, key]) +.. function:: max(iterable, *[, key, default]) max(arg1, arg2, *args[, key]) Return the largest item in an iterable or the largest of two or more arguments. - If one positional argument is provided, *iterable* must be a non-empty - iterable (such as a non-empty string, tuple or list). The largest item - in the iterable is returned. If two or more positional arguments are - provided, the largest of the positional arguments is returned. + If one positional argument is provided, it should be an :term:`iterable`. + The largest item in the iterable is returned. If two or more positional + arguments are provided, the largest of the positional arguments is + returned. - The optional keyword-only *key* argument specifies a one-argument ordering - function like that used for :meth:`list.sort`. + There are two optional keyword-only arguments. The *key* argument specifies + a one-argument ordering function like that used for :meth:`list.sort`. The + *default* argument specifies an object to return if the provided iterable is + empty. If the iterable is empty and *default* is not provided, a + :exc:`ValueError` is raised. If multiple items are maximal, the function returns the first one encountered. This is consistent with other sort-stability preserving tools such as ``sorted(iterable, key=keyfunc, reverse=True)[0]`` and ``heapq.nlargest(1, iterable, key=keyfunc)``. + .. versionadded:: 3.4 + The *default* keyword-only argument. + .. _func-memoryview: .. function:: memoryview(obj) @@ -786,25 +807,32 @@ are always available. They are listed here in alphabetical order. :ref:`typememoryview` for more information. -.. function:: min(iterable, *[, key]) +.. function:: min(iterable, *[, key, default]) min(arg1, arg2, *args[, key]) Return the smallest item in an iterable or the smallest of two or more arguments. - If one positional argument is provided, *iterable* must be a non-empty - iterable (such as a non-empty string, tuple or list). The smallest item - in the iterable is returned. If two or more positional arguments are - provided, the smallest of the positional arguments is returned. + If one positional argument is provided, it should be an :term:`iterable`. + The smallest item in the iterable is returned. If two or more positional + arguments are provided, the smallest of the positional arguments is + returned. - The optional keyword-only *key* argument specifies a one-argument ordering - function like that used for :meth:`list.sort`. + There are two optional keyword-only arguments. The *key* argument specifies + a one-argument ordering function like that used for :meth:`list.sort`. The + *default* argument specifies an object to return if the provided iterable is + empty. If the iterable is empty and *default* is not provided, a + :exc:`ValueError` is raised. If multiple items are minimal, the function returns the first one encountered. This is consistent with other sort-stability preserving tools such as ``sorted(iterable, key=keyfunc)[0]`` and ``heapq.nsmallest(1, iterable, key=keyfunc)``. + .. versionadded:: 3.4 + The *default* keyword-only argument. + + .. function:: next(iterator[, default]) Retrieve the next item from the *iterator* by calling its @@ -866,8 +894,7 @@ are always available. They are listed here in alphabetical order. ``'b'`` binary mode ``'t'`` text mode (default) ``'+'`` open a disk file for updating (reading and writing) - ``'U'`` universal newlines mode (for backwards compatibility; should - not be used in new code) + ``'U'`` :term:`universal newlines` mode (deprecated) ========= =============================================================== The default mode is ``'r'`` (open for reading text, synonym of ``'rt'``). @@ -973,6 +1000,8 @@ are always available. They are listed here in alphabetical order. :mod:`os.open` as *opener* results in functionality similar to passing ``None``). + The newly created file is :ref:`non-inheritable <fd_inheritance>`. + The following example uses the :ref:`dir_fd <dir_fd>` parameter of the :func:`os.open` function to open a file relative to a given directory:: @@ -986,10 +1015,6 @@ are always available. They are listed here in alphabetical order. ... >>> os.close(dir_fd) # don't leak a file descriptor - .. versionchanged:: 3.3 - The *opener* parameter was added. - The ``'x'`` mode was added. - The type of :term:`file object` returned by the :func:`open` function depends on the mode. When :func:`open` is used to open a file in a text mode (``'w'``, ``'r'``, ``'wt'``, ``'rt'``, etc.), it returns a subclass of @@ -1016,10 +1041,18 @@ are always available. They are listed here in alphabetical order. and :mod:`shutil`. .. versionchanged:: 3.3 + The *opener* parameter was added. + The ``'x'`` mode was added. :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`. :exc:`FileExistsError` is now raised if the file opened in exclusive creation mode (``'x'``) already exists. + .. versionchanged:: 3.4 + The file is now non-inheritable. + + .. deprecated-removed:: 3.4 4.0 + The ``'U'`` mode. + .. XXX works for bytes too, but should it? .. function:: ord(c) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 3c946e3..c46b799 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -6,6 +6,7 @@ .. moduleauthor:: Peter Harris <scav@blueyonder.co.uk> .. moduleauthor:: Raymond Hettinger <python@rcn.com> .. moduleauthor:: Nick Coghlan <ncoghlan@gmail.com> +.. moduleauthor:: Łukasz Langa <lukasz@langa.pl> .. sectionauthor:: Peter Harris <scav@blueyonder.co.uk> **Source code:** :source:`Lib/functools.py` @@ -133,15 +134,34 @@ The :mod:`functools` module defines the following functions: @total_ordering class Student: + def _is_valid_operand(self, other): + return (hasattr(other, "lastname") and + hasattr(other, "firstname")) def __eq__(self, other): + if not self._is_valid_operand(other): + return NotImplemented return ((self.lastname.lower(), self.firstname.lower()) == (other.lastname.lower(), other.firstname.lower())) def __lt__(self, other): + if not self._is_valid_operand(other): + return NotImplemented return ((self.lastname.lower(), self.firstname.lower()) < (other.lastname.lower(), other.firstname.lower())) + .. note:: + + While this decorator makes it easy to create well behaved totally + ordered types, it *does* come at the cost of slower execution and + more complex stack traces for the derived comparison methods. If + performance benchmarking indicates this is a bottleneck for a given + application, implementing all six rich comparison methods instead is + likely to provide an easy speed boost. + .. versionadded:: 3.2 + .. versionchanged:: 3.4 + Returning NotImplemented from the underlying comparison function for + unrecognised types is now supported. .. function:: partial(func, *args, **keywords) @@ -174,6 +194,50 @@ The :mod:`functools` module defines the following functions: 18 +.. class:: partialmethod(func, *args, **keywords) + + Return a new :class:`partialmethod` descriptor which behaves + like :class:`partial` except that it is designed to be used as a method + definition rather than being directly callable. + + *func* must be a :term:`descriptor` or a callable (objects which are both, + like normal functions, are handled as descriptors). + + When *func* is a descriptor (such as a normal Python function, + :func:`classmethod`, :func:`staticmethod`, :func:`abstractmethod` or + another instance of :class:`partialmethod`), calls to ``__get__`` are + delegated to the underlying descriptor, and an appropriate + :class:`partial` object returned as the result. + + When *func* is a non-descriptor callable, an appropriate bound method is + created dynamically. This behaves like a normal Python function when + used as a method: the *self* argument will be inserted as the first + positional argument, even before the *args* and *keywords* supplied to + the :class:`partialmethod` constructor. + + Example:: + + >>> class Cell(object): + ... def __init__(self): + ... self._alive = False + ... @property + ... def alive(self): + ... return self._alive + ... def set_state(self, state): + ... self._alive = bool(state) + ... set_alive = partialmethod(set_state, True) + ... set_dead = partialmethod(set_state, False) + ... + >>> c = Cell() + >>> c.alive + False + >>> c.set_alive() + >>> c.alive + True + + .. versionadded:: 3.4 + + .. function:: reduce(function, iterable[, initializer]) Apply *function* of two arguments cumulatively to the items of *sequence*, from @@ -198,6 +262,111 @@ The :mod:`functools` module defines the following functions: return value +.. decorator:: singledispatch(default) + + Transforms a function into a :term:`single-dispatch <single + dispatch>` :term:`generic function`. + + To define a generic function, decorate it with the ``@singledispatch`` + decorator. Note that the dispatch happens on the type of the first argument, + create your function accordingly:: + + >>> from functools import singledispatch + >>> @singledispatch + ... def fun(arg, verbose=False): + ... if verbose: + ... print("Let me just say,", end=" ") + ... print(arg) + + To add overloaded implementations to the function, use the :func:`register` + attribute of the generic function. It is a decorator, taking a type + parameter and decorating a function implementing the operation for that + type:: + + >>> @fun.register(int) + ... def _(arg, verbose=False): + ... if verbose: + ... print("Strength in numbers, eh?", end=" ") + ... print(arg) + ... + >>> @fun.register(list) + ... def _(arg, verbose=False): + ... if verbose: + ... print("Enumerate this:") + ... for i, elem in enumerate(arg): + ... print(i, elem) + + To enable registering lambdas and pre-existing functions, the + :func:`register` attribute can be used in a functional form:: + + >>> def nothing(arg, verbose=False): + ... print("Nothing.") + ... + >>> fun.register(type(None), nothing) + + The :func:`register` attribute returns the undecorated function which + enables decorator stacking, pickling, as well as creating unit tests for + each variant independently:: + + >>> @fun.register(float) + ... @fun.register(Decimal) + ... def fun_num(arg, verbose=False): + ... if verbose: + ... print("Half of your number:", end=" ") + ... print(arg / 2) + ... + >>> fun_num is fun + False + + When called, the generic function dispatches on the type of the first + argument:: + + >>> fun("Hello, world.") + Hello, world. + >>> fun("test.", verbose=True) + Let me just say, test. + >>> fun(42, verbose=True) + Strength in numbers, eh? 42 + >>> fun(['spam', 'spam', 'eggs', 'spam'], verbose=True) + Enumerate this: + 0 spam + 1 spam + 2 eggs + 3 spam + >>> fun(None) + Nothing. + >>> fun(1.23) + 0.615 + + Where there is no registered implementation for a specific type, its + method resolution order is used to find a more generic implementation. + The original function decorated with ``@singledispatch`` is registered + for the base ``object`` type, which means it is used if no better + implementation is found. + + To check which implementation will the generic function choose for + a given type, use the ``dispatch()`` attribute:: + + >>> fun.dispatch(float) + <function fun_num at 0x1035a2840> + >>> fun.dispatch(dict) # note: default implementation + <function fun at 0x103fe0000> + + To access all registered implementations, use the read-only ``registry`` + attribute:: + + >>> fun.registry.keys() + dict_keys([<class 'NoneType'>, <class 'int'>, <class 'object'>, + <class 'decimal.Decimal'>, <class 'list'>, + <class 'float'>]) + >>> fun.registry[float] + <function fun_num at 0x1035a2840> + >>> fun.registry[object] + <function fun at 0x103fe0000> + + .. versionadded:: 3.4 + + .. function:: update_wrapper(wrapper, wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) Update a *wrapper* function to look like the *wrapped* function. The optional @@ -212,8 +381,8 @@ The :mod:`functools` module defines the following functions: To allow access to the original function for introspection and other purposes (e.g. bypassing a caching decorator such as :func:`lru_cache`), this function - automatically adds a __wrapped__ attribute to the wrapper that refers to - the original function. + automatically adds a ``__wrapped__`` attribute to the wrapper that refers to + the function being wrapped. The main intended use for this function is in :term:`decorator` functions which wrap the decorated function and return the wrapper. If the wrapper function is @@ -236,6 +405,11 @@ The :mod:`functools` module defines the following functions: .. versionchanged:: 3.2 Missing attributes no longer trigger an :exc:`AttributeError`. + .. versionchanged:: 3.4 + The ``__wrapped__`` attribute now always refers to the wrapped + function, even if that function defined a ``__wrapped__`` attribute. + (see :issue:`17482`) + .. decorator:: wraps(wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) @@ -301,4 +475,3 @@ differences. For instance, the :attr:`__name__` and :attr:`__doc__` attributes are not created automatically. Also, :class:`partial` objects defined in classes behave like static methods and do not transform into bound methods during instance attribute look-up. - diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 71449a3..8135542 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -67,6 +67,25 @@ The :mod:`gc` module provides the following functions: returned. +.. function:: get_stats() + + Return a list of three per-generation dictionaries containing collection + statistics since interpreter start. The number of keys may change + in the future, but currently each dictionary will contain the following + items: + + * ``collections`` is the number of times this generation was collected; + + * ``collected`` is the total number of objects collected inside this + generation; + + * ``uncollectable`` is the total number of objects which were found + to be uncollectable (and were therefore moved to the :data:`garbage` + list) inside this generation. + + .. versionadded:: 3.4 + + .. function:: set_threshold(threshold0[, threshold1[, threshold2]]) Set the garbage collection thresholds (the collection frequency). Setting @@ -158,24 +177,13 @@ values but should not rebind them): .. data:: garbage - A list of objects which the collector found to be unreachable but could not be - freed (uncollectable objects). By default, this list contains only objects with - :meth:`__del__` methods. Objects that have :meth:`__del__` methods and are - part of a reference cycle cause the entire reference cycle to be uncollectable, - including objects not necessarily in the cycle but reachable only from it. - Python doesn't collect such cycles automatically because, in general, it isn't - possible for Python to guess a safe order in which to run the :meth:`__del__` - methods. If you know a safe order, you can force the issue by examining the - *garbage* list, and explicitly breaking cycles due to your objects within the - list. Note that these objects are kept alive even so by virtue of being in the - *garbage* list, so they should be removed from *garbage* too. For example, - after breaking cycles, do ``del gc.garbage[:]`` to empty the list. It's - generally better to avoid the issue by not creating cycles containing objects - with :meth:`__del__` methods, and *garbage* can be examined in that case to - verify that no such cycles are being created. - - If :const:`DEBUG_SAVEALL` is set, then all unreachable objects will be added - to this list rather than freed. + A list of objects which the collector found to be unreachable but could + not be freed (uncollectable objects). Starting with Python 3.4, this + list should be empty most of the time, except when using instances of + C extension types with a non-NULL ``tp_del`` slot. + + If :const:`DEBUG_SAVEALL` is set, then all unreachable objects will be + added to this list rather than freed. .. versionchanged:: 3.2 If this list is non-empty at interpreter shutdown, a @@ -183,6 +191,10 @@ values but should not rebind them): :const:`DEBUG_UNCOLLECTABLE` is set, in addition all uncollectable objects are printed. + .. versionchanged:: 3.4 + Following :pep:`442`, objects with a :meth:`__del__` method don't end + up in :attr:`gc.garbage` anymore. + .. data:: callbacks A list of callbacks that will be invoked by the garbage collector before and diff --git a/Doc/library/getpass.rst b/Doc/library/getpass.rst index ffe2b12..211563e 100644 --- a/Doc/library/getpass.rst +++ b/Doc/library/getpass.rst @@ -13,10 +13,11 @@ The :mod:`getpass` module provides two functions: .. function:: getpass(prompt='Password: ', stream=None) Prompt the user for a password without echoing. The user is prompted using - the string *prompt*, which defaults to ``'Password: '``. On Unix, the prompt - is written to the file-like object *stream*. *stream* defaults to the - controlling terminal (:file:`/dev/tty`) or if that is unavailable to - ``sys.stderr`` (this argument is ignored on Windows). + the string *prompt*, which defaults to ``'Password: '``. On Unix, the + prompt is written to the file-like object *stream* using the replace error + handler if needed. *stream* defaults to the controlling terminal + (:file:`/dev/tty`) or if that is unavailable to ``sys.stderr`` (this + argument is ignored on Windows). If echo free input is unavailable getpass() falls back to printing a warning message to *stream* and reading from ``sys.stdin`` and diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index eff138b..abcbf38 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -25,6 +25,10 @@ For a literal match, wrap the meta-characters in brackets. For example, ``'[?]'`` matches the character ``'?'``. +.. seealso:: + The :mod:`pathlib` module offers high-level path objects. + + .. function:: glob(pathname) Return a possibly-empty list of path names that match *pathname*, which must be @@ -40,6 +44,17 @@ For example, ``'[?]'`` matches the character ``'?'``. without actually storing them all simultaneously. +.. function:: escape(pathname) + + Escape all special characters (``'?'``, ``'*'`` and ``'['``). + This is useful if you want to match an arbitrary literal string that may + have special characters in it. Special characters in drive/UNC + sharepoints are not escaped, e.g. on Windows + ``escape('//?/c:/Quo vadis?.txt')`` returns ``'//?/c:/Quo vadis[?].txt'``. + + .. versionadded:: 3.4 + + For example, consider a directory containing only the following files: :file:`1.gif`, :file:`2.txt`, and :file:`card.gif`. :func:`glob` will produce the following results. Notice how any leading components of the path are diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index 354deed..78536fa 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -35,8 +35,8 @@ The module defines the following items: :class:`bytes` object), or an existing file object to read from or write to. The *mode* argument can be any of ``'r'``, ``'rb'``, ``'a'``, ``'ab'``, - ``'w'``, or ``'wb'`` for binary mode, or ``'rt'``, ``'at'``, or ``'wt'`` for - text mode. The default is ``'rb'``. + ``'w'``, ``'wb'``, ``'x'`` or ``'xb'`` for binary mode, or ``'rt'``, + ``'at'``, ``'wt'``, or ``'xt'`` for text mode. The default is ``'rb'``. The *compresslevel* argument is an integer from 0 to 9, as for the :class:`GzipFile` constructor. @@ -53,6 +53,9 @@ The module defines the following items: Added support for *filename* being a file object, support for text mode, and the *encoding*, *errors* and *newline* arguments. + .. versionchanged:: 3.4 + Added support for the ``'x'``, ``'xb'`` and ``'xt'`` modes. + .. class:: GzipFile(filename=None, mode=None, compresslevel=9, fileobj=None, mtime=None) @@ -73,8 +76,9 @@ The module defines the following items: original filename is not included in the header. The *mode* argument can be any of ``'r'``, ``'rb'``, ``'a'``, ``'ab'``, ``'w'``, - or ``'wb'``, depending on whether the file will be read or written. The default - is the mode of *fileobj* if discernible; otherwise, the default is ``'rb'``. + ``'wb'``, ``'x'``, or ``'xb'``, depending on whether the file will be read or + written. The default is the mode of *fileobj* if discernible; otherwise, the + default is ``'rb'``. Note that the file is always opened in binary mode. To open a compressed file in text mode, use :func:`.open` (or wrap your :class:`GzipFile` with an @@ -130,6 +134,9 @@ The module defines the following items: .. versionchanged:: 3.3 The :meth:`io.BufferedIOBase.read1` method is now implemented. + .. versionchanged:: 3.4 + Added support for the ``'x'`` and ``'xb'`` modes. + .. function:: compress(data, compresslevel=9) diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index a693f7e..e0a877a 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -32,6 +32,12 @@ digests. The modern term is secure hash. Some algorithms have known hash collision weaknesses, refer to the "See also" section at the end. + +.. _hash-algorithms: + +Hash algorithms +--------------- + There is one constructor method named for each type of :dfn:`hash`. All return a hash object with the same simple interface. For example: use :func:`sha1` to create a SHA1 hash object. You can now feed this object with :term:`bytes-like @@ -53,9 +59,9 @@ concatenation of the data fed to it so far using the :meth:`digest` or .. index:: single: OpenSSL; (use in module hashlib) Constructors for hash algorithms that are always present in this module are -:func:`md5`, :func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, and -:func:`sha512`. Additional algorithms may also be available depending upon the -OpenSSL library that Python uses on your platform. +:func:`md5`, :func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, +and :func:`sha512`. Additional algorithms may also be available depending upon +the OpenSSL library that Python uses on your platform. For example, to obtain the digest of the byte string ``b'Nobody inspects the spammish repetition'``:: @@ -122,6 +128,18 @@ returned by the constructors: The internal block size of the hash algorithm in bytes. +A hash object has the following attributes: + +.. attribute:: hash.name + + The canonical name of this hash, always lowercase and always suitable as a + parameter to :func:`new` to create another hash of this type. + + .. versionchanged:: 3.4 + The name attribute has been present in CPython since its inception, but + until Python 3.4 was not formally specified, so may not exist on some + platforms. + A hash object has the following methods: @@ -158,6 +176,46 @@ A hash object has the following methods: compute the digests of data sharing a common initial substring. +Key Derivation Function +----------------------- + +Key derivation and key stretching algorithms are designed for secure password +hashing. Naive algorithms such as ``sha1(password)`` are not resistant against +brute-force attacks. A good password hashing function must be tunable, slow, and +include a `salt <https://en.wikipedia.org/wiki/Salt_%28cryptography%29>`_. + + +.. function:: pbkdf2_hmac(name, password, salt, rounds, dklen=None) + + The function provides PKCS#5 password-based key derivation function 2. It + uses HMAC as pseudorandom function. + + The string *name* is the desired name of the hash digest algorithm for + HMAC, e.g. 'sha1' or 'sha256'. *password* and *salt* are interpreted as + buffers of bytes. Applications and libraries should limit *password* to + a sensible value (e.g. 1024). *salt* should be about 16 or more bytes from + a proper source, e.g. :func:`os.urandom`. + + The number of *rounds* should be chosen based on the hash algorithm and + computing power. As of 2013, at least 100,000 rounds of SHA-256 is suggested. + + *dklen* is the length of the derived key. If *dklen* is ``None`` then the + digest size of the hash algorithm *name* is used, e.g. 64 for SHA-512. + + >>> import hashlib, binascii + >>> dk = hashlib.pbkdf2_hmac('sha256', b'password', b'salt', 100000) + >>> binascii.hexlify(dk) + b'0394a2ede332c9a13eb82e9b24631604c31df978b4e2f0fbd2c549944f9d79a5' + + .. versionadded:: 3.4 + + .. note:: + + A fast implementation of *pbkdf2_hmac* is available with OpenSSL. The + Python implementation uses an inline version of :mod:`hmac`. It is about + three times slower and doesn't release the GIL. + + .. seealso:: Module :mod:`hmac` @@ -173,3 +231,5 @@ A hash object has the following methods: Wikipedia article with information on which algorithms have known issues and what that means regarding their use. + http://www.ietf.org/rfc/rfc2898.txt + PKCS #5: Password-Based Cryptography Specification Version 2.0 diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index 9575693..4858235 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -16,20 +16,32 @@ This module implements the HMAC algorithm as described by :rfc:`2104`. .. function:: new(key, msg=None, digestmod=None) - Return a new hmac object. *key* is a bytes object giving the secret key. If - *msg* is present, the method call ``update(msg)`` is made. *digestmod* is - the digest constructor or module for the HMAC object to use. It defaults to - the :data:`hashlib.md5` constructor. + Return a new hmac object. *key* is a bytes or bytearray object giving the + secret key. If *msg* is present, the method call ``update(msg)`` is made. + *digestmod* is the digest name, digest constructor or module for the HMAC + object to use. It supports any name suitable to :func:`hashlib.new` and + defaults to the :data:`hashlib.md5` constructor. + + .. versionchanged:: 3.4 + Parameter *key* can be a bytes or bytearray object. + Parameter *msg* can be of any type supported by :mod:`hashlib`. + Paramter *digestmod* can be the name of a hash algorithm. + + .. deprecated:: 3.4 + MD5 as implicit default digest for *digestmod* is deprecated. An HMAC object has the following methods: .. method:: HMAC.update(msg) - Update the hmac object with the bytes object *msg*. Repeated calls are - equivalent to a single call with the concatenation of all the arguments: + Update the hmac object with *msg*. Repeated calls are equivalent to a + single call with the concatenation of all the arguments: ``m.update(a); m.update(b)`` is equivalent to ``m.update(a + b)``. + .. versionchanged:: 3.4 + Parameter *msg* can be of any type supported by :mod:`hashlib`. + .. method:: HMAC.digest() @@ -66,6 +78,25 @@ An HMAC object has the following methods: compute the digests of strings that share a common initial substring. +A hash object has the following attributes: + +.. attribute:: HMAC.digest_size + + The size of the resulting HMAC digest in bytes. + +.. attribute:: HMAC.block_size + + The internal block size of the hash algorithm in bytes. + + .. versionadded:: 3.4 + +.. attribute:: HMAC.name + + The canonical name of this HMAC, always lowercase, e.g. ``hmac-md5``. + + .. versionadded:: 3.4 + + This module also provides the following helper function: .. function:: compare_digest(a, b) diff --git a/Doc/library/html.entities.rst b/Doc/library/html.entities.rst index 0aabbad..09b0abc 100644 --- a/Doc/library/html.entities.rst +++ b/Doc/library/html.entities.rst @@ -20,6 +20,7 @@ This module defines four dictionaries, :data:`html5`, Note that the trailing semicolon is included in the name (e.g. ``'gt;'``), however some of the names are accepted by the standard even without the semicolon: in this case the name is present with and without the ``';'``. + See also :func:`html.unescape`. .. versionadded:: 3.3 diff --git a/Doc/library/html.parser.rst b/Doc/library/html.parser.rst index e4154ef..44b7d6e 100644 --- a/Doc/library/html.parser.rst +++ b/Doc/library/html.parser.rst @@ -16,14 +16,21 @@ This module defines a class :class:`HTMLParser` which serves as the basis for parsing text files formatted in HTML (HyperText Mark-up Language) and XHTML. -.. class:: HTMLParser(strict=False) +.. class:: HTMLParser(strict=False, *, convert_charrefs=False) - Create a parser instance. If *strict* is ``False`` (the default), the parser - will accept and parse invalid markup. If *strict* is ``True`` the parser - will raise an :exc:`~html.parser.HTMLParseError` exception instead [#]_ when - it's not able to parse the markup. - The use of ``strict=True`` is discouraged and the *strict* argument is - deprecated. + Create a parser instance. + + If *convert_charrefs* is ``True`` (default: ``False``), all character + references (except the ones in ``script``/``style`` elements) are + automatically converted to the corresponding Unicode characters. + The use of ``convert_charrefs=True`` is encouraged and will become + the default in Python 3.5. + + If *strict* is ``False`` (the default), the parser will accept and parse + invalid markup. If *strict* is ``True`` the parser will raise an + :exc:`~html.parser.HTMLParseError` exception instead [#]_ when it's not + able to parse the markup. The use of ``strict=True`` is discouraged and + the *strict* argument is deprecated. An :class:`.HTMLParser` instance is fed HTML data and calls handler methods when start tags, end tags, text, comments, and other markup elements are @@ -34,12 +41,15 @@ parsing text files formatted in HTML (HyperText Mark-up Language) and XHTML. handler for elements which are closed implicitly by closing an outer element. .. versionchanged:: 3.2 - *strict* keyword added. + *strict* argument added. .. deprecated-removed:: 3.3 3.5 The *strict* argument and the strict mode have been deprecated. The parser is now able to accept and parse invalid markup too. + .. versionchanged:: 3.4 + *convert_charrefs* keyword argument added. + An exception is defined as well: @@ -74,7 +84,7 @@ as they are encountered:: def handle_data(self, data): print("Encountered some data :", data) - parser = MyHTMLParser(strict=False) + parser = MyHTMLParser() parser.feed('<html><head><title>Test</title></head>' '<body><h1>Parse me!</h1></body></html>') @@ -181,7 +191,8 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`): This method is called to process a named character reference of the form ``&name;`` (e.g. ``>``), where *name* is a general entity reference - (e.g. ``'gt'``). + (e.g. ``'gt'``). This method is never called if *convert_charrefs* is + ``True``. .. method:: HTMLParser.handle_charref(name) @@ -189,7 +200,8 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`): This method is called to process decimal and hexadecimal numeric character references of the form ``&#NNN;`` and ``&#xNNN;``. For example, the decimal equivalent for ``>`` is ``>``, whereas the hexadecimal is ``>``; - in this case the method will receive ``'62'`` or ``'x3E'``. + in this case the method will receive ``'62'`` or ``'x3E'``. This method + is never called if *convert_charrefs* is ``True``. .. method:: HTMLParser.handle_comment(data) @@ -272,7 +284,7 @@ examples:: def handle_decl(self, data): print("Decl :", data) - parser = MyHTMLParser(strict=False) + parser = MyHTMLParser() Parsing a doctype:: @@ -324,7 +336,8 @@ correct char (note: these 3 references are all equivalent to ``'>'``):: Num ent : > Feeding incomplete chunks to :meth:`~HTMLParser.feed` works, but -:meth:`~HTMLParser.handle_data` might be called more than once:: +:meth:`~HTMLParser.handle_data` might be called more than once +(unless *convert_charrefs* is set to ``True``):: >>> for chunk in ['<sp', 'an>buff', 'ered ', 'text</s', 'pan>']: ... parser.feed(chunk) diff --git a/Doc/library/html.rst b/Doc/library/html.rst index 1107ca9..d0706bc 100644 --- a/Doc/library/html.rst +++ b/Doc/library/html.rst @@ -20,6 +20,17 @@ This module defines utilities to manipulate HTML. .. versionadded:: 3.2 + +.. function:: unescape(s) + + Convert all named and numeric character references (e.g. ``>``, + ``>``, ``&x3e;``) in the string *s* to the corresponding unicode + characters. This function uses the rules defined by the HTML 5 standard + for both valid and invalid character references, and the :data:`list of + HTML 5 named character references <html.entities.html5>`. + + .. versionadded:: 3.4 + -------------- Submodules in the ``html`` package are: diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index bb30f24..9f6bcd1 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -27,7 +27,7 @@ HTTPS protocols. It is normally not used directly --- the module The module provides the following classes: -.. class:: HTTPConnection(host, port=None[, strict][, timeout], \ +.. class:: HTTPConnection(host, port=None[, timeout], \ source_address=None) An :class:`HTTPConnection` instance represents one transaction with an HTTP @@ -43,44 +43,48 @@ The module provides the following classes: For example, the following calls all create instances that connect to the server at the same host and port:: - >>> h1 = http.client.HTTPConnection('www.cwi.nl') - >>> h2 = http.client.HTTPConnection('www.cwi.nl:80') - >>> h3 = http.client.HTTPConnection('www.cwi.nl', 80) - >>> h3 = http.client.HTTPConnection('www.cwi.nl', 80, timeout=10) + >>> h1 = http.client.HTTPConnection('www.python.org') + >>> h2 = http.client.HTTPConnection('www.python.org:80') + >>> h3 = http.client.HTTPConnection('www.python.org', 80) + >>> h4 = http.client.HTTPConnection('www.python.org', 80, timeout=10) .. versionchanged:: 3.2 *source_address* was added. - .. deprecated-removed:: 3.2 3.4 - The *strict* parameter is deprecated. HTTP 0.9-style "Simple Responses" - are not supported anymore. + .. versionchanged:: 3.4 + The *strict* parameter was removed. HTTP 0.9-style "Simple Responses" are + not longer supported. .. class:: HTTPSConnection(host, port=None, key_file=None, \ - cert_file=None[, strict][, timeout], \ + cert_file=None[, timeout], \ source_address=None, *, context=None, \ check_hostname=None) A subclass of :class:`HTTPConnection` that uses SSL for communication with secure servers. Default port is ``443``. If *context* is specified, it must be a :class:`ssl.SSLContext` instance describing the various SSL - options. If *context* is specified and has a :attr:`~ssl.SSLContext.verify_mode` - of either :data:`~ssl.CERT_OPTIONAL` or :data:`~ssl.CERT_REQUIRED`, then - by default *host* is matched against the host name(s) allowed by the - server's certificate. If you want to change that behaviour, you can - explicitly set *check_hostname* to False. + options. *key_file* and *cert_file* are deprecated, please use - :meth:`ssl.SSLContext.load_cert_chain` instead. + :meth:`ssl.SSLContext.load_cert_chain` instead, or let + :func:`ssl.create_default_context` select the system's trusted CA + certificates for you. - If you access arbitrary hosts on the Internet, it is recommended to - require certificate checking and feed the *context* with a set of - trusted CA certificates:: + The recommended way to connect to HTTPS hosts on the Internet is as + follows:: - context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - context.verify_mode = ssl.CERT_REQUIRED - context.load_verify_locations('/etc/pki/tls/certs/ca-bundle.crt') - h = client.HTTPSConnection('svn.python.org', 443, context=context) + context = ssl.create_default_context() + h = client.HTTPSConnection('www.python.org', 443, context=context) + + Please read :ref:`ssl-security` for more information on best practices. + + .. note:: + If *context* is specified and has a :attr:`~ssl.SSLContext.verify_mode` + of either :data:`~ssl.CERT_OPTIONAL` or :data:`~ssl.CERT_REQUIRED`, then + by default *host* is matched against the host name(s) allowed by the + server's certificate. If you want to change that behaviour, you can + explicitly set *check_hostname* to False. .. versionchanged:: 3.2 *source_address*, *context* and *check_hostname* were added. @@ -89,19 +93,19 @@ The module provides the following classes: This class now supports HTTPS virtual hosts if possible (that is, if :data:`ssl.HAS_SNI` is true). - .. deprecated-removed:: 3.2 3.4 - The *strict* parameter is deprecated. HTTP 0.9-style "Simple Responses" - are not supported anymore. + .. versionchanged:: 3.4 + The *strict* parameter was removed. HTTP 0.9-style "Simple Responses" are + no longer supported. -.. class:: HTTPResponse(sock, debuglevel=0[, strict], method=None, url=None) +.. class:: HTTPResponse(sock, debuglevel=0, method=None, url=None) Class whose instances are returned upon successful connection. Not instantiated directly by user. - .. deprecated-removed:: 3.2 3.4 - The *strict* parameter is deprecated. HTTP 0.9-style "Simple Responses" - are not supported anymore. + .. versionchanged:: 3.4 + The *strict* parameter was removed. HTTP 0.9 style "Simple Responses" are + no longer supported. The following exceptions are raised as appropriate: @@ -656,7 +660,7 @@ request using http.client:: >>> # This creates an HTTP message >>> # with the content of BODY as the enclosed representation - >>> # for the resource http://localhost:8080/foobar + >>> # for the resource http://localhost:8080/file ... >>> import http.client >>> BODY = "***filecontents***" diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 5aeb719..0d8e7fe 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -81,7 +81,10 @@ of which this module provides three different variants: Holds an instance of the class specified by the :attr:`MessageClass` class variable. This instance parses and manages the headers in the HTTP - request. + request. The :func:`~http.client.parse_headers` function from + :mod:`http.client` is used to parse the headers and it requires that the + HTTP request provide a valid :rfc:`2822` style header. + .. attribute:: rfile @@ -116,7 +119,7 @@ of which this module provides three different variants: HTTP error code value. *message* should be a string containing a (detailed) error message of what occurred, and *explain* should be an explanation of the error code number. Default *message* and *explain* - values can found in the *responses* class variable. + values can found in the :attr:`responses` class variable. .. attribute:: error_content_type @@ -170,12 +173,22 @@ of which this module provides three different variants: .. versionadded:: 3.2 - .. method:: send_error(code, message=None) + .. method:: send_error(code, message=None, explain=None) Sends and logs a complete error reply to the client. The numeric *code* - specifies the HTTP error code, with *message* as optional, more specific text. A - complete set of headers is sent, followed by text composed using the - :attr:`error_message_format` class variable. + specifies the HTTP error code, with *message* as an optional, short, human + readable description of the error. The *explain* argument can be used to + provide more detailed information about the error; it will be formatted + using the :attr:`error_message_format` class variable and emitted, after + a complete set of headers, as the response body. The :attr:`responses` + class variable holds the default values for *message* and *explain* that + will be used if no value is provided; for unknown codes the default value + for both is the string ``???``. + + .. versionchanged:: 3.4 + The error response includes a Content-Length header. + Added the *explain* argument. + .. method:: send_response(code, message=None) @@ -341,7 +354,7 @@ of which this module provides three different variants: The :class:`SimpleHTTPRequestHandler` class can be used in the following manner in order to create a very basic webserver serving files relative to -the current directory. :: +the current directory:: import http.server import socketserver @@ -355,12 +368,23 @@ the current directory. :: print("serving at port", PORT) httpd.serve_forever() +.. _http-server-cli: + :mod:`http.server` can also be invoked directly using the :option:`-m` switch of the interpreter with a ``port number`` argument. Similar to -the previous example, this serves files relative to the current directory. :: +the previous example, this serves files relative to the current directory:: python -m http.server 8000 +By default, server binds itself to all interfaces. The option ``-b/--bind`` +specifies a specific address to which it should bind. For example, the +following command causes the server to bind to localhost only:: + + python -m http.server 8000 --bind 127.0.0.1 + +.. versionadded:: 3.4 + ``--bind`` argument was introduced. + .. class:: CGIHTTPRequestHandler(request, client_address, server) @@ -403,7 +427,7 @@ the previous example, this serves files relative to the current directory. :: reasons. Problems with the CGI script will be translated to error 403. :class:`CGIHTTPRequestHandler` can be enabled in the command line by passing -the ``--cgi`` option.:: +the ``--cgi`` option:: python -m http.server --cgi 8000 diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 36d78b0..2718cef 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -16,70 +16,82 @@ IDLE has the following features: * coded in 100% pure Python, using the :mod:`tkinter` GUI toolkit -* cross-platform: works on Windows and Unix +* cross-platform: works on Windows, Unix, and Mac OS X -* multi-window text editor with multiple undo, Python colorizing and many other - features, e.g. smart indent and call tips +* multi-window text editor with multiple undo, Python colorizing, + smart indent, call tips, and many other features * Python shell window (a.k.a. interactive interpreter) -* debugger (not complete, but you can set breakpoints, view and step) +* debugger (not complete, but you can set breakpoints, view and step) Menus ----- +IDLE has two window types, the Shell window and the Editor window. It is +possible to have multiple editor windows simultaneously. IDLE's +menus dynamically change based on which window is currently selected. Each menu +documented below indicates which window type it is associated with. Click on +the dotted line at the top of a menu to "tear it off": a separate window +containing the menu is created (for Unix and Windows only). -File menu -^^^^^^^^^ +File menu (Shell and Editor) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ New file - create a new file editing window + Create a new file editing window Open... - open an existing file + Open an existing file Open module... - open an existing module (searches sys.path) + Open an existing module (searches sys.path) + +Recent Files + Open a list of recent files Class browser - show classes and methods in current file + Show classes and methods in current file Path browser - show sys.path directories, modules, classes and methods + Show sys.path directories, modules, classes and methods .. index:: single: Class browser single: Path browser Save - save current window to the associated file (unsaved windows have a \* before and - after the window title) + Save current window to the associated file (unsaved windows have a + \* before and after the window title) Save As... - save current window to new file, which becomes the associated file + Save current window to new file, which becomes the associated file Save Copy As... - save current window to different file without changing the associated file + Save current window to different file without changing the associated file + +Print Window + Print the current window Close - close current window (asks to save if unsaved) + Close current window (asks to save if unsaved) Exit - close all windows and quit IDLE (asks to save if unsaved) + Close all windows and quit IDLE (asks to save if unsaved) -Edit menu -^^^^^^^^^ +Edit menu (Shell and Editor) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Undo - Undo last change to current window (max 1000 changes) + Undo last change to current window (a maximum of 1000 changes may be undone) Redo Redo last undone change to current window Cut - Copy selection into system-wide clipboard; then delete selection + Copy selection into system-wide clipboard; then delete the selection Copy Copy selection into system-wide clipboard @@ -108,11 +120,30 @@ Replace... Go to line Ask for a line number and show that line +Expand word + Expand the word you have typed to match another word in the same buffer; + repeat to get a different expansion + +Show call tip + After an unclosed parenthesis for a function, open a small window with + function parameter hints + +Show surrounding parens + Highlight the surrounding parenthesis + +Show Completions + Open a scroll window allowing selection keywords and attributes. See + Completions below. + + +Format menu (Editor window only) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Indent region - Shift selected lines right 4 spaces + Shift selected lines right by the indent width (default 4 spaces) Dedent region - Shift selected lines left 4 spaces + Shift selected lines left by the indent width (default 4 spaces) Comment out region Insert ## in front of selected lines @@ -121,67 +152,121 @@ Uncomment region Remove leading # or ## from selected lines Tabify region - Turns *leading* stretches of spaces into tabs + Turns *leading* stretches of spaces into tabs. (Note: We recommend using + 4 space blocks to indent Python code.) Untabify region - Turn *all* tabs into the right number of spaces + Turn *all* tabs into the correct number of spaces -Expand word - Expand the word you have typed to match another word in the same buffer; repeat - to get a different expansion +Toggle tabs + Open a dialog to switch between indenting with spaces and tabs. -Format Paragraph - Reformat the current blank-line-separated paragraph +New Indent Width + Open a dialog to change indent width. The accepted default by the Python + community is 4 spaces. -Import module - Import or reload the current module +Format Paragraph + Reformat the current blank-line-separated paragraph. All lines in the + paragraph will be formatted to less than 80 columns. -Run script - Execute the current file in the __main__ namespace +Strip trailing whitespace + Removes any space characters after the end of the last non-space character .. index:: single: Import module single: Run script -Windows menu -^^^^^^^^^^^^ +Run menu (Editor window only) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Zoom Height - toggles the window between normal size (24x80) and maximum height. +Python Shell + Open or wake up the Python Shell window -The rest of this menu lists the names of all open windows; select one to bring -it to the foreground (deiconifying it if necessary). +Check module + Check the syntax of the module currently open in the Editor window. If the + module has not been saved IDLE will prompt the user to save the code. + +Run module + Restart the shell to clean the environment, then execute the currently + open module. If the module has not been saved IDLE will prompt the user + to save the code. +Shell menu (Shell window only) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Debug menu -^^^^^^^^^^ +View Last Restart + Scroll the shell window to the last Shell restart -* in the Python Shell window only +Restart Shell + Restart the shell to clean the environment + +Debug menu (Shell window only) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Go to file/line Look around the insert point for a filename and line number, open the file, and show the line. Useful to view the source lines referenced in an - exception traceback. + exception traceback. Available in the context menu of the Shell window. -Debugger - Run commands in the shell under the debugger. +Debugger (toggle) + This feature is not complete and considered experimental. Run commands in + the shell under the debugger Stack viewer - Show the stack traceback of the last exception. + Show the stack traceback of the last exception Auto-open Stack Viewer - Open stack viewer on traceback. + Toggle automatically opening the stack viewer on unhandled exception .. index:: single: stack viewer single: debugger +Options menu (Shell and Editor) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Configure IDLE + Open a configuration dialog. Fonts, indentation, keybindings, and color + themes may be altered. Startup Preferences may be set, and additional + help sources can be specified. + +Code Context (toggle)(Editor Window only) + Open a pane at the top of the edit window which shows the block context + of the section of code which is scrolling off the top of the window. + +Windows menu (Shell and Editor) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Zoom Height + Toggles the window between normal size (40x80 initial setting) and maximum + height. The initial size is in the Configure IDLE dialog under the general + tab. + +The rest of this menu lists the names of all open windows; select one to bring +it to the foreground (deiconifying it if necessary). + +Help menu (Shell and Editor) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +About IDLE + Version, copyright, license, credits + +IDLE Help + Display a help file for IDLE detailing the menu options, basic editing and + navigation, and other tips. + +Python Docs + Access local Python documentation, if installed. Or will start a web browser + and open docs.python.org showing the latest Python documentation. -Edit context menu -^^^^^^^^^^^^^^^^^ +Additional help sources may be added here with the Configure IDLE dialog under +the General tab. -* Right-click in Edit window (Control-click on OS X) +Editor Window context menu +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* Right-click in Editor window (Control-click on OS X) Cut Copy selection into system-wide clipboard; then delete selection @@ -207,8 +292,8 @@ Clear Breakpoint single: breakpoints -Shell context menu -^^^^^^^^^^^^^^^^^^ +Shell Window context menu +^^^^^^^^^^^^^^^^^^^^^^^^^ * Right-click in Python Shell window (Control-click on OS X) @@ -225,19 +310,44 @@ Go to file/line Same as in Debug menu. -Basic editing and navigation ----------------------------- +Editing and navigation +---------------------- * :kbd:`Backspace` deletes to the left; :kbd:`Del` deletes to the right +* :kbd:`C-Backspace` delete word left; :kbd:`C-Del` delete word to the right + * Arrow keys and :kbd:`Page Up`/:kbd:`Page Down` to move around +* :kbd:`C-LeftArrow` and :kbd:`C-RightArrow` moves by words + * :kbd:`Home`/:kbd:`End` go to begin/end of line * :kbd:`C-Home`/:kbd:`C-End` go to begin/end of file -* Some :program:`Emacs` bindings may also work, including :kbd:`C-B`, - :kbd:`C-P`, :kbd:`C-A`, :kbd:`C-E`, :kbd:`C-D`, :kbd:`C-L` +* Some useful Emacs bindings are inherited from Tcl/Tk: + + * :kbd:`C-a` beginning of line + + * :kbd:`C-e` end of line + + * :kbd:`C-k` kill line (but doesn't put it in clipboard) + + * :kbd:`C-l` center window around the insertion point + + * :kbd:`C-b` go backwards one character without deleting (usually you can + also use the cursor key for this) + + * :kbd:`C-f` go forward one character without deleting (usually you can + also use the cursor key for this) + + * :kbd:`C-p` go up one line (usually you can also use the cursor key for + this) + + * :kbd:`C-d` delete next character + +Standard keybindings (like :kbd:`C-c` to copy and :kbd:`C-v` to paste) +may work. Keybindings are selected in the Configure IDLE dialog. Automatic indentation @@ -246,27 +356,76 @@ Automatic indentation After a block-opening statement, the next line is indented by 4 spaces (in the Python Shell window by one tab). After certain keywords (break, return etc.) the next line is dedented. In leading indentation, :kbd:`Backspace` deletes up -to 4 spaces if they are there. :kbd:`Tab` inserts 1-4 spaces (in the Python -Shell window one tab). See also the indent/dedent region commands in the edit -menu. - +to 4 spaces if they are there. :kbd:`Tab` inserts spaces (in the Python +Shell window one tab), number depends on Indent width. Currently tabs +are restricted to four spaces due to Tcl/Tk limitations. + +See also the indent/dedent region commands in the edit menu. + +Completions +^^^^^^^^^^^ + +Completions are supplied for functions, classes, and attributes of classes, +both built-in and user-defined. Completions are also provided for +filenames. + +The AutoCompleteWindow (ACW) will open after a predefined delay (default is +two seconds) after a '.' or (in a string) an os.sep is typed. If after one +of those characters (plus zero or more other characters) a tab is typed +the ACW will open immediately if a possible continuation is found. + +If there is only one possible completion for the characters entered, a +:kbd:`Tab` will supply that completion without opening the ACW. + +'Show Completions' will force open a completions window, by default the +:kbd:`C-space` will open a completions window. In an empty +string, this will contain the files in the current directory. On a +blank line, it will contain the built-in and user-defined functions and +classes in the current name spaces, plus any modules imported. If some +characters have been entered, the ACW will attempt to be more specific. + +If a string of characters is typed, the ACW selection will jump to the +entry most closely matching those characters. Entering a :kbd:`tab` will +cause the longest non-ambiguous match to be entered in the Editor window or +Shell. Two :kbd:`tab` in a row will supply the current ACW selection, as +will return or a double click. Cursor keys, Page Up/Down, mouse selection, +and the scroll wheel all operate on the ACW. + +"Hidden" attributes can be accessed by typing the beginning of hidden +name after a '.', e.g. '_'. This allows access to modules with +``__all__`` set, or to class-private attributes. + +Completions and the 'Expand Word' facility can save a lot of typing! + +Completions are currently limited to those in the namespaces. Names in +an Editor window which are not via ``__main__`` and :data:`sys.modules` will +not be found. Run the module once with your imports to correct this situation. +Note that IDLE itself places quite a few modules in sys.modules, so +much can be found by default, e.g. the re module. + +If you don't like the ACW popping up unbidden, simply make the delay +longer or disable the extension. Or another option is the delay could +be set to zero. Another alternative to preventing ACW popups is to +disable the call tips extension. Python Shell window ^^^^^^^^^^^^^^^^^^^ -* :kbd:`C-C` interrupts executing command +* :kbd:`C-c` interrupts executing command -* :kbd:`C-D` sends end-of-file; closes window if typed at a ``>>>`` prompt +* :kbd:`C-d` sends end-of-file; closes window if typed at a ``>>>`` prompt + (this is :kbd:`C-z` on Windows). -* :kbd:`Alt-p` retrieves previous command matching what you have typed +* :kbd:`Alt-/` (Expand word) is also useful to reduce typing -* :kbd:`Alt-n` retrieves next + Command history -* :kbd:`Return` while on any previous command retrieves that command + * :kbd:`Alt-p` retrieves previous command matching what you have typed. On + OS X use :kbd:`C-p`. -* :kbd:`Alt-/` (Expand word) is also useful here + * :kbd:`Alt-n` retrieves next. On OS X use :kbd:`C-n`. -.. index:: single: indentation + * :kbd:`Return` while on any previous command retrieves that command Syntax colors @@ -308,17 +467,17 @@ Startup Upon startup with the ``-s`` option, IDLE will execute the file referenced by the environment variables :envvar:`IDLESTARTUP` or :envvar:`PYTHONSTARTUP`. -Idle first checks for ``IDLESTARTUP``; if ``IDLESTARTUP`` is present the file -referenced is run. If ``IDLESTARTUP`` is not present, Idle checks for +IDLE first checks for ``IDLESTARTUP``; if ``IDLESTARTUP`` is present the file +referenced is run. If ``IDLESTARTUP`` is not present, IDLE checks for ``PYTHONSTARTUP``. Files referenced by these environment variables are -convenient places to store functions that are used frequently from the Idle +convenient places to store functions that are used frequently from the IDLE shell, or for executing import statements to import common modules. In addition, ``Tk`` also loads a startup file if it is present. Note that the Tk file is loaded unconditionally. This additional file is ``.Idle.py`` and is looked for in the user's home directory. Statements in this file will be executed in the Tk namespace, so this file is not useful for importing functions -to be used from Idle's Python shell. +to be used from IDLE's Python shell. Command line usage @@ -349,3 +508,45 @@ If there are arguments: the arguments are still available in ``sys.argv``. +Additional help sources +----------------------- + +IDLE includes a help menu entry called "Python Docs" that will open the +extensive sources of help, including tutorials, available at docs.python.org. +Selected URLs can be added or removed from the help menu at any time using the +Configure IDLE dialog. See the IDLE help option in the help menu of IDLE for +more information. + + +Other preferences +----------------- + +The font preferences, highlighting, keys, and general preferences can be +changed via the Configure IDLE menu option. Be sure to note that +keys can be user defined, IDLE ships with four built in key sets. In +addition a user can create a custom key set in the Configure IDLE dialog +under the keys tab. + +Extensions +---------- + +IDLE contains an extension facility. See the beginning of +config-extensions.def in the idlelib directory for further information. The +default extensions are currently: + +* FormatParagraph + +* AutoExpand + +* ZoomHeight + +* ScriptBinding + +* CallTips + +* ParenMatch + +* AutoComplete + +* CodeContext + diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 01236fb..fa736fe 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -69,17 +69,25 @@ There's also a subclass for secure connections: This is a subclass derived from :class:`IMAP4` that connects over an SSL encrypted socket (to use this class you need a socket module that was compiled with SSL support). If *host* is not specified, ``''`` (the local host) is used. - If *port* is omitted, the standard IMAP4-over-SSL port (993) is used. *keyfile* - and *certfile* are also optional - they can contain a PEM formatted private key - and certificate chain file for the SSL connection. *ssl_context* parameter is a - :class:`ssl.SSLContext` object which allows bundling SSL configuration - options, certificates and private keys into a single (potentially long-lived) - structure. Note that the *keyfile*/*certfile* parameters are mutually exclusive with *ssl_context*, - a :class:`ValueError` is raised if *keyfile*/*certfile* is provided along with *ssl_context*. + If *port* is omitted, the standard IMAP4-over-SSL port (993) is used. + *ssl_context* is a :class:`ssl.SSLContext` object which allows bundling + SSL configuration options, certificates and private keys into a single + (potentially long-lived) structure. Please read :ref:`ssl-security` for + best practices. + + *keyfile* and *certfile* are a legacy alternative to *ssl_context* - they + can point to PEM-formatted private key and certificate chain files for + the SSL connection. Note that the *keyfile*/*certfile* parameters are + mutually exclusive with *ssl_context*, a :class:`ValueError` is raised + if *keyfile*/*certfile* is provided along with *ssl_context*. .. versionchanged:: 3.3 *ssl_context* parameter added. + .. versionchanged:: 3.4 + The class now supports hostname check with + :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see + :data:`ssl.HAS_SNI`). The second subclass allows for connections created by a child process: @@ -433,10 +441,16 @@ An :class:`IMAP4` instance has the following methods: Send a ``STARTTLS`` command. The *ssl_context* argument is optional and should be a :class:`ssl.SSLContext` object. This will enable - encryption on the IMAP connection. + encryption on the IMAP connection. Please read :ref:`ssl-security` for + best practices. .. versionadded:: 3.2 + .. versionchanged:: 3.4 + The method now supports hostname check with + :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see + :data:`ssl.HAS_SNI`). + .. method:: IMAP4.status(mailbox, names) diff --git a/Doc/library/imp.rst b/Doc/library/imp.rst index 364d81e..c2dbdc5 100644 --- a/Doc/library/imp.rst +++ b/Doc/library/imp.rst @@ -3,7 +3,10 @@ .. module:: imp :synopsis: Access the implementation of the import statement. + :deprecated: +.. deprecated:: 3.4 + The :mod:`imp` package is pending deprecation in favor of :mod:`importlib`. .. index:: statement: import @@ -11,10 +14,6 @@ This module provides an interface to the mechanisms used to implement the :keyword:`import` statement. It defines the following constants and functions: -.. note:: - New programs should use :mod:`importlib` rather than this module. - - .. function:: get_magic() .. index:: pair: file; byte-code @@ -22,6 +21,9 @@ This module provides an interface to the mechanisms used to implement the Return the magic string value used to recognize byte-compiled code files (:file:`.pyc` files). (This value may be different for each Python version.) + .. deprecated:: 3.4 + Use :attr:`importlib.util.MAGIC_NUMBER` instead. + .. function:: get_suffixes() @@ -77,7 +79,9 @@ This module provides an interface to the mechanisms used to implement the When *P* itself has a dotted name, apply this recipe recursively. .. deprecated:: 3.3 - Use :func:`importlib.find_loader` instead. + Use :func:`importlib.util.find_spec` instead unless Python 3.3 + compatibility is required, in which case use + :func:`importlib.find_loader`. .. function:: load_module(name, file, pathname, description) @@ -101,8 +105,12 @@ This module provides an interface to the mechanisms used to implement the using a :keyword:`try` ... :keyword:`finally` statement. .. deprecated:: 3.3 - Unneeded as loaders should be used to load modules and - :func:`find_module` is deprecated. + If previously used in conjunction with :func:`imp.find_module` then + consider using :func:`importlib.import_module`, otherwise use the loader + returned by the replacement you chose for :func:`imp.find_module`. If you + called :func:`imp.load_module` and related functions directly then use the + classes in :mod:`importlib.machinery`, e.g. + ``importlib.machinery.SourceFileLoader(name, path).load_module()``. .. function:: new_module(name) @@ -110,6 +118,9 @@ This module provides an interface to the mechanisms used to implement the Return a new empty module object called *name*. This object is *not* inserted in ``sys.modules``. + .. deprecated:: 3.4 + Use :class:`types.ModuleType` instead. + .. function:: reload(module) @@ -176,6 +187,9 @@ This module provides an interface to the mechanisms used to implement the Relies on both ``__name__`` and ``__loader__`` being defined on the module being reloaded instead of just ``__name__``. + .. deprecated:: 3.4 + Use :func:`importlib.reload` instead. + The following functions are conveniences for handling :pep:`3147` byte-compiled file paths. @@ -201,6 +215,9 @@ file paths. If :attr:`sys.implementation.cache_tag` is ``None``, then :exc:`NotImplementedError` is raised. + .. deprecated:: 3.4 + Use :func:`importlib.util.cache_from_source` instead. + .. function:: source_from_cache(path) @@ -216,14 +233,17 @@ file paths. Raise :exc:`NotImplementedError` when :attr:`sys.implementation.cache_tag` is not defined. + .. deprecated:: 3.4 + Use :func:`importlib.util.source_from_cache` instead. + .. function:: get_tag() Return the :pep:`3147` magic tag string matching this version of Python's magic number, as returned by :func:`get_magic`. - .. note:: - You may use :attr:`sys.implementation.cache_tag` directly starting + .. deprecated:: 3.4 + Use :attr:`sys.implementation.cache_tag` directly starting in Python 3.3. @@ -246,10 +266,12 @@ that circular imports work without any deadlocks. exception is made for circular imports, which by construction have to expose an incomplete module object at some point. -.. versionchanged:: 3.3 - The locking scheme has changed to per-module locks for - the most part. A global import lock is kept for some critical tasks, - such as initializing the per-module locks. + .. versionchanged:: 3.3 + The locking scheme has changed to per-module locks for + the most part. A global import lock is kept for some critical tasks, + such as initializing the per-module locks. + + .. deprecated:: 3.4 .. function:: acquire_lock() @@ -264,10 +286,12 @@ that circular imports work without any deadlocks. On platforms without threads, this function does nothing. -.. versionchanged:: 3.3 - The locking scheme has changed to per-module locks for - the most part. A global import lock is kept for some critical tasks, - such as initializing the per-module locks. + .. versionchanged:: 3.3 + The locking scheme has changed to per-module locks for + the most part. A global import lock is kept for some critical tasks, + such as initializing the per-module locks. + + .. deprecated:: 3.4 .. function:: release_lock() @@ -275,10 +299,12 @@ that circular imports work without any deadlocks. Release the interpreter's global import lock. On platforms without threads, this function does nothing. -.. versionchanged:: 3.3 - The locking scheme has changed to per-module locks for - the most part. A global import lock is kept for some critical tasks, - such as initializing the per-module locks. + .. versionchanged:: 3.3 + The locking scheme has changed to per-module locks for + the most part. A global import lock is kept for some critical tasks, + such as initializing the per-module locks. + + .. deprecated:: 3.4 The following constants with integer values, defined in this module, are used @@ -345,6 +371,9 @@ to indicate the search result of :func:`find_module`. ``None`` is inserted into ``sys.path_importer_cache`` instead of an instance of :class:`NullImporter`. + .. deprecated:: 3.4 + Insert ``None`` into ``sys.path_importer_cache`` instead. + .. _examples-imp: diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 5f740a2..09a5d71 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -95,7 +95,7 @@ Functions Find the loader for a module, optionally within the specified *path*. If the module is in :attr:`sys.modules`, then ``sys.modules[name].__loader__`` is - returned (unless the loader would be ``None``, in which case + returned (unless the loader would be ``None`` or is not set, in which case :exc:`ValueError` is raised). Otherwise a search using :attr:`sys.meta_path` is done. ``None`` is returned if no loader is found. @@ -104,6 +104,15 @@ Functions will need to import all parent packages of the submodule and use the correct argument to *path*. + .. versionadded:: 3.3 + + .. versionchanged:: 3.4 + If ``__loader__`` is not set, raise :exc:`ValueError`, just like when the + attribute is set to ``None``. + + .. deprecated:: 3.4 + Use :func:`importlib.util.find_spec` instead. + .. function:: invalidate_caches() Invalidate the internal caches of finders stored at @@ -114,6 +123,74 @@ Functions .. versionadded:: 3.3 +.. function:: reload(module) + + Reload a previously imported *module*. The argument must be a module object, + so it must have been successfully imported before. This is useful if you + have edited the module source file using an external editor and want to try + out the new version without leaving the Python interpreter. The return value + is the module object (which can be different if re-importing causes a + different object to be placed in :data:`sys.modules`). + + When :func:`reload` is executed: + + * Python module's code is recompiled and the module-level code re-executed, + defining a new set of objects which are bound to names in the module's + dictionary by reusing the :term:`loader` which originally loaded the + module. The ``init`` function of extension modules is not called a second + time. + + * As with all other objects in Python the old objects are only reclaimed + after their reference counts drop to zero. + + * The names in the module namespace are updated to point to any new or + changed objects. + + * Other references to the old objects (such as names external to the module) are + not rebound to refer to the new objects and must be updated in each namespace + where they occur if that is desired. + + There are a number of other caveats: + + If a module is syntactically correct but its initialization fails, the first + :keyword:`import` statement for it does not bind its name locally, but does + store a (partially initialized) module object in ``sys.modules``. To reload + the module you must first :keyword:`import` it again (this will bind the name + to the partially initialized module object) before you can :func:`reload` it. + + When a module is reloaded, its dictionary (containing the module's global + variables) is retained. Redefinitions of names will override the old + definitions, so this is generally not a problem. If the new version of a + module does not define a name that was defined by the old version, the old + definition remains. This feature can be used to the module's advantage if it + maintains a global table or cache of objects --- with a :keyword:`try` + statement it can test for the table's presence and skip its initialization if + desired:: + + try: + cache + except NameError: + cache = {} + + It is legal though generally not very useful to reload built-in or + dynamically loaded modules (this is not true for e.g. :mod:`sys`, + :mod:`__main__`, :mod:`builtins` and other key modules where reloading is + frowned upon). In many cases, however, extension modules are not designed to + be initialized more than once, and may fail in arbitrary ways when reloaded. + + If a module imports objects from another module using :keyword:`from` ... + :keyword:`import` ..., calling :func:`reload` for the other module does not + redefine the objects imported from it --- one way around this is to + re-execute the :keyword:`from` statement, another is to use :keyword:`import` + and qualified names (*module.name*) instead. + + If a module instantiates instances of a class, reloading the module that + defines the class does not affect the method definitions of the instances --- + they continue to use the old class definition. The same is true for derived + classes. + + .. versionadded:: 3.4 + :mod:`importlib.abc` -- Abstract base classes related to import --------------------------------------------------------------- @@ -137,8 +214,6 @@ ABC hierarchy:: +-- ExecutionLoader --+ +-- FileLoader +-- SourceLoader - +-- PyLoader (deprecated) - +-- PyPycLoader (deprecated) .. class:: Finder @@ -154,6 +229,10 @@ ABC hierarchy:: module. Originally specified in :pep:`302`, this method was meant for use in :data:`sys.meta_path` and in the path-based import subsystem. + .. versionchanged:: 3.4 + Returns ``None`` when called instead of raising + :exc:`NotImplementedError`. + .. class:: MetaPathFinder @@ -162,20 +241,46 @@ ABC hierarchy:: .. versionadded:: 3.3 + .. method:: find_spec(fullname, path, target=None) + + An abstract method for finding a :term:`spec <module spec>` for + the specified module. If this is a top-level import, *path* will + be ``None``. Otherwise, this is a search for a subpackage or + module and *path* will be the value of :attr:`__path__` from the + parent package. If a spec cannot be found, ``None`` is returned. + When passed in, ``target`` is a module object that the finder may + use to make a more educated about what spec to return. + + .. versionadded:: 3.4 + .. method:: find_module(fullname, path) - An abstract method for finding a :term:`loader` for the specified + A legacy method for finding a :term:`loader` for the specified module. If this is a top-level import, *path* will be ``None``. Otherwise, this is a search for a subpackage or module and *path* will be the value of :attr:`__path__` from the parent package. If a loader cannot be found, ``None`` is returned. + If :meth:`find_spec` is defined, backwards-compatible functionality is + provided. + + .. versionchanged:: 3.4 + Returns ``None`` when called instead of raising + :exc:`NotImplementedError`. Can use :meth:`find_spec` to provide + functionality. + + .. deprecated:: 3.4 + Use :meth:`find_spec` instead. + .. method:: invalidate_caches() An optional method which, when called, should invalidate any internal cache used by the finder. Used by :func:`importlib.invalidate_caches` when invalidating the caches of all finders on :data:`sys.meta_path`. + .. versionchanged:: 3.4 + Returns ``None`` when called instead of ``NotImplemented``. + .. class:: PathEntryFinder @@ -183,27 +288,51 @@ ABC hierarchy:: it bears some similarities to :class:`MetaPathFinder`, ``PathEntryFinder`` is meant for use only within the path-based import subsystem provided by :class:`PathFinder`. This ABC is a subclass of :class:`Finder` for - compatibility. + compatibility reasons only. .. versionadded:: 3.3 + .. method:: find_spec(fullname, target=None) + + An abstract method for finding a :term:`spec <module spec>` for + the specified module. The finder will search for the module only + within the :term:`path entry` to which it is assigned. If a spec + cannot be found, ``None`` is returned. When passed in, ``target`` + is a module object that the finder may use to make a more educated + about what spec to return. + + .. versionadded:: 3.4 + .. method:: find_loader(fullname) - An abstract method for finding a :term:`loader` for the specified + A legacy method for finding a :term:`loader` for the specified module. Returns a 2-tuple of ``(loader, portion)`` where ``portion`` is a sequence of file system locations contributing to part of a namespace package. The loader may be ``None`` while specifying ``portion`` to signify the contribution of the file system locations to a namespace package. An empty list can be used for ``portion`` to signify the loader - is not part of a package. If ``loader`` is ``None`` and ``portion`` is - the empty list then no loader or location for a namespace package were - found (i.e. failure to find anything for the module). + is not part of a namespace package. If ``loader`` is ``None`` and + ``portion`` is the empty list then no loader or location for a namespace + package were found (i.e. failure to find anything for the module). + + If :meth:`find_spec` is defined then backwards-compatible functionality is + provided. + + .. versionchanged:: 3.4 + Returns ``(None, [])`` instead of raising :exc:`NotImplementedError`. + Uses :meth:`find_spec` when available to provide functionality. + + .. deprecated:: 3.4 + Use :meth:`find_spec` instead. .. method:: find_module(fullname) A concrete implementation of :meth:`Finder.find_module` which is equivalent to ``self.find_loader(fullname)[0]``. + .. deprecated:: 3.4 + Use :meth:`find_spec` instead. + .. method:: invalidate_caches() An optional method which, when called, should invalidate any internal @@ -216,9 +345,26 @@ ABC hierarchy:: An abstract base class for a :term:`loader`. See :pep:`302` for the exact definition for a loader. + .. method:: create_module(spec) + + An optional method that returns the module object to use when + importing a module. create_module() may also return ``None``, + indicating that the default module creation should take place + instead. + + .. versionadded:: 3.4 + + .. method:: exec_module(module) + + An abstract method that executes the module in its own namespace + when a module is imported or reloaded. The module should already + be initialized when exec_module() is called. + + .. versionadded:: 3.4 + .. method:: load_module(fullname) - An abstract method for loading a module. If the module cannot be + A legacy method for loading a module. If the module cannot be loaded, :exc:`ImportError` is raised, otherwise the loaded module is returned. @@ -229,12 +375,11 @@ ABC hierarchy:: from the import. If the loader inserted a module and the load fails, it must be removed by the loader from :data:`sys.modules`; modules already in :data:`sys.modules` before the loader began execution should be left - alone. The :func:`importlib.util.module_for_loader` decorator handles - all of these details. + alone (see :func:`importlib.util.module_for_loader`). The loader should set several attributes on the module. (Note that some of these attributes can change when a module is - reloaded.) + reloaded): - :attr:`__name__` The name of the module. @@ -254,20 +399,42 @@ ABC hierarchy:: - :attr:`__package__` The parent package for the module/package. If the module is top-level then it has a value of the empty string. The - :func:`importlib.util.set_package` decorator can handle the details - for :attr:`__package__`. + :func:`importlib.util.module_for_loader` decorator can handle the + details for :attr:`__package__`. - :attr:`__loader__` - The loader used to load the module. - (This is not set by the built-in import machinery, - but it should be set whenever a :term:`loader` is used.) + The loader used to load the module. The + :func:`importlib.util.module_for_loader` decorator can handle the + details for :attr:`__package__`. + + When :meth:`exec_module` is available then backwards-compatible + functionality is provided. + + .. versionchanged:: 3.4 + Raise :exc:`ImportError` when called instead of + :exc:`NotImplementedError`. Functionality provided when + :meth:`exec_module` is available. + + .. deprecated:: 3.4 + The recommended API for loading a module is :meth:`exec_module` + (and optionally :meth:`create_module`). Loaders should implement + it instead of load_module(). The import machinery takes care of + all the other responsibilities of load_module() when exec_module() + is implemented. .. method:: module_repr(module) - An abstract method which when implemented calculates and returns the - given module's repr, as a string. + A legacy method which when implemented calculates and returns the + given module's repr, as a string. The module type's default repr() will + use the result of this method as appropriate. + + .. versionadded:: 3.3 - .. versionadded: 3.3 + .. versionchanged:: 3.4 + Made optional instead of an abstractmethod. + + .. deprecated:: 3.4 + The import machinery now takes care of this automatically. .. class:: ResourceLoader @@ -282,10 +449,13 @@ ABC hierarchy:: Loaders that have a file-like storage back-end that allows storing arbitrary data can implement this abstract method to give direct access - to the data stored. :exc:`IOError` is to be raised if the *path* cannot + to the data stored. :exc:`OSError` is to be raised if the *path* cannot be found. The *path* is expected to be constructed using a module's :attr:`__file__` attribute or an item from a package's :attr:`__path__`. + .. versionchanged:: 3.4 + Raises :exc:`OSError` instead of :exc:`NotImplementedError`. + .. class:: InspectLoader @@ -294,14 +464,21 @@ ABC hierarchy:: .. method:: get_code(fullname) - An abstract method to return the :class:`code` object for a module. - ``None`` is returned if the module does not have a code object - (e.g. built-in module). :exc:`ImportError` is raised if loader cannot - find the requested module. + Return the code object for a module, or ``None`` if the module does not + have a code object (as would be the case, for example, for a built-in + module). Raise an :exc:`ImportError` if loader cannot find the + requested module. + + .. note:: + While the method has a default implementation, it is suggested that + it be overridden if possible for performance. .. index:: single: universal newlines; importlib.abc.InspectLoader.get_source method + .. versionchanged:: 3.4 + No longer abstract and a concrete implementation is provided. + .. method:: get_source(fullname) An abstract method to return the source of a module. It is returned as @@ -310,12 +487,42 @@ ABC hierarchy:: if no source is available (e.g. a built-in module). Raises :exc:`ImportError` if the loader cannot find the module specified. + .. versionchanged:: 3.4 + Raises :exc:`ImportError` instead of :exc:`NotImplementedError`. + .. method:: is_package(fullname) An abstract method to return a true value if the module is a package, a false value otherwise. :exc:`ImportError` is raised if the :term:`loader` cannot find the module. + .. versionchanged:: 3.4 + Raises :exc:`ImportError` instead of :exc:`NotImplementedError`. + + .. method:: source_to_code(data, path='<string>') + + Create a code object from Python source. + + The *data* argument can be whatever the :func:`compile` function + supports (i.e. string or bytes). The *path* argument should be + the "path" to where the source code originated from, which can be an + abstract concept (e.g. location in a zip file). + + .. versionadded:: 3.4 + + .. method:: exec_module(module) + + Implementation of :meth:`Loader.exec_module`. + + .. versionadded:: 3.4 + + .. method:: load_module(fullname) + + Implementation of :meth:`Loader.load_module`. + + .. deprecated:: 3.4 + use :meth:`exec_module` instead. + .. class:: ExecutionLoader @@ -333,6 +540,9 @@ ABC hierarchy:: the source file, regardless of whether a bytecode was used to load the module. + .. versionchanged:: 3.4 + Raises :exc:`ImportError` instead of :exc:`NotImplementedError`. + .. class:: FileLoader(fullname, path) @@ -357,13 +567,16 @@ ABC hierarchy:: Calls super's ``load_module()``. + .. deprecated:: 3.4 + Use :meth:`Loader.exec_module` instead. + .. method:: get_filename(fullname) Returns :attr:`path`. .. method:: get_data(path) - Returns the open, binary file for *path*. + Reads *path* as a binary file and returns the bytes from it. .. class:: SourceLoader @@ -378,7 +591,8 @@ ABC hierarchy:: loading is not supported. The abstract methods defined by this class are to add optional bytecode - file support. Not implementing these optional methods causes the loader to + file support. Not implementing these optional methods (or causing them to + raise :exc:`NotImplementedError`) causes the loader to only work with source code. Implementing the methods allows the loader to work with source *and* bytecode files; it does not allow for *sourceless* loading where only bytecode is provided. Bytecode files are an @@ -395,10 +609,13 @@ ABC hierarchy:: - ``'size'`` (optional): the size in bytes of the source code. Any other keys in the dictionary are ignored, to allow for future - extensions. + extensions. If the path cannot be handled, :exc:`OSError` is raised. .. versionadded:: 3.3 + .. versionchanged:: 3.4 + Raise :exc:`OSError` instead of :exc:`NotImplementedError`. + .. method:: path_mtime(path) Optional abstract method which returns the modification time for the @@ -407,7 +624,10 @@ ABC hierarchy:: .. deprecated:: 3.3 This method is deprecated in favour of :meth:`path_stats`. You don't have to implement it, but it is still available for compatibility - purposes. + purposes. Raise :exc:`OSError` if the path cannot be handled. + + .. versionchanged:: 3.4 + Raise :exc:`OSError` instead of :exc:`NotImplementedError`. .. method:: set_data(path, data) @@ -419,13 +639,25 @@ ABC hierarchy:: (:attr:`errno.EACCES`/:exc:`PermissionError`), do not propagate the exception. + .. versionchanged:: 3.4 + No longer raises :exc:`NotImplementedError` when called. + .. method:: get_code(fullname) Concrete implementation of :meth:`InspectLoader.get_code`. + .. method:: exec_module(module) + + Concrete implementation of :meth:`Loader.exec_module`. + + .. versionadded:: 3.4 + .. method:: load_module(fullname) - Concrete implementation of :meth:`Loader.load_module`. + Concrete implementation of :meth:`Loader.load_module`. + + .. deprecated:: 3.4 + Use :meth:`exec_module` instead. .. method:: get_source(fullname) @@ -440,142 +672,6 @@ ABC hierarchy:: itself does not end in ``__init__``. -.. class:: PyLoader - - An abstract base class inheriting from - :class:`ExecutionLoader` and - :class:`ResourceLoader` designed to ease the loading of - Python source modules (bytecode is not handled; see - :class:`SourceLoader` for a source/bytecode ABC). A subclass - implementing this ABC will only need to worry about exposing how the source - code is stored; all other details for loading Python source code will be - handled by the concrete implementations of key methods. - - .. deprecated:: 3.2 - This class has been deprecated in favor of :class:`SourceLoader` and is - slated for removal in Python 3.4. See below for how to create a - subclass that is compatible with Python 3.1 onwards. - - If compatibility with Python 3.1 is required, then use the following idiom - to implement a subclass that will work with Python 3.1 onwards (make sure - to implement :meth:`ExecutionLoader.get_filename`):: - - try: - from importlib.abc import SourceLoader - except ImportError: - from importlib.abc import PyLoader as SourceLoader - - - class CustomLoader(SourceLoader): - def get_filename(self, fullname): - """Return the path to the source file.""" - # Implement ... - - def source_path(self, fullname): - """Implement source_path in terms of get_filename.""" - try: - return self.get_filename(fullname) - except ImportError: - return None - - def is_package(self, fullname): - """Implement is_package by looking for an __init__ file - name as returned by get_filename.""" - filename = os.path.basename(self.get_filename(fullname)) - return os.path.splitext(filename)[0] == '__init__' - - - .. method:: source_path(fullname) - - An abstract method that returns the path to the source code for a - module. Should return ``None`` if there is no source code. - Raises :exc:`ImportError` if the loader knows it cannot handle the - module. - - .. method:: get_filename(fullname) - - A concrete implementation of - :meth:`importlib.abc.ExecutionLoader.get_filename` that - relies on :meth:`source_path`. If :meth:`source_path` returns - ``None``, then :exc:`ImportError` is raised. - - .. method:: load_module(fullname) - - A concrete implementation of :meth:`importlib.abc.Loader.load_module` - that loads Python source code. All needed information comes from the - abstract methods required by this ABC. The only pertinent assumption - made by this method is that when loading a package - :attr:`__path__` is set to ``[os.path.dirname(__file__)]``. - - .. method:: get_code(fullname) - - A concrete implementation of - :meth:`importlib.abc.InspectLoader.get_code` that creates code objects - from Python source code, by requesting the source code (using - :meth:`source_path` and :meth:`get_data`) and compiling it with the - built-in :func:`compile` function. - - .. method:: get_source(fullname) - - A concrete implementation of - :meth:`importlib.abc.InspectLoader.get_source`. Uses - :meth:`importlib.abc.ResourceLoader.get_data` and :meth:`source_path` - to get the source code. It tries to guess the source encoding using - :func:`tokenize.detect_encoding`. - - -.. class:: PyPycLoader - - An abstract base class inheriting from :class:`PyLoader`. - This ABC is meant to help in creating loaders that support both Python - source and bytecode. - - .. deprecated:: 3.2 - This class has been deprecated in favor of :class:`SourceLoader` and to - properly support :pep:`3147`. If compatibility is required with - Python 3.1, implement both :class:`SourceLoader` and :class:`PyLoader`; - instructions on how to do so are included in the documentation for - :class:`PyLoader`. Do note that this solution will not support - sourceless/bytecode-only loading; only source *and* bytecode loading. - - .. versionchanged:: 3.3 - Updated to parse (but not use) the new source size field in bytecode - files when reading and to write out the field properly when writing. - - .. method:: source_mtime(fullname) - - An abstract method which returns the modification time for the source - code of the specified module. The modification time should be an - integer. If there is no source code, return ``None``. If the - module cannot be found then :exc:`ImportError` is raised. - - .. method:: bytecode_path(fullname) - - An abstract method which returns the path to the bytecode for the - specified module, if it exists. It returns ``None`` - if no bytecode exists (yet). - Raises :exc:`ImportError` if the loader knows it cannot handle the - module. - - .. method:: get_filename(fullname) - - A concrete implementation of - :meth:`ExecutionLoader.get_filename` that relies on - :meth:`PyLoader.source_path` and :meth:`bytecode_path`. - If :meth:`source_path` returns a path, then that value is returned. - Else if :meth:`bytecode_path` returns a path, that path will be - returned. If a path is not available from both methods, - :exc:`ImportError` is raised. - - .. method:: write_bytecode(fullname, bytecode) - - An abstract method which has the loader write *bytecode* for future - use. If the bytecode is written, return ``True``. Return - ``False`` if the bytecode could not be written. This method - should not be called if :data:`sys.dont_write_bytecode` is true. - The *bytecode* argument should be a bytes string or bytes array. - - :mod:`importlib.machinery` -- Importers and path hooks ------------------------------------------------------ @@ -642,6 +738,10 @@ find and load modules. Only class methods are defined by this class to alleviate the need for instantiation. + .. note:: + Due to limitations in the extension module C-API, for now + BuiltinImporter does not implement :meth:`Loader.exec_module`. + .. class:: FrozenImporter @@ -672,24 +772,37 @@ find and load modules. Only class methods are defined by this class to alleviate the need for instantiation. + .. classmethod:: find_spec(fullname, path=None, target=None) + + Class method that attempts to find a :term:`spec <module spec>` + for the module specified by *fullname* on :data:`sys.path` or, if + defined, on *path*. For each path entry that is searched, + :data:`sys.path_importer_cache` is checked. If a non-false object + is found then it is used as the :term:`path entry finder` to look + for the module being searched for. If no entry is found in + :data:`sys.path_importer_cache`, then :data:`sys.path_hooks` is + searched for a finder for the path entry and, if found, is stored + in :data:`sys.path_importer_cache` along with being queried about + the module. If no finder is ever found then ``None`` is both + stored in the cache and returned. + + .. versionadded:: 3.4 + .. classmethod:: find_module(fullname, path=None) - Class method that attempts to find a :term:`loader` for the module - specified by *fullname* on :data:`sys.path` or, if defined, on - *path*. For each path entry that is searched, - :data:`sys.path_importer_cache` is checked. If a non-false object is - found then it is used as the :term:`path entry finder` to look for the - module being searched for. If no entry is found in - :data:`sys.path_importer_cache`, then :data:`sys.path_hooks` is - searched for a finder for the path entry and, if found, is stored in - :data:`sys.path_importer_cache` along with being queried about the - module. If no finder is ever found then ``None`` is both stored in - the cache and returned. + A legacy wrapper around :meth:`find_spec`. + + .. deprecated:: 3.4 + Use :meth:`find_spec` instead. .. classmethod:: invalidate_caches() - Calls :meth:`importlib.abc.PathEntryFinder.invalidate_caches` on all - finders stored in :attr:`sys.path_importer_cache`. + Calls :meth:`importlib.abc.PathEntryFinder.invalidate_caches` on all + finders stored in :attr:`sys.path_importer_cache`. + + .. versionchanged:: 3.4 + Calls objects in :data:`sys.path_hooks` with the current working + directory for ``''`` (i.e. the empty string). .. class:: FileFinder(path, \*loader_details) @@ -721,6 +834,12 @@ find and load modules. The path the finder will search in. + .. method:: find_spec(fullname, target=None) + + Attempt to find the spec to handle *fullname* within :attr:`path`. + + .. versionadded:: 3.4 + .. method:: find_loader(fullname) Attempt to find the loader to handle *fullname* within :attr:`path`. @@ -768,6 +887,11 @@ find and load modules. Concrete implementation of :meth:`importlib.abc.SourceLoader.set_data`. + .. method:: load_module(name=None) + + Concrete implementation of :meth:`importlib.abc.Loader.load_module` where + specifying the name of the module to load is optional. + .. class:: SourcelessFileLoader(fullname, path) @@ -802,10 +926,15 @@ find and load modules. Returns ``None`` as bytecode files have no source when this loader is used. + .. method:: load_module(name=None) + + Concrete implementation of :meth:`importlib.abc.Loader.load_module` where + specifying the name of the module to load is optional. + .. class:: ExtensionFileLoader(fullname, path) - A concrete implementation of :class:`importlib.abc.InspectLoader` for + A concrete implementation of :class:`importlib.abc.ExecutionLoader` for extension modules. The *fullname* argument specifies the name of the module the loader is to @@ -821,11 +950,15 @@ find and load modules. Path to the extension module. - .. method:: load_module(fullname) + .. method:: load_module(name=None) Loads the extension module if and only if *fullname* is the same as :attr:`name` or is ``None``. + .. note:: + Due to limitations in the extension module C-API, for now + ExtensionFileLoader does not implement :meth:`Loader.exec_module`. + .. method:: is_package(fullname) Returns ``True`` if the file path points to a package's ``__init__`` @@ -839,6 +972,70 @@ find and load modules. Returns ``None`` as extension modules do not have source code. + .. method:: get_filename(fullname) + + Returns :attr:`path`. + + .. versionadded:: 3.4 + + +.. class:: ModuleSpec(name, loader, *, origin=None, loader_state=None, is_package=None) + + A specification for a module's import-system-related state. + + .. versionadded:: 3.4 + + .. attribute:: name + + (``__name__``) + + A string for the fully-qualified name of the module. + + .. attribute:: loader + + (``__loader__``) + + The loader to use for loading. For namespace packages this should be + set to None. + + .. attribute:: origin + + (``__file__``) + + Name of the place from which the module is loaded, e.g. "builtin" for + built-in modules and the filename for modules loaded from source. + Normally "origin" should be set, but it may be None (the default) + which indicates it is unspecified. + + .. attribute:: submodule_search_locations + + (``__path__``) + + List of strings for where to find submodules, if a package (None + otherwise). + + .. attribute:: loader_state + + Container of extra module-specific data for use during loading (or + None). + + .. attribute:: cached + + (``__cached__``) + + String for where the compiled module should be stored (or None). + + .. attribute:: parent + + (``__package__``) + + (Read-only) Fully-qualified name of the package to which the module + belongs as a submodule (or None). + + .. attribute:: has_location + + Boolean indicating whether or not the module's "origin" + attribute refers to a loadable location. :mod:`importlib.util` -- Utility code for importers --------------------------------------------------- @@ -849,6 +1046,51 @@ find and load modules. This module contains the various objects that help in the construction of an :term:`importer`. +.. attribute:: MAGIC_NUMBER + + The bytes which represent the bytecode version number. If you need help with + loading/writing bytecode then consider :class:`importlib.abc.SourceLoader`. + + .. versionadded:: 3.4 + +.. function:: cache_from_source(path, debug_override=None) + + Return the :pep:`3147` path to the byte-compiled file associated with the + source *path*. For example, if *path* is ``/foo/bar/baz.py`` the return + value would be ``/foo/bar/__pycache__/baz.cpython-32.pyc`` for Python 3.2. + The ``cpython-32`` string comes from the current magic tag (see + :func:`get_tag`; if :attr:`sys.implementation.cache_tag` is not defined then + :exc:`NotImplementedError` will be raised). The returned path will end in + ``.pyc`` when ``__debug__`` is ``True`` or ``.pyo`` for an optimized Python + (i.e. ``__debug__`` is ``False``). By passing in ``True`` or ``False`` for + *debug_override* you can override the system's value for ``__debug__`` for + extension selection. + + *path* need not exist. + + .. versionadded:: 3.4 + + +.. function:: source_from_cache(path) + + Given the *path* to a :pep:`3147` file name, return the associated source code + file path. For example, if *path* is + ``/foo/bar/__pycache__/baz.cpython-32.pyc`` the returned path would be + ``/foo/bar/baz.py``. *path* need not exist, however if it does not conform + to :pep:`3147` format, a ``ValueError`` is raised. If + :attr:`sys.implementation.cache_tag` is not defined, + :exc:`NotImplementedError` is raised. + + .. versionadded:: 3.4 + +.. function:: decode_source(source_bytes) + + Decode the given bytes representing source code and return it as a string + with universal newlines (as required by + :meth:`importlib.abc.InspectLoader.get_source`). + + .. versionadded:: 3.4 + .. function:: resolve_name(name, package) Resolve a relative module name to an absolute one. @@ -865,9 +1107,25 @@ an :term:`importer`. .. versionadded:: 3.3 +.. function:: find_spec(name, package=None) + + Find the :term:`spec <module spec>` for a module, optionally relative to + the specified **package** name. If the module is in :attr:`sys.modules`, + then ``sys.modules[name].__spec__`` is returned (unless the spec would be + ``None`` or is not set, in which case :exc:`ValueError` is raised). + Otherwise a search using :attr:`sys.meta_path` is done. ``None`` is + returned if no spec is found. + + If **name** is for a submodule (contains a dot), the parent module is + automatically imported. + + **name** and **package** work the same as for :func:`import_module`. + + .. versionadded:: 3.4 + .. decorator:: module_for_loader - A :term:`decorator` for a :term:`loader` method, + A :term:`decorator` for :meth:`importlib.abc.Loader.load_module` to handle selecting the proper module object to load with. The decorated method is expected to have a call signature taking two positional arguments @@ -878,55 +1136,68 @@ an :term:`importer`. The decorated method will take in the **name** of the module to be loaded as expected for a :term:`loader`. If the module is not found in - :data:`sys.modules` then a new one is constructed with its - :attr:`__name__` attribute set to **name**, :attr:`__loader__` set to - **self**, and :attr:`__package__` set if - :meth:`importlib.abc.InspectLoader.is_package` is defined for **self** and - does not raise :exc:`ImportError` for **name**. If a new module is not - needed then the module found in :data:`sys.modules` will be passed into the - method. + :data:`sys.modules` then a new one is constructed. Regardless of where the + module came from, :attr:`__loader__` set to **self** and :attr:`__package__` + is set based on what :meth:`importlib.abc.InspectLoader.is_package` returns + (if available). These attributes are set unconditionally to support + reloading. If an exception is raised by the decorated method and a module was added to - :data:`sys.modules` it will be removed to prevent a partially initialized - module from being in left in :data:`sys.modules`. If the module was already - in :data:`sys.modules` then it is left alone. - - Use of this decorator handles all the details of which module object a - loader should initialize as specified by :pep:`302` as best as possible. + :data:`sys.modules`, then the module will be removed to prevent a partially + initialized module from being in left in :data:`sys.modules`. If the module + was already in :data:`sys.modules` then it is left alone. .. versionchanged:: 3.3 :attr:`__loader__` and :attr:`__package__` are automatically set (when possible). -.. decorator:: set_loader + .. versionchanged:: 3.4 + Set :attr:`__name__`, :attr:`__loader__` :attr:`__package__` + unconditionally to support reloading. - A :term:`decorator` for a :term:`loader` method, - to set the :attr:`__loader__` - attribute on loaded modules. If the attribute is already set the decorator - does nothing. It is assumed that the first positional argument to the - wrapped method (i.e. ``self``) is what :attr:`__loader__` should be set to. + .. deprecated:: 3.4 + The import machinery now directly performs all the functionality + provided by this function. - .. note:: +.. decorator:: set_loader - It is recommended that :func:`module_for_loader` be used over this - decorator as it subsumes this functionality. + A :term:`decorator` for :meth:`importlib.abc.Loader.load_module` + to set the :attr:`__loader__` + attribute on the returned module. If the attribute is already set the + decorator does nothing. It is assumed that the first positional argument to + the wrapped method (i.e. ``self``) is what :attr:`__loader__` should be set + to. + .. versionchanged:: 3.4 + Set ``__loader__`` if set to ``None``, as if the attribute does not + exist. + + .. deprecated:: 3.4 + The import machinery takes care of this automatically. .. decorator:: set_package - A :term:`decorator` for a :term:`loader` to set the :attr:`__package__` - attribute on the module returned by the loader. If :attr:`__package__` is - set and has a value other than ``None`` it will not be changed. - Note that the module returned by the loader is what has the attribute - set on and not the module found in :data:`sys.modules`. + A :term:`decorator` for :meth:`importlib.abc.Loader.load_module` to set the :attr:`__package__` attribute on the returned module. If :attr:`__package__` + is set and has a value other than ``None`` it will not be changed. + + .. deprecated:: 3.4 + The import machinery takes care of this automatically. + +.. function:: spec_from_loader(name, loader, *, origin=None, is_package=None) + + A factory function for creating a :class:`ModuleSpec` instance based + on a loader. The parameters have the same meaning as they do for + ModuleSpec. The function uses available :term:`loader` APIs, such as + :meth:`InspectLoader.is_package`, to fill in any missing + information on the spec. + + .. versionadded:: 3.4 - Reliance on this decorator is discouraged when it is possible to set - :attr:`__package__` before importing. By - setting it beforehand the code for the module is executed with the - attribute set and thus can be used by global level code during - initialization. +.. function:: spec_from_file_location(name, location, *, loader=None, submodule_search_locations=None) - .. note:: + A factory function for creating a :class:`ModuleSpec` instance based + on the path to a file. Missing information will be filled in on the + spec by making use of loader APIs and by the implication that the + module will be file-based. - It is recommended that :func:`module_for_loader` be used over this - decorator as it subsumes this functionality. + .. versionadded:: 3.4 diff --git a/Doc/library/index.rst b/Doc/library/index.rst index 1b25c6e..81289a5 100644 --- a/Doc/library/index.rst +++ b/Doc/library/index.rst @@ -65,6 +65,7 @@ the `Python Package Index <http://pypi.python.org/pypi>`_. tk.rst development.rst debug.rst + distribution.rst python.rst custominterp.rst modules.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index d4cf905..0c08712 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -69,7 +69,12 @@ attributes: | | | :term:`bytecode` | +-----------+-----------------+---------------------------+ | | __defaults__ | tuple of any default | -| | | values for arguments | +| | | values for positional or | +| | | keyword parameters | ++-----------+-----------------+---------------------------+ +| | __kwdefaults__ | mapping of any default | +| | | values for keyword-only | +| | | parameters | +-----------+-----------------+---------------------------+ | | __globals__ | global namespace in which | | | | this function was defined | @@ -173,8 +178,9 @@ attributes: .. note:: - :func:`getmembers` does not return metaclass attributes when the argument - is a class (this behavior is inherited from the :func:`dir` function). + :func:`getmembers` will only return class attributes defined in the + metaclass when the argument is a class and those attributes have been + listed in the metaclass' custom :meth:`__dir__`. .. function:: getmoduleinfo(path) @@ -428,11 +434,14 @@ function. Accepts a wide range of python callables, from plain functions and classes to :func:`functools.partial` objects. + Raises :exc:`ValueError` if no signature can be provided, and + :exc:`TypeError` if that type of object is not supported. + .. note:: Some callables may not be introspectable in certain implementations of - Python. For example, in CPython, built-in functions defined in C provide - no metadata about their arguments. + Python. For example, in CPython, some built-in functions defined in + C provide no metadata about their arguments. .. class:: Signature(parameters=None, \*, return_annotation=Signature.empty) @@ -510,9 +519,8 @@ function. .. attribute:: Parameter.name - The name of the parameter as a string. Must be a valid python identifier - name (with the exception of ``POSITIONAL_ONLY`` parameters, which can have - it set to ``None``). + The name of the parameter as a string. The name must be a valid + Python identifier. .. attribute:: Parameter.default @@ -596,6 +604,10 @@ function. >>> str(param.replace(default=Parameter.empty, annotation='spam')) "foo:'spam'" + .. versionchanged:: 3.4 + In Python 3.3 Parameter objects were allowed to have ``name`` set + to ``None`` if their ``kind`` was set to ``POSITIONAL_ONLY``. + This is no longer permitted. .. class:: BoundArguments @@ -717,6 +729,11 @@ Classes and functions Consider using the new :ref:`Signature Object <inspect-signature-object>` interface, which provides a better way of introspecting functions. + .. versionchanged:: 3.4 + This function is now based on :func:`signature`, but still ignores + ``__wrapped__`` attributes and includes the already bound first + parameter in the signature output for bound methods. + .. function:: getargvalues(frame) @@ -806,6 +823,23 @@ Classes and functions .. versionadded:: 3.3 +.. function:: unwrap(func, *, stop=None) + + Get the object wrapped by *func*. It follows the chain of :attr:`__wrapped__` + attributes returning the last object in the chain. + + *stop* is an optional callback accepting an object in the wrapper chain + as its sole argument that allows the unwrapping to be terminated early if + the callback returns a true value. If the callback never returns a true + value, the last object in the chain is returned as usual. For example, + :func:`signature` uses this to stop unwrapping if any object in the + chain has a ``__signature__`` attribute defined. + + :exc:`ValueError` is raised if a cycle is encountered. + + .. versionadded:: 3.4 + + .. _inspect-stack: The interpreter stack @@ -838,6 +872,10 @@ index of the current line within that list. finally: del frame + If you want to keep the frame around (for example to print a traceback + later), you can also break reference cycles by using the + :meth:`frame.clear` method. + The optional *context* argument supported by most of these functions specifies the number of lines of context to return, which are centered around the current line. @@ -994,3 +1032,22 @@ updated as expected: return an empty dictionary. .. versionadded:: 3.3 + + +.. _inspect-module-cli: + +Command Line Interface +---------------------- + +The :mod:`inspect` module also provides a basic introspection capability +from the command line. + +.. program:: inspect + +By default, accepts the name of a module and prints the source of that +module. A class or function within the module can be printed instead by +appended a colon and the qualified name of the target object. + +.. cmdoption:: --details + + Print information about the specified object rather than the source code diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 716767f..79f65e0 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -283,10 +283,10 @@ I/O Base Classes Return ``True`` if the stream can be read from. If ``False``, :meth:`read` will raise :exc:`OSError`. - .. method:: readline(limit=-1) + .. method:: readline(size=-1) - Read and return one line from the stream. If *limit* is specified, at - most *limit* bytes will be read. + Read and return one line from the stream. If *size* is specified, at + most *size* bytes will be read. The line terminator is always ``b'\n'`` for binary files; for text files, the *newlines* argument to :func:`open` can be used to select the line @@ -366,14 +366,14 @@ I/O Base Classes In addition to the attributes and methods from :class:`IOBase`, :class:`RawIOBase` provides the following methods: - .. method:: read(n=-1) + .. method:: read(size=-1) - Read up to *n* bytes from the object and return them. As a convenience, - if *n* is unspecified or -1, :meth:`readall` is called. Otherwise, - only one system call is ever made. Fewer than *n* bytes may be - returned if the operating system call returns fewer than *n* bytes. + Read up to *size* bytes from the object and return them. As a convenience, + if *size* is unspecified or -1, :meth:`readall` is called. Otherwise, + only one system call is ever made. Fewer than *size* bytes may be + returned if the operating system call returns fewer than *size* bytes. - If 0 bytes are returned, and *n* was not 0, this indicates end of file. + If 0 bytes are returned, and *size* was not 0, this indicates end of file. If the object is in non-blocking mode and no bytes are available, ``None`` is returned. @@ -442,10 +442,10 @@ I/O Base Classes .. versionadded:: 3.1 - .. method:: read(n=-1) + .. method:: read(size=-1) - Read and return up to *n* bytes. If the argument is omitted, ``None``, or - negative, data is read and returned until EOF is reached. An empty + Read and return up to *size* bytes. If the argument is omitted, ``None``, + or negative, data is read and returned until EOF is reached. An empty :class:`bytes` object is returned if the stream is already at EOF. If the argument is positive, and the underlying raw stream is not @@ -457,9 +457,9 @@ I/O Base Classes A :exc:`BlockingIOError` is raised if the underlying raw stream is in non blocking-mode, and has no data available at the moment. - .. method:: read1(n=-1) + .. method:: read1(size=-1) - Read and return up to *n* bytes, with at most one call to the underlying + Read and return up to *size* bytes, with at most one call to the underlying raw stream's :meth:`~RawIOBase.read` method. This can be useful if you are implementing your own buffering on top of a :class:`BufferedIOBase` object. @@ -522,6 +522,8 @@ Raw File I/O :mod:`os.open` as *opener* results in functionality similar to passing ``None``). + The newly created file is :ref:`non-inheritable <fd_inheritance>`. + See the :func:`open` built-in function for examples on using the *opener* parameter. @@ -529,6 +531,9 @@ Raw File I/O The *opener* parameter was added. The ``'x'`` mode was added. + .. versionchanged:: 3.4 + The file is now non-inheritable. + In addition to the attributes and methods from :class:`IOBase` and :class:`RawIOBase`, :class:`FileIO` provides the following data attributes: @@ -601,21 +606,21 @@ than raw I/O does. :class:`BufferedReader` provides or overrides these methods in addition to those from :class:`BufferedIOBase` and :class:`IOBase`: - .. method:: peek([n]) + .. method:: peek([size]) Return bytes from the stream without advancing the position. At most one single read on the raw stream is done to satisfy the call. The number of bytes returned may be less or more than requested. - .. method:: read([n]) + .. method:: read([size]) - Read and return *n* bytes, or if *n* is not given or negative, until EOF - or if the read call would block in non-blocking mode. + Read and return *size* bytes, or if *size* is not given or negative, until + EOF or if the read call would block in non-blocking mode. - .. method:: read1(n) + .. method:: read1(size) - Read and return up to *n* bytes with only one call on the raw stream. If - at least one byte is buffered, only buffered bytes are returned. + Read and return up to *size* bytes with only one call on the raw stream. + If at least one byte is buffered, only buffered bytes are returned. Otherwise, one raw stream read call is made. @@ -735,17 +740,17 @@ Text I/O .. versionadded:: 3.1 - .. method:: read(n) + .. method:: read(size) - Read and return at most *n* characters from the stream as a single - :class:`str`. If *n* is negative or ``None``, reads until EOF. + Read and return at most *size* characters from the stream as a single + :class:`str`. If *size* is negative or ``None``, reads until EOF. - .. method:: readline(limit=-1) + .. method:: readline(size=-1) Read until newline or EOF and return a single ``str``. If the stream is already at EOF, an empty string is returned. - If *limit* is specified, at most *limit* characters will be read. + If *size* is specified, at most *size* characters will be read. .. method:: seek(offset, whence=SEEK_SET) diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 8eac92f..9625e71 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -9,13 +9,6 @@ -------------- -.. note:: - - The ``ipaddress`` module has been included in the standard library on a - :term:`provisional basis <provisional package>`. Backwards incompatible - changes (up to and including removal of the package) may occur if deemed - necessary by the core developers. - :mod:`ipaddress` provides the capabilities to create, manipulate and operate on IPv4 and IPv6 addresses and networks. @@ -161,7 +154,16 @@ write code that handles both IP versions correctly. .. attribute:: is_private ``True`` if the address is allocated for private networks. See - :RFC:`1918` (for IPv4) or :RFC:`4193` (for IPv6). + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6). + + .. attribute:: is_global + + ``True`` if the address is allocated for public networks. See + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6). + + .. versionadded:: 3.4 .. attribute:: is_unspecified @@ -182,6 +184,9 @@ write code that handles both IP versions correctly. ``True`` if the address is reserved for link-local usage. See :RFC:`3927`. +.. _iana-ipv4-special-registry: http://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml +.. _iana-ipv6-special-registry: http://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml + .. class:: IPv6Address(address) @@ -216,18 +221,23 @@ write code that handles both IP versions correctly. The long form of the address representation, with all leading zeroes and groups consisting entirely of zeroes included. + + For the following attributes, see the corresponding documention of the + :class:`IPv4Address` class: + .. attribute:: packed .. attribute:: version .. attribute:: max_prefixlen .. attribute:: is_multicast .. attribute:: is_private + .. attribute:: is_global .. attribute:: is_unspecified .. attribute:: is_reserved .. attribute:: is_loopback .. attribute:: is_link_local - Refer to the corresponding attribute documentation in - :class:`IPv4Address` + .. versionadded:: 3.4 + is_global .. attribute:: is_site_local diff --git a/Doc/library/ipc.rst b/Doc/library/ipc.rst index 91ec693..6b17563 100644 --- a/Doc/library/ipc.rst +++ b/Doc/library/ipc.rst @@ -9,7 +9,7 @@ to communicate. Some modules only work for two processes that are on the same machine, e.g. :mod:`signal` and :mod:`mmap`. Other modules support networking protocols -that two or more processes can used to communicate across machines. +that two or more processes can use to communicate across machines. The list of modules described in this chapter is: @@ -18,6 +18,9 @@ The list of modules described in this chapter is: socket.rst ssl.rst + select.rst + selectors.rst + asyncio.rst asyncore.rst asynchat.rst signal.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 5d3e50a..f489535 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -784,6 +784,19 @@ which incur interpreter overhead. except exception: pass + def first_true(iterable, default=False, pred=None): + """Returns the first true value in the iterable. + + If no true value is found, returns *default* + + If *pred* is not None, returns the first item + for which pred(item) is true. + + """ + # first_true([a,b,c], x) --> a or b or c or x + # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x + return next(filter(pred, iterable), default) + def random_product(*args, repeat=1): "Random selection from itertools.product(*args, **kwds)" pools = [tuple(pool) for pool in args] * repeat diff --git a/Doc/library/json.rst b/Doc/library/json.rst index f652039..5d97ee8 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -42,8 +42,7 @@ Compact encoding:: Pretty printing:: >>> import json - >>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, - ... indent=4, separators=(',', ': '))) + >>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4)) { "4": 5, "6": 7 @@ -158,15 +157,13 @@ Basic Usage .. versionchanged:: 3.2 Allow strings for *indent* in addition to integers. - .. note:: - - Since the default item separator is ``', '``, the output might include - trailing whitespace when *indent* is specified. You can use - ``separators=(',', ': ')`` to avoid this. + If specified, *separators* should be an ``(item_separator, key_separator)`` + tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and + ``(',', ': ')`` otherwise. To get the most compact JSON representation, + you should specify ``(',', ':')`` to eliminate whitespace. - If *separators* is an ``(item_separator, dict_separator)`` tuple, then it - will be used instead of the default ``(', ', ': ')`` separators. ``(',', - ':')`` is the most compact JSON representation. + .. versionchanged:: 3.4 + Use ``(',', ': ')`` as default if *indent* is not ``None``. *default(obj)* is a function that should return a serializable version of *obj* or raise :exc:`TypeError`. The default simply raises :exc:`TypeError`. @@ -248,6 +245,8 @@ Basic Usage kwarg; otherwise :class:`JSONDecoder` is used. Additional keyword arguments will be passed to the constructor of the class. + If the data being deserialized is not a valid JSON document, a + :exc:`ValueError` will be raised. .. function:: loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw) @@ -257,6 +256,8 @@ Basic Usage The other arguments have the same meaning as in :func:`load`, except *encoding* which is ignored and deprecated. + If the data being deserialized is not a valid JSON document, a + :exc:`ValueError` will be raised. Encoders and Decoders --------------------- @@ -354,23 +355,26 @@ Encoders and Decoders .. _py-to-json-table: - +-------------------+---------------+ - | Python | JSON | - +===================+===============+ - | dict | object | - +-------------------+---------------+ - | list, tuple | array | - +-------------------+---------------+ - | str | string | - +-------------------+---------------+ - | int, float | number | - +-------------------+---------------+ - | True | true | - +-------------------+---------------+ - | False | false | - +-------------------+---------------+ - | None | null | - +-------------------+---------------+ + +----------------------------------------+---------------+ + | Python | JSON | + +========================================+===============+ + | dict | object | + +----------------------------------------+---------------+ + | list, tuple | array | + +----------------------------------------+---------------+ + | str | string | + +----------------------------------------+---------------+ + | int, float, int- & float-derived Enums | number | + +----------------------------------------+---------------+ + | True | true | + +----------------------------------------+---------------+ + | False | false | + +----------------------------------------+---------------+ + | None | null | + +----------------------------------------+---------------+ + + .. versionchanged:: 3.4 + Added support for int- and float-derived Enum classes. To extend this to recognize other objects, subclass and implement a :meth:`default` method with another method that returns a serializable object @@ -410,15 +414,13 @@ Encoders and Decoders .. versionchanged:: 3.2 Allow strings for *indent* in addition to integers. - .. note:: - - Since the default item separator is ``', '``, the output might include - trailing whitespace when *indent* is specified. You can use - ``separators=(',', ': ')`` to avoid this. - If specified, *separators* should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')``. To get the most compact JSON - representation, you should specify ``(',', ':')`` to eliminate whitespace. + tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and + ``(',', ': ')`` otherwise. To get the most compact JSON representation, + you should specify ``(',', ':')`` to eliminate whitespace. + + .. versionchanged:: 3.4 + Use ``(',', ': ')`` as default if *indent* is not ``None``. If specified, *default* is a function that gets called for objects that can't otherwise be serialized. It should return a JSON encodable version of the diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 303b4d8..171b0a3 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -80,12 +80,25 @@ in :mod:`logging` itself) and defining handlers which are declared either in .. function:: fileConfig(fname, defaults=None, disable_existing_loggers=True) - Reads the logging configuration from a :mod:`configparser`\-format file - named *fname*. The format of the file should be as described in - :ref:`logging-config-fileformat`. This function can be called several times - from an application, allowing an end user to select from various pre-canned - configurations (if the developer provides a mechanism to present the choices - and load the chosen configuration). + Reads the logging configuration from a :mod:`configparser`\-format file. The + format of the file should be as described in + :ref:`logging-config-fileformat`. + This function can be called several times from an application, allowing an + end user to select from various pre-canned configurations (if the developer + provides a mechanism to present the choices and load the chosen + configuration). + + :param fname: A filename, or a file-like object, or an instance derived + from :class:`~configparser.RawConfigParser`. If a + ``RawConfigParser``-derived instance is passed, it is used as + is. Otherwise, a :class:`~configparser.Configparser` is + instantiated, and the configuration read by it from the + object passed in ``fname``. If that has a :meth:`readline` + method, it is assumed to be a file-like object and read using + :meth:`~configparser.ConfigParser.read_file`; otherwise, + it is assumed to be a filename and passed to + :meth:`~configparser.ConfigParser.read`. + :param defaults: Defaults to be passed to the ConfigParser can be specified in this argument. @@ -99,8 +112,17 @@ in :mod:`logging` itself) and defining handlers which are declared either in their ancestors are explicitly named in the logging configuration. + .. versionchanged:: 3.4 + An instance of a subclass of :class:`~configparser.RawConfigParser` is + now accepted as a value for ``fname``. This facilitates: + + * Use of a configuration file where logging configuration is just part + of the overall application configuration. + * Use of a configuration read from a file, and then modified by the using + application (e.g. based on command-line parameters or other aspects + of the runtime environment) before being passed to ``fileConfig``. -.. function:: listen(port=DEFAULT_LOGGING_CONFIG_PORT) +.. function:: listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None) Starts up a socket server on the specified port, and listens for new configurations. If no port is specified, the module's default @@ -111,6 +133,17 @@ in :mod:`logging` itself) and defining handlers which are declared either in :meth:`~threading.Thread.join` when appropriate. To stop the server, call :func:`stopListening`. + The ``verify`` argument, if specified, should be a callable which should + verify whether bytes received across the socket are valid and should be + processed. This could be done by encrypting and/or signing what is sent + across the socket, such that the ``verify`` callable can perform + signature verification and/or decryption. The ``verify`` callable is called + with a single argument - the bytes received across the socket - and should + return the bytes to be processed, or None to indicate that the bytes should + be discarded. The returned bytes could be the same as the passed in bytes + (e.g. when only verification is done), or they could be completely different + (perhaps if decryption were performed). + To send a configuration to the socket, read in the configuration file and send it to the socket as a string of bytes preceded by a four-byte length string packed in binary using ``struct.pack('>L', n)``. @@ -129,7 +162,12 @@ in :mod:`logging` itself) and defining handlers which are declared either in :func:`listen` socket and sending a configuration which runs whatever code the attacker wants to have executed in the victim's process. This is especially easy to do if the default port is used, but not hard even if a - different port is used). + different port is used). To avoid the risk of this happening, use the + ``verify`` argument to :func:`listen` to prevent unrecognised + configurations from being applied. + + .. versionchanged:: 3.4. + The ``verify`` argument was added. .. function:: stopListening() diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 415e397..a2b14de 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -300,7 +300,7 @@ The :class:`TimedRotatingFileHandler` class, located in the timed intervals. -.. class:: TimedRotatingFileHandler(filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False) +.. class:: TimedRotatingFileHandler(filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False, atTime=None) Returns a new instance of the :class:`TimedRotatingFileHandler` class. The specified file is opened and used as the stream for logging. On rotating it also @@ -350,6 +350,12 @@ timed intervals. If *delay* is true, then file opening is deferred until the first call to :meth:`emit`. + If *atTime* is not ``None``, it must be a ``datetime.time`` instance which + specifies the time of day when rollover occurs, for the cases where rollover + is set to happen "at midnight" or "on a particular weekday". + + .. versionchanged:: 3.4 + *atTime* parameter was added. .. method:: doRollover() @@ -375,6 +381,9 @@ sends logging output to a network socket. The base class uses a TCP socket. Returns a new instance of the :class:`SocketHandler` class intended to communicate with a remote machine whose address is given by *host* and *port*. + .. versionchanged:: 3.4 + If ``port`` is specified as ``None``, a Unix domain socket is created + using the value in ``host`` - otherwise, a TCP socket is created. .. method:: close() @@ -460,6 +469,9 @@ over UDP sockets. Returns a new instance of the :class:`DatagramHandler` class intended to communicate with a remote machine whose address is given by *host* and *port*. + .. versionchanged:: 3.4 + If ``port`` is specified as ``None``, a Unix domain socket is created + using the value in ``host`` - otherwise, a TCP socket is created. .. method:: emit() @@ -838,10 +850,27 @@ supports sending logging messages to a Web server, using either ``GET`` or credentials, you should also specify secure=True so that your userid and password are not passed in cleartext across the wire. + .. method:: mapLogRecord(record) + + Provides a dictionary, based on ``record``, which is to be URL-encoded + and sent to the web server. The default implementation just returns + ``record.__dict__``. This method can be overridden if e.g. only a + subset of :class:`~logging.LogRecord` is to be sent to the web server, or + if more specific customization of what's sent to the server is required. .. method:: emit(record) - Sends the record to the Web server as a percent-encoded dictionary. + Sends the record to the Web server as an URL-encoded dictionary. The + :meth:`mapLogRecord` method is used to convert the record to the + dictionary to be sent. + + .. note:: Since preparing a record for sending it to a Web server is not + the same as a generic formatting operation, using + :meth:`~logging.Handler.setFormatter` to specify a + :class:`~logging.Formatter` for a :class:`HTTPHandler` has no effect. + Instead of calling :meth:`~logging.Handler.format`, this handler calls + :meth:`mapLogRecord` and then :func:`urllib.parse.urlencode` to encode the + dictionary in a form suitable for sending to a Web server. .. _queue-handler: diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 8d093e1..a3e2770 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -250,7 +250,7 @@ is the module's name in the Python package namespace. interpreted as for :meth:`debug`. -.. method:: Logger.exception(msg, *args) +.. method:: Logger.exception(msg, *args, **kwargs) Logs a message with level :const:`ERROR` on this logger. The arguments are interpreted as for :meth:`debug`. Exception info is added to the logging @@ -992,7 +992,7 @@ functions. are interpreted as for :func:`debug`. -.. function:: exception(msg, *args) +.. function:: exception(msg, *args, **kwargs) Logs a message with level :const:`ERROR` on the root logger. The arguments are interpreted as for :func:`debug`. Exception info is added to the logging @@ -1049,6 +1049,11 @@ functions. of the defined levels is passed in, the corresponding string representation is returned. Otherwise, the string 'Level %s' % lvl is returned. + .. versionchanged:: 3.4 + In Python versions earlier than 3.4, this function could also be passed a + text level, and would return the corresponding numeric value of the level. + This undocumented behaviour was a mistake, and has been removed in Python + 3.4. .. function:: makeLogRecord(attrdict) diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 07b69eb..b71051d 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -39,8 +39,8 @@ Reading and writing compressed files opened, or it can be an existing file object to read from or write to. The *mode* argument can be any of ``"r"``, ``"rb"``, ``"w"``, ``"wb"``, - ``"a"`` or ``"ab"`` for binary mode, or ``"rt"``, ``"wt"``, or ``"at"`` for - text mode. The default is ``"rb"``. + ``"x"``, ``"xb"``, ``"a"`` or ``"ab"`` for binary mode, or ``"rt"``, + ``"wt"``, ``"xt"``, or ``"at"`` for text mode. The default is ``"rb"``. When opening a file for reading, the *format* and *filters* arguments have the same meanings as for :class:`LZMADecompressor`. In this case, the *check* @@ -57,6 +57,9 @@ Reading and writing compressed files :class:`io.TextIOWrapper` instance with the specified encoding, error handling behavior, and line ending(s). + .. versionchanged:: 3.4 + Added support for the ``"x"``, ``"xb"`` and ``"xt"`` modes. + .. class:: LZMAFile(filename=None, mode="r", \*, format=None, check=-1, preset=None, filters=None) @@ -69,8 +72,9 @@ Reading and writing compressed files file will not be closed when the :class:`LZMAFile` is closed. The *mode* argument can be either ``"r"`` for reading (default), ``"w"`` for - overwriting, or ``"a"`` for appending. These can equivalently be given as - ``"rb"``, ``"wb"``, and ``"ab"`` respectively. + overwriting, ``"x"`` for exclusive creation, or ``"a"`` for appending. These + can equivalently be given as ``"rb"``, ``"wb"``, ``"xb"`` and ``"ab"`` + respectively. If *filename* is a file object (rather than an actual file name), a mode of ``"w"`` does not truncate the file, and is instead equivalent to ``"a"``. @@ -103,6 +107,9 @@ Reading and writing compressed files file object (e.g. if the :class:`LZMAFile` was constructed by passing a file object for *filename*). + .. versionchanged:: 3.4 + Added support for the ``"x"`` and ``"xb"`` modes. + Compressing and decompressing data in memory -------------------------------------------- diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index 3b9e3d2..124eb61 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -40,10 +40,11 @@ this module. The following types are supported: booleans, integers, floating point numbers, complex numbers, strings, bytes, bytearrays, tuples, lists, sets, frozensets, dictionaries, and code objects, where it should be understood that tuples, lists, sets, frozensets and dictionaries are only supported as long as -the values contained therein are themselves supported; and recursive lists, sets -and dictionaries should not be written (they will cause infinite loops). The +the values contained therein are themselves supported. singletons :const:`None`, :const:`Ellipsis` and :exc:`StopIteration` can also be marshalled and unmarshalled. +For format *version* lower than 3, recursive lists, sets and dictionaries cannot +be written (see below). There are functions that read/write files as well as functions operating on strings. @@ -103,7 +104,9 @@ In addition, the following constants are defined: Indicates the format that the module uses. Version 0 is the historical format, version 1 shares interned strings and version 2 uses a binary format - for floating point numbers. The current version is 2. + for floating point numbers. + Version 3 adds support for object instancing and recursion. + The current version is 3. .. rubric:: Footnotes diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 5f50cf1..5fac730 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -93,11 +93,112 @@ To show the individual process IDs involved, here is an expanded example:: p.start() p.join() -For an explanation of why (on Windows) the ``if __name__ == '__main__'`` part is +For an explanation of why the ``if __name__ == '__main__'`` part is necessary, see :ref:`multiprocessing-programming`. +Contexts and start methods +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _multiprocessing-start-methods: + +Depending on the platform, :mod:`multiprocessing` supports three ways +to start a process. These *start methods* are + + *spawn* + The parent process starts a fresh python interpreter process. The + child process will only inherit those resources necessary to run + the process objects :meth:`~Process.run` method. In particular, + unnecessary file descriptors and handles from the parent process + will not be inherited. Starting a process using this method is + rather slow compared to using *fork* or *forkserver*. + + Available on Unix and Windows. The default on Windows. + + *fork* + The parent process uses :func:`os.fork` to fork the Python + interpreter. The child process, when it begins, is effectively + identical to the parent process. All resources of the parent are + inherited by the child process. Note that safely forking a + multithreaded process is problematic. + + Available on Unix only. The default on Unix. + + *forkserver* + When the program starts and selects the *forkserver* start method, + a server process is started. From then on, whenever a new process + is needed, the parent process connects to the server and requests + that it fork a new process. The fork server process is single + threaded so it is safe for it to use :func:`os.fork`. No + unnecessary resources are inherited. + + Available on Unix platforms which support passing file descriptors + over Unix pipes. + +.. versionchanged:: 3.4 + *spawn* added on all unix platforms, and *forkserver* added for + some unix platforms. + Child processes no longer inherit all of the parents inheritable + handles on Windows. + +On Unix using the *spawn* or *forkserver* start methods will also +start a *semaphore tracker* process which tracks the unlinked named +semaphores created by processes of the program. When all processes +have exited the semaphore tracker unlinks any remaining semaphores. +Usually there should be none, but if a process was killed by a signal +there may some "leaked" semaphores. (Unlinking the named semaphores +is a serious matter since the system allows only a limited number, and +they will not be automatically unlinked until the next reboot.) + +To select a start method you use the :func:`set_start_method` in +the ``if __name__ == '__main__'`` clause of the main module. For +example:: + + import multiprocessing as mp + + def foo(q): + q.put('hello') + + if __name__ == '__main__': + mp.set_start_method('spawn') + q = mp.Queue() + p = mp.Process(target=foo, args=(q,)) + p.start() + print(q.get()) + p.join() + +:func:`set_start_method` should not be used more than once in the +program. + +Alternatively, you can use :func:`get_context` to obtain a context +object. Context objects have the same API as the multiprocessing +module, and allow one to use multiple start methods in the same +program. :: + + import multiprocessing as mp + + def foo(q): + q.put('hello') + + if __name__ == '__main__': + ctx = mp.get_context('spawn') + q = ctx.Queue() + p = ctx.Process(target=foo, args=(q,)) + p.start() + print(q.get()) + p.join() + +Note that objects related to one context may not be compatible with +processes for a different context. In particular, locks created using +the *fork* context cannot be passed to a processes started using the +*spawn* or *forkserver* start methods. + +A library which wants to use a particular start method should probably +use :func:`get_context` to avoid interfering with the choice of the +library user. + + Exchanging objects between processes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -274,15 +375,31 @@ processes in a few different ways. For example:: from multiprocessing import Pool + from time import sleep def f(x): return x*x if __name__ == '__main__': - with Pool(processes=4) as pool: # start 4 worker processes - result = pool.apply_async(f, [10]) # evaluate "f(10)" asynchronously - print(result.get(timeout=1)) # prints "100" unless your computer is *very* slow - print(pool.map(f, range(10))) # prints "[0, 1, 4,..., 81]" + # start 4 worker processes + with Pool(processes=4) as pool: + + # print "[0, 1, 4,..., 81]" + print(pool.map(f, range(10))) + + # print same numbers in arbitrary order + for i in pool.imap_unordered(f, range(10)): + print(i) + + # evaluate "f(10)" asynchronously + res = pool.apply_async(f, [10]) + print(res.get(timeout=1)) # prints "100" + + # make worker sleep for 10 secs + res = pool.apply_async(sleep, 10) + print(res.get(timeout=1)) # raises multiprocessing.TimeoutError + + # exiting the 'with'-block has stopped the pool Note that the methods of a pool should only ever be used by the process which created it. @@ -731,6 +848,9 @@ Miscellaneous Return the number of CPUs in the system. May raise :exc:`NotImplementedError`. + .. seealso:: + :func:`os.cpu_count` + .. function:: current_process() Return the :class:`Process` object corresponding to the current process. @@ -761,6 +881,43 @@ Miscellaneous If the module is being run normally by the Python interpreter then :func:`freeze_support` has no effect. +.. function:: get_all_start_methods() + + Returns a list of the supported start methods, the first of which + is the default. The possible start methods are ``'fork'``, + ``'spawn'`` and ``'forkserver'``. On Windows only ``'spawn'`` is + available. On Unix ``'fork'`` and ``'spawn'`` are always + supported, with ``'fork'`` being the default. + + .. versionadded:: 3.4 + +.. function:: get_context(method=None) + + Return a context object which has the same attributes as the + :mod:`multiprocessing` module. + + If *method* is *None* then the default context is returned. + Otherwise *method* should be ``'fork'``, ``'spawn'``, + ``'forkserver'``. :exc:`ValueError` is raised if the specified + start method is not available. + + .. versionadded:: 3.4 + +.. function:: get_start_method(allow_none=False) + + Return the name of start method used for starting processes. + + If the start method has not been fixed and *allow_none* is false, + then the start method is fixed to the default and the name is + returned. If the start method has not been fixed and *allow_none* + is true then *None* is returned. + + The return value can be ``'fork'``, ``'spawn'``, ``'forkserver'`` + or *None*. ``'fork'`` is the default on Unix, while ``'spawn'`` is + the default on Windows. + + .. versionadded:: 3.4 + .. function:: set_executable() Sets the path of the Python interpreter to use when starting a child process. @@ -769,8 +926,21 @@ Miscellaneous set_executable(os.path.join(sys.exec_prefix, 'pythonw.exe')) - before they can create child processes. (Windows only) + before they can create child processes. + + .. versionchanged:: 3.4 + Now supported on Unix when the ``'spawn'`` start method is used. +.. function:: set_start_method(method) + + Set the method which should be used to start child processes. + *method* can be ``'fork'``, ``'spawn'`` or ``'forkserver'``. + + Note that this should be called at most once, and it should be + protected inside the ``if __name__ == '__main__'`` clause of the + main module. + + .. versionadded:: 3.4 .. note:: @@ -1678,25 +1848,37 @@ Process Pools One can create a pool of processes which will carry out tasks submitted to it with the :class:`Pool` class. -.. class:: Pool([processes[, initializer[, initargs[, maxtasksperchild]]]]) +.. class:: Pool([processes[, initializer[, initargs[, maxtasksperchild [, context]]]]]) A process pool object which controls a pool of worker processes to which jobs can be submitted. It supports asynchronous results with timeouts and callbacks and has a parallel map implementation. *processes* is the number of worker processes to use. If *processes* is - ``None`` then the number returned by :func:`cpu_count` is used. If - *initializer* is not ``None`` then each worker process will call + ``None`` then the number returned by :func:`os.cpu_count` is used. + + If *initializer* is not ``None`` then each worker process will call ``initializer(*initargs)`` when it starts. + *maxtasksperchild* is the number of tasks a worker process can complete + before it will exit and be replaced with a fresh worker process, to enable + unused resources to be freed. The default *maxtasksperchild* is None, which + means worker processes will live as long as the pool. + + *context* can be used to specify the context used for starting + the worker processes. Usually a pool is created using the + function :func:`multiprocessing.Pool` or the :meth:`Pool` method + of a context object. In both cases *context* is set + appropriately. + Note that the methods of the pool object should only be called by the process which created the pool. .. versionadded:: 3.2 - *maxtasksperchild* is the number of tasks a worker process can complete - before it will exit and be replaced with a fresh worker process, to enable - unused resources to be freed. The default *maxtasksperchild* is None, which - means worker processes will live as long as the pool. + *maxtasksperchild* + + .. versionadded:: 3.4 + *context* .. note:: @@ -2189,43 +2371,8 @@ Below is an example session with logging turned on:: [INFO/MainProcess] sending shutdown message to manager [INFO/SyncManager-...] manager exiting with exitcode 0 -In addition to having these two logging functions, the multiprocessing also -exposes two additional logging level attributes. These are :const:`SUBWARNING` -and :const:`SUBDEBUG`. The table below illustrates where theses fit in the -normal level hierarchy. - -+----------------+----------------+ -| Level | Numeric value | -+================+================+ -| ``SUBWARNING`` | 25 | -+----------------+----------------+ -| ``SUBDEBUG`` | 5 | -+----------------+----------------+ - For a full table of logging levels, see the :mod:`logging` module. -These additional logging levels are used primarily for certain debug messages -within the multiprocessing module. Below is the same example as above, except -with :const:`SUBDEBUG` enabled:: - - >>> import multiprocessing, logging - >>> logger = multiprocessing.log_to_stderr() - >>> logger.setLevel(multiprocessing.SUBDEBUG) - >>> logger.warning('doomed') - [WARNING/MainProcess] doomed - >>> m = multiprocessing.Manager() - [INFO/SyncManager-...] child process calling self.run() - [INFO/SyncManager-...] created temp directory /.../pymp-... - [INFO/SyncManager-...] manager serving at '/.../pymp-djGBXN/listener-...' - >>> del m - [SUBDEBUG/MainProcess] finalizer calling ... - [INFO/MainProcess] sending shutdown message to manager - [DEBUG/SyncManager-...] manager received shutdown message - [SUBDEBUG/SyncManager-...] calling <Finalize object, callback=unlink, ... - [SUBDEBUG/SyncManager-...] finalizer calling <built-in function unlink> ... - [SUBDEBUG/SyncManager-...] calling <Finalize object, dead> - [SUBDEBUG/SyncManager-...] finalizer calling <function rmtree at 0x5aa730> ... - [INFO/SyncManager-...] manager exiting with exitcode 0 The :mod:`multiprocessing.dummy` module ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -2246,8 +2393,10 @@ There are certain guidelines and idioms which should be adhered to when using :mod:`multiprocessing`. -All platforms -~~~~~~~~~~~~~ +All start methods +~~~~~~~~~~~~~~~~~ + +The following applies to all start methods. Avoid shared state @@ -2281,11 +2430,13 @@ Joining zombie processes Better to inherit than pickle/unpickle - On Windows many types from :mod:`multiprocessing` need to be picklable so - that child processes can use them. However, one should generally avoid - sending shared objects to other processes using pipes or queues. Instead - you should arrange the program so that a process which needs access to a - shared resource created elsewhere can inherit it from an ancestor process. + When using the *spawn* or *forkserver* start methods many types + from :mod:`multiprocessing` need to be picklable so that child + processes can use them. However, one should generally avoid + sending shared objects to other processes using pipes or queues. + Instead you should arrange the program so that a process which + needs access to a shared resource created elsewhere can inherit it + from an ancestor process. Avoid terminating processes @@ -2332,15 +2483,17 @@ Joining processes that use queues Explicitly pass resources to child processes - On Unix a child process can make use of a shared resource created in a - parent process using a global resource. However, it is better to pass the - object as an argument to the constructor for the child process. + On Unix using the *fork* start method, a child process can make + use of a shared resource created in a parent process using a + global resource. However, it is better to pass the object as an + argument to the constructor for the child process. - Apart from making the code (potentially) compatible with Windows this also - ensures that as long as the child process is still alive the object will not - be garbage collected in the parent process. This might be important if some - resource is freed when the object is garbage collected in the parent - process. + Apart from making the code (potentially) compatible with Windows + and the other start methods this also ensures that as long as the + child process is still alive the object will not be garbage + collected in the parent process. This might be important if some + resource is freed when the object is garbage collected in the + parent process. So for instance :: @@ -2399,17 +2552,19 @@ Beware of replacing :data:`sys.stdin` with a "file like object" For more information, see :issue:`5155`, :issue:`5313` and :issue:`5331` -Windows -~~~~~~~ +The *spawn* and *forkserver* start methods +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Since Windows lacks :func:`os.fork` it has a few extra restrictions: +There are a few extra restriction which don't apply to the *fork* +start method. More picklability - Ensure that all arguments to :meth:`Process.__init__` are picklable. This - means, in particular, that bound or unbound methods cannot be used directly - as the ``target`` argument on Windows --- just define a function and use - that instead. + Ensure that all arguments to :meth:`Process.__init__` are + picklable. This means, in particular, that bound or unbound + methods cannot be used directly as the ``target`` (unless you use + the *fork* start method) --- just define a function and use that + instead. Also, if you subclass :class:`~multiprocessing.Process` then make sure that instances will be picklable when the :meth:`Process.start @@ -2431,7 +2586,8 @@ Safe importing of main module interpreter without causing unintended side effects (such a starting a new process). - For example, under Windows running the following module would fail with a + For example, using the *spawn* or *forkserver* start method + running the following module would fail with a :exc:`RuntimeError`:: from multiprocessing import Process @@ -2445,13 +2601,14 @@ Safe importing of main module Instead one should protect the "entry point" of the program by using ``if __name__ == '__main__':`` as follows:: - from multiprocessing import Process, freeze_support + from multiprocessing import Process, freeze_support, set_start_method def foo(): print('hello') if __name__ == '__main__': freeze_support() + set_start_method('spawn') p = Process(target=foo) p.start() @@ -2482,26 +2639,7 @@ Using :class:`~multiprocessing.pool.Pool`: :language: python3 -Synchronization types like locks, conditions and queues: - -.. literalinclude:: ../includes/mp_synchronize.py - :language: python3 - - An example showing how to use queues to feed tasks to a collection of worker processes and collect the results: .. literalinclude:: ../includes/mp_workers.py - - -An example of how a pool of worker processes can each run a -:class:`~http.server.SimpleHTTPRequestHandler` instance while sharing a single -listening socket. - -.. literalinclude:: ../includes/mp_webserver.py - - -Some simple benchmarks comparing :mod:`multiprocessing` with :mod:`threading`: - -.. literalinclude:: ../includes/mp_benchmarks.py - diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index 564f101..23ffed6 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -29,7 +29,7 @@ the Unix :program:`ftp` program and other FTP clients. This implements security behavior equivalent to that of ftp and other programs that use :file:`.netrc`. - .. versionchanged:: 3.3.3 Added the POSIX permission check. + .. versionchanged:: 3.4 Added the POSIX permission check. .. exception:: NetrcParseError diff --git a/Doc/library/nntplib.rst b/Doc/library/nntplib.rst index a15d7f7..3943f2c 100644 --- a/Doc/library/nntplib.rst +++ b/Doc/library/nntplib.rst @@ -1,4 +1,3 @@ - :mod:`nntplib` --- NNTP protocol client ======================================= @@ -71,7 +70,7 @@ The module itself defines the following classes: reader-specific commands, such as ``group``. If you get unexpected :exc:`NNTPPermanentError`\ s, you might need to set *readermode*. :class:`NNTP` class supports the :keyword:`with` statement to - unconditionally consume :exc:`socket.error` exceptions and to close the NNTP + unconditionally consume :exc:`OSError` exceptions and to close the NNTP connection when done. Here is a sample on how using it: >>> from nntplib import NNTP @@ -95,6 +94,7 @@ The module itself defines the following classes: port *port*. :class:`NNTP_SSL` objects have the same methods as :class:`NNTP` objects. If *port* is omitted, port 563 (NNTPS) is used. *ssl_context* is also optional, and is a :class:`~ssl.SSLContext` object. + Please read :ref:`ssl-security` for best practices. All other parameters behave the same as for :class:`NNTP`. Note that SSL-on-563 is discouraged per :rfc:`4642`, in favor of @@ -103,6 +103,10 @@ The module itself defines the following classes: .. versionadded:: 3.2 + .. versionchanged:: 3.4 + The class now supports hostname check with + :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see + :data:`ssl.HAS_SNI`). .. exception:: NNTPError @@ -231,9 +235,10 @@ tuples or objects that the method normally returns will be empty. .. method:: NNTP.starttls(ssl_context=None) - Send a ``STARTTLS`` command. The *ssl_context* argument is optional - and should be a :class:`ssl.SSLContext` object. This will enable - encryption on the NNTP connection. + Send a ``STARTTLS`` command. This will enable encryption on the NNTP + connection. The *ssl_context* argument is optional and should be a + :class:`ssl.SSLContext` object. Please read :ref:`ssl-security` for best + practices. Note that this may not be done after authentication information has been transmitted, and authentication occurs by default if possible during a @@ -242,6 +247,10 @@ tuples or objects that the method normally returns will be empty. .. versionadded:: 3.2 + .. versionchanged:: 3.4 + The method now supports hostname check with + :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see + :data:`ssl.HAS_SNI`). .. method:: NNTP.newgroups(date, *, file=None) diff --git a/Doc/library/numeric.rst b/Doc/library/numeric.rst index 2732a84..7c76a47 100644 --- a/Doc/library/numeric.rst +++ b/Doc/library/numeric.rst @@ -23,3 +23,4 @@ The following modules are documented in this chapter: decimal.rst fractions.rst random.rst + statistics.rst diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 24becf9..3bcbaa4 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -11,6 +11,9 @@ import operator from operator import itemgetter, iadd +**Source code:** :source:`Lib/operator.py` + +-------------- The :mod:`operator` module exports a set of efficient functions corresponding to the intrinsic operators of Python. For example, ``operator.add(x, y)`` is @@ -235,6 +238,14 @@ their character equivalents. .. XXX: find a better, readable, example +.. function:: length_hint(obj, default=0) + + Return an estimated length for the object *o*. First try to return its + actual length, then an estimate using :meth:`object.__length_hint__`, and + finally return the default value. + + .. versionadded:: 3.4 + The :mod:`operator` module also defines tools for generalized attribute and item lookups. These are useful for making fast field extractors as arguments for :func:`map`, :func:`sorted`, :meth:`itertools.groupby`, or other functions that diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index 13395b6..72145aa 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -8,8 +8,8 @@ .. sectionauthor:: Greg Ward <gward@python.net> .. deprecated:: 3.2 - The :mod:`optparse` module is deprecated and will not be developed further; - development will continue with the :mod:`argparse` module. + The :mod:`optparse` module is deprecated and will not be developed further; + development will continue with the :mod:`argparse` module. **Source code:** :source:`Lib/optparse.py` diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 57a6bca..6b3a3b6 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -22,6 +22,11 @@ Functions such as :func:`expanduser` and :func:`expandvars` can be invoked explicitly when an application desires shell-like path expansion. (See also the :mod:`glob` module.) + +.. seealso:: + The :mod:`pathlib` module offers high-level path objects. + + .. note:: All of these functions accept either only bytes or only string objects as @@ -42,7 +47,6 @@ the :mod:`glob` module.) * :mod:`posixpath` for UNIX-style paths * :mod:`ntpath` for Windows paths * :mod:`macpath` for old-style MacOS paths - * :mod:`os2emxpath` for OS/2 EMX paths .. function:: abspath(path) @@ -189,11 +193,17 @@ the :mod:`glob` module.) .. function:: ismount(path) - Return ``True`` if pathname *path* is a :dfn:`mount point`: a point in a file - system where a different file system has been mounted. The function checks - whether *path*'s parent, :file:`path/..`, is on a different device than *path*, - or whether :file:`path/..` and *path* point to the same i-node on the same - device --- this should detect mount points for all Unix and POSIX variants. + Return ``True`` if pathname *path* is a :dfn:`mount point`: a point in a + file system where a different file system has been mounted. On POSIX, the + function checks whether *path*'s parent, :file:`path/..`, is on a different + device than *path*, or whether :file:`path/..` and *path* point to the same + i-node on the same device --- this should detect mount points for all Unix + and POSIX variants. On Windows, a drive letter root and a share UNC are + always mount points, and for any other path ``GetVolumePathName`` is called + to see if it is different from the input path. + + .. versionadded:: 3.4 + Support for detecting non-root mount points on Windows. .. function:: join(path1[, path2[, ...]]) @@ -232,7 +242,7 @@ the :mod:`glob` module.) links encountered in the path (if they are supported by the operating system). -.. function:: relpath(path, start=None) +.. function:: relpath(path, start=os.curdir) Return a relative filepath to *path* either from the current directory or from an optional *start* directory. This is a path computation: the @@ -247,18 +257,17 @@ the :mod:`glob` module.) .. function:: samefile(path1, path2) Return ``True`` if both pathname arguments refer to the same file or directory. - On Unix, this is determined by the device number and i-node number and raises an + This is determined by the device number and i-node number and raises an exception if a :func:`os.stat` call on either pathname fails. - On Windows, two files are the same if they resolve to the same final path - name using the Windows API call GetFinalPathNameByHandle. This function - raises an exception if handles cannot be obtained to either file. - Availability: Unix, Windows. .. versionchanged:: 3.2 Added Windows support. + .. versionchanged:: 3.4 + Windows now uses the same implementation as all other platforms. + .. function:: sameopenfile(fp1, fp2) @@ -277,7 +286,10 @@ the :mod:`glob` module.) :func:`os.lstat`, or :func:`os.stat`. This function implements the underlying comparison used by :func:`samefile` and :func:`sameopenfile`. - Availability: Unix. + Availability: Unix, Windows. + + .. versionchanged:: 3.4 + Added Windows support. .. function:: split(path) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index d7b9829..20f3b25 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -54,7 +54,7 @@ Notes on the availability of these functions: The name of the operating system dependent module imported. The following names have currently been registered: ``'posix'``, ``'nt'``, ``'mac'``, - ``'os2'``, ``'ce'``, ``'java'``. + ``'ce'``, ``'java'``. .. seealso:: :attr:`sys.platform` has a finer granularity. :func:`os.uname` gives @@ -379,7 +379,7 @@ process and user. .. index:: single: user; id - Return the current process's user id. + Return the current process's real user id. Availability: Unix. @@ -687,17 +687,30 @@ as internal buffering of data. .. function:: dup(fd) - Return a duplicate of file descriptor *fd*. + Return a duplicate of file descriptor *fd*. The new file descriptor is + :ref:`non-inheritable <fd_inheritance>`. + + On Windows, when duplicating a standard stream (0: stdin, 1: stdout, + 2: stderr), the new file descriptor is :ref:`inheritable + <fd_inheritance>`. Availability: Unix, Windows. + .. versionchanged:: 3.4 + The new file descriptor is now non-inheritable. + -.. function:: dup2(fd, fd2) +.. function:: dup2(fd, fd2, inheritable=True) Duplicate file descriptor *fd* to *fd2*, closing the latter first if necessary. + The file descriptor *fd2* is :ref:`inheritable <fd_inheritance>` by default, + or non-inheritable if *inheritable* is ``False``. Availability: Unix, Windows. + .. versionchanged:: 3.4 + Add the optional *inheritable* parameter. + .. function:: fchmod(fd, mode) @@ -848,17 +861,21 @@ as internal buffering of data. Open the file *file* and set various flags according to *flags* and possibly its mode according to *mode*. When computing *mode*, the current umask value is first masked out. Return the file descriptor for the newly opened file. + The new file descriptor is :ref:`non-inheritable <fd_inheritance>`. For a description of the flag and mode values, see the C run-time documentation; flag constants (like :const:`O_RDONLY` and :const:`O_WRONLY`) are defined in - this module too (see :ref:`open-constants`). In particular, on Windows adding + the :mod:`os` module. In particular, on Windows adding :const:`O_BINARY` is needed to open files in binary mode. This function can support :ref:`paths relative to directory descriptors - <dir_fd>`. + <dir_fd>` with the *dir_fd* parameter. Availability: Unix, Windows. + .. versionchanged:: 3.4 + The new file descriptor is now non-inheritable. + .. note:: This function is intended for low-level I/O. For normal usage, use the @@ -869,25 +886,93 @@ as internal buffering of data. .. versionadded:: 3.3 The *dir_fd* argument. +The following constants are options for the *flags* parameter to the +:func:`~os.open` function. They can be combined using the bitwise OR operator +``|``. Some of them are not available on all platforms. For descriptions of +their availability and use, consult the :manpage:`open(2)` manual page on Unix +or `the MSDN <http://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windows. + + +.. data:: O_RDONLY + O_WRONLY + O_RDWR + O_APPEND + O_CREAT + O_EXCL + O_TRUNC + + These constants are available on Unix and Windows. + + +.. data:: O_DSYNC + O_RSYNC + O_SYNC + O_NDELAY + O_NONBLOCK + O_NOCTTY + O_SHLOCK + O_EXLOCK + O_CLOEXEC + + These constants are only available on Unix. + + .. versionchanged:: 3.3 + Add :data:`O_CLOEXEC` constant. + +.. data:: O_BINARY + O_NOINHERIT + O_SHORT_LIVED + O_TEMPORARY + O_RANDOM + O_SEQUENTIAL + O_TEXT + + These constants are only available on Windows. + + +.. data:: O_ASYNC + O_DIRECT + O_DIRECTORY + O_NOFOLLOW + O_NOATIME + O_PATH + O_TMPFILE + + These constants are GNU extensions and not present if they are not defined by + the C library. + + .. versionchanged:: 3.4 + Add :data:`O_PATH` on systems that support it. + Add :data:`O_TMPFILE`, only available on Linux Kernel 3.11 + or newer. + .. function:: openpty() .. index:: module: pty - Open a new pseudo-terminal pair. Return a pair of file descriptors ``(master, - slave)`` for the pty and the tty, respectively. For a (slightly) more portable - approach, use the :mod:`pty` module. + Open a new pseudo-terminal pair. Return a pair of file descriptors + ``(master, slave)`` for the pty and the tty, respectively. The new file + descriptors are :ref:`non-inheritable <fd_inheritance>`. For a (slightly) more + portable approach, use the :mod:`pty` module. Availability: some flavors of Unix. + .. versionchanged:: 3.4 + The new file descriptors are now non-inheritable. + .. function:: pipe() - Create a pipe. Return a pair of file descriptors ``(r, w)`` usable for reading - and writing, respectively. + Create a pipe. Return a pair of file descriptors ``(r, w)`` usable for + reading and writing, respectively. The new file descriptor is + :ref:`non-inheritable <fd_inheritance>`. Availability: Unix, Windows. + .. versionchanged:: 3.4 + The new file descriptors are now non-inheritable. + .. function:: pipe2(flags) @@ -1027,7 +1112,7 @@ as internal buffering of data. sequence to hold the rest of the data. :func:`~os.readv` returns the total number of bytes read (which may be less than the total capacity of all the objects). - + Availability: Unix. .. versionadded:: 3.3 @@ -1080,80 +1165,8 @@ as internal buffering of data. sequence of :term:`bytes-like objects <bytes-like object>`. :func:`~os.writev` writes the contents of each object to the file descriptor and returns the total number of bytes written. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. _open-constants: - -``open()`` flag constants -~~~~~~~~~~~~~~~~~~~~~~~~~ - -The following constants are options for the *flags* parameter to the -:func:`~os.open` function. They can be combined using the bitwise OR operator -``|``. Some of them are not available on all platforms. For descriptions of -their availability and use, consult the :manpage:`open(2)` manual page on Unix -or `the MSDN <http://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windows. - - -.. data:: O_RDONLY - O_WRONLY - O_RDWR - O_APPEND - O_CREAT - O_EXCL - O_TRUNC - - These constants are available on Unix and Windows. - - -.. data:: O_DSYNC - O_RSYNC - O_SYNC - O_NDELAY - O_NONBLOCK - O_NOCTTY - O_SHLOCK - O_EXLOCK - O_CLOEXEC - - These constants are only available on Unix. - - .. versionchanged:: 3.3 - Add :data:`O_CLOEXEC` constant. -.. data:: O_BINARY - O_NOINHERIT - O_SHORT_LIVED - O_TEMPORARY - O_RANDOM - O_SEQUENTIAL - O_TEXT - - These constants are only available on Windows. - - -.. data:: O_ASYNC - O_DIRECT - O_DIRECTORY - O_NOFOLLOW - O_NOATIME - - These constants are GNU extensions and not present if they are not defined by - the C library. - - -.. data:: RTLD_LAZY - RTLD_NOW - RTLD_GLOBAL - RTLD_LOCAL - RTLD_NODELETE - RTLD_NOLOAD - RTLD_DEEPBIND - - See the Unix manual page :manpage:`dlopen(3)`. + Availability: Unix. .. versionadded:: 3.3 @@ -1195,6 +1208,49 @@ Querying the size of a terminal Height of the terminal window in characters. +.. _fd_inheritance: + +Inheritance of File Descriptors +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 3.4 + +A file descriptor has an "inheritable" flag which indicates if the file descriptor +can be inherited by child processes. Since Python 3.4, file descriptors +created by Python are non-inheritable by default. + +On UNIX, non-inheritable file descriptors are closed in child processes at the +execution of a new program, other file descriptors are inherited. + +On Windows, non-inheritable handles and file descriptors are closed in child +processes, except for standard streams (file descriptors 0, 1 and 2: stdin, stdout +and stderr), which are always inherited. Using :func:`spawn\* <spawnl>` functions, +all inheritable handles and all inheritable file descriptors are inherited. +Using the :mod:`subprocess` module, all file descriptors except standard +streams are closed, and inheritable handles are only inherited if the +*close_fds* parameter is ``False``. + +.. function:: get_inheritable(fd) + + Get the "inheritable" flag of the specified file descriptor (a boolean). + +.. function:: set_inheritable(fd, inheritable) + + Set the "inheritable" flag of the specified file descriptor. + +.. function:: get_handle_inheritable(handle) + + Get the "inheritable" flag of the specified handle (a boolean). + + Availability: Windows. + +.. function:: set_handle_inheritable(handle, inheritable) + + Set the "inheritable" flag of the specified handle. + + Availability: Windows. + + .. _os-file-dir: Files and Directories @@ -1551,7 +1607,7 @@ features: The *dir_fd* argument. -.. function:: makedirs(path, mode=0o777, exist_ok=False) +.. function:: makedirs(name, mode=0o777, exist_ok=False) .. index:: single: directory; creating @@ -1576,12 +1632,12 @@ features: .. versionadded:: 3.2 The *exist_ok* parameter. - .. versionchanged:: 3.3.6 + .. versionchanged:: 3.4.1 - Before Python 3.3.6, if *exist_ok* was ``True`` and the directory existed, + Before Python 3.4.1, if *exist_ok* was ``True`` and the directory existed, :func:`makedirs` would still raise an error if *mode* did not match the mode of the existing directory. Since this behavior was impossible to - implement safely, it was removed in Python 3.3.6. See :issue:`21082`. + implement safely, it was removed in Python 3.4.1. See :issue:`21082`. .. function:: mkfifo(path, mode=0o666, *, dir_fd=None) @@ -1711,7 +1767,7 @@ features: The *dir_fd* argument. -.. function:: removedirs(path) +.. function:: removedirs(name) .. index:: single: directory; deleting @@ -1935,11 +1991,26 @@ features: read-only, and if :const:`ST_NOSUID` is set, the semantics of setuid/setgid bits are disabled or not supported. + Additional module-level constants are defined for GNU/glibc based systems. + These are :const:`ST_NODEV` (disallow access to device special files), + :const:`ST_NOEXEC` (disallow program execution), :const:`ST_SYNCHRONOUS` + (writes are synced at once), :const:`ST_MANDLOCK` (allow mandatory locks on an FS), + :const:`ST_WRITE` (write on file/directory/symlink), :const:`ST_APPEND` + (append-only file), :const:`ST_IMMUTABLE` (immutable file), :const:`ST_NOATIME` + (do not update access times), :const:`ST_NODIRATIME` (do not update directory access + times), :const:`ST_RELATIME` (update atime relative to mtime/ctime). + This function can support :ref:`specifying a file descriptor <path_fd>`. .. versionchanged:: 3.2 The :const:`ST_RDONLY` and :const:`ST_NOSUID` constants were added. + .. versionchanged:: 3.4 + The :const:`ST_NODEV`, :const:`ST_NOEXEC`, :const:`ST_SYNCHRONOUS`, + :const:`ST_MANDLOCK`, :const:`ST_WRITE`, :const:`ST_APPEND`, + :const:`ST_IMMUTABLE`, :const:`ST_NOATIME`, :const:`ST_NODIRATIME`, + and :const:`ST_RELATIME` constants were added. + Availability: Unix. .. versionadded:: 3.3 @@ -2589,7 +2660,7 @@ written in Python, such as a mail server's external command delivery program. Fork a child process. Return ``0`` in the child and the child's process id in the parent. If an error occurs :exc:`OSError` is raised. - Note that some platforms including FreeBSD <= 6.3, Cygwin and OS/2 EMX have + Note that some platforms including FreeBSD <= 6.3 and Cygwin have known issues when using fork() from a thread. .. warning:: @@ -2835,7 +2906,6 @@ written in Python, such as a mail server's external command delivery program. :manpage:`times(2)` or the corresponding Windows Platform API documentation. On Windows, only :attr:`user` and :attr:`system` are known; the other attributes are zero. - On OS/2, only :attr:`elapsed` is known; the other attributes are zero. Availability: Unix, Windows. @@ -3057,7 +3127,7 @@ information, consult your Unix manpages. .. versionadded:: 3.3 -The following scheduling policies are exposed if they are a supported by the +The following scheduling policies are exposed if they are supported by the operating system. .. data:: SCHED_OTHER @@ -3166,10 +3236,6 @@ operating system. Return the set of CPUs the process with PID *pid* (or the current process if zero) is restricted to. - .. seealso:: - :func:`multiprocessing.cpu_count` returns the number of CPUs in the - system. - .. _os-path: @@ -3207,6 +3273,13 @@ Miscellaneous System Information Availability: Unix. +.. function:: cpu_count() + + Return the number of CPUs in the system. Returns None if undetermined. + + .. versionadded:: 3.4 + + .. function:: getloadavg() Return the number of processes in the system run queue averaged over the last @@ -3305,6 +3378,19 @@ Higher-level operations on pathnames are defined in the :mod:`os.path` module. The file path of the null device. For example: ``'/dev/null'`` for POSIX, ``'nul'`` for Windows. Also available via :mod:`os.path`. +.. data:: RTLD_LAZY + RTLD_NOW + RTLD_GLOBAL + RTLD_LOCAL + RTLD_NODELETE + RTLD_NOLOAD + RTLD_DEEPBIND + + Flags for use with the :func:`~sys.setdlopenflags` and + :func:`~sys.getdlopenflags` functions. See the Unix manual page + :manpage:`dlopen(3)` for what the different flags mean. + + .. versionadded:: 3.3 .. _os-miscfunc: diff --git a/Doc/library/pathlib-inheritance.png b/Doc/library/pathlib-inheritance.png Binary files differnew file mode 100644 index 0000000..b81c3de --- /dev/null +++ b/Doc/library/pathlib-inheritance.png diff --git a/Doc/library/pathlib-inheritance.svg b/Doc/library/pathlib-inheritance.svg new file mode 100644 index 0000000..9f42005 --- /dev/null +++ b/Doc/library/pathlib-inheritance.svg @@ -0,0 +1,4 @@ +<?xml version="1.0" standalone="yes"?> + +<svg version="1.1" viewBox="0.0 0.0 538.0 496.0" fill="none" stroke="none" stroke-linecap="square" stroke-miterlimit="10" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><clipPath id="p.0"><path d="m0 0l538.0 0l0 496.0l-538.0 0l0 -496.0z" clip-rule="nonzero"></path></clipPath><g clip-path="url(#p.0)"><path fill="#000000" fill-opacity="0.0" d="m0 0l538.916 0l0 496.08398l-538.916 0z" fill-rule="nonzero"></path><path fill="#ffffff" d="m176.0 24.0l187.43307 0l0 80.53543l-187.43307 0z" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m176.0 24.0l187.43307 0l0 80.53543l-187.43307 0z" fill-rule="nonzero"></path><path fill="#000000" d="m235.18527 66.877716l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm13.9296875 -2.234375l0 5.484375q0.515625 0 0.75 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-1.515625 0l0 -0.375q-0.6875 0.3125 -1.3125 0.46875q-0.625 0.171875 -1.1875 0.171875q-0.796875 0 -1.375 -0.328125q-0.578125 -0.34375 -0.90625 -0.9375q-0.25 -0.421875 -0.25 -1.046875l0 -3.453125l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.875 0l0 4.765625q0 0.5 0.234375 0.75q0.25 0.234375 0.765625 0.234375q0.484375 0 1.03125 -0.1875q0.5625 -0.203125 1.390625 -0.703125l0 -3.265625l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l2.171875 0zm5.8671875 0l0 1.0q1.015625 -0.734375 1.59375 -0.96875q0.578125 -0.25 1.09375 -0.25q0.78125 0 1.515625 0.578125q0.5 0.390625 0.5 0.796875q0 0.34375 -0.25 0.59375q-0.234375 0.234375 -0.5625 0.234375q-0.296875 0 -0.625 -0.296875q-0.328125 -0.296875 -0.59375 -0.296875q-0.328125 0 -1.0 0.421875q-0.671875 0.421875 -1.671875 1.265625l0 2.40625l2.28125 0q0.578125 0 0.828125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-4.828125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.953125 0l0 -3.890625l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l2.171875 0zm13.9609375 4.359375l-6.578125 0q0.25 0.625 0.890625 1.015625q0.640625 0.375 1.71875 0.375q0.890625 0 2.375 -0.390625q0.609375 -0.15625 0.84375 -0.15625q0.3125 0 0.53125 0.234375q0.21875 0.21875 0.21875 0.5625q0 0.3125 -0.234375 0.53125q-0.3125 0.296875 -1.53125 0.5625q-1.203125 0.265625 -2.3125 0.265625q-1.921875 0 -3.078125 -1.09375q-1.15625 -1.09375 -1.15625 -2.671875q0 -1.6875 1.25 -2.75q1.25 -1.0625 2.875 -1.0625q0.96875 0 1.78125 0.34375q0.828125 0.34375 1.21875 0.75q0.5625 0.578125 0.9375 1.421875q0.25 0.59375 0.25 1.375l0 0.6875zm-1.78125 -1.609375q-0.359375 -0.6875 -0.953125 -1.015625q-0.59375 -0.34375 -1.421875 -0.34375q-0.8125 0 -1.40625 0.34375q-0.59375 0.328125 -0.96875 1.015625l4.75 0zm6.4296875 1.09375l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm11.9765625 4.859375l0 -0.375q-0.609375 0.3125 -1.34375 0.46875q-0.71875 0.171875 -1.3125 0.171875q-1.28125 0 -2.09375 -0.6875q-0.796875 -0.6875 -0.796875 -1.515625q0 -1.0 1.015625 -1.859375q1.03125 -0.875 2.84375 -0.875q0.734375 0 1.6875 0.15625l0 -0.375q0 -0.359375 -0.3125 -0.578125q-0.3125 -0.234375 -1.171875 -0.234375q-0.71875 0 -1.84375 0.28125q-0.421875 0.09375 -0.65625 0.09375q-0.328125 0 -0.546875 -0.21875q-0.21875 -0.234375 -0.21875 -0.59375q0 -0.203125 0.078125 -0.34375q0.078125 -0.15625 0.21875 -0.25q0.140625 -0.09375 0.578125 -0.21875q0.59375 -0.15625 1.203125 -0.25q0.625 -0.109375 1.125 -0.109375q1.5 0 2.3125 0.65625q0.828125 0.640625 0.828125 1.75l0 3.296875l0.28125 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.875 0zm0 -2.875q-0.96875 -0.1875 -1.78125 -0.1875q-0.96875 0 -1.671875 0.484375q-0.4375 0.296875 -0.4375 0.609375q0 0.234375 0.203125 0.375q0.390625 0.25 1.078125 0.25q0.578125 0 1.296875 -0.21875q0.734375 -0.234375 1.3125 -0.625l0 -0.6875zm7.7578125 -2.625l0 3.21875q0 0.515625 0.21875 0.671875q0.328125 0.265625 1.171875 0.265625q1.21875 0 2.265625 -0.53125q0.390625 -0.203125 0.625 -0.203125q0.3125 0 0.53125 0.234375q0.234375 0.234375 0.234375 0.578125q0 0.3125 -0.25 0.53125q-0.375 0.375 -1.515625 0.6875q-1.125 0.3125 -1.890625 0.3125q-1.5 0 -2.25 -0.640625q-0.734375 -0.65625 -0.734375 -1.59375l0 -3.53125l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l0.578125 0l0 -1.453125q0 -0.578125 0.21875 -0.8125q0.21875 -0.25 0.578125 -0.25q0.359375 0 0.578125 0.25q0.21875 0.234375 0.21875 0.8125l0 1.453125l2.96875 0q0.578125 0 0.8125 0.21875q0.25 0.21875 0.25 0.578125q0 0.359375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-2.96875 0zm8.3515625 -4.640625l0 3.421875q0.5 -0.296875 1.0 -0.4375q0.515625 -0.15625 1.046875 -0.15625q0.828125 0 1.46875 0.28125q0.65625 0.28125 1.078125 0.890625q0.421875 0.609375 0.421875 1.53125l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.5625 0 -0.8125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.4375 0.375 -0.671875q0.203125 -0.125 0.796875 -0.125l0 -2.890625q0 -0.625 -0.28125 -0.875q-0.359375 -0.328125 -1.078125 -0.328125q-0.53125 0 -0.953125 0.203125q-0.40625 0.203125 -1.09375 0.890625l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.4375 0.375 -0.671875q0.1875 -0.125 0.796875 -0.125l0 -6.921875l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l1.875 0z" fill-rule="nonzero"></path><path fill="#ffffff" d="m16.0 144.0l187.43306 0l0 80.53543l-187.43306 0z" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m16.0 144.0l187.43306 0l0 80.53543l-187.43306 0z" fill-rule="nonzero"></path><path fill="#000000" d="m51.181374 186.87772l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm13.9296875 -2.234375l0 5.484375q0.515625 0 0.75 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-1.515625 0l0 -0.375q-0.6875 0.3125 -1.3125 0.46875q-0.625 0.171875 -1.1875 0.171875q-0.796875 0 -1.375 -0.328125q-0.578125 -0.34375 -0.90625 -0.9375q-0.25 -0.421875 -0.25 -1.046875l0 -3.453125l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.875 0l0 4.765625q0 0.5 0.234375 0.75q0.25 0.234375 0.765625 0.234375q0.484375 0 1.03125 -0.1875q0.5625 -0.203125 1.390625 -0.703125l0 -3.265625l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l2.171875 0zm5.8671875 0l0 1.0q1.015625 -0.734375 1.59375 -0.96875q0.578125 -0.25 1.09375 -0.25q0.78125 0 1.515625 0.578125q0.5 0.390625 0.5 0.796875q0 0.34375 -0.25 0.59375q-0.234375 0.234375 -0.5625 0.234375q-0.296875 0 -0.625 -0.296875q-0.328125 -0.296875 -0.59375 -0.296875q-0.328125 0 -1.0 0.421875q-0.671875 0.421875 -1.671875 1.265625l0 2.40625l2.28125 0q0.578125 0 0.828125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-4.828125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.953125 0l0 -3.890625l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l2.171875 0zm13.9609375 4.359375l-6.578125 0q0.25 0.625 0.890625 1.015625q0.640625 0.375 1.71875 0.375q0.890625 0 2.375 -0.390625q0.609375 -0.15625 0.84375 -0.15625q0.3125 0 0.53125 0.234375q0.21875 0.21875 0.21875 0.5625q0 0.3125 -0.234375 0.53125q-0.3125 0.296875 -1.53125 0.5625q-1.203125 0.265625 -2.3125 0.265625q-1.921875 0 -3.078125 -1.09375q-1.15625 -1.09375 -1.15625 -2.671875q0 -1.6875 1.25 -2.75q1.25 -1.0625 2.875 -1.0625q0.96875 0 1.78125 0.34375q0.828125 0.34375 1.21875 0.75q0.5625 0.578125 0.9375 1.421875q0.25 0.59375 0.25 1.375l0 0.6875zm-1.78125 -1.609375q-0.359375 -0.6875 -0.953125 -1.015625q-0.59375 -0.34375 -1.421875 -0.34375q-0.8125 0 -1.40625 0.34375q-0.59375 0.328125 -0.96875 1.015625l4.75 0zm6.4296875 1.09375l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm14.6953125 1.4375q0 0.921875 -0.515625 1.796875q-0.515625 0.859375 -1.53125 1.375q-1.0 0.515625 -2.109375 0.515625q-1.09375 0 -2.09375 -0.5q-1.0 -0.515625 -1.53125 -1.375q-0.515625 -0.875 -0.515625 -1.828125q0 -0.953125 0.53125 -1.875q0.53125 -0.9375 1.53125 -1.46875q1.0 -0.53125 2.078125 -0.53125q1.09375 0 2.109375 0.546875q1.015625 0.546875 1.53125 1.46875q0.515625 0.90625 0.515625 1.875zm-1.609375 0.015625q0 -0.78125 -0.546875 -1.421875q-0.765625 -0.875 -2.0 -0.875q-1.078125 0 -1.8125 0.703125q-0.71875 0.6875 -0.71875 1.59375q0 0.75 0.734375 1.40625q0.734375 0.65625 1.796875 0.65625q1.078125 0 1.8125 -0.65625q0.734375 -0.65625 0.734375 -1.40625zm8.7734375 -1.8125q-0.390625 -0.25 -0.828125 -0.359375q-0.421875 -0.125 -0.890625 -0.125q-0.921875 0 -1.46875 0.296875q-0.25 0.140625 -0.25 0.296875q0 0.171875 0.328125 0.34375q0.25 0.125 1.125 0.25q1.59375 0.21875 2.21875 0.4375q0.8125 0.28125 1.25 0.859375q0.453125 0.5625 0.453125 1.203125q0 0.859375 -0.75 1.4375q-1.09375 0.84375 -2.828125 0.84375q-0.6875 0 -1.28125 -0.125q-0.59375 -0.125 -1.078125 -0.359375q-0.125 0.09375 -0.265625 0.15625q-0.125 0.046875 -0.265625 0.046875q-0.375 0 -0.59375 -0.234375q-0.21875 -0.25 -0.21875 -0.828125l0 -0.546875q0 -0.578125 0.21875 -0.8125q0.21875 -0.25 0.578125 -0.25q0.296875 0 0.484375 0.15625q0.203125 0.15625 0.3125 0.546875q0.359375 0.3125 0.875 0.484375q0.515625 0.15625 1.1875 0.15625q1.109375 0 1.71875 -0.34375q0.28125 -0.171875 0.28125 -0.359375q0 -0.3125 -0.40625 -0.515625q-0.421875 -0.203125 -1.71875 -0.34375q-1.921875 -0.203125 -2.578125 -0.78125q-0.640625 -0.578125 -0.640625 -1.40625q0 -0.859375 0.71875 -1.4375q0.984375 -0.78125 2.578125 -0.78125q0.5625 0 1.0625 0.109375q0.515625 0.109375 0.984375 0.328125q0.15625 -0.109375 0.28125 -0.15625q0.125 -0.0625 0.234375 -0.0625q0.328125 0 0.546875 0.25q0.21875 0.234375 0.21875 0.8125l0 0.390625q0 0.53125 -0.125 0.71875q-0.25 0.359375 -0.671875 0.359375q-0.296875 0 -0.515625 -0.171875q-0.21875 -0.1875 -0.28125 -0.484375zm8.4453125 -4.921875l0 1.703125l-1.90625 0l0 -1.703125l1.90625 0zm0.21875 3.046875l0 5.484375l1.921875 0q0.578125 0 0.828125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-5.4375 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l1.921875 0l0 -3.890625l-1.296875 0q-0.5625 0 -0.8125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.234375 -0.578125q0.25 -0.21875 0.828125 -0.21875l2.890625 0zm10.007805 3.40625l2.4375 2.078125q0.4375 0.03125 0.65625 0.25q0.21875 0.21875 0.21875 0.5625q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-1.8125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.28125 0.171875 -0.5q0.1875 -0.21875 0.484375 -0.296875l-1.1875 -1.03125l-1.2187424 1.03125q0.35936737 0.09375 0.5156174 0.296875q0.171875 0.1875 0.171875 0.5q0 0.359375 -0.25 0.59375q-0.23436737 0.21875 -0.8124924 0.21875l-1.8125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.328125 0.21875 -0.546875q0.21875 -0.21875 0.65625 -0.25l2.375 -2.09375l-2.109375 -1.796875q-0.40625 -0.03125 -0.625 -0.25q-0.21875 -0.21875 -0.21875 -0.546875q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.5 0q0.578125 0 0.8124924 0.21875q0.25 0.21875 0.25 0.5625q0 0.46875 -0.43749237 0.75l0.9687424 0.8125l0.953125 -0.828125q-0.421875 -0.296875 -0.421875 -0.703125q0 -0.375 0.234375 -0.59375q0.25 -0.21875 0.828125 -0.21875l1.484375 0q0.578125 0 0.828125 0.21875q0.25 0.21875 0.25 0.578125q0 0.328125 -0.21875 0.546875q-0.21875 0.21875 -0.640625 0.25l-2.109375 1.8125zm7.4765625 0.4375l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm11.9765625 4.859375l0 -0.375q-0.609375 0.3125 -1.34375 0.46875q-0.71875 0.171875 -1.3125 0.171875q-1.28125 0 -2.09375 -0.6875q-0.796875 -0.6875 -0.796875 -1.515625q0 -1.0 1.015625 -1.859375q1.03125 -0.875 2.84375 -0.875q0.734375 0 1.6875 0.15625l0 -0.375q0 -0.359375 -0.3125 -0.578125q-0.3125 -0.234375 -1.171875 -0.234375q-0.71875 0 -1.84375 0.28125q-0.421875 0.09375 -0.65625 0.09375q-0.328125 0 -0.546875 -0.21875q-0.21875 -0.234375 -0.21875 -0.59375q0 -0.203125 0.078125 -0.34375q0.078125 -0.15625 0.21875 -0.25q0.140625 -0.09375 0.578125 -0.21875q0.59375 -0.15625 1.203125 -0.25q0.625 -0.109375 1.125 -0.109375q1.5 0 2.3125 0.65625q0.828125 0.640625 0.828125 1.75l0 3.296875l0.28125 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.875 0zm0 -2.875q-0.96875 -0.1875 -1.78125 -0.1875q-0.96875 0 -1.671875 0.484375q-0.4375 0.296875 -0.4375 0.609375q0 0.234375 0.203125 0.375q0.390625 0.25 1.078125 0.25q0.578125 0 1.296875 -0.21875q0.734375 -0.234375 1.3125 -0.625l0 -0.6875zm7.7578125 -2.625l0 3.21875q0 0.515625 0.21875 0.671875q0.328125 0.265625 1.171875 0.265625q1.21875 0 2.265625 -0.53125q0.390625 -0.203125 0.625 -0.203125q0.3125 0 0.53125 0.234375q0.234375 0.234375 0.234375 0.578125q0 0.3125 -0.25 0.53125q-0.375 0.375 -1.515625 0.6875q-1.125 0.3125 -1.890625 0.3125q-1.5 0 -2.25 -0.640625q-0.734375 -0.65625 -0.734375 -1.59375l0 -3.53125l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l0.578125 0l0 -1.453125q0 -0.578125 0.21875 -0.8125q0.21875 -0.25 0.578125 -0.25q0.359375 0 0.578125 0.25q0.21875 0.234375 0.21875 0.8125l0 1.453125l2.96875 0q0.578125 0 0.8125 0.21875q0.25 0.21875 0.25 0.578125q0 0.359375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-2.96875 0zm8.3515625 -4.640625l0 3.421875q0.5 -0.296875 1.0 -0.4375q0.515625 -0.15625 1.046875 -0.15625q0.828125 0 1.46875 0.28125q0.65625 0.28125 1.078125 0.890625q0.421875 0.609375 0.421875 1.53125l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.5625 0 -0.8125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.4375 0.375 -0.671875q0.203125 -0.125 0.796875 -0.125l0 -2.890625q0 -0.625 -0.28125 -0.875q-0.359375 -0.328125 -1.078125 -0.328125q-0.53125 0 -0.953125 0.203125q-0.40625 0.203125 -1.09375 0.890625l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.4375 0.375 -0.671875q0.1875 -0.125 0.796875 -0.125l0 -6.921875l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l1.875 0z" fill-rule="nonzero"></path><path fill="#ffffff" d="m336.0 143.46457l187.43304 0l0 80.53543l-187.43304 0z" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m336.0 143.46457l187.43304 0l0 80.53543l-187.43304 0z" fill-rule="nonzero"></path><path fill="#000000" d="m361.5798 186.34229l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm13.9296875 -2.234375l0 5.484375q0.515625 0 0.75 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-1.515625 0l0 -0.375q-0.6875 0.3125 -1.3125 0.46875q-0.625 0.171875 -1.1875 0.171875q-0.796875 0 -1.375 -0.328125q-0.578125 -0.34375 -0.90625 -0.9375q-0.25 -0.421875 -0.25 -1.046875l0 -3.453125l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.875 0l0 4.765625q0 0.5 0.234375 0.75q0.25 0.234375 0.765625 0.234375q0.484375 0 1.03125 -0.1875q0.5625 -0.203125 1.390625 -0.703125l0 -3.265625l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l2.171875 0zm5.8671875 0l0 1.0q1.015625 -0.734375 1.59375 -0.96875q0.578125 -0.25 1.09375 -0.25q0.78125 0 1.515625 0.578125q0.5 0.390625 0.5 0.796875q0 0.34375 -0.25 0.59375q-0.234375 0.234375 -0.5625 0.234375q-0.296875 0 -0.625 -0.296875q-0.328125 -0.296875 -0.59375 -0.296875q-0.328125 0 -1.0 0.421875q-0.671875 0.421875 -1.671875 1.265625l0 2.40625l2.28125 0q0.578125 0 0.828125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-4.828125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.953125 0l0 -3.890625l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l2.171875 0zm13.9609375 4.359375l-6.578125 0q0.25 0.625 0.890625 1.015625q0.640625 0.375 1.71875 0.375q0.890625 0 2.375 -0.390625q0.609375 -0.15625 0.84375 -0.15625q0.3125 0 0.53125 0.234375q0.21875 0.21875 0.21875 0.5625q0 0.3125 -0.234375 0.53125q-0.3125 0.296875 -1.53125 0.5625q-1.203125 0.265625 -2.3125 0.265625q-1.921875 0 -3.078125 -1.09375q-1.15625 -1.09375 -1.15625 -2.671875q0 -1.6875 1.25 -2.75q1.25 -1.0625 2.875 -1.0625q0.96875 0 1.78125 0.34375q0.828125 0.34375 1.21875 0.75q0.5625 0.578125 0.9375 1.421875q0.25 0.59375 0.25 1.375l0 0.6875zm-1.78125 -1.609375q-0.359375 -0.6875 -0.953125 -1.015625q-0.59375 -0.34375 -1.421875 -0.34375q-0.8125 0 -1.40625 0.34375q-0.59375 0.328125 -0.96875 1.015625l4.75 0zm7.3671875 -0.203125l-1.484375 4.546875l-1.796875 0l-0.953125 -7.875q-0.375 -0.046875 -0.5625 -0.25q-0.1875 -0.203125 -0.1875 -0.53125q0 -0.375 0.25 -0.59375q0.25 -0.234375 0.828125 -0.234375l2.125 0.015625q0.578125 0 0.828125 0.21875q0.25 0.21875 0.25 0.578125q0 0.359375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-0.828125 0l0.53125 4.484375l1.25 -3.734375l1.671875 0l1.25 3.734375l0.53125 -4.484375l-0.84375 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.21875 -0.234375 -0.578125q0 -0.359375 0.234375 -0.578125q0.25 -0.234375 0.828125 -0.234375l2.125 0.015625q0.578125 0 0.828125 0.21875q0.25 0.21875 0.25 0.578125q0 0.296875 -0.203125 0.515625q-0.1875 0.21875 -0.5625 0.28125l-0.921875 7.875l-1.765625 0l-1.53125 -4.546875zm10.1640625 -5.59375l0 1.703125l-1.90625 0l0 -1.703125l1.90625 0zm0.21875 3.046875l0 5.484375l1.921875 0q0.578125 0 0.828125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-5.4375 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l1.921875 0l0 -3.890625l-1.296875 0q-0.5625 0 -0.8125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.234375 -0.578125q0.25 -0.21875 0.828125 -0.21875l2.890625 0zm7.1953125 0l0 0.53125q0.4375 -0.375 0.953125 -0.5625q0.53125 -0.1875 1.15625 -0.1875q1.421875 0 2.25 0.890625q0.65625 0.703125 0.65625 1.84375l0 2.96875q0.5 0 0.734375 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.453125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.34375 0.234375 -0.5625q0.25 -0.234375 0.75 -0.234375l0 -3.015625q0 -0.53125 -0.28125 -0.765625q-0.359375 -0.3125 -1.09375 -0.3125q-0.5625 0 -0.984375 0.21875q-0.40625 0.203125 -1.046875 0.90625l0 2.96875q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.4375 0.375 -0.671875q0.1875 -0.125 0.796875 -0.125l0 -3.890625q-0.5 0 -0.75 -0.21875q-0.234375 -0.234375 -0.234375 -0.578125q0 -0.359375 0.234375 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.53125 0zm14.8984375 -3.046875l0 8.53125l0.265625 0q0.578125 0 0.828125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-1.875 0l0 -0.390625q-0.546875 0.3125 -1.140625 0.484375q-0.59375 0.171875 -1.234375 0.171875q-1.8125 0 -2.921875 -1.046875q-1.09375 -1.046875 -1.09375 -2.609375q0 -1.625 1.15625 -2.765625q1.15625 -1.15625 2.828125 -1.15625q0.625 0 1.21875 0.203125q0.609375 0.1875 1.1875 0.5625l0 -1.984375l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l1.875 0zm-1.609375 6.796875q0 -0.984375 -0.703125 -1.671875q-0.6875 -0.6875 -1.6875 -0.6875q-1.0 0 -1.703125 0.6875q-0.6875 0.6875 -0.6875 1.65625q0 0.875 0.625 1.453125q0.625 0.5625 1.765625 0.5625q1.125 0 1.75 -0.5625q0.640625 -0.578125 0.640625 -1.4375zm11.6953125 -0.078125q0 0.921875 -0.515625 1.796875q-0.515625 0.859375 -1.53125 1.375q-1.0 0.515625 -2.109375 0.515625q-1.09375 0 -2.09375 -0.5q-1.0 -0.515625 -1.53125 -1.375q-0.515625 -0.875 -0.515625 -1.828125q0 -0.953125 0.53125 -1.875q0.53125 -0.9375 1.53125 -1.46875q1.0 -0.53125 2.078125 -0.53125q1.09375 0 2.109375 0.546875q1.015625 0.546875 1.53125 1.46875q0.515625 0.90625 0.515625 1.875zm-1.609375 0.015625q0 -0.78125 -0.546875 -1.421875q-0.765625 -0.875 -2.0 -0.875q-1.078125 0 -1.8125 0.703125q-0.71875 0.6875 -0.71875 1.59375q0 0.75 0.734375 1.40625q0.734375 0.65625 1.796875 0.65625q1.078125 0 1.8125 -0.65625q0.734375 -0.65625 0.734375 -1.40625zm7.0546875 0.453125l-1.109375 2.953125l-1.5 0l-1.34375 -5.5q-0.4375 -0.015625 -0.671875 -0.234375q-0.21875 -0.234375 -0.21875 -0.5625q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.484375 0q0.578125 0 0.828125 0.21875q0.25 0.21875 0.25 0.578125q0 0.359375 -0.28125 0.609375q-0.21875 0.1875 -0.828125 0.1875l0.609375 2.546875l0.984375 -2.609375l1.421875 0l1.0 2.609375l0.625 -2.546875q-0.59375 0 -0.78125 -0.109375q-0.375 -0.25 -0.375 -0.6875q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.5 0q0.578125 0 0.828125 0.21875q0.25 0.21875 0.25 0.578125q0 0.328125 -0.21875 0.546875q-0.21875 0.21875 -0.640625 0.25l-1.3125 5.5l-1.484375 0l-1.171875 -2.953125zm11.3203125 -2.265625q-0.390625 -0.25 -0.828125 -0.359375q-0.421875 -0.125 -0.890625 -0.125q-0.921875 0 -1.46875 0.296875q-0.25 0.140625 -0.25 0.296875q0 0.171875 0.328125 0.34375q0.25 0.125 1.125 0.25q1.59375 0.21875 2.21875 0.4375q0.8125 0.28125 1.25 0.859375q0.453125 0.5625 0.453125 1.203125q0 0.859375 -0.75 1.4375q-1.09375 0.84375 -2.828125 0.84375q-0.6875 0 -1.28125 -0.125q-0.59375 -0.125 -1.078125 -0.359375q-0.125 0.09375 -0.265625 0.15625q-0.125 0.046875 -0.265625 0.046875q-0.375 0 -0.59375 -0.234375q-0.21875 -0.25 -0.21875 -0.828125l0 -0.546875q0 -0.578125 0.21875 -0.8125q0.21875 -0.25 0.578125 -0.25q0.296875 0 0.484375 0.15625q0.203125 0.15625 0.3125 0.546875q0.359375 0.3125 0.875 0.484375q0.515625 0.15625 1.1875 0.15625q1.109375 0 1.71875 -0.34375q0.28125 -0.171875 0.28125 -0.359375q0 -0.3125 -0.40625 -0.515625q-0.421875 -0.203125 -1.71875 -0.34375q-1.921875 -0.203125 -2.578125 -0.78125q-0.640625 -0.578125 -0.640625 -1.40625q0 -0.859375 0.71875 -1.4375q0.984375 -0.78125 2.578125 -0.78125q0.5625 0 1.0625 0.109375q0.515625 0.109375 0.984375 0.328125q0.15625 -0.109375 0.28125 -0.15625q0.125 -0.0625 0.234375 -0.0625q0.328125 0 0.546875 0.25q0.21875 0.234375 0.21875 0.8125l0 0.390625q0 0.53125 -0.125 0.71875q-0.25 0.359375 -0.671875 0.359375q-0.296875 0 -0.515625 -0.171875q-0.21875 -0.1875 -0.28125 -0.484375zm6.9453125 1.96875l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm11.9765625 4.859375l0 -0.375q-0.609375 0.3125 -1.34375 0.46875q-0.71875 0.171875 -1.3125 0.171875q-1.28125 0 -2.09375 -0.6875q-0.796875 -0.6875 -0.796875 -1.515625q0 -1.0 1.015625 -1.859375q1.03125 -0.875 2.84375 -0.875q0.734375 0 1.6875 0.15625l0 -0.375q0 -0.359375 -0.3125 -0.578125q-0.3125 -0.234375 -1.171875 -0.234375q-0.71875 0 -1.84375 0.28125q-0.421875 0.09375 -0.65625 0.09375q-0.328125 0 -0.546875 -0.21875q-0.21875 -0.234375 -0.21875 -0.59375q0 -0.203125 0.078125 -0.34375q0.078125 -0.15625 0.21875 -0.25q0.140625 -0.09375 0.578125 -0.21875q0.59375 -0.15625 1.203125 -0.25q0.625 -0.109375 1.125 -0.109375q1.5 0 2.3125 0.65625q0.828125 0.640625 0.828125 1.75l0 3.296875l0.28125 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.875 0zm0 -2.875q-0.96875 -0.1875 -1.78125 -0.1875q-0.96875 0 -1.671875 0.484375q-0.4375 0.296875 -0.4375 0.609375q0 0.234375 0.203125 0.375q0.390625 0.25 1.078125 0.25q0.578125 0 1.296875 -0.21875q0.734375 -0.234375 1.3125 -0.625l0 -0.6875zm7.7578125 -2.625l0 3.21875q0 0.515625 0.21875 0.671875q0.328125 0.265625 1.171875 0.265625q1.21875 0 2.265625 -0.53125q0.390625 -0.203125 0.625 -0.203125q0.3125 0 0.53125 0.234375q0.234375 0.234375 0.234375 0.578125q0 0.3125 -0.25 0.53125q-0.375 0.375 -1.515625 0.6875q-1.125 0.3125 -1.890625 0.3125q-1.5 0 -2.25 -0.640625q-0.734375 -0.65625 -0.734375 -1.59375l0 -3.53125l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l0.578125 0l0 -1.453125q0 -0.578125 0.21875 -0.8125q0.21875 -0.25 0.578125 -0.25q0.359375 0 0.578125 0.25q0.21875 0.234375 0.21875 0.8125l0 1.453125l2.96875 0q0.578125 0 0.8125 0.21875q0.25 0.21875 0.25 0.578125q0 0.359375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-2.96875 0zm8.3515625 -4.640625l0 3.421875q0.5 -0.296875 1.0 -0.4375q0.515625 -0.15625 1.046875 -0.15625q0.828125 0 1.46875 0.28125q0.65625 0.28125 1.078125 0.890625q0.421875 0.609375 0.421875 1.53125l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.5625 0 -0.8125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.4375 0.375 -0.671875q0.203125 -0.125 0.796875 -0.125l0 -2.890625q0 -0.625 -0.28125 -0.875q-0.359375 -0.328125 -1.078125 -0.328125q-0.53125 0 -0.953125 0.203125q-0.40625 0.203125 -1.09375 0.890625l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.4375 0.375 -0.671875q0.1875 -0.125 0.796875 -0.125l0 -6.921875l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l1.875 0z" fill-rule="nonzero"></path><path fill="#ffffff" d="m176.0 271.46457l187.43307 0l0 80.53543l-187.43307 0z" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m176.0 271.46457l187.43307 0l0 80.53543l-187.43307 0z" fill-rule="nonzero"></path><path fill="#000000" d="m254.3884 314.3423l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm11.9765625 4.859375l0 -0.375q-0.609375 0.3125 -1.34375 0.46875q-0.71875 0.171875 -1.3125 0.171875q-1.28125 0 -2.09375 -0.6875q-0.796875 -0.6875 -0.796875 -1.515625q0 -1.0 1.015625 -1.859375q1.03125 -0.875 2.84375 -0.875q0.734375 0 1.6875 0.15625l0 -0.375q0 -0.359375 -0.3125 -0.578125q-0.3125 -0.234375 -1.171875 -0.234375q-0.71875 0 -1.84375 0.28125q-0.421875 0.09375 -0.65625 0.09375q-0.328125 0 -0.546875 -0.21875q-0.21875 -0.234375 -0.21875 -0.59375q0 -0.203125 0.078125 -0.34375q0.078125 -0.15625 0.21875 -0.25q0.140625 -0.09375 0.578125 -0.21875q0.59375 -0.15625 1.203125 -0.25q0.625 -0.109375 1.125 -0.109375q1.5 0 2.3125 0.65625q0.828125 0.640625 0.828125 1.75l0 3.296875l0.28125 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.875 0zm0 -2.875q-0.96875 -0.1875 -1.78125 -0.1875q-0.96875 0 -1.671875 0.484375q-0.4375 0.296875 -0.4375 0.609375q0 0.234375 0.203125 0.375q0.390625 0.25 1.078125 0.25q0.578125 0 1.296875 -0.21875q0.734375 -0.234375 1.3125 -0.625l0 -0.6875zm7.7578125 -2.625l0 3.21875q0 0.515625 0.21875 0.671875q0.328125 0.265625 1.171875 0.265625q1.21875 0 2.265625 -0.53125q0.390625 -0.203125 0.625 -0.203125q0.3125 0 0.53125 0.234375q0.234375 0.234375 0.234375 0.578125q0 0.3125 -0.25 0.53125q-0.375 0.375 -1.515625 0.6875q-1.125 0.3125 -1.890625 0.3125q-1.5 0 -2.25 -0.640625q-0.734375 -0.65625 -0.734375 -1.59375l0 -3.53125l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l0.578125 0l0 -1.453125q0 -0.578125 0.21875 -0.8125q0.21875 -0.25 0.578125 -0.25q0.359375 0 0.578125 0.25q0.21875 0.234375 0.21875 0.8125l0 1.453125l2.96875 0q0.578125 0 0.8125 0.21875q0.25 0.21875 0.25 0.578125q0 0.359375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-2.96875 0zm8.3515625 -4.640625l0 3.421875q0.5 -0.296875 1.0 -0.4375q0.515625 -0.15625 1.046875 -0.15625q0.828125 0 1.46875 0.28125q0.65625 0.28125 1.078125 0.890625q0.421875 0.609375 0.421875 1.53125l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.5625 0 -0.8125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.4375 0.375 -0.671875q0.203125 -0.125 0.796875 -0.125l0 -2.890625q0 -0.625 -0.28125 -0.875q-0.359375 -0.328125 -1.078125 -0.328125q-0.53125 0 -0.953125 0.203125q-0.40625 0.203125 -1.09375 0.890625l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.4375 0.375 -0.671875q0.1875 -0.125 0.796875 -0.125l0 -6.921875l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l1.875 0z" fill-rule="nonzero"></path><path fill="#ffffff" d="m16.0 400.0l187.43306 0l0 80.53543l-187.43306 0z" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m16.0 400.0l187.43306 0l0 80.53543l-187.43306 0z" fill-rule="nonzero"></path><path fill="#000000" d="m70.3845 442.87772l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm14.6953125 1.4375q0 0.921875 -0.515625 1.796875q-0.515625 0.859375 -1.53125 1.375q-1.0 0.515625 -2.109375 0.515625q-1.09375 0 -2.09375 -0.5q-1.0 -0.515625 -1.53125 -1.375q-0.515625 -0.875 -0.515625 -1.828125q0 -0.953125 0.53125 -1.875q0.53125 -0.9375 1.53125 -1.46875q1.0 -0.53125 2.078125 -0.53125q1.09375 0 2.109375 0.546875q1.015625 0.546875 1.53125 1.46875q0.515625 0.90625 0.515625 1.875zm-1.609375 0.015625q0 -0.78125 -0.546875 -1.421875q-0.765625 -0.875 -2.0 -0.875q-1.078125 0 -1.8125 0.703125q-0.71875 0.6875 -0.71875 1.59375q0 0.75 0.734375 1.40625q0.734375 0.65625 1.796875 0.65625q1.078125 0 1.8125 -0.65625q0.734375 -0.65625 0.734375 -1.40625zm8.7734375 -1.8125q-0.390625 -0.25 -0.828125 -0.359375q-0.421875 -0.125 -0.890625 -0.125q-0.921875 0 -1.46875 0.296875q-0.25 0.140625 -0.25 0.296875q0 0.171875 0.328125 0.34375q0.25 0.125 1.125 0.25q1.59375 0.21875 2.21875 0.4375q0.8125 0.28125 1.25 0.859375q0.453125 0.5625 0.453125 1.203125q0 0.859375 -0.75 1.4375q-1.09375 0.84375 -2.828125 0.84375q-0.6875 0 -1.28125 -0.125q-0.59375 -0.125 -1.078125 -0.359375q-0.125 0.09375 -0.265625 0.15625q-0.125 0.046875 -0.265625 0.046875q-0.375 0 -0.59375 -0.234375q-0.21875 -0.25 -0.21875 -0.828125l0 -0.546875q0 -0.578125 0.21875 -0.8125q0.21875 -0.25 0.578125 -0.25q0.296875 0 0.484375 0.15625q0.203125 0.15625 0.3125 0.546875q0.359375 0.3125 0.875 0.484375q0.515625 0.15625 1.1875 0.15625q1.109375 0 1.71875 -0.34375q0.28125 -0.171875 0.28125 -0.359375q0 -0.3125 -0.40625 -0.515625q-0.421875 -0.203125 -1.71875 -0.34375q-1.921875 -0.203125 -2.578125 -0.78125q-0.640625 -0.578125 -0.640625 -1.40625q0 -0.859375 0.71875 -1.4375q0.984375 -0.78125 2.578125 -0.78125q0.5625 0 1.0625 0.109375q0.515625 0.109375 0.984375 0.328125q0.15625 -0.109375 0.28125 -0.15625q0.125 -0.0625 0.234375 -0.0625q0.328125 0 0.546875 0.25q0.21875 0.234375 0.21875 0.8125l0 0.390625q0 0.53125 -0.125 0.71875q-0.25 0.359375 -0.671875 0.359375q-0.296875 0 -0.515625 -0.171875q-0.21875 -0.1875 -0.28125 -0.484375zm8.4453125 -4.921875l0 1.703125l-1.90625 0l0 -1.703125l1.90625 0zm0.21875 3.046875l0 5.484375l1.921875 0q0.578125 0 0.828125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-5.4375 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l1.921875 0l0 -3.890625l-1.296875 0q-0.5625 0 -0.8125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.234375 -0.578125q0.25 -0.21875 0.828125 -0.21875l2.890625 0zm10.0078125 3.40625l2.4375 2.078125q0.4375 0.03125 0.65625 0.25q0.21875 0.21875 0.21875 0.5625q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-1.8125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.28125 0.171875 -0.5q0.1875 -0.21875 0.484375 -0.296875l-1.1875 -1.03125l-1.21875 1.03125q0.359375 0.09375 0.515625 0.296875q0.171875 0.1875 0.171875 0.5q0 0.359375 -0.25 0.59375q-0.234375 0.21875 -0.8125 0.21875l-1.8125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.328125 0.21875 -0.546875q0.21875 -0.21875 0.65625 -0.25l2.375 -2.09375l-2.109375 -1.796875q-0.40625 -0.03125 -0.625 -0.25q-0.21875 -0.21875 -0.21875 -0.546875q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.5 0q0.578125 0 0.8125 0.21875q0.25 0.21875 0.25 0.5625q0 0.46875 -0.4375 0.75l0.96875 0.8125l0.953125 -0.828125q-0.421875 -0.296875 -0.421875 -0.703125q0 -0.375 0.234375 -0.59375q0.25 -0.21875 0.828125 -0.21875l1.484375 0q0.578125 0 0.828125 0.21875q0.25 0.21875 0.25 0.578125q0 0.328125 -0.21875 0.546875q-0.21875 0.21875 -0.640625 0.25l-2.109375 1.8125zm7.4765625 0.4375l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm11.976555 4.859375l0 -0.375q-0.609375 0.3125 -1.34375 0.46875q-0.71875 0.171875 -1.3124924 0.171875q-1.28125 0 -2.09375 -0.6875q-0.796875 -0.6875 -0.796875 -1.515625q0 -1.0 1.015625 -1.859375q1.03125 -0.875 2.8437424 -0.875q0.734375 0 1.6875 0.15625l0 -0.375q0 -0.359375 -0.3125 -0.578125q-0.3125 -0.234375 -1.171875 -0.234375q-0.71875 0 -1.8437424 0.28125q-0.421875 0.09375 -0.65625 0.09375q-0.328125 0 -0.546875 -0.21875q-0.21875 -0.234375 -0.21875 -0.59375q0 -0.203125 0.078125 -0.34375q0.078125 -0.15625 0.21875 -0.25q0.140625 -0.09375 0.578125 -0.21875q0.59375 -0.15625 1.203125 -0.25q0.6249924 -0.109375 1.1249924 -0.109375q1.5 0 2.3125 0.65625q0.828125 0.640625 0.828125 1.75l0 3.296875l0.28125 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.875 0zm0 -2.875q-0.96875 -0.1875 -1.78125 -0.1875q-0.9687424 0 -1.6718674 0.484375q-0.4375 0.296875 -0.4375 0.609375q0 0.234375 0.203125 0.375q0.390625 0.25 1.078125 0.25q0.5781174 0 1.2968674 -0.21875q0.734375 -0.234375 1.3125 -0.625l0 -0.6875zm7.7578125 -2.625l0 3.21875q0 0.515625 0.21875 0.671875q0.328125 0.265625 1.171875 0.265625q1.21875 0 2.265625 -0.53125q0.390625 -0.203125 0.625 -0.203125q0.3125 0 0.53125 0.234375q0.234375 0.234375 0.234375 0.578125q0 0.3125 -0.25 0.53125q-0.375 0.375 -1.515625 0.6875q-1.125 0.3125 -1.890625 0.3125q-1.5 0 -2.25 -0.640625q-0.734375 -0.65625 -0.734375 -1.59375l0 -3.53125l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l0.578125 0l0 -1.453125q0 -0.578125 0.21875 -0.8125q0.21875 -0.25 0.578125 -0.25q0.359375 0 0.578125 0.25q0.21875 0.234375 0.21875 0.8125l0 1.453125l2.96875 0q0.578125 0 0.8125 0.21875q0.25 0.21875 0.25 0.578125q0 0.359375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-2.96875 0zm8.3515625 -4.640625l0 3.421875q0.5 -0.296875 1.0 -0.4375q0.515625 -0.15625 1.046875 -0.15625q0.828125 0 1.46875 0.28125q0.65625 0.28125 1.078125 0.890625q0.421875 0.609375 0.421875 1.53125l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.5625 0 -0.8125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.4375 0.375 -0.671875q0.203125 -0.125 0.796875 -0.125l0 -2.890625q0 -0.625 -0.28125 -0.875q-0.359375 -0.328125 -1.078125 -0.328125q-0.53125 0 -0.953125 0.203125q-0.40625 0.203125 -1.09375 0.890625l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.4375 0.375 -0.671875q0.1875 -0.125 0.796875 -0.125l0 -6.921875l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l1.875 0z" fill-rule="nonzero"></path><path fill="#ffffff" d="m336.0 400.0l187.43304 0l0 80.53543l-187.43304 0z" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m336.0 400.0l187.43304 0l0 80.53543l-187.43304 0z" fill-rule="nonzero"></path><path fill="#000000" d="m381.72043 441.58084l-1.484375 4.546875l-1.796875 0l-0.953125 -7.875q-0.375 -0.046875 -0.5625 -0.25q-0.1875 -0.203125 -0.1875 -0.53125q0 -0.375 0.25 -0.59375q0.25 -0.234375 0.828125 -0.234375l2.125 0.015625q0.578125 0 0.828125 0.21875q0.25 0.21875 0.25 0.578125q0 0.359375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-0.828125 0l0.53125 4.484375l1.25 -3.734375l1.671875 0l1.25 3.734375l0.53125 -4.484375l-0.84375 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.21875 -0.234375 -0.578125q0 -0.359375 0.234375 -0.578125q0.25 -0.234375 0.828125 -0.234375l2.125 0.015625q0.578125 0 0.828125 0.21875q0.25 0.21875 0.25 0.578125q0 0.296875 -0.203125 0.515625q-0.1875 0.21875 -0.5625 0.28125l-0.921875 7.875l-1.765625 0l-1.53125 -4.546875zm10.1640625 -5.59375l0 1.703125l-1.90625 0l0 -1.703125l1.90625 0zm0.21875 3.046875l0 5.484375l1.921875 0q0.578125 0 0.828125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-5.4375 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l1.921875 0l0 -3.890625l-1.296875 0q-0.5625 0 -0.8125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.234375 -0.578125q0.25 -0.21875 0.828125 -0.21875l2.890625 0zm7.1953125 0l0 0.53125q0.4375 -0.375 0.953125 -0.5625q0.53125 -0.1875 1.15625 -0.1875q1.421875 0 2.25 0.890625q0.65625 0.703125 0.65625 1.84375l0 2.96875q0.5 0 0.734375 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.453125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.34375 0.234375 -0.5625q0.25 -0.234375 0.75 -0.234375l0 -3.015625q0 -0.53125 -0.28125 -0.765625q-0.359375 -0.3125 -1.09375 -0.3125q-0.5625 0 -0.984375 0.21875q-0.40625 0.203125 -1.046875 0.90625l0 2.96875q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.4375 0.375 -0.671875q0.1875 -0.125 0.796875 -0.125l0 -3.890625q-0.5 0 -0.75 -0.21875q-0.234375 -0.234375 -0.234375 -0.578125q0 -0.359375 0.234375 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.53125 0zm14.8984375 -3.046875l0 8.53125l0.265625 0q0.578125 0 0.828125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-1.875 0l0 -0.390625q-0.546875 0.3125 -1.140625 0.484375q-0.59375 0.171875 -1.234375 0.171875q-1.8125 0 -2.921875 -1.046875q-1.09375 -1.046875 -1.09375 -2.609375q0 -1.625 1.15625 -2.765625q1.15625 -1.15625 2.828125 -1.15625q0.625 0 1.21875 0.203125q0.609375 0.1875 1.1875 0.5625l0 -1.984375l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l1.875 0zm-1.609375 6.796875q0 -0.984375 -0.703125 -1.671875q-0.6875 -0.6875 -1.6875 -0.6875q-1.0 0 -1.703125 0.6875q-0.6875 0.6875 -0.6875 1.65625q0 0.875 0.625 1.453125q0.625 0.5625 1.765625 0.5625q1.125 0 1.75 -0.5625q0.640625 -0.578125 0.640625 -1.4375zm11.6953125 -0.078125q0 0.921875 -0.515625 1.796875q-0.515625 0.859375 -1.53125 1.375q-1.0 0.515625 -2.109375 0.515625q-1.09375 0 -2.09375 -0.5q-1.0 -0.515625 -1.53125 -1.375q-0.515625 -0.875 -0.515625 -1.828125q0 -0.953125 0.53125 -1.875q0.53125 -0.9375 1.53125 -1.46875q1.0 -0.53125 2.078125 -0.53125q1.09375 0 2.109375 0.546875q1.015625 0.546875 1.53125 1.46875q0.515625 0.90625 0.515625 1.875zm-1.609375 0.015625q0 -0.78125 -0.546875 -1.421875q-0.765625 -0.875 -2.0 -0.875q-1.078125 0 -1.8125 0.703125q-0.71875 0.6875 -0.71875 1.59375q0 0.75 0.734375 1.40625q0.734375 0.65625 1.796875 0.65625q1.078125 0 1.8125 -0.65625q0.734375 -0.65625 0.734375 -1.40625zm7.0546875 0.453125l-1.109375 2.953125l-1.5 0l-1.34375 -5.5q-0.4375 -0.015625 -0.671875 -0.234375q-0.21875 -0.234375 -0.21875 -0.5625q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.484375 0q0.578125 0 0.828125 0.21875q0.25 0.21875 0.25 0.578125q0 0.359375 -0.28125 0.609375q-0.21875 0.1875 -0.828125 0.1875l0.609375 2.546875l0.984375 -2.609375l1.421875 0l1.0 2.609375l0.625 -2.546875q-0.59375 0 -0.78125 -0.109375q-0.375 -0.25 -0.375 -0.6875q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.5 0q0.578125 0 0.828125 0.21875q0.25 0.21875 0.25 0.578125q0 0.328125 -0.21875 0.546875q-0.21875 0.21875 -0.640625 0.25l-1.3125 5.5l-1.484375 0l-1.171875 -2.953125zm11.3203125 -2.265625q-0.390625 -0.25 -0.828125 -0.359375q-0.421875 -0.125 -0.890625 -0.125q-0.921875 0 -1.46875 0.296875q-0.25 0.140625 -0.25 0.296875q0 0.171875 0.328125 0.34375q0.25 0.125 1.125 0.25q1.59375 0.21875 2.21875 0.4375q0.8125 0.28125 1.25 0.859375q0.453125 0.5625 0.453125 1.203125q0 0.859375 -0.75 1.4375q-1.09375 0.84375 -2.828125 0.84375q-0.6875 0 -1.28125 -0.125q-0.59375 -0.125 -1.078125 -0.359375q-0.125 0.09375 -0.265625 0.15625q-0.125 0.046875 -0.265625 0.046875q-0.375 0 -0.59375 -0.234375q-0.21875 -0.25 -0.21875 -0.828125l0 -0.546875q0 -0.578125 0.21875 -0.8125q0.21875 -0.25 0.578125 -0.25q0.296875 0 0.484375 0.15625q0.203125 0.15625 0.3125 0.546875q0.359375 0.3125 0.875 0.484375q0.515625 0.15625 1.1875 0.15625q1.109375 0 1.71875 -0.34375q0.28125 -0.171875 0.28125 -0.359375q0 -0.3125 -0.40625 -0.515625q-0.421875 -0.203125 -1.71875 -0.34375q-1.921875 -0.203125 -2.578125 -0.78125q-0.640625 -0.578125 -0.640625 -1.40625q0 -0.859375 0.71875 -1.4375q0.984375 -0.78125 2.578125 -0.78125q0.5625 0 1.0625 0.109375q0.515625 0.109375 0.984375 0.328125q0.15625 -0.109375 0.28125 -0.15625q0.125 -0.0625 0.234375 -0.0625q0.328125 0 0.546875 0.25q0.21875 0.234375 0.21875 0.8125l0 0.390625q0 0.53125 -0.125 0.71875q-0.25 0.359375 -0.671875 0.359375q-0.296875 0 -0.515625 -0.171875q-0.21875 -0.1875 -0.28125 -0.484375zm6.9453125 1.96875l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm11.9765625 4.859375l0 -0.375q-0.609375 0.3125 -1.34375 0.46875q-0.71875 0.171875 -1.3125 0.171875q-1.28125 0 -2.09375 -0.6875q-0.796875 -0.6875 -0.796875 -1.515625q0 -1.0 1.015625 -1.859375q1.03125 -0.875 2.84375 -0.875q0.734375 0 1.6875 0.15625l0 -0.375q0 -0.359375 -0.3125 -0.578125q-0.3125 -0.234375 -1.171875 -0.234375q-0.71875 0 -1.84375 0.28125q-0.421875 0.09375 -0.65625 0.09375q-0.328125 0 -0.546875 -0.21875q-0.21875 -0.234375 -0.21875 -0.59375q0 -0.203125 0.078125 -0.34375q0.078125 -0.15625 0.21875 -0.25q0.140625 -0.09375 0.578125 -0.21875q0.59375 -0.15625 1.203125 -0.25q0.625 -0.109375 1.125 -0.109375q1.5 0 2.3125 0.65625q0.828125 0.640625 0.828125 1.75l0 3.296875l0.28125 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.875 0zm0 -2.875q-0.96875 -0.1875 -1.78125 -0.1875q-0.96875 0 -1.671875 0.484375q-0.4375 0.296875 -0.4375 0.609375q0 0.234375 0.203125 0.375q0.390625 0.25 1.078125 0.25q0.578125 0 1.296875 -0.21875q0.734375 -0.234375 1.3125 -0.625l0 -0.6875zm7.7578125 -2.625l0 3.21875q0 0.515625 0.21875 0.671875q0.328125 0.265625 1.171875 0.265625q1.21875 0 2.265625 -0.53125q0.390625 -0.203125 0.625 -0.203125q0.3125 0 0.53125 0.234375q0.234375 0.234375 0.234375 0.578125q0 0.3125 -0.25 0.53125q-0.375 0.375 -1.515625 0.6875q-1.125 0.3125 -1.890625 0.3125q-1.5 0 -2.25 -0.640625q-0.734375 -0.65625 -0.734375 -1.59375l0 -3.53125l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l0.578125 0l0 -1.453125q0 -0.578125 0.21875 -0.8125q0.21875 -0.25 0.578125 -0.25q0.359375 0 0.578125 0.25q0.21875 0.234375 0.21875 0.8125l0 1.453125l2.96875 0q0.578125 0 0.8125 0.21875q0.25 0.21875 0.25 0.578125q0 0.359375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-2.96875 0zm8.3515625 -4.640625l0 3.421875q0.5 -0.296875 1.0 -0.4375q0.515625 -0.15625 1.046875 -0.15625q0.828125 0 1.46875 0.28125q0.65625 0.28125 1.078125 0.890625q0.421875 0.609375 0.421875 1.53125l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.5625 0 -0.8125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.4375 0.375 -0.671875q0.203125 -0.125 0.796875 -0.125l0 -2.890625q0 -0.625 -0.28125 -0.875q-0.359375 -0.328125 -1.078125 -0.328125q-0.53125 0 -0.953125 0.203125q-0.40625 0.203125 -1.09375 0.890625l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.4375 0.375 -0.671875q0.1875 -0.125 0.796875 -0.125l0 -6.921875l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l1.875 0z" fill-rule="nonzero"></path><path fill="#000000" fill-opacity="0.0" d="m96.0 144.0l0 -80.0l80.0 0" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m96.0 144.0l0 -80.0l68.0 0" fill-rule="evenodd"></path><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" d="m164.0 67.30347l9.076202 -3.3034668l-9.076202 -3.303463z" fill-rule="evenodd"></path><path fill="#000000" fill-opacity="0.0" d="m431.5013 143.08398l0 -78.99999l-68.0 0" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m431.5013 143.08398l0 -78.99999l-56.0 0" fill-rule="evenodd"></path><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" d="m375.5013 60.780525l-9.076202 3.3034668l9.076202 3.3034592z" fill-rule="evenodd"></path><path fill="#000000" fill-opacity="0.0" d="m269.71652 104.53543l0 166.92914" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m269.71652 116.53543l0 154.92914" fill-rule="evenodd"></path><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" d="m273.02 116.53543l-3.3034668 -9.076195l-3.3034668 9.076195z" fill-rule="evenodd"></path><path fill="#000000" fill-opacity="0.0" d="m65.0 223.08398l-1.0078735 176.91339" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m64.93163 235.08379l-0.93950653 164.91359" fill-rule="evenodd"></path><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" d="m68.23505 235.10262l-3.2517014 -9.094864l-3.3551178 9.05722z" fill-rule="evenodd"></path><path fill="#000000" fill-opacity="0.0" d="m471.0 224.38058l-1.0078735 176.91336" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m470.93164 236.38037l-0.93951416 164.9136" fill-rule="evenodd"></path><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" d="m474.23505 236.3992l-3.251709 -9.094864l-3.3551025 9.05722z" fill-rule="evenodd"></path><path fill="#000000" fill-opacity="0.0" d="m128.50131 400.00266l0 -86.168l47.496002 0" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m128.50131 400.00266l0 -86.168l35.496002 0" fill-rule="evenodd"></path><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" d="m163.99731 317.13812l9.076202 -3.3034668l-9.076202 -3.3034668z" fill-rule="evenodd"></path><path fill="#000000" fill-opacity="0.0" d="m414.71902 400.00266l0 -86.168l-51.216003 0" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m414.71902 400.00266l0 -86.168l-39.216003 0" fill-rule="evenodd"></path><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" d="m375.50302 310.5312l-9.076172 3.3034668l9.076172 3.3034668z" fill-rule="evenodd"></path></g></svg> + diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst new file mode 100644 index 0000000..ec1dc4f --- /dev/null +++ b/Doc/library/pathlib.rst @@ -0,0 +1,919 @@ + +:mod:`pathlib` --- Object-oriented filesystem paths +=================================================== + +.. module:: pathlib + :synopsis: Object-oriented filesystem paths + +.. index:: single: path; operations + +.. versionadded:: 3.4 + +This module offers classes representing filesystem paths with semantics +appropriate for different operating systems. Path classes are divided +between :ref:`pure paths <pure-paths>`, which provide purely computational +operations without I/O, and :ref:`concrete paths <concrete-paths>`, which +inherit from pure paths but also provide I/O operations. + +.. image:: pathlib-inheritance.png + :align: center + +If you've never used this module before or just aren't sure which class is +right for your task, :class:`Path` is most likely what you need. It instantiates +a :ref:`concrete path <concrete-paths>` for the platform the code is running on. + +Pure paths are useful in some special cases; for example: + +#. If you want to manipulate Windows paths on a Unix machine (or vice versa). + You cannot instantiate a :class:`WindowsPath` when running on Unix, but you + can instantiate :class:`PureWindowsPath`. +#. You want to make sure that your code only manipulates paths without actually + accessing the OS. In this case, instantiating one of the pure classes may be + useful since those simply don't have any OS-accessing operations. + +.. note:: + This module has been included in the standard library on a + :term:`provisional basis <provisional package>`. Backwards incompatible + changes (up to and including removal of the package) may occur if deemed + necessary by the core developers. + +.. seealso:: + :pep:`428`: The pathlib module -- object-oriented filesystem paths. + +.. seealso:: + For low-level path manipulation on strings, you can also use the + :mod:`os.path` module. + + +Basic use +--------- + +Importing the main class:: + + >>> from pathlib import Path + +Listing subdirectories:: + + >>> p = Path('.') + >>> [x for x in p.iterdir() if x.is_dir()] + [PosixPath('.hg'), PosixPath('docs'), PosixPath('dist'), + PosixPath('__pycache__'), PosixPath('build')] + +Listing Python source files in this directory tree:: + + >>> list(p.glob('**/*.py')) + [PosixPath('test_pathlib.py'), PosixPath('setup.py'), + PosixPath('pathlib.py'), PosixPath('docs/conf.py'), + PosixPath('build/lib/pathlib.py')] + +Navigating inside a directory tree:: + + >>> p = Path('/etc') + >>> q = p / 'init.d' / 'reboot' + >>> q + PosixPath('/etc/init.d/reboot') + >>> q.resolve() + PosixPath('/etc/rc.d/init.d/halt') + +Querying path properties:: + + >>> q.exists() + True + >>> q.is_dir() + False + +Opening a file:: + + >>> with q.open() as f: f.readline() + ... + '#!/bin/bash\n' + + +.. _pure-paths: + +Pure paths +---------- + +Pure path objects provide path-handling operations which don't actually +access a filesystem. There are three ways to access these classes, which +we also call *flavours*: + +.. class:: PurePath(*pathsegments) + + A generic class that represents the system's path flavour (instantiating + it creates either a :class:`PurePosixPath` or a :class:`PureWindowsPath`):: + + >>> PurePath('setup.py') # Running on a Unix machine + PurePosixPath('setup.py') + + Each element of *pathsegments* can be either a string or bytes object + representing a path segment; it can also be another path object:: + + >>> PurePath('foo', 'some/path', 'bar') + PurePosixPath('foo/some/path/bar') + >>> PurePath(Path('foo'), Path('bar')) + PurePosixPath('foo/bar') + + When *pathsegments* is empty, the current directory is assumed:: + + >>> PurePath() + PurePosixPath('.') + + When several absolute paths are given, the last is taken as an anchor + (mimicking :func:`os.path.join`'s behaviour):: + + >>> PurePath('/etc', '/usr', 'lib64') + PurePosixPath('/usr/lib64') + >>> PureWindowsPath('c:/Windows', 'd:bar') + PureWindowsPath('d:bar') + + However, in a Windows path, changing the local root doesn't discard the + previous drive setting:: + + >>> PureWindowsPath('c:/Windows', '/Program Files') + PureWindowsPath('c:/Program Files') + + Spurious slashes and single dots are collapsed, but double dots (``'..'``) + are not, since this would change the meaning of a path in the face of + symbolic links:: + + >>> PurePath('foo//bar') + PurePosixPath('foo/bar') + >>> PurePath('foo/./bar') + PurePosixPath('foo/bar') + >>> PurePath('foo/../bar') + PurePosixPath('foo/../bar') + + (a naïve approach would make ``PurePosixPath('foo/../bar')`` equivalent + to ``PurePosixPath('bar')``, which is wrong if ``foo`` is a symbolic link + to another directory) + +.. class:: PurePosixPath(*pathsegments) + + A subclass of :class:`PurePath`, this path flavour represents non-Windows + filesystem paths:: + + >>> PurePosixPath('/etc') + PurePosixPath('/etc') + + *pathsegments* is specified similarly to :class:`PurePath`. + +.. class:: PureWindowsPath(*pathsegments) + + A subclass of :class:`PurePath`, this path flavour represents Windows + filesystem paths:: + + >>> PureWindowsPath('c:/Program Files/') + PureWindowsPath('c:/Program Files') + + *pathsegments* is specified similarly to :class:`PurePath`. + +Regardless of the system you're running on, you can instantiate all of +these classes, since they don't provide any operation that does system calls. + + +General properties +^^^^^^^^^^^^^^^^^^ + +Paths are immutable and hashable. Paths of a same flavour are comparable +and orderable. These properties respect the flavour's case-folding +semantics:: + + >>> PurePosixPath('foo') == PurePosixPath('FOO') + False + >>> PureWindowsPath('foo') == PureWindowsPath('FOO') + True + >>> PureWindowsPath('FOO') in { PureWindowsPath('foo') } + True + >>> PureWindowsPath('C:') < PureWindowsPath('d:') + True + +Paths of a different flavour compare unequal and cannot be ordered:: + + >>> PureWindowsPath('foo') == PurePosixPath('foo') + False + >>> PureWindowsPath('foo') < PurePosixPath('foo') + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + TypeError: unorderable types: PureWindowsPath() < PurePosixPath() + + +Operators +^^^^^^^^^ + +The slash operator helps create child paths, similarly to :func:`os.path.join`:: + + >>> p = PurePath('/etc') + >>> p + PurePosixPath('/etc') + >>> p / 'init.d' / 'apache2' + PurePosixPath('/etc/init.d/apache2') + >>> q = PurePath('bin') + >>> '/usr' / q + PurePosixPath('/usr/bin') + +The string representation of a path is the raw filesystem path itself +(in native form, e.g. with backslashes under Windows), which you can +pass to any function taking a file path as a string:: + + >>> p = PurePath('/etc') + >>> str(p) + '/etc' + >>> p = PureWindowsPath('c:/Program Files') + >>> str(p) + 'c:\\Program Files' + +Similarly, calling :class:`bytes` on a path gives the raw filesystem path as a +bytes object, as encoded by :func:`os.fsencode`:: + + >>> bytes(p) + b'/etc' + +.. note:: + Calling :class:`bytes` is only recommended under Unix. Under Windows, + the unicode form is the canonical representation of filesystem paths. + + +Accessing individual parts +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To access the individual "parts" (components) of a path, use the following +property: + +.. data:: PurePath.parts + + A tuple giving access to the path's various components:: + + >>> p = PurePath('/usr/bin/python3') + >>> p.parts + ('/', 'usr', 'bin', 'python3') + + >>> p = PureWindowsPath('c:/Program Files/PSF') + >>> p.parts + ('c:\\', 'Program Files', 'PSF') + + (note how the drive and local root are regrouped in a single part) + + +Methods and properties +^^^^^^^^^^^^^^^^^^^^^^ + +Pure paths provide the following methods and properties: + +.. data:: PurePath.drive + + A string representing the drive letter or name, if any:: + + >>> PureWindowsPath('c:/Program Files/').drive + 'c:' + >>> PureWindowsPath('/Program Files/').drive + '' + >>> PurePosixPath('/etc').drive + '' + + UNC shares are also considered drives:: + + >>> PureWindowsPath('//host/share/foo.txt').drive + '\\\\host\\share' + +.. data:: PurePath.root + + A string representing the (local or global) root, if any:: + + >>> PureWindowsPath('c:/Program Files/').root + '\\' + >>> PureWindowsPath('c:Program Files/').root + '' + >>> PurePosixPath('/etc').root + '/' + + UNC shares always have a root:: + + >>> PureWindowsPath('//host/share').root + '\\' + +.. data:: PurePath.anchor + + The concatenation of the drive and root:: + + >>> PureWindowsPath('c:/Program Files/').anchor + 'c:\\' + >>> PureWindowsPath('c:Program Files/').anchor + 'c:' + >>> PurePosixPath('/etc').anchor + '/' + >>> PureWindowsPath('//host/share').anchor + '\\\\host\\share\\' + + +.. data:: PurePath.parents + + An immutable sequence providing access to the logical ancestors of + the path:: + + >>> p = PureWindowsPath('c:/foo/bar/setup.py') + >>> p.parents[0] + PureWindowsPath('c:/foo/bar') + >>> p.parents[1] + PureWindowsPath('c:/foo') + >>> p.parents[2] + PureWindowsPath('c:/') + + +.. data:: PurePath.parent + + The logical parent of the path:: + + >>> p = PurePosixPath('/a/b/c/d') + >>> p.parent + PurePosixPath('/a/b/c') + + You cannot go past an anchor, or empty path:: + + >>> p = PurePosixPath('/') + >>> p.parent + PurePosixPath('/') + >>> p = PurePosixPath('.') + >>> p.parent + PurePosixPath('.') + + .. note:: + This is a purely lexical operation, hence the following behaviour:: + + >>> p = PurePosixPath('foo/..') + >>> p.parent + PurePosixPath('foo') + + If you want to walk an arbitrary filesystem path upwards, it is + recommended to first call :meth:`Path.resolve` so as to resolve + symlinks and eliminate `".."` components. + + +.. data:: PurePath.name + + A string representing the final path component, excluding the drive and + root, if any:: + + >>> PurePosixPath('my/library/setup.py').name + 'setup.py' + + UNC drive names are not considered:: + + >>> PureWindowsPath('//some/share/setup.py').name + 'setup.py' + >>> PureWindowsPath('//some/share').name + '' + + +.. data:: PurePath.suffix + + The file extension of the final component, if any:: + + >>> PurePosixPath('my/library/setup.py').suffix + '.py' + >>> PurePosixPath('my/library.tar.gz').suffix + '.gz' + >>> PurePosixPath('my/library').suffix + '' + + +.. data:: PurePath.suffixes + + A list of the path's file extensions:: + + >>> PurePosixPath('my/library.tar.gar').suffixes + ['.tar', '.gar'] + >>> PurePosixPath('my/library.tar.gz').suffixes + ['.tar', '.gz'] + >>> PurePosixPath('my/library').suffixes + [] + + +.. data:: PurePath.stem + + The final path component, without its suffix:: + + >>> PurePosixPath('my/library.tar.gz').stem + 'library.tar' + >>> PurePosixPath('my/library.tar').stem + 'library' + >>> PurePosixPath('my/library').stem + 'library' + + +.. method:: PurePath.as_posix() + + Return a string representation of the path with forward slashes (``/``):: + + >>> p = PureWindowsPath('c:\\windows') + >>> str(p) + 'c:\\windows' + >>> p.as_posix() + 'c:/windows' + + +.. method:: PurePath.as_uri() + + Represent the path as a ``file`` URI. :exc:`ValueError` is raised if + the path isn't absolute. + + >>> p = PurePosixPath('/etc/passwd') + >>> p.as_uri() + 'file:///etc/passwd' + >>> p = PureWindowsPath('c:/Windows') + >>> p.as_uri() + 'file:///c:/Windows' + + +.. method:: PurePath.is_absolute() + + Return whether the path is absolute or not. A path is considered absolute + if it has both a root and (if the flavour allows) a drive:: + + >>> PurePosixPath('/a/b').is_absolute() + True + >>> PurePosixPath('a/b').is_absolute() + False + + >>> PureWindowsPath('c:/a/b').is_absolute() + True + >>> PureWindowsPath('/a/b').is_absolute() + False + >>> PureWindowsPath('c:').is_absolute() + False + >>> PureWindowsPath('//some/share').is_absolute() + True + + +.. method:: PurePath.is_reserved() + + With :class:`PureWindowsPath`, return ``True`` if the path is considered + reserved under Windows, ``False`` otherwise. With :class:`PurePosixPath`, + ``False`` is always returned. + + >>> PureWindowsPath('nul').is_reserved() + True + >>> PurePosixPath('nul').is_reserved() + False + + File system calls on reserved paths can fail mysteriously or have + unintended effects. + + +.. method:: PurePath.joinpath(*other) + + Calling this method is equivalent to combining the path with each of + the *other* arguments in turn:: + + >>> PurePosixPath('/etc').joinpath('passwd') + PurePosixPath('/etc/passwd') + >>> PurePosixPath('/etc').joinpath(PurePosixPath('passwd')) + PurePosixPath('/etc/passwd') + >>> PurePosixPath('/etc').joinpath('init.d', 'apache2') + PurePosixPath('/etc/init.d/apache2') + >>> PureWindowsPath('c:').joinpath('/Program Files') + PureWindowsPath('c:/Program Files') + + +.. method:: PurePath.match(pattern) + + Match this path against the provided glob-style pattern. Return ``True`` + if matching is successful, ``False`` otherwise. + + If *pattern* is relative, the path can be either relative or absolute, + and matching is done from the right:: + + >>> PurePath('a/b.py').match('*.py') + True + >>> PurePath('/a/b/c.py').match('b/*.py') + True + >>> PurePath('/a/b/c.py').match('a/*.py') + False + + If *pattern* is absolute, the path must be absolute, and the whole path + must match:: + + >>> PurePath('/a.py').match('/*.py') + True + >>> PurePath('a/b.py').match('/*.py') + False + + As with other methods, case-sensitivity is observed:: + + >>> PureWindowsPath('b.py').match('*.PY') + True + + +.. method:: PurePath.relative_to(*other) + + Compute a version of this path relative to the path represented by + *other*. If it's impossible, ValueError is raised:: + + >>> p = PurePosixPath('/etc/passwd') + >>> p.relative_to('/') + PurePosixPath('etc/passwd') + >>> p.relative_to('/etc') + PurePosixPath('passwd') + >>> p.relative_to('/usr') + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + File "pathlib.py", line 694, in relative_to + .format(str(self), str(formatted))) + ValueError: '/etc/passwd' does not start with '/usr' + + +.. method:: PurePath.with_name(name) + + Return a new path with the :attr:`name` changed. If the original path + doesn't have a name, ValueError is raised:: + + >>> p = PureWindowsPath('c:/Downloads/pathlib.tar.gz') + >>> p.with_name('setup.py') + PureWindowsPath('c:/Downloads/setup.py') + >>> p = PureWindowsPath('c:/') + >>> p.with_name('setup.py') + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + File "/home/antoine/cpython/default/Lib/pathlib.py", line 751, in with_name + raise ValueError("%r has an empty name" % (self,)) + ValueError: PureWindowsPath('c:/') has an empty name + + +.. method:: PurePath.with_suffix(suffix) + + Return a new path with the :attr:`suffix` changed. If the original path + doesn't have a suffix, the new *suffix* is appended instead:: + + >>> p = PureWindowsPath('c:/Downloads/pathlib.tar.gz') + >>> p.with_suffix('.bz2') + PureWindowsPath('c:/Downloads/pathlib.tar.bz2') + >>> p = PureWindowsPath('README') + >>> p.with_suffix('.txt') + PureWindowsPath('README.txt') + + +.. _concrete-paths: + + +Concrete paths +-------------- + +Concrete paths are subclasses of the pure path classes. In addition to +operations provided by the latter, they also provide methods to do system +calls on path objects. There are three ways to instantiate concrete paths: + +.. class:: Path(*pathsegments) + + A subclass of :class:`PurePath`, this class represents concrete paths of + the system's path flavour (instantiating it creates either a + :class:`PosixPath` or a :class:`WindowsPath`):: + + >>> Path('setup.py') + PosixPath('setup.py') + + *pathsegments* is specified similarly to :class:`PurePath`. + +.. class:: PosixPath(*pathsegments) + + A subclass of :class:`Path` and :class:`PurePosixPath`, this class + represents concrete non-Windows filesystem paths:: + + >>> PosixPath('/etc') + PosixPath('/etc') + + *pathsegments* is specified similarly to :class:`PurePath`. + +.. class:: WindowsPath(*pathsegments) + + A subclass of :class:`Path` and :class:`PureWindowsPath`, this class + represents concrete Windows filesystem paths:: + + >>> WindowsPath('c:/Program Files/') + WindowsPath('c:/Program Files') + + *pathsegments* is specified similarly to :class:`PurePath`. + +You can only instantiate the class flavour that corresponds to your system +(allowing system calls on non-compatible path flavours could lead to +bugs or failures in your application):: + + >>> import os + >>> os.name + 'posix' + >>> Path('setup.py') + PosixPath('setup.py') + >>> PosixPath('setup.py') + PosixPath('setup.py') + >>> WindowsPath('setup.py') + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + File "pathlib.py", line 798, in __new__ + % (cls.__name__,)) + NotImplementedError: cannot instantiate 'WindowsPath' on your system + + +Methods +^^^^^^^ + +Concrete paths provide the following methods in addition to pure paths +methods. Many of these methods can raise an :exc:`OSError` if a system +call fails (for example because the path doesn't exist): + +.. classmethod:: Path.cwd() + + Return a new path object representing the current directory (as returned + by :func:`os.getcwd`):: + + >>> Path.cwd() + PosixPath('/home/antoine/pathlib') + + +.. method:: Path.stat() + + Return information about this path (similarly to :func:`os.stat`). + The result is looked up at each call to this method. + + >>> p = Path('setup.py') + >>> p.stat().st_size + 956 + >>> p.stat().st_mtime + 1327883547.852554 + + +.. method:: Path.chmod(mode) + + Change the file mode and permissions, like :func:`os.chmod`:: + + >>> p = Path('setup.py') + >>> p.stat().st_mode + 33277 + >>> p.chmod(0o444) + >>> p.stat().st_mode + 33060 + + +.. method:: Path.exists() + + Whether the path points to an existing file or directory:: + + >>> Path('.').exists() + True + >>> Path('setup.py').exists() + True + >>> Path('/etc').exists() + True + >>> Path('nonexistentfile').exists() + False + + .. note:: + If the path points to a symlink, :meth:`exists` returns whether the + symlink *points to* an existing file or directory. + + +.. method:: Path.glob(pattern) + + Glob the given *pattern* in the directory represented by this path, + yielding all matching files (of any kind):: + + >>> sorted(Path('.').glob('*.py')) + [PosixPath('pathlib.py'), PosixPath('setup.py'), PosixPath('test_pathlib.py')] + >>> sorted(Path('.').glob('*/*.py')) + [PosixPath('docs/conf.py')] + + The "``**``" pattern means "this directory and all subdirectories, + recursively". In other words, it enables recursive globbing:: + + >>> sorted(Path('.').glob('**/*.py')) + [PosixPath('build/lib/pathlib.py'), + PosixPath('docs/conf.py'), + PosixPath('pathlib.py'), + PosixPath('setup.py'), + PosixPath('test_pathlib.py')] + + .. note:: + Using the "``**``" pattern in large directory trees may consume + an inordinate amount of time. + + +.. method:: Path.group() + + Return the name of the group owning the file. :exc:`KeyError` is raised + if the file's gid isn't found in the system database. + + +.. method:: Path.is_dir() + + Return ``True`` if the path points to a directory (or a symbolic link + pointing to a directory), ``False`` if it points to another kind of file. + + ``False`` is also returned if the path doesn't exist or is a broken symlink; + other errors (such as permission errors) are propagated. + + +.. method:: Path.is_file() + + Return ``True`` if the path points to a regular file (or a symbolic link + pointing to a regular file), ``False`` if it points to another kind of file. + + ``False`` is also returned if the path doesn't exist or is a broken symlink; + other errors (such as permission errors) are propagated. + + +.. method:: Path.is_symlink() + + Return ``True`` if the path points to a symbolic link, ``False`` otherwise. + + ``False`` is also returned if the path doesn't exist; other errors (such + as permission errors) are propagated. + + +.. method:: Path.is_socket() + + Return ``True`` if the path points to a Unix socket (or a symbolic link + pointing to a Unix socket), ``False`` if it points to another kind of file. + + ``False`` is also returned if the path doesn't exist or is a broken symlink; + other errors (such as permission errors) are propagated. + + +.. method:: Path.is_fifo() + + Return ``True`` if the path points to a FIFO (or a symbolic link + pointing to a FIFO), ``False`` if it points to another kind of file. + + ``False`` is also returned if the path doesn't exist or is a broken symlink; + other errors (such as permission errors) are propagated. + + +.. method:: Path.is_block_device() + + Return ``True`` if the path points to a block device (or a symbolic link + pointing to a block device), ``False`` if it points to another kind of file. + + ``False`` is also returned if the path doesn't exist or is a broken symlink; + other errors (such as permission errors) are propagated. + + +.. method:: Path.is_char_device() + + Return ``True`` if the path points to a character device (or a symbolic link + pointing to a character device), ``False`` if it points to another kind of file. + + ``False`` is also returned if the path doesn't exist or is a broken symlink; + other errors (such as permission errors) are propagated. + + +.. method:: Path.iterdir() + + When the path points to a directory, yield path objects of the directory + contents:: + + >>> p = Path('docs') + >>> for child in p.iterdir(): child + ... + PosixPath('docs/conf.py') + PosixPath('docs/_templates') + PosixPath('docs/make.bat') + PosixPath('docs/index.rst') + PosixPath('docs/_build') + PosixPath('docs/_static') + PosixPath('docs/Makefile') + +.. method:: Path.lchmod(mode) + + Like :meth:`Path.chmod` but, if the path points to a symbolic link, the + symbolic link's mode is changed rather than its target's. + + +.. method:: Path.lstat() + + Like :meth:`Path.stat` but, if the path points to a symbolic link, return + the symbolic link's information rather than its target's. + + +.. method:: Path.mkdir(mode=0o777, parents=False) + + Create a new directory at this given path. If *mode* is given, it is + combined with the process' ``umask`` value to determine the file mode + and access flags. If the path already exists, :exc:`FileExistsError` + is raised. + + If *parents* is true, any missing parents of this path are created + as needed; they are created with the default permissions without taking + *mode* into account (mimicking the POSIX ``mkdir -p`` command). + + If *parents* is false (the default), a missing parent raises + :exc:`FileNotFoundError`. + + +.. method:: Path.open(mode='r', buffering=-1, encoding=None, errors=None, newline=None) + + Open the file pointed to by the path, like the built-in :func:`open` + function does:: + + >>> p = Path('setup.py') + >>> with p.open() as f: + ... f.readline() + ... + '#!/usr/bin/env python3\n' + + +.. method:: Path.owner() + + Return the name of the user owning the file. :exc:`KeyError` is raised + if the file's uid isn't found in the system database. + + +.. method:: Path.rename(target) + + Rename this file or directory to the given *target*. *target* can be + either a string or another path object:: + + >>> p = Path('foo') + >>> p.open('w').write('some text') + 9 + >>> target = Path('bar') + >>> p.rename(target) + >>> target.open().read() + 'some text' + + +.. method:: Path.replace(target) + + Rename this file or directory to the given *target*. If *target* points + to an existing file or directory, it will be unconditionally replaced. + + +.. method:: Path.resolve() + + Make the path absolute, resolving any symlinks. A new path object is + returned:: + + >>> p = Path() + >>> p + PosixPath('.') + >>> p.resolve() + PosixPath('/home/antoine/pathlib') + + `".."` components are also eliminated (this is the only method to do so):: + + >>> p = Path('docs/../setup.py') + >>> p.resolve() + PosixPath('/home/antoine/pathlib/setup.py') + + If the path doesn't exist, :exc:`FileNotFoundError` is raised. If an + infinite loop is encountered along the resolution path, + :exc:`RuntimeError` is raised. + + +.. method:: Path.rglob(pattern) + + This is like calling :meth:`glob` with "``**``" added in front of the + given *pattern*: + + >>> sorted(Path().rglob("*.py")) + [PosixPath('build/lib/pathlib.py'), + PosixPath('docs/conf.py'), + PosixPath('pathlib.py'), + PosixPath('setup.py'), + PosixPath('test_pathlib.py')] + + +.. method:: Path.rmdir() + + Remove this directory. The directory must be empty. + + +.. method:: Path.symlink_to(target, target_is_directory=False) + + Make this path a symbolic link to *target*. Under Windows, + *target_is_directory* must be true (default ``False``) if the link's target + is a directory. Under POSIX, *target_is_directory*'s value is ignored. + + >>> p = Path('mylink') + >>> p.symlink_to('setup.py') + >>> p.resolve() + PosixPath('/home/antoine/pathlib/setup.py') + >>> p.stat().st_size + 956 + >>> p.lstat().st_size + 8 + + .. note:: + The order of arguments (link, target) is the reverse + of :func:`os.symlink`'s. + + +.. method:: Path.touch(mode=0o777, exist_ok=True) + + Create a file at this given path. If *mode* is given, it is combined + with the process' ``umask`` value to determine the file mode and access + flags. If the file already exists, the function succeeds if *exist_ok* + is true (and its modification time is updated to the current time), + otherwise :exc:`FileExistsError` is raised. + + +.. method:: Path.unlink() + + Remove this file or symbolic link. If the path points to a directory, + use :func:`Path.rmdir` instead. diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index f4e37ac..48a8a6b 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -41,7 +41,7 @@ of the debugger is:: .. versionchanged:: 3.3 Tab-completion via the :mod:`readline` module is available for commands and command arguments, e.g. the current global and local names are offered as - arguments of the ``print`` command. + arguments of the ``p`` command. :file:`pdb.py` can also be invoked as a script to debug other scripts. For example:: @@ -309,7 +309,7 @@ by the local file. ``end`` to terminate the commands. An example:: (Pdb) commands 1 - (com) print some_variable + (com) p some_variable (com) end (Pdb) @@ -403,13 +403,19 @@ by the local file. Print the argument list of the current function. -.. pdbcommand:: p(rint) expression +.. pdbcommand:: p expression Evaluate the *expression* in the current context and print its value. + .. note:: + + ``print()`` can also be used, but is not a debugger command --- this executes the + Python :func:`print` function. + + .. pdbcommand:: pp expression - Like the :pdbcmd:`print` command, except the value of the expression is + Like the :pdbcmd:`p` command, except the value of the expression is pretty-printed using the :mod:`pprint` module. .. pdbcommand:: whatis expression diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 273fb34..ce5467f 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -94,6 +94,9 @@ There are fundamental differences between the pickle protocols and The :mod:`json` module: a standard library module allowing JSON serialization and deserialization. + +.. _pickle-protocols: + Data stream format ------------------ @@ -113,7 +116,9 @@ The module :mod:`pickletools` contains tools for analyzing data streams generated by :mod:`pickle`. :mod:`pickletools` source code has extensive comments about opcodes used by pickle protocols. -There are currently 4 different protocols which can be used for pickling. +There are currently 5 different protocols which can be used for pickling. +The higher the protocol used, the more recent the version of Python needed +to read the pickle produced. * Protocol version 0 is the original "human-readable" protocol and is backwards compatible with earlier versions of Python. @@ -125,10 +130,15 @@ There are currently 4 different protocols which can be used for pickling. efficient pickling of :term:`new-style class`\es. Refer to :pep:`307` for information about improvements brought by protocol 2. -* Protocol version 3 was added in Python 3. It has explicit support for +* Protocol version 3 was added in Python 3.0. It has explicit support for :class:`bytes` objects and cannot be unpickled by Python 2.x. This is - the default as well as the current recommended protocol; use it whenever - possible. + the default protocol, and the recommended protocol when compatibility with + other Python 3 versions is required. + +* Protocol version 4 was added in Python 3.4. It adds support for very large + objects, pickling more kinds of objects, and some data format + optimizations. Refer to :pep:`3154` for information about improvements + brought by protocol 4. .. note:: Serialization is a more primitive notion than persistence; although @@ -156,13 +166,16 @@ The :mod:`pickle` module provides the following constants: .. data:: HIGHEST_PROTOCOL - The highest protocol version available. This value can be passed as a - *protocol* value. + An integer, the highest :ref:`protocol version <pickle-protocols>` + available. This value can be passed as a *protocol* value to functions + :func:`dump` and :func:`dumps` as well as the :class:`Pickler` + constructor. .. data:: DEFAULT_PROTOCOL - The default protocol used for pickling. May be less than HIGHEST_PROTOCOL. - Currently the default protocol is 3, a new protocol designed for Python 3.0. + An integer, the default :ref:`protocol version <pickle-protocols>` used + for pickling. May be less than :data:`HIGHEST_PROTOCOL`. Currently the + default protocol is 3, a new protocol designed for Python 3. The :mod:`pickle` module provides the following functions to make the pickling @@ -173,13 +186,10 @@ process more convenient: Write a pickled representation of *obj* to the open :term:`file object` *file*. This is equivalent to ``Pickler(file, protocol).dump(obj)``. - The optional *protocol* argument tells the pickler to use the given protocol; - supported protocols are 0, 1, 2, 3. The default protocol is 3; a - backward-incompatible protocol designed for Python 3.0. - - Specifying a negative protocol version selects the highest protocol version - supported. The higher the protocol used, the more recent the version of - Python needed to read the pickle produced. + The optional *protocol* argument, an integer, tells the pickler to use + the given protocol; supported protocols are 0 to :data:`HIGHEST_PROTOCOL`. + If not specified, the default is :data:`DEFAULT_PROTOCOL`. If a negative + number is specified, :data:`HIGHEST_PROTOCOL` is selected. The *file* argument must have a write() method that accepts a single bytes argument. It can thus be an on-disk file opened for binary writing, a @@ -187,64 +197,57 @@ process more convenient: interface. If *fix_imports* is true and *protocol* is less than 3, pickle will try to - map the new Python 3.x names to the old module names used in Python 2.x, - so that the pickle data stream is readable with Python 2.x. + map the new Python 3 names to the old module names used in Python 2, so + that the pickle data stream is readable with Python 2. .. function:: dumps(obj, protocol=None, \*, fix_imports=True) - Return the pickled representation of the object as a :class:`bytes` - object, instead of writing it to a file. - - The optional *protocol* argument tells the pickler to use the given protocol; - supported protocols are 0, 1, 2, 3. The default protocol is 3; a - backward-incompatible protocol designed for Python 3.0. + Return the pickled representation of the object as a :class:`bytes` object, + instead of writing it to a file. - Specifying a negative protocol version selects the highest protocol version - supported. The higher the protocol used, the more recent the version of - Python needed to read the pickle produced. - - If *fix_imports* is true and *protocol* is less than 3, pickle will try to - map the new Python 3.x names to the old module names used in Python 2.x, - so that the pickle data stream is readable with Python 2.x. + Arguments *protocol* and *fix_imports* have the same meaning as in + :func:`dump`. .. function:: load(file, \*, fix_imports=True, encoding="ASCII", errors="strict") - Read a pickled object representation from the open :term:`file object` *file* - and return the reconstituted object hierarchy specified therein. This is - equivalent to ``Unpickler(file).load()``. + Read a pickled object representation from the open :term:`file object` + *file* and return the reconstituted object hierarchy specified therein. + This is equivalent to ``Unpickler(file).load()``. - The protocol version of the pickle is detected automatically, so no protocol - argument is needed. Bytes past the pickled object's representation are - ignored. + The protocol version of the pickle is detected automatically, so no + protocol argument is needed. Bytes past the pickled object's + representation are ignored. The argument *file* must have two methods, a read() method that takes an integer argument, and a readline() method that requires no arguments. Both - methods should return bytes. Thus *file* can be an on-disk file opened - for binary reading, a :class:`io.BytesIO` object, or any other custom object + methods should return bytes. Thus *file* can be an on-disk file opened for + binary reading, a :class:`io.BytesIO` object, or any other custom object that meets this interface. Optional keyword arguments are *fix_imports*, *encoding* and *errors*, which are used to control compatibility support for pickle stream generated - by Python 2.x. If *fix_imports* is true, pickle will try to map the old - Python 2.x names to the new names used in Python 3.x. The *encoding* and + by Python 2. If *fix_imports* is true, pickle will try to map the old + Python 2 names to the new names used in Python 3. The *encoding* and *errors* tell pickle how to decode 8-bit string instances pickled by Python - 2.x; these default to 'ASCII' and 'strict', respectively. + 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can + be 'bytes' to read these 8-bit string instances as bytes objects. .. function:: loads(bytes_object, \*, fix_imports=True, encoding="ASCII", errors="strict") Read a pickled object hierarchy from a :class:`bytes` object and return the reconstituted object hierarchy specified therein - The protocol version of the pickle is detected automatically, so no protocol - argument is needed. Bytes past the pickled object's representation are - ignored. + The protocol version of the pickle is detected automatically, so no + protocol argument is needed. Bytes past the pickled object's + representation are ignored. Optional keyword arguments are *fix_imports*, *encoding* and *errors*, which are used to control compatibility support for pickle stream generated - by Python 2.x. If *fix_imports* is true, pickle will try to map the old - Python 2.x names to the new names used in Python 3.x. The *encoding* and + by Python 2. If *fix_imports* is true, pickle will try to map the old + Python 2 names to the new names used in Python 3. The *encoding* and *errors* tell pickle how to decode 8-bit string instances pickled by Python - 2.x; these default to 'ASCII' and 'strict', respectively. + 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can + be 'bytes' to read these 8-bit string instances as bytes objects. The :mod:`pickle` module defines three exceptions: @@ -279,21 +282,19 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and This takes a binary file for writing a pickle data stream. - The optional *protocol* argument tells the pickler to use the given protocol; - supported protocols are 0, 1, 2, 3. The default protocol is 3; a - backward-incompatible protocol designed for Python 3.0. - - Specifying a negative protocol version selects the highest protocol version - supported. The higher the protocol used, the more recent the version of - Python needed to read the pickle produced. + The optional *protocol* argument, an integer, tells the pickler to use + the given protocol; supported protocols are 0 to :data:`HIGHEST_PROTOCOL`. + If not specified, the default is :data:`DEFAULT_PROTOCOL`. If a negative + number is specified, :data:`HIGHEST_PROTOCOL` is selected. The *file* argument must have a write() method that accepts a single bytes argument. It can thus be an on-disk file opened for binary writing, a - :class:`io.BytesIO` instance, or any other custom object that meets this interface. + :class:`io.BytesIO` instance, or any other custom object that meets this + interface. If *fix_imports* is true and *protocol* is less than 3, pickle will try to - map the new Python 3.x names to the old module names used in Python 2.x, - so that the pickle data stream is readable with Python 2.x. + map the new Python 3 names to the old module names used in Python 2, so + that the pickle data stream is readable with Python 2. .. method:: dump(obj) @@ -355,16 +356,17 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and The argument *file* must have two methods, a read() method that takes an integer argument, and a readline() method that requires no arguments. Both - methods should return bytes. Thus *file* can be an on-disk file object opened - for binary reading, a :class:`io.BytesIO` object, or any other custom object - that meets this interface. + methods should return bytes. Thus *file* can be an on-disk file object + opened for binary reading, a :class:`io.BytesIO` object, or any other + custom object that meets this interface. Optional keyword arguments are *fix_imports*, *encoding* and *errors*, which are used to control compatibility support for pickle stream generated - by Python 2.x. If *fix_imports* is true, pickle will try to map the old - Python 2.x names to the new names used in Python 3.x. The *encoding* and + by Python 2. If *fix_imports* is true, pickle will try to map the old + Python 2 names to the new names used in Python 3. The *encoding* and *errors* tell pickle how to decode 8-bit string instances pickled by Python - 2.x; these default to 'ASCII' and 'strict', respectively. + 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can + be 'bytes' to read these ß8-bit string instances as bytes objects. .. method:: load() @@ -409,7 +411,8 @@ The following types can be pickled: * tuples, lists, sets, and dictionaries containing only picklable objects -* functions defined at the top level of a module +* functions defined at the top level of a module (using :keyword:`def`, not + :keyword:`lambda`) * built-in functions defined at the top level of a module @@ -427,7 +430,7 @@ raised in this case. You can carefully raise this limit with :func:`sys.setrecursionlimit`. Note that functions (built-in and user-defined) are pickled by "fully qualified" -name reference, not by value. This means that only the function name is +name reference, not by value. [#]_ This means that only the function name is pickled, along with the name of the module the function is defined in. Neither the function's code, nor any of its function attributes are pickled. Thus the defining module must be importable in the unpickling environment, and the module @@ -483,12 +486,29 @@ implementation of this behaviour:: Classes can alter the default behaviour by providing one or several special methods: +.. method:: object.__getnewargs_ex__() + + In protocols 4 and newer, classes that implements the + :meth:`__getnewargs_ex__` method can dictate the values passed to the + :meth:`__new__` method upon unpickling. The method must return a pair + ``(args, kwargs)`` where *args* is a tuple of positional arguments + and *kwargs* a dictionary of named arguments for constructing the + object. Those will be passed to the :meth:`__new__` method upon + unpickling. + + You should implement this method if the :meth:`__new__` method of your + class requires keyword-only arguments. Otherwise, it is recommended for + compatibility to implement :meth:`__getnewargs__`. + + .. method:: object.__getnewargs__() - In protocol 2 and newer, classes that implements the :meth:`__getnewargs__` - method can dictate the values passed to the :meth:`__new__` method upon - unpickling. This is often needed for classes whose :meth:`__new__` method - requires arguments. + This method serve a similar purpose as :meth:`__getnewargs_ex__` but + for protocols 2 and newer. It must return a tuple of arguments `args` + which will be passed to the :meth:`__new__` method upon unpickling. + + In protocols 4 and newer, :meth:`__getnewargs__` will not be called if + :meth:`__getnewargs_ex__` is defined. .. method:: object.__getstate__() @@ -520,10 +540,10 @@ the methods :meth:`__getstate__` and :meth:`__setstate__`. At unpickling time, some methods like :meth:`__getattr__`, :meth:`__getattribute__`, or :meth:`__setattr__` may be called upon the - instance. In case those methods rely on some internal invariant being true, - the type should implement :meth:`__getnewargs__` to establish such an - invariant; otherwise, neither :meth:`__new__` nor :meth:`__init__` will be - called. + instance. In case those methods rely on some internal invariant being + true, the type should implement :meth:`__getnewargs__` or + :meth:`__getnewargs_ex__` to establish such an invariant; otherwise, + neither :meth:`__new__` nor :meth:`__init__` will be called. .. index:: pair: copy; protocol @@ -535,7 +555,7 @@ objects. [#]_ Although powerful, implementing :meth:`__reduce__` directly in your classes is error prone. For this reason, class designers should use the high-level -interface (i.e., :meth:`__getnewargs__`, :meth:`__getstate__` and +interface (i.e., :meth:`__getnewargs_ex__`, :meth:`__getstate__` and :meth:`__setstate__`) whenever possible. We will show, however, cases where using :meth:`__reduce__` is the only option or leads to more efficient pickling or both. @@ -883,6 +903,9 @@ The following example reads the resulting pickled data. :: .. [#] Don't confuse this with the :mod:`marshal` module +.. [#] This is why :keyword:`lambda` functions cannot be pickled: all + :keyword:`lambda` functions share the same name: ``<lambda>``. + .. [#] The exception raised will likely be an :exc:`ImportError` or an :exc:`AttributeError` but it could be something else. diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 7c4f912..13ea7b9 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -74,15 +74,17 @@ support. Retrieve a :pep:`302` module loader for the given *fullname*. - This is a convenience wrapper around :func:`importlib.find_loader` that - sets the *path* argument correctly when searching for submodules, and - also ensures parent packages (if any) are imported before searching for - submodules. + This is a backwards compatibility wrapper around + :func:`importlib.util.find_spec` that converts most failures to + :exc:`ImportError` and only returns the loader rather than the full + :class:`ModuleSpec`. .. versionchanged:: 3.3 Updated to be based directly on :mod:`importlib` rather than relying on the package internal PEP 302 import emulation. + .. versionchanged:: 3.4 + Updated to be based on :pep:`451` .. function:: get_importer(path_item) @@ -109,14 +111,13 @@ support. not already imported, its containing package (if any) is imported, in order to establish the package ``__path__``. - This function uses :func:`iter_importers`, and is thus subject to the same - limitations regarding platform-specific special import locations such as the - Windows registry. - .. versionchanged:: 3.3 Updated to be based directly on :mod:`importlib` rather than relying on the package internal PEP 302 import emulation. + .. versionchanged:: 3.4 + Updated to be based on :pep:`451` + .. function:: iter_importers(fullname='') diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index a267368..6a2d6b4 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -16,26 +16,24 @@ -------------- This module provides an interface for reading and writing the "property list" -XML files used mainly by Mac OS X. +files used mainly by Mac OS X and supports both binary and XML plist files. -The property list (``.plist``) file format is a simple XML pickle supporting +The property list (``.plist``) file format is a simple serialization supporting basic object types, like dictionaries, lists, numbers and strings. Usually the top level object is a dictionary. -To write out and to parse a plist file, use the :func:`writePlist` and -:func:`readPlist` functions. +To write out and to parse a plist file, use the :func:`dump` and +:func:`load` functions. -To work with plist data in bytes objects, use :func:`writePlistToBytes` -and :func:`readPlistFromBytes`. +To work with plist data in bytes objects, use :func:`dumps` +and :func:`loads`. Values can be strings, integers, floats, booleans, tuples, lists, dictionaries -(but only with string keys), :class:`Data` or :class:`datetime.datetime` -objects. String values (including dictionary keys) have to be unicode strings -- -they will be written out as UTF-8. +(but only with string keys), :class:`Data`, :class:`bytes`, :class:`bytesarray` +or :class:`datetime.datetime` objects. -The ``<data>`` plist type is supported through the :class:`Data` class. This is -a thin wrapper around a Python bytes object. Use :class:`Data` if your strings -contain control characters. +.. versionchanged:: 3.4 + New API, old API deprecated. Support for binary format plists added. .. seealso:: @@ -45,37 +43,147 @@ contain control characters. This module defines the following functions: -.. function:: readPlist(pathOrFile) +.. function:: load(fp, \*, fmt=None, use_builtin_types=True, dict_type=dict) - Read a plist file. *pathOrFile* may either be a file name or a (readable and - binary) file object. Return the unpacked root object (which usually is a + Read a plist file. *fp* should be a readable and binary file object. + Return the unpacked root object (which usually is a dictionary). - The XML data is parsed using the Expat parser from :mod:`xml.parsers.expat` - -- see its documentation for possible exceptions on ill-formed XML. - Unknown elements will simply be ignored by the plist parser. + The *fmt* is the format of the file and the following values are valid: + + * :data:`None`: Autodetect the file format + + * :data:`FMT_XML`: XML file format + + * :data:`FMT_BINARY`: Binary plist format + + If *use_builtin_types* is true (the default) binary data will be returned + as instances of :class:`bytes`, otherwise it is returned as instances of + :class:`Data`. + + The *dict_type* is the type used for dictionaries that are read from the + plist file. The exact structure of the plist can be recovered by using + :class:`collections.OrderedDict` (although the order of keys shouldn't be + important in plist files). + + XML data for the :data:`FMT_XML` format is parsed using the Expat parser + from :mod:`xml.parsers.expat` -- see its documentation for possible + exceptions on ill-formed XML. Unknown elements will simply be ignored + by the plist parser. + + The parser for the binary format raises :exc:`InvalidFileException` + when the file cannot be parsed. + + .. versionadded:: 3.4 + + +.. function:: loads(data, \*, fmt=None, use_builtin_types=True, dict_type=dict) + + Load a plist from a bytes object. See :func:`load` for an explanation of + the keyword arguments. + + .. versionadded:: 3.4 + + +.. function:: dump(value, fp, \*, fmt=FMT_XML, sort_keys=True, skipkeys=False) + + Write *value* to a plist file. *Fp* should be a writable, binary + file object. + + The *fmt* argument specifies the format of the plist file and can be + one of the following values: + + * :data:`FMT_XML`: XML formatted plist file + + * :data:`FMT_BINARY`: Binary formatted plist file + + When *sort_keys* is true (the default) the keys for dictionaries will be + written to the plist in sorted order, otherwise they will be written in + the iteration order of the dictionary. + + When *skipkeys* is false (the default) the function raises :exc:`TypeError` + when a key of a dictionary is not a string, otherwise such keys are skipped. + + A :exc:`TypeError` will be raised if the object is of an unsupported type or + a container that contains objects of unsupported types. + + An :exc:`OverflowError` will be raised for integer values that cannot + be represented in (binary) plist files. + + .. versionadded:: 3.4 + + +.. function:: dumps(value, \*, fmt=FMT_XML, sort_keys=True, skipkeys=False) + + Return *value* as a plist-formatted bytes object. See + the documentation for :func:`dump` for an explanation of the keyword + arguments of this function. + + .. versionadded:: 3.4 + +The following functions are deprecated: + +.. function:: readPlist(pathOrFile) + + Read a plist file. *pathOrFile* may be either a file name or a (readable + and binary) file object. Returns the unpacked root object (which usually + is a dictionary). + + This function calls :func:`load` to do the actual work, the the documentation + of :func:`that function <load>` for an explanation of the keyword arguments. + + .. note:: + + Dict values in the result have a ``__getattr__`` method that defers + to ``__getitem_``. This means that you can use attribute access to + access items of these dictionaries. + + .. deprecated:: 3.4 Use :func:`load` instead. .. function:: writePlist(rootObject, pathOrFile) - Write *rootObject* to a plist file. *pathOrFile* may either be a file name - or a (writable and binary) file object. + Write *rootObject* to an XML plist file. *pathOrFile* may be either a file name + or a (writable and binary) file object - A :exc:`TypeError` will be raised if the object is of an unsupported type or - a container that contains objects of unsupported types. + .. deprecated:: 3.4 Use :func:`dump` instead. .. function:: readPlistFromBytes(data) Read a plist data from a bytes object. Return the root object. + See :func:`load` for a description of the keyword arguments. + + .. note:: + + Dict values in the result have a ``__getattr__`` method that defers + to ``__getitem_``. This means that you can use attribute access to + access items of these dictionaries. + + .. deprecated:: 3.4 Use :func:`loads` instead. + .. function:: writePlistToBytes(rootObject) - Return *rootObject* as a plist-formatted bytes object. + Return *rootObject* as an XML plist-formatted bytes object. + + .. deprecated:: 3.4 Use :func:`dumps` instead. + +The following classes are available: + +.. class:: Dict([dict]): + + Return an extended mapping object with the same value as dictionary + *dict*. + + This class is a subclass of :class:`dict` where attribute access can + be used to access items. That is, ``aDict.key`` is the same as + ``aDict['key']`` for getting, setting and deleting items in the mapping. + + .. deprecated:: 3.0 -The following class is available: .. class:: Data(data) @@ -86,6 +194,24 @@ The following class is available: It has one attribute, :attr:`data`, that can be used to retrieve the Python bytes object stored in it. + .. deprecated:: 3.4 Use a :class:`bytes` object instead + + +The following constants are available: + +.. data:: FMT_XML + + The XML format for plist files. + + .. versionadded:: 3.4 + + +.. data:: FMT_BINARY + + The binary format for plist files + + .. versionadded:: 3.4 + Examples -------- @@ -103,13 +229,15 @@ Generating a plist:: aTrueValue = True, aFalseValue = False, ), - someData = Data(b"<binary gunk>"), - someMoreData = Data(b"<lots of binary gunk>" * 10), + someData = b"<binary gunk>", + someMoreData = b"<lots of binary gunk>" * 10, aDate = datetime.datetime.fromtimestamp(time.mktime(time.gmtime())), ) - writePlist(pl, fileName) + with open(fileName, 'wb') as fp: + dump(pl, fp) Parsing a plist:: - pl = readPlist(pathOrFile) + with open(fileName, 'rb') as fp: + pl = load(fp) print(pl["aKey"]) diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst index b4080d6..bc7b3e7 100644 --- a/Doc/library/poplib.rst +++ b/Doc/library/poplib.rst @@ -13,8 +13,11 @@ -------------- This module defines a class, :class:`POP3`, which encapsulates a connection to a -POP3 server and implements the protocol as defined in :rfc:`1725`. The -:class:`POP3` class supports both the minimal and optional command sets. +POP3 server and implements the protocol as defined in :rfc:`1939`. The +:class:`POP3` class supports both the minimal and optional command sets from +:rfc:`1939`. The :class:`POP3` class also supports the `STLS` command introduced +in :rfc:`2595` to enable encrypted communication on an already established connection. + Additionally, this module provides a class :class:`POP3_SSL`, which provides support for connecting to POP3 servers that use SSL as an underlying protocol layer. @@ -40,16 +43,23 @@ The :mod:`poplib` module provides two classes: This is a subclass of :class:`POP3` that connects to the server over an SSL encrypted socket. If *port* is not specified, 995, the standard POP3-over-SSL - port is used. *keyfile* and *certfile* are also optional - they can contain a - PEM formatted private key and certificate chain file for the SSL connection. - *timeout* works as in the :class:`POP3` constructor. *context* parameter is a - :class:`ssl.SSLContext` object which allows bundling SSL configuration - options, certificates and private keys into a single (potentially long-lived) - structure. + port is used. *timeout* works as in the :class:`POP3` constructor. + *context* is an optional :class:`ssl.SSLContext` object which allows + bundling SSL configuration options, certificates and private keys into a + single (potentially long-lived) structure. Please read :ref:`ssl-security` + for best practices. + + *keyfile* and *certfile* are a legacy alternative to *context* - they can + point to PEM-formatted private key and certificate chain files, + respectively, for the SSL connection. .. versionchanged:: 3.2 *context* parameter added. + .. versionchanged:: 3.4 + The class now supports hostname check with + :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see + :data:`ssl.HAS_SNI`). One exception is defined as an attribute of the :mod:`poplib` module: @@ -97,6 +107,14 @@ An :class:`POP3` instance has the following methods: Returns the greeting string sent by the POP3 server. +.. method:: POP3.capa() + + Query the server's capabilities as specified in :rfc:`2449`. + Returns a dictionary in the form ``{'name': ['param'...]}``. + + .. versionadded:: 3.4 + + .. method:: POP3.user(username) Send user command, response should indicate that a password is required. @@ -176,6 +194,23 @@ An :class:`POP3` instance has the following methods: the unique id for that message in the form ``'response mesgnum uid``, otherwise result is list ``(response, ['mesgnum uid', ...], octets)``. +.. method:: POP3.stls(context=None) + + Start a TLS session on the active connection as specified in :rfc:`2595`. + This is only allowed before user authentication + + *context* parameter is a :class:`ssl.SSLContext` object which allows + bundling SSL configuration options, certificates and private keys into + a single (potentially long-lived) structure. Please read :ref:`ssl-security` + for best practices. + + This method supports hostname checking via + :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see + :data:`ssl.HAS_SNI`). + + .. versionadded:: 3.4 + + Instances of :class:`POP3_SSL` have no additional methods. The interface of this subclass is identical to its parent. diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index 3a86331..447f8f7 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -14,8 +14,8 @@ The :mod:`pprint` module provides a capability to "pretty-print" arbitrary Python data structures in a form which can be used as input to the interpreter. If the formatted structures include objects which are not fundamental Python types, the representation may not be loadable. This may be the case if objects -such as files, sockets, classes, or instances are included, as well as many -other built-in objects which are not representable as Python constants. +such as files, sockets or classes are included, as well as many other +objects which are not representable as Python literals. The formatted representation keeps objects on a single line if it can, and breaks them onto multiple lines if they don't fit within the allowed width. @@ -29,14 +29,14 @@ The :mod:`pprint` module defines one class: .. First the implementation class: -.. class:: PrettyPrinter(indent=1, width=80, depth=None, stream=None) +.. class:: PrettyPrinter(indent=1, width=80, depth=None, stream=None, *, \ + compact=False) Construct a :class:`PrettyPrinter` instance. This constructor understands several keyword parameters. An output stream may be set using the *stream* keyword; the only method used on the stream object is the file protocol's :meth:`write` method. If not specified, the :class:`PrettyPrinter` adopts - ``sys.stdout``. Three additional parameters may be used to control the - formatted representation. The keywords are *indent*, *depth*, and *width*. The + ``sys.stdout``. The amount of indentation added for each recursive level is specified by *indent*; the default is one. Other values can cause output to look a little odd, but can make nesting easier to spot. The number of levels which may be printed is @@ -45,7 +45,12 @@ The :mod:`pprint` module defines one class: the depth of the objects being formatted. The desired output width is constrained using the *width* parameter; the default is 80 characters. If a structure cannot be formatted within the constrained width, a best effort will - be made. + be made. If *compact* is false (the default) each item of a long sequence + will be formatted on a separate line. If *compact* is true, as many items + as will fit within the *width* will be formatted on each output line. + + .. versionchanged:: 3.4 + Added the *compact* parameter. >>> import pprint >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] @@ -58,6 +63,12 @@ The :mod:`pprint` module defines one class: 'lumberjack', 'knights', 'ni'] + >>> pp = pprint.PrettyPrinter(width=41, compact=True) + >>> pp.pprint(stuff) + [['spam', 'eggs', 'lumberjack', + 'knights', 'ni'], + 'spam', 'eggs', 'lumberjack', 'knights', + 'ni'] >>> tup = ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', ... ('parrot', ('fresh fruit',)))))))) >>> pp = pprint.PrettyPrinter(depth=6) @@ -65,23 +76,30 @@ The :mod:`pprint` module defines one class: ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', (...))))))) -The :class:`PrettyPrinter` class supports several derivative functions: +The :mod:`pprint` module also provides several shortcut functions: -.. function:: pformat(object, indent=1, width=80, depth=None) +.. function:: pformat(object, indent=1, width=80, depth=None, *, compact=False) - Return the formatted representation of *object* as a string. *indent*, *width* - and *depth* will be passed to the :class:`PrettyPrinter` constructor as - formatting parameters. + Return the formatted representation of *object* as a string. *indent*, + *width*, *depth* and *compact* will be passed to the :class:`PrettyPrinter` + constructor as formatting parameters. + .. versionchanged:: 3.4 + Added the *compact* parameter. -.. function:: pprint(object, stream=None, indent=1, width=80, depth=None) + +.. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ + compact=False) Prints the formatted representation of *object* on *stream*, followed by a newline. If *stream* is ``None``, ``sys.stdout`` is used. This may be used in the interactive interpreter instead of the :func:`print` function for inspecting values (you can even reassign ``print = pprint.pprint`` for use - within a scope). *indent*, *width* and *depth* will be passed to the - :class:`PrettyPrinter` constructor as formatting parameters. + within a scope). *indent*, *width*, *depth* and *compact* will be passed + to the :class:`PrettyPrinter` constructor as formatting parameters. + + .. versionchanged:: 3.4 + Added the *compact* parameter. >>> import pprint >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] @@ -193,101 +211,141 @@ Example ------- To demonstrate several uses of the :func:`pprint` function and its parameters, -let's fetch information about a project from PyPI:: +let's fetch information about a project from `PyPI <https://pypi.python.org>`_:: >>> import json >>> import pprint >>> from urllib.request import urlopen - >>> with urlopen('http://pypi.python.org/pypi/configparser/json') as url: + >>> with urlopen('http://pypi.python.org/pypi/Twisted/json') as url: ... http_info = url.info() ... raw_data = url.read().decode(http_info.get_content_charset()) >>> project_info = json.loads(raw_data) - >>> result = {'headers': http_info.items(), 'body': project_info} In its basic form, :func:`pprint` shows the whole object:: - >>> pprint.pprint(result) - {'body': {'info': {'_pypi_hidden': False, - '_pypi_ordering': 12, - 'classifiers': ['Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Natural Language :: English', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Topic :: Software Development :: Libraries', - 'Topic :: Software Development :: Libraries :: Python Modules'], - 'download_url': 'UNKNOWN', - 'home_page': 'http://docs.python.org/py3k/library/configparser.html', - 'keywords': 'configparser ini parsing conf cfg configuration file', - 'license': 'MIT', - 'name': 'configparser', - 'package_url': 'http://pypi.python.org/pypi/configparser', - 'platform': 'any', - 'release_url': 'http://pypi.python.org/pypi/configparser/3.2.0r3', - 'requires_python': None, - 'stable_version': None, - 'summary': 'This library brings the updated configparser from Python 3.2+ to Python 2.6-2.7.', - 'version': '3.2.0r3'}, - 'urls': [{'comment_text': '', - 'downloads': 47, - 'filename': 'configparser-3.2.0r3.tar.gz', - 'has_sig': False, - 'md5_digest': '8500fd87c61ac0de328fc996fce69b96', - 'packagetype': 'sdist', - 'python_version': 'source', - 'size': 32281, - 'upload_time': '2011-05-10T16:28:50', - 'url': 'http://pypi.python.org/packages/source/c/configparser/configparser-3.2.0r3.tar.gz'}]}, - 'headers': [('Date', 'Sat, 14 May 2011 12:48:52 GMT'), - ('Server', 'Apache/2.2.16 (Debian)'), - ('Content-Disposition', 'inline'), - ('Connection', 'close'), - ('Transfer-Encoding', 'chunked'), - ('Content-Type', 'application/json; charset="UTF-8"')]} + >>> pprint.pprint(project_info) + {'info': {'_pypi_hidden': False, + '_pypi_ordering': 125, + 'author': 'Glyph Lefkowitz', + 'author_email': 'glyph@twistedmatrix.com', + 'bugtrack_url': '', + 'cheesecake_code_kwalitee_id': None, + 'cheesecake_documentation_id': None, + 'cheesecake_installability_id': None, + 'classifiers': ['Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 2 :: Only'], + 'description': 'An extensible framework for Python programming, ' + 'with special focus\r\n' + 'on event-based network programming and ' + 'multiprotocol integration.', + 'docs_url': '', + 'download_url': 'UNKNOWN', + 'home_page': 'http://twistedmatrix.com/', + 'keywords': '', + 'license': 'MIT', + 'maintainer': '', + 'maintainer_email': '', + 'name': 'Twisted', + 'package_url': 'http://pypi.python.org/pypi/Twisted', + 'platform': 'UNKNOWN', + 'release_url': 'http://pypi.python.org/pypi/Twisted/12.3.0', + 'requires_python': None, + 'stable_version': None, + 'summary': 'An asynchronous networking framework written in Python', + 'version': '12.3.0'}, + 'urls': [{'comment_text': '', + 'downloads': 71844, + 'filename': 'Twisted-12.3.0.tar.bz2', + 'has_sig': False, + 'md5_digest': '6e289825f3bf5591cfd670874cc0862d', + 'packagetype': 'sdist', + 'python_version': 'source', + 'size': 2615733, + 'upload_time': '2012-12-26T12:47:03', + 'url': 'https://pypi.python.org/packages/source/T/Twisted/Twisted-12.3.0.tar.bz2'}, + {'comment_text': '', + 'downloads': 5224, + 'filename': 'Twisted-12.3.0.win32-py2.7.msi', + 'has_sig': False, + 'md5_digest': '6b778f5201b622a5519a2aca1a2fe512', + 'packagetype': 'bdist_msi', + 'python_version': '2.7', + 'size': 2916352, + 'upload_time': '2012-12-26T12:48:15', + 'url': 'https://pypi.python.org/packages/2.7/T/Twisted/Twisted-12.3.0.win32-py2.7.msi'}]} The result can be limited to a certain *depth* (ellipsis is used for deeper contents):: - >>> pprint.pprint(result, depth=3) - {'body': {'info': {'_pypi_hidden': False, - '_pypi_ordering': 12, - 'classifiers': [...], - 'download_url': 'UNKNOWN', - 'home_page': 'http://docs.python.org/py3k/library/configparser.html', - 'keywords': 'configparser ini parsing conf cfg configuration file', - 'license': 'MIT', - 'name': 'configparser', - 'package_url': 'http://pypi.python.org/pypi/configparser', - 'platform': 'any', - 'release_url': 'http://pypi.python.org/pypi/configparser/3.2.0r3', - 'requires_python': None, - 'stable_version': None, - 'summary': 'This library brings the updated configparser from Python 3.2+ to Python 2.6-2.7.', - 'version': '3.2.0r3'}, - 'urls': [{...}]}, - 'headers': [('Date', 'Sat, 14 May 2011 12:48:52 GMT'), - ('Server', 'Apache/2.2.16 (Debian)'), - ('Content-Disposition', 'inline'), - ('Connection', 'close'), - ('Transfer-Encoding', 'chunked'), - ('Content-Type', 'application/json; charset="UTF-8"')]} - -Additionally, maximum *width* can be suggested. If a long object cannot be -split, the specified width will be exceeded:: - - >>> pprint.pprint(result['headers'], width=30) - [('Date', - 'Sat, 14 May 2011 12:48:52 GMT'), - ('Server', - 'Apache/2.2.16 (Debian)'), - ('Content-Disposition', - 'inline'), - ('Connection', 'close'), - ('Transfer-Encoding', - 'chunked'), - ('Content-Type', - 'application/json; charset="UTF-8"')] + >>> pprint.pprint(project_info, depth=2) + {'info': {'_pypi_hidden': False, + '_pypi_ordering': 125, + 'author': 'Glyph Lefkowitz', + 'author_email': 'glyph@twistedmatrix.com', + 'bugtrack_url': '', + 'cheesecake_code_kwalitee_id': None, + 'cheesecake_documentation_id': None, + 'cheesecake_installability_id': None, + 'classifiers': [...], + 'description': 'An extensible framework for Python programming, ' + 'with special focus\r\n' + 'on event-based network programming and ' + 'multiprotocol integration.', + 'docs_url': '', + 'download_url': 'UNKNOWN', + 'home_page': 'http://twistedmatrix.com/', + 'keywords': '', + 'license': 'MIT', + 'maintainer': '', + 'maintainer_email': '', + 'name': 'Twisted', + 'package_url': 'http://pypi.python.org/pypi/Twisted', + 'platform': 'UNKNOWN', + 'release_url': 'http://pypi.python.org/pypi/Twisted/12.3.0', + 'requires_python': None, + 'stable_version': None, + 'summary': 'An asynchronous networking framework written in Python', + 'version': '12.3.0'}, + 'urls': [{...}, {...}]} + +Additionally, maximum character *width* can be suggested. If a long object +cannot be split, the specified width will be exceeded:: + + >>> pprint.pprint(project_info, depth=2, width=50) + {'info': {'_pypi_hidden': False, + '_pypi_ordering': 125, + 'author': 'Glyph Lefkowitz', + 'author_email': 'glyph@twistedmatrix.com', + 'bugtrack_url': '', + 'cheesecake_code_kwalitee_id': None, + 'cheesecake_documentation_id': None, + 'cheesecake_installability_id': None, + 'classifiers': [...], + 'description': 'An extensible ' + 'framework for ' + 'Python programming, ' + 'with special ' + 'focus\r\n' + 'on event-based ' + 'network programming ' + 'and multiprotocol ' + 'integration.', + 'docs_url': '', + 'download_url': 'UNKNOWN', + 'home_page': 'http://twistedmatrix.com/', + 'keywords': '', + 'license': 'MIT', + 'maintainer': '', + 'maintainer_email': '', + 'name': 'Twisted', + 'package_url': 'http://pypi.python.org/pypi/Twisted', + 'platform': 'UNKNOWN', + 'release_url': 'http://pypi.python.org/pypi/Twisted/12.3.0', + 'requires_python': None, + 'stable_version': None, + 'summary': 'An asynchronous ' + 'networking framework ' + 'written in Python', + 'version': '12.3.0'}, + 'urls': [{...}, {...}]} diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst index 2b9385b..90baec5 100644 --- a/Doc/library/pty.rst +++ b/Doc/library/pty.rst @@ -45,6 +45,9 @@ The :mod:`pty` module defines the following functions: a file descriptor. The defaults try to read 1024 bytes each time they are called. + .. versionchanged:: 3.4 + :func:`spawn` now returns the status value from :func:`os.waitpid` + on the child process. Example ------- diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index 07ddc25..bae8450 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -28,7 +28,7 @@ byte-code cache files in the directory containing the source code. .. function:: compile(file, cfile=None, dfile=None, doraise=False, optimize=-1) - Compile a source file to byte-code and write out the byte-code cache file. + Compile a source file to byte-code and write out the byte-code cache file. The source code is loaded from the file name *file*. The byte-code is written to *cfile*, which defaults to the :PEP:`3147` path, ending in ``.pyc`` (``.pyo`` if optimization is enabled in the current interpreter). @@ -41,6 +41,13 @@ byte-code cache files in the directory containing the source code. is raised. This function returns the path to byte-compiled file, i.e. whatever *cfile* value was used. + If the path that *cfile* becomes (either explicitly specified or computed) + is a symlink or non-regular file, :exc:`FileExistsError` will be raised. + This is to act as a warning that import will turn those paths into regular + files if it is allowed to write byte-compiled files to those paths. This is + a side-effect of import using file renaming to place the final byte-compiled + file into place to prevent concurrent file writing issues. + *optimize* controls the optimization level and is passed to the built-in :func:`compile` function. The default of ``-1`` selects the optimization level of the current interpreter. @@ -50,6 +57,13 @@ byte-code cache files in the directory containing the source code. default was *file* + ``'c'`` (``'o'`` if optimization was enabled). Also added the *optimize* parameter. + .. versionchanged:: 3.4 + Changed code to use :mod:`importlib` for the byte-code cache file writing. + This means file creation/writing semantics now match what :mod:`importlib` + does, e.g. permissions, write-and-move semantics, etc. Also added the + caveat that :exc:`FileExistsError` is raised if *cfile* is a symlink or + non-regular file. + .. function:: main(args=None) diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index e100865..3f520e8 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -84,3 +84,8 @@ Reference Manual pages. .. versionchanged:: 3.2 Added the ``-b`` option, deprecated the ``-g`` option. + +.. versionchanged:: 3.4 + :mod:`pydoc` now uses :func:`inspect.signature` rather than + :func:`inspect.getfullargspec` to extract signature information from + callables. diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index 3d88d85..cb8ab65 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -100,6 +100,11 @@ The :mod:`xml.parsers.expat` module contains two functions: http://www.python.org/ns/ elem1 elem2 + Due to limitations in the ``Expat`` library used by :mod:`pyexpat`, + the :class:`xmlparser` instance returned can only be used to parse a single + XML document. Call ``ParserCreate`` for each document to provide unique + parser instances. + .. seealso:: @@ -119,7 +124,9 @@ XMLParser Objects Parses the contents of the string *data*, calling the appropriate handler functions to process the parsed data. *isfinal* must be true on the final call - to this method. *data* can be the empty string at any time. + to this method; it allows the parsing of a single file in fragments, + not the submission of multiple files. + *data* can be the empty string at any time. .. method:: xmlparser.ParseFile(file) diff --git a/Doc/library/python.rst b/Doc/library/python.rst index b67fbfc..f307d7d 100644 --- a/Doc/library/python.rst +++ b/Doc/library/python.rst @@ -25,4 +25,3 @@ overview: inspect.rst site.rst fpectl.rst - distutils.rst diff --git a/Doc/library/re.rst b/Doc/library/re.rst index dd790e7..7679bc2 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -481,7 +481,7 @@ form. .. note:: The compiled versions of the most recent patterns passed to - :func:`re.match`, :func:`re.search` or :func:`re.compile` are cached, so + :func:`re.compile` and the module-level matching functions are cached, so programs that use only a few regular expressions at a time needn't worry about compiling regular expressions. @@ -563,7 +563,7 @@ form. .. function:: search(pattern, string, flags=0) - Scan through *string* looking for a location where the regular expression + Scan through *string* looking for the first location where the regular expression *pattern* produces a match, and return a corresponding :ref:`match object <match-objects>`. Return ``None`` if no position in the string matches the pattern; note that this is different from finding a zero-length match at some @@ -584,6 +584,16 @@ form. instead (see also :ref:`search-vs-match`). +.. function:: fullmatch(pattern, string, flags=0) + + If the whole *string* matches the regular expression *pattern*, return a + corresponding :ref:`match object <match-objects>`. Return ``None`` if the + string does not match the pattern; note that this is different from a + zero-length match. + + .. versionadded:: 3.4 + + .. function:: split(pattern, string, maxsplit=0, flags=0) Split *string* by the occurrences of *pattern*. If capturing parentheses are @@ -755,7 +765,7 @@ attributes: >>> pattern = re.compile("d") >>> pattern.search("dog") # Match at index 0 - <_sre.SRE_Match object at ...> + <_sre.SRE_Match object; span=(0, 1), match='d'> >>> pattern.search("dog", 1) # No match; search doesn't include the "d" @@ -772,12 +782,30 @@ attributes: >>> pattern = re.compile("o") >>> pattern.match("dog") # No match as "o" is not at the start of "dog". >>> pattern.match("dog", 1) # Match as "o" is the 2nd character of "dog". - <_sre.SRE_Match object at ...> + <_sre.SRE_Match object; span=(1, 2), match='o'> If you want to locate a match anywhere in *string*, use :meth:`~regex.search` instead (see also :ref:`search-vs-match`). +.. method:: regex.fullmatch(string[, pos[, endpos]]) + + If the whole *string* matches this regular expression, return a corresponding + :ref:`match object <match-objects>`. Return ``None`` if the string does not + match the pattern; note that this is different from a zero-length match. + + The optional *pos* and *endpos* parameters have the same meaning as for the + :meth:`~regex.search` method. + + >>> pattern = re.compile("o[gh]") + >>> pattern.fullmatch("dog") # No match as "o" is not at the start of "dog". + >>> pattern.fullmatch("ogre") # No match as not the full string matches. + >>> pattern.fullmatch("doggie", 1, 3) # Matches within given limits. + <_sre.SRE_Match object; span=(1, 3), match='og'> + + .. versionadded:: 3.4 + + .. method:: regex.split(string, maxsplit=0) Identical to the :func:`split` function, using the compiled pattern. @@ -1139,7 +1167,7 @@ For example:: >>> re.match("c", "abcdef") # No match >>> re.search("c", "abcdef") # Match - <_sre.SRE_Match object at ...> + <_sre.SRE_Match object; span=(2, 3), match='c'> Regular expressions beginning with ``'^'`` can be used with :func:`search` to restrict the match at the beginning of the string:: @@ -1147,7 +1175,7 @@ restrict the match at the beginning of the string:: >>> re.match("c", "abcdef") # No match >>> re.search("^c", "abcdef") # No match >>> re.search("^a", "abcdef") # Match - <_sre.SRE_Match object at ...> + <_sre.SRE_Match object; span=(0, 1), match='a'> Note however that in :const:`MULTILINE` mode :func:`match` only matches at the beginning of the string, whereas using :func:`search` with a regular expression @@ -1155,7 +1183,7 @@ beginning with ``'^'`` will match at the beginning of each line. >>> re.match('X', 'A\nB\nX', re.MULTILINE) # No match >>> re.search('^X', 'A\nB\nX', re.MULTILINE) # Match - <_sre.SRE_Match object at ...> + <_sre.SRE_Match object; span=(4, 5), match='X'> Making a Phonebook @@ -1274,9 +1302,9 @@ another one to escape it. For example, the two following lines of code are functionally identical: >>> re.match(r"\W(.)\1\W", " ff ") - <_sre.SRE_Match object at ...> + <_sre.SRE_Match object; span=(0, 4), match=' ff '> >>> re.match("\\W(.)\\1\\W", " ff ") - <_sre.SRE_Match object at ...> + <_sre.SRE_Match object; span=(0, 4), match=' ff '> When one wants to match a literal backslash, it must be escaped in the regular expression. With raw string notation, this means ``r"\\"``. Without raw string @@ -1284,9 +1312,9 @@ notation, one must use ``"\\\\"``, making the following lines of code functionally identical: >>> re.match(r"\\", r"\\") - <_sre.SRE_Match object at ...> + <_sre.SRE_Match object; span=(0, 1), match='\\'> >>> re.match("\\\\", r"\\") - <_sre.SRE_Match object at ...> + <_sre.SRE_Match object; span=(0, 1), match='\\'> Writing a Tokenizer diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 1134619..692310b 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -190,28 +190,32 @@ Example The following example demonstrates how to use the :mod:`readline` module's history reading and writing functions to automatically load and save a history -file named :file:`.pyhist` from the user's home directory. The code below would -normally be executed automatically during interactive sessions from the user's -:envvar:`PYTHONSTARTUP` file. :: +file named :file:`.python_history` from the user's home directory. The code +below would normally be executed automatically during interactive sessions +from the user's :envvar:`PYTHONSTARTUP` file. :: + import atexit import os import readline - histfile = os.path.join(os.path.expanduser("~"), ".pyhist") + + histfile = os.path.join(os.path.expanduser("~"), ".python_history") try: readline.read_history_file(histfile) except FileNotFoundError: pass - import atexit + atexit.register(readline.write_history_file, histfile) - del os, histfile + +This code is actually automatically run when Python is run in +:ref:`interactive mode <tut-interactive>` (see :ref:`rlcompleter-config`). The following example extends the :class:`code.InteractiveConsole` class to support history save/restore. :: - import code - import readline import atexit + import code import os + import readline class HistoryConsole(code.InteractiveConsole): def __init__(self, locals=None, filename="<console>", diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index ed85850..f8112cc 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -74,6 +74,27 @@ this module for those platforms. ``setrlimit`` may also raise :exc:`error` if the underlying system call fails. +.. function:: prlimit(pid, resource[, limits]) + + Combines :func:`setrlimit` and :func:`getrlimit` in one function and + supports to get and set the resources limits of an arbitrary process. If + *pid* is 0, then the call applies to the current process. *resource* and + *limits* have the same meaning as in :func:`setrlimit`, except that + *limits* is optional. + + When *limits* is not given the function returns the *resource* limit of the + process *pid*. When *limits* is given the *resource* limit of the process is + set and the former resource limit is returned. + + Raises :exc:`ProcessLookupError` when *pid* can't be found and + :exc:`PermissionError` when the user doesn't have ``CAP_SYS_RESOURCE`` for + the process. + + Availability: Linux 2.6.36 or later with glibc 2.13 or later + + .. versionadded:: 3.4 + + These symbols define resources whose consumption can be controlled using the :func:`setrlimit` and :func:`getrlimit` functions described below. The values of these symbols are exactly the constants used by C programs. @@ -151,6 +172,80 @@ platform. The maximum area (in bytes) of address space which may be taken by the process. +.. data:: RLIMIT_MSGQUEUE + + The number of bytes that can be allocated for POSIX message queues. + + Availability: Linux 2.6.8 or later. + + .. versionadded:: 3.4 + + +.. data:: RLIMIT_NICE + + The ceiling for the process's nice level (calculated as 20 - rlim_cur). + + Availability: Linux 2.6.12 or later. + + .. versionadded:: 3.4 + + +.. data:: RLIMIT_RTPRIO + + The ceiling of the real-time priority. + + Availability: Linux 2.6.12 or later. + + .. versionadded:: 3.4 + + +.. data:: RLIMIT_RTTIME + + The time limit (in microseconds) on CPU time that a process can spend + under real-time scheduling without making a blocking syscall. + + Availability: Linux 2.6.25 or later. + + .. versionadded:: 3.4 + + +.. data:: RLIMIT_SIGPENDING + + The number of signals which the process may queue. + + Availability: Linux 2.6.8 or later. + + .. versionadded:: 3.4 + +.. data:: RLIMIT_SBSIZE + + The maximum size (in bytes) of socket buffer usage for this user. + This limits the amount of network memory, and hence the amount of mbufs, + that this user may hold at any time. + + Availability: FreeBSD 9 or later. + + .. versionadded:: 3.4 + +.. data:: RLIMIT_SWAP + + The maximum size (in bytes) of the swap space that may be reserved or + used by all of this user id's processes. + This limit is enforced only if bit 1 of the vm.overcommit sysctl is set. + Please see :manpage:`tuning(7)` for a complete description of this sysctl. + + Availability: FreeBSD 9 or later. + + .. versionadded:: 3.4 + +.. data:: RLIMIT_NPTS + + The maximum number of pseudo-terminals created by this user id. + + Availability: FreeBSD 9 or later. + + .. versionadded:: 3.4 + Resource Usage -------------- diff --git a/Doc/library/rlcompleter.rst b/Doc/library/rlcompleter.rst index 633088d..9ed01c7 100644 --- a/Doc/library/rlcompleter.rst +++ b/Doc/library/rlcompleter.rst @@ -27,18 +27,10 @@ Example:: readline.__name__ readline.parse_and_bind( >>> readline. -The :mod:`rlcompleter` module is designed for use with Python's interactive -mode. A user can add the following lines to his or her initialization file -(identified by the :envvar:`PYTHONSTARTUP` environment variable) to get -automatic :kbd:`Tab` completion:: - - try: - import readline - except ImportError: - print("Module readline not available.") - else: - import rlcompleter - readline.parse_and_bind("tab: complete") +The :mod:`rlcompleter` module is designed for use with Python's +:ref:`interactive mode <tut-interactive>`. Unless Python is run with the +:option:`-S` option, the module is automatically imported and configured +(see :ref:`rlcompleter-config`). On platforms without :mod:`readline`, the :class:`Completer` class defined by this module can still be used for custom purposes. diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index 6919bc0..ee9fbcf 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -44,28 +44,22 @@ The :mod:`runpy` module provides two functions: below are defined in the supplied dictionary, those definitions are overridden by :func:`run_module`. - The special global variables ``__name__``, ``__file__``, ``__cached__``, - ``__loader__`` - and ``__package__`` are set in the globals dictionary before the module - code is executed (Note that this is a minimal set of variables - other - variables may be set implicitly as an interpreter implementation detail). + The special global variables ``__name__``, ``__spec__``, ``__file__``, + ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals + dictionary before the module code is executed (Note that this is a + minimal set of variables - other variables may be set implicitly as an + interpreter implementation detail). ``__name__`` is set to *run_name* if this optional argument is not :const:`None`, to ``mod_name + '.__main__'`` if the named module is a package and to the *mod_name* argument otherwise. - ``__file__`` is set to the name provided by the module loader. If the - loader does not make filename information available, this variable is set - to :const:`None`. + ``__spec__`` will be set appropriately for the *actually* imported + module (that is, ``__spec__.name`` will always be *mod_name* or + ``mod_name + '.__main__``, never *run_name*). - ``__cached__`` will be set to ``None``. - - ``__loader__`` is set to the :pep:`302` module loader used to retrieve the - code for the module (This loader may be a wrapper around the standard - import mechanism). - - ``__package__`` is set to *mod_name* if the named module is a package and - to ``mod_name.rpartition('.')[0]`` otherwise. + ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` are + :ref:`set as normal <import-mod-attrs>` based on the module spec. If the argument *alter_sys* is supplied and evaluates to :const:`True`, then ``sys.argv[0]`` is updated with the value of ``__file__`` and @@ -83,8 +77,13 @@ The :mod:`runpy` module provides two functions: Added ability to execute packages by looking for a ``__main__`` submodule. .. versionchanged:: 3.2 - Added ``__cached__`` global variable (see :PEP:`3147`). + Added ``__cached__`` global variable (see :pep:`3147`). + .. versionchanged:: 3.4 + Updated to take advantage of the module spec feature added by + :pep:`451`. This allows ``__cached__`` to be set correctly for modules + run this way, as well as ensuring the real module name is always + accessible as ``__spec__.name``. .. function:: run_path(file_path, init_globals=None, run_name=None) @@ -108,23 +107,28 @@ The :mod:`runpy` module provides two functions: below are defined in the supplied dictionary, those definitions are overridden by :func:`run_path`. - The special global variables ``__name__``, ``__file__``, ``__loader__`` - and ``__package__`` are set in the globals dictionary before the module - code is executed (Note that this is a minimal set of variables - other - variables may be set implicitly as an interpreter implementation detail). + The special global variables ``__name__``, ``__spec__``, ``__file__``, + ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals + dictionary before the module code is executed (Note that this is a + minimal set of variables - other variables may be set implicitly as an + interpreter implementation detail). ``__name__`` is set to *run_name* if this optional argument is not :const:`None` and to ``'<run_path>'`` otherwise. - ``__file__`` is set to the name provided by the module loader. If the - loader does not make filename information available, this variable is set - to :const:`None`. For a simple script, this will be set to ``file_path``. + If the supplied path directly references a script file (whether as source + or as precompiled byte code), then ``__file__`` will be set to the + supplied path, and ``__spec__``, ``__cached__``, ``__loader__`` and + ``__package__`` will all be set to :const:`None`. - ``__loader__`` is set to the :pep:`302` module loader used to retrieve the - code for the module (This loader may be a wrapper around the standard - import mechanism). For a simple script, this will be set to :const:`None`. + ``__spec__`` will be set to :const:`None` if the supplied path is a + direct path to a script (as source or as precompiled bytecode). - ``__package__`` is set to ``__name__.rpartition('.')[0]``. + If the supplied path is a reference to a valid sys.path entry, then + ``__spec__`` will be set appropriately for the imported ``__main__`` + module (that is, ``__spec__.name`` will always be ``__main__``). + ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` will be + :ref:`set as normal <import-mod-attrs>` based on the module spec. A number of alterations are also made to the :mod:`sys` module. Firstly, ``sys.path`` may be altered as described above. ``sys.argv[0]`` is updated @@ -141,6 +145,12 @@ The :mod:`runpy` module provides two functions: .. versionadded:: 3.2 + .. versionchanged:: 3.4 + Updated to take advantage of the module spec feature added by + :pep:`451`. This allows ``__cached__`` to be set correctly in the + case where ``__main__`` is imported from a valid sys.path entry rather + than being executed directly. + .. seealso:: :pep:`338` - Executing modules as scripts @@ -149,6 +159,9 @@ The :mod:`runpy` module provides two functions: :pep:`366` - Main module explicit relative imports PEP written and implemented by Nick Coghlan. + :pep:`451` - A ModuleSpec Type for the Import System + PEP written and implemented by Eric Snow + :ref:`using-on-general` - CPython command line details The :func:`importlib.import_module` function diff --git a/Doc/library/select.rst b/Doc/library/select.rst index b1fd3a8..973a0cc 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -14,6 +14,14 @@ it also works for other file types (in particular, on Unix, it works on pipes). It cannot be used on regular files to determine whether a file has grown since it was last read. +.. note:: + + The :mod:`selectors` module allows high-level and efficient I/O + multiplexing, built upon the :mod:`select` module primitives. Users are + encouraged to use the :mod:`selectors` module instead, unless they want + precise control over the OS-level primitives used. + + The module defines the following: @@ -37,21 +45,37 @@ The module defines the following: increases this value, :c:func:`devpoll` may return an incomplete list of active file descriptors. + The new file descriptor is :ref:`non-inheritable <fd_inheritance>`. + .. versionadded:: 3.3 + .. versionchanged:: 3.4 + The new file descriptor is now non-inheritable. + .. function:: epoll(sizehint=-1, flags=0) (Only supported on Linux 2.5.44 and newer.) Return an edge polling object, which can be used as Edge or Level Triggered interface for I/O events. *sizehint* is deprecated and completely ignored. *flags* can be set to :const:`EPOLL_CLOEXEC`, which causes the epoll descriptor to be closed - automatically when :func:`os.execve` is called. See section - :ref:`epoll-objects` below for the methods supported by epolling objects. + automatically when :func:`os.execve` is called. + + See the :ref:`epoll-objects` section below for the methods supported by + epolling objects. + + ``epoll`` objects support the context management protocol: when used in a + :keyword:`with` statement, the new file descriptor is automatically closed + at the end of the block. + The new file descriptor is :ref:`non-inheritable <fd_inheritance>`. .. versionchanged:: 3.3 Added the *flags* parameter. + .. versionchanged:: 3.4 + Support for the :keyword:`with` statement was added. + The new file descriptor is now non-inheritable. + .. function:: poll() @@ -66,6 +90,11 @@ The module defines the following: (Only supported on BSD.) Returns a kernel queue object; see section :ref:`kqueue-objects` below for the methods supported by kqueue objects. + The new file descriptor is :ref:`non-inheritable <fd_inheritance>`. + + .. versionchanged:: 3.4 + The new file descriptor is now non-inheritable. + .. function:: kevent(ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=0) @@ -144,6 +173,27 @@ descriptors), ``/dev/poll`` is O(active file descriptors). object. +.. method:: devpoll.close() + + Close the file descriptor of the polling object. + + .. versionadded:: 3.4 + + +.. attribute:: devpoll.closed + + ``True`` if the polling object is closed. + + .. versionadded:: 3.4 + + +.. method:: devpoll.fileno() + + Return the file descriptor number of the polling object. + + .. versionadded:: 3.4 + + .. method:: devpoll.register(fd[, eventmask]) Register a file descriptor with the polling object. Future calls to the @@ -241,6 +291,11 @@ Edge and Level Trigger Polling (epoll) Objects Close the control file descriptor of the epoll object. +.. attribute:: epoll.closed + + ``True`` if the epoll object is closed. + + .. method:: epoll.fileno() Return the file descriptor number of the control fd. @@ -258,7 +313,7 @@ Edge and Level Trigger Polling (epoll) Objects .. method:: epoll.modify(fd, eventmask) - Modify a register file descriptor. + Modify a registered file descriptor. .. method:: epoll.unregister(fd) @@ -322,7 +377,7 @@ linearly scanned again. :c:func:`select` is O(highest file descriptor), while Modifies an already registered fd. This has the same effect as ``register(fd, eventmask)``. Attempting to modify a file descriptor - that was never registered causes an :exc:`IOError` exception with errno + that was never registered causes an :exc:`OSError` exception with errno :const:`ENOENT` to be raised. @@ -360,6 +415,11 @@ Kqueue Objects Close the control file descriptor of the kqueue object. +.. attribute:: kqueue.closed + + ``True`` if the kqueue object is closed. + + .. method:: kqueue.fileno() Return the file descriptor number of the control fd. diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst new file mode 100644 index 0000000..98377c8 --- /dev/null +++ b/Doc/library/selectors.rst @@ -0,0 +1,257 @@ +:mod:`selectors` -- High-level I/O multiplexing +=============================================== + +.. module:: selectors + :synopsis: High-level I/O multiplexing. + +.. versionadded:: 3.4 + + +Introduction +------------ + +This module allows high-level and efficient I/O multiplexing, built upon the +:mod:`select` module primitives. Users are encouraged to use this module +instead, unless they want precise control over the OS-level primitives used. + +It defines a :class:`BaseSelector` abstract base class, along with several +concrete implementations (:class:`KqueueSelector`, :class:`EpollSelector`...), +that can be used to wait for I/O readiness notification on multiple file +objects. In the following, "file object" refers to any object with a +:meth:`fileno()` method, or a raw file descriptor. See :term:`file object`. + +:class:`DefaultSelector` is an alias to the most efficient implementation +available on the current platform: this should be the default choice for most +users. + +.. note:: + The type of file objects supported depends on the platform: on Windows, + sockets are supported, but not pipes, whereas on Unix, both are supported + (some other types may be supported as well, such as fifos or special file + devices). + +.. seealso:: + + :mod:`select` + Low-level I/O multiplexing module. + + +Classes +------- + +Classes hierarchy:: + + BaseSelector + +-- SelectSelector + +-- PollSelector + +-- EpollSelector + +-- KqueueSelector + + +In the following, *events* is a bitwise mask indicating which I/O events should +be waited for on a given file object. It can be a combination of the constants +below: + + +-----------------------+-----------------------------------------------+ + | Constant | Meaning | + +=======================+===============================================+ + | :const:`EVENT_READ` | Available for read | + +-----------------------+-----------------------------------------------+ + | :const:`EVENT_WRITE` | Available for write | + +-----------------------+-----------------------------------------------+ + + +.. class:: SelectorKey + + A :class:`SelectorKey` is a :class:`~collections.namedtuple` used to + associate a file object to its underlying file decriptor, selected event + mask and attached data. It is returned by several :class:`BaseSelector` + methods. + + .. attribute:: fileobj + + File object registered. + + .. attribute:: fd + + Underlying file descriptor. + + .. attribute:: events + + Events that must be waited for on this file object. + + .. attribute:: data + + Optional opaque data associated to this file object: for example, this + could be used to store a per-client session ID. + + +.. class:: BaseSelector + + A :class:`BaseSelector` is used to wait for I/O event readiness on multiple + file objects. It supports file stream registration, unregistration, and a + method to wait for I/O events on those streams, with an optional timeout. + It's an abstract base class, so cannot be instantiated. Use + :class:`DefaultSelector` instead, or one of :class:`SelectSelector`, + :class:`KqueueSelector` etc. if you want to specifically use an + implementation, and your platform supports it. + :class:`BaseSelector` and its concrete implementations support the + :term:`context manager` protocol. + + .. method:: register(fileobj, events, data=None) + + Register a file object for selection, monitoring it for I/O events. + + *fileobj* is the file object to monitor. It may either be an integer + file descriptor or an object with a ``fileno()`` method. + *events* is a bitwise mask of events to monitor. + *data* is an opaque object. + + This returns a new :class:`SelectorKey` instance, or raises a + :exc:`ValueError` in case of invalid event mask or file descriptor, or + :exc:`KeyError` if the file object is already registered. + + .. method:: unregister(fileobj) + + Unregister a file object from selection, removing it from monitoring. A + file object shall be unregistered prior to being closed. + + *fileobj* must be a file object previously registered. + + This returns the associated :class:`SelectorKey` instance, or raises a + :exc:`KeyError` if *fileobj* is not registered. It will raise + :exc:`ValueError` if *fileobj* is invalid (e.g. it has no ``fileno()`` + method or its ``fileno()`` method has an invalid return value). + + .. method:: modify(fileobj, events, data=None) + + Change a registered file object's monitored events or attached data. + + This is equivalent to :meth:`BaseSelector.unregister(fileobj)` followed + by :meth:`BaseSelector.register(fileobj, events, data)`, except that it + can be implemented more efficiently. + + This returns a new :class:`SelectorKey` instance, or raises a + :exc:`ValueError` in case of invalid event mask or file descriptor, or + :exc:`KeyError` if the file object is not registered. + + .. method:: select(timeout=None) + + Wait until some registered file objects become ready, or the timeout + expires. + + If ``timeout > 0``, this specifies the maximum wait time, in seconds. + If ``timeout <= 0``, the call won't block, and will report the currently + ready file objects. + If *timeout* is ``None``, the call will block until a monitored file object + becomes ready. + + This returns a list of ``(key, events)`` tuples, one for each ready file + object. + + *key* is the :class:`SelectorKey` instance corresponding to a ready file + object. + *events* is a bitmask of events ready on this file object. + + .. note:: + This method can return before any file object becomes ready or the + timeout has elapsed if the current process receives a signal: in this + case, an empty list will be returned. + + .. method:: close() + + Close the selector. + + This must be called to make sure that any underlying resource is freed. + The selector shall not be used once it has been closed. + + .. method:: get_key(fileobj) + + Return the key associated with a registered file object. + + This returns the :class:`SelectorKey` instance associated to this file + object, or raises :exc:`KeyError` if the file object is not registered. + + .. method:: get_map() + + Return a mapping of file objects to selector keys. + + This returns a :class:`~collections.abc.Mapping` instance mapping + registered file objects to their associated :class:`SelectorKey` + instance. + + +.. class:: DefaultSelector() + + The default selector class, using the most efficient implementation + available on the current platform. This should be the default choice for + most users. + + +.. class:: SelectSelector() + + :func:`select.select`-based selector. + + +.. class:: PollSelector() + + :func:`select.poll`-based selector. + + +.. class:: EpollSelector() + + :func:`select.epoll`-based selector. + + .. method:: fileno() + + This returns the file descriptor used by the underlying + :func:`select.epoll` object. + + +.. class:: KqueueSelector() + + :func:`select.kqueue`-based selector. + + .. method:: fileno() + + This returns the file descriptor used by the underlying + :func:`select.kqueue` object. + + +Examples +-------- + +Here is a simple echo server implementation:: + + import selectors + import socket + + sel = selectors.DefaultSelector() + + def accept(sock, mask): + conn, addr = sock.accept() # Should be ready + print('accepted', conn, 'from', addr) + conn.setblocking(False) + sel.register(conn, selectors.EVENT_READ, read) + + def read(conn, mask): + data = conn.recv(1000) # Should be ready + if data: + print('echoing', repr(data), 'to', conn) + conn.send(data) # Hope it won't block + else: + print('closing', conn) + sel.unregister(conn) + conn.close() + + sock = socket.socket() + sock.bind(('localhost', 1234)) + sock.listen(100) + sock.setblocking(False) + sel.register(sock, selectors.EVENT_READ, accept) + + while True: + events = sel.select() + for key, mask in events: + callback = key.data + callback(key.fileobj, mask) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 048f80a..22e202d 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -44,8 +44,11 @@ lots of shared sub-objects. The keys are ordinary strings. .. note:: Do not rely on the shelf being closed automatically; always call - :meth:`close` explicitly when you don't need it any more, or use a - :keyword:`with` statement with :func:`contextlib.closing`. + :meth:`~Shelf.close` explicitly when you don't need it any more, or + use :func:`shelve.open` as a context manager:: + + with shelve.open('spam') as db: + db['eggs'] = 'eggs' .. warning:: @@ -118,10 +121,16 @@ Restrictions The *keyencoding* parameter is the encoding used to encode keys before they are used with the underlying dict. - .. versionadded:: 3.2 - The *keyencoding* parameter; previously, keys were always encoded in + A :class:`Shelf` object can also be used as a context manager, in which + case it will be automatically closed when the :keyword:`with` block ends. + + .. versionchanged:: 3.2 + Added the *keyencoding* parameter; previously, keys were always encoded in UTF-8. + .. versionchanged:: 3.4 + Added context manager support. + .. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8') diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index a1457d0..e4f348c 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -53,7 +53,7 @@ Directory and files operations *dst* and return *dst*. *src* and *dst* are path names given as strings. *dst* must be the complete target file name; look at :func:`shutil.copy` for a copy that accepts a target directory path. If *src* and *dst* - specify the same file, :exc:`Error` is raised. + specify the same file, :exc:`SameFileError` is raised. The destination location must be writable; otherwise, an :exc:`OSError` exception will be raised. If *dst* already exists, it will be replaced. @@ -69,6 +69,19 @@ Directory and files operations Added *follow_symlinks* argument. Now returns *dst*. + .. versionchanged:: 3.4 + Raise :exc:`SameFileError` instead of :exc:`Error`. Since the former is + a subclass of the latter, this change is backward compatible. + + +.. exception:: SameFileError + + This exception is raised if source and destination in :func:`copyfile` + are the same file. + + .. versionadded:: 3.4 + + .. function:: copymode(src, dst, *, follow_symlinks=True) Copy the permission bits from *src* to *dst*. The file contents, owner, and @@ -380,11 +393,10 @@ provided by this module. :: errors.extend(err.args[0]) try: copystat(src, dst) - except WindowsError: - # can't copy file access times on Windows - pass except OSError as why: - errors.extend((src, dst, str(why))) + # can't copy file access times on Windows + if why.winerror is None: + errors.extend((src, dst, str(why))) if errors: raise Error(errors) diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 36b80c3..e57b8cc 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -38,6 +38,9 @@ Unix and Macintosh). For each of the distinct head-tail combinations, it sees if it refers to an existing directory, and if so, adds it to ``sys.path`` and also inspects the newly added path for configuration files. +.. deprecated:: 3.4 + Support for the "site-python" directory will be removed in 3.5. + If a file named "pyvenv.cfg" exists one directory above sys.executable, sys.prefix and sys.exec_prefix are set to that directory and it is also checked for site-packages and site-python (sys.base_prefix and @@ -96,7 +99,11 @@ After these path manipulations, an attempt is made to import a module named :mod:`sitecustomize`, which can perform arbitrary site-specific customizations. It is typically created by a system administrator in the site-packages directory. If this import fails with an :exc:`ImportError` exception, it is -silently ignored. +silently ignored. If Python is started without output streams available, as +with :file:`pythonw.exe` on Windows (which is used by default to start IDLE), +attempted output from :mod:`sitecustomize` is ignored. Any exception other +than :exc:`ImportError` causes a silent and perhaps mysterious failure of the +process. .. index:: module: usercustomize @@ -111,6 +118,27 @@ empty, and the path manipulations are skipped; however the import of :mod:`sitecustomize` and :mod:`usercustomize` is still attempted. +.. _rlcompleter-config: + +Readline configuration +---------------------- + +On systems that support :mod:`readline`, this module will also import and +configure the :mod:`rlcompleter` module, if Python is started in +:ref:`interactive mode <tut-interactive>` and without the :option:`-S` option. +The default behavior is enable tab-completion and to use +:file:`~/.python_history` as the history save file. To disable it, delete (or +override) the :data:`sys.__interactivehook__` attribute in your +:mod:`sitecustomize` or :mod:`usercustomize` module or your +:envvar:`PYTHONSTARTUP` file. + +.. versionchanged:: 3.4 + Activation of rlcompleter and history was made automatic. + + +Module contents +--------------- + .. data:: PREFIXES A list of prefixes for site-packages directories. @@ -153,8 +181,7 @@ empty, and the path manipulations are skipped; however the import of Adds all the standard site-specific directories to the module search path. This function is called automatically when this module is imported, - unless the :program:`python` interpreter was started with the :option:`-S` - flag. + unless the Python interpreter was started with the :option:`-S` flag. .. versionchanged:: 3.3 This function used to be called unconditionnally. diff --git a/Doc/library/smtpd.rst b/Doc/library/smtpd.rst index f305818..3ebed06 100644 --- a/Doc/library/smtpd.rst +++ b/Doc/library/smtpd.rst @@ -27,7 +27,8 @@ SMTPServer Objects ------------------ -.. class:: SMTPServer(localaddr, remoteaddr, data_size_limit=33554432) +.. class:: SMTPServer(localaddr, remoteaddr, data_size_limit=33554432,\ + map=None) Create a new :class:`SMTPServer` object, which binds to local address *localaddr*. It will treat *remoteaddr* as an upstream SMTP relayer. It @@ -38,6 +39,8 @@ SMTPServer Objects accepted in a ``DATA`` command. A value of ``None`` or ``0`` means no limit. + A dictionary can be specified in *map* to avoid using a global socket map. + .. method:: process_message(peer, mailfrom, rcpttos, data) Raise :exc:`NotImplementedError` exception. Override this in subclasses to @@ -53,6 +56,9 @@ SMTPServer Objects Override this in subclasses to use a custom :class:`SMTPChannel` for managing SMTP clients. + .. versionchanged:: 3.4 + The *map* argument was added. + DebuggingServer Objects ----------------------- @@ -90,11 +96,20 @@ MailmanProxy Objects SMTPChannel Objects ------------------- -.. class:: SMTPChannel(server, conn, addr) +.. class:: SMTPChannel(server, conn, addr, data_size_limit=33554432,\ + map=None)) Create a new :class:`SMTPChannel` object which manages the communication between the server and a single SMTP client. + *conn* and *addr* are as per the instance variables described below. + + *data_size_limit* specifies the maximum number of bytes that will be + accepted in a ``DATA`` command. A value of ``None`` or ``0`` means no + limit. + + A dictionary can be specified in *map* to avoid using a global socket map. + To use a custom SMTPChannel implementation you need to override the :attr:`SMTPServer.channel_class` of your :class:`SMTPServer`. diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index addc6be..8e1bfb5 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -32,7 +32,8 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). than a success code, an :exc:`SMTPConnectError` is raised. The optional *timeout* parameter specifies a timeout in seconds for blocking operations like the connection attempt (if not specified, the global default timeout - setting will be used). The optional source_address parameter allows to bind + setting will be used). If the timeout expires, :exc:`socket.timeout` is + raised. The optional source_address parameter allows to bind to some specific source address in a machine with multiple network interfaces, and/or to some specific source TCP port. It takes a 2-tuple (host, port), for the socket to bind to as its source address before @@ -69,20 +70,15 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). required from the beginning of the connection and using :meth:`starttls` is not appropriate. If *host* is not specified, the local host is used. If *port* is zero, the standard SMTP-over-SSL port (465) is used. The optional - arguments *local_hostname* and *source_address* have the same meaning as - they do in the :class:`SMTP` class. *keyfile* and *certfile* are also - optional, and can contain a PEM formatted private key and certificate chain - file for the SSL connection. *context* also optional, can contain a - SSLContext, and is an alternative to keyfile and certfile; If it is - specified both keyfile and certfile must be None. The optional *timeout* - parameter specifies a timeout in seconds for blocking operations like the - connection attempt (if not specified, the global default timeout setting - will be used). The optional source_address parameter allows to bind to some - specific source address in a machine with multiple network interfaces, - and/or to some specific source tcp port. It takes a 2-tuple (host, port), - for the socket to bind to as its source address before connecting. If - omitted (or if host or port are ``''`` and/or 0 respectively) the OS default - behavior will be used. + arguments *local_hostname*, *timeout* and *source_address* have the same + meaning as they do in the :class:`SMTP` class. *context*, also optional, + can contain a :class:`~ssl.SSLContext` and allows to configure various + aspects of the secure connection. Please read :ref:`ssl-security` for + best practices. + + *keyfile* and *certfile* are a legacy alternative to *context*, and can + point to a PEM formatted private key and certificate chain file for the + SSL connection. .. versionchanged:: 3.3 *context* was added. @@ -90,6 +86,10 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). .. versionchanged:: 3.3 source_address argument was added. + .. versionchanged:: 3.4 + The class now supports hostname check with + :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see + :data:`ssl.HAS_SNI`). .. class:: LMTP(host='', port=LMTP_PORT, local_hostname=None, source_address=None) @@ -110,8 +110,11 @@ A nice selection of exceptions is defined as well: .. exception:: SMTPException - The base exception class for all the other exceptions provided by this - module. + Subclass of :exc:`OSError` that is the base exception class for all + the other exceptions provided by this module. + + .. versionchanged:: 3.4 + SMTPException became subclass of :exc:`OSError` .. exception:: SMTPServerDisconnected @@ -316,6 +319,11 @@ An :class:`SMTP` instance has the following methods: .. versionchanged:: 3.3 *context* was added. + .. versionchanged:: 3.4 + The method now supports hostname check with + :attr:`SSLContext.check_hostname` and *Server Name Indicator* (see + :data:`~ssl.HAS_SNI`). + .. method:: SMTP.sendmail(from_addr, to_addrs, msg, mail_options=[], rcpt_options=[]) diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 3a49eed..54c6bad 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -6,8 +6,7 @@ This module provides access to the BSD *socket* interface. It is available on -all modern Unix systems, Windows, MacOS, OS/2, and probably additional -platforms. +all modern Unix systems, Windows, MacOS, and probably additional platforms. .. note:: @@ -107,8 +106,8 @@ created. Socket addresses are represented as follows: .. versionadded:: 3.3 -- Certain other address families (:const:`AF_BLUETOOTH`, :const:`AF_PACKET`) - support specific representations. +- Certain other address families (:const:`AF_BLUETOOTH`, :const:`AF_PACKET`, + :const:`AF_CAN`) support specific representations. .. XXX document them! @@ -264,6 +263,16 @@ Constants .. versionadded:: 3.3 +.. data:: CAN_BCM + CAN_BCM_* + + CAN_BCM, in the CAN protocol family, is the broadcast manager (BCM) protocol. + Broadcast manager constants, documented in the Linux documentation, are also + defined in the socket module. + + Availability: Linux >= 2.6.25. + + .. versionadded:: 3.4 .. data:: AF_RDS PF_RDS @@ -290,6 +299,11 @@ Constants TIPC related constants, matching the ones exported by the C socket API. See the TIPC documentation for more information. +.. data:: AF_LINK + + Availability: BSD, OSX. + + .. versionadded:: 3.4 .. data:: has_ipv6 @@ -306,20 +320,29 @@ Creating sockets The following functions all create :ref:`socket objects <socket-objects>`. -.. function:: socket([family[, type[, proto]]]) +.. function:: socket(family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None) Create a new socket using the given address family, socket type and protocol number. The address family should be :const:`AF_INET` (the default), :const:`AF_INET6`, :const:`AF_UNIX`, :const:`AF_CAN` or :const:`AF_RDS`. The socket type should be :const:`SOCK_STREAM` (the default), :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other ``SOCK_`` - constants. The protocol number is usually zero and may be omitted in that - case or :const:`CAN_RAW` in case the address family is :const:`AF_CAN`. + constants. The protocol number is usually zero and may be omitted or in the + case where the address family is :const:`AF_CAN` the protocol should be one + of :const:`CAN_RAW` or :const:`CAN_BCM`. + + The newly created socket is :ref:`non-inheritable <fd_inheritance>`. .. versionchanged:: 3.3 The AF_CAN family was added. The AF_RDS family was added. + .. versionchanged:: 3.4 + The CAN_BCM protocol was added. + + .. versionchanged:: 3.4 + The returned socket is now non-inheritable. + .. function:: socketpair([family[, type[, proto]]]) @@ -329,10 +352,15 @@ The following functions all create :ref:`socket objects <socket-objects>`. if defined on the platform; otherwise, the default is :const:`AF_INET`. Availability: Unix. + The newly created sockets are :ref:`non-inheritable <fd_inheritance>`. + .. versionchanged:: 3.2 The returned socket objects now support the whole socket API, rather than a subset. + .. versionchanged:: 3.4 + The returned sockets are now non-inheritable. + .. function:: create_connection(address[, timeout[, source_address]]) @@ -360,7 +388,7 @@ The following functions all create :ref:`socket objects <socket-objects>`. support for the :keyword:`with` statement was added. -.. function:: fromfd(fd, family, type[, proto]) +.. function:: fromfd(fd, family, type, proto=0) Duplicate the file descriptor *fd* (an integer as returned by a file object's :meth:`fileno` method) and build a socket object from the result. Address @@ -371,6 +399,11 @@ The following functions all create :ref:`socket objects <socket-objects>`. a socket passed to a program as standard input or output (such as a server started by the Unix inet daemon). The socket is assumed to be in blocking mode. + The newly created socket is :ref:`non-inheritable <fd_inheritance>`. + + .. versionchanged:: 3.4 + The returned socket is now non-inheritable. + .. function:: fromshare(data) @@ -598,7 +631,10 @@ The :mod:`socket` module also offers various network-related services: both the value of *address_family* and the underlying implementation of :c:func:`inet_pton`. - Availability: Unix (maybe not all platforms). + Availability: Unix (maybe not all platforms), Windows. + + .. versionchanged:: 3.4 + Windows support added .. function:: inet_ntop(address_family, packed_ip) @@ -614,7 +650,10 @@ The :mod:`socket` module also offers various network-related services: specified address family, :exc:`ValueError` will be raised. A :exc:`OSError` is raised for errors from the call to :func:`inet_ntop`. - Availability: Unix (maybe not all platforms). + Availability: Unix (maybe not all platforms), Windows. + + .. versionchanged:: 3.4 + Windows support added .. @@ -735,6 +774,11 @@ to sockets. *new* socket object usable to send and receive data on the connection, and *address* is the address bound to the socket on the other end of the connection. + The newly created socket is :ref:`non-inheritable <fd_inheritance>`. + + .. versionchanged:: 3.4 + The socket is now non-inheritable. + .. method:: socket.bind(address) @@ -787,6 +831,16 @@ to sockets. .. versionadded:: 3.2 +.. method:: socket.dup() + + Duplicate the socket. + + The newly created socket is :ref:`non-inheritable <fd_inheritance>`. + + .. versionchanged:: 3.4 + The socket is now non-inheritable. + + .. method:: socket.fileno() Return the socket's file descriptor (a small integer). This is useful with @@ -797,6 +851,15 @@ to sockets. this limitation. +.. method:: socket.get_inheritable() + + Get the :ref:`inheritable flag <fd_inheritance>` of the socket's file + descriptor or socket's handle: ``True`` if the socket can be inherited in + child processes, ``False`` if it cannot. + + .. versionadded:: 3.4 + + .. method:: socket.getpeername() Return the remote address to which the socket is connected. This is useful to @@ -1083,6 +1146,14 @@ to sockets. .. versionadded:: 3.3 +.. method:: socket.set_inheritable(inheritable) + + Set the :ref:`inheritable flag <fd_inheritance>` of the socket's file + descriptor or socket's handle. + + .. versionadded:: 3.4 + + .. method:: socket.setblocking(flag) Set blocking or non-blocking mode of the socket: if *flag* is false, the @@ -1364,7 +1435,16 @@ the interface:: s.ioctl(socket.SIO_RCVALL, socket.RCVALL_OFF) The last example shows how to use the socket interface to communicate to a CAN -network. This example might require special priviledge:: +network using the raw socket protocol. To use CAN with the broadcast +manager protocol instead, open a socket with:: + + socket.socket(socket.AF_CAN, socket.SOCK_DGRAM, socket.CAN_BCM) + +After binding (:const:`CAN_RAW`) or connecting (:const:`CAN_BCM`) the socket, you +can use the :meth:`socket.send`, and the :meth:`socket.recv` operations (and +their counterparts) on the socket object as usual. + +This example might require special priviledge:: import socket import struct diff --git a/Doc/library/spwd.rst b/Doc/library/spwd.rst index add611f..58be78f 100644 --- a/Doc/library/spwd.rst +++ b/Doc/library/spwd.rst @@ -19,9 +19,9 @@ below, see ``<shadow.h>``): +-------+---------------+---------------------------------+ | Index | Attribute | Meaning | +=======+===============+=================================+ -| 0 | ``sp_nam`` | Login name | +| 0 | ``sp_namp`` | Login name | +-------+---------------+---------------------------------+ -| 1 | ``sp_pwd`` | Encrypted password | +| 1 | ``sp_pwdp`` | Encrypted password | +-------+---------------+---------------------------------+ | 2 | ``sp_lstchg`` | Date of last change | +-------+---------------+---------------------------------+ @@ -36,15 +36,15 @@ below, see ``<shadow.h>``): +-------+---------------+---------------------------------+ | 6 | ``sp_inact`` | Number of days after password | | | | expires until account is | -| | | blocked | +| | | disabled | +-------+---------------+---------------------------------+ | 7 | ``sp_expire`` | Number of days since 1970-01-01 | -| | | until account is disabled | +| | | when account expires | +-------+---------------+---------------------------------+ | 8 | ``sp_flag`` | Reserved | +-------+---------------+---------------------------------+ -The sp_nam and sp_pwd items are strings, all others are integers. +The sp_namp and sp_pwdp items are strings, all others are integers. :exc:`KeyError` is raised if the entry asked for cannot be found. The following functions are defined: diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 8a50f87..6097e7a 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -13,8 +13,8 @@ SQLite for internal data storage. It's also possible to prototype an application using SQLite and then port the code to a larger database such as PostgreSQL or Oracle. -sqlite3 was written by Gerhard Häring and provides a SQL interface compliant -with the DB-API 2.0 specification described by :pep:`249`. +The sqlite3 module was written by Gerhard Häring. It provides a SQL interface +compliant with the DB-API 2.0 specification described by :pep:`249`. To use the module, you must first create a :class:`Connection` object that represents the database. Here the data will be stored in the @@ -31,23 +31,29 @@ and call its :meth:`~Cursor.execute` method to perform SQL commands:: c = conn.cursor() # Create table - c.execute('''create table stocks - (date text, trans text, symbol text, - qty real, price real)''') + c.execute('''CREATE TABLE stocks + (date text, trans text, symbol text, qty real, price real)''') # Insert a row of data - c.execute("""insert into stocks - values ('2006-01-05','BUY','RHAT',100,35.14)""") + c.execute("INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14)") # Save (commit) the changes conn.commit() - # We can also close the cursor if we are done with it - c.close() + # We can also close the connection if we are done with it. + # Just be sure any changes have been committed or they will be lost. + conn.close() + +The data you've saved is persistent and is available in subsequent sessions:: + + import sqlite3 + conn = sqlite3.connect('example.db') + c = conn.cursor() Usually your SQL operations will need to use values from Python variables. You shouldn't assemble your query using Python's string operations because doing so -is insecure; it makes your program vulnerable to an SQL injection attack. +is insecure; it makes your program vulnerable to an SQL injection attack +(see http://xkcd.com/327/ for humorous example of what can go wrong). Instead, use the DB-API's parameter substitution. Put ``?`` as a placeholder wherever you want to use a value, and then provide a tuple of values as the @@ -56,19 +62,20 @@ modules may use a different placeholder, such as ``%s`` or ``:1``.) For example:: # Never do this -- insecure! - symbol = 'IBM' - c.execute("select * from stocks where symbol = '%s'" % symbol) + symbol = 'RHAT' + c.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol) # Do this instead - t = ('IBM',) - c.execute('select * from stocks where symbol=?', t) + t = ('RHAT',) + c.execute('SELECT * FROM stocks WHERE symbol=?', t) + print(c.fetchone()) - # Larger example - for t in [('2006-03-28', 'BUY', 'IBM', 1000, 45.00), - ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00), - ('2006-04-06', 'SELL', 'IBM', 500, 53.00), - ]: - c.execute('insert into stocks values (?,?,?,?,?)', t) + # Larger example that inserts many records at a time + purchases = [('2006-03-28', 'BUY', 'IBM', 1000, 45.00), + ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00), + ('2006-04-06', 'SELL', 'IBM', 500, 53.00), + ] + c.executemany('INSERT INTO stocks VALUES (?,?,?,?,?)', purchases) To retrieve data after executing a SELECT statement, you can either treat the cursor as an :term:`iterator`, call the cursor's :meth:`~Cursor.fetchone` method to @@ -77,16 +84,13 @@ matching rows. This example uses the iterator form:: - >>> c = conn.cursor() - >>> c.execute('select * from stocks order by price') - >>> for row in c: - ... print(row) - ... + >>> for row in c.execute('SELECT * FROM stocks ORDER BY price'): + print(row) + ('2006-01-05', 'BUY', 'RHAT', 100, 35.14) ('2006-03-28', 'BUY', 'IBM', 1000, 45.0) ('2006-04-06', 'SELL', 'IBM', 500, 53.0) - ('2006-04-05', 'BUY', 'MSOFT', 1000, 72.0) - >>> + ('2006-04-05', 'BUY', 'MSFT', 1000, 72.0) .. seealso:: @@ -99,6 +103,9 @@ This example uses the iterator form:: The SQLite web page; the documentation describes the syntax and the available data types for the supported SQL dialect. + http://www.w3schools.com/sql/ + Tutorial, reference and examples for learning SQL syntax. + :pep:`249` - Database API Specification 2.0 PEP written by Marc-André Lemburg. @@ -159,7 +166,7 @@ Module functions and constants first blank for the column name: the column name would simply be "x". -.. function:: connect(database[, timeout, detect_types, isolation_level, check_same_thread, factory, cached_statements]) +.. function:: connect(database[, timeout, detect_types, isolation_level, check_same_thread, factory, cached_statements, uri]) Opens a connection to the SQLite database file *database*. You can use ``":memory:"`` to open a database connection to a database that resides in RAM @@ -195,6 +202,18 @@ Module functions and constants for the connection, you can set the *cached_statements* parameter. The currently implemented default is to cache 100 statements. + If *uri* is true, *database* is interpreted as a URI. This allows you + to specify options. For example, to open a database in read-only mode + you can use:: + + db = sqlite3.connect('file:path/to/database?mode=ro', uri=True) + + More information about this feature, including a list of recognized options, can + be found in the `SQLite URI documentation <http://www.sqlite.org/uri.html>`_. + + .. versionchanged:: 3.4 + Added the *uri* parameter. + .. function:: register_converter(typename, callable) @@ -505,7 +524,7 @@ Cursor Objects .. method:: execute(sql, [parameters]) - Executes an SQL statement. The SQL statement may be parametrized (i. e. + Executes an SQL statement. The SQL statement may be parameterized (i. e. placeholders instead of SQL literals). The :mod:`sqlite3` module supports two kinds of placeholders: question marks (qmark style) and named placeholders (named style). @@ -627,7 +646,7 @@ Row Objects .. method:: keys - This method returns a tuple of column names. Immediately after a query, + This method returns a list of column names. Immediately after a query, it is the first member of each tuple in :attr:`Cursor.description`. Let's assume we initialize a table as in the example given above:: @@ -702,19 +721,20 @@ The following Python types can thus be sent to SQLite without any problem: This is how SQLite types are converted to Python types by default: -+-------------+---------------------------------------------+ -| SQLite type | Python type | -+=============+=============================================+ -| ``NULL`` | :const:`None` | -+-------------+---------------------------------------------+ -| ``INTEGER`` | :class:`int` | -+-------------+---------------------------------------------+ -| ``REAL`` | :class:`float` | -+-------------+---------------------------------------------+ -| ``TEXT`` | depends on text_factory, str by default | -+-------------+---------------------------------------------+ -| ``BLOB`` | :class:`bytes` | -+-------------+---------------------------------------------+ ++-------------+----------------------------------------------+ +| SQLite type | Python type | ++=============+==============================================+ +| ``NULL`` | :const:`None` | ++-------------+----------------------------------------------+ +| ``INTEGER`` | :class:`int` | ++-------------+----------------------------------------------+ +| ``REAL`` | :class:`float` | ++-------------+----------------------------------------------+ +| ``TEXT`` | depends on :attr:`~Connection.text_factory`, | +| | :class:`str` by default | ++-------------+----------------------------------------------+ +| ``BLOB`` | :class:`bytes` | ++-------------+----------------------------------------------+ The type system of the :mod:`sqlite3` module is extensible in two ways: you can store additional Python types in a SQLite database via object adaptation, and @@ -730,9 +750,6 @@ use other Python types with SQLite, you must **adapt** them to one of the sqlite3 module's supported types for SQLite: one of NoneType, int, float, str, bytes. -The :mod:`sqlite3` module uses Python object adaptation, as described in -:pep:`246` for this. The protocol to use is :class:`PrepareProtocol`. - There are two ways to enable the :mod:`sqlite3` module to adapt a custom Python type to one of the supported ones. @@ -788,8 +805,8 @@ and constructs a :class:`Point` object from it. .. note:: - Converter functions **always** get called with a string, no matter under which - data type you sent the value to SQLite. + Converter functions **always** get called with a :class:`bytes` object, no + matter under which data type you sent the value to SQLite. :: diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index aa5a52f..b473c45 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -26,7 +26,8 @@ probably additional platforms, as long as OpenSSL is installed on that platform. Some behavior may be platform dependent, since calls are made to the operating system socket APIs. The installed version of OpenSSL may also - cause variations in behavior. + cause variations in behavior. For example, TLSv1.1 and TLSv1.2 come with + openssl version 1.0.1. .. warning:: Don't use this module without reading the :ref:`ssl-security`. Doing so @@ -186,14 +187,16 @@ instead. .. table:: - ======================== ========= ========= ========== ========= - *client* / **server** **SSLv2** **SSLv3** **SSLv23** **TLSv1** - ------------------------ --------- --------- ---------- --------- - *SSLv2* yes no yes no - *SSLv3* no yes yes no - *SSLv23* yes no yes no - *TLSv1* no no yes yes - ======================== ========= ========= ========== ========= + ======================== ========= ========= ========== ========= =========== =========== + *client* / **server** **SSLv2** **SSLv3** **SSLv23** **TLSv1** **TLSv1.1** **TLSv1.2** + ------------------------ --------- --------- ---------- --------- ----------- ----------- + *SSLv2* yes no yes no no no + *SSLv3* no yes yes no no no + *SSLv23* yes no yes no no no + *TLSv1* no no yes yes no no + *TLSv1.1* no no yes no yes no + *TLSv1.2* no no yes no no yes + ======================== ========= ========= ========== ========= =========== =========== .. note:: @@ -227,6 +230,58 @@ instead. .. versionchanged:: 3.2 New optional argument *ciphers*. + +Context creation +^^^^^^^^^^^^^^^^ + +A convenience function helps create :class:`SSLContext` objects for common +purposes. + +.. function:: create_default_context(purpose=Purpose.SERVER_AUTH, cafile=None, capath=None, cadata=None) + + Return a new :class:`SSLContext` object with default settings for + the given *purpose*. The settings are chosen by the :mod:`ssl` module, + and usually represent a higher security level than when calling the + :class:`SSLContext` constructor directly. + + *cafile*, *capath*, *cadata* represent optional CA certificates to + trust for certificate verification, as in + :meth:`SSLContext.load_verify_locations`. If all three are + :const:`None`, this function can choose to trust the system's default + CA certificates instead. + + The settings in Python 3.4 are: :data:`PROTOCOL_SSLv23`, :data:`OP_NO_SSLv2`, + and :data:`OP_NO_SSLv3` with high encryption cipher suites without RC4 and + without unauthenticated cipher suites. Passing :data:`~Purpose.SERVER_AUTH` + as *purpose* sets :data:`~SSLContext.verify_mode` to :data:`CERT_REQUIRED` + and either loads CA certificates (when at least one of *cafile*, *capath* or + *cadata* is given) or uses :meth:`SSLContext.load_default_certs` to load + default CA certificates. + + .. note:: + The protocol, options, cipher and other settings may change to more + restrictive values anytime without prior deprecation. The values + represent a fair balance between compatibility and security. + + If your application needs specific settings, you should create a + :class:`SSLContext` and apply the settings yourself. + + .. note:: + If you find that when certain older clients or servers attempt to connect + with a :class:`SSLContext` created by this function that they get an + error stating "Protocol or cipher suite mismatch", it may be that they + only support SSL3.0 which this function excludes using the + :data:`OP_NO_SSLv3`. SSL3.0 has problematic security due to a number of + poor implementations and it's reliance on MD5 within the protocol. If you + wish to continue to use this function but still allow SSL 3.0 connections + you can re-enable them using:: + + ctx = ssl.create_default_context(Purpose.CLIENT_AUTH) + ctx.options &= ~ssl.OP_NO_SSLv3 + + .. versionadded:: 3.4 + + Random generation ^^^^^^^^^^^^^^^^^ @@ -356,6 +411,61 @@ Certificate handling Given a certificate as an ASCII PEM string, returns a DER-encoded sequence of bytes for that same certificate. +.. function:: get_default_verify_paths() + + Returns a named tuple with paths to OpenSSL's default cafile and capath. + The paths are the same as used by + :meth:`SSLContext.set_default_verify_paths`. The return value is a + :term:`named tuple` ``DefaultVerifyPaths``: + + * :attr:`cafile` - resolved path to cafile or None if the file doesn't exist, + * :attr:`capath` - resolved path to capath or None if the directory doesn't exist, + * :attr:`openssl_cafile_env` - OpenSSL's environment key that points to a cafile, + * :attr:`openssl_cafile` - hard coded path to a cafile, + * :attr:`openssl_capath_env` - OpenSSL's environment key that points to a capath, + * :attr:`openssl_capath` - hard coded path to a capath directory + + .. versionadded:: 3.4 + +.. function:: enum_certificates(store_name) + + Retrieve certificates from Windows' system cert store. *store_name* may be + one of ``CA``, ``ROOT`` or ``MY``. Windows may provide additional cert + stores, too. + + The function returns a list of (cert_bytes, encoding_type, trust) tuples. + The encoding_type specifies the encoding of cert_bytes. It is either + :const:`x509_asn` for X.509 ASN.1 data or :const:`pkcs_7_asn` for + PKCS#7 ASN.1 data. Trust specifies the purpose of the certificate as a set + of OIDS or exactly ``True`` if the certificate is trustworthy for all + purposes. + + Example:: + + >>> ssl.enum_certificates("CA") + [(b'data...', 'x509_asn', {'1.3.6.1.5.5.7.3.1', '1.3.6.1.5.5.7.3.2'}), + (b'data...', 'x509_asn', True)] + + Availability: Windows. + + .. versionadded:: 3.4 + +.. function:: enum_crls(store_name) + + Retrieve CRLs from Windows' system cert store. *store_name* may be + one of ``CA``, ``ROOT`` or ``MY``. Windows may provide additional cert + stores, too. + + The function returns a list of (cert_bytes, encoding_type, trust) tuples. + The encoding_type specifies the encoding of cert_bytes. It is either + :const:`x509_asn` for X.509 ASN.1 data or :const:`pkcs_7_asn` for + PKCS#7 ASN.1 data. + + Availability: Windows. + + .. versionadded:: 3.4 + + Constants ^^^^^^^^^ @@ -392,6 +502,38 @@ Constants be passed, either to :meth:`SSLContext.load_verify_locations` or as a value of the ``ca_certs`` parameter to :func:`wrap_socket`. +.. data:: VERIFY_DEFAULT + + Possible value for :attr:`SSLContext.verify_flags`. In this mode, + certificate revocation lists (CRLs) are not checked. By default OpenSSL + does neither require nor verify CRLs. + + .. versionadded:: 3.4 + +.. data:: VERIFY_CRL_CHECK_LEAF + + Possible value for :attr:`SSLContext.verify_flags`. In this mode, only the + peer cert is check but non of the intermediate CA certificates. The mode + requires a valid CRL that is signed by the peer cert's issuer (its direct + ancestor CA). If no proper has been loaded + :attr:`SSLContext.load_verify_locations`, validation will fail. + + .. versionadded:: 3.4 + +.. data:: VERIFY_CRL_CHECK_CHAIN + + Possible value for :attr:`SSLContext.verify_flags`. In this mode, CRLs of + all certificates in the peer cert chain are checked. + + .. versionadded:: 3.4 + +.. data:: VERIFY_X509_STRICT + + Possible value for :attr:`SSLContext.verify_flags` to disable workarounds + for broken X.509 certificates. + + .. versionadded:: 3.4 + .. data:: PROTOCOL_SSLv2 Selects SSL version 2 as the channel encryption protocol. @@ -417,9 +559,22 @@ Constants .. data:: PROTOCOL_TLSv1 - Selects TLS version 1 as the channel encryption protocol. This is the most + Selects TLS version 1.0 as the channel encryption protocol. + +.. data:: PROTOCOL_TLSv1_1 + + Selects TLS version 1.1 as the channel encryption protocol. + Available only with openssl version 1.0.1+. + + .. versionadded:: 3.4 + +.. data:: PROTOCOL_TLSv1_2 + + Selects TLS version 1.2 as the channel encryption protocol. This is the most modern version, and probably the best choice for maximum protection, if both - sides can speak it. + sides can speak it. Available only with openssl version 1.0.1+. + + .. versionadded:: 3.4 .. data:: OP_ALL @@ -453,6 +608,22 @@ Constants .. versionadded:: 3.2 +.. data:: OP_NO_TLSv1_1 + + Prevents a TLSv1.1 connection. This option is only applicable in conjunction + with :const:`PROTOCOL_SSLv23`. It prevents the peers from choosing TLSv1.1 as + the protocol version. Available only with openssl version 1.0.1+. + + .. versionadded:: 3.4 + +.. data:: OP_NO_TLSv1_2 + + Prevents a TLSv1.2 connection. This option is only applicable in conjunction + with :const:`PROTOCOL_SSLv23`. It prevents the peers from choosing TLSv1.2 as + the protocol version. Available only with openssl version 1.0.1+. + + .. versionadded:: 3.4 + .. data:: OP_CIPHER_SERVER_PREFERENCE Use the server's cipher ordering preference, rather than the client's. @@ -549,6 +720,37 @@ Constants .. versionadded:: 3.2 +.. data:: ALERT_DESCRIPTION_HANDSHAKE_FAILURE + ALERT_DESCRIPTION_INTERNAL_ERROR + ALERT_DESCRIPTION_* + + Alert Descriptions from :rfc:`5246` and others. The `IANA TLS Alert Registry + <http://www.iana.org/assignments/tls-parameters/tls-parameters.xml#tls-parameters-6>`_ + contains this list and references to the RFCs where their meaning is defined. + + Used as the return value of the callback function in + :meth:`SSLContext.set_servername_callback`. + + .. versionadded:: 3.4 + +.. data:: Purpose.SERVER_AUTH + + Option for :func:`create_default_context` and + :meth:`SSLContext.load_default_certs`. This value indicates that the + context may be used to authenticate Web servers (therefore, it will + be used to create client-side sockets). + + .. versionadded:: 3.4 + +.. data:: Purpose.CLIENT_AUTH + + Option for :func:`create_default_context` and + :meth:`SSLContext.load_default_certs`. This value indicates that the + context may be used to authenticate Web clients (therefore, it will + be used to create server-side sockets). + + .. versionadded:: 3.4 + SSL Sockets ----------- @@ -584,10 +786,16 @@ SSL sockets also have the following additional methods and attributes: Perform the SSL setup handshake. + .. versionchanged:: 3.4 + The handshake method also performce :func:`match_hostname` when the + :attr:`~SSLContext.check_hostname` attribute of the socket's + :attr:`~SSLSocket.context` is true. + .. method:: SSLSocket.getpeercert(binary_form=False) If there is no certificate for the peer on the other end of the connection, - returns ``None``. + return ``None``. If the SSL handshake hasn't been done yet, raise + :exc:`ValueError`. If the ``binary_form`` parameter is :const:`False`, and a certificate was received from the peer, this method returns a :class:`dict` instance. If the @@ -645,6 +853,11 @@ SSL sockets also have the following additional methods and attributes: The returned dictionary includes additional items such as ``issuer`` and ``notBefore``. + .. versionchanged:: 3.4 + :exc:`ValueError` is raised when the handshake isn't done. + The returned dictionary includes additional X509v3 extension items + such as ``crlDistributionPoints``, ``caIssuers`` and ``OCSP`` URIs. + .. method:: SSLSocket.cipher() Returns a three-value tuple containing the name of the cipher being used, the @@ -715,11 +928,30 @@ to speed up repeated connections from the same clients. Create a new SSL context. You must pass *protocol* which must be one of the ``PROTOCOL_*`` constants defined in this module. - :data:`PROTOCOL_SSLv23` is recommended for maximum interoperability. + :data:`PROTOCOL_SSLv23` is currently recommended for maximum + interoperability. + + .. seealso:: + :func:`create_default_context` lets the :mod:`ssl` module choose + security settings for a given purpose. :class:`SSLContext` objects have the following methods and attributes: +.. method:: SSLContext.cert_store_stats() + + Get statistics about quantities of loaded X.509 certificates, count of + X.509 certificates flagged as CA certificates and certificate revocation + lists as dictionary. + + Example for a context with one CA cert and one other cert:: + + >>> context.cert_store_stats() + {'crl': 0, 'x509_ca': 1, 'x509': 2} + + .. versionadded:: 3.4 + + .. method:: SSLContext.load_cert_chain(certfile, keyfile=None, password=None) Load a private key and the corresponding certificate. The *certfile* @@ -750,12 +982,32 @@ to speed up repeated connections from the same clients. .. versionchanged:: 3.3 New optional argument *password*. -.. method:: SSLContext.load_verify_locations(cafile=None, capath=None) +.. method:: SSLContext.load_default_certs(purpose=Purpose.SERVER_AUTH) + + Load a set of default "certification authority" (CA) certificates from + default locations. On Windows it loads CA certs from the ``CA`` and + ``ROOT`` system stores. On other systems it calls + :meth:`SSLContext.set_default_verify_paths`. In the future the method may + load CA certificates from other locations, too. + + The *purpose* flag specifies what kind of CA certificates are loaded. The + default settings :data:`Purpose.SERVER_AUTH` loads certificates, that are + flagged and trusted for TLS web server authentication (client side + sockets). :data:`Purpose.CLIENT_AUTH` loads CA certificates for client + certificate verification on the server side. + + .. versionadded:: 3.4 + +.. method:: SSLContext.load_verify_locations(cafile=None, capath=None, cadata=None) Load a set of "certification authority" (CA) certificates used to validate other peers' certificates when :data:`verify_mode` is other than :data:`CERT_NONE`. At least one of *cafile* or *capath* must be specified. + This method can also load certification revocation lists (CRLs) in PEM or + or DER format. In order to make use of CRLs, :attr:`SSLContext.verify_flags` + must be configured properly. + The *cafile* string, if present, is the path to a file of concatenated CA certificates in PEM format. See the discussion of :ref:`ssl-certificates` for more information about how to arrange the @@ -766,6 +1018,25 @@ to speed up repeated connections from the same clients. following an `OpenSSL specific layout <http://www.openssl.org/docs/ssl/SSL_CTX_load_verify_locations.html>`_. + The *cadata* object, if present, is either an ASCII string of one or more + PEM-encoded certificates or a bytes-like object of DER-encoded + certificates. Like with *capath* extra lines around PEM-encoded + certificates are ignored but at least one certificate must be present. + + .. versionchanged:: 3.4 + New optional argument *cadata* + +.. method:: SSLContext.get_ca_certs(binary_form=False) + + Get a list of loaded "certification authority" (CA) certificates. If the + ``binary_form`` parameter is :const:`False` each list + entry is a dict like the output of :meth:`SSLSocket.getpeercert`. Otherwise + the method returns a list of DER-encoded certificates. The returned list + does not contain certificates from *capath* unless a certificate was + requested and loaded by a SSL connection. + + .. versionadded:: 3.4 + .. method:: SSLContext.set_default_verify_paths() Load a set of default "certification authority" (CA) certificates from @@ -803,6 +1074,56 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.3 +.. method:: SSLContext.set_servername_callback(server_name_callback) + + Register a callback function that will be called after the TLS Client Hello + handshake message has been received by the SSL/TLS server when the TLS client + specifies a server name indication. The server name indication mechanism + is specified in :rfc:`6066` section 3 - Server Name Indication. + + Only one callback can be set per ``SSLContext``. If *server_name_callback* + is ``None`` then the callback is disabled. Calling this function a + subsequent time will disable the previously registered callback. + + The callback function, *server_name_callback*, will be called with three + arguments; the first being the :class:`ssl.SSLSocket`, the second is a string + that represents the server name that the client is intending to communicate + (or :const:`None` if the TLS Client Hello does not contain a server name) + and the third argument is the original :class:`SSLContext`. The server name + argument is the IDNA decoded server name. + + A typical use of this callback is to change the :class:`ssl.SSLSocket`'s + :attr:`SSLSocket.context` attribute to a new object of type + :class:`SSLContext` representing a certificate chain that matches the server + name. + + Due to the early negotiation phase of the TLS connection, only limited + methods and attributes are usable like + :meth:`SSLSocket.selected_npn_protocol` and :attr:`SSLSocket.context`. + :meth:`SSLSocket.getpeercert`, :meth:`SSLSocket.getpeercert`, + :meth:`SSLSocket.cipher` and :meth:`SSLSocket.compress` methods require that + the TLS connection has progressed beyond the TLS Client Hello and therefore + will not contain return meaningful values nor can they be called safely. + + The *server_name_callback* function must return ``None`` to allow the + TLS negotiation to continue. If a TLS failure is required, a constant + :const:`ALERT_DESCRIPTION_* <ALERT_DESCRIPTION_INTERNAL_ERROR>` can be + returned. Other return values will result in a TLS fatal error with + :const:`ALERT_DESCRIPTION_INTERNAL_ERROR`. + + If there is a IDNA decoding error on the server name, the TLS connection + will terminate with an :const:`ALERT_DESCRIPTION_INTERNAL_ERROR` fatal TLS + alert message to the client. + + If an exception is raised from the *server_name_callback* function the TLS + connection will terminate with a fatal TLS alert message + :const:`ALERT_DESCRIPTION_HANDSHAKE_FAILURE`. + + This method will raise :exc:`NotImplementedError` if the OpenSSL library + had OPENSSL_NO_TLSEXT defined when it was built. + + .. versionadded:: 3.4 + .. method:: SSLContext.load_dh_params(dhfile) Load the key generation parameters for Diffie-Helman (DH) key exchange. @@ -869,6 +1190,45 @@ to speed up repeated connections from the same clients. >>> stats['hits'], stats['misses'] (0, 0) +.. method:: SSLContext.get_ca_certs(binary_form=False) + + Returns a list of dicts with information of loaded CA certs. If the + optional argument is true, returns a DER-encoded copy of the CA + certificate. + + .. note:: + Certificates in a capath directory aren't loaded unless they have + been used at least once. + + .. versionadded:: 3.4 + +.. attribute:: SSLContext.check_hostname + + Wether to match the peer cert's hostname with :func:`match_hostname` in + :meth:`SSLSocket.do_handshake`. The context's + :attr:`~SSLContext.verify_mode` must be set to :data:`CERT_OPTIONAL` or + :data:`CERT_REQUIRED`, and you must pass *server_hostname* to + :meth:`~SSLContext.wrap_socket` in order to match the hostname. + + Example:: + + import socket, ssl + + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + context.load_default_certs() + + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ssl_sock = context.wrap_socket(s, server_hostname='www.verisign.com'): + ssl_sock.connect(('www.verisign.com', 443)) + + .. versionadded:: 3.4 + + .. note:: + + This features requires OpenSSL 0.9.8f or newer. + .. attribute:: SSLContext.options An integer representing the set of SSL options enabled on this context. @@ -885,6 +1245,15 @@ to speed up repeated connections from the same clients. The protocol version chosen when constructing the context. This attribute is read-only. +.. attribute:: SSLContext.verify_flags + + The flags for certificate verification operations. You can set flags like + :data:`VERIFY_CRL_CHECK_LEAF` by ORing them together. By default OpenSSL + does neither require nor verify certificate revocation lists (CRLs). + Available only with openssl version 0.9.8+. + + .. versionadded:: 3.4 + .. attribute:: SSLContext.verify_mode Whether to try to verify other peers' certificates and how to behave @@ -970,20 +1339,9 @@ If you are going to require validation of the other side of the connection's certificate, you need to provide a "CA certs" file, filled with the certificate chains for each issuer you are willing to trust. Again, this file just contains these chains concatenated together. For validation, Python will use the first -chain it finds in the file which matches. Some "standard" root certificates are -available from various certification authorities: `CACert.org -<http://www.cacert.org/index.php?id=3>`_, `Thawte -<http://www.thawte.com/roots/>`_, `Verisign -<http://www.verisign.com/support/roots.html>`_, `Positive SSL -<http://www.PositiveSSL.com/ssl-certificate-support/cert_installation/UTN-USERFirst-Hardware.crt>`_ -(used by python.org), `Equifax and GeoTrust -<http://www.geotrust.com/resources/root_certificates/index.asp>`_. - -In general, if you are using SSL3 or TLS1, you don't need to put the full chain -in your "CA certs" file; you only need the root certificates, and the remote -peer is supposed to furnish the other certificates necessary to chain from its -certificate to a root certificate. See :rfc:`4158` for more discussion of the -way in which certification chains can be built. +chain it finds in the file which matches. The platform's certificates file can +be used by calling :meth:`SSLContext.load_default_certs`, this is done +automatically with :func:`.create_default_context`. Combined key and certificate ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1186,7 +1544,7 @@ waiting for clients to connect:: import socket, ssl - context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.load_cert_chain(certfile="mycertfile", keyfile="mykeyfile") bindsocket = socket.socket() @@ -1230,8 +1588,19 @@ the sockets in non-blocking mode and use an event loop). Notes on non-blocking sockets ----------------------------- -When working with non-blocking sockets, there are several things you need -to be aware of: +SSL sockets behave slightly different than regular sockets in +non-blocking mode. When working with non-blocking sockets, there are +thus several things you need to be aware of: + +- Most :class:`SSLSocket` methods will raise either + :exc:`SSLWantWriteError` or :exc:`SSLWantReadError` instead of + :exc:`BlockingIOError` if an I/O operation would + block. :exc:`SSLWantReadError` will be raised if a read operation on + the underlying socket is necessary, and :exc:`SSLWantWriteError` for + a write operation on the underlying socket. Note that attempts to + *write* to an SSL socket may require *reading* from the underlying + socket first, and attempts to *read* from the SSL socket may require + a prior *write* to the underlying socket. - Calling :func:`~select.select` tells you that the OS-level socket can be read from (or written to), but it does not imply that there is sufficient @@ -1240,8 +1609,14 @@ to be aware of: and :meth:`SSLSocket.send` failures, and retry after another call to :func:`~select.select`. +- Conversely, since the SSL layer has its own framing, a SSL socket may + still have data available for reading without :func:`~select.select` + being aware of it. Therefore, you should first call + :meth:`SSLSocket.recv` to drain any potentially available data, and then + only block on a :func:`~select.select` call if still necessary. + (of course, similar provisions apply when using other primitives such as - :func:`~select.poll`) + :func:`~select.poll`, or those in the :mod:`selectors` module) - The SSL handshake itself will be non-blocking: the :meth:`SSLSocket.do_handshake` method has to be retried until it returns @@ -1263,9 +1638,40 @@ to be aware of: Security considerations ----------------------- +Best defaults +^^^^^^^^^^^^^ + +For **client use**, if you don't have any special requirements for your +security policy, it is highly recommended that you use the +:func:`create_default_context` function to create your SSL context. +It will load the system's trusted CA certificates, enable certificate +validation and hostname checking, and try to choose reasonably secure +protocol and cipher settings. + +For example, here is how you would use the :class:`smtplib.SMTP` class to +create a trusted, secure connection to a SMTP server:: + + >>> import ssl, smtplib + >>> smtp = smtplib.SMTP("mail.python.org", port=587) + >>> context = ssl.create_default_context() + >>> smtp.starttls(context=context) + (220, b'2.0.0 Ready to start TLS') + +If a client certificate is needed for the connection, it can be added with +:meth:`SSLContext.load_cert_chain`. + +By contrast, if you create the SSL context by calling the :class:`SSLContext` +constructor yourself, it will not have certificate validation nor hostname +checking enabled by default. If you do so, please read the paragraphs below +to achieve a good security level. + +Manual settings +^^^^^^^^^^^^^^^ + Verifying certificates -^^^^^^^^^^^^^^^^^^^^^^ +'''''''''''''''''''''' +When calling the the :class:`SSLContext` constructor directly, :const:`CERT_NONE` is the default. Since it does not authenticate the other peer, it can be insecure, especially in client mode where most of time you would like to ensure the authenticity of the server you're talking to. @@ -1274,7 +1680,9 @@ Therefore, when in client mode, it is highly recommended to use have to check that the server certificate, which can be obtained by calling :meth:`SSLSocket.getpeercert`, matches the desired service. For many protocols and applications, the service can be identified by the hostname; -in this case, the :func:`match_hostname` function can be used. +in this case, the :func:`match_hostname` function can be used. This common +check is automatically performed when :attr:`SSLContext.check_hostname` is +enabled. In server mode, if you want to authenticate your clients using the SSL layer (rather than using a higher-level authentication mechanism), you'll also have @@ -1287,7 +1695,7 @@ to specify :const:`CERT_REQUIRED` and similarly check the client certificate. by default). Protocol versions -^^^^^^^^^^^^^^^^^ +''''''''''''''''' SSL version 2 is considered insecure and is therefore dangerous to use. If you want maximum compatibility between clients and servers, it is recommended @@ -1297,27 +1705,20 @@ SSLv2 explicitly using the :data:`SSLContext.options` attribute:: context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.options |= ssl.OP_NO_SSLv2 -The SSL context created above will allow SSLv3 and TLSv1 connections, but -not SSLv2. +The SSL context created above will allow SSLv3 and TLSv1 (and later, if +supported by your system) connections, but not SSLv2. Cipher selection -^^^^^^^^^^^^^^^^ +'''''''''''''''' If you have advanced security requirements, fine-tuning of the ciphers enabled when negotiating a SSL session is possible through the :meth:`SSLContext.set_ciphers` method. Starting from Python 3.2.3, the ssl module disables certain weak ciphers by default, but you may want -to further restrict the cipher choice. For example:: - - context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - context.set_ciphers('HIGH:!aNULL:!eNULL') - -The ``!aNULL:!eNULL`` part of the cipher spec is necessary to disable ciphers -which don't provide both encryption and authentication. Be sure to read -OpenSSL's documentation about the `cipher list -format <http://www.openssl.org/docs/apps/ciphers.html#CIPHER_LIST_FORMAT>`_. -If you want to check which ciphers are enabled by a given cipher list, -use the ``openssl ciphers`` command on your system. +to further restrict the cipher choice. Be sure to read OpenSSL's documentation +about the `cipher list format <http://www.openssl.org/docs/apps/ciphers.html#CIPHER_LIST_FORMAT>`_. +If you want to check which ciphers are enabled by a given cipher list, use the +``openssl ciphers`` command on your system. Multi-processing ^^^^^^^^^^^^^^^^ @@ -1350,3 +1751,12 @@ successful call of :func:`~ssl.RAND_add`, :func:`~ssl.RAND_bytes` or `RFC 4366: Transport Layer Security (TLS) Extensions <http://www.ietf.org/rfc/rfc4366>`_ Blake-Wilson et. al. + + `RFC 5246: The Transport Layer Security (TLS) Protocol Version 1.2 <http://www.ietf.org/rfc/rfc5246>`_ + T. Dierks et. al. + + `RFC 6066: Transport Layer Security (TLS) Extensions <http://www.ietf.org/rfc/rfc6066>`_ + D. Eastlake + + `IANA TLS: Transport Layer Security (TLS) Parameters <http://www.iana.org/assignments/tls-parameters/tls-parameters.xml>`_ + IANA diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index 6c20aa2..24769f6 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -6,7 +6,8 @@ os.lstat() and os.fstat(). .. sectionauthor:: Skip Montanaro <skip@automatrix.com> -**Source code:** :source:`Lib/stat.py` +**Source code:** :source:`Modules/_stat.c` + :source:`Lib/stat.py` -------------- @@ -15,6 +16,9 @@ results of :func:`os.stat`, :func:`os.fstat` and :func:`os.lstat` (if they exist). For complete details about the :c:func:`stat`, :c:func:`fstat` and :c:func:`lstat` calls, consult the documentation for your system. +.. versionchanged:: 3.4 + The stat module is backed by a C implementation. + The :mod:`stat` module defines the following functions to test for specific file types: @@ -53,6 +57,24 @@ types: Return non-zero if the mode is from a socket. +.. function:: S_ISDOOR(mode) + + Return non-zero if the mode is from a door. + + .. versionadded:: 3.4 + +.. function:: S_ISPORT(mode) + + Return non-zero if the mode is from an event port. + + .. versionadded:: 3.4 + +.. function:: S_ISWHT(mode) + + Return non-zero if the mode is from a whiteout. + + .. versionadded:: 3.4 + Two additional functions are defined for more general manipulation of the file's mode: @@ -113,6 +135,10 @@ readable string: .. versionadded:: 3.3 + .. versionchanged:: 3.4 + The function supports :data:`S_IFDOOR`, :data:`S_IFPORT` and + :data:`S_IFWHT`. + All the variables below are simply symbolic indexes into the 10-tuple returned by :func:`os.stat`, :func:`os.fstat` or :func:`os.lstat`. @@ -210,6 +236,29 @@ Use of the functions above is more portable than use of the first set of flags: FIFO. +.. data:: S_IFDOOR + + Door. + + .. versionadded:: 3.4 + +.. data:: S_IFPORT + + Event port. + + .. versionadded:: 3.4 + +.. data:: S_IFWHT + + Whiteout. + + .. versionadded:: 3.4 + +.. note:: + + :data:`S_IFDOOR`, :data:`S_IFPORT` or :data:`S_IFWHT` are defined as + 0 when the platform does not have support for the file types. + The following flags can also be used in the *mode* argument of :func:`os.chmod`: .. data:: S_ISUID diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst new file mode 100644 index 0000000..4e77838 --- /dev/null +++ b/Doc/library/statistics.rst @@ -0,0 +1,418 @@ +:mod:`statistics` --- Mathematical statistics functions +======================================================= + +.. module:: statistics + :synopsis: mathematical statistics functions +.. moduleauthor:: Steven D'Aprano <steve+python@pearwood.info> +.. sectionauthor:: Steven D'Aprano <steve+python@pearwood.info> + +.. versionadded:: 3.4 + +.. testsetup:: * + + from statistics import * + __name__ = '<doctest>' + +**Source code:** :source:`Lib/statistics.py` + +-------------- + +This module provides functions for calculating mathematical statistics of +numeric (:class:`Real`-valued) data. + +.. note:: + + Unless explicitly noted otherwise, these functions support :class:`int`, + :class:`float`, :class:`decimal.Decimal` and :class:`fractions.Fraction`. + Behaviour with other types (whether in the numeric tower or not) is + currently unsupported. Mixed types are also undefined and + implementation-dependent. If your input data consists of mixed types, + you may be able to use :func:`map` to ensure a consistent result, e.g. + ``map(float, input_data)``. + +Averages and measures of central location +----------------------------------------- + +These functions calculate an average or typical value from a population +or sample. + +======================= ============================================= +:func:`mean` Arithmetic mean ("average") of data. +:func:`median` Median (middle value) of data. +:func:`median_low` Low median of data. +:func:`median_high` High median of data. +:func:`median_grouped` Median, or 50th percentile, of grouped data. +:func:`mode` Mode (most common value) of discrete data. +======================= ============================================= + +Measures of spread +------------------ + +These functions calculate a measure of how much the population or sample +tends to deviate from the typical or average values. + +======================= ============================================= +:func:`pstdev` Population standard deviation of data. +:func:`pvariance` Population variance of data. +:func:`stdev` Sample standard deviation of data. +:func:`variance` Sample variance of data. +======================= ============================================= + + +Function details +---------------- + +Note: The functions do not require the data given to them to be sorted. +However, for reading convenience, most of the examples show sorted sequences. + +.. function:: mean(data) + + Return the sample arithmetic mean of *data*, a sequence or iterator of + real-valued numbers. + + The arithmetic mean is the sum of the data divided by the number of data + points. It is commonly called "the average", although it is only one of many + different mathematical averages. It is a measure of the central location of + the data. + + If *data* is empty, :exc:`StatisticsError` will be raised. + + Some examples of use: + + .. doctest:: + + >>> mean([1, 2, 3, 4, 4]) + 2.8 + >>> mean([-1.0, 2.5, 3.25, 5.75]) + 2.625 + + >>> from fractions import Fraction as F + >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) + Fraction(13, 21) + + >>> from decimal import Decimal as D + >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) + Decimal('0.5625') + + .. note:: + + The mean is strongly affected by outliers and is not a robust estimator + for central location: the mean is not necessarily a typical example of the + data points. For more robust, although less efficient, measures of + central location, see :func:`median` and :func:`mode`. (In this case, + "efficient" refers to statistical efficiency rather than computational + efficiency.) + + The sample mean gives an unbiased estimate of the true population mean, + which means that, taken on average over all the possible samples, + ``mean(sample)`` converges on the true mean of the entire population. If + *data* represents the entire population rather than a sample, then + ``mean(data)`` is equivalent to calculating the true population mean μ. + + +.. function:: median(data) + + Return the median (middle value) of numeric data, using the common "mean of + middle two" method. If *data* is empty, :exc:`StatisticsError` is raised. + + The median is a robust measure of central location, and is less affected by + the presence of outliers in your data. When the number of data points is + odd, the middle data point is returned: + + .. doctest:: + + >>> median([1, 3, 5]) + 3 + + When the number of data points is even, the median is interpolated by taking + the average of the two middle values: + + .. doctest:: + + >>> median([1, 3, 5, 7]) + 4.0 + + This is suited for when your data is discrete, and you don't mind that the + median may not be an actual data point. + + .. seealso:: :func:`median_low`, :func:`median_high`, :func:`median_grouped` + + +.. function:: median_low(data) + + Return the low median of numeric data. If *data* is empty, + :exc:`StatisticsError` is raised. + + The low median is always a member of the data set. When the number of data + points is odd, the middle value is returned. When it is even, the smaller of + the two middle values is returned. + + .. doctest:: + + >>> median_low([1, 3, 5]) + 3 + >>> median_low([1, 3, 5, 7]) + 3 + + Use the low median when your data are discrete and you prefer the median to + be an actual data point rather than interpolated. + + +.. function:: median_high(data) + + Return the high median of data. If *data* is empty, :exc:`StatisticsError` + is raised. + + The high median is always a member of the data set. When the number of data + points is odd, the middle value is returned. When it is even, the larger of + the two middle values is returned. + + .. doctest:: + + >>> median_high([1, 3, 5]) + 3 + >>> median_high([1, 3, 5, 7]) + 5 + + Use the high median when your data are discrete and you prefer the median to + be an actual data point rather than interpolated. + + +.. function:: median_grouped(data, interval=1) + + Return the median of grouped continuous data, calculated as the 50th + percentile, using interpolation. If *data* is empty, :exc:`StatisticsError` + is raised. + + .. doctest:: + + >>> median_grouped([52, 52, 53, 54]) + 52.5 + + In the following example, the data are rounded, so that each value represents + the midpoint of data classes, e.g. 1 is the midpoint of the class 0.5-1.5, 2 + is the midpoint of 1.5-2.5, 3 is the midpoint of 2.5-3.5, etc. With the data + given, the middle value falls somewhere in the class 3.5-4.5, and + interpolation is used to estimate it: + + .. doctest:: + + >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5]) + 3.7 + + Optional argument *interval* represents the class interval, and defaults + to 1. Changing the class interval naturally will change the interpolation: + + .. doctest:: + + >>> median_grouped([1, 3, 3, 5, 7], interval=1) + 3.25 + >>> median_grouped([1, 3, 3, 5, 7], interval=2) + 3.5 + + This function does not check whether the data points are at least + *interval* apart. + + .. impl-detail:: + + Under some circumstances, :func:`median_grouped` may coerce data points to + floats. This behaviour is likely to change in the future. + + .. seealso:: + + * "Statistics for the Behavioral Sciences", Frederick J Gravetter and + Larry B Wallnau (8th Edition). + + * Calculating the `median <http://www.ualberta.ca/~opscan/median.html>`_. + + * The `SSMEDIAN + <https://projects.gnome.org/gnumeric/doc/gnumeric-function-SSMEDIAN.shtml>`_ + function in the Gnome Gnumeric spreadsheet, including `this discussion + <https://mail.gnome.org/archives/gnumeric-list/2011-April/msg00018.html>`_. + + +.. function:: mode(data) + + Return the most common data point from discrete or nominal *data*. The mode + (when it exists) is the most typical value, and is a robust measure of + central location. + + If *data* is empty, or if there is not exactly one most common value, + :exc:`StatisticsError` is raised. + + ``mode`` assumes discrete data, and returns a single value. This is the + standard treatment of the mode as commonly taught in schools: + + .. doctest:: + + >>> mode([1, 1, 2, 3, 3, 3, 3, 4]) + 3 + + The mode is unique in that it is the only statistic which also applies + to nominal (non-numeric) data: + + .. doctest:: + + >>> mode(["red", "blue", "blue", "red", "green", "red", "red"]) + 'red' + + +.. function:: pstdev(data, mu=None) + + Return the population standard deviation (the square root of the population + variance). See :func:`pvariance` for arguments and other details. + + .. doctest:: + + >>> pstdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) + 0.986893273527251 + + +.. function:: pvariance(data, mu=None) + + Return the population variance of *data*, a non-empty iterable of real-valued + numbers. Variance, or second moment about the mean, is a measure of the + variability (spread or dispersion) of data. A large variance indicates that + the data is spread out; a small variance indicates it is clustered closely + around the mean. + + If the optional second argument *mu* is given, it should be the mean of + *data*. If it is missing or ``None`` (the default), the mean is + automatically calculated. + + Use this function to calculate the variance from the entire population. To + estimate the variance from a sample, the :func:`variance` function is usually + a better choice. + + Raises :exc:`StatisticsError` if *data* is empty. + + Examples: + + .. doctest:: + + >>> data = [0.0, 0.25, 0.25, 1.25, 1.5, 1.75, 2.75, 3.25] + >>> pvariance(data) + 1.25 + + If you have already calculated the mean of your data, you can pass it as the + optional second argument *mu* to avoid recalculation: + + .. doctest:: + + >>> mu = mean(data) + >>> pvariance(data, mu) + 1.25 + + This function does not attempt to verify that you have passed the actual mean + as *mu*. Using arbitrary values for *mu* may lead to invalid or impossible + results. + + Decimals and Fractions are supported: + + .. doctest:: + + >>> from decimal import Decimal as D + >>> pvariance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) + Decimal('24.815') + + >>> from fractions import Fraction as F + >>> pvariance([F(1, 4), F(5, 4), F(1, 2)]) + Fraction(13, 72) + + .. note:: + + When called with the entire population, this gives the population variance + σ². When called on a sample instead, this is the biased sample variance + s², also known as variance with N degrees of freedom. + + If you somehow know the true population mean μ, you may use this function + to calculate the variance of a sample, giving the known population mean as + the second argument. Provided the data points are representative + (e.g. independent and identically distributed), the result will be an + unbiased estimate of the population variance. + + +.. function:: stdev(data, xbar=None) + + Return the sample standard deviation (the square root of the sample + variance). See :func:`variance` for arguments and other details. + + .. doctest:: + + >>> stdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) + 1.0810874155219827 + + +.. function:: variance(data, xbar=None) + + Return the sample variance of *data*, an iterable of at least two real-valued + numbers. Variance, or second moment about the mean, is a measure of the + variability (spread or dispersion) of data. A large variance indicates that + the data is spread out; a small variance indicates it is clustered closely + around the mean. + + If the optional second argument *xbar* is given, it should be the mean of + *data*. If it is missing or ``None`` (the default), the mean is + automatically calculated. + + Use this function when your data is a sample from a population. To calculate + the variance from the entire population, see :func:`pvariance`. + + Raises :exc:`StatisticsError` if *data* has fewer than two values. + + Examples: + + .. doctest:: + + >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] + >>> variance(data) + 1.3720238095238095 + + If you have already calculated the mean of your data, you can pass it as the + optional second argument *xbar* to avoid recalculation: + + .. doctest:: + + >>> m = mean(data) + >>> variance(data, m) + 1.3720238095238095 + + This function does not attempt to verify that you have passed the actual mean + as *xbar*. Using arbitrary values for *xbar* can lead to invalid or + impossible results. + + Decimal and Fraction values are supported: + + .. doctest:: + + >>> from decimal import Decimal as D + >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) + Decimal('31.01875') + + >>> from fractions import Fraction as F + >>> variance([F(1, 6), F(1, 2), F(5, 3)]) + Fraction(67, 108) + + .. note:: + + This is the sample variance s² with Bessel's correction, also known as + variance with N-1 degrees of freedom. Provided that the data points are + representative (e.g. independent and identically distributed), the result + should be an unbiased estimate of the true population variance. + + If you somehow know the actual population mean μ you should pass it to the + :func:`pvariance` function as the *mu* parameter to get the variance of a + sample. + +Exceptions +---------- + +A single exception is defined: + +.. exception:: StatisticsError + + Subclass of :exc:`ValueError` for statistics-related exceptions. + +.. + # This modelines must appear within the last ten lines of the file. + kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index ab78cdc..e41fa6e 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1526,7 +1526,7 @@ expression support in the :mod:`re` module). at that position. -.. method:: str.expandtabs([tabsize]) +.. method:: str.expandtabs(tabsize=8) Return a copy of the string where all tab characters are replaced by one or more spaces, depending on the current column and the given tab size. Tab @@ -2462,6 +2462,10 @@ copying. .. versionchanged:: 3.3 One-dimensional memoryviews with formats 'B', 'b' or 'c' are now hashable. + .. versionchanged:: 3.4 + memoryview is now registered automatically with + :class:`collections.abc.Sequence` + :class:`memoryview` has several methods: .. method:: __eq__(exporter) @@ -3027,8 +3031,8 @@ pairs within braces, for example: ``{'jack': 4098, 'sjoerd': 4127}`` or ``{4098: If no positional argument is given, an empty dictionary is created. If a positional argument is given and it is a mapping object, a dictionary is created with the same key-value pairs as the mapping object. Otherwise, - the positional argument must be an :term:`iterator` object. Each item in - the iterable must itself be an iterator with exactly two objects. The + the positional argument must be an :term:`iterable` object. Each item in + the iterable must itself be an iterable with exactly two objects. The first object of each item becomes a key in the new dictionary, and the second object the corresponding value. If a key occurs more than once, the last value for that key becomes the corresponding value in the new diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index dde32b9..ec2e1be 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -66,6 +66,19 @@ The module defines the following exception and functions: format (``len(buffer[offset:])`` must be at least ``calcsize(fmt)``). +.. function:: iter_unpack(fmt, buffer) + + Iteratively unpack from the buffer *buffer* according to the format + string *fmt*. This function returns an iterator which will read + equally-sized chunks from the buffer until all its contents have been + consumed. The buffer's size in bytes must be a multiple of the amount + of data required by the format, as reflected by :func:`calcsize`. + + Each iteration yields a tuple as specified by the format string. + + .. versionadded:: 3.4 + + .. function:: calcsize(fmt) Return the size of the struct (and hence of the bytes object produced by @@ -388,6 +401,13 @@ The :mod:`struct` module also defines the following type: (``len(buffer[offset:])`` must be at least :attr:`self.size`). + .. method:: iter_unpack(buffer) + + Identical to the :func:`iter_unpack` function, using the compiled format. + (``len(buffer)`` must be a multiple of :attr:`self.size`). + + .. versionadded:: 3.4 + .. attribute:: format The format string used to construct this Struct object. diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index a9ee970..854993c 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -54,18 +54,12 @@ use cases, the underlying :class:`Popen` interface can be used directly. >>> subprocess.call("exit 1", shell=True) 1 - .. warning:: - - Invoking the system shell with ``shell=True`` can be a security hazard - if combined with untrusted input. See the warning under - :ref:`frequently-used-arguments` for details. - .. note:: - Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this function. As - the pipes are not being read in the current process, the child - process may block if it generates enough output to a pipe to fill up - the OS pipe buffer. + Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this + function. The child process will block if it generates enough + output to a pipe to fill up the OS pipe buffer as the pipes are + not being read from. .. versionchanged:: 3.3 *timeout* was added. @@ -99,24 +93,18 @@ use cases, the underlying :class:`Popen` interface can be used directly. ... subprocess.CalledProcessError: Command 'exit 1' returned non-zero exit status 1 - .. warning:: - - Invoking the system shell with ``shell=True`` can be a security hazard - if combined with untrusted input. See the warning under - :ref:`frequently-used-arguments` for details. - .. note:: - Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this function. As - the pipes are not being read in the current process, the child - process may block if it generates enough output to a pipe to fill up - the OS pipe buffer. + Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this + function. The child process will block if it generates enough + output to a pipe to fill up the OS pipe buffer as the pipes are + not being read from. .. versionchanged:: 3.3 *timeout* was added. -.. function:: check_output(args, *, stdin=None, stderr=None, shell=False, universal_newlines=False, timeout=None) +.. function:: check_output(args, *, input=None, stdin=None, stderr=None, shell=False, universal_newlines=False, timeout=None) Run command with arguments and return its output. @@ -129,15 +117,21 @@ use cases, the underlying :class:`Popen` interface can be used directly. in :ref:`frequently-used-arguments` (hence the use of keyword-only notation in the abbreviated signature). The full function signature is largely the same as that of the :class:`Popen` constructor - this functions passes all - supplied arguments other than *timeout* directly through to that interface. - In addition, *stdout* is not permitted as an argument, as it is used - internally to collect the output from the subprocess. + supplied arguments other than *input* and *timeout* directly through to + that interface. In addition, *stdout* is not permitted as an argument, as + it is used internally to collect the output from the subprocess. The *timeout* argument is passed to :meth:`Popen.wait`. If the timeout expires, the child process will be killed and then waited for again. The :exc:`TimeoutExpired` exception will be re-raised after the child process has terminated. + The *input* argument is passed to :meth:`Popen.communicate` and thus to the + subprocess's stdin. If used it must be a byte sequence, or a string if + ``universal_newlines=True``. When used, the internal :class:`Popen` object + is automatically created with ``stdin=PIPE``, and the *stdin* argument may + not be used as well. + Examples:: >>> subprocess.check_output(["echo", "Hello World!"]) @@ -146,6 +140,10 @@ use cases, the underlying :class:`Popen` interface can be used directly. >>> subprocess.check_output(["echo", "Hello World!"], universal_newlines=True) 'Hello World!\n' + >>> subprocess.check_output(["sed", "-e", "s/foo/bar/"], + ... input=b"when in the course of fooman events\n") + b'when in the course of barman events\n' + >>> subprocess.check_output("exit 1", shell=True) Traceback (most recent call last): ... @@ -167,25 +165,20 @@ use cases, the underlying :class:`Popen` interface can be used directly. ... shell=True) 'ls: non_existent_file: No such file or directory\n' - .. versionadded:: 3.1 - - .. - - .. warning:: - - Invoking the system shell with ``shell=True`` can be a security hazard - if combined with untrusted input. See the warning under - :ref:`frequently-used-arguments` for details. - .. note:: - Do not use ``stderr=PIPE`` with this function. As the pipe is not being - read in the current process, the child process may block if it - generates enough output to the pipe to fill up the OS pipe buffer. + Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this + function. The child process will block if it generates enough + output to a pipe to fill up the OS pipe buffer as the pipes are + not being read from. + + .. versionadded:: 3.1 .. versionchanged:: 3.3 *timeout* was added. + .. versionchanged:: 3.4 + *input* was added. .. data:: DEVNULL @@ -200,7 +193,7 @@ use cases, the underlying :class:`Popen` interface can be used directly. Special value that can be used as the *stdin*, *stdout* or *stderr* argument to :class:`Popen` and indicates that a pipe to the standard stream should be - opened. + opened. Most useful with :meth:`Popen.communicate`. .. data:: STDOUT @@ -326,28 +319,9 @@ default values. The arguments that are most commonly needed are: instead of ``locale.getpreferredencoding()``. See the :class:`io.TextIOWrapper` class for more information on this change. - .. warning:: - - Executing shell commands that incorporate unsanitized input from an - untrusted source makes a program vulnerable to `shell injection - <http://en.wikipedia.org/wiki/Shell_injection#Shell_injection>`_, - a serious security flaw which can result in arbitrary command execution. - For this reason, the use of ``shell=True`` is **strongly discouraged** - in cases where the command string is constructed from external input:: - - >>> from subprocess import call - >>> filename = input("What file would you like to display?\n") - What file would you like to display? - non_existent; rm -rf / # - >>> call("cat " + filename, shell=True) # Uh-oh. This will end badly... - - ``shell=False`` disables all shell based features, but does not suffer - from this vulnerability; see the Note in the :class:`Popen` constructor - documentation for helpful hints in getting ``shell=False`` to work. + .. note:: - When using ``shell=True``, :func:`shlex.quote` can be used to properly - escape whitespace and shell metacharacters in strings that are going to - be used to construct shell commands. + Read the `Security Considerations`_ section before using ``shell=True``. These options, along with all of the other options, are described in more detail in the :class:`Popen` constructor documentation. @@ -368,7 +342,7 @@ functions. startupinfo=None, creationflags=0, restore_signals=True, \ start_new_session=False, pass_fds=()) - Execute a child program in a new process. On Unix, the class uses + Execute a child program in a new process. On POSIX, the class uses :meth:`os.execvp`-like behavior to execute the child program. On Windows, the class uses the Windows ``CreateProcess()`` function. The arguments to :class:`Popen` are as follows. @@ -380,7 +354,7 @@ functions. arguments for additional differences from the default behavior. Unless otherwise stated, it is recommended to pass *args* as a sequence. - On Unix, if *args* is a string, the string is interpreted as the name or + On POSIX, if *args* is a string, the string is interpreted as the name or path of the program to execute. However, this can only be done if not passing arguments to the program. @@ -411,7 +385,7 @@ functions. the shell as the program to execute. If *shell* is *True*, it is recommended to pass *args* as a string rather than as a sequence. - On Unix with ``shell=True``, the shell defaults to :file:`/bin/sh`. If + On POSIX with ``shell=True``, the shell defaults to :file:`/bin/sh`. If *args* is a string, the string specifies the command to execute through the shell. This means that the string must be formatted exactly as it would be when typed at the shell prompt. This @@ -428,11 +402,9 @@ functions. into the shell (e.g. :command:`dir` or :command:`copy`). You do not need ``shell=True`` to run a batch file or console-based executable. - .. warning:: + .. note:: - Passing ``shell=True`` can be a security hazard if combined with - untrusted input. See the warning under :ref:`frequently-used-arguments` - for details. + Read the `Security Considerations`_ section before using ``shell=True``. *bufsize* will be supplied as the corresponding argument to the :func:`open` function when creating the stdin/stdout/stderr pipe file objects: :const:`0` @@ -453,9 +425,9 @@ functions. program to execute specified by *args*. However, the original *args* is still passed to the program. Most programs treat the program specified by *args* as the command name, which can then be different from the program - actually executed. On Unix, the *args* name + actually executed. On POSIX, the *args* name becomes the display name for the executable in utilities such as - :program:`ps`. If ``shell=True``, on Unix the *executable* argument + :program:`ps`. If ``shell=True``, on POSIX the *executable* argument specifies a replacement shell for the default :file:`/bin/sh`. *stdin*, *stdout* and *stderr* specify the executed program's standard input, @@ -471,7 +443,7 @@ functions. If *preexec_fn* is set to a callable object, this object will be called in the child process just before the child is executed. - (Unix only) + (POSIX only) .. warning:: @@ -489,8 +461,8 @@ functions. common use of *preexec_fn* to call os.setsid() in the child. If *close_fds* is true, all file descriptors except :const:`0`, :const:`1` and - :const:`2` will be closed before the child process is executed. (Unix only). - The default varies by platform: Always true on Unix. On Windows it is + :const:`2` will be closed before the child process is executed. (POSIX only). + The default varies by platform: Always true on POSIX. On Windows it is true when *stdin*/*stdout*/*stderr* are :const:`None`, false otherwise. On Windows, if *close_fds* is true then no handles will be inherited by the child process. Note that on Windows, you cannot set *close_fds* to true and @@ -502,7 +474,7 @@ functions. *pass_fds* is an optional sequence of file descriptors to keep open between the parent and child. Providing any *pass_fds* forces - *close_fds* to be :const:`True`. (Unix only) + *close_fds* to be :const:`True`. (POSIX only) .. versionadded:: 3.2 The *pass_fds* parameter was added. @@ -515,13 +487,13 @@ functions. If *restore_signals* is true (the default) all signals that Python has set to SIG_IGN are restored to SIG_DFL in the child process before the exec. Currently this includes the SIGPIPE, SIGXFZ and SIGXFSZ signals. - (Unix only) + (POSIX only) .. versionchanged:: 3.2 *restore_signals* was added. If *start_new_session* is true the setsid() system call will be made in the - child process prior to the execution of the subprocess. (Unix only) + child process prior to the execution of the subprocess. (POSIX only) .. versionchanged:: 3.2 *start_new_session* was added. @@ -588,14 +560,21 @@ Exceptions defined in this module all inherit from :exc:`SubprocessError`. The :exc:`SubprocessError` base class was added. -Security -^^^^^^^^ +Security Considerations +----------------------- + +Unlike some other popen functions, this implementation will never +implicitly call a system shell. This means that all characters, +including shell metacharacters, can safely be passed to child processes. +If the shell is invoked explicitly, via ``shell=True``, it is the application's +responsibility to ensure that all whitespace and metacharacters are +quoted appropriately to avoid +`shell injection <http://en.wikipedia.org/wiki/Shell_injection#Shell_injection>`_ +vulnerabilities. -Unlike some other popen functions, this implementation will never call a -system shell implicitly. This means that all characters, including shell -metacharacters, can safely be passed to child processes. Obviously, if the -shell is invoked explicitly, then it is the application's responsibility to -ensure that all whitespace and metacharacters are quoted appropriately. +When using ``shell=True``, the :func:`shlex.quote` function can be +used to properly escape whitespace and shell metacharacters in strings +that are going to be used to construct shell commands. Popen Objects @@ -619,16 +598,27 @@ Instances of the :class:`Popen` class have the following methods: :exc:`TimeoutExpired` exception. It is safe to catch this exception and retry the wait. - .. warning:: + .. note:: + + This will deadlock when using ``stdout=PIPE`` or ``stderr=PIPE`` + and the child process generates enough output to a pipe such that + it blocks waiting for the OS pipe buffer to accept more data. + Use :meth:`Popen.communicate` when using pipes to avoid that. + + .. note:: - This will deadlock when using ``stdout=PIPE`` and/or - ``stderr=PIPE`` and the child process generates enough output to - a pipe such that it blocks waiting for the OS pipe buffer to - accept more data. Use :meth:`communicate` to avoid that. + The function is implemented using a busy loop (non-blocking call and + short sleeps). Use the :mod:`asyncio` module for an asynchronous wait: + see :class:`asyncio.create_subprocess_exec`. .. versionchanged:: 3.3 *timeout* was added. + .. deprecated:: 3.4 + + Do not use the *endtime* parameter. It is was unintentionally + exposed in 3.3 but was left undocumented as it was intended to be + private for internal use. Use *timeout* instead. .. method:: Popen.communicate(input=None, timeout=None) @@ -695,13 +685,6 @@ Instances of the :class:`Popen` class have the following methods: The following attributes are also available: -.. warning:: - - Use :meth:`~Popen.communicate` rather than :attr:`.stdin.write <Popen.stdin>`, - :attr:`.stdout.read <Popen.stdout>` or :attr:`.stderr.read <Popen.stderr>` to avoid - deadlocks due to any of the other OS pipe buffers filling up and blocking the - child process. - .. attribute:: Popen.args The *args* argument as it was passed to :class:`Popen` -- a @@ -735,6 +718,13 @@ The following attributes are also available: ``True``, the stream is a text stream, otherwise it is a byte stream. If the *stderr* argument was not :data:`PIPE`, this attribute is ``None``. +.. warning:: + + Use :meth:`~Popen.communicate` rather than :attr:`.stdin.write <Popen.stdin>`, + :attr:`.stdout.read <Popen.stdout>` or :attr:`.stderr.read <Popen.stderr>` to avoid + deadlocks due to any of the other OS pipe buffers filling up and blocking the + child process. + .. attribute:: Popen.pid @@ -751,7 +741,7 @@ The following attributes are also available: hasn't terminated yet. A negative value ``-N`` indicates that the child was terminated by signal - ``N`` (Unix only). + ``N`` (POSIX only). Windows Popen Helpers @@ -847,8 +837,6 @@ The :mod:`subprocess` module exposes the following constants. The new process has a new console, instead of inheriting its parent's console (the default). - This flag is always set when :class:`Popen` is created with ``shell=True``. - .. data:: CREATE_NEW_PROCESS_GROUP A :class:`Popen` ``creationflags`` parameter to specify that a new process @@ -1025,7 +1013,7 @@ Replacing functions from the :mod:`popen2` module (child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode) ==> - p = Popen(["somestring"], shell=True, bufsize=bufsize, + p = Popen("somestring", shell=True, bufsize=bufsize, stdin=PIPE, stdout=PIPE, close_fds=True) (child_stdout, child_stdin) = (p.stdout, p.stdin) @@ -1063,9 +1051,9 @@ handling consistency are valid for these functions. Return ``(status, output)`` of executing *cmd* in a shell. - Execute the string *cmd* in a shell with :class:`Popen` and return a 2-tuple - ``(status, output)`` via :func:`Popen.communicate`. Universal newlines mode - is used; see the notes on :ref:`frequently-used-arguments` for more details. + Execute the string *cmd* in a shell with :meth:`Popen.check_output` and + return a 2-tuple ``(status, output)``. Universal newlines mode is used; + see the notes on :ref:`frequently-used-arguments` for more details. A trailing newline is stripped from the output. The exit status for the command can be interpreted @@ -1078,7 +1066,7 @@ handling consistency are valid for these functions. >>> subprocess.getstatusoutput('/bin/junk') (256, 'sh: /bin/junk: not found') - Availability: Unix & Windows + Availability: POSIX & Windows .. versionchanged:: 3.3.4 Windows support added @@ -1094,7 +1082,7 @@ handling consistency are valid for these functions. >>> subprocess.getoutput('ls /bin/ls') '/bin/ls' - Availability: Unix & Windows + Availability: POSIX & Windows .. versionchanged:: 3.3.4 Windows support added diff --git a/Doc/library/sunau.rst b/Doc/library/sunau.rst index 4bdb99b..a94ae08 100644 --- a/Doc/library/sunau.rst +++ b/Doc/library/sunau.rst @@ -150,8 +150,9 @@ AU_read objects, as returned by :func:`.open` above, have the following methods: .. method:: AU_read.getparams() - Returns a tuple ``(nchannels, sampwidth, framerate, nframes, comptype, - compname)``, equivalent to output of the :meth:`get\*` methods. + Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth, + framerate, nframes, comptype, compname)``, equivalent to output of the + :meth:`get\*` methods. .. method:: AU_read.readframes(n) @@ -211,6 +212,9 @@ AU_write objects, as returned by :func:`.open` above, have the following methods Set the sample width (in bytes.) + .. versionchanged:: 3.4 + Added support for 24-bit samples. + .. method:: AU_write.setframerate(n) @@ -246,11 +250,17 @@ AU_write objects, as returned by :func:`.open` above, have the following methods Write audio frames, without correcting *nframes*. + .. versionchanged:: 3.4 + Any :term:`bytes-like object` is now accepted. + .. method:: AU_write.writeframes(data) Write audio frames and make sure *nframes* is correct. + .. versionchanged:: 3.4 + Any :term:`bytes-like object` is now accepted. + .. method:: AU_write.close() diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 36c608c..bee309e 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -12,11 +12,12 @@ always available. .. data:: abiflags - On POSIX systems where Python is build with the standard ``configure`` + On POSIX systems where Python was built with the standard ``configure`` script, this contains the ABI flags as specified by :pep:`3149`. .. versionadded:: 3.2 + .. data:: argv The list of command line arguments passed to a Python script. ``argv[0]`` is the @@ -382,6 +383,21 @@ always available. .. versionadded:: 3.1 +.. function:: getallocatedblocks() + + Return the number of memory blocks currently allocated by the interpreter, + regardless of their size. This function is mainly useful for tracking + and debugging memory leaks. Because of the interpreter's internal + caches, the result can vary from call to call; you may have to call + :func:`_clear_type_cache()` and :func:`gc.collect()` to get more + predictable results. + + If a Python build or implementation cannot reasonably compute this + information, :func:`getallocatedblocks()` is allowed to return 0 instead. + + .. versionadded:: 3.4 + + .. function:: getcheckinterval() Return the interpreter's "check interval"; see :func:`setcheckinterval`. @@ -398,9 +414,10 @@ always available. .. function:: getdlopenflags() - Return the current value of the flags that are used for :c:func:`dlopen` calls. - The flag constants are defined in the :mod:`ctypes` and :mod:`DLFCN` modules. - Availability: Unix. + Return the current value of the flags that are used for + :c:func:`dlopen` calls. Symbolic names for the flag values can be + found in the :mod:`os` module (``RTLD_xxx`` constants, e.g. + :data:`os.RTLD_LAZY`). Availability: Unix. .. function:: getfilesystemencoding() @@ -579,9 +596,20 @@ always available. | :const:`imag` | multiplier used for the imaginary part of a | | | complex number | +---------------------+--------------------------------------------------+ + | :const:`algorithm` | name of the algorithm for hashing of str, bytes, | + | | and memoryview | + +---------------------+--------------------------------------------------+ + | :const:`hash_bits` | internal output size of the hash algorithm | + +---------------------+--------------------------------------------------+ + | :const:`seed_bits` | size of the seed key of the hash algorithm | + +---------------------+--------------------------------------------------+ + .. versionadded:: 3.2 + .. versionchanged:: 3.4 + Added *algorithm*, *hash_bits* and *seed_bits* + .. data:: hexversion @@ -665,6 +693,17 @@ always available. .. versionadded:: 3.1 +.. data:: __interactivehook__ + + When this attribute exists, its value is automatically called (with no + arguments) when the interpreter is launched in :ref:`interactive mode + <tut-interactive>`. This is done after the :envvar:`PYTHONSTARTUP` file is + read, so that you can set this hook there. The :mod:`site` module + :ref:`sets this <rlcompleter-config>`. + + .. versionadded:: 3.4 + + .. function:: intern(string) Enter *string* in the table of "interned" strings and return the interned string @@ -812,8 +851,6 @@ always available. Windows ``'win32'`` Windows/Cygwin ``'cygwin'`` Mac OS X ``'darwin'`` - OS/2 ``'os2'`` - OS/2 EMX ``'os2emx'`` ================ =========================== .. versionchanged:: 3.3 @@ -884,7 +921,7 @@ always available. the interpreter loads extension modules. Among other things, this will enable a lazy resolving of symbols when importing a module, if called as ``sys.setdlopenflags(0)``. To share symbols across extension modules, call as - ``sys.setdlopenflags(os.RTLD_GLOBAL)``. Symbolic names for the flag modules + ``sys.setdlopenflags(os.RTLD_GLOBAL)``. Symbolic names for the flag values can be found in the :mod:`os` module (``RTLD_xxx`` constants, e.g. :data:`os.RTLD_LAZY`). @@ -1029,8 +1066,9 @@ always available. statements and for the prompts of :func:`input`; * The interpreter's own prompts and its error messages go to ``stderr``. - By default, these streams are regular text streams as returned by the - :func:`open` function. Their parameters are chosen as follows: + These streams are regular :term:`text files <text file>` like those + returned by the :func:`open` function. Their parameters are chosen as + follows: * The character encoding is platform-dependent. Under Windows, if the stream is interactive (that is, if its :meth:`isatty` method returns ``True``), the @@ -1038,26 +1076,22 @@ always available. platforms, the locale encoding is used (see :meth:`locale.getpreferredencoding`). Under all platforms though, you can override this value by setting the - :envvar:`PYTHONIOENCODING` environment variable. + :envvar:`PYTHONIOENCODING` environment variable before starting Python. * When interactive, standard streams are line-buffered. Otherwise, they are block-buffered like regular text files. You can override this value with the :option:`-u` command-line option. - To write or read binary data from/to the standard streams, use the - underlying binary :data:`~io.TextIOBase.buffer`. For example, to write - bytes to :data:`stdout`, use ``sys.stdout.buffer.write(b'abc')``. Using - :meth:`io.TextIOBase.detach`, streams can be made binary by default. This - function sets :data:`stdin` and :data:`stdout` to binary:: + .. note:: - def make_streams_binary(): - sys.stdin = sys.stdin.detach() - sys.stdout = sys.stdout.detach() + To write or read binary data from/to the standard streams, use the + underlying binary :data:`~io.TextIOBase.buffer` object. For example, to + write bytes to :data:`stdout`, use ``sys.stdout.buffer.write(b'abc')``. - Note that the streams may be replaced with objects (like :class:`io.StringIO`) - that do not support the :attr:`~io.BufferedIOBase.buffer` attribute or the - :meth:`~io.BufferedIOBase.detach` method and can raise :exc:`AttributeError` - or :exc:`io.UnsupportedOperation`. + However, if you are writing a library (and do not control in which + context its code will be executed), be aware that the standard streams + may be replaced with file-like objects like :class:`io.StringIO` which + do not support the :attr:`~io.BufferedIOBase.buffer` attribute. .. data:: __stdin__ @@ -1094,7 +1128,6 @@ always available. | :const:`name` | Name of the thread implementation: | | | | | | * ``'nt'``: Windows threads | - | | * ``'os2'``: OS/2 threads | | | * ``'pthread'``: POSIX threads | | | * ``'solaris'``: Solaris threads | +------------------+---------------------------------------------------------+ diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index c47dcce..535ac54 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -83,8 +83,6 @@ Python currently supports seven schemes: located under the user home directory. - *nt*: scheme for NT platforms like Windows. - *nt_user*: scheme for NT platforms, when the *user* option is used. -- *os2*: scheme for OS/2 platforms. -- *os2_home*: scheme for OS/2 patforms, when the *user* option is used. Each scheme is itself composed of a series of paths and each path has a unique identifier. Python currently uses eight paths: diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 1ead71c..15b88f8 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -81,6 +81,10 @@ Some facts and figures: If *fileobj* is specified, it is used as an alternative to a :term:`file object` opened in binary mode for *name*. It is supposed to be at position 0. + For modes ``'w:gz'``, ``'r:gz'``, ``'w:bz2'``, ``'r:bz2'``, :func:`tarfile.open` + accepts the keyword argument *compresslevel* to specify the compression level of + the file. + For special purposes, there is a second format for *mode*: ``'filemode|[compression]'``. :func:`tarfile.open` will return a :class:`TarFile` object that processes its data as a stream of blocks. No random seeking will @@ -292,7 +296,7 @@ be finalized; only the internally used file object will be closed. See the will be added as a pax global header if *format* is :const:`PAX_FORMAT`. -.. method:: TarFile.open(...) +.. classmethod:: TarFile.open(...) Alternative constructor. The :func:`tarfile.open` function is actually a shortcut to this classmethod. @@ -509,7 +513,7 @@ A ``TarInfo`` object has the following public data attributes: :const:`AREGTYPE`, :const:`LNKTYPE`, :const:`SYMTYPE`, :const:`DIRTYPE`, :const:`FIFOTYPE`, :const:`CONTTYPE`, :const:`CHRTYPE`, :const:`BLKTYPE`, :const:`GNUTYPE_SPARSE`. To determine the type of a :class:`TarInfo` object - more conveniently, use the ``is_*()`` methods below. + more conveniently, use the ``is*()`` methods below. .. attribute:: TarInfo.linkname @@ -591,6 +595,67 @@ A :class:`TarInfo` object also provides some convenient query methods: Return :const:`True` if it is one of character device, block device or FIFO. +.. _tarfile-commandline: + +Command Line Interface +---------------------- + +.. versionadded:: 3.4 + +The :mod:`tarfile` module provides a simple command line interface to interact +with tar archives. + +If you want to create a new tar archive, specify its name after the :option:`-c` +option and then list the filename(s) that should be included:: + + $ python -m tarfile -c monty.tar spam.txt eggs.txt + +Passing a directory is also acceptable:: + + $ python -m tarfile -c monty.tar life-of-brian_1979/ + +If you want to extract a tar archive into the current directory, use +the :option:`-e` option:: + + $ python -m tarfile -e monty.tar + +You can also extract a tar archive into a different directory by passing the +directory's name:: + + $ python -m tarfile -e monty.tar other-dir/ + +For a list of the files in a tar archive, use the :option:`-l` option:: + + $ python -m tarfile -l monty.tar + + +Command line options +~~~~~~~~~~~~~~~~~~~~ + +.. cmdoption:: -l <tarfile> + --list <tarfile> + + List files in a tarfile. + +.. cmdoption:: -c <tarfile> <source1> <sourceN> + --create <tarfile> <source1> <sourceN> + + Create tarfile from source files. + +.. cmdoption:: -e <tarfile> [<output_dir>] + --extract <tarfile> [<output_dir>] + + Extract tarfile into the current directory if *output_dir* is not specified. + +.. cmdoption:: -t <tarfile> + --test <tarfile> + + Test whether the tarfile is valid or not. + +.. cmdoption:: -v, --verbose + + Verbose output + .. _tar-examples: Examples diff --git a/Doc/library/test.rst b/Doc/library/test.rst index c28536d..83026d8 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -443,13 +443,6 @@ The :mod:`test.support` module defines the following functions: A decorator for running tests that require support for symbolic links. -.. function:: suppress_crash_popup() - - A context manager that disables Windows Error Reporting dialogs using - `SetErrorMode <http://msdn.microsoft.com/en-us/library/windows/desktop/ms680621%28v=vs.85%29.aspx>`_. - On other platforms it's a no-op. - - .. decorator:: anticipate_failure(condition) A decorator to conditionally mark tests with @@ -594,6 +587,21 @@ The :mod:`test.support` module defines the following classes: Temporarily unset the environment variable ``envvar``. +.. class:: SuppressCrashReport() + + A context manager used to try to prevent crash dialog popups on tests that + are expected to crash a subprocess. + + On Windows, it disables Windows Error Reporting dialogs using + `SetErrorMode <http://msdn.microsoft.com/en-us/library/windows/desktop/ms680621.aspx>`_. + + On UNIX, :func:`resource.setrlimit` is used to set + :attr:`resource.RLIMIT_CORE`'s soft limit to 0 to prevent coredump file + creation. + + On both platforms, the old value is restored by :meth:`__exit__`. + + .. class:: WarningsRecorder() Class used to record warnings for unit tests. See documentation of diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index c625254..edf1fd6 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -10,11 +10,11 @@ -------------- -The :mod:`textwrap` module provides two convenience functions, :func:`wrap` and -:func:`fill`, as well as :class:`TextWrapper`, the class that does all the work, -and two utility functions, :func:`dedent` and :func:`indent`. If you're just wrapping or filling one -or two text strings, the convenience functions should be good enough; -otherwise, you should use an instance of :class:`TextWrapper` for efficiency. +The :mod:`textwrap` module provides some convenience functions, +as well as :class:`TextWrapper`, the class that does all the work. +If you're just wrapping or filling one or two text strings, the convenience +functions should be good enough; otherwise, you should use an instance of +:class:`TextWrapper` for efficiency. .. function:: wrap(text, width=70, **kwargs) @@ -39,19 +39,31 @@ otherwise, you should use an instance of :class:`TextWrapper` for efficiency. In particular, :func:`fill` accepts exactly the same keyword arguments as :func:`wrap`. -Both :func:`wrap` and :func:`fill` work by creating a :class:`TextWrapper` -instance and calling a single method on it. That instance is not reused, so for -applications that wrap/fill many text strings, it will be more efficient for you -to create your own :class:`TextWrapper` object. -Text is preferably wrapped on whitespaces and right after the hyphens in -hyphenated words; only then will long words be broken if necessary, unless -:attr:`TextWrapper.break_long_words` is set to false. +.. function:: shorten(text, width, **kwargs) + + Collapse and truncate the given *text* to fit in the given *width*. + + First the whitespace in *text* is collapsed (all whitespace is replaced by + single spaces). If the result fits in the *width*, it is returned. + Otherwise, enough words are dropped from the end so that the remaining words + plus the :attr:`placeholder` fit within :attr:`width`:: + + >>> textwrap.shorten("Hello world!", width=12) + 'Hello world!' + >>> textwrap.shorten("Hello world!", width=11) + 'Hello [...]' + >>> textwrap.shorten("Hello world", width=10, placeholder="...") + 'Hello...' + + Optional keyword arguments correspond to the instance attributes of + :class:`TextWrapper`, documented below. Note that the whitespace is + collapsed before the text is passed to the :class:`TextWrapper` :meth:`fill` + function, so changing the value of :attr:`.tabsize`, :attr:`.expand_tabs`, + :attr:`.drop_whitespace`, and :attr:`.replace_whitespace` will have no effect. + + .. versionadded:: 3.4 -Two additional utility function, :func:`dedent` and :func:`indent`, are -provided to remove indentation from strings that have unwanted whitespace -to the left of the text and to add an arbitrary prefix to selected lines -in a block of text. .. function:: dedent(text) @@ -102,6 +114,16 @@ in a block of text. + world +:func:`wrap`, :func:`fill` and :func:`shorten` work by creating a +:class:`TextWrapper` instance and calling a single method on it. That +instance is not reused, so for applications that process many text +strings using :func:`wrap` and/or :func:`fill`, it may be more efficient to +create your own :class:`TextWrapper` object. + +Text is preferably wrapped on whitespaces and right after the hyphens in +hyphenated words; only then will long words be broken if necessary, unless +:attr:`TextWrapper.break_long_words` is set to false. + .. class:: TextWrapper(**kwargs) The :class:`TextWrapper` constructor accepts a number of optional keyword @@ -235,7 +257,23 @@ in a block of text. was to always allow breaking hyphenated words. - :class:`TextWrapper` also provides two public methods, analogous to the + .. attribute:: max_lines + + (default: ``None``) If not ``None``, then the output will contain at most + *max_lines* lines, with *placeholder* appearing at the end of the output. + + .. versionadded:: 3.4 + + + .. attribute:: placeholder + + (default: ``' [...]'``) String that will appear at the end of the output + text if it has been truncated. + + .. versionadded:: 3.4 + + + :class:`TextWrapper` also provides some public methods, analogous to the module-level convenience functions: .. method:: wrap(text) @@ -251,4 +289,3 @@ in a block of text. Wraps the single paragraph in *text*, and returns a single string containing the wrapped paragraph. - diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 7c326d1..4a3b3ea 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -57,6 +57,15 @@ This module defines the following functions: and threads that have not yet been started. +.. function:: main_thread() + + Return the main :class:`Thread` object. In normal conditions, the + main thread is the thread from which the Python interpreter was + started. + + .. versionadded:: 3.4 + + .. function:: settrace(func) .. index:: single: trace function diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index a487917..824a8a3 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -63,6 +63,12 @@ The module defines three convenience functions and a public class: Create a :class:`Timer` instance with the given statement, *setup* code and *timer* function and run its :meth:`.timeit` method with *number* executions. + .. note:: + + Because :meth:`.timeit` is executing *stmt*, placing a return statement + in *stmt* will prevent :meth:`.timeit` from returning execution time. + It will instead return the data specified by your return statement. + .. function:: repeat(stmt='pass', setup='pass', timer=<default timer>, repeat=3, number=1000000) @@ -151,7 +157,7 @@ The module defines three convenience functions and a public class: t = Timer(...) # outside the try/except try: t.timeit(...) # or t.repeat(...) - except: + except Exception: t.print_exc() The advantage over the standard traceback is that source lines in the diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index eeb1f80..0715a43 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -15,7 +15,7 @@ this should open a window demonstrating a simple Tk interface. .. seealso:: - `Python Tkinter Resources <http://www.python.org/topics/tkinter/>`_ + `Python Tkinter Resources <https://wiki.python.org/moin/TkInter>`_ The Python Tkinter Topic Guide provides a great deal of information on using Tk from Python and links to other sources of information on Tk. @@ -747,32 +747,6 @@ Entry widget indexes (index, view index, etc.) displayed. You can use these :mod:`tkinter` functions to access these special points in text widgets: -.. function:: AtEnd() - refers to the last position in the text - - .. deprecated:: 3.3 - -.. function:: AtInsert() - refers to the point where the text cursor is - - .. deprecated:: 3.3 - -.. function:: AtSelFirst() - indicates the beginning point of the selected text - - .. deprecated:: 3.3 - -.. function:: AtSelLast() - denotes the last point of the selected text and finally - - .. deprecated:: 3.3 - -.. function:: At(x[, y]) - refers to the character at pixel location *x*, *y* (with *y* not used in the - case of a text entry widget, which contains a single line of text). - - .. deprecated:: 3.3 - Text widget indexes The index notation for Text widgets is very rich and is best described in the Tk man pages. diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst index 37d9f41..bd6b121 100644 --- a/Doc/library/tokenize.rst +++ b/Doc/library/tokenize.rst @@ -131,6 +131,24 @@ function it uses to do this is available: .. versionadded:: 3.2 +.. exception:: TokenError + + Raised when either a docstring or expression that may be split over several + lines is not completed anywhere in the file, for example:: + + """Beginning of + docstring + + or:: + + [1, + 2, + 3 + +Note that unclosed single-quoted strings do not cause an error to be +raised. They are tokenized as ``ERRORTOKEN``, followed by the tokenization of +their contents. + .. _tokenize-cli: diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 32e5733..15fbedc 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -72,7 +72,7 @@ The module defines the following functions: Return a list of up to *limit* "pre-processed" stack trace entries extracted from the traceback object *traceback*. It is useful for alternate formatting of stack traces. If *limit* is omitted or ``None``, all entries are extracted. A - "pre-processed" stack trace entry is a quadruple (*filename*, *line number*, + "pre-processed" stack trace entry is a 4-tuple (*filename*, *line number*, *function name*, *text*) representing the information that is usually printed for a stack trace. The *text* is a string with leading and trailing whitespace stripped; if the source is not available it is ``None``. @@ -129,6 +129,13 @@ The module defines the following functions: A shorthand for ``format_list(extract_stack(f, limit))``. +.. function:: clear_frames(tb) + + Clears the local variables of all the stack frames in a traceback *tb* + by calling the :meth:`clear` method of each frame object. + + .. versionadded:: 3.4 + .. _traceback-example: @@ -146,7 +153,7 @@ module. :: source = input(">>> ") try: exec(source, envdir) - except: + except Exception: print("Exception in user code:") print("-"*60) traceback.print_exc(file=sys.stdout) diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst new file mode 100644 index 0000000..3405518 --- /dev/null +++ b/Doc/library/tracemalloc.rst @@ -0,0 +1,634 @@ +:mod:`tracemalloc` --- Trace memory allocations +=============================================== + +.. module:: tracemalloc + :synopsis: Trace memory allocations. + +.. versionadded:: 3.4 + +The tracemalloc module is a debug tool to trace memory blocks allocated by +Python. It provides the following information: + +* Traceback where an object was allocated +* Statistics on allocated memory blocks per filename and per line number: + total size, number and average size of allocated memory blocks +* Compute the differences between two snapshots to detect memory leaks + +To trace most memory blocks allocated by Python, the module should be started +as early as possible by setting the :envvar:`PYTHONTRACEMALLOC` environment +variable to ``1``, or by using :option:`-X` ``tracemalloc`` command line +option. The :func:`tracemalloc.start` function can be called at runtime to +start tracing Python memory allocations. + +By default, a trace of an allocated memory block only stores the most recent +frame (1 frame). To store 25 frames at startup: set the +:envvar:`PYTHONTRACEMALLOC` environment variable to ``25``, or use the +:option:`-X` ``tracemalloc=25`` command line option. + + +Examples +======== + +Display the top 10 +------------------ + +Display the 10 files allocating the most memory:: + + import tracemalloc + + tracemalloc.start() + + # ... run your application ... + + snapshot = tracemalloc.take_snapshot() + top_stats = snapshot.statistics('lineno') + + print("[ Top 10 ]") + for stat in top_stats[:10]: + print(stat) + + +Example of output of the Python test suite:: + + [ Top 10 ] + <frozen importlib._bootstrap>:716: size=4855 KiB, count=39328, average=126 B + <frozen importlib._bootstrap>:284: size=521 KiB, count=3199, average=167 B + /usr/lib/python3.4/collections/__init__.py:368: size=244 KiB, count=2315, average=108 B + /usr/lib/python3.4/unittest/case.py:381: size=185 KiB, count=779, average=243 B + /usr/lib/python3.4/unittest/case.py:402: size=154 KiB, count=378, average=416 B + /usr/lib/python3.4/abc.py:133: size=88.7 KiB, count=347, average=262 B + <frozen importlib._bootstrap>:1446: size=70.4 KiB, count=911, average=79 B + <frozen importlib._bootstrap>:1454: size=52.0 KiB, count=25, average=2131 B + <string>:5: size=49.7 KiB, count=148, average=344 B + /usr/lib/python3.4/sysconfig.py:411: size=48.0 KiB, count=1, average=48.0 KiB + +We can see that Python loaded ``4.8 MiB`` data (bytecode and constants) from +modules and that the :mod:`collections` module allocated ``244 KiB`` to build +:class:`~collections.namedtuple` types. + +See :meth:`Snapshot.statistics` for more options. + + +Compute differences +------------------- + +Take two snapshots and display the differences:: + + import tracemalloc + tracemalloc.start() + # ... start your application ... + + snapshot1 = tracemalloc.take_snapshot() + # ... call the function leaking memory ... + snapshot2 = tracemalloc.take_snapshot() + + top_stats = snapshot2.compare_to(snapshot1, 'lineno') + + print("[ Top 10 differences ]") + for stat in top_stats[:10]: + print(stat) + +Example of output before/after running some tests of the Python test suite:: + + [ Top 10 differences ] + <frozen importlib._bootstrap>:716: size=8173 KiB (+4428 KiB), count=71332 (+39369), average=117 B + /usr/lib/python3.4/linecache.py:127: size=940 KiB (+940 KiB), count=8106 (+8106), average=119 B + /usr/lib/python3.4/unittest/case.py:571: size=298 KiB (+298 KiB), count=589 (+589), average=519 B + <frozen importlib._bootstrap>:284: size=1005 KiB (+166 KiB), count=7423 (+1526), average=139 B + /usr/lib/python3.4/mimetypes.py:217: size=112 KiB (+112 KiB), count=1334 (+1334), average=86 B + /usr/lib/python3.4/http/server.py:848: size=96.0 KiB (+96.0 KiB), count=1 (+1), average=96.0 KiB + /usr/lib/python3.4/inspect.py:1465: size=83.5 KiB (+83.5 KiB), count=109 (+109), average=784 B + /usr/lib/python3.4/unittest/mock.py:491: size=77.7 KiB (+77.7 KiB), count=143 (+143), average=557 B + /usr/lib/python3.4/urllib/parse.py:476: size=71.8 KiB (+71.8 KiB), count=969 (+969), average=76 B + /usr/lib/python3.4/contextlib.py:38: size=67.2 KiB (+67.2 KiB), count=126 (+126), average=546 B + +We can see that Python has loaded ``8.2 MiB`` of module data (bytecode and +constants), and that this is ``4.4 MiB`` more than had been loaded before the +tests, when the previous snapshot was taken. Similarly, the :mod:`linecache` +module has cached ``940 KiB`` of Python source code to format tracebacks, all +of it since the previous snapshot. + +If the system has little free memory, snapshots can be written on disk using +the :meth:`Snapshot.dump` method to analyze the snapshot offline. Then use the +:meth:`Snapshot.load` method reload the snapshot. + + +Get the traceback of a memory block +----------------------------------- + +Code to display the traceback of the biggest memory block:: + + import tracemalloc + + # Store 25 frames + tracemalloc.start(25) + + # ... run your application ... + + snapshot = tracemalloc.take_snapshot() + top_stats = snapshot.statistics('traceback') + + # pick the biggest memory block + stat = top_stats[0] + print("%s memory blocks: %.1f KiB" % (stat.count, stat.size / 1024)) + for line in stat.traceback.format(): + print(line) + +Example of output of the Python test suite (traceback limited to 25 frames):: + + 903 memory blocks: 870.1 KiB + File "<frozen importlib._bootstrap>", line 716 + File "<frozen importlib._bootstrap>", line 1036 + File "<frozen importlib._bootstrap>", line 934 + File "<frozen importlib._bootstrap>", line 1068 + File "<frozen importlib._bootstrap>", line 619 + File "<frozen importlib._bootstrap>", line 1581 + File "<frozen importlib._bootstrap>", line 1614 + File "/usr/lib/python3.4/doctest.py", line 101 + import pdb + File "<frozen importlib._bootstrap>", line 284 + File "<frozen importlib._bootstrap>", line 938 + File "<frozen importlib._bootstrap>", line 1068 + File "<frozen importlib._bootstrap>", line 619 + File "<frozen importlib._bootstrap>", line 1581 + File "<frozen importlib._bootstrap>", line 1614 + File "/usr/lib/python3.4/test/support/__init__.py", line 1728 + import doctest + File "/usr/lib/python3.4/test/test_pickletools.py", line 21 + support.run_doctest(pickletools) + File "/usr/lib/python3.4/test/regrtest.py", line 1276 + test_runner() + File "/usr/lib/python3.4/test/regrtest.py", line 976 + display_failure=not verbose) + File "/usr/lib/python3.4/test/regrtest.py", line 761 + match_tests=ns.match_tests) + File "/usr/lib/python3.4/test/regrtest.py", line 1563 + main() + File "/usr/lib/python3.4/test/__main__.py", line 3 + regrtest.main_in_temp_cwd() + File "/usr/lib/python3.4/runpy.py", line 73 + exec(code, run_globals) + File "/usr/lib/python3.4/runpy.py", line 160 + "__main__", fname, loader, pkg_name) + +We can see that the most memory was allocated in the :mod:`importlib` module to +load data (bytecode and constants) from modules: ``870 KiB``. The traceback is +where the :mod:`importlib` loaded data most recently: on the ``import pdb`` +line of the :mod:`doctest` module. The traceback may change if a new module is +loaded. + + +Pretty top +---------- + +Code to display the 10 lines allocating the most memory with a pretty output, +ignoring ``<frozen importlib._bootstrap>`` and ``<unknown>`` files:: + + import linecache + import os + import tracemalloc + + def display_top(snapshot, group_by='lineno', limit=10): + snapshot = snapshot.filter_traces(( + tracemalloc.Filter(False, "<frozen importlib._bootstrap>"), + tracemalloc.Filter(False, "<unknown>"), + )) + top_stats = snapshot.statistics(group_by) + + print("Top %s lines" % limit) + for index, stat in enumerate(top_stats[:limit], 1): + frame = stat.traceback[0] + # replace "/path/to/module/file.py" with "module/file.py" + filename = os.sep.join(frame.filename.split(os.sep)[-2:]) + print("#%s: %s:%s: %.1f KiB" + % (index, filename, frame.lineno, stat.size / 1024)) + line = linecache.getline(frame.filename, frame.lineno).strip() + if line: + print(' %s' % line) + + other = top_stats[limit:] + if other: + size = sum(stat.size for stat in other) + print("%s other: %.1f KiB" % (len(other), size / 1024)) + total = sum(stat.size for stat in top_stats) + print("Total allocated size: %.1f KiB" % (total / 1024)) + + tracemalloc.start() + + # ... run your application ... + + snapshot = tracemalloc.take_snapshot() + display_top(snapshot) + +Example of output of the Python test suite:: + + Top 10 lines + #1: Lib/base64.py:414: 419.8 KiB + _b85chars2 = [(a + b) for a in _b85chars for b in _b85chars] + #2: Lib/base64.py:306: 419.8 KiB + _a85chars2 = [(a + b) for a in _a85chars for b in _a85chars] + #3: collections/__init__.py:368: 293.6 KiB + exec(class_definition, namespace) + #4: Lib/abc.py:133: 115.2 KiB + cls = super().__new__(mcls, name, bases, namespace) + #5: unittest/case.py:574: 103.1 KiB + testMethod() + #6: Lib/linecache.py:127: 95.4 KiB + lines = fp.readlines() + #7: urllib/parse.py:476: 71.8 KiB + for a in _hexdig for b in _hexdig} + #8: <string>:5: 62.0 KiB + #9: Lib/_weakrefset.py:37: 60.0 KiB + self.data = set() + #10: Lib/base64.py:142: 59.8 KiB + _b32tab2 = [a + b for a in _b32tab for b in _b32tab] + 6220 other: 3602.8 KiB + Total allocated size: 5303.1 KiB + +See :meth:`Snapshot.statistics` for more options. + + +API +=== + +Functions +--------- + +.. function:: clear_traces() + + Clear traces of memory blocks allocated by Python. + + See also :func:`stop`. + + +.. function:: get_object_traceback(obj) + + Get the traceback where the Python object *obj* was allocated. + Return a :class:`Traceback` instance, or ``None`` if the :mod:`tracemalloc` + module is not tracing memory allocations or did not trace the allocation of + the object. + + See also :func:`gc.get_referrers` and :func:`sys.getsizeof` functions. + + +.. function:: get_traceback_limit() + + Get the maximum number of frames stored in the traceback of a trace. + + The :mod:`tracemalloc` module must be tracing memory allocations to + get the limit, otherwise an exception is raised. + + The limit is set by the :func:`start` function. + + +.. function:: get_traced_memory() + + Get the current size and peak size of memory blocks traced by the + :mod:`tracemalloc` module as a tuple: ``(current: int, peak: int)``. + + +.. function:: get_tracemalloc_memory() + + Get the memory usage in bytes of the :mod:`tracemalloc` module used to store + traces of memory blocks. + Return an :class:`int`. + + +.. function:: is_tracing() + + ``True`` if the :mod:`tracemalloc` module is tracing Python memory + allocations, ``False`` otherwise. + + See also :func:`start` and :func:`stop` functions. + + +.. function:: start(nframe: int=1) + + Start tracing Python memory allocations: install hooks on Python memory + allocators. Collected tracebacks of traces will be limited to *nframe* + frames. By default, a trace of a memory block only stores the most recent + frame: the limit is ``1``. *nframe* must be greater or equal to ``1``. + + Storing more than ``1`` frame is only useful to compute statistics grouped + by ``'traceback'`` or to compute cumulative statistics: see the + :meth:`Snapshot.compare_to` and :meth:`Snapshot.statistics` methods. + + Storing more frames increases the memory and CPU overhead of the + :mod:`tracemalloc` module. Use the :func:`get_tracemalloc_memory` function + to measure how much memory is used by the :mod:`tracemalloc` module. + + The :envvar:`PYTHONTRACEMALLOC` environment variable + (``PYTHONTRACEMALLOC=NFRAME``) and the :option:`-X` ``tracemalloc=NFRAME`` + command line option can be used to start tracing at startup. + + See also :func:`stop`, :func:`is_tracing` and :func:`get_traceback_limit` + functions. + + +.. function:: stop() + + Stop tracing Python memory allocations: uninstall hooks on Python memory + allocators. Also clears all previously collected traces of memory blocks + allocated by Python. + + Call :func:`take_snapshot` function to take a snapshot of traces before + clearing them. + + See also :func:`start`, :func:`is_tracing` and :func:`clear_traces` + functions. + + +.. function:: take_snapshot() + + Take a snapshot of traces of memory blocks allocated by Python. Return a new + :class:`Snapshot` instance. + + The snapshot does not include memory blocks allocated before the + :mod:`tracemalloc` module started to trace memory allocations. + + Tracebacks of traces are limited to :func:`get_traceback_limit` frames. Use + the *nframe* parameter of the :func:`start` function to store more frames. + + The :mod:`tracemalloc` module must be tracing memory allocations to take a + snapshot, see the the :func:`start` function. + + See also the :func:`get_object_traceback` function. + + +Filter +------ + +.. class:: Filter(inclusive: bool, filename_pattern: str, lineno: int=None, all_frames: bool=False) + + Filter on traces of memory blocks. + + See the :func:`fnmatch.fnmatch` function for the syntax of + *filename_pattern*. The ``'.pyc'`` and ``'.pyo'`` file extensions are + replaced with ``'.py'``. + + Examples: + + * ``Filter(True, subprocess.__file__)`` only includes traces of the + :mod:`subprocess` module + * ``Filter(False, tracemalloc.__file__)`` excludes traces of the + :mod:`tracemalloc` module + * ``Filter(False, "<unknown>")`` excludes empty tracebacks + + .. attribute:: inclusive + + If *inclusive* is ``True`` (include), only trace memory blocks allocated + in a file with a name matching :attr:`filename_pattern` at line number + :attr:`lineno`. + + If *inclusive* is ``False`` (exclude), ignore memory blocks allocated in + a file with a name matching :attr:`filename_pattern` at line number + :attr:`lineno`. + + .. attribute:: lineno + + Line number (``int``) of the filter. If *lineno* is ``None``, the filter + matches any line number. + + .. attribute:: filename_pattern + + Filename pattern of the filter (``str``). + + .. attribute:: all_frames + + If *all_frames* is ``True``, all frames of the traceback are checked. If + *all_frames* is ``False``, only the most recent frame is checked. + + This attribute has no effect if the traceback limit is ``1``. See the + :func:`get_traceback_limit` function and :attr:`Snapshot.traceback_limit` + attribute. + + +Frame +----- + +.. class:: Frame + + Frame of a traceback. + + The :class:`Traceback` class is a sequence of :class:`Frame` instances. + + .. attribute:: filename + + Filename (``str``). + + .. attribute:: lineno + + Line number (``int``). + + +Snapshot +-------- + +.. class:: Snapshot + + Snapshot of traces of memory blocks allocated by Python. + + The :func:`take_snapshot` function creates a snapshot instance. + + .. method:: compare_to(old_snapshot: Snapshot, group_by: str, cumulative: bool=False) + + Compute the differences with an old snapshot. Get statistics as a sorted + list of :class:`StatisticDiff` instances grouped by *group_by*. + + See the :meth:`statistics` method for *group_by* and *cumulative* + parameters. + + The result is sorted from the biggest to the smallest by: absolute value + of :attr:`StatisticDiff.size_diff`, :attr:`StatisticDiff.size`, absolute + value of :attr:`StatisticDiff.count_diff`, :attr:`Statistic.count` and + then by :attr:`StatisticDiff.traceback`. + + + .. method:: dump(filename) + + Write the snapshot into a file. + + Use :meth:`load` to reload the snapshot. + + + .. method:: filter_traces(filters) + + Create a new :class:`Snapshot` instance with a filtered :attr:`traces` + sequence, *filters* is a list of :class:`Filter` instances. If *filters* + is an empty list, return a new :class:`Snapshot` instance with a copy of + the traces. + + All inclusive filters are applied at once, a trace is ignored if no + inclusive filters match it. A trace is ignored if at least one exclusive + filter matchs it. + + + .. classmethod:: load(filename) + + Load a snapshot from a file. + + See also :meth:`dump`. + + + .. method:: statistics(group_by: str, cumulative: bool=False) + + Get statistics as a sorted list of :class:`Statistic` instances grouped + by *group_by*: + + ===================== ======================== + group_by description + ===================== ======================== + ``'filename'`` filename + ``'lineno'`` filename and line number + ``'traceback'`` traceback + ===================== ======================== + + If *cumulative* is ``True``, cumulate size and count of memory blocks of + all frames of the traceback of a trace, not only the most recent frame. + The cumulative mode can only be used with *group_by* equals to + ``'filename'`` and ``'lineno'``. + + The result is sorted from the biggest to the smallest by: + :attr:`Statistic.size`, :attr:`Statistic.count` and then by + :attr:`Statistic.traceback`. + + + .. attribute:: traceback_limit + + Maximum number of frames stored in the traceback of :attr:`traces`: + result of the :func:`get_traceback_limit` when the snapshot was taken. + + .. attribute:: traces + + Traces of all memory blocks allocated by Python: sequence of + :class:`Trace` instances. + + The sequence has an undefined order. Use the :meth:`Snapshot.statistics` + method to get a sorted list of statistics. + + +Statistic +--------- + +.. class:: Statistic + + Statistic on memory allocations. + + :func:`Snapshot.statistics` returns a list of :class:`Statistic` instances. + + See also the :class:`StatisticDiff` class. + + .. attribute:: count + + Number of memory blocks (``int``). + + .. attribute:: size + + Total size of memory blocks in bytes (``int``). + + .. attribute:: traceback + + Traceback where the memory block was allocated, :class:`Traceback` + instance. + + +StatisticDiff +------------- + +.. class:: StatisticDiff + + Statistic difference on memory allocations between an old and a new + :class:`Snapshot` instance. + + :func:`Snapshot.compare_to` returns a list of :class:`StatisticDiff` + instances. See also the :class:`Statistic` class. + + .. attribute:: count + + Number of memory blocks in the new snapshot (``int``): ``0`` if + the memory blocks have been released in the new snapshot. + + .. attribute:: count_diff + + Difference of number of memory blocks between the old and the new + snapshots (``int``): ``0`` if the memory blocks have been allocated in + the new snapshot. + + .. attribute:: size + + Total size of memory blocks in bytes in the new snapshot (``int``): + ``0`` if the memory blocks have been released in the new snapshot. + + .. attribute:: size_diff + + Difference of total size of memory blocks in bytes between the old and + the new snapshots (``int``): ``0`` if the memory blocks have been + allocated in the new snapshot. + + .. attribute:: traceback + + Traceback where the memory blocks were allocated, :class:`Traceback` + instance. + + +Trace +----- + +.. class:: Trace + + Trace of a memory block. + + The :attr:`Snapshot.traces` attribute is a sequence of :class:`Trace` + instances. + + .. attribute:: size + + Size of the memory block in bytes (``int``). + + .. attribute:: traceback + + Traceback where the memory block was allocated, :class:`Traceback` + instance. + + +Traceback +--------- + +.. class:: Traceback + + Sequence of :class:`Frame` instances sorted from the most recent frame to + the oldest frame. + + A traceback contains at least ``1`` frame. If the ``tracemalloc`` module + failed to get a frame, the filename ``"<unknown>"`` at line number ``0`` is + used. + + When a snapshot is taken, tracebacks of traces are limited to + :func:`get_traceback_limit` frames. See the :func:`take_snapshot` function. + + The :attr:`Trace.traceback` attribute is an instance of :class:`Traceback` + instance. + + .. method:: format(limit=None) + + Format the traceback as a list of lines with newlines. Use the + :mod:`linecache` module to retrieve lines from the source code. If + *limit* is set, only format the *limit* most recent frames. + + Similar to the :func:`traceback.format_tb` function, except that + :meth:`format` does not include newlines. + + Example:: + + print("Traceback (most recent call first):") + for line in traceback: + print(line) + + Output:: + + Traceback (most recent call first): + File "test.py", line 9 + obj = Object() + File "test.py", line 12 + tb = tracemalloc.get_object_traceback(f()) + diff --git a/Doc/library/tulip_coro.dia b/Doc/library/tulip_coro.dia Binary files differnew file mode 100644 index 0000000..eccf4fa --- /dev/null +++ b/Doc/library/tulip_coro.dia diff --git a/Doc/library/tulip_coro.png b/Doc/library/tulip_coro.png Binary files differnew file mode 100644 index 0000000..65b6951 --- /dev/null +++ b/Doc/library/tulip_coro.png diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 695480f..abdb939 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -15,6 +15,9 @@ It also defines names for some object types that are used by the standard Python interpreter, but not exposed as builtins like :class:`int` or :class:`str` are. +Finally, it provides some additional type-related utility classes and functions +that are not fundamental enough to be builtins. + Dynamic Type Creation --------------------- @@ -107,9 +110,35 @@ Standard names are defined for the following types: C".) -.. data:: ModuleType +.. class:: ModuleType(name, doc=None) + + The type of :term:`modules <module>`. Constructor takes the name of the + module to be created and optionally its :term:`docstring`. + + .. attribute:: __doc__ + + The :term:`docstring` of the module. Defaults to ``None``. + + .. attribute:: __loader__ + + The :term:`loader` which loaded the module. Defaults to ``None``. + + .. versionchanged:: 3.4 + Defaults to ``None``. Previously the attribute was optional. + + .. attribute:: __name__ + + The name of the module. - The type of modules. + .. attribute:: __package__ + + Which :term:`package` a module belongs to. If the module is top-level + (i.e. not a part of any specific package) then the attribute should be set + to ``''``, else it should be set to the name of the package (which can be + :attr:`__name__` if the module is a package itself). Defaults to ``None``. + + .. versionchanged:: 3.4 + Defaults to ``None``. Previously the attribute was optional. .. data:: TracebackType @@ -194,6 +223,9 @@ Standard names are defined for the following types: Return a new view of the underlying mapping's values. +Additional Utility Classes and Functions +---------------------------------------- + .. class:: SimpleNamespace A simple :class:`object` subclass that provides attribute access to its @@ -212,9 +244,26 @@ Standard names are defined for the following types: keys = sorted(self.__dict__) items = ("{}={!r}".format(k, self.__dict__[k]) for k in keys) return "{}({})".format(type(self).__name__, ", ".join(items)) + def __eq__(self, other): + return self.__dict__ == other.__dict__ ``SimpleNamespace`` may be useful as a replacement for ``class NS: pass``. However, for a structured record type use :func:`~collections.namedtuple` instead. .. versionadded:: 3.3 + + +.. function:: DynamicClassAttribute(fget=None, fset=None, fdel=None, doc=None) + + Route attribute access on a class to __getattr__. + + This is a descriptor, used to define attributes that act differently when + accessed through an instance and through a class. Instance access remains + normal, but access to an attribute through a class will be routed to the + class's __getattr__ method; this is done by raising AttributeError. + + This allows one to have properties active on an instance, and have virtual + attributes on the class with the same name (see Enum for an example). + + .. versionadded:: 3.4 diff --git a/Doc/library/undoc.rst b/Doc/library/undoc.rst index 80386d2..20830e2 100644 --- a/Doc/library/undoc.rst +++ b/Doc/library/undoc.rst @@ -20,7 +20,7 @@ These modules are used to implement the :mod:`os.path` module, and are not documented beyond this mention. There's little need to document these. :mod:`ntpath` - --- Implementation of :mod:`os.path` on Win32, Win64, WinCE, and OS/2 platforms. + --- Implementation of :mod:`os.path` on Win32, Win64, and WinCE platforms. :mod:`posixpath` --- Implementation of :mod:`os.path` on POSIX. diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index ad70dd9..3b3d3a0 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -15,8 +15,8 @@ This module provides access to the Unicode Character Database (UCD) which defines character properties for all Unicode characters. The data contained in -this database is compiled from the `UCD version 6.1.0 -<http://www.unicode.org/Public/6.1.0/ucd>`_. +this database is compiled from the `UCD version 6.3.0 +<http://www.unicode.org/Public/6.3.0/ucd>`_. The module uses the same names and symbols as defined by Unicode Standard Annex #44, `"Unicode Character Database" @@ -166,6 +166,6 @@ Examples: .. rubric:: Footnotes -.. [#] http://www.unicode.org/Public/6.1.0/ucd/NameAliases.txt +.. [#] http://www.unicode.org/Public/6.3.0/ucd/NameAliases.txt -.. [#] http://www.unicode.org/Public/6.1.0/ucd/NamedSequences.txt +.. [#] http://www.unicode.org/Public/6.3.0/ucd/NamedSequences.txt diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index 554ef11..1c43bfe 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -277,6 +277,20 @@ instantiate the class in those tests. ... AttributeError: object has no attribute 'old_method' +Using a specification also enables a smarter matching of calls made to the +mock, regardless of whether some parameters were passed as positional or +named arguments:: + + >>> def f(a, b, c): pass + ... + >>> mock = Mock(spec=f) + >>> mock(1, 2, 3) + <Mock name='mock()' id='140161580456576'> + >>> mock.assert_called_with(a=1, b=2, c=3) + +If you want this smarter matching to also work with method calls on the mock, +you can use :ref:`auto-speccing <auto-speccing>`. + If you want a stronger form of specification that prevents the setting of arbitrary attributes as well as the getting of them then you can use `spec_set` instead of `spec`. @@ -920,8 +934,8 @@ After the `MagicMock` has been used we can use attributes like the magic methods you specifically want: >>> mock = Mock() - >>> mock.__setitem__ = Mock(side_effect=getitem) - >>> mock.__getitem__ = Mock(side_effect=setitem) + >>> mock.__getitem__ = Mock(side_effect=getitem) + >>> mock.__setitem__ = Mock(side_effect=setitem) A *third* option is to use `MagicMock` but passing in `dict` as the `spec` (or `spec_set`) argument so that the `MagicMock` created only has diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index f805df3..cb72a68 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -264,7 +264,6 @@ the `new_callable` argument to `patch`. <Mock name='mock.method()' id='...'> >>> mock.method.assert_called_with(1, 2, 3, test='wow') - .. method:: assert_called_once_with(*args, **kwargs) Assert that the mock was called exactly once and with the specified @@ -685,6 +684,27 @@ have to create a dictionary and unpack it using `**`: ... KeyError +A callable mock which was created with a *spec* (or a *spec_set*) will +introspect the specification object's signature when matching calls to +the mock. Therefore, it can match the actual call's arguments regardless +of whether they were passed positionally or by name:: + + >>> def f(a, b, c): pass + ... + >>> mock = Mock(spec=f) + >>> mock(1, 2, c=3) + <Mock name='mock()' id='140161580456576'> + >>> mock.assert_called_with(1, 2, 3) + >>> mock.assert_called_with(a=1, b=2, c=3) + +This applies to :meth:`~Mock.assert_called_with`, +:meth:`~Mock.assert_called_once_with`, :meth:`~Mock.assert_has_calls` and +:meth:`~Mock.assert_any_call`. When :ref:`auto-speccing`, it will also +apply to method calls on the mock object. + + .. versionchanged:: 3.4 + Added signature introspection on specced and autospecced mock objects. + .. class:: PropertyMock(*args, **kwargs) @@ -1969,8 +1989,13 @@ mock_open default) then a `MagicMock` will be created for you, with the API limited to methods or attributes available on standard file handles. - `read_data` is a string for the `~io.IOBase.read` method of the file handle - to return. This is an empty string by default. + `read_data` is a string for the :meth:`~io.IOBase.read`, + :meth:`~io.IOBase.readline`, and :meth:`~io.IOBase.readlines` methods + of the file handle to return. Calls to those methods will take data from + `read_data` until it is depleted. The mock of these methods is pretty + simplistic. If you need more control over the data that you are feeding to + the tested code you will need to customize this mock for yourself. + `read_data` is an empty string by default. Using `open` as a context manager is a great way to ensure your file handles are closed properly and is becoming common:: diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index fdc9409..52f6e3f 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -239,9 +239,10 @@ Test Discovery Unittest supports simple test discovery. In order to be compatible with test discovery, all of the test files must be :ref:`modules <tut-modules>` or -:ref:`packages <tut-packages>` importable from the top-level directory of -the project (this means that their filenames must be valid -:ref:`identifiers <identifiers>`). +:ref:`packages <tut-packages>` (including :term:`namespace packages +<namespace package>`) importable from the top-level directory of +the project (this means that their filenames must be valid :ref:`identifiers +<identifiers>`). Test discovery is implemented in :meth:`TestLoader.discover`, but can also be used from the command line. The basic command-line usage is:: @@ -306,6 +307,9 @@ as the start directory. Test modules and packages can customize test loading and discovery by through the `load_tests protocol`_. +.. versionchanged:: 3.4 + Test discovery supports :term:`namespace packages <namespace package>`. + .. _organizing-tests: @@ -559,8 +563,71 @@ The following decorators implement test skipping and expected failures: Usually you can use :meth:`TestCase.skipTest` or one of the skipping decorators instead of raising this directly. -Skipped tests will not have :meth:`setUp` or :meth:`tearDown` run around them. -Skipped classes will not have :meth:`setUpClass` or :meth:`tearDownClass` run. +Skipped tests will not have :meth:`~TestCase.setUp` or :meth:`~TestCase.tearDown` run around them. +Skipped classes will not have :meth:`~TestCase.setUpClass` or :meth:`~TestCase.tearDownClass` run. +Skipped modules will not have :func:`setUpModule` or :func:`tearDownModule` run. + + +.. _subtests: + +Distinguishing test iterations using subtests +--------------------------------------------- + +.. versionadded:: 3.4 + +When some of your tests differ only by a some very small differences, for +instance some parameters, unittest allows you to distinguish them inside +the body of a test method using the :meth:`~TestCase.subTest` context manager. + +For example, the following test:: + + class NumbersTest(unittest.TestCase): + + def test_even(self): + """ + Test that numbers between 0 and 5 are all even. + """ + for i in range(0, 6): + with self.subTest(i=i): + self.assertEqual(i % 2, 0) + +will produce the following output:: + + ====================================================================== + FAIL: test_even (__main__.NumbersTest) (i=1) + ---------------------------------------------------------------------- + Traceback (most recent call last): + File "subtests.py", line 32, in test_even + self.assertEqual(i % 2, 0) + AssertionError: 1 != 0 + + ====================================================================== + FAIL: test_even (__main__.NumbersTest) (i=3) + ---------------------------------------------------------------------- + Traceback (most recent call last): + File "subtests.py", line 32, in test_even + self.assertEqual(i % 2, 0) + AssertionError: 1 != 0 + + ====================================================================== + FAIL: test_even (__main__.NumbersTest) (i=5) + ---------------------------------------------------------------------- + Traceback (most recent call last): + File "subtests.py", line 32, in test_even + self.assertEqual(i % 2, 0) + AssertionError: 1 != 0 + +Without using a subtest, execution would stop after the first failure, +and the error would be less easy to diagnose because the value of ``i`` +wouldn't be displayed:: + + ====================================================================== + FAIL: test_even (__main__.NumbersTest) + ---------------------------------------------------------------------- + Traceback (most recent call last): + File "subtests.py", line 32, in test_even + self.assertEqual(i % 2, 0) + AssertionError: 1 != 0 .. _unittest-contents: @@ -607,9 +674,9 @@ Test cases .. method:: setUp() Method called to prepare the test fixture. This is called immediately - before calling the test method; any exception raised by this method will - be considered an error rather than a test failure. The default - implementation does nothing. + before calling the test method; other than :exc:`AssertionError` or :exc:`SkipTest`, + any exception raised by this method will be considered an error rather than + a test failure. The default implementation does nothing. .. method:: tearDown() @@ -617,10 +684,10 @@ Test cases Method called immediately after the test method has been called and the result recorded. This is called even if the test method raised an exception, so the implementation in subclasses may need to be particularly - careful about checking internal state. Any exception raised by this - method will be considered an error rather than a test failure. This - method will only be called if the :meth:`setUp` succeeds, regardless of - the outcome of the test method. The default implementation does nothing. + careful about checking internal state. Any exception, other than :exc:`AssertionError` + or :exc:`SkipTest`, raised by this method will be considered an error rather than a + test failure. This method will only be called if the :meth:`setUp` succeeds, + regardless of the outcome of the test method. The default implementation does nothing. .. method:: setUpClass() @@ -676,6 +743,21 @@ Test cases .. versionadded:: 3.1 + .. method:: subTest(msg=None, **params) + + Return a context manager which executes the enclosed code block as a + subtest. *msg* and *params* are optional, arbitrary values which are + displayed whenever a subtest fails, allowing you to identify them + clearly. + + A test case can contain any number of subtest declarations, and + they can be arbitrarily nested. + + See :ref:`subtests` for more information. + + .. versionadded:: 3.4 + + .. method:: debug() Run the test without collecting the result. This allows exceptions raised @@ -806,8 +888,8 @@ Test cases - It is also possible to check that exceptions and warnings are raised using - the following methods: + It is also possible to check the production of exceptions, warnings and + log messages using the following methods: +---------------------------------------------------------+--------------------------------------+------------+ | Method | Checks that | New in | @@ -824,6 +906,9 @@ Test cases | :meth:`assertWarnsRegex(warn, r, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *warn* | 3.2 | | <TestCase.assertWarnsRegex>` | and the message matches regex *r* | | +---------------------------------------------------------+--------------------------------------+------------+ + | :meth:`assertLogs(logger, level) | The ``with`` block logs on *logger* | 3.4 | + | <TestCase.assertLogs>` | with minimum *level* | | + +---------------------------------------------------------+--------------------------------------+------------+ .. method:: assertRaises(exception, callable, *args, **kwds) assertRaises(exception, msg=None) @@ -954,6 +1039,47 @@ Test cases .. versionchanged:: 3.3 Added the *msg* keyword argument when used as a context manager. + .. method:: assertLogs(logger=None, level=None) + + A context manager to test that at least one message is logged on + the *logger* or one of its children, with at least the given + *level*. + + If given, *logger* should be a :class:`logging.Logger` object or a + :class:`str` giving the name of a logger. The default is the root + logger, which will catch all messages. + + If given, *level* should be either a numeric logging level or + its string equivalent (for example either ``"ERROR"`` or + :attr:`logging.ERROR`). The default is :attr:`logging.INFO`. + + The test passes if at least one message emitted inside the ``with`` + block matches the *logger* and *level* conditions, otherwise it fails. + + The object returned by the context manager is a recording helper + which keeps tracks of the matching log messages. It has two + attributes: + + .. attribute:: records + + A list of :class:`logging.LogRecord` objects of the matching + log messages. + + .. attribute:: output + + A list of :class:`str` objects with the formatted output of + matching messages. + + Example:: + + with self.assertLogs('foo', level='INFO') as cm: + logging.getLogger('foo').info('first message') + logging.getLogger('foo.bar').error('second message') + self.assertEqual(cm.output, ['INFO:foo:first message', + 'ERROR:foo.bar:second message']) + + .. versionadded:: 3.4 + There are also other methods used to perform more specific checks, such as: @@ -1393,15 +1519,24 @@ Grouping tests Tests grouped by a :class:`TestSuite` are always accessed by iteration. Subclasses can lazily provide tests by overriding :meth:`__iter__`. Note - that this method maybe called several times on a single suite - (for example when counting tests or comparing for equality) - so the tests returned must be the same for repeated iterations. + that this method may be called several times on a single suite (for + example when counting tests or comparing for equality) so the tests + returned by repeated iterations before :meth:`TestSuite.run` must be the + same for each call iteration. After :meth:`TestSuite.run`, callers should + not rely on the tests returned by this method unless the caller uses a + subclass that overrides :meth:`TestSuite._removeTestAtIndex` to preserve + test references. .. versionchanged:: 3.2 In earlier versions the :class:`TestSuite` accessed tests directly rather than through iteration, so overriding :meth:`__iter__` wasn't sufficient for providing tests. + .. versionchanged:: 3.4 + In earlier versions the :class:`TestSuite` held references to each + :class:`TestCase` after :meth:`TestSuite.run`. Subclasses can restore + that behavior by overriding :meth:`TestSuite._removeTestAtIndex`. + In the typical usage of a :class:`TestSuite` object, the :meth:`run` method is invoked by a :class:`TestRunner` rather than by the end-user test harness. @@ -1500,7 +1635,9 @@ Loading and running tests directory must be specified separately. If importing a module fails, for example due to a syntax error, then this - will be recorded as a single error and discovery will continue. + will be recorded as a single error and discovery will continue. If the + import failure is due to :exc:`SkipTest` being raised, it will be recorded + as a skip instead of an error. If a test package name (directory with :file:`__init__.py`) matches the pattern then the package will be checked for a ``load_tests`` @@ -1519,6 +1656,14 @@ Loading and running tests .. versionadded:: 3.2 + .. versionchanged:: 3.4 + Modules that raise :exc:`SkipTest` on import are recorded as skips, + not errors. + Discovery works for :term:`namespace packages <namespace package>`. + Paths are sorted before being imported so that execution order is + the same even if the underlying file system's ordering is not + dependent on file name. + The following attributes of a :class:`TestLoader` can be configured either by subclassing or assignment on an instance: @@ -1630,6 +1775,10 @@ Loading and running tests Return ``True`` if all tests run so far have passed, otherwise returns ``False``. + .. versionchanged:: 3.4 + Returns ``False`` if there were any :attr:`unexpectedSuccesses` + from tests marked with the :func:`expectedFailure` decorator. + .. method:: stop() @@ -1658,14 +1807,14 @@ Loading and running tests Called after the test case *test* has been executed, regardless of the outcome. - .. method:: startTestRun(test) + .. method:: startTestRun() Called once before any tests are executed. .. versionadded:: 3.1 - .. method:: stopTestRun(test) + .. method:: stopTestRun() Called once after all tests are executed. @@ -1728,6 +1877,22 @@ Loading and running tests :attr:`unexpectedSuccesses` attribute. + .. method:: addSubTest(test, subtest, outcome) + + Called when a subtest finishes. *test* is the test case + corresponding to the test method. *subtest* is a custom + :class:`TestCase` instance describing the subtest. + + If *outcome* is :const:`None`, the subtest succeeded. Otherwise, + it failed with an exception where *outcome* is a tuple of the form + returned by :func:`sys.exc_info`: ``(type, value, traceback)``. + + The default implementation does nothing when the outcome is a + success, and records subtest failures as normal failures. + + .. versionadded:: 3.4 + + .. class:: TextTestResult(stream, descriptions, verbosity) A concrete implementation of :class:`TestResult` used by the @@ -1783,6 +1948,14 @@ Loading and running tests stream, descriptions, verbosity + .. method:: run(test) + + This method is the main public interface to the `TextTestRunner`. This + method takes a :class:`TestSuite` or :class:`TestCase` instance. A + :class:`TestResult` is created by calling + :func:`_makeResult` and the test(s) are run and the + results printed to stdout. + .. function:: main(module='__main__', defaultTest=None, argv=None, testRunner=None, \ testLoader=unittest.defaultTestLoader, exit=True, verbosity=1, \ @@ -1802,9 +1975,10 @@ Loading and running tests if __name__ == '__main__': unittest.main(verbosity=2) - The *defaultTest* argument is the name of the test to run if no test names - are specified via *argv*. If not specified or ``None`` and no test names are - provided via *argv*, all tests found in *module* are run. + The *defaultTest* argument is either the name of a single test or an + iterable of test names to run if no test names are specified via *argv*. If + not specified or ``None`` and no test names are provided via *argv*, all + tests found in *module* are run. The *argv* argument can be a list of options passed to the program, with the first element being the program name. If not specified or ``None``, @@ -1842,6 +2016,10 @@ Loading and running tests The *verbosity*, *failfast*, *catchbreak*, *buffer* and *warnings* parameters were added. + .. versionchanged:: 3.4 + The *defaultTest* parameter was changed to also accept an iterable of + test names. + load_tests Protocol ################### diff --git a/Doc/library/urllib.error.rst b/Doc/library/urllib.error.rst index 25c13bd..f7f0c97 100644 --- a/Doc/library/urllib.error.rst +++ b/Doc/library/urllib.error.rst @@ -46,6 +46,13 @@ The following exceptions are raised by :mod:`urllib.error` as appropriate: This is usually a string explaining the reason for this error. + .. attribute:: headers + + The HTTP response headers for the HTTP request that caused the + :exc:`HTTPError`. + + .. versionadded:: 3.4 + .. exception:: ContentTooShortError(msg, content) This exception is raised when the :func:`~urllib.request.urlretrieve` diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 6493a29..019f59c 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -218,12 +218,17 @@ The following classes are provided: fetching of the image, this should be true. *method* should be a string that indicates the HTTP request method that - will be used (e.g. ``'HEAD'``). Its value is stored in the + will be used (e.g. ``'HEAD'``). If provided, its value is stored in the :attr:`~Request.method` attribute and is used by :meth:`get_method()`. + Subclasses may indicate a default method by setting the + :attr:`~Request.method` attribute in the class itself. .. versionchanged:: 3.3 :attr:`Request.method` argument is added to the Request class. + .. versionchanged:: 3.4 + Default :attr:`Request.method` may be indicated at the class level. + .. class:: OpenerDirector() @@ -356,6 +361,11 @@ The following classes are provided: Open local files. +.. class:: DataHandler() + + Open data URLs. + + .. versionadded:: 3.4 .. class:: FTPHandler() @@ -391,6 +401,12 @@ request. The original URL passed to the constructor. + .. versionchanged:: 3.4 + + Request.full_url is a property with setter, getter and a deleter. Getting + :attr:`~Request.full_url` returns the original request URL with the + fragment, if it was present. + .. attribute:: Request.type The URI scheme. @@ -413,6 +429,10 @@ request. The entity body for the request, or None if not specified. + .. versionchanged:: 3.4 + Changing value of :attr:`Request.data` now deletes "Content-Length" + header if it was previously set or calculated. + .. attribute:: Request.unverifiable boolean, indicates whether the request is unverifiable as defined @@ -420,13 +440,20 @@ request. .. attribute:: Request.method - The HTTP request method to use. This value is used by - :meth:`~Request.get_method` to override the computed HTTP request - method that would otherwise be returned. This attribute is initialized with - the value of the *method* argument passed to the constructor. + The HTTP request method to use. By default its value is :const:`None`, + which means that :meth:`~Request.get_method` will do its normal computation + of the method to be used. Its value can be set (thus overriding the default + computation in :meth:`~Request.get_method`) either by providing a default + value by setting it at the class level in a :class:`Request` subclass, or by + passing a value in to the :class:`Request` constructor via the *method* + argument. .. versionadded:: 3.3 + .. versionchanged:: 3.4 + A default value can now be set in subclasses; previously it could only + be set via the constructor argument. + .. method:: Request.get_method() @@ -461,65 +488,29 @@ request. unredirected). -.. method:: Request.get_full_url() - - Return the URL given in the constructor. - - -.. method:: Request.set_proxy(host, type) - - Prepare the request by connecting to a proxy server. The *host* and *type* will - replace those of the instance, and the instance's selector will be the original - URL given in the constructor. - - -.. method:: Request.add_data(data) - - Set the :class:`Request` data to *data*. This is ignored by all handlers except - HTTP handlers --- and there it should be a byte string, and will change the - request to be ``POST`` rather than ``GET``. Deprecated in 3.3, use - :attr:`Request.data`. - - .. deprecated-removed:: 3.3 3.4 - - -.. method:: Request.has_data() - - Return whether the instance has a non-\ ``None`` data. Deprecated in 3.3, - use :attr:`Request.data`. - - .. deprecated-removed:: 3.3 3.4 - +.. method:: Request.remove_header(header) -.. method:: Request.get_data() + Remove named header from the request instance (both from regular and + unredirected headers). - Return the instance's data. Deprecated in 3.3, use :attr:`Request.data`. + .. versionadded:: 3.4 - .. deprecated-removed:: 3.3 3.4 - - -.. method:: Request.get_type() - - Return the type of the URL --- also known as the scheme. Deprecated in 3.3, - use :attr:`Request.type`. - - .. deprecated-removed:: 3.3 3.4 +.. method:: Request.get_full_url() -.. method:: Request.get_host() + Return the URL given in the constructor. - Return the host to which a connection will be made. Deprecated in 3.3, use - :attr:`Request.host`. + .. versionchanged:: 3.4 - .. deprecated-removed:: 3.3 3.4 + Returns :attr:`Request.full_url` -.. method:: Request.get_selector() +.. method:: Request.set_proxy(host, type) - Return the selector --- the part of the URL that is sent to the server. - Deprecated in 3.3, use :attr:`Request.selector`. + Prepare the request by connecting to a proxy server. The *host* and *type* will + replace those of the instance, and the instance's selector will be the original + URL given in the constructor. - .. deprecated-removed:: 3.3 3.4 .. method:: Request.get_header(header_name, default=None) @@ -531,25 +522,10 @@ request. Return a list of tuples (header_name, header_value) of the Request headers. - -.. method:: Request.set_proxy(host, type) - -.. method:: Request.get_origin_req_host() - - Return the request-host of the origin transaction, as defined by - :rfc:`2965`. See the documentation for the :class:`Request` constructor. - Deprecated in 3.3, use :attr:`Request.origin_req_host`. - - .. deprecated-removed:: 3.3 3.4 - - -.. method:: Request.is_unverifiable() - - Return whether the request is unverifiable, as defined by RFC 2965. See the - documentation for the :class:`Request` constructor. Deprecated in 3.3, use - :attr:`Request.unverifiable`. - - .. deprecated-removed:: 3.3 3.4 +.. versionchanged:: 3.4 + The request methods add_data, has_data, get_data, get_type, get_host, + get_selector, get_origin_req_host and is_unverifiable that were deprecated + since 3.3 have been removed. .. _opener-director-objects: @@ -983,6 +959,21 @@ FileHandler Objects hostname is given, an :exc:`~urllib.error.URLError` is raised. +.. _data-handler-objects: + +DataHandler Objects +------------------- + +.. method:: DataHandler.data_open(req) + + Read a data URL. This kind of URL contains the content encoded in the URL + itself. The data URL syntax is specified in :rfc:`2397`. This implementation + ignores white spaces in base64 encoded data URLs so the URL may be wrapped + in whatever source file it comes from. But even though some browsers don't + mind about a missing padding at the end of a base64 encoded data URL, this + implementation will raise an :exc:`ValueError` in that case. + + .. _ftp-handler-objects: FTPHandler Objects @@ -1398,7 +1389,9 @@ some point in the future. pair: FTP; protocol * Currently, only the following protocols are supported: HTTP (versions 0.9 and - 1.0), FTP, and local files. + 1.0), FTP, local files, and data URLs. + + .. versionchanged:: 3.4 Added support for data URLs. * The caching feature of :func:`urlretrieve` has been disabled until someone finds the time to hack proper processing of Expiration time headers. diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index be0cad4..e9ede8b 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -76,6 +76,8 @@ Creating virtual environments without there needing to be any reference to its venv in ``PATH``. +.. _venv-api: + API --- @@ -85,7 +87,8 @@ The high-level method described above makes use of a simple API which provides mechanisms for third-party virtual environment creators to customize environment creation according to their needs, the :class:`EnvBuilder` class. -.. class:: EnvBuilder(system_site_packages=False, clear=False, symlinks=False, upgrade=False) +.. class:: EnvBuilder(system_site_packages=False, clear=False, \ + symlinks=False, upgrade=False, with_pip=False) The :class:`EnvBuilder` class accepts the following keyword arguments on instantiation: @@ -93,8 +96,8 @@ creation according to their needs, the :class:`EnvBuilder` class. * ``system_site_packages`` -- a Boolean value indicating that the system Python site-packages should be available to the environment (defaults to ``False``). - * ``clear`` -- a Boolean value which, if true, will delete any existing target - directory instead of raising an exception (defaults to ``False``). + * ``clear`` -- a Boolean value which, if true, will delete the contents of + any existing target directory, before creating the environment. * ``symlinks`` -- a Boolean value indicating whether to attempt to symlink the Python binary (and any necessary DLLs or other binaries, @@ -105,6 +108,12 @@ creation according to their needs, the :class:`EnvBuilder` class. environment with the running Python - for use when that Python has been upgraded in-place (defaults to ``False``). + * ``with_pip`` -- a Boolean value which, if true, ensures pip is + installed in the virtual environment. This uses :mod:`ensurepip` with + the ``--default-pip`` option. + + .. versionchanged:: 3.4 + Added the ``with_pip`` parameter Creators of third-party virtual environment tools will be free to use the @@ -188,6 +197,9 @@ creation according to their needs, the :class:`EnvBuilder` class. * ``__VENV_NAME__`` is replaced with the environment name (final path segment of environment directory). + * ``__VENV_PROMPT__`` is replaced with the prompt (the environment + name surrounded by parentheses and with a following space) + * ``__VENV_BIN_NAME__`` is replaced with the name of the bin directory (either ``bin`` or ``Scripts``). @@ -199,11 +211,15 @@ creation according to their needs, the :class:`EnvBuilder` class. There is also a module-level convenience function: -.. function:: create(env_dir, system_site_packages=False, clear=False, symlinks=False) +.. function:: create(env_dir, system_site_packages=False, clear=False, \ + symlinks=False, with_pip=False) Create an :class:`EnvBuilder` with the given keyword arguments, and call its :meth:`~EnvBuilder.create` method with the *env_dir* argument. + .. versionchanged:: 3.4 + Added the ``with_pip`` parameter + An example of extending ``EnvBuilder`` -------------------------------------- diff --git a/Doc/library/wave.rst b/Doc/library/wave.rst index afafb45..ab64978 100644 --- a/Doc/library/wave.rst +++ b/Doc/library/wave.rst @@ -19,26 +19,32 @@ The :mod:`wave` module defines the following function and exception: .. function:: open(file, mode=None) If *file* is a string, open the file by that name, otherwise treat it as a - seekable file-like object. *mode* can be any of + file-like object. *mode* can be: - ``'r'``, ``'rb'`` + ``'rb'`` Read only mode. - ``'w'``, ``'wb'`` + ``'wb'`` Write only mode. Note that it does not allow read/write WAV files. - A *mode* of ``'r'`` or ``'rb'`` returns a :class:`Wave_read` object, while a - *mode* of ``'w'`` or ``'wb'`` returns a :class:`Wave_write` object. If - *mode* is omitted and a file-like object is passed as *file*, ``file.mode`` - is used as the default value for *mode* (the ``'b'`` flag is still added if - necessary). + A *mode* of ``'rb'`` returns a :class:`Wave_read` object, while a *mode* of + ``'wb'`` returns a :class:`Wave_write` object. If *mode* is omitted and a + file-like object is passed as *file*, ``file.mode`` is used as the default + value for *mode*. If you pass in a file-like object, the wave object will not close it when its :meth:`close` method is called; it is the caller's responsibility to close the file object. + The :func:`.open` function may be used in a :keyword:`with` statement. When + the :keyword:`with` block completes, the :meth:`Wave_read.close() + <wave.Wave_read.close>` or :meth:`Wave_write.close() + <wave.Wave_write.close()>` method is called. + + .. versionchanged:: 3.4 + Added support for unseekable files. .. function:: openfp(file, mode) @@ -98,8 +104,9 @@ Wave_read objects, as returned by :func:`.open`, have the following methods: .. method:: Wave_read.getparams() - Returns a tuple ``(nchannels, sampwidth, framerate, nframes, comptype, - compname)``, equivalent to output of the :meth:`get\*` methods. + Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth, + framerate, nframes, comptype, compname)``, equivalent to output of the + :meth:`get\*` methods. .. method:: Wave_read.readframes(n) @@ -143,13 +150,30 @@ them, and is otherwise implementation dependent. Wave_write Objects ------------------ +For seekable output streams, the ``wave`` header will automatically be updated +to reflect the number of frames actually written. For unseekable streams, the +*nframes* value must be accurate when the first frame data is written. An +accurate *nframes* value can be achieved either by calling +:meth:`~Wave_write.setnframes` or :meth:`~Wave_write.setparams` with the number +of frames that will be written before :meth:`~Wave_write.close` is called and +then using :meth:`~Wave_write.writeframesraw` to write the frame data, or by +calling :meth:`~Wave_write.writeframes` with all of the frame data to be +written. In the latter case :meth:`~Wave_write.writeframes` will calculate +the number of frames in the data and set *nframes* accordingly before writing +the frame data. + Wave_write objects, as returned by :func:`.open`, have the following methods: +.. versionchanged:: 3.4 + Added support for unseekable files. + .. method:: Wave_write.close() Make sure *nframes* is correct, and close the file if it was opened by - :mod:`wave`. This method is called upon object collection. + :mod:`wave`. This method is called upon object collection. It will raise + an exception if the output stream is not seekable and *nframes* does not + match the number of frames actually written. .. method:: Wave_write.setnchannels(n) @@ -173,8 +197,9 @@ Wave_write objects, as returned by :func:`.open`, have the following methods: .. method:: Wave_write.setnframes(n) - Set the number of frames to *n*. This will be changed later if more frames are - written. + Set the number of frames to *n*. This will be changed later if the number + of frames actually written is different (this update attempt will + raise an error if the output stream is not seekable). .. method:: Wave_write.setcomptype(type, name) @@ -200,10 +225,19 @@ Wave_write objects, as returned by :func:`.open`, have the following methods: Write audio frames, without correcting *nframes*. + .. versionchanged:: 3.4 + Any :term:`bytes-like object` is now accepted. + .. method:: Wave_write.writeframes(data) - Write audio frames and make sure *nframes* is correct. + Write audio frames and make sure *nframes* is correct. It will raise an + error if the output stream is not seekable and the total number of frames + that have been written after *data* has been written does not match the + previously set value for *nframes*. + + .. versionchanged:: 3.4 + Any :term:`bytes-like object` is now accepted. Note that it is invalid to set any parameters after calling :meth:`writeframes` diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 224f442..9ca60a9 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -51,10 +51,16 @@ garbage collection. :class:`WeakSet` implements the :class:`set` interface, but keeps weak references to its elements, just like a :class:`WeakKeyDictionary` does. -Most programs should find that using one of these weak container types is all -they need -- it's not usually necessary to create your own weak references -directly. The low-level machinery used by the weak dictionary implementations -is exposed by the :mod:`weakref` module for the benefit of advanced uses. +:class:`finalize` provides a straight forward way to register a +cleanup function to be called when an object is garbage collected. +This is simpler to use than setting up a callback function on a raw +weak reference, since the module automatically ensures that the finalizer +remains alive until the object is collected. + +Most programs should find that using one of these weak container types +or :class:`finalize` is all they need -- it's not usually necessary to +create your own weak references directly. The low-level machinery is +exposed by the :mod:`weakref` module for the benefit of advanced uses. Not all objects can be weakly referenced; those objects which can include class instances, functions written in Python (but not in C), instance methods, sets, @@ -111,6 +117,15 @@ Extension types can easily be made to support weak references; see This is a subclassable type rather than a factory function. + .. attribute:: __callback__ + + This read-only attribute returns the callback currently associated to the + weakref. If there is no callback or if the referent of the weakref is + no longer alive then this attribute will have value ``None``. + + .. versionchanged:: 3.4 + Added the :attr:`__callback__` attribute. + .. function:: proxy(object[, callback]) @@ -192,6 +207,98 @@ These method have the same issues as the and :meth:`keyrefs` method of discarded when no strong reference to it exists any more. +.. class:: WeakMethod(method) + + A custom :class:`ref` subclass which simulates a weak reference to a bound + method (i.e., a method defined on a class and looked up on an instance). + Since a bound method is ephemeral, a standard weak reference cannot keep + hold of it. :class:`WeakMethod` has special code to recreate the bound + method until either the object or the original function dies:: + + >>> class C: + ... def method(self): + ... print("method called!") + ... + >>> c = C() + >>> r = weakref.ref(c.method) + >>> r() + >>> r = weakref.WeakMethod(c.method) + >>> r() + <bound method C.method of <__main__.C object at 0x7fc859830220>> + >>> r()() + method called! + >>> del c + >>> gc.collect() + 0 + >>> r() + >>> + + .. versionadded:: 3.4 + +.. class:: finalize(obj, func, *args, **kwargs) + + Return a callable finalizer object which will be called when *obj* + is garbage collected. Unlike an ordinary weak reference, a finalizer + will always survive until the reference object is collected, greatly + simplifying lifecycle management. + + A finalizer is considered *alive* until it is called (either explicitly + or at garbage collection), and after that it is *dead*. Calling a live + finalizer returns the result of evaluating ``func(*arg, **kwargs)``, + whereas calling a dead finalizer returns :const:`None`. + + Exceptions raised by finalizer callbacks during garbage collection + will be shown on the standard error output, but cannot be + propagated. They are handled in the same way as exceptions raised + from an object's :meth:`__del__` method or a weak reference's + callback. + + When the program exits, each remaining live finalizer is called + unless its :attr:`atexit` attribute has been set to false. They + are called in reverse order of creation. + + A finalizer will never invoke its callback during the later part of + the interpreter shutdown when module globals are liable to have + been replaced by :const:`None`. + + .. method:: __call__() + + If *self* is alive then mark it as dead and return the result of + calling ``func(*args, **kwargs)``. If *self* is dead then return + :const:`None`. + + .. method:: detach() + + If *self* is alive then mark it as dead and return the tuple + ``(obj, func, args, kwargs)``. If *self* is dead then return + :const:`None`. + + .. method:: peek() + + If *self* is alive then return the tuple ``(obj, func, args, + kwargs)``. If *self* is dead then return :const:`None`. + + .. attribute:: alive + + Property which is true if the finalizer is alive, false otherwise. + + .. attribute:: atexit + + A writable boolean property which by default is true. When the + program exits, it calls all remaining live finalizers for which + :attr:`.atexit` is true. They are called in reverse order of + creation. + + .. note:: + + It is important to ensure that *func*, *args* and *kwargs* do + not own any references to *obj*, either directly or indirectly, + since otherwise *obj* will never be garbage collected. In + particular, *func* should not be a bound method of *obj*. + + .. versionadded:: 3.4 + + .. data:: ReferenceType The type object for weak references objects. @@ -232,8 +339,9 @@ These method have the same issues as the and :meth:`keyrefs` method of Weak Reference Objects ---------------------- -Weak reference objects have no attributes or methods, but do allow the referent -to be obtained, if it still exists, by calling it: +Weak reference objects have no methods and no attributes besides +:attr:`ref.__callback__`. A weak reference object allows the referent to be +obtained, if it still exists, by calling it: >>> import weakref >>> class Object: @@ -326,3 +434,140 @@ objects can still be retrieved by ID if they do. def id2obj(oid): return _id2obj_dict[oid] + +.. _finalize-examples: + +Finalizer Objects +----------------- + +The main benefit of using :class:`finalize` is that it makes it simple +to register a callback without needing to preserve the returned finalizer +object. For instance + + >>> import weakref + >>> class Object: + ... pass + ... + >>> kenny = Object() + >>> weakref.finalize(kenny, print, "You killed Kenny!") #doctest:+ELLIPSIS + <finalize object at ...; for 'Object' at ...> + >>> del kenny + You killed Kenny! + +The finalizer can be called directly as well. However the finalizer +will invoke the callback at most once. + + >>> def callback(x, y, z): + ... print("CALLBACK") + ... return x + y + z + ... + >>> obj = Object() + >>> f = weakref.finalize(obj, callback, 1, 2, z=3) + >>> assert f.alive + >>> assert f() == 6 + CALLBACK + >>> assert not f.alive + >>> f() # callback not called because finalizer dead + >>> del obj # callback not called because finalizer dead + +You can unregister a finalizer using its :meth:`~finalize.detach` +method. This kills the finalizer and returns the arguments passed to +the constructor when it was created. + + >>> obj = Object() + >>> f = weakref.finalize(obj, callback, 1, 2, z=3) + >>> f.detach() #doctest:+ELLIPSIS + (<__main__.Object object ...>, <function callback ...>, (1, 2), {'z': 3}) + >>> newobj, func, args, kwargs = _ + >>> assert not f.alive + >>> assert newobj is obj + >>> assert func(*args, **kwargs) == 6 + CALLBACK + +Unless you set the :attr:`~finalize.atexit` attribute to +:const:`False`, a finalizer will be called when the program exits if it +is still alive. For instance + + >>> obj = Object() + >>> weakref.finalize(obj, print, "obj dead or exiting") #doctest:+ELLIPSIS + <finalize object at ...; for 'Object' at ...> + >>> exit() #doctest:+SKIP + obj dead or exiting + + +Comparing finalizers with :meth:`__del__` methods +------------------------------------------------- + +Suppose we want to create a class whose instances represent temporary +directories. The directories should be deleted with their contents +when the first of the following events occurs: + +* the object is garbage collected, +* the object's :meth:`remove` method is called, or +* the program exits. + +We might try to implement the class using a :meth:`__del__` method as +follows:: + + class TempDir: + def __init__(self): + self.name = tempfile.mkdtemp() + + def remove(self): + if self.name is not None: + shutil.rmtree(self.name) + self.name = None + + @property + def removed(self): + return self.name is None + + def __del__(self): + self.remove() + +Starting with Python 3.4, :meth:`__del__` methods no longer prevent +reference cycles from being garbage collected, and module globals are +no longer forced to :const:`None` during interpreter shutdown. So this +code should work without any issues on CPython. + +However, handling of :meth:`__del__` methods is notoriously implementation +specific, since it depends on internal details of the interpreter's garbage +collector implementation. + +A more robust alternative can be to define a finalizer which only references +the specific functions and objects that it needs, rather than having access +to the full state of the object:: + + class TempDir: + def __init__(self): + self.name = tempfile.mkdtemp() + self._finalizer = weakref.finalize(self, shutil.rmtree, self.name) + + def remove(self): + self._finalizer() + + @property + def removed(self): + return not self._finalizer.alive + +Defined like this, our finalizer only receives a reference to the details +it needs to clean up the directory appropriately. If the object never gets +garbage collected the finalizer will still be called at exit. + +The other advantage of weakref based finalizers is that they can be used to +register finalizers for classes where the definition is controlled by a +third party, such as running code when a module is unloaded:: + + import weakref, sys + def unloading_module(): + # implicit reference to the module globals from the function body + weakref.finalize(sys.modules[__name__], unloading_module) + + +.. note:: + + If you create a finalizer object in a daemonic thread just as the + the program exits then there is the possibility that the finalizer + does not get called at exit. However, in a daemonic thread + :func:`atexit.register`, ``try: ... finally: ...`` and ``with: ...`` + do not guarantee that cleanup occurs either. diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index 9c2b3ab..ef63769 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -19,12 +19,12 @@ will be used if graphical browsers are not available or an X11 display isn't available. If text-mode browsers are used, the calling process will block until the user exits the browser. -If the environment variable :envvar:`BROWSER` exists, it is interpreted to -override the platform default list of browsers, as a :data:`os.pathsep`-separated -list of browsers to try in order. When the value of a list part contains the -string ``%s``, then it is interpreted as a literal browser command line to be -used with the argument URL substituted for ``%s``; if the part does not contain -``%s``, it is simply interpreted as the name of the browser to launch. [1]_ +If the environment variable :envvar:`BROWSER` exists, it is interpreted as the +:data:`os.pathsep`-separated list of browsers to try ahead of the the platform +defaults. When the value of a list part contains the string ``%s``, then it is +interpreted as a literal browser command line to be used with the argument URL +substituted for ``%s``; if the part does not contain ``%s``, it is simply +interpreted as the name of the browser to launch. [1]_ For non-Unix platforms, or when a remote browser is available on Unix, the controlling process will not wait for the user to finish with the browser, but diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 3f2dcbb..fc0b79e 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -105,6 +105,59 @@ Children are nested, and we can access specific child nodes by index:: >>> root[0][1].text '2008' + +.. note:: + + Not all elements of the XML input will end up as elements of the + parsed tree. Currently, this module skips over any XML comments, + processing instructions, and document type declarations in the + input. Nevertheless, trees built using this module's API rather + than parsing from XML text can have comments and processing + instructions in them; they will be included when generating XML + output. A document type declaration may be accessed by passing a + custom :class:`TreeBuilder` instance to the :class:`XMLParser` + constructor. + + +.. _elementtree-pull-parsing: + +Pull API for non-blocking parsing +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Most parsing functions provided by this module require the whole document +to be read at once before returning any result. It is possible to use an +:class:`XMLParser` and feed data into it incrementally, but it is a push API that +calls methods on a callback target, which is too low-level and inconvenient for +most needs. Sometimes what the user really wants is to be able to parse XML +incrementally, without blocking operations, while enjoying the convenience of +fully constructed :class:`Element` objects. + +The most powerful tool for doing this is :class:`XMLPullParser`. It does not +require a blocking read to obtain the XML data, and is instead fed with data +incrementally with :meth:`XMLPullParser.feed` calls. To get the parsed XML +elements, call :meth:`XMLPullParser.read_events`. Here is an example:: + + >>> parser = ET.XMLPullParser(['start', 'end']) + >>> parser.feed('<mytag>sometext') + >>> list(parser.read_events()) + [('start', <Element 'mytag' at 0x7fa66db2be58>)] + >>> parser.feed(' more text</mytag>') + >>> for event, elem in parser.read_events(): + ... print(event) + ... print(elem.tag, 'text=', elem.text) + ... + end + +The obvious use case is applications that operate in a non-blocking fashion +where the XML data is being received from a socket or read incrementally from +some storage device. In such cases, blocking reads are unacceptable. + +Because it's so flexible, :class:`XMLPullParser` can be inconvenient to use for +simpler use-cases. If you don't mind your application blocking on reading XML +data but would still like to have incremental parsing capabilities, take a look +at :func:`iterparse`. It can be useful when you're reading a large XML document +and don't want to hold it wholly in memory. + Finding interesting elements ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -285,7 +338,7 @@ Supported XPath syntax +=======================+======================================================+ | ``tag`` | Selects all child elements with the given tag. | | | For example, ``spam`` selects all child elements | -| | named ``spam``, ``spam/egg`` selects all | +| | named ``spam``, and ``spam/egg`` selects all | | | grandchildren named ``egg`` in all children named | | | ``spam``. | +-----------------------+------------------------------------------------------+ @@ -341,6 +394,10 @@ Functions string containing the comment string. Returns an element instance representing a comment. + Note that :class:`XMLParser` skips over comments in the input + instead of creating comment objects for them. An :class:`ElementTree` will + only contain comment nodes if they have been inserted into to + the tree using one of the :class:`Element` methods. .. function:: dump(elem) @@ -379,25 +436,32 @@ Functions Parses an XML section into an element tree incrementally, and reports what's going on to the user. *source* is a filename or :term:`file object` - containing XML data. *events* is a tuple of events to report back. The - supported events are the strings ``"start"``, ``"end"``, ``"start-ns"`` - and ``"end-ns"`` (the "ns" events are used to get detailed namespace + containing XML data. *events* is a sequence of events to report back. The + supported events are the strings ``"start"``, ``"end"``, ``"start-ns"`` and + ``"end-ns"`` (the "ns" events are used to get detailed namespace information). If *events* is omitted, only ``"end"`` events are reported. *parser* is an optional parser instance. If not given, the standard - :class:`XMLParser` parser is used. *parser* can only use the default - :class:`TreeBuilder` as a target. Returns an :term:`iterator` providing - ``(event, elem)`` pairs. + :class:`XMLParser` parser is used. *parser* must be a subclass of + :class:`XMLParser` and can only use the default :class:`TreeBuilder` as a + target. Returns an :term:`iterator` providing ``(event, elem)`` pairs. + + Note that while :func:`iterparse` builds the tree incrementally, it issues + blocking reads on *source* (or the file it names). As such, it's unsuitable + for applications where blocking reads can't be made. For fully non-blocking + parsing, see :class:`XMLPullParser`. .. note:: - :func:`iterparse` only guarantees that it has seen the ">" - character of a starting tag when it emits a "start" event, so the - attributes are defined, but the contents of the text and tail attributes - are undefined at that point. The same applies to the element children; - they may or may not be present. + :func:`iterparse` only guarantees that it has seen the ">" character of a + starting tag when it emits a "start" event, so the attributes are defined, + but the contents of the text and tail attributes are undefined at that + point. The same applies to the element children; they may or may not be + present. If you need a fully populated element, look for "end" events instead. + .. deprecated:: 3.4 + The *parser* argument. .. function:: parse(source, parser=None) @@ -414,6 +478,11 @@ Functions containing the PI target. *text* is a string containing the PI contents, if given. Returns an element instance, representing a processing instruction. + Note that :class:`XMLParser` skips over processing instructions + in the input instead of creating comment objects for them. An + :class:`ElementTree` will only contain processing instruction nodes if + they have been inserted into to the tree using one of the + :class:`Element` methods. .. function:: register_namespace(prefix, uri) @@ -438,29 +507,39 @@ Functions arguments. Returns an element instance. -.. function:: tostring(element, encoding="us-ascii", method="xml") +.. function:: tostring(element, encoding="us-ascii", method="xml", *, \ + short_empty_elements=True) Generates a string representation of an XML element, including all subelements. *element* is an :class:`Element` instance. *encoding* [1]_ is the output encoding (default is US-ASCII). Use ``encoding="unicode"`` to generate a Unicode string (otherwise, a bytestring is generated). *method* is either ``"xml"``, ``"html"`` or ``"text"`` (default is ``"xml"``). + *short_empty_elements* has the same meaning as in :meth:`ElementTree.write`. Returns an (optionally) encoded string containing the XML data. + .. versionadded:: 3.4 + The *short_empty_elements* parameter. -.. function:: tostringlist(element, encoding="us-ascii", method="xml") + +.. function:: tostringlist(element, encoding="us-ascii", method="xml", *, \ + short_empty_elements=True) Generates a string representation of an XML element, including all subelements. *element* is an :class:`Element` instance. *encoding* [1]_ is the output encoding (default is US-ASCII). Use ``encoding="unicode"`` to generate a Unicode string (otherwise, a bytestring is generated). *method* is either ``"xml"``, ``"html"`` or ``"text"`` (default is ``"xml"``). + *short_empty_elements* has the same meaning as in :meth:`ElementTree.write`. Returns a list of (optionally) encoded strings containing the XML data. It does not guarantee any specific sequence, except that ``b"".join(tostringlist(element)) == tostring(element)``. .. versionadded:: 3.2 + .. versionadded:: 3.4 + The *short_empty_elements* parameter. + .. function:: XML(text, parser=None) @@ -753,7 +832,8 @@ ElementTree Objects .. method:: write(file, encoding="us-ascii", xml_declaration=None, \ - default_namespace=None, method="xml") + default_namespace=None, method="xml", *, \ + short_empty_elements=True) Writes the element tree to a file, as XML. *file* is a file name, or a :term:`file object` opened for writing. *encoding* [1]_ is the output @@ -764,6 +844,10 @@ ElementTree Objects *default_namespace* sets the default XML namespace (for "xmlns"). *method* is either ``"xml"``, ``"html"`` or ``"text"`` (default is ``"xml"``). + The keyword-only *short_empty_elements* parameter controls the formatting + of elements that contain no content. If *True* (the default), they are + emitted as a single self-closed tag, otherwise they are emitted as a pair + of start/end tags. The output is either a string (:class:`str`) or binary (:class:`bytes`). This is controlled by the *encoding* argument. If *encoding* is @@ -772,6 +856,9 @@ ElementTree Objects :term:`file object`; make sure you do not try to write a string to a binary stream and vice versa. + .. versionadded:: 3.4 + The *short_empty_elements* parameter. + This is the XML file that is going to be manipulated:: @@ -817,6 +904,7 @@ QName Objects :class:`QName` instances are opaque. + .. _elementtree-treebuilder-objects: TreeBuilder Objects @@ -876,13 +964,18 @@ XMLParser Objects .. class:: XMLParser(html=0, target=None, encoding=None) - :class:`Element` structure builder for XML source data, based on the expat - parser. *html* are predefined HTML entities. This flag is not supported by - the current implementation. *target* is the target object. If omitted, the - builder uses an instance of the standard :class:`TreeBuilder` class. - *encoding* [1]_ is optional. If given, the value overrides the encoding + This class is the low-level building block of the module. It uses + :mod:`xml.parsers.expat` for efficient, event-based parsing of XML. It can + be fed XML data incrementall with the :meth:`feed` method, and parsing events + are translated to a push API - by invoking callbacks on the *target* object. + If *target* is omitted, the standard :class:`TreeBuilder` is used. The + *html* argument was historically used for backwards compatibility and is now + deprecated. If *encoding* [1]_ is given, the value overrides the encoding specified in the XML file. + .. deprecated:: 3.4 + The *html* argument. The remaining arguments should be passed via + keywword to prepare for the removal of the *html* argument. .. method:: close() @@ -902,12 +995,12 @@ XMLParser Objects Feeds data to the parser. *data* is encoded data. - :meth:`XMLParser.feed` calls *target*\'s ``start()`` method - for each opening tag, its ``end()`` method for each closing tag, - and data is processed by method ``data()``. :meth:`XMLParser.close` - calls *target*\'s method ``close()``. - :class:`XMLParser` can be used not only for building a tree structure. - This is an example of counting the maximum depth of an XML file:: + :meth:`XMLParser.feed` calls *target*\'s ``start(tag, attrs_dict)`` method + for each opening tag, its ``end(tag)`` method for each closing tag, and data + is processed by method ``data(data)``. :meth:`XMLParser.close` calls + *target*\'s method ``close()``. :class:`XMLParser` can be used not only for + building a tree structure. This is an example of counting the maximum depth + of an XML file:: >>> from xml.etree.ElementTree import XMLParser >>> class MaxDepth: # The target object of the parser @@ -941,6 +1034,60 @@ XMLParser Objects >>> parser.close() 4 + +.. _elementtree-xmlpullparser-objects: + +XMLPullParser Objects +^^^^^^^^^^^^^^^^^^^^^ + +.. class:: XMLPullParser(events=None) + + A pull parser suitable for non-blocking applications. Its input-side API is + similar to that of :class:`XMLParser`, but instead of pushing calls to a + callback target, :class:`XMLPullParser` collects an internal list of parsing + events and lets the user read from it. *events* is a sequence of events to + report back. The supported events are the strings ``"start"``, ``"end"``, + ``"start-ns"`` and ``"end-ns"`` (the "ns" events are used to get detailed + namespace information). If *events* is omitted, only ``"end"`` events are + reported. + + .. method:: feed(data) + + Feed the given bytes data to the parser. + + .. method:: close() + + Signal the parser that the data stream is terminated. Unlike + :meth:`XMLParser.close`, this method always returns :const:`None`. + Any events not yet retrieved when the parser is closed can still be + read with :meth:`read_events`. + + .. method:: read_events() + + Return an iterator over the events which have been encountered in the + data fed to the + parser. The iterator yields ``(event, elem)`` pairs, where *event* is a + string representing the type of event (e.g. ``"end"``) and *elem* is the + encountered :class:`Element` object. + + Events provided in a previous call to :meth:`read_events` will not be + yielded again. Events are consumed from the internal queue only when + they are retrieved from the iterator, so multiple readers iterating in + parallel over iterators obtained from :meth:`read_events` will have + unpredictable results. + + .. note:: + + :class:`XMLPullParser` only guarantees that it has seen the ">" + character of a starting tag when it emits a "start" event, so the + attributes are defined, but the contents of the text and tail attributes + are undefined at that point. The same applies to the element children; + they may or may not be present. + + If you need a fully populated element, look for "end" events instead. + + .. versionadded:: 3.4 + Exceptions ^^^^^^^^^^ diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 7a6482b..1d23a7c 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -130,7 +130,7 @@ ZipFile Objects --------------- -.. class:: ZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=False) +.. class:: ZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True) Open a ZIP file, where *file* can be either a path to a file (a string) or a file-like object. The *mode* parameter should be ``'r'`` to read an existing @@ -144,15 +144,12 @@ ZipFile Objects and should be :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2` or :const:`ZIP_LZMA`; unrecognized values will cause :exc:`RuntimeError` to be raised. If :const:`ZIP_DEFLATED`, - :const:`ZIP_BZIP2` or :const:`ZIP_LZMA` is specified but the corresponded module + :const:`ZIP_BZIP2` or :const:`ZIP_LZMA` is specified but the corresponding module (:mod:`zlib`, :mod:`bz2` or :mod:`lzma`) is not available, :exc:`RuntimeError` is also raised. The default is :const:`ZIP_STORED`. If *allowZip64* is - ``True`` zipfile will create ZIP files that use the ZIP64 extensions when - the zipfile is larger than 2 GiB. If it is false (the default) :mod:`zipfile` + ``True`` (the default) zipfile will create ZIP files that use the ZIP64 + extensions when the zipfile is larger than 2 GiB. If it is false :mod:`zipfile` will raise an exception when the ZIP file would require ZIP64 extensions. - ZIP64 extensions are disabled by default because the default :program:`zip` - and :program:`unzip` commands on Unix (the InfoZIP utilities) don't support - these extensions. If the file is created with mode ``'a'`` or ``'w'`` and then :meth:`closed <close>` without adding any files to the archive, the appropriate @@ -171,6 +168,9 @@ ZipFile Objects .. versionchanged:: 3.3 Added support for :mod:`bzip2 <bz2>` and :mod:`lzma` compression. + .. versionchanged:: 3.4 + ZIP64 extensions are enabled by default. + .. method:: ZipFile.close() @@ -234,6 +234,9 @@ ZipFile Objects or a :class:`ZipInfo` object. You will appreciate this when trying to read a ZIP file that contains members with duplicate names. + .. deprecated-removed:: 3.4 3.6 + The ``'U'`` or ``'rU'`` mode. Use :class:`io.TextIOWrapper` for reading + compressed text files in :term:`universal newlines` mode. .. method:: ZipFile.extract(member, path=None, pwd=None) @@ -266,10 +269,8 @@ ZipFile Objects Never extract archives from untrusted sources without prior inspection. It is possible that files are created outside of *path*, e.g. members that have absolute filenames starting with ``"/"`` or filenames with two - dots ``".."``. - - .. versionchanged:: 3.3.1 - The zipfile module attempts to prevent that. See :meth:`extract` note. + dots ``".."``. This module attempts to prevent that. + See :meth:`extract` note. .. method:: ZipFile.printdir() @@ -376,15 +377,18 @@ PyZipFile Objects The :class:`PyZipFile` constructor takes the same parameters as the :class:`ZipFile` constructor, and one additional parameter, *optimize*. -.. class:: PyZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=False, \ +.. class:: PyZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True, \ optimize=-1) .. versionadded:: 3.2 The *optimize* parameter. + .. versionchanged:: 3.4 + ZIP64 extensions are enabled by default. + Instances have one method in addition to those of :class:`ZipFile` objects: - .. method:: PyZipFile.writepy(pathname, basename='') + .. method:: PyZipFile.writepy(pathname, basename='', filterfunc=None) Search for files :file:`\*.py` and add the corresponding file to the archive. @@ -397,16 +401,33 @@ The :class:`PyZipFile` constructor takes the same parameters as the ``2``, only files with that optimization level (see :func:`compile`) are added to the archive, compiling if necessary. - If the pathname is a file, the filename must end with :file:`.py`, and + If *pathname* is a file, the filename must end with :file:`.py`, and just the (corresponding :file:`\*.py[co]`) file is added at the top level - (no path information). If the pathname is a file that does not end with + (no path information). If *pathname* is a file that does not end with :file:`.py`, a :exc:`RuntimeError` will be raised. If it is a directory, and the directory is not a package directory, then all the files :file:`\*.py[co]` are added at the top level. If the directory is a package directory, then all :file:`\*.py[co]` are added under the package name as a file path, and if any subdirectories are package directories, - all of these are added recursively. *basename* is intended for internal - use only. The :meth:`writepy` method makes archives with file names like + all of these are added recursively. + + *basename* is intended for internal use only. + + *filterfunc*, if given, must be a function taking a single string + argument. It will be passed each path (including each individual full + file path) before it is added to the archive. If *filterfunc* returns a + false value, the path will not be added, and if it is a directory its + contents will be ignored. For example, if our test files are all either + in ``test`` directories or start with the string ``test_``, we can use a + *filterfunc* to exclude them:: + + >>> zf = PyZipFile('myprog.zip') + >>> def notests(s): + ... fn = os.path.basename(s) + ... return (not (fn == 'test' or fn.startswith('test_'))) + >>> zf.writepy('myprog', filterfunc=notests) + + The :meth:`writepy` method makes archives with file names like this:: string.pyc # Top level name @@ -415,6 +436,9 @@ The :class:`PyZipFile` constructor takes the same parameters as the test/bogus/__init__.pyc # Subpackage directory test/bogus/myfile.pyc # Submodule test.bogus.myfile + .. versionadded:: 3.4 + The *filterfunc* parameter. + .. _zipinfo-objects: diff --git a/Doc/license.rst b/Doc/license.rst index 4440c7b..6190fe0 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -84,9 +84,9 @@ Terms and conditions for accessing or otherwise using Python analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python |release| alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of - copyright, i.e., "Copyright © 2001-2014 Python Software Foundation; All Rights - Reserved" are retained in Python |release| alone or in any derivative version - prepared by Licensee. + copyright, i.e., "Copyright © 2001-2014 Python Software Foundation; All + Rights Reserved" are retained in Python |release| alone or in any derivative + version prepared by Licensee. #. In the event Licensee prepares a derivative work that is based on or incorporates Python |release| or any part thereof, and wants to make the @@ -590,6 +590,35 @@ The :mod:`select` and contains the following notice for the kqueue interface:: SUCH DAMAGE. +SipHash24 +--------- + +The file :file:`Python/pyhash.c` contains Marek Majkowski' implementation of +Dan Bernstein's SipHash24 algorithm. The contains the following note:: + + <MIT License> + Copyright (c) 2013 Marek Majkowski <marek@popcount.org> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + </MIT License> + + Original location: + https://github.com/majek/csiphash/ + + Solution inspired by code from: + Samuel Neves (supercop/crypto_auth/siphash24/little) + djb (supercop/crypto_auth/siphash24/little2) + Jean-Philippe Aumasson (https://131002.net/siphash/siphash24.c) + + strtod and dtoa --------------- @@ -846,6 +875,47 @@ used for the build:: jloup@gzip.org madler@alumni.caltech.edu +cfuhash +------- + +The implementation of the hash table used by the :mod:`tracemalloc` is based +on the cfuhash project:: + + Copyright (c) 2005 Don Owens + All rights reserved. + + This code is released under the BSD license: + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + OF THE POSSIBILITY OF SUCH DAMAGE. + + libmpdec -------- diff --git a/Doc/make.bat b/Doc/make.bat index d6f7074..b26826e 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -1,59 +1,124 @@ -@@echo off +@echo off setlocal -set SVNROOT=http://svn.python.org/projects -if "%PYTHON%" EQU "" set PYTHON=py -2 -if "%HTMLHELP%" EQU "" set HTMLHELP=%ProgramFiles%\HTML Help Workshop\hhc.exe +pushd %~dp0 + +set this=%~n0 + +if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build +if "%PYTHON%" EQU "" set PYTHON=py + +if DEFINED ProgramFiles(x86) set _PRGMFLS=%ProgramFiles(x86)% +if NOT DEFINED ProgramFiles(x86) set _PRGMFLS=%ProgramFiles% +if "%HTMLHELP%" EQU "" set HTMLHELP=%_PRGMFLS%\HTML Help Workshop\hhc.exe + if "%DISTVERSION%" EQU "" for /f "usebackq" %%v in (`%PYTHON% tools/sphinxext/patchlevel.py`) do set DISTVERSION=%%v +if "%BUILDDIR%" EQU "" set BUILDDIR=build + +rem Targets that don't require sphinx-build if "%1" EQU "" goto help -if "%1" EQU "html" goto build -if "%1" EQU "htmlhelp" goto build -if "%1" EQU "latex" goto build -if "%1" EQU "text" goto build -if "%1" EQU "suspicious" goto build -if "%1" EQU "linkcheck" goto build -if "%1" EQU "changes" goto build -if "%1" EQU "checkout" goto checkout -if "%1" EQU "update" goto update +if "%1" EQU "help" goto help +if "%1" EQU "check" goto check +if "%1" EQU "serve" goto serve +if "%1" == "clean" ( + rmdir /q /s %BUILDDIR% + goto end +) + +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + goto end +) + +rem Targets that do require sphinx-build and have their own label +if "%1" EQU "htmlview" goto htmlview + +rem Everything else +goto build :help -set this=%~n0 -echo HELP +echo.usage: %this% BUILDER [filename ...] +echo. +echo.Call %this% with the desired Sphinx builder as the first argument, e.g. +echo.``%this% html`` or ``%this% doctest``. Interesting targets that are +echo.always available include: echo. -echo %this% checkout -echo %this% update -echo %this% html -echo %this% htmlhelp -echo %this% latex -echo %this% text -echo %this% suspicious -echo %this% linkcheck -echo %this% changes +echo. Provided by Sphinx: +echo. html, htmlhelp, latex, text +echo. suspicious, linkcheck, changes, doctest +echo. Provided by this script: +echo. clean, check, serve, htmlview echo. +echo.All arguments past the first one are passed through to sphinx-build as +echo.filenames to build or are ignored. See README.txt in this directory or +echo.the documentation for your version of Sphinx for more exhaustive lists +echo.of available targets and descriptions of each. +echo. +echo.This script assumes that the SPHINXBUILD environment variable contains +echo.a legitimate command for calling sphinx-build, or that sphinx-build is +echo.on your PATH if SPHINXBUILD is not set. Options for sphinx-build can +echo.be passed by setting the SPHINXOPTS environment variable. +goto end + +:build +if NOT "%PAPER%" == "" ( + set SPHINXOPTS=-D latex_paper_size=%PAPER% %SPHINXOPTS% +) +cmd /C %SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%* + +if "%1" EQU "htmlhelp" ( + if not exist "%HTMLHELP%" ( + echo. + echo.The HTML Help Workshop was not found. Set the HTMLHELP variable + echo.to the path to hhc.exe or download and install it from + echo.http://msdn.microsoft.com/en-us/library/ms669985 + rem Set errorlevel to 1 and exit + cmd /C exit /b 1 + goto end + ) + cmd /C "%HTMLHELP%" build\htmlhelp\python%DISTVERSION:.=%.hhp + rem hhc.exe seems to always exit with code 1, reset to 0 for less than 2 + if not errorlevel 2 cmd /C exit /b 0 +) + +echo. +if errorlevel 1 ( + echo.Build failed (exit code %ERRORLEVEL%^), check for error messages + echo.above. Any output will be found in %BUILDDIR%\%1 +) else ( + echo.Build succeeded. All output should be in %BUILDDIR%\%1 +) goto end -:checkout -svn co %SVNROOT%/external/Sphinx-1.2/sphinx tools/sphinx -svn co %SVNROOT%/external/docutils-0.11/docutils tools/docutils -svn co %SVNROOT%/external/Jinja-2.3.1/jinja2 tools/jinja2 -svn co %SVNROOT%/external/Pygments-1.6/pygments tools/pygments +:htmlview +if NOT "%2" EQU "" ( + echo.Can't specify filenames to build with htmlview target, ignoring. +) +cmd /C %this% html + +if EXIST %BUILDDIR%\html\index.html ( + echo.Opening %BUILDDIR%\html\index.html in the default web browser... + start %BUILDDIR%\html\index.html +) + goto end -:update -svn update tools/sphinx -svn update tools/docutils -svn update tools/jinja2 -svn update tools/pygments +:check +cmd /C %PYTHON% tools\rstlint.py -i tools goto end -:build -if not exist build mkdir build -if not exist build\%1 mkdir build\%1 -if not exist build\doctrees mkdir build\doctrees -cmd /C %PYTHON% --version -cmd /C %PYTHON% tools\sphinx-build.py -b%1 -dbuild\doctrees . build\%* -if "%1" EQU "htmlhelp" "%HTMLHELP%" build\htmlhelp\python%DISTVERSION:.=%.hhp +:serve +cmd /C %PYTHON% ..\Tools\scripts\serve.py %BUILDDIR%\html goto end :end +popd diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 8afc69e..94b07f0 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -22,14 +22,14 @@ also syntactically compound statements. single: clause single: suite -Compound statements consist of one or more 'clauses.' A clause consists of a +A compound statement consists of one or more 'clauses.' A clause consists of a header and a 'suite.' The clause headers of a particular compound statement are all at the same indentation level. Each clause header begins with a uniquely identifying keyword and ends with a colon. A suite is a group of statements controlled by a clause. A suite can be one or more semicolon-separated simple statements on the same line as the header, following the header's colon, or it can be one or more indented statements on subsequent lines. Only the latter -form of suite can contain nested compound statements; the following is illegal, +form of a suite can contain nested compound statements; the following is illegal, mostly because it wouldn't be clear to which :keyword:`if` clause a following :keyword:`else` clause would belong:: @@ -156,8 +156,8 @@ The :keyword:`for` statement is used to iterate over the elements of a sequence The expression list is evaluated once; it should yield an iterable object. An iterator is created for the result of the ``expression_list``. The suite is -then executed once for each item provided by the iterator, in the order of -ascending indices. Each item in turn is assigned to the target list using the +then executed once for each item provided by the iterator, in the order returned +by the iterator. Each item in turn is assigned to the target list using the standard rules for assignments (see :ref:`assignment`), and then the suite is executed. When the items are exhausted (which is immediately when the sequence is empty or an iterator raises a :exc:`StopIteration` exception), the suite in @@ -170,17 +170,25 @@ the :keyword:`else` clause, if present, is executed, and the loop terminates. A :keyword:`break` statement executed in the first suite terminates the loop without executing the :keyword:`else` clause's suite. A :keyword:`continue` statement executed in the first suite skips the rest of the suite and continues -with the next item, or with the :keyword:`else` clause if there was no next +with the next item, or with the :keyword:`else` clause if there is no next item. -The suite may assign to the variable(s) in the target list; this does not affect -the next item assigned to it. +The for-loop makes assignments to the variables(s) in the target list. +This overwrites all previous assignments to those variables including +those made in the suite of the for-loop:: + + for i in range(10): + print(i) + i = 5 # this will not affect the for-loop + # because i will be overwritten with the next + # index in the range + .. index:: builtin: range Names in the target list are not deleted when the loop is finished, but if the -sequence is empty, it will not have been assigned to at all by the loop. Hint: +sequence is empty, they will not have been assigned to at all by the loop. Hint: the built-in function :func:`range` returns an iterator of integers suitable to emulate the effect of Pascal's ``for i := a to b do``; e.g., ``list(range(3))`` returns the list ``[0, 1, 2]``. @@ -284,7 +292,7 @@ keeping all locals in that frame alive until the next garbage collection occurs. object: traceback Before an except clause's suite is executed, details about the exception are -stored in the :mod:`sys` module and can be access via :func:`sys.exc_info`. +stored in the :mod:`sys` module and can be accessed via :func:`sys.exc_info`. :func:`sys.exc_info` returns a 3-tuple consisting of the exception class, the exception instance and a traceback object (see section :ref:`types`) identifying the point in the program where the exception occurred. :func:`sys.exc_info` @@ -313,14 +321,14 @@ exception, the saved exception is set as the context of the new exception. If the :keyword:`finally` clause executes a :keyword:`return` or :keyword:`break` statement, the saved exception is discarded:: - def f(): - try: - 1/0 - finally: - return 42 - - >>> f() - 42 + >>> def f(): + ... try: + ... 1/0 + ... finally: + ... return 42 + ... + >>> f() + 42 The exception information is not available to the program during execution of the :keyword:`finally` clause. @@ -337,6 +345,20 @@ statement, the :keyword:`finally` clause is also executed 'on the way out.' A reason is a problem with the current implementation --- this restriction may be lifted in the future). +The return value of a function is determined by the last :keyword:`return` +statement executed. Since the :keyword:`finally` clause always executes, a +:keyword:`return` statement executed in the :keyword:`finally` clause will +always be the last one executed:: + + >>> def foo(): + ... try: + ... return 'try' + ... finally: + ... return 'finally' + ... + >>> foo() + 'finally' + Additional information on exceptions can be found in section :ref:`exceptions`, and information on using the :keyword:`raise` statement to generate exceptions may be found in section :ref:`raise`. @@ -348,7 +370,9 @@ may be found in section :ref:`raise`. The :keyword:`with` statement ============================= -.. index:: statement: with +.. index:: + statement: with + single: as; with statement The :keyword:`with` statement is used to wrap the execution of a block with methods defined by a context manager (see section :ref:`context-managers`). @@ -445,7 +469,7 @@ A function definition defines a user-defined function object (see section decorator: "@" `dotted_name` ["(" [`parameter_list` [","]] ")"] NEWLINE dotted_name: `identifier` ("." `identifier`)* parameter_list: (`defparameter` ",")* - : ( "*" [`parameter`] ("," `defparameter`)* ["," "**" `parameter`] + : | "*" [`parameter`] ("," `defparameter`)* ["," "**" `parameter`] : | "**" `parameter` : | `defparameter` [","] ) parameter: `identifier` [":" `expression`] @@ -493,14 +517,15 @@ case the parameter's default value is substituted. If a parameter has a default value, all following parameters up until the "``*``" must also have a default value --- this is a syntactic restriction that is not expressed by the grammar. -**Default parameter values are evaluated when the function definition is -executed.** This means that the expression is evaluated once, when the function -is defined, and that the same "pre-computed" value is used for each call. This -is especially important to understand when a default parameter is a mutable -object, such as a list or a dictionary: if the function modifies the object -(e.g. by appending an item to a list), the default value is in effect modified. -This is generally not what was intended. A way around this is to use ``None`` -as the default, and explicitly test for it in the body of the function, e.g.:: +**Default parameter values are evaluated from left to right when the function +definition is executed.** This means that the expression is evaluated once, when +the function is defined, and that the same "pre-computed" value is used for each +call. This is especially important to understand when a default parameter is a +mutable object, such as a list or a dictionary: if the function modifies the +object (e.g. by appending an item to a list), the default value is in effect +modified. This is generally not what was intended. A way around this is to use +``None`` as the default, and explicitly test for it in the body of the function, +e.g.:: def whats_on_the_telly(penguin=None): if penguin is None: diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 160af30..d401ee0 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -77,7 +77,7 @@ are still reachable. module for information on controlling the collection of cyclic garbage. Other implementations act differently and CPython may change. Do not depend on immediate finalization of objects when they become - unreachable (ex: always close files). + unreachable (so you should always close files explicitly). Note that the use of the implementation's tracing or debugging facilities may keep objects alive that would normally be collectable. Also note that catching @@ -285,16 +285,17 @@ Sequences single: integer single: Unicode - A string is a sequence of values that represent Unicode codepoints. - All the codepoints in range ``U+0000 - U+10FFFF`` can be represented - in a string. Python doesn't have a :c:type:`chr` type, and - every character in the string is represented as a string object - with length ``1``. The built-in function :func:`ord` converts a - character to its codepoint (as an integer); :func:`chr` converts - an integer in range ``0 - 10FFFF`` to the corresponding character. + A string is a sequence of values that represent Unicode code points. + All the code points in the range ``U+0000 - U+10FFFF`` can be + represented in a string. Python doesn't have a :c:type:`char` type; + instead, every code point in the string is represented as a string + object with length ``1``. The built-in function :func:`ord` + converts a code point from its string form to an integer in the + range ``0 - 10FFFF``; :func:`chr` converts an integer in the range + ``0 - 10FFFF`` to the corresponding length ``1`` string object. :meth:`str.encode` can be used to convert a :class:`str` to - :class:`bytes` using the given encoding, and :meth:`bytes.decode` can - be used to achieve the opposite. + :class:`bytes` using the given text encoding, and + :meth:`bytes.decode` can be used to achieve the opposite. Tuples .. index:: @@ -323,8 +324,6 @@ Sequences object: mutable sequence object: mutable pair: assignment; statement - single: delete - statement: del single: subscription single: slicing @@ -934,6 +933,20 @@ Internal types frame). A debugger can implement a Jump command (aka Set Next Statement) by writing to f_lineno. + Frame objects support one method: + + .. method:: frame.clear() + + This method clears all references to local variables held by the + frame. Also, if the frame belonged to a generator, the generator + is finalized. This helps break reference cycles involving frame + objects (for example when catching an exception and storing its + traceback for later use). + + :exc:`RuntimeError` is raised if the frame is currently executing. + + .. versionadded:: 3.4 + Traceback objects .. index:: object: traceback @@ -1121,12 +1134,10 @@ Basic customization ``sys.last_traceback`` keeps the stack frame alive). The first situation can only be remedied by explicitly breaking the cycles; the latter two situations can be resolved by storing ``None`` in ``sys.last_traceback``. - Circular references which are garbage are detected when the option cycle - detector is enabled (it's on by default), but can only be cleaned up if - there are no Python- level :meth:`__del__` methods involved. Refer to the - documentation for the :mod:`gc` module for more information about how - :meth:`__del__` methods are handled by the cycle detector, particularly - the description of the ``garbage`` value. + Circular references which are garbage are detected and cleaned up when + the cyclic garbage collector is enabled (it's on by default). Refer to the + documentation for the :mod:`gc` module for more information about this + topic. .. warning:: @@ -1214,6 +1225,10 @@ Basic customization The return value must be a string object. + .. versionchanged:: 3.4 + The __format__ method of ``object`` itself raises a :exc:`TypeError` + if passed any non-empty string. + .. _richcmpfuncs: .. method:: object.__lt__(self, other) @@ -1451,6 +1466,14 @@ class' :attr:`__dict__`. Called to delete the attribute on an instance *instance* of the owner class. +The attribute :attr:`__objclass__` is interpreted by the :mod:`inspect` module +as specifying the class where this object was defined (setting this +appropriately can assist in runtime introspection of dynamic class attributes). +For callables, it may indicate that an instance of the given type (or a +subclass) is expected or required as the first positional argument (for example, +CPython sets this attribute for unbound methods that are implemented in C). + + .. _descriptor-invocation: Invoking Descriptors @@ -1631,6 +1654,8 @@ of these candidate metaclasses. If none of the candidate metaclasses meets that criterion, then the class definition will fail with ``TypeError``. +.. _prepare: + Preparing the class namespace ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1698,7 +1723,7 @@ property creation, proxies, frameworks, and automatic resource locking/synchronization. Here is an example of a metaclass that uses an :class:`collections.OrderedDict` -to remember the order that class members were defined:: +to remember the order that class variables are defined:: class OrderedClass(type): @@ -1830,6 +1855,15 @@ through the container; for mappings, :meth:`__iter__` should be the same as considered to be false in a Boolean context. +.. method:: object.__length_hint__(self) + + Called to implement :func:`operator.length_hint`. Should return an estimated + length for the object (which may be greater or less than the actual length). + The length must be an integer ``>=`` 0. This method is purely an + optimization and is never required for correctness. + + .. versionadded:: 3.4 + .. note:: Slicing is done exclusively with the following three methods. A call like :: @@ -2061,9 +2095,17 @@ left undefined. .. method:: object.__index__(self) - Called to implement :func:`operator.index`. Also called whenever Python needs - an integer object (such as in slicing, or in the built-in :func:`bin`, - :func:`hex` and :func:`oct` functions). Must return an integer. + Called to implement :func:`operator.index`, and whenever Python needs to + losslessly convert the numeric object to an integer object (such as in + slicing, or in the built-in :func:`bin`, :func:`hex` and :func:`oct` + functions). Presence of this method indicates that the numeric object is + an integer type. Must return an integer. + + .. note:: + + In order to have a coherent integer type class, when :meth:`__index__` is + defined :meth:`__int__` should also be defined, and both should return + the same value. .. _context-managers: diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index 82e37a2..664d736 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -31,11 +31,11 @@ that name established in the innermost function block containing the use. A :dfn:`block` is a piece of Python program text that is executed as a unit. The following are blocks: a module, a function body, and a class definition. Each command typed interactively is a block. A script file (a file given as -standard input to the interpreter or specified on the interpreter command line -the first argument) is a code block. A script command (a command specified on -the interpreter command line with the '**-c**' option) is a code block. The -string argument passed to the built-in functions :func:`eval` and :func:`exec` -is a code block. +standard input to the interpreter or specified as a command line argument to the +interpreter) is a code block. A script command (a command specified on the +interpreter command line with the '**-c**' option) is a code block. The string +argument passed to the built-in functions :func:`eval` and :func:`exec` is a +code block. .. index:: pair: execution; frame @@ -77,7 +77,7 @@ global.) If a variable is used in a code block but not defined there, it is a single: UnboundLocalError When a name is not found at all, a :exc:`NameError` exception is raised. If the -name refers to a local variable that has not been bound, a +name refers to a local variable that has not been bound, an :exc:`UnboundLocalError` exception is raised. :exc:`UnboundLocalError` is a subclass of :exc:`NameError`. diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 06baba0..ddc3286 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -29,7 +29,7 @@ Arithmetic conversions When a description of an arithmetic operator below uses the phrase "the numeric arguments are converted to a common type," this means that the operator -implementation for built-in types works that way: +implementation for built-in types works as follows: * If either argument is a complex number, the other is converted to complex; @@ -38,8 +38,9 @@ implementation for built-in types works that way: * otherwise, both must be integers and no conversion is necessary. -Some additional rules apply for certain operators (e.g., a string left argument -to the '%' operator). Extensions must define their own conversion behavior. +Some additional rules apply for certain operators (e.g., a string as a left +argument to the '%' operator). Extensions must define their own conversion +behavior. .. _atoms: @@ -183,7 +184,7 @@ nesting from left to right, and evaluating the expression to produce an element each time the innermost block is reached. Note that the comprehension is executed in a separate scope, so names assigned -to in the target list don't "leak" in the enclosing scope. +to in the target list don't "leak" into the enclosing scope. .. _lists: @@ -293,7 +294,7 @@ for comprehensions, except that it is enclosed in parentheses instead of brackets or curly braces. Variables used in the generator expression are evaluated lazily when the -:meth:`~generator.__next__` method is called for generator object (in the same +:meth:`~generator.__next__` method is called for the generator object (in the same fashion as normal generators). However, the leftmost :keyword:`for` clause is immediately evaluated, so that an error produced by it can be seen before any other possible error in the code that handles the generator expression. @@ -302,7 +303,7 @@ may depend on the previous :keyword:`for` loop. For example: ``(x*y for x in range(10) for y in bar(x))``. The parentheses can be omitted on calls with only one argument. See section -:ref:`calls` for the detail. +:ref:`calls` for details. .. _yieldexpr: @@ -327,12 +328,12 @@ When a generator function is called, it returns an iterator known as a generator. That generator then controls the execution of a generator function. The execution starts when one of the generator's methods is called. At that time, the execution proceeds to the first yield expression, where it is -suspended again, returning the value of :token:`expression_list` to generator's +suspended again, returning the value of :token:`expression_list` to the generator's caller. By suspended, we mean that all local state is retained, including the current bindings of local variables, the instruction pointer, and the internal evaluation stack. When the execution is resumed by calling one of the generator's methods, the function can proceed exactly as if the yield expression -was just another external call. The value of the yield expression after +were just another external call. The value of the yield expression after resuming depends on the method which resumed the execution. If :meth:`~generator.__next__` is used (typically via either a :keyword:`for` or the :func:`next` builtin) then the result is :const:`None`. Otherwise, if @@ -344,10 +345,10 @@ that method. All of this makes generator functions quite similar to coroutines; they yield multiple times, they have more than one entry point and their execution can be suspended. The only difference is that a generator function cannot control -where should the execution continue after it yields; the control is always +where the execution should continue after it yields; the control is always transferred to the generator's caller. -yield expressions are allowed in the :keyword:`try` clause of a :keyword:`try` +Yield expressions are allowed in the :keyword:`try` clause of a :keyword:`try` ... :keyword:`finally` construct. If the generator is not resumed before it is finalized (by reaching a zero reference count or by being garbage collected), the generator-iterator's :meth:`~generator.close` method will be called, @@ -430,7 +431,7 @@ is already executing raises a :exc:`ValueError` exception. .. method:: generator.throw(type[, value[, traceback]]) - Raises an exception of type ``type`` at the point where generator was paused, + Raises an exception of type ``type`` at the point where the generator was paused, and returns the next value yielded by the generator function. If the generator exits without yielding another value, a :exc:`StopIteration` exception is raised. If the generator function does not catch the passed-in exception, or @@ -520,11 +521,11 @@ An attribute reference is a primary followed by a period and a name: The primary must evaluate to an object of a type that supports attribute references, which most objects do. This object is then asked to produce the -attribute whose name is the identifier (which can be customized by overriding -the :meth:`__getattr__` method). If this attribute is not available, the -exception :exc:`AttributeError` is raised. Otherwise, the type and value of the -object produced is determined by the object. Multiple evaluations of the same -attribute reference may yield different objects. +attribute whose name is the identifier. This production can be customized by +overriding the :meth:`__getattr__` method. If this attribute is not available, +the exception :exc:`AttributeError` is raised. Otherwise, the type and value of +the object produced is determined by the object. Multiple evaluations of the +same attribute reference may yield different objects. .. _subscriptions: @@ -549,9 +550,9 @@ A subscription selects an item of a sequence (string, tuple or list) or mapping .. productionlist:: subscription: `primary` "[" `expression_list` "]" -The primary must evaluate to an object that supports subscription, e.g. a list -or dictionary. User-defined objects can support subscription by defining a -:meth:`__getitem__` method. +The primary must evaluate to an object that supports subscription (lists or +dictionaries for example). User-defined objects can support subscription by +defining a :meth:`__getitem__` method. For built-in objects, there are two types of objects that support subscription: @@ -660,8 +661,8 @@ series of :term:`arguments <argument>`: keyword_arguments: `keyword_item` ("," `keyword_item`)* keyword_item: `identifier` "=" `expression` -A trailing comma may be present after the positional and keyword arguments but -does not affect the semantics. +An optional trailing comma may be present after the positional and keyword arguments +but does not affect the semantics. .. index:: single: parameter; call semantics @@ -943,9 +944,9 @@ point number using the :func:`abs` function if appropriate. .. index:: single: addition The ``+`` (addition) operator yields the sum of its arguments. The arguments -must either both be numbers or both sequences of the same type. In the former -case, the numbers are converted to a common type and then added together. In -the latter case, the sequences are concatenated. +must either both be numbers or both be sequences of the same type. In the +former case, the numbers are converted to a common type and then added together. +In the latter case, the sequences are concatenated. .. index:: single: subtraction @@ -1106,7 +1107,7 @@ Comparison of objects of the same type depends on the type: another one is made arbitrarily but consistently within one execution of a program. -Comparison of objects of the differing types depends on whether either of the +Comparison of objects of differing types depends on whether either of the types provide explicit support for the comparison. Most numeric types can be compared with one another. When cross-type comparison is not supported, the comparison method returns ``NotImplemented``. @@ -1116,7 +1117,7 @@ comparison method returns ``NotImplemented``. The operators :keyword:`in` and :keyword:`not in` test for membership. ``x in s`` evaluates to true if *x* is a member of *s*, and false otherwise. ``x not in s`` returns the negation of ``x in s``. All built-in sequences and set types -support this as well as dictionary, for which :keyword:`in` tests whether a the +support this as well as dictionary, for which :keyword:`in` tests whether the dictionary has a given key. For container types such as list, tuple, set, frozenset, dict, or collections.deque, the expression ``x in y`` is equivalent to ``any(x is e or x == e for e in y)``. @@ -1202,9 +1203,9 @@ returned; otherwise, *y* is evaluated and the resulting value is returned. they return to ``False`` and ``True``, but rather return the last evaluated argument. This is sometimes useful, e.g., if ``s`` is a string that should be replaced by a default value if it is empty, the expression ``s or 'foo'`` yields -the desired value. Because :keyword:`not` has to invent a value anyway, it does -not bother to return a value of the same type as its argument, so e.g., ``not -'foo'`` yields ``False``, not ``''``.) +the desired value. Because :keyword:`not` has to create a new value, it +returns a boolean value regardless of the type of its argument +(for example, ``not 'foo'`` produces ``False`` rather than ``''``.) Conditional expressions @@ -1222,8 +1223,8 @@ Conditional expressions Conditional expressions (sometimes called a "ternary operator") have the lowest priority of all Python operations. -The expression ``x if C else y`` first evaluates the condition, *C* (*not* *x*); -if *C* is true, *x* is evaluated and its value is returned; otherwise, *y* is +The expression ``x if C else y`` first evaluates the condition, *C* rather than *x*. +If *C* is true, *x* is evaluated and its value is returned; otherwise, *y* is evaluated and its value is returned. See :pep:`308` for more details about conditional expressions. @@ -1244,10 +1245,9 @@ Lambdas lambda_expr: "lambda" [`parameter_list`]: `expression` lambda_expr_nocond: "lambda" [`parameter_list`]: `expression_nocond` -Lambda expressions (sometimes called lambda forms) have the same syntactic position as -expressions. They are a shorthand to create anonymous functions; the expression -``lambda arguments: expression`` yields a function object. The unnamed object -behaves like a function object defined with :: +Lambda expressions (sometimes called lambda forms) are used to create anonymous +functions. The expression ``lambda arguments: expression`` yields a function +object. The unnamed object behaves like a function object defined with :: def <lambda>(arguments): return expression @@ -1310,13 +1310,15 @@ Operator precedence .. index:: pair: operator; precedence -The following table summarizes the operator precedences in Python, from lowest +The following table summarizes the operator precedence in Python, from lowest precedence (least binding) to highest precedence (most binding). Operators in the same box have the same precedence. Unless the syntax is explicitly given, operators are binary. Operators in the same box group left to right (except for -comparisons, including tests, which all have the same precedence and chain from -left to right --- see section :ref:`comparisons` --- and exponentiation, which -groups from right to left). +exponentiation, which groups from right to left). + +Note that comparisons, membership tests, and identity tests, all have the same +precedence and have a left-to-right chaining feature as described in the +:ref:`comparisons` section. +-----------------------------------------------+-------------------------------------+ diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index eb497a1..645802d 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -210,6 +210,7 @@ Finders and loaders .. index:: single: finder single: loader + single: module spec If the named module is not found in :data:`sys.modules`, then Python's import protocol is invoked to find and load the module. This protocol consists of @@ -230,13 +231,17 @@ The import machinery is extensible, so new finders can be added to extend the range and scope of module searching. Finders do not actually load modules. If they can find the named module, they -return a :term:`loader`, which the import machinery then invokes to load the -module and create the corresponding module object. +return a :dfn:`module spec`, an encapsulation of the module's import-related +information, which the import machinery then uses when loading the module. The following sections describe the protocol for finders and loaders in more detail, including how you can create and register new ones to extend the import machinery. +.. versionchanged:: 3.4 + In previous versions of Python, finders returned :term:`loaders <loader>` + directly, whereas now they return module specs which *contain* loaders. + Loaders are still used during import but have fewer responsibilities. Import hooks ------------ @@ -270,41 +275,43 @@ The meta path .. index:: single: sys.meta_path - pair: finder; find_module - pair: finder; find_loader + pair: finder; find_spec When the named module is not found in :data:`sys.modules`, Python next searches :data:`sys.meta_path`, which contains a list of meta path finder objects. These finders are queried in order to see if they know how to handle the named module. Meta path finders must implement a method called -:meth:`find_module()` which takes two arguments, a name and an import path. -The meta path finder can use any strategy it wants to determine whether it can -handle the named module or not. +:meth:`~importlib.abc.MetaPathFinder.find_spec()` which takes three arguments: +a name, an import path, and (optionally) a target module. The meta path +finder can use any strategy it wants to determine whether it can handle +the named module or not. If the meta path finder knows how to handle the named module, it returns a -loader object. If it cannot handle the named module, it returns ``None``. If +spec object. If it cannot handle the named module, it returns ``None``. If :data:`sys.meta_path` processing reaches the end of its list without returning -a loader, then an :exc:`ImportError` is raised. Any other exceptions raised +a spec, then an :exc:`ImportError` is raised. Any other exceptions raised are simply propagated up, aborting the import process. -The :meth:`find_module()` method of meta path finders is called with two -arguments. The first is the fully qualified name of the module being -imported, for example ``foo.bar.baz``. The second argument is the path -entries to use for the module search. For top-level modules, the second -argument is ``None``, but for submodules or subpackages, the second -argument is the value of the parent package's ``__path__`` attribute. If -the appropriate ``__path__`` attribute cannot be accessed, an -:exc:`ImportError` is raised. +The :meth:`~importlib.abc.MetaPathFinder.find_spec()` method of meta path +finders is called with two or three arguments. The first is the fully +qualified name of the module being imported, for example ``foo.bar.baz``. +The second argument is the path entries to use for the module search. For +top-level modules, the second argument is ``None``, but for submodules or +subpackages, the second argument is the value of the parent package's +``__path__`` attribute. If the appropriate ``__path__`` attribute cannot +be accessed, an :exc:`ImportError` is raised. The third argument is an +existing module object that will be the target of loading later. The +import system passes in a target module only during reload. The meta path may be traversed multiple times for a single import request. For example, assuming none of the modules involved has already been cached, importing ``foo.bar.baz`` will first perform a top level import, calling -``mpf.find_module("foo", None)`` on each meta path finder (``mpf``). After +``mpf.find_spec("foo", None, None)`` on each meta path finder (``mpf``). After ``foo`` has been imported, ``foo.bar`` will be imported by traversing the meta path a second time, calling -``mpf.find_module("foo.bar", foo.__path__)``. Once ``foo.bar`` has been +``mpf.find_spec("foo.bar", foo.__path__, None)``. Once ``foo.bar`` has been imported, the final traversal will call -``mpf.find_module("foo.bar.baz", foo.bar.__path__)``. +``mpf.find_spec("foo.bar.baz", foo.bar.__path__, None)``. Some meta path finders only support top level imports. These importers will always return ``None`` when anything other than ``None`` is passed as the @@ -315,129 +322,239 @@ knows how to import built-in modules, one that knows how to import frozen modules, and one that knows how to import modules from an :term:`import path` (i.e. the :term:`path based finder`). +.. versionchanged:: 3.4 + The :meth:`~importlib.abc.MetaPathFinder.find_spec` method of meta path + finders replaced :meth:`~importlib.abc.MetaPathFinder.find_module`, which + is now deprecated. While it will continue to work without change, the + import machinery will try it only if the finder does not implement + ``find_spec()``. -Loaders -======= - -If and when a module loader is found its -:meth:`~importlib.abc.Loader.load_module` method is called, with a single -argument, the fully qualified name of the module being imported. This method -has several responsibilities, and should return the module object it has -loaded [#fnlo]_. If it cannot load the module, it should raise an -:exc:`ImportError`, although any other exception raised during -:meth:`load_module()` will be propagated. -In many cases, the finder and loader can be the same object; in such cases the -:meth:`finder.find_module()` would just return ``self``. +Loading +======= -Loaders must satisfy the following requirements: +If and when a module spec is found, the import machinery will use it (and +the loader it contains) when loading the module. Here is an approximation +of what happens during the loading portion of import:: + + module = None + if spec.loader is not None and hasattr(spec.loader, 'create_module'): + module = spec.loader.create_module(spec) + if module is None: + module = ModuleType(spec.name) + # The import-related module attributes get set here: + _init_module_attrs(spec, module) + + if spec.loader is None: + if spec.submodule_search_locations is not None: + # namespace package + sys.modules[spec.name] = module + else: + # unsupported + raise ImportError + elif not hasattr(spec.loader, 'exec_module'): + module = spec.loader.load_module(spec.name) + # Set __loader__ and __package__ if missing. + else: + sys.modules[spec.name] = module + try: + spec.loader.exec_module(module) + except BaseException: + try: + del sys.modules[spec.name] + except KeyError: + pass + raise + return sys.modules[spec.name] + +Note the following details: * If there is an existing module object with the given name in - :data:`sys.modules`, the loader must use that existing module. (Otherwise, - :func:`imp.reload` will not work correctly.) If the named module does - not exist in :data:`sys.modules`, the loader must create a new module - object and add it to :data:`sys.modules`. + :data:`sys.modules`, import will have already returned it. - Note that the module *must* exist in :data:`sys.modules` before the loader + * The module will exist in :data:`sys.modules` before the loader executes the module code. This is crucial because the module code may (directly or indirectly) import itself; adding it to :data:`sys.modules` beforehand prevents unbounded recursion in the worst case and multiple loading in the best. - If loading fails, the loader must remove any modules it has inserted into - :data:`sys.modules`, but it must remove **only** the failing module, and - only if the loader itself has loaded it explicitly. Any module already in - the :data:`sys.modules` cache, and any module that was successfully loaded - as a side-effect, must remain in the cache. - - * The loader may set the ``__file__`` attribute of the module. If set, this - attribute's value must be a string. The loader may opt to leave - ``__file__`` unset if it has no semantic meaning (e.g. a module loaded from - a database). If ``__file__`` is set, it may also be appropriate to set the - ``__cached__`` attribute which is the path to any compiled version of the - code (e.g. byte-compiled file). The file does not need to exist to set this - attribute; the path can simply point to whether the compiled file would - exist (see :pep:`3147`). - - * The loader may set the ``__name__`` attribute of the module. While not - required, setting this attribute is highly recommended so that the - :meth:`repr()` of the module is more informative. - - * If the module is a package (either regular or namespace), the loader must - set the module object's ``__path__`` attribute. The value must be - iterable, but may be empty if ``__path__`` has no further significance - to the loader. If ``__path__`` is not empty, it must produce strings - when iterated over. More details on the semantics of ``__path__`` are - given :ref:`below <package-path-rules>`. - - * The ``__loader__`` attribute must be set to the loader object that loaded - the module. This is mostly for introspection and reloading, but can be - used for additional loader-specific functionality, for example getting - data associated with a loader. - - * The module's ``__package__`` attribute should be set. Its value must be a - string, but it can be the same value as its ``__name__``. If the attribute - is set to ``None`` or is missing, the import system will fill it in with a - more appropriate value. When the module is a package, its ``__package__`` - value should be set to its ``__name__``. When the module is not a package, - ``__package__`` should be set to the empty string for top-level modules, or - for submodules, to the parent package's name. See :pep:`366` for further - details. + * If loading fails, the failing module -- and only the failing module -- + gets removed from :data:`sys.modules`. Any module already in the + :data:`sys.modules` cache, and any module that was successfully loaded + as a side-effect, must remain in the cache. This contrasts with + reloading where even the failing module is left in :data:`sys.modules`. - This attribute is used instead of ``__name__`` to calculate explicit - relative imports for main modules, as defined in :pep:`366`. + * After the module is created but before execution, the import machinery + sets the import-related module attributes ("_init_module_attrs" in + the pseudo-code example above), as summarized in a + :ref:`later section <import-mod-attrs>`. + + * Module execution is the key moment of loading in which the module's + namespace gets populated. Execution is entirely delegated to the + loader, which gets to decide what gets populated and how. + + * The module created during loading and passed to exec_module() may + not be the one returned at the end of import [#fnlo]_. + +.. versionchanged:: 3.4 + The import system has taken over the boilerplate responsibilities of + loaders. These were previously performed by the + :meth:`importlib.abc.Loader.load_module` method. + +Loaders +------- + +Module loaders provide the critical function of loading: module execution. +The import machinery calls the :meth:`importlib.abc.Loader.exec_module` +method with a single argument, the module object to execute. Any value +returned from :meth:`~importlib.abc.Loader.exec_module` is ignored. + +Loaders must satisfy the following requirements: * If the module is a Python module (as opposed to a built-in module or a dynamically loaded extension), the loader should execute the module's code in the module's global name space (``module.__dict__``). + * If the loader cannot execute the module, it should raise an + :exc:`ImportError`, although any other exception raised during + :meth:`~importlib.abc.Loader.exec_module` will be propagated. -Module reprs ------------- +In many cases, the finder and loader can be the same object; in such cases the +:meth:`~importlib.abc.MetaPathFinder.find_spec` method would just return a +spec with the loader set to ``self``. -By default, all modules have a usable repr, however depending on the -attributes set above, and hooks in the loader, you can more explicitly control -the repr of module objects. - -Loaders may implement a :meth:`module_repr()` method which takes a single -argument, the module object. When ``repr(module)`` is called for a module -with a loader supporting this protocol, whatever is returned from -``module.__loader__.module_repr(module)`` is returned as the module's repr -without further processing. This return value must be a string. - -If the module has no ``__loader__`` attribute, or the loader has no -:meth:`module_repr()` method, then the module object implementation itself -will craft a default repr using whatever information is available. It will -try to use the ``module.__name__``, ``module.__file__``, and -``module.__loader__`` as input into the repr, with defaults for whatever -information is missing. +Module loaders may opt in to creating the module object during loading +by implementing a :meth:`~importlib.abc.Loader.create_module` method. +It takes one argument, the module spec, and returns the new module object +to use during loading. ``create_module()`` does not need to set any attributes +on the module object. If the loader does not define ``create_module()``, the +import machinery will create the new module itself. -Here are the exact rules used: +.. versionadded:: 3.4 + The create_module() method of loaders. - * If the module has a ``__loader__`` and that loader has a - :meth:`module_repr()` method, call it with a single argument, which is the - module object. The value returned is used as the module's repr. +.. versionchanged:: 3.4 + The :meth:`~importlib.abc.Loader.load_module` method was replaced by + :meth:`~importlib.abc.Loader.exec_module` and the import + machinery assumed all the boilerplate responsibilities of loading. - * If an exception occurs in :meth:`module_repr()`, the exception is caught - and discarded, and the calculation of the module's repr continues as if - :meth:`module_repr()` did not exist. + For compatibility with existing loaders, the import machinery will use + the ``load_module()`` method of loaders if it exists and the loader does + not also implement ``exec_module()``. However, ``load_module()`` has been + deprecated and loaders should implement ``exec_module()`` instead. - * If the module has a ``__file__`` attribute, this is used as part of the - module's repr. + The ``load_module()`` method must implement all the boilerplate loading + functionality described above in addition to executing the module. All + the same constraints apply, with some additional clarification: - * If the module has no ``__file__`` but does have a ``__loader__``, then the - loader's repr is used as part of the module's repr. + * If there is an existing module object with the given name in + :data:`sys.modules`, the loader must use that existing module. + (Otherwise, :func:`importlib.reload` will not work correctly.) If the + named module does not exist in :data:`sys.modules`, the loader + must create a new module object and add it to :data:`sys.modules`. - * Otherwise, just use the module's ``__name__`` in the repr. + * The module *must* exist in :data:`sys.modules` before the loader + executes the module code, to prevent unbounded recursion or multiple + loading. + + * If loading fails, the loader must remove any modules it has inserted + into :data:`sys.modules`, but it must remove **only** the failing + module, and only if the loader itself has loaded it explicitly. + +Module spec +----------- + +The import machinery uses a variety of information about each module +during import, especially before loading. Most of the information is +common to all modules. The purpose of a module's spec is to encapsulate +this import-related information on a per-module basis. + +Using a spec during import allows state to be transferred between import +system components, e.g. between the finder that creates the module spec +and the loader that executes it. Most importantly, it allows the +import machinery to perform the boilerplate operations of loading, +whereas without a module spec the loader had that responsibility. + +See :class:`~importlib.machinery.ModuleSpec` for more specifics on what +information a module's spec may hold. + +.. versionadded:: 3.4 + +.. _import-mod-attrs: + +Import-related module attributes +-------------------------------- + +The import machinery fills in these attributes on each module object +during loading, based on the module's spec, before the loader executes +the module. + +.. attribute:: __name__ + + The ``__name__`` attribute must be set to the fully-qualified name of + the module. This name is used to uniquely identify the module in + the import system. + +.. attribute:: __loader__ + + The ``__loader__`` attribute must be set to the loader object that + the import machinery used when loading the module. This is mostly + for introspection, but can be used for additional loader-specific + functionality, for example getting data associated with a loader. + +.. attribute:: __package__ + + The module's ``__package__`` attribute must be set. Its value must + be a string, but it can be the same value as its ``__name__``. When + the module is a package, its ``__package__`` value should be set to + its ``__name__``. When the module is not a package, ``__package__`` + should be set to the empty string for top-level modules, or for + submodules, to the parent package's name. See :pep:`366` for further + details. -This example, from :pep:`420` shows how a loader can craft its own module -repr:: + This attribute is used instead of ``__name__`` to calculate explicit + relative imports for main modules, as defined in :pep:`366`. + +.. attribute:: __spec__ + + The ``__spec__`` attribute must be set to the module spec that was + used when importing the module. This is used primarily for + introspection and during reloading. Setting ``__spec__`` + appropriately applies equally to :ref:`modules initialized during + interpreter startup <programs>`. The one exception is ``__main__``, + where ``__spec__`` is :ref:`set to None in some cases <main_spec>`. + + .. versionadded:: 3.4 + +.. attribute:: __path__ - class NamespaceLoader: - @classmethod - def module_repr(cls, module): - return "<module '{}' (namespace)>".format(module.__name__) + If the module is a package (either regular or namespace), the module + object's ``__path__`` attribute must be set. The value must be + iterable, but may be empty if ``__path__`` has no further significance. + If ``__path__`` is not empty, it must produce strings when iterated + over. More details on the semantics of ``__path__`` are given + :ref:`below <package-path-rules>`. + Non-package modules should not have a ``__path__`` attribute. + +.. attribute:: __file__ +.. attribute:: __cached__ + + ``__file__`` is optional. If set, this attribute's value must be a + string. The import system may opt to leave ``__file__`` unset if it + has no semantic meaning (e.g. a module loaded from a database). + + If ``__file__`` is set, it may also be appropriate to set the + ``__cached__`` attribute which is the path to any compiled version of + the code (e.g. byte-compiled file). The file does not need to exist + to set this attribute; the path can simply point to where the + compiled file would exist (see :pep:`3147`). + + It is also appropriate to set ``__cached__`` when ``__file__`` is not + set. However, that scenario is quite atypical. Ultimately, the + loader is what makes use of ``__file__`` and/or ``__cached__``. So + if a loader can load from a cached module but otherwise does not load + from a file, that atypical scenario may be appropriate. .. _package-path-rules: @@ -462,9 +579,47 @@ A package's ``__init__.py`` file may set or alter the package's ``__path__`` attribute, and this was typically the way namespace packages were implemented prior to :pep:`420`. With the adoption of :pep:`420`, namespace packages no longer need to supply ``__init__.py`` files containing only ``__path__`` -manipulation code; the namespace loader automatically sets ``__path__`` +manipulation code; the import machinery automatically sets ``__path__`` correctly for the namespace package. +Module reprs +------------ + +By default, all modules have a usable repr, however depending on the +attributes set above, and in the module's spec, you can more explicitly +control the repr of module objects. + +If the module has a spec (``__spec__``), the import machinery will try +to generate a repr from it. If that fails or there is no spec, the import +system will craft a default repr using whatever information is available +on the module. It will try to use the ``module.__name__``, +``module.__file__``, and ``module.__loader__`` as input into the repr, +with defaults for whatever information is missing. + +Here are the exact rules used: + + * If the module has a ``__spec__`` attribute, the information in the spec + is used to generate the repr. The "name", "loader", "origin", and + "has_location" attributes are consulted. + + * If the module has a ``__file__`` attribute, this is used as part of the + module's repr. + + * If the module has no ``__file__`` but does have a ``__loader__`` that is not + ``None``, then the loader's repr is used as part of the module's repr. + + * Otherwise, just use the module's ``__name__`` in the repr. + +.. versionchanged:: 3.4 + Use of :meth:`loader.module_repr() <importlib.abc.Loader.module_repr>` + has been deprecated and the module spec is now used by the import + machinery to generate a module repr. + + For backward compatibility with Python 3.3, the module repr will be + generated by calling the loader's + :meth:`~importlib.abc.Loader.module_repr` method, if defined, before + trying either approach described above. However, the method is deprecated. + The Path Based Finder ===================== @@ -473,8 +628,9 @@ The Path Based Finder single: path based finder As mentioned previously, Python comes with several default meta path finders. -One of these, called the :term:`path based finder`, searches an :term:`import -path`, which contains a list of :term:`path entries <path entry>`. Each path +One of these, called the :term:`path based finder` +(:class:`~importlib.machinery.PathFinder`), searches an :term:`import path`, +which contains a list of :term:`path entries <path entry>`. Each path entry names a location to search for modules. The path based finder itself doesn't know how to import anything. Instead, it @@ -523,15 +679,15 @@ Path entry finders single: sys.path_importer_cache single: PYTHONPATH -The :term:`path based finder` is responsible for finding and loading Python -modules and packages whose location is specified with a string :term:`path -entry`. Most path entries name locations in the file system, but they need -not be limited to this. +The :term:`path based finder` is responsible for finding and loading +Python modules and packages whose location is specified with a string +:term:`path entry`. Most path entries name locations in the file system, +but they need not be limited to this. As a meta path finder, the :term:`path based finder` implements the -:meth:`find_module()` protocol previously described, however it exposes -additional hooks that can be used to customize how modules are found and -loaded from the :term:`import path`. +:meth:`~importlib.abc.MetaPathFinder.find_spec` protocol previously +described, however it exposes additional hooks that can be used to +customize how modules are found and loaded from the :term:`import path`. Three variables are used by the :term:`path based finder`, :data:`sys.path`, :data:`sys.path_hooks` and :data:`sys.path_importer_cache`. The ``__path__`` @@ -551,14 +707,16 @@ finder>`. The :term:`path based finder` is a :term:`meta path finder`, so the import machinery begins the :term:`import path` search by calling the path -based finder's :meth:`find_module()` method as described previously. When -the ``path`` argument to :meth:`find_module()` is given, it will be a +based finder's :meth:`~importlib.machinery.PathFinder.find_spec` method as +described previously. When the ``path`` argument to +:meth:`~importlib.machinery.PathFinder.find_spec` is given, it will be a list of string paths to traverse - typically a package's ``__path__`` -attribute for an import within that package. If the ``path`` argument -is ``None``, this indicates a top level import and :data:`sys.path` is used. +attribute for an import within that package. If the ``path`` argument is +``None``, this indicates a top level import and :data:`sys.path` is used. The path based finder iterates over every entry in the search path, and -for each of these, looks for an appropriate :term:`path entry finder` for the +for each of these, looks for an appropriate :term:`path entry finder` +(:class:`~importlib.abc.PathEntryFinder`) for the path entry. Because this can be an expensive operation (e.g. there may be `stat()` call overheads for this search), the path based finder maintains a cache mapping path entries to path entry finders. This cache is maintained @@ -575,59 +733,82 @@ hooks <path entry hook>` in this list is called with a single argument, the path entry to be searched. This callable may either return a :term:`path entry finder` that can handle the path entry, or it may raise :exc:`ImportError`. An :exc:`ImportError` is used by the path based finder to -signal that the hook cannot find a :term:`path entry finder` for that -:term:`path entry`. The exception is ignored and :term:`import path` -iteration continues. The hook should expect either a string or bytes object; -the encoding of bytes objects is up to the hook (e.g. it may be a file system -encoding, UTF-8, or something else), and if the hook cannot decode the -argument, it should raise :exc:`ImportError`. +signal that the hook cannot find a :term:`path entry finder`. +for that :term:`path entry`. The +exception is ignored and :term:`import path` iteration continues. The hook +should expect either a string or bytes object; the encoding of bytes objects +is up to the hook (e.g. it may be a file system encoding, UTF-8, or something +else), and if the hook cannot decode the argument, it should raise +:exc:`ImportError`. If :data:`sys.path_hooks` iteration ends with no :term:`path entry finder` -being returned, then the path based finder's :meth:`find_module()` method -will store ``None`` in :data:`sys.path_importer_cache` (to indicate that -there is no finder for this path entry) and return ``None``, indicating that -this :term:`meta path finder` could not find the module. +being returned, then the path based finder's +:meth:`~importlib.machinery.PathFinder.find_spec` method will store ``None`` +in :data:`sys.path_importer_cache` (to indicate that there is no finder for +this path entry) and return ``None``, indicating that this +:term:`meta path finder` could not find the module. If a :term:`path entry finder` *is* returned by one of the :term:`path entry hook` callables on :data:`sys.path_hooks`, then the following protocol is used -to ask the finder for a module loader, which is then used to load the module. - +to ask the finder for a module spec, which is then used when loading the +module. Path entry finder protocol -------------------------- In order to support imports of modules and initialized packages and also to contribute portions to namespace packages, path entry finders must implement -the :meth:`find_loader()` method. - -:meth:`find_loader()` takes one argument, the fully qualified name of the -module being imported. :meth:`find_loader()` returns a 2-tuple where the -first item is the loader and the second item is a namespace :term:`portion`. -When the first item (i.e. the loader) is ``None``, this means that while the -path entry finder does not have a loader for the named module, it knows that the -path entry contributes to a namespace portion for the named module. This will -almost always be the case where Python is asked to import a namespace package -that has no physical presence on the file system. When a path entry finder -returns ``None`` for the loader, the second item of the 2-tuple return value -must be a sequence, although it can be empty. - -If :meth:`find_loader()` returns a non-``None`` loader value, the portion is -ignored and the loader is returned from the path based finder, terminating -the search through the path entries. - -For backwards compatibility with other implementations of the import -protocol, many path entry finders also support the same, -traditional :meth:`find_module()` method that meta path finders support. -However path entry finder :meth:`find_module()` methods are never called -with a ``path`` argument (they are expected to record the appropriate -path information from the initial call to the path hook). - -The :meth:`find_module()` method on path entry finders is deprecated, -as it does not allow the path entry finder to contribute portions to -namespace packages. Instead path entry finders should implement the -:meth:`find_loader()` method as described above. If it exists on the path -entry finder, the import system will always call :meth:`find_loader()` -in preference to :meth:`find_module()`. +the :meth:`~importlib.abc.PathEntryFinder.find_spec` method. + +:meth:`~importlib.abc.PathEntryFinder.find_spec` takes two argument, the +fully qualified name of the module being imported, and the (optional) target +module. ``find_spec()`` returns a fully populated spec for the module. +This spec will always have "loader" set (with one exception). + +To indicate to the import machinery that the spec represents a namespace +:term:`portion`. the path entry finder sets "loader" on the spec to +``None`` and "submodule_search_locations" to a list containing the +portion. + +.. versionchanged:: 3.4 + :meth:`~importlib.abc.PathEntryFinder.find_spec` replaced + :meth:`~importlib.abc.PathEntryFinder.find_loader` and + :meth:`~importlib.abc.PathEntryFinder.find_module`, both of which + are now deprecated, but will be used if ``find_spec()`` is not defined. + + Older path entry finders may implement one of these two deprecated methods + instead of ``find_spec()``. The methods are still respected for the + sake of backward compatibility. Howevever, if ``find_spec()`` is + implemented on the path entry finder, the legacy methods are ignored. + + :meth:`~importlib.abc.PathEntryFinder.find_loader` takes one argument, the + fully qualified name of the module being imported. ``find_loader()`` + returns a 2-tuple where the first item is the loader and the second item + is a namespace :term:`portion`. When the first item (i.e. the loader) is + ``None``, this means that while the path entry finder does not have a + loader for the named module, it knows that the path entry contributes to + a namespace portion for the named module. This will almost always be the + case where Python is asked to import a namespace package that has no + physical presence on the file system. When a path entry finder returns + ``None`` for the loader, the second item of the 2-tuple return value must + be a sequence, although it can be empty. + + If ``find_loader()`` returns a non-``None`` loader value, the portion is + ignored and the loader is returned from the path based finder, terminating + the search through the path entries. + + For backwards compatibility with other implementations of the import + protocol, many path entry finders also support the same, + traditional ``find_module()`` method that meta path finders support. + However path entry finder ``find_module()`` methods are never called + with a ``path`` argument (they are expected to record the appropriate + path information from the initial call to the path hook). + + The ``find_module()`` method on path entry finders is deprecated, + as it does not allow the path entry finder to contribute portions to + namespace packages. If both ``find_loader()`` and ``find_module()`` + exist on a path entry finder, the import system will always call + ``find_loader()`` in preference to ``find_module()``. Replacing the standard import system @@ -646,9 +827,54 @@ import statements within that module. To selectively prevent import of some modules from a hook early on the meta path (rather than disabling the standard import system entirely), it is sufficient to raise :exc:`ImportError` directly from -:meth:`find_module` instead of returning ``None``. The latter indicates -that the meta path search should continue. while raising an exception -terminates it immediately. +:meth:`~importlib.abc.MetaPathFinder.find_spec` instead of returning +``None``. The latter indicates that the meta path search should continue, +while raising an exception terminates it immediately. + + +Special considerations for __main__ +=================================== + +The :mod:`__main__` module is a special case relative to Python's import +system. As noted :ref:`elsewhere <programs>`, the ``__main__`` module +is directly initialized at interpreter startup, much like :mod:`sys` and +:mod:`builtins`. However, unlike those two, it doesn't strictly +qualify as a built-in module. This is because the manner in which +``__main__`` is initialized depends on the flags and other options with +which the interpreter is invoked. + +.. _main_spec: + +__main__.__spec__ +----------------- + +Depending on how :mod:`__main__` is initialized, ``__main__.__spec__`` +gets set appropriately or to ``None``. + +When Python is started with the :option:`-m` option, ``__spec__`` is set +to the module spec of the corresponding module or package. ``__spec__`` is +also populated when the ``__main__`` module is loaded as part of executing a +directory, zipfile or other :data:`sys.path` entry. + +In :ref:`the remaining cases <using-on-interface-options>` +``__main__.__spec__`` is set to ``None``, as the code used to populate the +:mod:`__main__` does not correspond directly with an importable module: + +- interactive prompt +- -c switch +- running from stdin +- running directly from a source or bytecode file + +Note that ``__main__.__spec__`` is always ``None`` in the last case, +*even if* the file could technically be imported directly as a module +instead. Use the :option:`-m` switch if valid module metadata is desired +in :mod:`__main__`. + +Note also that even when ``__main__`` corresponds with an importable module +and ``__main__.__spec__`` is set accordingly, they're still considered +*distinct* modules. This is due to the fact that blocks guarded by +``if __name__ == "__main__":`` checks only execute when the module is used +to populate the ``__main__`` namespace, and not during normal import. Open issues @@ -663,6 +889,12 @@ related entries in the data model reference page? XXX runpy, pkgutil, et al in the library manual should all get "See Also" links at the top pointing to the new import system section. +XXX Add more explanation regarding the different ways in which +``__main__`` is initialized? + +XXX Add more info on ``__main__`` quirks/pitfalls (i.e. copy from +:pep:`395`). + References ========== @@ -688,6 +920,11 @@ proposed ``__name__`` for semantics :pep:`366` would eventually specify for :pep:`338` defines executing modules as scripts. +:pep:`451` adds the encapsulation of per-module import state in spec +objects. It also off-loads most of the boilerplate responsibilities of +loaders back onto the import machinery. These changes allow the +deprecation of several APIs in the import system and also addition of new +methods to finders and loaders. .. rubric:: Footnotes diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 0ed3d3b..6617c3b 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -76,7 +76,7 @@ are ignored by the syntax; they are not tokens. Encoding declarations --------------------- -.. index:: source character set, encodings +.. index:: source character set, encoding declarations (source file) If a comment in the first or second line of the Python script matches the regular expression ``coding[=:]\s*([-\w.]+)``, this comment is processed as an diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 40bbc39..1748d5a 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -7,7 +7,7 @@ Simple statements .. index:: pair: simple; statement -Simple statements are comprised within a single logical line. Several simple +A simple statement is comprised within a single logical line. Several simple statements may occur on a single line separated by semicolons. The syntax for simple statements is: @@ -70,6 +70,7 @@ Assignment statements ===================== .. index:: + single: =; assignment statement pair: assignment; statement pair: binding; name pair: rebinding; name @@ -90,8 +91,8 @@ attributes or items of mutable objects: : | `slicing` : | "*" `target` -(See section :ref:`primaries` for the syntax definitions for the last three -symbols.) +(See section :ref:`primaries` for the syntax definitions for *attributeref*, +*subscription*, and *slicing*.) An assignment statement evaluates the expression list (remember that this can be a single expression or a comma-separated list, the latter yielding a tuple) and @@ -227,7 +228,7 @@ Assignment of an object to a single target is recursively defined as follows. inclusive. Finally, the sequence object is asked to replace the slice with the items of the assigned sequence. The length of the slice may be different from the length of the assigned sequence, thus changing the length of the - target sequence, if the object allows it. + target sequence, if the target sequence allows it. .. impl-detail:: @@ -235,14 +236,15 @@ Assignment of an object to a single target is recursively defined as follows. as for expressions, and invalid syntax is rejected during the code generation phase, causing less detailed error messages. -WARNING: Although the definition of assignment implies that overlaps between the -left-hand side and the right-hand side are 'safe' (for example ``a, b = b, a`` -swaps two variables), overlaps *within* the collection of assigned-to variables -are not safe! For instance, the following program prints ``[0, 2]``:: +Although the definition of assignment implies that overlaps between the +left-hand side and the right-hand side are 'simultanenous' (for example ``a, b = +b, a`` swaps two variables), overlaps *within* the collection of assigned-to +variables occur left-to-right, sometimes resulting in confusion. For instance, +the following program prints ``[0, 2]``:: x = [0, 1] i = 0 - i, x[i] = 1, 2 + i, x[i] = 1, 2 # i is updated, then x[i] is updated print(x) @@ -260,6 +262,18 @@ Augmented assignment statements .. index:: pair: augmented; assignment single: statement; assignment, augmented + single: +=; augmented assignment + single: -=; augmented assignment + single: *=; augmented assignment + single: /=; augmented assignment + single: %=; augmented assignment + single: &=; augmented assignment + single: ^=; augmented assignment + single: |=; augmented assignment + single: **=; augmented assignment + single: //=; augmented assignment + single: >>=; augmented assignment + single: <<=; augmented assignment Augmented assignment is the combination, in a single statement, of a binary operation and an assignment statement: @@ -270,7 +284,7 @@ operation and an assignment statement: augop: "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**=" : | ">>=" | "<<=" | "&=" | "^=" | "|=" -(See section :ref:`primaries` for the syntax definitions for the last three +(See section :ref:`primaries` for the syntax definitions of the last three symbols.) An augmented assignment evaluates the target (which, unlike normal assignment @@ -284,6 +298,11 @@ version, ``x`` is only evaluated once. Also, when possible, the actual operation is performed *in-place*, meaning that rather than creating a new object and assigning that to the target, the old object is modified instead. +Unlike normal assignments, augmented assignments evaluate the left-hand side +*before* evaluating the right-hand side. For example, ``a[i] += f(x)`` first +looks-up ``a[i]``, then it evaluates ``f(x)`` and performs the addition, and +lastly, it writes the result back to ``a[i]``. + With the exception of assigning to tuples and multiple targets in a single statement, the assignment done by augmented assignment statements is handled the same way as normal assignments. Similarly, with the exception of the possible @@ -645,7 +664,7 @@ commas) the two steps are carried out separately for each clause, just as though the clauses had been separated out into individiual import statements. -The details of the first step, finding and loading modules is described in +The details of the first step, finding and loading modules are described in greater detail in the section on the :ref:`import system <importsystem>`, which also describes the various types of packages and modules that can be imported, as well as all the hooks that can be used to customize @@ -656,6 +675,8 @@ initializing the module, which includes execution of the module's code. If the requested module is retrieved successfully, it will be made available in the local namespace in one of three ways: +.. index:: single: as; import statement + * If the module name is followed by :keyword:`as`, then the name following :keyword:`as` is bound directly to the imported module. * If no other name is specified, and the module being imported is a top @@ -674,7 +695,7 @@ available in the local namespace in one of three ways: The :keyword:`from` form uses a slightly more complex process: -#. find the module specified in the :keyword:`from` clause loading and +#. find the module specified in the :keyword:`from` clause, loading and initializing it if necessary; #. for each of the identifiers specified in the :keyword:`import` clauses: @@ -682,7 +703,7 @@ The :keyword:`from` form uses a slightly more complex process: #. if not, attempt to import a submodule with that name and then check the imported module again for that attribute #. if the attribute is not found, :exc:`ImportError` is raised. - #. otherwise, a reference to that value is bound in the local namespace, + #. otherwise, a reference to that value is stored in the local namespace, using the name in the :keyword:`as` clause if it is present, otherwise using the attribute name @@ -711,9 +732,9 @@ to avoid accidentally exporting items that are not part of the API (such as library modules which were imported and used within the module). The :keyword:`from` form with ``*`` may only occur in a module scope. The wild -card form of import --- ``import *`` --- is only allowed at the module level. -Attempting to use it in class or function definitions will raise a -:exc:`SyntaxError`. +card form of import --- ``from module import *`` --- is only allowed at the +module level. Attempting to use it in class or function definitions will raise +a :exc:`SyntaxError`. .. index:: single: relative; import @@ -732,7 +753,7 @@ import mod`` from within ``pkg.subpkg1`` you will import ``pkg.subpkg2.mod``. The specification for relative imports is contained within :pep:`328`. :func:`importlib.import_module` is provided to support applications that -determine which modules need to be loaded dynamically. +determine dynamically the modules to be loaded. .. _future: @@ -744,10 +765,12 @@ Future statements A :dfn:`future statement` is a directive to the compiler that a particular module should be compiled using syntax or semantics that will be available in a -specified future release of Python. The future statement is intended to ease -migration to future versions of Python that introduce incompatible changes to -the language. It allows use of the new features on a per-module basis before -the release in which the feature becomes standard. +specified future release of Python where the feature becomes standard. + +The future statement is intended to ease migration to future versions of Python +that introduce incompatible changes to the language. It allows use of the new +features on a per-module basis before the release in which the feature becomes +standard. .. productionlist:: * future_statement: "from" "__future__" "import" feature ["as" name] @@ -842,7 +865,7 @@ definition, function definition, or :keyword:`import` statement. .. impl-detail:: - The current implementation does not enforce the latter two restrictions, but + The current implementation does not enforce the two restrictions, but programs should not abuse this freedom, as future implementations may enforce them or silently change the meaning of the program. @@ -875,16 +898,16 @@ The :keyword:`nonlocal` statement : | "nonlocal" identifier augop expression_list The :keyword:`nonlocal` statement causes the listed identifiers to refer to -previously bound variables in the nearest enclosing scope. This is important -because the default behavior for binding is to search the local namespace -first. The statement allows encapsulated code to rebind variables outside of -the local scope besides the global (module) scope. +previously bound variables in the nearest enclosing scope excluding globals. +This is important because the default behavior for binding is to search the +local namespace first. The statement allows encapsulated code to rebind +variables outside of the local scope besides the global (module) scope. .. XXX not implemented The :keyword:`nonlocal` statement may prepend an assignment or augmented assignment, but not an expression. -Names listed in a :keyword:`nonlocal` statement, unlike to those listed in a +Names listed in a :keyword:`nonlocal` statement, unlike those listed in a :keyword:`global` statement, must refer to pre-existing bindings in an enclosing scope (the scope in which a new binding should be created cannot be determined unambiguously). diff --git a/Doc/tools/sphinx-build.py b/Doc/tools/sphinx-build.py deleted file mode 100644 index d3fe702..0000000 --- a/Doc/tools/sphinx-build.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -""" - Sphinx - Python documentation toolchain - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - :copyright: 2007-2010 by Georg Brandl. - :license: Python license. -""" - -import sys -import warnings - -# Get rid of UserWarnings reported by pkg_resources. -warnings.filterwarnings('ignore', category=UserWarning, module='jinja2') - -if __name__ == '__main__': - - if sys.version_info[:3] < (2, 4, 0) or sys.version_info[:3] > (3, 0, 0): - sys.stderr.write("""\ -Error: Sphinx needs to be executed with Python 2.4 or newer (not 3.x though). -(If you run this from the Makefile, you can set the PYTHON variable -to the path of an alternative interpreter executable, e.g., -``make html PYTHON=python2.5``). -""") - sys.exit(1) - - from sphinx import main - sys.exit(main(sys.argv)) diff --git a/Doc/tools/sphinxext/c_annotations.py b/Doc/tools/sphinxext/c_annotations.py index 8b5167a..cf9ad9e 100644 --- a/Doc/tools/sphinxext/c_annotations.py +++ b/Doc/tools/sphinxext/c_annotations.py @@ -81,7 +81,10 @@ class Annotations(dict): continue if not par[0].has_key('names') or not par[0]['names']: continue - entry = self.get(par[0]['names'][0]) + name = par[0]['names'][0] + if name.startswith("c."): + name = name[2:] + entry = self.get(name) if not entry: continue elif entry.result_type not in ("PyObject*", "PyVarObject*"): diff --git a/Doc/tools/sphinxext/download.html b/Doc/tools/sphinxext/download.html index 31a53cf..0550111 100644 --- a/Doc/tools/sphinxext/download.html +++ b/Doc/tools/sphinxext/download.html @@ -34,8 +34,8 @@ in the table are the size of the download files in megabytes.</p> <td><a href="{{ dlbase }}/python-{{ release }}-docs-text.tar.bz2">Download</a> (ca. 1.5 MB)</td> </tr> <tr><td>EPUB</td> - <td><a href="{{ dlbase }}/python-{{ release }}-docs-epub.zip">Download</a> (ca. 3.5 MB)</td> - <td><a href="{{ dlbase }}/python-{{ release }}-docs-epub.tar.bz2">Download</a> (ca. 3.5 MB)</td> + <td><a href="{{ dlbase }}/python-{{ release }}-docs.epub">Download</a> (ca. 4.5 MB)</td> + <td></td> </tr> </table> diff --git a/Doc/tools/sphinxext/indexcontent.html b/Doc/tools/sphinxext/indexcontent.html index 7f85470..969099a 100644 --- a/Doc/tools/sphinxext/indexcontent.html +++ b/Doc/tools/sphinxext/indexcontent.html @@ -16,14 +16,14 @@ <p class="biglink"><a class="biglink" href="{{ pathto("howto/index") }}">Python HOWTOs</a><br/> <span class="linkdescr">in-depth documents on specific topics</span></p> </td><td width="50%"> + <p class="biglink"><a class="biglink" href="{{ pathto("installing/index") }}">Installing Python Modules</a><br/> + <span class="linkdescr">installing from the Python Package Index & other sources</span></p> + <p class="biglink"><a class="biglink" href="{{ pathto("distributing/index") }}">Distributing Python Modules</a><br/> + <span class="linkdescr">publishing modules for installation by others</span></p> <p class="biglink"><a class="biglink" href="{{ pathto("extending/index") }}">Extending and Embedding</a><br/> <span class="linkdescr">tutorial for C/C++ programmers</span></p> <p class="biglink"><a class="biglink" href="{{ pathto("c-api/index") }}">Python/C API</a><br/> <span class="linkdescr">reference for C/C++ programmers</span></p> - <p class="biglink"><a class="biglink" href="{{ pathto("install/index") }}">Installing Python Modules</a><br/> - <span class="linkdescr">information for installers & sys-admins</span></p> - <p class="biglink"><a class="biglink" href="{{ pathto("distutils/index") }}">Distributing Python Modules</a><br/> - <span class="linkdescr">sharing modules with others</span></p> <p class="biglink"><a class="biglink" href="{{ pathto("faq/index") }}">FAQs</a><br/> <span class="linkdescr">frequently asked questions (with answers!)</span></p> </td></tr> diff --git a/Doc/tools/sphinxext/indexsidebar.html b/Doc/tools/sphinxext/indexsidebar.html index a0ec32f..83ba123 100644 --- a/Doc/tools/sphinxext/indexsidebar.html +++ b/Doc/tools/sphinxext/indexsidebar.html @@ -1,17 +1,18 @@ - <h3>Download</h3> - <p><a href="{{ pathto('download') }}">Download these documents</a></p> - <h3>Docs for other versions</h3> - <ul> - <li><a href="http://docs.python.org/2.7/">Python 2.7 (stable)</a></li> - <li><a href="http://docs.python.org/3.4/">Python 3.4 (in development)</a></li> - <li><a href="http://www.python.org/doc/versions/">Old versions</a></li> - </ul> +<h3>Download</h3> +<p><a href="{{ pathto('download') }}">Download these documents</a></p> +<h3>Docs for other versions</h3> +<ul> + <li><a href="http://docs.python.org/2.7/">Python 2.7 (stable)</a></li> + <li><a href="http://docs.python.org/3.3/">Python 3.3 (stable)</a></li> + <li><a href="http://docs.python.org/3.5/">Python 3.5 (in development)</a></li> + <li><a href="http://www.python.org/doc/versions/">Old versions</a></li> +</ul> - <h3>Other resources</h3> - <ul> - {# XXX: many of these should probably be merged in the main docs #} - <li><a href="http://www.python.org/dev/peps/">PEP Index</a></li> - <li><a href="http://wiki.python.org/moin/BeginnersGuide">Beginner's Guide</a></li> - <li><a href="http://wiki.python.org/moin/PythonBooks">Book List</a></li> - <li><a href="http://www.python.org/doc/av/">Audio/Visual Talks</a></li> - </ul> +<h3>Other resources</h3> +<ul> + {# XXX: many of these should probably be merged in the main docs #} + <li><a href="http://www.python.org/dev/peps/">PEP Index</a></li> + <li><a href="http://wiki.python.org/moin/BeginnersGuide">Beginner's Guide</a></li> + <li><a href="http://wiki.python.org/moin/PythonBooks">Book List</a></li> + <li><a href="http://www.python.org/doc/av/">Audio/Visual Talks</a></li> +</ul> diff --git a/Doc/tools/sphinxext/layout.html b/Doc/tools/sphinxext/layout.html index 16a9212..d3b2801 100644 --- a/Doc/tools/sphinxext/layout.html +++ b/Doc/tools/sphinxext/layout.html @@ -12,6 +12,8 @@ {%- endif %} </li> {% endblock %} +{% block relbar1 %} {% if builder != 'qthelp' %} {{ relbar() }} {% endif %} {% endblock %} +{% block relbar2 %} {% if builder != 'qthelp' %} {{ relbar() }} {% endif %} {% endblock %} {% block extrahead %} <link rel="shortcut icon" type="image/png" href="{{ pathto('_static/py.png', 1) }}" /> {% if not embedded %}<script type="text/javascript" src="{{ pathto('_static/copybutton.js', 1) }}"></script>{% endif %} diff --git a/Doc/tools/sphinxext/pyspecific.py b/Doc/tools/sphinxext/pyspecific.py index f2d3ca1..31d8c06 100644 --- a/Doc/tools/sphinxext/pyspecific.py +++ b/Doc/tools/sphinxext/pyspecific.py @@ -10,7 +10,7 @@ """ ISSUE_URI = 'http://bugs.python.org/issue%s' -SOURCE_URI = 'http://hg.python.org/cpython/file/3.3/%s' +SOURCE_URI = 'http://hg.python.org/cpython/file/3.4/%s' from docutils import nodes, utils diff --git a/Doc/tools/sphinxext/susp-ignored.csv b/Doc/tools/sphinxext/susp-ignored.csv index abc3ca8..1769023 100644 --- a/Doc/tools/sphinxext/susp-ignored.csv +++ b/Doc/tools/sphinxext/susp-ignored.csv @@ -79,9 +79,8 @@ howto/logging,,:Started,INFO:root:Started howto/logging,,:This,DEBUG:root:This message should go to the log file howto/logging,,:This,DEBUG:This message should appear on the console howto/logging,,:Watch,WARNING:root:Watch out! -howto/pyporting,75,::,# make sure to use :: Python *and* :: Python :: 3 so -howto/pyporting,75,::,"'Programming Language :: Python'," -howto/pyporting,75,::,'Programming Language :: Python :: 3' +howto/pyporting,,::,Programming Language :: Python :: 2 +howto/pyporting,,::,Programming Language :: Python :: 3 howto/regex,,::, howto/regex,,:foo,(?:foo) howto/urllib2,,:example,"for example ""joe@password:example.com""" @@ -141,15 +140,8 @@ library/linecache,,:sys,"sys:x:3:3:sys:/dev:/bin/sh" library/logging.handlers,,:port,host:port library/mmap,,:i2,obj[i1:i2] library/multiprocessing,,`,# Add more tasks using `put()` -library/multiprocessing,,`,# A test file for the `multiprocessing` package -library/multiprocessing,,`,# A test of `multiprocessing.Pool` class -library/multiprocessing,,`,# `BaseManager`. -library/multiprocessing,,`,# in the original order then consider using `Pool.map()` or library/multiprocessing,,`,">>> l._callmethod('__getitem__', (20,)) # equiv to `l[20]`" library/multiprocessing,,`,">>> l._callmethod('__getslice__', (2, 7)) # equiv to `l[2:7]`" -library/multiprocessing,,`,# Not sure if we should synchronize access to `socket.accept()` method by -library/multiprocessing,,`,# object. (We import `multiprocessing.reduction` to enable this pickling.) -library/multiprocessing,,`,# `Pool.imap()` (which will save on the amount of code needed anyway). library/multiprocessing,,:queue,">>> QueueManager.register('get_queue', callable=lambda:queue)" library/multiprocessing,,`,# register the Foo class; make `f()` and `g()` accessible via proxy library/multiprocessing,,`,# register the Foo class; make `g()` and `_h()` accessible via proxy @@ -158,20 +150,16 @@ library/nntplib,,:bytes,:bytes library/nntplib,,:lines,:lines library/optparse,,:len,"del parser.rargs[:len(value)]" library/os.path,,:foo,c:foo +library/pathlib,,:bar,">>> PureWindowsPath('c:/Windows', 'd:bar')" +library/pathlib,,:bar,PureWindowsPath('d:bar') +library/pathlib,,:Program,>>> PureWindowsPath('c:Program Files/').root +library/pathlib,,:Program,>>> PureWindowsPath('c:Program Files/').anchor library/pdb,,:lineno,filename:lineno library/pickle,,:memory,"conn = sqlite3.connect("":memory:"")" library/posix,,`,"CFLAGS=""`getconf LFS_CFLAGS`"" OPT=""-g -O2 $CFLAGS""" -library/pprint,209,::,"'classifiers': ['Development Status :: 4 - Beta'," -library/pprint,209,::,"'Intended Audience :: Developers'," -library/pprint,209,::,"'License :: OSI Approved :: MIT License'," -library/pprint,209,::,"'Natural Language :: English'," -library/pprint,209,::,"'Operating System :: OS Independent'," -library/pprint,209,::,"'Programming Language :: Python'," -library/pprint,209,::,"'Programming Language :: Python :: 2'," -library/pprint,209,::,"'Programming Language :: Python :: 2.6'," -library/pprint,209,::,"'Programming Language :: Python :: 2.7'," -library/pprint,209,::,"'Topic :: Software Development :: Libraries'," -library/pprint,209,::,"'Topic :: Software Development :: Libraries :: Python Modules']," +library/pprint,,::,"'Programming Language :: Python :: 2 :: Only']," +library/pprint,,::,"'Programming Language :: Python :: 2.6'," +library/pprint,,::,"'Programming Language :: Python :: 2.7'," library/profile,,:lineno,filename:lineno(function) library/pyexpat,,:elem1,<py:elem1 /> library/pyexpat,,:py,"xmlns:py = ""http://www.python.org/ns/"">" @@ -183,6 +171,7 @@ library/socket,,:len,fds.fromstring(cmsg_data[:len(cmsg_data) - (len(cmsg_data) library/sqlite3,,:age,"cur.execute(""select * from people where name_last=:who and age=:age"", {""who"": who, ""age"": age})" library/sqlite3,,:memory, library/sqlite3,,:who,"cur.execute(""select * from people where name_last=:who and age=:age"", {""who"": who, ""age"": age})" +library/sqlite3,,:path,"db = sqlite3.connect('file:path/to/database?mode=ro', uri=True)" library/ssl,,:My,"Organizational Unit Name (eg, section) []:My Group" library/ssl,,:My,"Organization Name (eg, company) [Internet Widgits Pty Ltd]:My Organization, Inc." library/ssl,,:myserver,"Common Name (eg, YOUR name) []:myserver.mygroup.myorganization.com" @@ -207,7 +196,12 @@ library/tarfile,,:xz,'r:xz' library/tarfile,,:xz,'w:xz' library/time,,:mm, library/time,,:ss, +library/tracemalloc,,:limit,"for index, stat in enumerate(top_stats[:limit], 1):" library/turtle,,::,Example:: +library/unittest,1412,:foo,"self.assertEqual(cm.output, ['INFO:foo:first message'," +library/unittest,1412,:first,"self.assertEqual(cm.output, ['INFO:foo:first message'," +library/unittest,1412,:foo,'ERROR:foo.bar:second message']) +library/unittest,1412,:second,'ERROR:foo.bar:second message']) library/urllib.request,,:close,Connection:close library/urllib.request,,:lang,"xmlns=""http://www.w3.org/1999/xhtml"" xml:lang=""en"" lang=""en"">\n\n<head>\n" library/urllib.request,,:password,"""joe:password@python.org""" @@ -283,8 +277,8 @@ whatsnew/3.2,,:gz,">>> with tarfile.open(name='myarchive.tar.gz', mode='w:gz') a whatsnew/3.2,,:location,zope9-location = ${zope9:location} whatsnew/3.2,,:prefix,zope-conf = ${custom:prefix}/etc/zope.conf whatsnew/changelog,,:platform,:platform: +whatsnew/changelog,,:gz,": TarFile opened with external fileobj and ""w:gz"" mode didn't" whatsnew/changelog,,:PythonCmd,"With Tk < 8.5 _tkinter.c:PythonCmd() raised UnicodeDecodeError, caused" whatsnew/changelog,,::,": Fix FTP tests for IPv6, bind to ""::1"" instead of ""localhost""." whatsnew/changelog,,::,": Use ""127.0.0.1"" or ""::1"" instead of ""localhost"" as much as" whatsnew/changelog,,:password,user:password -whatsnew/changelog,,:gz,w:gz diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 97aea4f..ef50731 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -370,7 +370,7 @@ defined to allow. For example:: return False retries = retries - 1 if retries < 0: - raise IOError('uncooperative user') + raise OSError('uncooperative user') print(complaint) This function can be called in several ways: diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 24d2d2e..f2b66f7 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -111,10 +111,15 @@ An example that uses most of the list methods:: >>> a.sort() >>> a [-1, 1, 66.25, 333, 333, 1234.5] + >>> a.pop() + 1234.5 + >>> a + [-1, 1, 66.25, 333, 333] You might have noticed that methods like ``insert``, ``remove`` or ``sort`` that -modify the list have no return value printed -- they return ``None``. [1]_ This -is a design principle for all mutable data structures in Python. +only modify the list have no return value printed -- they return the default +``None``. [1]_ This is a design principle for all mutable data structures in +Python. .. _tut-lists-as-stacks: diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index 4282151..d048ae9 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -131,8 +131,8 @@ the exception (allowing a caller to handle the exception as well):: f = open('myfile.txt') s = f.readline() i = int(s.strip()) - except IOError as err: - print("I/O error: {0}".format(err)) + except OSError as err: + print("OS error: {0}".format(err)) except ValueError: print("Could not convert data to an integer.") except: diff --git a/Doc/tutorial/interactive.rst b/Doc/tutorial/interactive.rst index 36acb06..abf30f0 100644 --- a/Doc/tutorial/interactive.rst +++ b/Doc/tutorial/interactive.rst @@ -7,140 +7,27 @@ Interactive Input Editing and History Substitution Some versions of the Python interpreter support editing of the current input line and history substitution, similar to facilities found in the Korn shell and the GNU Bash shell. This is implemented using the `GNU Readline`_ library, -which supports Emacs-style and vi-style editing. This library has its own -documentation which I won't duplicate here; however, the basics are easily -explained. The interactive editing and history described here are optionally -available in the Unix and Cygwin versions of the interpreter. - -This chapter does *not* document the editing facilities of Mark Hammond's -PythonWin package or the Tk-based environment, IDLE, distributed with Python. -The command line history recall which operates within DOS boxes on NT and some -other DOS and Windows flavors is yet another beast. - - -.. _tut-lineediting: - -Line Editing -============ - -If supported, input line editing is active whenever the interpreter prints a -primary or secondary prompt. The current line can be edited using the -conventional Emacs control characters. The most important of these are: -:kbd:`C-A` (Control-A) moves the cursor to the beginning of the line, :kbd:`C-E` -to the end, :kbd:`C-B` moves it one position to the left, :kbd:`C-F` to the -right. Backspace erases the character to the left of the cursor, :kbd:`C-D` the -character to its right. :kbd:`C-K` kills (erases) the rest of the line to the -right of the cursor, :kbd:`C-Y` yanks back the last killed string. -:kbd:`C-underscore` undoes the last change you made; it can be repeated for -cumulative effect. - - -.. _tut-history: - -History Substitution -==================== - -History substitution works as follows. All non-empty input lines issued are -saved in a history buffer, and when a new prompt is given you are positioned on -a new line at the bottom of this buffer. :kbd:`C-P` moves one line up (back) in -the history buffer, :kbd:`C-N` moves one down. Any line in the history buffer -can be edited; an asterisk appears in front of the prompt to mark a line as -modified. Pressing the :kbd:`Return` key passes the current line to the -interpreter. :kbd:`C-R` starts an incremental reverse search; :kbd:`C-S` starts -a forward search. +which supports various styles of editing. This library has its own +documentation which we won't duplicate here. .. _tut-keybindings: -Key Bindings -============ - -The key bindings and some other parameters of the Readline library can be -customized by placing commands in an initialization file called -:file:`~/.inputrc`. Key bindings have the form :: - - key-name: function-name - -or :: - - "string": function-name - -and options can be set with :: - - set option-name value - -For example:: - - # I prefer vi-style editing: - set editing-mode vi - - # Edit using a single line: - set horizontal-scroll-mode On +Tab Completion and History Editing +================================== - # Rebind some keys: - Meta-h: backward-kill-word - "\C-u": universal-argument - "\C-x\C-r": re-read-init-file - -Note that the default binding for :kbd:`Tab` in Python is to insert a :kbd:`Tab` -character instead of Readline's default filename completion function. If you -insist, you can override this by putting :: - - Tab: complete - -in your :file:`~/.inputrc`. (Of course, this makes it harder to type indented -continuation lines if you're accustomed to using :kbd:`Tab` for that purpose.) - -.. index:: - module: rlcompleter - module: readline - -Automatic completion of variable and module names is optionally available. To -enable it in the interpreter's interactive mode, add the following to your -startup file: [#]_ :: - - import rlcompleter, readline - readline.parse_and_bind('tab: complete') - -This binds the :kbd:`Tab` key to the completion function, so hitting the -:kbd:`Tab` key twice suggests completions; it looks at Python statement names, -the current local variables, and the available module names. For dotted -expressions such as ``string.a``, it will evaluate the expression up to the -final ``'.'`` and then suggest completions from the attributes of the resulting -object. Note that this may execute application-defined code if an object with a -:meth:`__getattr__` method is part of the expression. - -A more capable startup file might look like this example. Note that this -deletes the names it creates once they are no longer needed; this is done since -the startup file is executed in the same namespace as the interactive commands, -and removing the names avoids creating side effects in the interactive -environment. You may find it convenient to keep some of the imported modules, -such as :mod:`os`, which turn out to be needed in most sessions with the -interpreter. :: - - # Add auto-completion and a stored history file of commands to your Python - # interactive interpreter. Requires Python 2.0+, readline. Autocomplete is - # bound to the Esc key by default (you can change it - see readline docs). - # - # Store the file in ~/.pystartup, and set an environment variable to point - # to it: "export PYTHONSTARTUP=~/.pystartup" in bash. - - import atexit - import os - import readline - import rlcompleter - - historyPath = os.path.expanduser("~/.pyhistory") - - def save_history(historyPath=historyPath): - import readline - readline.write_history_file(historyPath) - - if os.path.exists(historyPath): - readline.read_history_file(historyPath) - - atexit.register(save_history) - del os, atexit, readline, rlcompleter, save_history, historyPath +Completion of variable and module names is +:ref:`automatically enabled <rlcompleter-config>` at interpreter startup so +that the :kbd:`Tab` key invokes the completion function; it looks at +Python statement names, the current local variables, and the available +module names. For dotted expressions such as ``string.a``, it will evaluate +the expression up to the final ``'.'`` and then suggest completions from +the attributes of the resulting object. Note that this may execute +application-defined code if an object with a :meth:`__getattr__` method +is part of the expression. The default configuration also saves your +history into a file named :file:`.python_history` in your user directory. +The history will be available again during the next interactive interpreter +session. .. _tut-commentary: @@ -162,14 +49,6 @@ into other applications. Another similar enhanced interactive environment is bpython_. -.. rubric:: Footnotes - -.. [#] Python will execute the contents of a file identified by the - :envvar:`PYTHONSTARTUP` environment variable when you start an interactive - interpreter. To customize Python even for non-interactive mode, see - :ref:`tut-customize`. - - .. _GNU Readline: http://tiswww.case.edu/php/chet/readline/rltop.html .. _IPython: http://ipython.scipy.org/ .. _bpython: http://www.bpython-interpreter.org/ diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index cdc2bf2..44dc6d1 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -10,13 +10,13 @@ Using the Python Interpreter Invoking the Interpreter ======================== -The Python interpreter is usually installed as :file:`/usr/local/bin/python3.3` +The Python interpreter is usually installed as :file:`/usr/local/bin/python3.4` on those machines where it is available; putting :file:`/usr/local/bin` in your Unix shell's search path makes it possible to start it by typing the command: .. code-block:: text - python3.3 + python3.4 to the shell. [#]_ Since the choice of the directory where the interpreter lives is an installation option, other places are possible; check with your local @@ -24,26 +24,25 @@ Python guru or system administrator. (E.g., :file:`/usr/local/python` is a popular alternative location.) On Windows machines, the Python installation is usually placed in -:file:`C:\\Python33`, though you can change this when you're running the +:file:`C:\\Python34`, though you can change this when you're running the installer. To add this directory to your path, you can type the following command into the command prompt in a DOS box:: - set path=%path%;C:\python33 + set path=%path%;C:\python34 Typing an end-of-file character (:kbd:`Control-D` on Unix, :kbd:`Control-Z` on Windows) at the primary prompt causes the interpreter to exit with a zero exit status. If that doesn't work, you can exit the interpreter by typing the following command: ``quit()``. -The interpreter's line-editing features usually aren't very sophisticated. On -Unix, whoever installed the interpreter may have enabled support for the GNU -readline library, which adds more elaborate interactive editing and history -features. Perhaps the quickest check to see whether command line editing is -supported is typing Control-P to the first Python prompt you get. If it beeps, -you have command line editing; see Appendix :ref:`tut-interacting` for an -introduction to the keys. If nothing appears to happen, or if ``^P`` is echoed, -command line editing isn't available; you'll only be able to use backspace to -remove characters from the current line. +The interpreter's line-editing features include interactive editing, history +substitution and code completion on systems that support readline. Perhaps the +quickest check to see whether command line editing is supported is typing +Control-P to the first Python prompt you get. If it beeps, you have command +line editing; see Appendix :ref:`tut-interacting` for an introduction to the +keys. If nothing appears to happen, or if ``^P`` is echoed, command line +editing isn't available; you'll only be able to use backspace to remove +characters from the current line. The interpreter operates somewhat like the Unix shell: when called with standard input connected to a tty device, it reads and executes commands interactively; @@ -95,9 +94,9 @@ with the *secondary prompt*, by default three dots (``...``). The interpreter prints a welcome message stating its version number and a copyright notice before printing the first prompt:: - $ python3.3 - Python 3.3 (default, Sep 24 2012, 09:25:04) - [GCC 4.6.3] on linux2 + $ python3.4 + Python 3.4 (default, Mar 16 2014, 09:25:04) + [GCC 4.8.2] on linux Type "help", "copyright", "credits" or "license" for more information. >>> @@ -106,7 +105,7 @@ before printing the first prompt:: Continuation lines are needed when entering a multi-line construct. As an example, take a look at this :keyword:`if` statement:: - >>> the_world_is_flat = 1 + >>> the_world_is_flat = True >>> if the_world_is_flat: ... print("Be careful not to fall off!") ... @@ -149,7 +148,7 @@ Executable Python Scripts On BSD'ish Unix systems, Python scripts can be made directly executable, like shell scripts, by putting the line :: - #! /usr/bin/env python3.3 + #! /usr/bin/env python3.4 (assuming that the interpreter is on the user's :envvar:`PATH`) at the beginning of the script and giving the file an executable mode. The ``#!`` must be the diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index 1902964..fd361ae 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -165,10 +165,16 @@ a built-in module with that name. If not found, it then searches for a file named :file:`spam.py` in a list of directories given by the variable :data:`sys.path`. :data:`sys.path` is initialized from these locations: -* the directory containing the input script (or the current directory). +* The directory containing the input script (or the current directory when no + file is specified). * :envvar:`PYTHONPATH` (a list of directory names, with the same syntax as the shell variable :envvar:`PATH`). -* the installation-dependent default. +* The installation-dependent default. + +.. note:: + On file systems which support symlinks, the directory containing the input + script is calculated after the symlink is followed. In other words the + directory containing the symlink is **not** added to the module search path. After initialization, Python programs can modify :data:`sys.path`. The directory containing the script being run is placed at the beginning of the @@ -278,24 +284,23 @@ defines. It returns a sorted list of strings:: >>> dir(fibo) ['__name__', 'fib', 'fib2'] >>> dir(sys) # doctest: +NORMALIZE_WHITESPACE - ['__displayhook__', '__doc__', '__egginsert', '__excepthook__', - '__loader__', '__name__', '__package__', '__plen', '__stderr__', - '__stdin__', '__stdout__', '_clear_type_cache', '_current_frames', - '_debugmallocstats', '_getframe', '_home', '_mercurial', '_xoptions', - 'abiflags', 'api_version', 'argv', 'base_exec_prefix', 'base_prefix', - 'builtin_module_names', 'byteorder', 'call_tracing', 'callstats', - 'copyright', 'displayhook', 'dont_write_bytecode', 'exc_info', - 'excepthook', 'exec_prefix', 'executable', 'exit', 'flags', 'float_info', - 'float_repr_style', 'getcheckinterval', 'getdefaultencoding', - 'getdlopenflags', 'getfilesystemencoding', 'getobjects', 'getprofile', - 'getrecursionlimit', 'getrefcount', 'getsizeof', 'getswitchinterval', - 'gettotalrefcount', 'gettrace', 'hash_info', 'hexversion', - 'implementation', 'int_info', 'intern', 'maxsize', 'maxunicode', - 'meta_path', 'modules', 'path', 'path_hooks', 'path_importer_cache', - 'platform', 'prefix', 'ps1', 'setcheckinterval', 'setdlopenflags', - 'setprofile', 'setrecursionlimit', 'setswitchinterval', 'settrace', - 'stderr', 'stdin', 'stdout', 'thread_info', 'version', 'version_info', - 'warnoptions'] + ['__displayhook__', '__doc__', '__excepthook__', '__loader__', '__name__', + '__package__', '__stderr__', '__stdin__', '__stdout__', + '_clear_type_cache', '_current_frames', '_debugmallocstats', '_getframe', + '_home', '_mercurial', '_xoptions', 'abiflags', 'api_version', 'argv', + 'base_exec_prefix', 'base_prefix', 'builtin_module_names', 'byteorder', + 'call_tracing', 'callstats', 'copyright', 'displayhook', + 'dont_write_bytecode', 'exc_info', 'excepthook', 'exec_prefix', + 'executable', 'exit', 'flags', 'float_info', 'float_repr_style', + 'getcheckinterval', 'getdefaultencoding', 'getdlopenflags', + 'getfilesystemencoding', 'getobjects', 'getprofile', 'getrecursionlimit', + 'getrefcount', 'getsizeof', 'getswitchinterval', 'gettotalrefcount', + 'gettrace', 'hash_info', 'hexversion', 'implementation', 'int_info', + 'intern', 'maxsize', 'maxunicode', 'meta_path', 'modules', 'path', + 'path_hooks', 'path_importer_cache', 'platform', 'prefix', 'ps1', + 'setcheckinterval', 'setdlopenflags', 'setprofile', 'setrecursionlimit', + 'setswitchinterval', 'settrace', 'stderr', 'stdin', 'stdout', + 'thread_info', 'version', 'version_info', 'warnoptions'] Without arguments, :func:`dir` lists the names you have defined currently:: diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 7e7a154..cd73bc2 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -15,7 +15,7 @@ operating system:: >>> import os >>> os.getcwd() # Return the current working directory - 'C:\\Python33' + 'C:\\Python34' >>> os.chdir('/server/accesslogs') # Change current working directory >>> os.system('mkdir today') # Run the command mkdir in the system shell 0 @@ -40,7 +40,9 @@ a higher level interface that is easier to use:: >>> import shutil >>> shutil.copyfile('data.db', 'archive.db') + 'archive.db' >>> shutil.move('/build/executables', 'installdir') + 'installdir' .. _tut-file-wildcards: diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index c1dd69a..c0197ea 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -277,7 +277,7 @@ applications include caching objects that are expensive to create:: Traceback (most recent call last): File "<stdin>", line 1, in <module> d['primary'] # entry was automatically removed - File "C:/python33/lib/weakref.py", line 46, in __getitem__ + File "C:/python34/lib/weakref.py", line 46, in __getitem__ o = self.data[key]() KeyError: 'primary' diff --git a/Doc/tutorial/whatnow.rst b/Doc/tutorial/whatnow.rst index 7fcbdc3f..979f587 100644 --- a/Doc/tutorial/whatnow.rst +++ b/Doc/tutorial/whatnow.rst @@ -21,8 +21,8 @@ the set are: and many other tasks. Skimming through the Library Reference will give you an idea of what's available. -* :ref:`install-index` explains how to install external modules written by other - Python users. +* :ref:`installing-index` explains how to install additional modules written + by other Python users. * :ref:`reference-index`: A detailed explanation of Python's syntax and semantics. It's heavy reading, but is useful as a complete guide to the diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 4e7168f..0c3c203 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -24,7 +24,7 @@ Command line When invoking Python, you may specify any of these options:: - python [-bBdEhiOqsSuvVWx?] [-c command | -m module-name | script | - ] [args] + python [-bBdEhiIOqsSuvVWx?] [-c command | -m module-name | script | - ] [args] The most common use case is, of course, a simple invocation of a script:: @@ -81,7 +81,8 @@ source. the implementation may not always enforce this (e.g. it may allow you to use a name that includes a hyphen). - Package names are also permitted. When a package name is supplied instead + Package names (including namespace packages) are also permitted. When a + package name is supplied instead of a normal module, the interpreter will execute ``<pkg>.__main__`` as the main module. This behaviour is deliberately similar to the handling of directories and zipfiles that are passed to the interpreter as the @@ -115,6 +116,9 @@ source. .. versionchanged:: 3.1 Supply the package name to run a ``__main__`` submodule. + .. versionchanged:: 3.4 + namespace packages are also supported + .. describe:: - @@ -147,7 +151,12 @@ source. If no interface option is given, :option:`-i` is implied, ``sys.argv[0]`` is an empty string (``""``) and the current directory will be added to the -start of :data:`sys.path`. +start of :data:`sys.path`. Also, tab-completion and history editing is +automatically enabled, if available on your platform (see +:ref:`rlcompleter-config`). + +.. versionchanged:: 3.4 + Automatic enabling of tab-completion and history editing. .. seealso:: :ref:`tut-invoking` @@ -170,6 +179,8 @@ Generic options Python 3.0 +.. _using-on-misc-options: + Miscellaneous options ~~~~~~~~~~~~~~~~~~~~~ @@ -208,6 +219,17 @@ Miscellaneous options raises an exception. See also :envvar:`PYTHONINSPECT`. +.. cmdoption:: -I + + Run Python in isolated mode. This also implies -E and -s. + In isolated mode :data:`sys.path` contains neither the script's directory nor + the user's site-packages directory. All :envvar:`PYTHON*` environment + variables are ignored, too. Further restrictions may be imposed to prevent + the user from injecting malicious code. + + .. versionadded:: 3.4 + + .. cmdoption:: -O Turn on basic optimizations. This changes the filename extension for @@ -358,9 +380,19 @@ Miscellaneous options .. cmdoption:: -X Reserved for various implementation-specific options. CPython currently - defines just one, you can use ``-X faulthandler`` to enable - :mod:`faulthandler`. It also allows to pass arbitrary values and retrieve - them through the :data:`sys._xoptions` dictionary. + defines the following possible values: + + * ``-X faulthandler`` to enable :mod:`faulthandler`; + * ``-X showrefcount`` to enable the output of the total reference count + and memory blocks (only works on debug builds); + * ``-X tracemalloc`` to start tracing Python memory allocations using the + :mod:`tracemalloc` module. By default, only the most recent frame is + stored in a traceback of a trace. Use ``-X tracemalloc=NFRAME`` to start + tracing with a traceback limit of *NFRAME* frames. See the + :func:`tracemalloc.start` for more information. + + It also allows to pass arbitrary values and retrieve them through the + :data:`sys._xoptions` dictionary. .. versionchanged:: 3.2 It is now allowed to pass :option:`-X` with CPython. @@ -368,6 +400,9 @@ Miscellaneous options .. versionadded:: 3.3 The ``-X faulthandler`` option. + .. versionadded:: 3.4 + The ``-X showrefcount`` and ``-X tracemalloc`` options. + Options you shouldn't use ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -385,7 +420,7 @@ Environment variables --------------------- These environment variables influence Python's behavior, they are processed -before the command-line switches other than -E. It is customary that +before the command-line switches other than -E or -I. It is customary that command-line switches override environmental variables where there is a conflict. @@ -430,7 +465,7 @@ conflict. is executed in the same namespace where interactive commands are executed so that objects defined or imported in it can be used without qualification in the interactive session. You can also change the prompts :data:`sys.ps1` and - :data:`sys.ps2` in this file. + :data:`sys.ps2` and the hook :data:`sys.__interactivehook__` in this file. .. envvar:: PYTHONY2K @@ -485,9 +520,9 @@ conflict. .. envvar:: PYTHONDONTWRITEBYTECODE - If this is set, Python won't try to write ``.pyc`` or ``.pyo`` files on the - import of source modules. This is equivalent to specifying the :option:`-B` - option. + If this is set to a non-empty string, Python won't try to write ``.pyc`` or + ``.pyo`` files on the import of source modules. This is equivalent to + specifying the :option:`-B` option. .. envvar:: PYTHONHASHSEED @@ -512,13 +547,16 @@ conflict. .. envvar:: PYTHONIOENCODING If this is set before running the interpreter, it overrides the encoding used - for stdin/stdout/stderr, in the syntax ``encodingname:errorhandler``. The - ``:errorhandler`` part is optional and has the same meaning as in - :func:`str.encode`. + for stdin/stdout/stderr, in the syntax ``encodingname:errorhandler``. Both + the ``encodingname`` and the ``:errorhandler`` parts are optional and have + the same meaning as in :func:`str.encode`. For stderr, the ``:errorhandler`` part is ignored; the handler will always be ``'backslashreplace'``. + .. versionchanged:: 3.4 + The ``encodingname`` part is now optional. + .. envvar:: PYTHONNOUSERSITE @@ -556,15 +594,34 @@ conflict. .. envvar:: PYTHONFAULTHANDLER - If this environment variable is set, :func:`faulthandler.enable` is called - at startup: install a handler for :const:`SIGSEGV`, :const:`SIGFPE`, - :const:`SIGABRT`, :const:`SIGBUS` and :const:`SIGILL` signals to dump the - Python traceback. This is equivalent to :option:`-X` ``faulthandler`` - option. + If this environment variable is set to a non-empty string, + :func:`faulthandler.enable` is called at startup: install a handler for + :const:`SIGSEGV`, :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and + :const:`SIGILL` signals to dump the Python traceback. This is equivalent to + :option:`-X` ``faulthandler`` option. .. versionadded:: 3.3 +.. envvar:: PYTHONTRACEMALLOC + + If this environment variable is set to a non-empty string, start tracing + Python memory allocations using the :mod:`tracemalloc` module. The value of + the variable is the maximum number of frames stored in a traceback of a + trace. For example, ``PYTHONTRACEMALLOC=1`` stores only the most recent + frame. See the :func:`tracemalloc.start` for more information. + + .. versionadded:: 3.4 + + +.. envvar:: PYTHONASYNCIODEBUG + + If this environment variable is set to a non-empty string, enable the debug + mode of the :mod:`asyncio` module. + + .. versionadded:: 3.4 + + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 3e1b74d..5be439f 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -25,7 +25,7 @@ there. What you get after installing is a number of things: -* A :file:`MacPython 3.3` folder in your :file:`Applications` folder. In here +* A :file:`MacPython 3.4` folder in your :file:`Applications` folder. In here you find IDLE, the development environment that is a standard part of official Python distributions; PythonLauncher, which handles double-clicking Python scripts from the Finder; and the "Build Applet" tool, which allows you to @@ -93,7 +93,7 @@ aware of: programs that talk to the Aqua window manager (in other words, anything that has a GUI) need to be run in a special way. Use :program:`pythonw` instead of :program:`python` to start such scripts. -With Python 3.3, you can use either :program:`python` or :program:`pythonw`. +With Python 3.4, you can use either :program:`python` or :program:`pythonw`. Configuration @@ -158,7 +158,7 @@ http://www.riverbankcomputing.co.uk/software/pyqt/intro. Distributing Python Applications on the Mac =========================================== -The "Build Applet" tool that is placed in the MacPython 3.3 folder is fine for +The "Build Applet" tool that is placed in the MacPython 3.4 folder is fine for packaging small Python scripts on your own machine to run as a standard Mac application. This tool, however, is not robust enough to distribute Python applications to other users. diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index 40635c6..3c7fc41 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -60,6 +60,8 @@ To install the newest Python versions on OpenSolaris, install `blastwave prompt. +.. _building-python-on-unix: + Building Python =============== diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc index 5fdbc9b..52cdda0 100644 --- a/Doc/using/venv-create.inc +++ b/Doc/using/venv-create.inc @@ -11,6 +11,11 @@ containing a copy of the ``python`` binary (or binaries, in the case of Windows). It also creates an (initially empty) ``lib/pythonX.Y/site-packages`` subdirectory (on Windows, this is ``Lib\site-packages``). +.. seealso:: + + `Python Packaging User Guide: Creating and using virtual environments + <http://packaging.python.org/en/latest/tutorial.html#creating-and-using-virtual-environments>`__ + .. highlight:: none On Windows, you may have to invoke the ``pyvenv`` script as follows, if you @@ -25,7 +30,7 @@ or equivalently:: The command, if run with ``-h``, will show the available options:: usage: pyvenv [-h] [--system-site-packages] [--symlinks] [--clear] - [--upgrade] ENV_DIR [ENV_DIR ...] + [--upgrade] [--without-pip] ENV_DIR [ENV_DIR ...] Creates virtual Python environments in one or more target directories. @@ -38,11 +43,19 @@ The command, if run with ``-h``, will show the available options:: virtual environment. --symlinks Try to use symlinks rather than copies, when symlinks are not the default for the platform. + --copies Try to use copies rather than symlinks, even when + symlinks are the default for the platform. --clear Delete the environment directory if it already exists. If not specified and the directory exists, an error is raised. --upgrade Upgrade the environment directory to use this version of Python, assuming Python has been upgraded in-place. + --without-pip Skips installing or upgrading pip in the virtual + environment (pip is bootstrapped by default) + +.. versionchanged:: 3.4 + Installs pip by default, added the ``--without-pip`` and ``--copies`` + options If the target directory already exists an error will be raised, unless the ``--clear`` or ``--upgrade`` option was provided. @@ -51,25 +64,29 @@ The created ``pyvenv.cfg`` file also includes the ``include-system-site-packages`` key, set to ``true`` if ``venv`` is run with the ``--system-site-packages`` option, ``false`` otherwise. +Unless the ``--without-pip`` option is given, :mod:`ensurepip` will be +invoked to bootstrap ``pip`` into the virtual environment. + Multiple paths can be given to ``pyvenv``, in which case an identical virtualenv will be created, according to the given options, at each provided path. Once a venv has been created, it can be "activated" using a script in the -venv's binary directory. The invocation of the script is platform-specific: on -a Posix platform, you would typically do:: - - $ source <venv>/bin/activate - -whereas on Windows, you might do:: - - C:\> <venv>/Scripts/activate - -if you are using the ``cmd.exe`` shell, or perhaps:: - - PS C:\> <venv>/Scripts/Activate.ps1 - -if you use PowerShell. +venv's binary directory. The invocation of the script is platform-specific: + ++-------------+-----------------+-----------------------------------------+ +| Platform | Shell | Command to activate virtual environment | ++=============+=================+=========================================+ +| Posix | bash/zsh | $ source <venv>/bin/activate | ++-------------+-----------------+-----------------------------------------+ +| | fish | $ . <venv>/bin/activate.fish | ++-------------+-----------------+-----------------------------------------+ +| | csh/tcsh | $ source <venv>/bin/activate.csh | ++-------------+-----------------+-----------------------------------------+ +| Windows | cmd.exe | C:\> <venv>/Scripts/activate.bat | ++-------------+-----------------+-----------------------------------------+ +| | PowerShell | PS C:\> <venv>/Scripts/Activate.ps1 | ++-------------+-----------------+-----------------------------------------+ You don't specifically *need* to activate an environment; activation just prepends the venv's binary directory to your path, so that "python" invokes the @@ -83,3 +100,5 @@ a "deactivate" function, whereas on Windows there are separate scripts called ``deactivate.bat`` and ``Deactivate.ps1`` which are installed when the venv is created. +.. versionadded:: 3.4 + ``fish`` and ``csh`` activation scripts. diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 6c4d16e..42a92af 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -11,6 +11,10 @@ This document aims to give an overview of Windows-specific behaviour you should know about when using Python on Microsoft Windows. +.. XXX (ncoghlan) + + This looks rather stale to me... + Installing Python ================= @@ -560,12 +564,6 @@ View the :file:`readme.txt` in their respective directories: +--------------------+--------------+-----------------------+ | Directory | MSVC version | Visual Studio version | +====================+==============+=======================+ -| :file:`PC/VC6/` | 6.0 | 97 | -+--------------------+--------------+-----------------------+ -| :file:`PC/VS7.1/` | 7.1 | 2003 | -+--------------------+--------------+-----------------------+ -| :file:`PC/VS8.0/` | 8.0 | 2005 | -+--------------------+--------------+-----------------------+ | :file:`PC/VS9.0/` | 9.0 | 2008 | +--------------------+--------------+-----------------------+ | :file:`PCbuild/` | 10.0 | 2010 | diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index b26c9b2..8e0d3aa 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -7,7 +7,6 @@ .. hyperlink all the methods & functions. .. T_STRING_INPLACE not described in main docs -.. "Format String Syntax" in string.rst could use many more examples. .. $Id$ Rules for maintenance: @@ -50,17 +49,16 @@ This saves the maintainer some effort going through the SVN logs when researching a change. -This article explains the new features in Python 2.7. The final -release of 2.7 is currently scheduled for July 2010; the detailed -schedule is described in :pep:`373`. +This article explains the new features in Python 2.7. Python 2.7 was released +on July 3, 2010. Numeric handling has been improved in many ways, for both -floating-point numbers and for the :class:`Decimal` class. There are -some useful additions to the standard library, such as a greatly -enhanced :mod:`unittest` module, the :mod:`argparse` module for -parsing command-line options, convenient ordered-dictionary and -:class:`Counter` classes in the :mod:`collections` module, and many -other improvements. +floating-point numbers and for the :class:`~decimal.Decimal` class. +There are some useful additions to the standard library, such as a +greatly enhanced :mod:`unittest` module, the :mod:`argparse` module +for parsing command-line options, convenient :class:`~collections.OrderedDict` +and :class:`~collections.Counter` classes in the :mod:`collections` module, +and many other improvements. Python 2.7 is planned to be the last of the 2.x releases, so we worked on making it a good release for the long term. To help with porting @@ -81,45 +79,91 @@ bug/patch item for each change. The Future for Python 2.x ========================= -Python 2.7 is intended to be the last major release in the 2.x series. -The Python maintainers are planning to focus their future efforts on -the Python 3.x series. - -This means that 2.7 will remain in place for a long time, running -production systems that have not been ported to Python 3.x. -Two consequences of the long-term significance of 2.7 are: - -* It's very likely the 2.7 release will have a longer period of - maintenance compared to earlier 2.x versions. Python 2.7 will - continue to be maintained while the transition to 3.x continues, and - the developers are planning to support Python 2.7 with bug-fix - releases beyond the typical two years. - -* A policy decision was made to silence warnings only of interest to - developers. :exc:`DeprecationWarning` and its - descendants are now ignored unless otherwise requested, preventing - users from seeing warnings triggered by an application. This change - was also made in the branch that will become Python 3.2. (Discussed - on stdlib-sig and carried out in :issue:`7319`.) - - In previous releases, :exc:`DeprecationWarning` messages were - enabled by default, providing Python developers with a clear - indication of where their code may break in a future major version - of Python. - - However, there are increasingly many users of Python-based - applications who are not directly involved in the development of - those applications. :exc:`DeprecationWarning` messages are - irrelevant to such users, making them worry about an application - that's actually working correctly and burdening application developers - with responding to these concerns. - - You can re-enable display of :exc:`DeprecationWarning` messages by - running Python with the :option:`-Wdefault` (short form: - :option:`-Wd`) switch, or by setting the :envvar:`PYTHONWARNINGS` - environment variable to ``"default"`` (or ``"d"``) before running - Python. Python code can also re-enable them - by calling ``warnings.simplefilter('default')``. +Python 2.7 is the last major release in the 2.x series, as the Python +maintainers have shifted the focus of their new feature development efforts +to the Python 3.x series. This means that while Python 2 continues to +receive bug fixes, and to be updated to build correctly on new hardware and +versions of supported operated systems, there will be no new full feature +releases for the language or standard library. + +However, while there is a large common subset between Python 2.7 and Python +3, and many of the changes involved in migrating to that common subset, or +directly to Python 3, can be safely automated, some other changes (notably +those associated with Unicode handling) may require careful consideration, +and preferably robust automated regression test suites, to migrate +effectively. + +This means that Python 2.7 will remain in place for a long time, providing a +stable and supported base platform for production systems that have not yet +been ported to Python 3. The full expected lifecycle of the Python 2.7 +series is detailed in :pep:`373`. + +Some key consequences of the long-term significance of 2.7 are: + +* As noted above, the 2.7 release has a much longer period of maintenance + when compared to earlier 2.x versions. Python 2.7 is currently expected to + remain supported by the core development team (receiving security updates + and other bug fixes) until at least 2020 (10 years after its initial + release, compared to the more typical support period of 18-24 months). + +* As the Python 2.7 standard library ages, making effective use of the + Python Package Index (either directly or via a redistributor) becomes + more important for Python 2 users. In addition to a wide variety of third + party packages for various tasks, the available packages include backports + of new modules and features from the Python 3 standard library that are + compatible with Python 2, as well as various tools and libraries that can + make it easier to migrate to Python 3. The `Python Packaging User Guide + <https://packaging.python.org>`__ provides guidance on downloading and + installing software from the Python Package Index. + +* While the preferred approach to enhancing Python 2 is now the publication + of new packages on the Python Package Index, this approach doesn't + necessarily work in all cases, especially those related to network + security. In exceptional cases that cannot be handled adequately by + publishing new or updated packages on PyPI, the Python Enhancement + Proposal process may be used to make the case for adding new features + directly to the Python 2 standard library. Any such additions, and the + maintenance releases where they were added, will be noted in the + :ref:`py27-maintenance-enhancements` section below. + +For projects wishing to migrate from Python 2 to Python 3, or for library +and framework developers wishing to support users on both Python 2 and +Python 3, there are a variety of tools and guides available to help decide +on a suitable approach and manage some of the technical details involved. +The recommended starting point is the :ref:`pyporting-howto` HOWTO guide. + + +Changes to the Handling of Deprecation Warnings +=============================================== + +For Python 2.7, a policy decision was made to silence warnings only of +interest to developers by default. :exc:`DeprecationWarning` and its +descendants are now ignored unless otherwise requested, preventing +users from seeing warnings triggered by an application. This change +was also made in the branch that became Python 3.2. (Discussed +on stdlib-sig and carried out in :issue:`7319`.) + +In previous releases, :exc:`DeprecationWarning` messages were +enabled by default, providing Python developers with a clear +indication of where their code may break in a future major version +of Python. + +However, there are increasingly many users of Python-based +applications who are not directly involved in the development of +those applications. :exc:`DeprecationWarning` messages are +irrelevant to such users, making them worry about an application +that's actually working correctly and burdening application developers +with responding to these concerns. + +You can re-enable display of :exc:`DeprecationWarning` messages by +running Python with the :option:`-Wdefault <-W>` (short form: +:option:`-Wd <-W>`) switch, or by setting the :envvar:`PYTHONWARNINGS` +environment variable to ``"default"`` (or ``"d"``) before running +Python. Python code can also re-enable them +by calling ``warnings.simplefilter('default')``. + +The ``unittest`` module also automatically reenables deprecation warnings +when running tests. Python 3.1 Features @@ -133,7 +177,7 @@ for migrating to the 3.x series. A partial list of 3.1 features that were backported to 2.7: * The syntax for set literals (``{1,2,3}`` is a mutable set). -* Dictionary and set comprehensions (``{ i: i*2 for i in range(3)}``). +* Dictionary and set comprehensions (``{i: i*2 for i in range(3)}``). * Multiple context managers in a single :keyword:`with` statement. * A new version of the :mod:`io` library, rewritten in C for performance. * The ordered-dictionary type described in :ref:`pep-0372`. @@ -155,7 +199,7 @@ Other new Python3-mode warnings include: * :func:`operator.isCallable` and :func:`operator.sequenceIncludes`, which are not supported in 3.x, now trigger warnings. * The :option:`-3` switch now automatically - enables the :option:`-Qwarn` switch that causes warnings + enables the :option:`-Qwarn <-Q>` switch that causes warnings about using classic division with integers and long integers. @@ -390,9 +434,10 @@ standard input or output. .. seealso:: - `argparse module documentation <http://docs.python.org/dev/library/argparse.html>`__ + :mod:`argparse` documentation + The documentation page of the argparse module. - `Upgrading optparse code to use argparse <http://docs.python.org/dev/library/argparse.html#upgrading-optparse-code>`__ + :ref:`upgrading-optparse-code` Part of the Python documentation, describing how to convert code that uses :mod:`optparse`. @@ -402,8 +447,6 @@ standard input or output. PEP 391: Dictionary-Based Configuration For Logging ==================================================== -.. XXX not documented in library reference yet; add link here once it's added. - The :mod:`logging` module is very flexible; applications can define a tree of logging subsystems, and each logger in this tree can filter out certain messages, format them differently, and direct messages to @@ -412,21 +455,21 @@ a varying number of handlers. All this flexibility can require a lot of configuration. You can write Python statements to create objects and set their properties, but a complex set-up requires verbose but boring code. -:mod:`logging` also supports a :func:`~logging.config.fileConfig` +:mod:`logging` also supports a :func:`~logging.fileConfig` function that parses a file, but the file format doesn't support configuring filters, and it's messier to generate programmatically. -Python 2.7 adds a :func:`~logging.config.dictConfig` function that +Python 2.7 adds a :func:`~logging.dictConfig` function that uses a dictionary to configure logging. There are many ways to produce a dictionary from different sources: construct one with code; parse a file containing JSON; or use a YAML parsing library if one is -installed. +installed. For more information see :ref:`logging-config-api`. The following example configures two loggers, the root logger and a -logger named "network". Messages sent to the root logger will be +logger named "network". Messages sent to the root logger will be sent to the system log using the syslog protocol, and messages to the "network" logger will be written to a :file:`network.log` file -that will be rotated once the log reaches 1Mb. +that will be rotated once the log reaches 1MB. :: @@ -445,7 +488,7 @@ that will be rotated once the log reaches 1Mb. 'filename': '/logs/network.log', 'formatter': 'standard', 'level': 'INFO', - 'maxBytes': 1024*1024}, + 'maxBytes': 1000000}, 'syslog': {'class': 'logging.handlers.SysLogHandler', 'formatter': 'standard', 'level': 'ERROR'}}, @@ -483,16 +526,19 @@ implemented by Vinay Sajip, are: for UDP or :const:`socket.SOCK_STREAM` for TCP. The default protocol remains UDP. -* :class:`Logger` instances gained a :meth:`getChild` method that retrieves a - descendant logger using a relative path. For example, - once you retrieve a logger by doing ``log = getLogger('app')``, +* :class:`~logging.Logger` instances gained a :meth:`~logging.Logger.getChild` + method that retrieves a descendant logger using a relative path. + For example, once you retrieve a logger by doing ``log = getLogger('app')``, calling ``log.getChild('network.listen')`` is equivalent to ``getLogger('app.network.listen')``. -* The :class:`LoggerAdapter` class gained a :meth:`isEnabledFor` method - that takes a *level* and returns whether the underlying logger would +* The :class:`~logging.LoggerAdapter` class gained a + :meth:`~logging.LoggerAdapter.isEnabledFor` method that takes a + *level* and returns whether the underlying logger would process a message of that level of importance. +.. XXX: Logger objects don't have a class declaration so the link don't work + .. seealso:: :pep:`391` - Dictionary-Based Configuration For Logging @@ -501,14 +547,15 @@ implemented by Vinay Sajip, are: PEP 3106: Dictionary Views ==================================================== -The dictionary methods :meth:`keys`, :meth:`values`, and :meth:`items` -are different in Python 3.x. They return an object called a :dfn:`view` -instead of a fully materialized list. +The dictionary methods :meth:`~dict.keys`, :meth:`~dict.values`, and +:meth:`~dict.items` are different in Python 3.x. They return an object +called a :dfn:`view` instead of a fully materialized list. -It's not possible to change the return values of :meth:`keys`, -:meth:`values`, and :meth:`items` in Python 2.7 because too much code -would break. Instead the 3.x versions were added under the new names -:meth:`viewkeys`, :meth:`viewvalues`, and :meth:`viewitems`. +It's not possible to change the return values of :meth:`~dict.keys`, +:meth:`~dict.values`, and :meth:`~dict.items` in Python 2.7 because +too much code would break. Instead the 3.x versions were added +under the new names :meth:`~dict.viewkeys`, :meth:`~dict.viewvalues`, +and :meth:`~dict.viewitems`. :: @@ -550,8 +597,8 @@ over the view:: RuntimeError: dictionary changed size during iteration You can use the view methods in Python 2.x code, and the 2to3 -converter will change them to the standard :meth:`keys`, -:meth:`values`, and :meth:`items` methods. +converter will change them to the standard :meth:`~dict.keys`, +:meth:`~dict.values`, and :meth:`~dict.items` methods. .. seealso:: @@ -624,7 +671,7 @@ Some smaller changes made to the core Python language are: ``{}`` continues to represent an empty dictionary; use ``set()`` for an empty set. - >>> {1,2,3,4,5} + >>> {1, 2, 3, 4, 5} set([1, 2, 3, 4, 5]) >>> set() # empty set set([]) @@ -794,7 +841,7 @@ Some smaller changes made to the core Python language are: ``None`` as its first argument. (Fixed by Georg Brandl; :issue:`4759`.) - .. bytearray doesn't seem to be documented + .. XXX bytearray doesn't seem to be documented * When using ``@classmethod`` and ``@staticmethod`` to wrap methods as class or static methods, the wrapper object now @@ -867,12 +914,6 @@ Optimizations Several performance enhancements have been added: -.. * A new :program:`configure` option, :option:`--with-computed-gotos`, - compiles the main bytecode interpreter loop using a new dispatch - mechanism that gives speedups of up to 20%, depending on the system - and benchmark. The new mechanism is only supported on certain - compilers, such as gcc, SunPro, and icc. - * A new opcode was added to perform the initial setup for :keyword:`with` statements, looking up the :meth:`__enter__` and :meth:`__exit__` methods. (Contributed by Benjamin Peterson.) @@ -1054,7 +1095,7 @@ changes, or look through the Subversion logs for all the details. :meth:`~collections.deque.count` method that returns the number of contained elements equal to the supplied argument *x*, and a :meth:`~collections.deque.reverse` method that reverses the elements - of the deque in-place. :class:`deque` also exposes its maximum + of the deque in-place. :class:`~collections.deque` also exposes its maximum length as the read-only :attr:`~collections.deque.maxlen` attribute. (Both features added by Raymond Hettinger.) @@ -1135,15 +1176,14 @@ changes, or look through the Subversion logs for all the details. ``Decimal('0.1000000000000000055511151231257827021181583404541015625')``. (Implemented by Raymond Hettinger; :issue:`4796`.) - Comparing instances of :class:`Decimal` with floating-point + Comparing instances of :class:`~decimal.Decimal` with floating-point numbers now produces sensible results based on the numeric values of the operands. Previously such comparisons would fall back to Python's default rules for comparing objects, which produced arbitrary results based on their type. Note that you still cannot combine :class:`Decimal` and floating-point in other operations such as addition, since you should be explicitly choosing how to convert between float and - :class:`Decimal`. - (Fixed by Mark Dickinson; :issue:`2531`.) + :class:`~decimal.Decimal`. (Fixed by Mark Dickinson; :issue:`2531`.) The constructor for :class:`~decimal.Decimal` now accepts floating-point numbers (added by Raymond Hettinger; :issue:`8257`) @@ -1195,8 +1235,8 @@ changes, or look through the Subversion logs for all the details. Ordering comparisons (``<``, ``<=``, ``>``, ``>=``) between fractions and complex numbers now raise a :exc:`TypeError`. - This fixes an oversight, making the :class:`Fraction` match the other - numeric types. + This fixes an oversight, making the :class:`~fractions.Fraction` + match the other numeric types. .. revision 79455 @@ -1210,7 +1250,7 @@ changes, or look through the Subversion logs for all the details. uploads thanks to an added *rest* parameter (patch by Pablo Mouzo; :issue:`6845`.) -* New class decorator: :func:`total_ordering` in the :mod:`functools` +* New class decorator: :func:`~functools.total_ordering` in the :mod:`functools` module takes a class that defines an :meth:`__eq__` method and one of :meth:`__lt__`, :meth:`__le__`, :meth:`__gt__`, or :meth:`__ge__`, and generates the missing comparison methods. Since the @@ -1218,7 +1258,7 @@ changes, or look through the Subversion logs for all the details. this decorator makes it easier to define ordered classes. (Added by Raymond Hettinger; :issue:`5479`.) - New function: :func:`cmp_to_key` will take an old-style comparison + New function: :func:`~functools.cmp_to_key` will take an old-style comparison function that expects two arguments and return a new callable that can be used as the *key* parameter to functions such as :func:`sorted`, :func:`min` and :func:`max`, etc. The primary @@ -1345,7 +1385,7 @@ changes, or look through the Subversion logs for all the details. with any object literal that decodes to a list of pairs. (Contributed by Raymond Hettinger; :issue:`5381`.) -* The :mod:`mailbox` module's :class:`Maildir` class now records the +* The :mod:`mailbox` module's :class:`~mailbox.Maildir` class now records the timestamp on the directories it reads, and only re-reads them if the modification time has subsequently changed. This improves performance by avoiding unneeded directory scans. (Fixed by @@ -1432,7 +1472,7 @@ changes, or look through the Subversion logs for all the details. * The :mod:`signal` module no longer re-installs the signal handler unless this is truly necessary, which fixes a bug that could make it impossible to catch the EINTR signal robustly. (Fixed by - Charles-François Natali; :issue:`8354`.) + Charles-Francois Natali; :issue:`8354`.) * New functions: in the :mod:`site` module, three new functions return various site- and user-specific paths. @@ -1466,10 +1506,10 @@ changes, or look through the Subversion logs for all the details. defaults to False; if overridden to be True, new request connections will have the TCP_NODELAY option set to prevent buffering many small sends into a single TCP packet. - The :attr:`~SocketServer.TCPServer.timeout` class attribute can hold + The :attr:`~SocketServer.BaseServer.timeout` class attribute can hold a timeout in seconds that will be applied to the request socket; if - no request is received within that time, :meth:`handle_timeout` - will be called and :meth:`handle_request` will return. + no request is received within that time, :meth:`~SocketServer.BaseServer.handle_timeout` + will be called and :meth:`~SocketServer.BaseServer.handle_request` will return. (Contributed by Kristján Valur Jónsson; :issue:`6192` and :issue:`6267`.) * Updated module: the :mod:`sqlite3` module has been updated to @@ -1479,7 +1519,7 @@ changes, or look through the Subversion logs for all the details. and then call :meth:`~sqlite3.Connection.load_extension` to load a particular shared library. (Updated by Gerhard Häring.) -* The :mod:`ssl` module's :class:`ssl.SSLSocket` objects now support the +* The :mod:`ssl` module's :class:`~ssl.SSLSocket` objects now support the buffer API, which fixed a test suite failure (fix by Antoine Pitrou; :issue:`7133`) and automatically set OpenSSL's :c:macro:`SSL_MODE_AUTO_RETRY`, which will prevent an error @@ -1535,7 +1575,7 @@ changes, or look through the Subversion logs for all the details. on receiving an :const:`EINTR` signal. (Reported by several people; final patch by Gregory P. Smith in :issue:`1068268`.) -* New function: :func:`~symtable.is_declared_global` in the :mod:`symtable` module +* New function: :func:`~symtable.Symbol.is_declared_global` in the :mod:`symtable` module returns true for variables that are explicitly declared to be global, false for ones that are implicitly global. (Contributed by Jeremy Hylton.) @@ -1716,7 +1756,7 @@ Some of the functions in the module are: Makefile and the :file:`pyconfig.h` file. * :func:`~sysconfig.get_config_vars` returns a dictionary containing all of the configuration variables. -* :func:`~sysconfig.getpath` returns the configured path for +* :func:`~sysconfig.get_path` returns the configured path for a particular type of module: the standard library, site-specific modules, platform-specific modules, etc. * :func:`~sysconfig.is_python_build` returns true if you're running a @@ -1778,7 +1818,7 @@ any importable test files named ``test*.py``:: Consult the :mod:`unittest` module documentation for more details. (Developed in :issue:`6001`.) -The :func:`main` function supports some other new options: +The :func:`~unittest.main` function supports some other new options: * :option:`-b` or :option:`--buffer` will buffer the standard output and standard error streams during each test. If the test passes, @@ -1796,7 +1836,7 @@ The :func:`main` function supports some other new options: being tested or the tests being run have defined a signal handler of their own, by noticing that a signal handler was already set and calling it. If this doesn't work for you, there's a - :func:`removeHandler` decorator that can be used to mark tests that + :func:`~unittest.removeHandler` decorator that can be used to mark tests that should have the control-C handling disabled. * :option:`-f` or :option:`--failfast` makes @@ -1923,7 +1963,7 @@ GvR worked on merging them into Python's version of :mod:`unittest`. :func:`unittest.main` now takes an optional ``exit`` argument. If False, :func:`~unittest.main` doesn't call :func:`sys.exit`, allowing -:func:`main` to be used from the interactive interpreter. +:func:`~unittest.main` to be used from the interactive interpreter. (Contributed by J. Pablo Fernández; :issue:`3379`.) :class:`~unittest.TestResult` has new :meth:`~unittest.TestResult.startTestRun` and @@ -2120,7 +2160,7 @@ Changes to Python's build process and to the C API include: :c:macro:`Py_ISSPACE`, :c:macro:`Py_ISUPPER`, :c:macro:`Py_ISXDIGIT`, - and :c:macro:`Py_TOLOWER`, :c:macro:`Py_TOUPPER`. + :c:macro:`Py_TOLOWER`, and :c:macro:`Py_TOUPPER`. All of these functions are analogous to the C standard macros for classifying characters, but ignore the current locale setting, because in @@ -2266,11 +2306,11 @@ Port-Specific Changes: Windows (Contributed by David Cournapeau; :issue:`4365`.) * The :mod:`_winreg` module for accessing the registry now implements - the :func:`CreateKeyEx` and :func:`DeleteKeyEx` functions, extended - versions of previously-supported functions that take several extra - arguments. The :func:`DisableReflectionKey`, - :func:`EnableReflectionKey`, and :func:`QueryReflectionKey` were also - tested and documented. + the :func:`~_winreg.CreateKeyEx` and :func:`~_winreg.DeleteKeyEx` + functions, extended versions of previously-supported functions that + take several extra arguments. The :func:`~_winreg.DisableReflectionKey`, + :func:`~_winreg.EnableReflectionKey`, and :func:`~_winreg.QueryReflectionKey` + were also tested and documented. (Implemented by Brian Curtin: :issue:`7347`.) * The new :c:func:`_beginthreadex` API is used to start threads, and @@ -2329,7 +2369,7 @@ Other Changes and Fixes attributes of the resulting code objects are overwritten when the original filename is obsolete. This can happen if the file has been renamed, moved, or is accessed through different paths. (Patch by - Žiga Seilnacht and Jean-Paul Calderone; :issue:`1180193`.) + Ziga Seilnacht and Jean-Paul Calderone; :issue:`1180193`.) * The :file:`regrtest.py` script now takes a :option:`--randseed=` switch that takes an integer that will be used as the random seed @@ -2387,20 +2427,20 @@ that may require changes to your code: In the standard library: -* Operations with :class:`datetime` instances that resulted in a year +* Operations with :class:`~datetime.datetime` instances that resulted in a year falling outside the supported range didn't always raise :exc:`OverflowError`. Such errors are now checked more carefully and will now raise the exception. (Reported by Mark Leander, patch by Anand B. Pillai and Alexander Belopolsky; :issue:`7150`.) -* When using :class:`Decimal` instances with a string's +* When using :class:`~decimal.Decimal` instances with a string's :meth:`format` method, the default alignment was previously left-alignment. This has been changed to right-alignment, which might change the output of your programs. (Changed by Mark Dickinson; :issue:`6857`.) Comparisons involving a signaling NaN value (or ``sNAN``) now signal - :const:`InvalidOperation` instead of silently returning a true or + :const:`~decimal.InvalidOperation` instead of silently returning a true or false value depending on the comparison operator. Quiet NaN values (or ``NaN``) are now hashable. (Fixed by Mark Dickinson; :issue:`7279`.) @@ -2411,7 +2451,7 @@ In the standard library: or comment (which looks like `<!-- comment -->`). (Patch by Neil Muller; :issue:`2746`.) -* The :meth:`readline` method of :class:`StringIO` objects now does +* The :meth:`~StringIO.StringIO.readline` method of :class:`~StringIO.StringIO` objects now does nothing when a negative length is requested, as other file-like objects do. (:issue:`7348`). @@ -2470,6 +2510,54 @@ For applications that embed Python: .. ====================================================================== +.. _py27-maintenance-enhancements: + +New Features Added to Python 2.7 Maintenance Releases +===================================================== + +New features may be added to Python 2.7 maintenance releases when the +situation genuinely calls for it. Any such additions must go through +the Python Enhancement Proposal process, and make a compelling case for why +they can't be adequately addressed by either adding the new feature solely to +Python 3, or else by publishing it on the Python Package Index. + +In addition to the specific proposals listed below, there is a general +exemption allowing new ``-3`` warnings to be added in any Python 2.7 +maintenance release. + + +PEP 434: IDLE Enhancement Exception for All Branches +---------------------------------------------------- + +:pep:`434` describes a general exemption for changes made to the IDLE +development environment shipped along with Python. This exemption makes it +possible for the IDLE developers to provide a more consistent user +experience across all supported versions of Python 2 and 3. + +For details of any IDLE changes, refer to the NEWS file for the specific +release. + + +PEP 466: Network Security Enhancements for Python 2.7 +----------------------------------------------------- + +:pep:`466` describes a number of network security enhancement proposals +that have been approved for inclusion in Python 2.7 maintenance releases, +with the first of those changes appearing in the Python 2.7.7 release. + +:pep:`466` related features added in Python 2.7.7: + +* :func:`hmac.compare_digest` was added to make a timing attack resistant + comparison operation broadly available to Python 2 applications + (backported by Alex Gaynor in :issue:`21306`) + +* the version of OpenSSL linked with the prebuilt Windows installers + published on python.org was updated to 1.0.1g (contributed by + Zachary Ware in :issue:`21462`) + + +.. ====================================================================== + .. _acks27: Acknowledgements diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst new file mode 100644 index 0000000..177208a --- /dev/null +++ b/Doc/whatsnew/3.4.rst @@ -0,0 +1,2506 @@ +**************************** + What's New In Python 3.4 +**************************** + +:Author: R. David Murray <rdmurray@bitdance.com> (Editor) + +.. Rules for maintenance: + + * Anyone can add text to this document, but the maintainer reserves the + right to rewrite any additions. In particular, for obscure or esoteric + features, the maintainer may reduce any addition to a simple reference to + the new documentation rather than explaining the feature inline. + + * While the maintainer will periodically go through Misc/NEWS + and add changes, it's best not to rely on this. We know from experience + that any changes that aren't in the What's New documentation around the + time of the original release will remain largely unknown to the community + for years, even if they're added later. We also know from experience that + other priorities can arise, and the maintainer will run out of time to do + updates -- in such cases, end users will be much better served by partial + notifications that at least give a hint about new features to + investigate. + + * This is not a complete list of every single change; completeness + is the purpose of Misc/NEWS. The What's New should focus on changes that + are visible to Python *users* and that *require* a feature release (i.e. + most bug fixes should only be recorded in Misc/NEWS) + + * PEPs should not be marked Final until they have an entry in What's New. + A placeholder entry that is just a section header and a link to the PEP + (e.g ":pep:`397` has been implemented") is acceptable. If a PEP has been + implemented and noted in What's New, don't forget to mark it as Final! + + * If you want to draw your new text to the attention of the + maintainer, add 'XXX' to the beginning of the paragraph or + section. + + * It's OK to add just a very brief note about a change. For + example: "The :ref:`~socket.transmogrify()` function was added to the + :mod:`socket` module." The maintainer will research the change and + write the necessary text (if appropriate). The advantage of doing this + is that even if no more descriptive text is ever added, readers will at + least have a notification that the new feature exists and a link to the + relevant documentation. + + * You can comment out your additions if you like, but it's not + necessary (especially when a final release is some months away). + + * Credit the author of a patch or bugfix. Just the name is + sufficient; the e-mail address isn't necessary. + + * It's helpful to add the bug/patch number as a comment: + + The :ref:`~socket.transmogrify()` function was added to the + :mod:`socket` module. (Contributed by P.Y. Developer in :issue:`12345`.) + + This saves the maintainer the effort of going through the Mercurial log + when researching a change. + + * Cross referencing tip: :ref:`mod.attr` will display as ``mod.attr``, + while :ref:`~mod.attr` will display as ``attr``. + +This article explains the new features in Python 3.4, compared to 3.3. +Python 3.4 was released on March 16, 2014. For full details, see the +`changelog <http://docs.python.org/3.4/whatsnew/changelog.html>`_. + + +.. seealso:: + + :pep:`429` -- Python 3.4 Release Schedule + + + +Summary -- Release Highlights +============================= + +.. This section singles out the most important changes in Python 3.4. + Brevity is key. + +New syntax features: + +* No new syntax features were added in Python 3.4. + +Other new features: + +* :ref:`pip should always be available <whatsnew-pep-453>` (:pep:`453`). +* :ref:`Newly created file descriptors are non-inheritable <whatsnew-pep-446>` + (:pep:`446`). +* command line option for :ref:`isolated mode <whatsnew-isolated-mode>` + (:issue:`16499`). +* :ref:`improvements in the handling of codecs <codec-handling-improvements>` + that are not text encodings (multiple issues). +* :ref:`A ModuleSpec Type <whatsnew-pep-451>` for the Import System + (:pep:`451`). (Affects importer authors.) +* The :mod:`marshal` format has been made :ref:`more compact and efficient + <whatsnew-marshal-3>` (:issue:`16475`). + +New library modules: + +* :mod:`asyncio`: :ref:`New provisional API for asynchronous IO + <whatsnew-asyncio>` (:pep:`3156`). +* :mod:`ensurepip`: :ref:`Bootstrapping the pip installer <whatsnew-ensurepip>` + (:pep:`453`). +* :mod:`enum`: :ref:`Support for enumeration types <whatsnew-enum>` + (:pep:`435`). +* :mod:`pathlib`: :ref:`Object-oriented filesystem paths <whatsnew-pathlib>` + (:pep:`428`). +* :mod:`selectors`: :ref:`High-level and efficient I/O multiplexing + <whatsnew-selectors>`, built upon the :mod:`select` module primitives (part + of :pep:`3156`). +* :mod:`statistics`: A basic :ref:`numerically stable statistics library + <whatsnew-statistics>` (:pep:`450`). +* :mod:`tracemalloc`: :ref:`Trace Python memory allocations + <whatsnew-tracemalloc>` (:pep:`454`). + +Significantly improved library modules: + +* :ref:`Single-dispatch generic functions <whatsnew-singledispatch>` in + :mod:`functools` (:pep:`443`). +* New :mod:`pickle` :ref:`protocol 4 <whatsnew-protocol-4>` (:pep:`3154`). +* :mod:`multiprocessing` now has :ref:`an option to avoid using os.fork + on Unix <whatsnew-multiprocessing-no-fork>` (:issue:`8713`). +* :mod:`email` has a new submodule, :mod:`~email.contentmanager`, and + a new :mod:`~email.message.Message` subclass + (:class:`~email.contentmanager.EmailMessage`) that :ref:`simplify MIME + handling <whatsnew_email_contentmanager>` (:issue:`18891`). +* The :mod:`inspect` and :mod:`pydoc` modules are now capable of + correct introspection of a much wider variety of callable objects, + which improves the output of the Python :func:`help` system. +* The :mod:`ipaddress` module API has been declared stable + +Security improvements: + +* :ref:`Secure and interchangeable hash algorithm <whatsnew-pep-456>` + (:pep:`456`). +* :ref:`Make newly created file descriptors non-inheritable <whatsnew-pep-446>` + (:pep:`446`) to avoid leaking file descriptors to child processes. +* New command line option for :ref:`isolated mode <whatsnew-isolated-mode>`, + (:issue:`16499`). +* :mod:`multiprocessing` now has :ref:`an option to avoid using os.fork + on Unix <whatsnew-multiprocessing-no-fork>`. *spawn* and *forkserver* are + more secure because they avoid sharing data with child processes. +* :mod:`multiprocessing` child processes on Windows no longer inherit + all of the parent's inheritable handles, only the necessary ones. +* A new :func:`hashlib.pbkdf2_hmac` function provides + the `PKCS#5 password-based key derivation function 2 + <http://en.wikipedia.org/wiki/PBKDF2>`_. +* :ref:`TLSv1.1 and TLSv1.2 support <whatsnew-tls-11-12>` for :mod:`ssl`. +* :ref:`Retrieving certificates from the Windows system cert store support + <whatsnew34-win-cert-store>` for :mod:`ssl`. +* :ref:`Server-side SNI (Server Name Indication) support + <whatsnew34-sni>` for :mod:`ssl`. +* The :class:`ssl.SSLContext` class has a :ref:`lot of improvements + <whatsnew34-sslcontext>`. +* All modules in the standard library that support SSL now support server + certificate verification, including hostname matching + (:func:`ssl.match_hostname`) and CRLs (Certificate Revocation lists, see + :func:`ssl.SSLContext.load_verify_locations`). + +CPython implementation improvements: + +* :ref:`Safe object finalization <whatsnew-pep-442>` (:pep:`442`). +* Leveraging :pep:`442`, in most cases :ref:`module globals are no longer set + to None during finalization <whatsnew-pep-442>` (:issue:`18214`). +* :ref:`Configurable memory allocators <whatsnew-pep-445>` (:pep:`445`). +* :ref:`Argument Clinic <whatsnew-pep-436>` (:pep:`436`). + +Please read on for a comprehensive list of user-facing changes, including many +other smaller improvements, CPython optimizations, deprecations, and potential +porting issues. + + + +New Features +============ + +.. _whatsnew-pep-453: + +PEP 453: Explicit Bootstrapping of PIP in Python Installations +-------------------------------------------------------------- + +Bootstrapping pip By Default +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The new :mod:`ensurepip` module (defined in :pep:`453`) provides a standard +cross-platform mechanism to bootstrap the pip installer into Python +installations and virtual environments. The version of ``pip`` included +with Python 3.4.0 is ``pip`` 1.5.4, and future 3.4.x maintenance releases +will update the bundled version to the latest version of ``pip`` that is +available at the time of creating the release candidate. + +By default, the commands ``pipX`` and ``pipX.Y`` will be installed on all +platforms (where X.Y stands for the version of the Python installation), +along with the ``pip`` Python package and its dependencies. On Windows and +in virtual environments on all platforms, the unversioned ``pip`` command +will also be installed. On other platforms, the system wide unversioned +``pip`` command typically refers to the separately installed Python 2 +version. + +The :ref:`pyvenv <scripts-pyvenv>` command line utility and the :mod:`venv` +module make use of the :mod:`ensurepip` module to make ``pip`` readily +available in virtual environments. When using the command line utility, +``pip`` is installed by default, while when using the :mod:`venv` module +:ref:`venv-api` installation of ``pip`` must be requested explicitly. + +For CPython :ref:`source builds on POSIX systems <building-python-on-unix>`, +the ``make install`` and ``make altinstall`` commands bootstrap ``pip`` by +default. This behaviour can be controlled through configure options, and +overridden through Makefile options. + +On Windows and Mac OS X, the CPython installers now default to installing +``pip`` along with CPython itself (users may opt out of installing it +during the installation process). Window users will need to opt in to the +automatic ``PATH`` modifications to have ``pip`` available from the command +line by default, otherwise it can still be accessed through the Python +launcher for Windows as ``py -m pip``. + +As `discussed in the PEP`__, platform packagers may choose not to install +these commands by default, as long as, when invoked, they provide clear and +simple directions on how to install them on that platform (usually using +the system package manager). + +__ http://www.python.org/dev/peps/pep-0453/#recommendations-for-downstream-distributors + +.. note:: + + To avoid conflicts between parallel Python 2 and Python 3 installations, + only the versioned ``pip3`` and ``pip3.4`` commands are bootstrapped by + default when ``ensurepip`` is invoked directly - the ``--default-pip`` + option is needed to also request the unversioned ``pip`` command. + ``pyvenv`` and the Windows installer ensure that the unqualified ``pip`` + command is made available in those environments, and ``pip`` can always be + invoked via the ``-m`` switch rather than directly to avoid ambiguity on + systems with multiple Python installations. + + +Documentation Changes +~~~~~~~~~~~~~~~~~~~~~ + +As part of this change, the :ref:`installing-index` and +:ref:`distributing-index` sections of the documentation have been +completely redesigned as short getting started and FAQ documents. Most +packaging documentation has now been moved out to the Python Packaging +Authority maintained `Python Packaging User Guide +<http://packaging.python.org>`__ and the documentation of the individual +projects. + +However, as this migration is currently still incomplete, the legacy +versions of those guides remaining available as :ref:`install-index` +and :ref:`distutils-index`. + +.. seealso:: + + :pep:`453` -- Explicit bootstrapping of pip in Python installations + PEP written by Donald Stufft and Nick Coghlan, implemented by + Donald Stufft, Nick Coghlan, Martin von Löwis and Ned Deily. + + +.. _whatsnew-pep-446: + +PEP 446: Newly Created File Descriptors Are Non-Inheritable +----------------------------------------------------------- + +:pep:`446` makes newly created file descriptors :ref:`non-inheritable +<fd_inheritance>`. In general, this is the behavior an application will +want: when launching a new process, having currently open files also +open in the new process can lead to all sorts of hard to find bugs, +and potentially to security issues. + +However, there are occasions when inheritance is desired. To support +these cases, the following new functions and methods are available: + +* :func:`os.get_inheritable`, :func:`os.set_inheritable` +* :func:`os.get_handle_inheritable`, :func:`os.set_handle_inheritable` +* :meth:`socket.socket.get_inheritable`, :meth:`socket.socket.set_inheritable` + +.. seealso:: + + :pep:`446` -- Make newly created file descriptors non-inheritable + PEP written and implemented by Victor Stinner. + + +.. _codec-handling-improvements: + +Improvements to Codec Handling +------------------------------ + +Since it was first introduced, the :mod:`codecs` module has always been +intended to operate as a type-neutral dynamic encoding and decoding +system. However, its close coupling with the Python text model, especially +the type restricted convenience methods on the builtin :class:`str`, +:class:`bytes` and :class:`bytearray` types, has historically obscured that +fact. + +As a key step in clarifying the situation, the :meth:`codecs.encode` and +:meth:`codecs.decode` convenience functions are now properly documented in +Python 2.7, 3.3 and 3.4. These functions have existed in the :mod:`codecs` +module (and have been covered by the regression test suite) since Python 2.4, +but were previously only discoverable through runtime introspection. + +Unlike the convenience methods on :class:`str`, :class:`bytes` and +:class:`bytearray`, the :mod:`codecs` convenience functions support arbitrary +codecs in both Python 2 and Python 3, rather than being limited to Unicode text +encodings (in Python 3) or ``basestring`` <-> ``basestring`` conversions (in +Python 2). + +In Python 3.4, the interpreter is able to identify the known non-text +encodings provided in the standard library and direct users towards these +general purpose convenience functions when appropriate:: + + >>> b"abcdef".decode("hex") + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + LookupError: 'hex' is not a text encoding; use codecs.decode() to handle arbitrary codecs + + >>> "hello".encode("rot13") + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + LookupError: 'rot13' is not a text encoding; use codecs.encode() to handle arbitrary codecs + + >>> open("foo.txt", encoding="hex") + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + LookupError: 'hex' is not a text encoding; use codecs.open() to handle arbitrary codecs + +In a related change, whenever it is feasible without breaking backwards +compatibility, exceptions raised during encoding and decoding operations +are wrapped in a chained exception of the same type that mentions the +name of the codec responsible for producing the error:: + + >>> import codecs + + >>> codecs.decode(b"abcdefgh", "hex") + Traceback (most recent call last): + File "/usr/lib/python3.4/encodings/hex_codec.py", line 20, in hex_decode + return (binascii.a2b_hex(input), len(input)) + binascii.Error: Non-hexadecimal digit found + + The above exception was the direct cause of the following exception: + + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + binascii.Error: decoding with 'hex' codec failed (Error: Non-hexadecimal digit found) + + >>> codecs.encode("hello", "bz2") + Traceback (most recent call last): + File "/usr/lib/python3.4/encodings/bz2_codec.py", line 17, in bz2_encode + return (bz2.compress(input), len(input)) + File "/usr/lib/python3.4/bz2.py", line 498, in compress + return comp.compress(data) + comp.flush() + TypeError: 'str' does not support the buffer interface + + The above exception was the direct cause of the following exception: + + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + TypeError: encoding with 'bz2' codec failed (TypeError: 'str' does not support the buffer interface) + +Finally, as the examples above show, these improvements have permitted +the restoration of the convenience aliases for the non-Unicode codecs that +were themselves restored in Python 3.2. This means that encoding binary data +to and from its hexadecimal representation (for example) can now be written +as:: + + >>> from codecs import encode, decode + >>> encode(b"hello", "hex") + b'68656c6c6f' + >>> decode(b"68656c6c6f", "hex") + b'hello' + +The binary and text transforms provided in the standard library are detailed +in :ref:`binary-transforms` and :ref:`text-transforms`. + +(Contributed by Nick Coghlan in :issue:`7475`, :issue:`17827`, +:issue:`17828` and :issue:`19619`) + + +.. _whatsnew-pep-451: + +PEP 451: A ModuleSpec Type for the Import System +------------------------------------------------ + +:pep:`451` provides an encapsulation of the information about a module that the +import machinery will use to load it (that is, a module specification). This +helps simplify both the import implementation and several import-related APIs. +The change is also a stepping stone for `several future import-related +improvements`__. + +__ https://mail.python.org/pipermail/python-dev/2013-November/130111.html + +The public-facing changes from the PEP are entirely backward-compatible. +Furthermore, they should be transparent to everyone but importer authors. Key +finder and loader methods have been deprecated, but they will continue working. +New importers should use the new methods described in the PEP. Existing +importers should be updated to implement the new methods. See the +:ref:`deprecated-3.4` section for a list of methods that should be replaced and +their replacements. + + +Other Language Changes +---------------------- + +Some smaller changes made to the core Python language are: + +* Unicode database updated to UCD version 6.3. + +* :func:`min` and :func:`max` now accept a *default* keyword-only argument that + can be used to specify the value they return if the iterable they are + evaluating has no elements. (Contributed by Julian Berman in + :issue:`18111`.) + +* Module objects are now :mod:`weakref`'able. + +* Module ``__file__`` attributes (and related values) should now always + contain absolute paths by default, with the sole exception of + ``__main__.__file__`` when a script has been executed directly using + a relative path (Contributed by Brett Cannon in :issue:`18416`). + +* All the UTF-\* codecs (except UTF-7) now reject surrogates during both + encoding and decoding unless the ``surrogatepass`` error handler is used, + with the exception of the UTF-16 decoder (which accepts valid surrogate pairs) + and the UTF-16 encoder (which produces them while encoding non-BMP characters). + Contributed by Victor Stinner, Kang-Hao (Kenny) Lu and Serhiy Storchaka in + :issue:`12892`. + +* New German EBCDIC :ref:`codec <standard-encodings>` ``cp273``. (Contributed + by Michael Bierenfeld and Andrew Kuchling in :issue:`1097797`.) + +* New Ukrainian :ref:`codec <standard-encodings>` ``cp1125``. (Contributed by + Serhiy Storchaka in :issue:`19668`.) + +* :class:`bytes`.join() and :class:`bytearray`.join() now accept arbitrary + buffer objects as arguments. (Contributed by Antoine Pitrou in + :issue:`15958`.) + +* The :class:`int` constructor now accepts any object that has an ``__index__`` + method for its *base* argument. (Contributed by Mark Dickinson in + :issue:`16772`.) + +* Frame objects now have a :func:`~frame.clear` method that clears all + references to local variables from the frame. (Contributed by Antoine Pitrou + in :issue:`17934`.) + +* :class:`memoryview` is now registered as a :class:`Sequence <collections.abc>`, + and supports the :func:`reversed` builtin. (Contributed by Nick Coghlan + and Claudiu Popa in :issue:`18690` and :issue:`19078`.) + +* Signatures reported by :func:`help` have been modified and improved in + several cases as a result of the introduction of Argument Clinic and other + changes to the :mod:`inspect` and :mod:`pydoc` modules. + +* :meth:`~object.__length_hint__` is now part of the formal language + specification (see :pep:`424`). (Contributed by Armin Ronacher in + :issue:`16148`.) + + +New Modules +=========== + + +.. _whatsnew-asyncio: + +asyncio +------- + +The new :mod:`asyncio` module (defined in :pep:`3156`) provides a standard +pluggable event loop model for Python, providing solid asynchronous IO +support in the standard library, and making it easier for other event loop +implementations to interoperate with the standard library and each other. + +For Python 3.4, this module is considered a :term:`provisional API`. + +.. seealso:: + + :pep:`3156` -- Asynchronous IO Support Rebooted: the "asyncio" Module + PEP written and implementation led by Guido van Rossum. + + +.. _whatsnew-ensurepip: + +ensurepip +--------- + +The new :mod:`ensurepip` module is the primary infrastructure for the +:pep:`453` implementation. In the normal course of events end users will not +need to interact with this module, but it can be used to manually bootstrap +``pip`` if the automated bootstrapping into an installation or virtual +environment was declined. + +:mod:`ensurepip` includes a bundled copy of ``pip``, up-to-date as of the first +release candidate of the release of CPython with which it ships (this applies +to both maintenance releases and feature releases). ``ensurepip`` does not +access the internet. If the installation has Internet access, after +``ensurepip`` is run the bundled ``pip`` can be used to upgrade ``pip`` to a +more recent release than the bundled one. (Note that such an upgraded version +of ``pip`` is considered to be a separately installed package and will not be +removed if Python is uninstalled.) + +The module is named *ensure*\ pip because if called when ``pip`` is already +installed, it does nothing. It also has an ``--upgrade`` option that will +cause it to install the bundled copy of ``pip`` if the existing installed +version of ``pip`` is older than the bundled copy. + + +.. _whatsnew-enum: + +enum +---- + +The new :mod:`enum` module (defined in :pep:`435`) provides a standard +implementation of enumeration types, allowing other modules (such as +:mod:`socket`) to provide more informative error messages and better +debugging support by replacing opaque integer constants with backwards +compatible enumeration values. + +.. seealso:: + + :pep:`435` -- Adding an Enum type to the Python standard library + PEP written by Barry Warsaw, Eli Bendersky and Ethan Furman, + implemented by Ethan Furman. + + +.. _whatsnew-pathlib: + +pathlib +------- + +The new :mod:`pathlib` module offers classes representing filesystem paths +with semantics appropriate for different operating systems. Path classes are +divided between *pure paths*, which provide purely computational operations +without I/O, and *concrete paths*, which inherit from pure paths but also +provide I/O operations. + +For Python 3.4, this module is considered a :term:`provisional API`. + +.. seealso:: + + :pep:`428` -- The pathlib module -- object-oriented filesystem paths + PEP written and implemented by Antoine Pitrou. + + +.. _whatsnew-selectors: + +selectors +--------- + +The new :mod:`selectors` module (created as part of implementing :pep:`3156`) +allows high-level and efficient I/O multiplexing, built upon the +:mod:`select` module primitives. + + +.. _whatsnew-statistics: + +statistics +---------- + +The new :mod:`statistics` module (defined in :pep:`450`) offers some core +statistics functionality directly in the standard library. This module +supports calculation of the mean, median, mode, variance and standard +deviation of a data series. + +.. seealso:: + + :pep:`450` -- Adding A Statistics Module To The Standard Library + PEP written and implemented by Steven D'Aprano + +.. _whatsnew-tracemalloc: + + +tracemalloc +----------- + +The new :mod:`tracemalloc` module (defined in :pep:`454`) is a debug tool to +trace memory blocks allocated by Python. It provides the following information: + +* Trace where an object was allocated +* Statistics on allocated memory blocks per filename and per line number: + total size, number and average size of allocated memory blocks +* Compute the differences between two snapshots to detect memory leaks + +.. seealso:: + + :pep:`454` -- Add a new tracemalloc module to trace Python memory allocations + PEP written and implemented by Victor Stinner + + + +Improved Modules +================ + + +abc +--- + +New function :func:`abc.get_cache_token` can be used to know when to invalidate +caches that are affected by changes in the object graph. (Contributed +by Łukasz Langa in :issue:`16832`.) + +New class :class:`~abc.ABC` has :class:`~abc.ABCMeta` as its meta class. +Using ``ABC`` as a base class has essentially the same effect as specifying +``metaclass=abc.ABCMeta``, but is simpler to type and easier to read. +(Contributed by Bruno Dupuis in :issue:`16049`.) + + +aifc +---- + +The :meth:`~aifc.aifc.getparams` method now returns a namedtuple rather than a +plain tuple. (Contributed by Claudiu Popa in :issue:`17818`.) + +:func:`aifc.open` now supports the context manager protocol: when used in a +:keyword:`with` block, the :meth:`~aifc.aifc.close` method of the returned +object will be called automatically at the end of the block. (Contributed by +Serhiy Storchacha in :issue:`16486`.) + +The :meth:`~aifc.aifc.writeframesraw` and :meth:`~aifc.aifc.writeframes` +methods now accept any :term:`bytes-like object`. (Contributed by Serhiy +Storchaka in :issue:`8311`.) + + +argparse +-------- + +The :class:`~argparse.FileType` class now accepts *encoding* and +*errors* arguments, which are passed through to :func:`open`. (Contributed +by Lucas Maystre in :issue:`11175`.) + + +audioop +------- + +:mod:`audioop` now supports 24-bit samples. (Contributed by Serhiy Storchaka +in :issue:`12866`.) + +New :func:`~audioop.byteswap` function converts big-endian samples to +little-endian and vice versa (Contributed by Serhiy Storchaka in +:issue:`19641`). + +All :mod:`audioop` functions now accept any :term:`bytes-like object`. Strings +are not accepted: they didn't work before, now they raise an error right away. +(Contributed by Serhiy Storchaka in :issue:`16685`.) + + +base64 +------ + +The encoding and decoding functions in :mod:`base64` now accept any +:term:`bytes-like object` in cases where it previously required a +:class:`bytes` or :class:`bytearray` instance. (Contributed by Nick Coghlan in +:issue:`17839`.) + +New functions :func:`~base64.a85encode`, :func:`~base64.a85decode`, +:func:`~base64.b85encode`, and :func:`~base64.b85decode` provide the ability to +encode and decode binary data from and to ``Ascii85`` and the git/mercurial +``Base85`` formats, respectively. The ``a85`` functions have options that can +be used to make them compatible with the variants of the ``Ascii85`` encoding, +including the Adobe variant. (Contributed by Martin Morrison, the Mercurial +project, Serhiy Storchaka, and Antoine Pitrou in :issue:`17618`.) + + +collections +----------- + +The :meth:`.ChainMap.new_child` method now accepts an *m* argument specifying +the child map to add to the chain. This allows an existing mapping and/or a +custom mapping type to be used for the child. (Contributed by Vinay Sajip in +:issue:`16613`.) + + +colorsys +-------- + +The number of digits in the coefficients for the RGB --- YIQ conversions have +been expanded so that they match the FCC NTSC versions. The change in +results should be less than 1% and may better match results found elsewhere. +(Contributed by Brian Landers and Serhiy Storchaka in :issue:`14323`.) + + +contextlib +---------- + +The new :class:`contextlib.suppress` context manager helps to clarify the +intent of code that deliberately suppresses exceptions from a single +statement. (Contributed by Raymond Hettinger in :issue:`15806` and +Zero Piraeus in :issue:`19266`) + +The new :func:`contextlib.redirect_stdout` context manager makes it easier +for utility scripts to handle inflexible APIs that write their output to +:data:`sys.stdout` and don't provide any options to redirect it. Using the +context manager, the :data:`sys.stdout` output can be redirected to any +other stream or, in conjunction with :class:`io.StringIO`, to a string. +The latter can be especially useful, for example, to capture output +from a function that was written to implement a command line interface. +It is recommended only for utility scripts because it affects the +global state of :data:`sys.stdout`. (Contributed by Raymond Hettinger +in :issue:`15805`) + +The :mod:`contextlib` documentation has also been updated to include a +:ref:`discussion <single-use-reusable-and-reentrant-cms>` of the +differences between single use, reusable and reentrant context managers. + + +dbm +--- + +:func:`dbm.open` objects now support the context management protocol. When +used in a :keyword:`with` statement, the ``close`` method of the database +object will be called automatically at the end of the block. (Contributed by +Claudiu Popa and Nick Coghlan in :issue:`19282`.) + + +dis +--- + +Functions :func:`~dis.show_code`, :func:`~dis.dis`, :func:`~dis.distb`, and +:func:`~dis.disassemble` now accept a keyword-only *file* argument that +controls where they write their output. + +The :mod:`dis` module is now built around an :class:`~dis.Instruction` class +that provides object oriented access to the details of each individual bytecode +operation. + +A new method, :func:`~dis.get_instructions`, provides an iterator that emits +the Instruction stream for a given piece of Python code. Thus it is now +possible to write a program that inspects and manipulates a bytecode +object in ways different from those provided by the :mod:`~dis` module +itself. For example:: + + >>> import dis + >>> for instr in dis.get_instructions(lambda x: x + 1): + ... print(instr.opname) + LOAD_FAST + LOAD_CONST + BINARY_ADD + RETURN_VALUE + +The various display tools in the :mod:`dis` module have been rewritten to use +these new components. + +In addition, a new application-friendly class :class:`~dis.Bytecode` provides +an object-oriented API for inspecting bytecode in both in human-readable form +and for iterating over instructions. The :class:`~dis.Bytecode` constructor +takes the same arguments that :func:`~dis.get_instruction` does (plus an +optional *current_offset*), and the resulting object can be iterated to produce +:class:`~dis.Instruction` objects. But it also has a :mod:`~dis.Bytecode.dis` +method, equivalent to calling :mod:`~dis.dis` on the constructor argument, but +returned as a multi-line string:: + + >>> bytecode = dis.Bytecode(lambda x: x +1, current_offset=3) + >>> for instr in bytecode: + ... print('{} ({})'.format(instr.opname, instr.opcode)) + LOAD_FAST (124) + LOAD_CONST (100) + BINARY_ADD (23) + RETURN_VALUE (83) + >>> bytecode.dis().splitlines() # doctest: +NORMALIZE_WHITESPACE + [' 1 0 LOAD_FAST 0 (x)', + ' --> 3 LOAD_CONST 1 (1)', + ' 6 BINARY_ADD', + ' 7 RETURN_VALUE'] + +:class:`~dis.Bytecode` also has a class method, +:meth:`~dis.Bytecode.from_traceback`, that provides the ability to manipulate a +traceback (that is, ``print(Bytecode.from_traceback(tb).dis())`` is equivalent +to ``distb(tb)``). + +(Contributed by Nick Coghlan, Ryan Kelly and Thomas Kluyver in :issue:`11816` +and Claudiu Popa in :issue:`17916`) + +New function :func:`~dis.stack_effect` computes the effect on the Python stack +of a given opcode and argument, information that is not otherwise available. +(Contributed by Larry Hastings in :issue:`19722`.) + + +doctest +------- + +A new :ref:`option flag <doctest-options>`, :data:`~doctest.FAIL_FAST`, halts +test running as soon as the first failure is detected. (Contributed by R. +David Murray and Daniel Urban in :issue:`16522`.) + +The :mod:`doctest` command line interface now uses :mod:`argparse`, and has two +new options, ``-o`` and ``-f``. ``-o`` allows :ref:`doctest options +<doctest-options>` to be specified on the command line, and ``-f`` is a +shorthand for ``-o FAIL_FAST`` (to parallel the similar option supported by the +:mod:`unittest` CLI). (Contributed by R. David Murray in :issue:`11390`.) + +:mod:`doctest` will now find doctests in extension module ``__doc__`` strings. +(Contributed by Zachary Ware in :issue:`3158`.) + + +email +----- + +:meth:`~email.message.Message.as_string` now accepts a *policy* argument to +override the default policy of the message when generating a string +representation of it. This means that ``as_string`` can now be used in more +circumstances, instead of having to create and use a :mod:`~email.generator` in +order to pass formatting parameters to its ``flatten`` method. (Contributed by +R. David Murray in :issue:`18600`.) + +New method :meth:`~email.message.Message.as_bytes` added to produce a bytes +representation of the message in a fashion similar to how ``as_string`` +produces a string representation. It does not accept the *maxheaderlen* +argument, but does accept the *unixfrom* and *policy* arguments. The +:class:`~email.message.Message` :meth:`~email.message.Message.__bytes__` method +calls it, meaning that ``bytes(mymsg)`` will now produce the intuitive +result: a bytes object containing the fully formatted message. (Contributed +by R. David Murray in :issue:`18600`.) + +The :meth:`.Message.set_param` message now accepts a *replace* keyword argument. +When specified, the associated header will be updated without changing +its location in the list of headers. For backward compatibility, the default +is ``False``. (Contributed by R. David Murray in :issue:`18891`.) + + +.. _whatsnew_email_contentmanager: + +A pair of new subclasses of :class:`~email.message.Message` have been added +(:class:`.EmailMessage` and :class:`.MIMEPart`), along with a new sub-module, +:mod:`~email.contentmanager` and a new :mod:`~email.policy` attribute +:attr:`~email.policy.EmailPolicy.content_manager`. All documentation is +currently in the new module, which is being added as part of email's new +:term:`provisional API`. These classes provide a number of new methods that +make extracting content from and inserting content into email messages much +easier. For details, see the :mod:`~email.contentmanager` documentation and +the :ref:`email-contentmanager-api-examples`. These API additions complete the +bulk of the work that was planned as part of the email6 project. The currently +provisional API is scheduled to become final in Python 3.5 (possibly with a few +minor additions in the area of error handling). (Contributed by R. David +Murray in :issue:`18891`.) + + +filecmp +------- + +A new :func:`~filecmp.clear_cache` function provides the ability to clear the +:mod:`filecmp` comparison cache, which uses :func:`os.stat` information to +determine if the file has changed since the last compare. This can be used, +for example, if the file might have been changed and re-checked in less time +than the resolution of a particular filesystem's file modification time field. +(Contributed by Mark Levitt in :issue:`18149`.) + +New module attribute :data:`~filecmp.DEFAULT_IGNORES` provides the list of +directories that are used as the default value for the *ignore* parameter of +the :func:`~filecmp.dircmp` function. (Contributed by Eli Bendersky in +:issue:`15442`.) + + +functools +--------- + +The new :func:`~functools.partialmethod` descriptor brings partial argument +application to descriptors, just as :func:`~functools.partial` provides +for normal callables. The new descriptor also makes it easier to get +arbitrary callables (including :func:`~functools.partial` instances) +to behave like normal instance methods when included in a class definition. +(Contributed by Alon Horev and Nick Coghlan in :issue:`4331`) + +.. _whatsnew-singledispatch: + +The new :func:`~functools.singledispatch` decorator brings support for +single-dispatch generic functions to the Python standard library. Where +object oriented programming focuses on grouping multiple operations on a +common set of data into a class, a generic function focuses on grouping +multiple implementations of an operation that allows it to work with +*different* kinds of data. + +.. seealso:: + + :pep:`443` -- Single-dispatch generic functions + PEP written and implemented by Łukasz Langa. + +:func:`~functools.total_ordering` now supports a return value of +:const:`NotImplemented` from the underlying comparison function. (Contributed +by Katie Miller in :issue:`10042`.) + +A pure-python version of the :func:`~functools.partial` function is now in the +stdlib; in CPython it is overridden by the C accelerated version, but it is +available for other implementations to use. (Contributed by Brian Thorne in +:issue:`12428`.) + + +gc +-- + +New function :func:`~gc.get_stats` returns a list of three per-generation +dictionaries containing the collections statistics since interpreter startup. +(Contributed by Antoine Pitrou in :issue:`16351`.) + + +glob +---- + +A new function :func:`~glob.escape` provides a way to escape special characters +in a filename so that they do not become part of the globbing expansion but are +instead matched literally. (Contributed by Serhiy Storchaka in :issue:`8402`.) + + +hashlib +------- + +A new :func:`hashlib.pbkdf2_hmac` function provides +the `PKCS#5 password-based key derivation function 2 +<http://en.wikipedia.org/wiki/PBKDF2>`_. (Contributed by Christian +Heimes in :issue:`18582`) + +The :attr:`~hashlib.hash.name` attribute of :mod:`hashlib` hash objects is now +a formally supported interface. It has always existed in CPython's +:mod:`hashlib` (although it did not return lower case names for all supported +hashes), but it was not a public interface and so some other Python +implementations have not previously supported it. (Contributed by Jason R. +Coombs in :issue:`18532`.) + + +hmac +---- + +:mod:`hmac` now accepts ``bytearray`` as well as ``bytes`` for the *key* +argument to the :func:`~hmac.new` function, and the *msg* parameter to both the +:func:`~hmac.new` function and the :meth:`~hmac.HMAC.update` method now +accepts any type supported by the :mod:`hashlib` module. (Contributed +by Jonas Borgström in :issue:`18240`.) + +The *digestmod* argument to the :func:`hmac.new` function may now be any hash +digest name recognized by :mod:`hashlib`. In addition, the current behavior in +which the value of *digestmod* defaults to ``MD5`` is deprecated: in a +future version of Python there will be no default value. (Contributed by +Christian Heimes in :issue:`17276`.) + +With the addition of :attr:`~hmac.HMAC.block_size` and :attr:`~hmac.HMAC.name` +attributes (and the formal documentation of the :attr:`~hmac.HMAC.digest_size` +attribute), the :mod:`hmac` module now conforms fully to the :pep:`247` API. +(Contributed by Christian Heimes in :issue:`18775`.) + + +html +---- + +New function :func:`~html.unescape` function converts HTML5 character references to +the corresponding Unicode characters. (Contributed by Ezio Melotti in +:issue:`2927`) + +:class:`~html.parser.HTMLParser` accepts a new keyword argument +*convert_charrefs* that, when ``True``, automatically converts all character +references. For backward-compatibility, its value defaults to ``False``, but +it will change to ``True`` in a future version of Python, so you are invited to +set it explicitly and update your code to use this new feature. (Contributed +by Ezio Melotti in :issue:`13633`) + +The *strict* argument of :class:`~html.parser.HTMLParser` is now deprecated. +(Contributed by Ezio Melotti in :issue:`15114`) + + +http +---- + +:meth:`~http.server.BaseHTTPRequestHandler.send_error` now accepts an +optional additional *explain* parameter which can be used to provide an +extended error description, overriding the hardcoded default if there is one. +This extended error description will be formatted using the +:attr:`~http.server.HTTP.error_message_format` attribute and sent as the body +of the error response. (Contributed by Karl Cow in :issue:`12921`.) + +The :mod:`http.server` :ref:`command line interface <http-server-cli>` now has +a ``-b/--bind`` option that causes the server to listen on a specific address. +(Contributed by Malte Swart in :issue:`17764`.) + + +importlib +--------- + +The :class:`~importlib.abc.InspectLoader` ABC defines a new method, +:meth:`~importlib.abc.InspectLoader.source_to_code` that accepts source +data and a path and returns a code object. The default implementation +is equivalent to ``compile(data, path, 'exec', dont_inherit=True)``. +(Contributed by Eric Snow and Brett Cannon in :issue:`15627`.) + +:class:`~importlib.abc.InspectLoader` also now has a default implementation +for the :meth:`~importlib.abc.InspectLoader.get_code` method. However, +it will normally be desirable to override the default implementation +for performance reasons. (Contributed by Brett Cannon in :issue:`18072`.) + +The :func:`~importlib.reload` function has been moved from :mod:`imp` to +:mod:`importlib` as part of the :mod:`imp` module deprecation. (Contributed by +Berker Peksag in :issue:`18193`.) + +:mod:`importlib.util` now has a :data:`~importlib.util.MAGIC_NUMBER` attribute +providing access to the bytecode version number. This replaces the +:func:`~imp.get_magic` function in the deprecated :mod:`imp` module. +(Contributed by Brett Cannon in :issue:`18192`.) + +New :mod:`importlib.util` functions :func:`~importlib.util.cache_from_source` +and :func:`~importlib.util.source_from_cache` replace the same-named functions +in the deprecated :mod:`imp` module. (Contributed by Brett Cannon in +:issue:`18194`.) + +The :mod:`importlib` bootstrap :class:`.NamespaceLoader` now conforms to +the :class:`.InspectLoader` ABC, which means that ``runpy`` and +``python -m`` can now be used with namespace packages. (Contributed +by Brett Cannon in :issue:`18058`.) + +:mod:`importlib.util` has a new function :func:`~importlib.util.decode_source` +that decodes source from bytes using universal newline processing. This is +useful for implementing :meth:`.InspectLoader.get_source` methods. + +:class:`importlib.machinery.ExtensionFileLoader` now has a +:meth:`~importlib.machinery.ExtensionFileLoader.get_filename` method. This was +inadvertently omitted in the original implementation. (Contributed by Eric +Snow in :issue:`19152`.) + + +inspect +------- + +The :mod:`inspect` module now offers a basic :ref:`command line interface +<inspect-module-cli>` to quickly display source code and other +information for modules, classes and functions. (Contributed by Claudiu Popa +and Nick Coghlan in :issue:`18626`) + +:func:`~inspect.unwrap` makes it easy to unravel wrapper function chains +created by :func:`functools.wraps` (and any other API that sets the +``__wrapped__`` attribute on a wrapper function). (Contributed by +Daniel Urban, Aaron Iles and Nick Coghlan in :issue:`13266`) + +As part of the implementation of the new :mod:`enum` module, the +:mod:`inspect` module now has substantially better support for custom +``__dir__`` methods and dynamic class attributes provided through +metaclasses (Contributed by Ethan Furman in :issue:`18929` and +:issue:`19030`) + +:func:`~inspect.getfullargspec` and :func:`~inspect.getargspec` +now use the :func:`~inspect.signature` API. This allows them to +support a much broader range of callables, including those with +``__signature__`` attributes, those with metadata provided by argument +clinic, :func:`functools.partial` objects and more. Note that, unlike +:func:`~inspect.signature`, these functions still ignore ``__wrapped__`` +attributes, and report the already bound first argument for bound methods, +so it is still necessary to update your code to use +:func:`~inspect.signature` directly if those features are desired. +(Contributed by Yury Selivanov in :issue:`17481`) + +:func:`~inspect.signature` now supports duck types of CPython functions, +which adds support for functions compiled with Cython. (Contributed +by Stefan Behnel and Yury Selivanov in :issue:`17159`) + + +ipaddress +--------- + +:mod:`ipaddress` was added to the standard library in Python 3.3 as a +:term:`provisional API`. With the release of Python 3.4, this qualification +has been removed: :mod:`ipaddress` is now considered a stable API, covered +by the normal standard library requirements to maintain backwards +compatibility. + +A new :attr:`~ipaddress.IPv4Address.is_global` property is ``True`` if +an address is globally routeable. (Contributed by Peter Moody in +:issue:`17400`.) + + +logging +------- + +The :class:`~logging.handlers.TimedRotatingFileHandler` has a new *atTime* +parameter that can be used to specify the time of day when rollover should +happen. (Contributed by Ronald Oussoren in :issue:`9556`.) + +:class:`~logging.handlers.SocketHandler` and +:class:`~logging.handlers.DatagramHandler` now support Unix domain sockets (by +setting *port* to ``None``). (Contributed by Vinay Sajip in commit +ce46195b56a9.) + +:func:`~logging.config.fileConfig` now accepts a +:class:`configparser.RawConfigParser` subclass instance for the *fname* +parameter. This facilitates using a configuration file when logging +configuration is just a part of the overall application configuration, or where +the application modifies the configuration before passing it to +:func:`~logging.config.fileConfig`. (Contributed by Vinay Sajip in +:issue:`16110`.) + +Logging configuration data received from a socket via the +:func:`logging.config.listen` function can now be validated before being +processed by supplying a verification function as the argument to the new +*verify* keyword argument. (Contributed by Vinay Sajip in :issue:`15452`.) + + +.. _whatsnew-marshal-3: + +marshal +------- + +The default :mod:`marshal` version has been bumped to 3. The code implementing +the new version restores the Python2 behavior of recording only one copy of +interned strings and preserving the interning on deserialization, and extends +this "one copy" ability to any object type (including handling recursive +references). This reduces both the size of ``.pyc`` files and the amount of +memory a module occupies in memory when it is loaded from a ``.pyc`` (or +``.pyo``) file. (Contributed by Kristján Valur Jónsson in :issue:`16475`, +with additional speedups by Antoine Pitrou in :issue:`19219`.) + + +mmap +---- + +mmap objects can now be :mod:`weakref`\ ed. (Contributed by Valerie Lambert in +:issue:`4885`.) + + +multiprocessing +--------------- + +.. _whatsnew-multiprocessing-no-fork: + +On Unix two new :ref:`start methods <multiprocessing-start-methods>`, +(``spawn`` and ``forkserver``, have been added for starting processes using +:mod:`multiprocessing`. These make the mixing of processes with threads more +robust, and the ``spawn`` method matches the semantics that multiprocessing has +always used on Windows. New function +:func:`~multiprocessing.get_all_start_methods` reports all start methods +available on the platform, :func:`~multiprocessing.get_start_method` reports +the current start method, and :func:`~multiprocessing.set_start_method` sets +the start method. (Contributed by Richard Oudkerk in :issue:`8713`). + +:mod:`multiprocessing` also now has the concept of a ``context``, which +determines how child processes are created. New function +:func:`~multiprocessing.get_context` returns a context that uses a specified +start method. It has the same API as the :mod:`multiprocessing` module itself, +so you can use it to create :class:`~multiprocessing.pool.Pool`\ s and other +objects that will operate within that context. This allows a framework and an +application or different parts of the same application to use multiprocessing +without interfering with each other. (Contributed by Richard Oudkerk in +:issue:`18999`.) + +Except when using the old *fork* start method, child processes no longer +inherit unneeded handles/file descriptors from their parents (part of +:issue:`8713`). + +:mod:`multiprocessing` now relies on :mod:`runpy` (which implements the +``-m`` switch) to initialise ``__main__`` appropriately in child processes +when using the ``spawn`` or ``forkserver`` start methods. This resolves some +edge cases where combining multiprocessing, the ``-m`` command line switch, +and explicit relative imports could cause obscure failures in child +processes. (Contributed by Nick Coghlan in :issue:`19946`) + + +operator +-------- + +New function :func:`~operator.length_hint` provides an implementation of the +specification for how the :meth:`~object.__length_hint__` special method should +be used, as part of the :pep:`424` formal specification of this language +feature. (Contributed by Armin Ronacher in :issue:`16148`.) + +There is now a pure-python version of the :mod:`operator` module available for +reference and for use by alternate implementations of Python. (Contributed by +Zachary Ware in :issue:`16694`.) + + +os +-- + +There are new functions to get and set the :ref:`inheritable flag +<fd_inheritance>` of a file descriptor (:func:`os.get_inheritable`, +:func:`os.set_inheritable`) or a Windows handle +(:func:`os.get_handle_inheritable`, :func:`os.set_handle_inheritable`). + +New function :func:`~os.cpu_count` reports the number of CPUs available on the +platform on which Python is running (or ``None`` if the count can't be +determined). The :func:`multiprocessing.cpu_count` function is now implemented +in terms of this function). (Contributed by Trent Nelson, Yogesh Chaudhari, +Victor Stinner, and Charles-François Natali in :issue:`17914`.) + +:func:`os.path.samestat` is now available on the Windows platform (and the +:func:`os.path.samefile` implementation is now shared between Unix and +Windows). (Contributed by Brian Curtin in :issue:`11939`.) + +:func:`os.path.ismount` now recognizes volumes mounted below a drive +root on Windows. (Contributed by Tim Golden in :issue:`9035`.) + +:func:`os.open` supports two new flags on platforms that provide them, +:data:`~os.O_PATH` (un-opened file descriptor), and :data:`~os.O_TMPFILE` +(unnamed temporary file; as of 3.4.0 release available only on Linux systems +with a kernel version of 3.11 or newer that have uapi headers). (Contributed +by Christian Heimes in :issue:`18673` and Benjamin Peterson, respectively.) + + +pdb +--- + +:mod:`pdb` has been enhanced to handle generators, :keyword:`yield`, and +``yield from`` in a more useful fashion. This is especially helpful when +debugging :mod:`asyncio` based programs. (Contributed by Andrew Svetlov and +Xavier de Gaye in :issue:`16596`.) + +The ``print`` command has been removed from :mod:`pdb`, restoring access to the +Python :func:`print` function from the pdb command line. Python2's ``pdb`` did +not have a ``print`` command; instead, entering ``print`` executed the +``print`` statement. In Python3 ``print`` was mistakenly made an alias for the +pdb :pdbcmd:`p` command. ``p``, however, prints the ``repr`` of its argument, +not the ``str`` like the Python2 ``print`` command did. Worse, the Python3 +``pdb print`` command shadowed the Python3 ``print`` function, making it +inaccessible at the ``pdb`` prompt. (Contributed by Connor Osborn in +:issue:`18764`.) + + +.. _whatsnew-protocol-4: + +pickle +------ + +:mod:`pickle` now supports (but does not use by default) a new pickle protocol, +protocol 4. This new protocol addresses a number of issues that were present +in previous protocols, such as the serialization of nested classes, very large +strings and containers, and classes whose :meth:`__new__` method takes +keyword-only arguments. It also provides some efficiency improvements. + +.. seealso:: + + :pep:`3154` -- Pickle protocol 4 + PEP written by Antoine Pitrou and implemented by Alexandre Vassalotti. + + +plistlib +-------- + +:mod:`plistlib` now has an API that is similar to the standard pattern for +stdlib serialization protocols, with new :func:`~plistlib.load`, +:func:`~plistlib.dump`, :func:`~plistlib.loads`, and :func:`~plistlib.dumps` +functions. (The older API is now deprecated.) In addition to the already +supported XML plist format (:data:`~plistlib.FMT_XML`), it also now supports +the binary plist format (:data:`~plistlib.FMT_BINARY`). (Contributed by Ronald +Oussoren and others in :issue:`14455`). + + +poplib +------ + +Two new methods have been added to :mod:`poplib`: :meth:`~poplib.POP3.capa`, +which returns the list of capabilities advertised by the POP server, and +:meth:`~poplib.POP3.stls`, which switches a clear-text POP3 session into an +encrypted POP3 session if the POP server supports it. (Contributed by Lorenzo +Catucci in :issue:`4473`.) + + +pprint +------ + +The :mod:`pprint` module's :class:`~pprint.PrettyPrinter` class and its +:func:`~pprint.pformat`, and :func:`~pprint.pprint` functions have a new +option, *compact*, that controls how the output is formatted. Currently +setting *compact* to ``True`` means that sequences will be printed with as many +sequence elements as will fit within *width* on each (indented) line. +(Contributed by Serhiy Storchaka in :issue:`19132`.) + +Long strings are now wrapped using Python's normal line continuation +syntax. (Contributed by Antoine Pitrou in :issue:`17150`). + + +pty +--- + +:func:`pty.spawn` now returns the status value from :func:`os.waitpid` on +the child process, instead of ``None``. (Contributed by Gregory P. Smith.) + + +pydoc +----- + +The :mod:`pydoc` module is now based directly on the :func:`inspect.signature` +introspection API, allowing it to provide signature information for a wider +variety of callable objects. This change also means that ``__wrapped__`` +attributes are now taken into account when displaying help information +(Contributed by Larry Hastings in :issue:`19674`) + +The :mod:`pydoc` module no longer displays the ``self`` parameter for +already bound methods. Instead, it aims to always display the exact current +signature of the supplied callable (Contributed by Larry Hastings in +:issue:`20710`) + +In addition to the changes that have been made to :mod:`pydoc` directly, +its handling of custom ``__dir__`` methods and various descriptor +behaviours has also been improved substantially by the underlying changes in +the :mod:`inspect` module. + +As the :func:`help` builtin is based on :mod:`pydoc`, the above changes also +affect the behaviour of :func:`help`. + + +re +-- + +New :func:`~re.fullmatch` function and :meth:`.regex.fullmatch` method anchor +the pattern at both ends of the string to match. This provides a way to be +explicit about the goal of the match, which avoids a class of subtle bugs where +``$`` characters get lost during code changes or the addition of alternatives +to an existing regular expression. (Contributed by Matthew Barnett in +:issue:`16203`.) + +The repr of :ref:`regex objects <re-objects>` now includes the pattern +and the flags; the repr of :ref:`match objects <match-objects>` now +includes the start, end, and the part of the string that matched. (Contributed +by Hugo Lopes Tavares and Serhiy Storchaka in :issue:`13592` and +:issue:`17087`.) + + +resource +-------- + +New :func:`~resource.prlimit` function, available on Linux platforms with a +kernel version of 2.6.36 or later and glibc of 2.13 or later, provides the +ability to query or set the resource limits for processes other than the one +making the call. (Contributed by Christian Heimes in :issue:`16595`.) + +On Linux kernel version 2.6.36 or later, there are there are also some new +Linux specific constants: :attr:`~resource.RLIMIT_MSGQUEUE`, +:attr:`~resource.RLIMIT_NICE`, :attr:`~resource.RLIMIT_RTPRIO`, +:attr:`~resource.RLIMIT_RTTIME`, and :attr:`~resource.RLIMIT_SIGPENDING`. +(Contributed by Christian Heimes in :issue:`19324`.) + +On FreeBSD version 9 and later, there some new FreeBSD specific constants: +:attr:`~resource.RLIMIT_SBSIZE`, :attr:`~resource.RLIMIT_SWAP`, and +:attr:`~resource.RLIMIT_NPTS`. (Contributed by Claudiu Popa in +:issue:`19343`.) + + +select +------ + +:class:`~select.epoll` objects now support the context management protocol. +When used in a :keyword:`with` statement, the :meth:`~select.epoll.close` +method will be called automatically at the end of the block. (Contributed +by Serhiy Storchaka in :issue:`16488`.) + +:class:`~select.devpoll` objects now have :meth:`~select.devpoll.fileno` and +:meth:`~select.devpoll.close` methods, as well as a new attribute +:attr:`~select.devpoll.closed`. (Contributed by Victor Stinner in +:issue:`18794`.) + + +shelve +------ + +:class:`~shelve.Shelf` instances may now be used in :keyword:`with` statements, +and will be automatically closed at the end of the :keyword:`with` block. +(Contributed by Filip Gruszczyński in :issue:`13896`.) + + +shutil +------ + +:func:`~shutil.copyfile` now raises a specific :exc:`~shutil.Error` subclass, +:exc:`~shutil.SameFileError`, when the source and destination are the same +file, which allows an application to take appropriate action on this specific +error. (Contributed by Atsuo Ishimoto and Hynek Schlawack in +:issue:`1492704`.) + + +smtpd +----- + +The :class:`~smtpd.SMTPServer` and :class:`~smtpd.SMTPChannel` classes now +accept a *map* keyword argument which, if specified, is passed in to +:class:`asynchat.async_chat` as its *map* argument. This allows an application +to avoid affecting the global socket map. (Contributed by Vinay Sajip in +:issue:`11959`.) + + +smtplib +------- + +:exc:`~smtplib.SMTPException` is now a subclass of :exc:`OSError`, which allows +both socket level errors and SMTP protocol level errors to be caught in one +try/except statement by code that only cares whether or not an error occurred. +(Contributed by Ned Jackson Lovely in :issue:`2118`). + + +socket +------ + +The socket module now supports the :data:`~socket.CAN_BCM` protocol on +platforms that support it. (Contributed by Brian Thorne in :issue:`15359`.) + +Socket objects have new methods to get or set their :ref:`inheritable flag +<fd_inheritance>`, :meth:`~socket.socket.get_inheritable` and +:meth:`~socket.socket.set_inheritable`. + +The ``socket.AF_*`` and ``socket.SOCK_*`` constants are now enumeration values +using the new :mod:`enum` module. This allows meaningful names to be printed +during debugging, instead of integer "magic numbers". + +The :data:`~socket.AF_LINK` constant is now available on BSD and OSX. + +:func:`~socket.inet_pton` and :func:`~socket.inet_ntop` are now supported +on Windows. (Contributed by Atsuo Ishimoto in :issue:`7171`.) + + +sqlite3 +------- + +A new boolean parameter to the :func:`~sqlite3.connect` function, *uri*, can be +used to indicate that the *database* parameter is a ``uri`` (see the `SQLite +URI documentation <http://www.sqlite.org/uri.html>`_). (Contributed by poq in +:issue:`13773`.) + + +ssl +--- + +.. _whatsnew-tls-11-12: + +:data:`~ssl.PROTOCOL_TLSv1_1` and :data:`~ssl.PROTOCOL_TLSv1_2` (TLSv1.1 and +TLSv1.2 support) have been added; support for these protocols is only available if +Python is linked with OpenSSL 1.0.1 or later. (Contributed by Michele Orrù and +Antoine Pitrou in :issue:`16692`) + +.. _whatsnew34-sslcontext: + +New function :func:`~ssl.create_default_context` provides a standard way to +obtain an :class:`~ssl.SSLContext` whose settings are intended to be a +reasonable balance between compatibility and security. These settings are +more stringent than the defaults provided by the :class:`~ssl.SSLContext` +constructor, and may be adjusted in the future, without prior deprecation, if +best-practice security requirements change. The new recommended best +practice for using stdlib libraries that support SSL is to use +:func:`~ssl.create_default_context` to obtain an :class:`~ssl.SSLContext` +object, modify it if needed, and then pass it as the *context* argument +of the appropriate stdlib API. (Contributed by Christian Heimes +in :issue:`19689`.) + +:class:`~ssl.SSLContext` method :meth:`~ssl.SSLContext.load_verify_locations` +accepts a new optional argument *cadata*, which can be used to provide PEM or +DER encoded certificates directly via strings or bytes, respectively. +(Contributed by Christian Heimes in :issue:`18138`.) + +New function :func:`~ssl.get_default_verify_paths` returns +a named tuple of the paths and environment variables that the +:meth:`~ssl.SSLContext.set_default_verify_paths` method uses to set +OpenSSL's default ``cafile`` and ``capath``. This can be an aid in +debugging default verification issues. (Contributed by Christian Heimes +in :issue:`18143`.) + +:class:`~ssl.SSLContext` has a new method, +:meth:`~ssl.SSLContext.cert_store_stats`, that reports the number of loaded +``X.509`` certs, ``X.509 CA`` certs, and certificate revocation lists (``crl``\ +s), as well as a :meth:`~ssl.SSLContext.get_ca_certs` method that returns a +list of the loaded ``CA`` certificates. (Contributed by Christian Heimes in +:issue:`18147`.) + +If OpenSSL 0.9.8 or later is available, :class:`~ssl.SSLContext` has an new +attribute :attr:`~ssl.SSLContext.verify_flags` that can be used to control the +certificate verification process by setting it to some combination of the new +constants :data:`~ssl.VERIFY_DEFAULT`, :data:`~ssl.VERIFY_CRL_CHECK_LEAF`, +:data:`~ssl.VERIFY_CRL_CHECK_CHAIN`, or :data:`~ssl.VERIFY_X509_STRICT`. +OpenSSL does not do any CRL verification by default. (Contributed by +Christien Heimes in :issue:`8813`.) + +New :class:`~ssl.SSLContext` method :meth:`~ssl.SSLContext.load_default_certs` +loads a set of default "certificate authority" (CA) certificates from default +locations, which vary according to the platform. It can be used to load both +TLS web server authentication certificates +(``purpose=``:data:`~ssl.Purpose.SERVER_AUTH`) for a client to use to verify a +server, and certificates for a server to use in verifying client certificates +(``purpose=``:data:`~ssl.Purpose.CLIENT_AUTH`). (Contributed by Christian +Heimes in :issue:`19292`.) + +.. _whatsnew34-win-cert-store: + +Two new windows-only functions, :func:`~ssl.enum_certificates` and +:func:`~ssl.enum_crls` provide the ability to retrieve certificates, +certificate information, and CRLs from the Windows cert store. (Contributed +by Christian Heimes in :issue:`17134`.) + +.. _whatsnew34-sni: + +Support for server-side SNI (Server Name Indication) using the new +:meth:`ssl.SSLContext.set_servername_callback` method. +(Contributed by Daniel Black in :issue:`8109`.) + +The dictionary returned by :meth:`.SSLSocket.getpeercert` contains additional +``X509v3`` extension items: ``crlDistributionPoints``, ``calIssuers``, and +``OCSP`` URIs. (Contributed by Christian Heimes in :issue:`18379`.) + + +stat +---- + +The :mod:`stat` module is now backed by a C implementation in :mod:`_stat`. A C +implementation is required as most of the values aren't standardized and +are platform-dependent. (Contributed by Christian Heimes in :issue:`11016`.) + +The module supports new :mod:`~stat.ST_MODE` flags, :mod:`~stat.S_IFDOOR`, +:attr:`~stat.S_IFPORT`, and :attr:`~stat.S_IFWHT`. (Contributed by +Christian Hiemes in :issue:`11016`.) + + +struct +------ + +New function :mod:`~struct.iter_unpack` and a new +:meth:`struct.Struct.iter_unpack` method on compiled formats provide streamed +unpacking of a buffer containing repeated instances of a given format of data. +(Contributed by Antoine Pitrou in :issue:`17804`.) + + +subprocess +---------- + +:func:`~subprocess.check_output` now accepts an *input* argument that can +be used to provide the contents of ``stdin`` for the command that is run. +(Contributed by Zack Weinberg in :issue:`16624`.) + +:func:`~subprocess.getstatus` and :func:`~subprocess.getstatusoutput` now +work on Windows. This change was actually inadvertently made in 3.3.4. +(Contributed by Tim Golden in :issue:`10197`.) + + +sunau +----- + +The :meth:`~sunau.getparams` method now returns a namedtuple rather than a +plain tuple. (Contributed by Claudiu Popa in :issue:`18901`.) + +:meth:`sunau.open` now supports the context manager protocol: when used in a +:keyword:`with` block, the ``close`` method of the returned object will be +called automatically at the end of the block. (Contributed by Serhiy Storchaka +in :issue:`18878`.) + +:meth:`.AU_write.setsampwidth` now supports 24 bit samples, thus adding +support for writing 24 sample using the module. (Contributed by +Serhiy Storchaka in :issue:`19261`.) + +The :meth:`~sunau.AU_write.writeframesraw` and +:meth:`~sunau.AU_write.writeframes` methods now accept any :term:`bytes-like +object`. (Contributed by Serhiy Storchaka in :issue:`8311`.) + + +sys +--- + +New function :func:`sys.getallocatedblocks` returns the current number of +blocks allocated by the interpreter. (In CPython with the default +``--with-pymalloc`` setting, this is allocations made through the +:c:func:`PyObject_Malloc` API.) This can be useful for tracking memory leaks, +especially if automated via a test suite. (Contributed by Antoine Pitrou +in :issue:`13390`.) + +When the Python interpreter starts in :ref:`interactive mode +<tut-interactive>`, it checks for an :data:`~sys.__interactivehook__` attribute +on the :mod:`sys` module. If the attribute exists, its value is called with no +arguments just before interactive mode is started. The check is made after the +:envvar:`PYTHONSTARTUP` file is read, so it can be set there. The :mod:`site` +module :ref:`sets it <rlcompleter-config>` to a function that enables tab +completion and history saving (in :file:`~/.python-history`) if the platform +supports :mod:`readline`. If you do not want this (new) behavior, you can +override it in :envvar:`PYTHONSTARTUP`, :mod:`sitecustomize`, or +:mod:`usercustomize` by deleting this attribute from :mod:`sys` (or setting it +to some other callable). (Contributed by Éric Araujo and Antoine Pitrou in +:issue:`5845`.) + + +tarfile +------- + +The :mod:`tarfile` module now supports a simple :ref:`tarfile-commandline` when +called as a script directly or via ``-m``. This can be used to create and +extract tarfile archives. (Contributed by Berker Peksag in :issue:`13477`.) + + +textwrap +-------- + +The :class:`~textwrap.TextWrapper` class has two new attributes/constructor +arguments: :attr:`~textwrap.TextWrapper.max_lines`, which limits the number of +lines in the output, and :attr:`~textwrap.TextWrapper.placeholder`, which is a +string that will appear at the end of the output if it has been truncated +because of *max_lines*. Building on these capabilities, a new convenience +function :func:`~textwrap.shorten` collapses all of the whitespace in the input +to single spaces and produces a single line of a given *width* that ends with +the *placeholder* (by default, ``[...]``). (Contributed by Antoine Pitrou and +Serhiy Storchaka in :issue:`18585` and :issue:`18725`.) + + +threading +--------- + +The :class:`~threading.Thread` object representing the main thread can be +obtained from the new :func:`~threading.main_thread` function. In normal +conditions this will be the thread from which the Python interpreter was +started. (Contributed by Andrew Svetlov in :issue:`18882`.) + + +traceback +--------- + +A new :func:`traceback.clear_frames` function takes a traceback object +and clears the local variables in all of the frames it references, +reducing the amount of memory consumed. (Contributed by Andrew Kuchling in +:issue:`1565525`). + + +types +----- + +A new :func:`~types.DynamicClassAttribute` descriptor provides a way to define +an attribute that acts normally when looked up through an instance object, but +which is routed to the *class* ``__getattr__`` when looked up through the +class. This allows one to have properties active on a class, and have virtual +attributes on the class with the same name (see :mod:`Enum` for an example). +(Contributed by Ethan Furman in :issue:`19030`.) + + +urllib +------ + +:mod:`urllib.request` now supports ``data:`` URLs via the +:class:`~urllib.request.DataHandler` class. (Contributed by Mathias Panzenböck +in :issue:`16423`.) + +The http method that will be used by a :class:`~urllib.request.Request` class +can now be specified by setting a :class:`~urllib.request.Request.method` +class attribute on the subclass. (Contributed by Jason R Coombs in +:issue:`18978`.) + +:class:`~urllib.request.Request` objects are now reusable: if the +:attr:`~urllib.request.Request.full_url` or :attr:`~urllib.request.Request.data` +attributes are modified, all relevant internal properties are updated. This +means, for example, that it is now possible to use the same +:class:`~urllib.request.Request` object in more than one +:meth:`.OpenerDirector.open` call with different *data* arguments, or to +modify a :class:`~urllib.request.Request`\ 's ``url`` rather than recomputing it +from scratch. There is also a new +:meth:`~urllib.request.Request.remove_header` method that can be used to remove +headers from a :class:`~urllib.request.Request`. (Contributed by Alexey +Kachayev in :issue:`16464`, Daniel Wozniak in :issue:`17485`, and Damien Brecht +and Senthil Kumaran in :issue:`17272`.) + +:class:`~urllib.error.HTTPError` objects now have a +:attr:`~urllib.error.HTTPError.headers` attribute that provides access to the +HTTP response headers associated with the error. (Contributed by +Berker Peksag in :issue:`15701`.) + + +unittest +-------- + +The :class:`~unittest.TestCase` class has a new method, +:meth:`~unittest.TestCase.subTest`, that produces a context manager whose +:keyword:`with` block becomes a "sub-test". This context manager allows a test +method to dynamically generate subtests by, say, calling the ``subTest`` +context manager inside a loop. A single test method can thereby produce an +indefinite number of separately-identified and separately-counted tests, all of +which will run even if one or more of them fail. For example:: + + class NumbersTest(unittest.TestCase): + def test_even(self): + for i in range(6): + with self.subTest(i=i): + self.assertEqual(i % 2, 0) + +will result in six subtests, each identified in the unittest verbose output +with a label consisting of the variable name ``i`` and a particular value for +that variable (``i=0``, ``i=1``, etc). See :ref:`subtests` for the full +version of this example. (Contributed by Antoine Pitrou in :issue:`16997`.) + +:func:`unittest.main` now accepts an iterable of test names for +*defaultTest*, where previously it only accepted a single test name as a +string. (Contributed by Jyrki Pulliainen in :issue:`15132`.) + +If :class:`~unittest.SkipTest` is raised during test discovery (that is, at the +module level in the test file), it is now reported as a skip instead of an +error. (Contributed by Zach Ware in :issue:`16935`.) + +:meth:`~unittest.TestLoader.discover` now sorts the discovered files to provide +consistent test ordering. (Contributed by Martin Melin and Jeff Ramnani in +:issue:`16709`.) + +:class:`~unittest.TestSuite` now drops references to tests as soon as the test +has been run, if the test is successful. On Python interpreters that do +garbage collection, this allows the tests to be garbage collected if nothing +else is holding a reference to the test. It is possible to override this +behavior by creating a :class:`~unittest.TestSuite` subclass that defines a +custom ``_removeTestAtIndex`` method. (Contributed by Tom Wardill, Matt +McClure, and Andrew Svetlov in :issue:`11798`.) + +A new test assertion context-manager, :meth:`~unittest.TestCase.assertLogs`, +will ensure that a given block of code emits a log message using the +:mod:`logging` module. By default the message can come from any logger and +have a priority of ``INFO`` or higher, but both the logger name and an +alternative minimum logging level may be specified. The object returned by the +context manager can be queried for the :class:`~logging.LogRecord`\ s and/or +formatted messages that were logged. (Contributed by Antoine Pitrou in +:issue:`18937`.) + +Test discovery now works with namespace packages (Contributed by Claudiu Popa +in :issue:`17457`.) + +:mod:`unittest.mock` objects now inspect their specification signatures when +matching calls, which means an argument can now be matched by either position +or name, instead of only by position. (Contributed by Antoine Pitrou in +:issue:`17015`.) + +:func:`~mock.mock_open` objects now have ``readline`` and ``readlines`` +methods. (Contributed by Toshio Kuratomi in :issue:`17467`.) + + +venv +---- + +:mod:`venv` now includes activation scripts for the ``csh`` and ``fish`` +shells (Contributed by Andrew Svetlov in :issue:`15417`.) + +:class:`~venv.EnvBuilder` and the :func:`~venv.create` convenience function +take a new keyword argument *with_pip*, which defaults to ``False``, that +controls whether or not :class:`~venv.EnvBuilder` ensures that ``pip`` is +installed in the virtual environment. (Contributed by Nick Coghlan in +:issue:`19552` as part of the :pep:`453` implementation.) + + +wave +---- + +The :meth:`~wave.getparams` method now returns a namedtuple rather than a +plain tuple. (Contributed by Claudiu Popa in :issue:`17487`.) + +:meth:`wave.open` now supports the context manager protocol. (Contributed +by Claudiu Popa in :issue:`17616`.) + +:mod:`wave` can now :ref:`write output to unseekable files +<wave-write-objects>`. (Contributed by David Jones, Guilherme Polo, and Serhiy +Storchaka in :issue:`5202`.) + +The :meth:`~wave.Wave_write.writeframesraw` and +:meth:`~wave.Wave_write.writeframes` methods now accept any :term:`bytes-like +object`. (Contributed by Serhiy Storchaka in :issue:`8311`.) + + +weakref +------- + +New :class:`~weakref.WeakMethod` class simulates weak references to bound +methods. (Contributed by Antoine Pitrou in :issue:`14631`.) + +New :class:`~weakref.finalize` class makes it possible to register a callback +to be invoked when an object is garbage collected, without needing to +carefully manage the lifecycle of the weak reference itself. (Contributed by +Richard Oudkerk in :issue:`15528`) + +The callback, if any, associated with a :class:`~weakref.ref` is now +exposed via the :attr:`~weakref.ref.__callback__` attribute. (Contributed +by Mark Dickinson in :issue:`17643`.) + + +xml.etree +--------- + +A new parser, :class:`~xml.etree.ElementTree.XMLPullParser`, allows a +non-blocking applications to parse XML documents. An example can be +seen at :ref:`elementtree-pull-parsing`. (Contributed by Antoine +Pitrou in :issue:`17741`.) + +The :mod:`xml.etree.ElementTree` :func:`~xml.etree.ElementTree.tostring` and +:func:`~xml.etree.ElementTree.tostringlist` functions, and the +:class:`~xml.etree.ElementTree.ElementTree` +:meth:`~xml.etree.ElementTree.ElementTree.write` method, now have a +*short_empty_elements* :ref:`keyword-only parameter <keyword-only_parameter>` +providing control over whether elements with no content are written in +abbreviated (``<tag />``) or expanded (``<tag></tag>``) form. (Contributed by +Ariel Poliak and Serhiy Storchaka in :issue:`14377`.) + + +zipfile +------- + +The :meth:`~zipfile.PyZipFile.writepy` method of the +:class:`~zipfile.PyZipFile` class has a new *filterfunc* option that can be +used to control which directories and files are added to the archive. For +example, this could be used to exclude test files from the archive. +(Contributed by Christian Tismer in :issue:`19274`.) + +The *allowZip64* parameter to :class:`~zipfile.ZipFile` and +:class:`~zipfile.PyZipfile` is now ``True`` by default. (Contributed by +William Mallard in :issue:`17201`.) + + + +CPython Implementation Changes +============================== + + +.. _whatsnew-pep-445: + +PEP 445: Customization of CPython Memory Allocators +--------------------------------------------------- + +:pep:`445` adds new C level interfaces to customize memory allocation in +the CPython interpreter. + +.. seealso:: + + :pep:`445` -- Add new APIs to customize Python memory allocators + PEP written and implemented by Victor Stinner. + + +.. _whatsnew-pep-442: + +PEP 442: Safe Object Finalization +--------------------------------- + +:pep:`442` removes the current limitations and quirks of object finalization +in CPython. With it, objects with :meth:`__del__` methods, as well as +generators with :keyword:`finally` clauses, can be finalized when they are +part of a reference cycle. + +As part of this change, module globals are no longer forcibly set to +:const:`None` during interpreter shutdown in most cases, instead relying +on the normal operation of the cyclic garbage collector. This avoids a +whole class of interpreter-shutdown-time errors, usually involving +``__del__`` methods, that have plagued Python since the cyclic GC +was first introduced. + +.. seealso:: + + :pep:`442` -- Safe object finalization + PEP written and implemented by Antoine Pitrou. + + +.. _whatsnew-pep-456: + +PEP 456: Secure and Interchangeable Hash Algorithm +-------------------------------------------------- + +:pep:`456` follows up on earlier security fix work done on Python's hash +algorithm to address certain DOS attacks to which public facing APIs backed by +dictionary lookups may be subject. (See :issue:`14621` for the start of the +current round of improvements.) The PEP unifies CPython's hash code to make it +easier for a packager to substitute a different hash algorithm, and switches +Python's default implementation to a SipHash implementation on platforms that +have a 64 bit data type. Any performance differences in comparison with the +older FNV algorithm are trivial. + +The PEP adds additional fields to the :attr:`sys.hash_info` struct sequence to +describe the hash algorithm in use by the currently executing binary. Otherwise, +the PEP does not alter any existing CPython APIs. + + +.. _whatsnew-pep-436: + +PEP 436: Argument Clinic +------------------------ + +"Argument Clinic" (:pep:`436`) is now part of the CPython build process +and can be used to simplify the process of defining and maintaining +accurate signatures for builtins and standard library extension modules +implemented in C. + +Some standard library extension modules have been converted to use Argument +Clinic in Python 3.4, and :mod:`pydoc` and :mod:`inspect` have been updated +accordingly. + +It is expected that signature metadata for programmatic introspection will +be added to additional callables implemented in C as part of Python 3.4 +maintenance releases. + +.. note:: + The Argument Clinic PEP is not fully up to date with the state of the + implementation. This has been deemed acceptable by the release manager + and core development team in this case, as Argument Clinic will not + be made available as a public API for third party use in Python 3.4. + +.. seealso:: + + :pep:`436` -- The Argument Clinic DSL + PEP written and implemented by Larry Hastings. + + +Other Build and C API Changes +----------------------------- + +* The new :c:func:`PyType_GetSlot` function has been added to the stable ABI, + allowing retrieval of function pointers from named type slots when using + the limited API. (Contributed by Martin von Löwis in :issue:`17162`) + +* The new :c:func:`Py_SetStandardStreamEncoding` pre-initialization API + allows applications embedding the CPython interpreter to reliably force + a particular encoding and error handler for the standard streams + (Contributed by Bastien Montagne and Nick Coghlan in :issue:`16129`) + +* Most Python C APIs that don't mutate string arguments are now correctly + marked as accepting ``const char *`` rather than ``char *`` (Contributed + by Serhiy Storchaka in :issue:`1772673`). + +* A new shell version of ``python-config`` can be used even when a python + interpreter is not available (for example, in cross compilation scenarios). + +* :c:func:`PyUnicode_FromFormat` now supports width and precision + specifications for ``%s``, ``%A``, ``%U``, ``%V``, ``%S``, and ``%R``. + (Contributed by Ysj Ray and Victor Stinner in :issue:`7330`.) + +* New function :c:func:`PyStructSequence_InitType2` supplements the + existing :c:func:`PyStructSequence_InitType` function. The difference + is that it returns ``0`` on success and ``-1`` on failure. + +* The CPython source can now be compiled using the address sanity checking + features of recent versions of GCC and clang: the false alarms in the small + object allocator have been silenced. (Contributed by Dhiru Kholia in + :issue:`18596`.) + +* The Windows build now uses `Address Space Layout Randomization + <http://en.wikipedia.org/wiki/ASLR>`_ and `Data Execution Prevention + <http://en.wikipedia.org/wiki/Data_Execution_Prevention>`_. (Contributed by + Christian Heimes in :issue:`16632`.) + +* New function :c:func:`PyObject_LengthHint` is the C API equivalent + of :func:`operator.length_hint`. (Contributed by Armin Ronacher in + :issue:`16148`.) + + +.. _other-improvements-3.4: + +Other Improvements +------------------ + +.. _whatsnew-isolated-mode: + +* The :ref:`python <using-on-cmdline>` command has a new :ref:`option + <using-on-misc-options>`, ``-I``, which causes it to run in "isolated mode", + which means that :data:`sys.path` contains neither the script's directory nor + the user's ``site-packages`` directory, and all :envvar:`PYTHON*` environment + variables are ignored (it implies both ``-s`` and ``-E``). Other + restrictions may also be applied in the future, with the goal being to + isolate the execution of a script from the user's environment. This is + appropriate, for example, when Python is used to run a system script. On + most POSIX systems it can and should be used in the ``#!`` line of system + scripts. (Contributed by Christian Heimes in :issue:`16499`.) + +* Tab-completion is now enabled by default in the interactive interpreter + on systems that support :mod:`readline`. History is also enabled by default, + and is written to (and read from) the file :file:`~/.python-history`. + (Contributed by Antoine Pitrou and Éric Araujo in :issue:`5845`.) + +* Invoking the Python interpreter with ``--version`` now outputs the version to + standard output instead of standard error (:issue:`18338`). Similar changes + were made to :mod:`argparse` (:issue:`18920`) and other modules that have + script-like invocation capabilities (:issue:`18922`). + +* The CPython Windows installer now adds ``.py`` to the :envvar:`PATHEXT` + variable when extensions are registered, allowing users to run a python + script at the windows command prompt by just typing its name without the + ``.py`` extension. (Contributed by Paul Moore in :issue:`18569`.) + +* A new ``make`` target `coverage-report + <http://docs.python.org/devguide/coverage.html#measuring-coverage-of-c-code-with-gcov-and-lcov>`_ + will build python, run the test suite, and generate an HTML coverage report + for the C codebase using ``gcov`` and `lcov + <http://ltp.sourceforge.net/coverage/lcov.php>`_. + +* The ``-R`` option to the :ref:`python regression test suite <regrtest>` now + also checks for memory allocation leaks, using + :func:`sys.getallocatedblocks()`. (Contributed by Antoine Pitrou in + :issue:`13390`). + +* ``python -m`` now works with namespace packages. + +* The :mod:`stat` module is now implemented in C, which means it gets the + values for its constants from the C header files, instead of having the + values hard-coded in the python module as was previously the case. + +* Loading multiple python modules from a single OS module (``.so``, ``.dll``) + now works correctly (previously it silently returned the first python + module in the file). (Contributed by Václav Šmilauer in :issue:`16421`.) + +* A new opcode, :opcode:`LOAD_CLASSDEREF`, has been added to fix a bug in the + loading of free variables in class bodies that could be triggered by certain + uses of :ref:`__prepare__ <prepare>`. (Contributed by Benjamin Peterson in + :issue:`17853`.) + +* A number of MemoryError-related crashes were identified and fixed by Victor + Stinner using his :pep:`445`-based ``pyfailmalloc`` tool (:issue:`18408`, + :issue:`18520`). + +* The :ref:`pyvenv <scripts-pyvenv>` command now accepts a ``--copies`` option + to use copies rather than symlinks even on systems where symlinks are the + default. (Contributed by Vinay Sajip in :issue:`18807`.) + +* The :ref:`pyvenv <scripts-pyvenv>` command also accepts a ``--without-pip`` + option to suppress the otherwise-automatic bootstrapping of pip into + the virtual environment. (Contributed by Nick Coghlan in :issue:`19552` + as part of the :pep:`453` implementation.) + +* The encoding name is now optional in the value set for the + :envvar:`PYTHONIOENCODING` environment variable. This makes it possible to + set just the error handler, without changing the default encoding. + (Contributed by Serhiy Storchaka in :issue:`18818`.) + +* The :mod:`bz2`, :mod:`lzma`, and :mod:`gzip` module ``open`` functions now + support ``x`` (exclusive creation) mode. (Contributed by Tim Heaney and + Vajrasky Kok in :issue:`19201`, :issue:`19222`, and :issue:`19223`.) + + +Significant Optimizations +------------------------- + +* The UTF-32 decoder is now 3x to 4x faster. (Contributed by Serhiy Storchaka + in :issue:`14625`.) + +* The cost of hash collisions for sets is now reduced. Each hash table + probe now checks a series of consecutive, adjacent key/hash pairs before + continuing to make random probes through the hash table. This exploits + cache locality to make collision resolution less expensive. + The collision resolution scheme can be described as a hybrid of linear + probing and open addressing. The number of additional linear probes + defaults to nine. This can be changed at compile-time by defining + LINEAR_PROBES to be any value. Set LINEAR_PROBES=0 to turn-off + linear probing entirely. (Contributed by Raymond Hettinger in + :issue:`18771`.) + +* The interpreter starts about 30% faster. A couple of measures lead to the + speedup. The interpreter loads fewer modules on startup, e.g. the :mod:`re`, + :mod:`collections` and :mod:`locale` modules and their dependencies are no + longer imported by default. The marshal module has been improved to load + compiled Python code faster. (Contributed by Antoine Pitrou, Christian + Heimes and Victor Stinner in :issue:`19219`, :issue:`19218`, :issue:`19209`, + :issue:`19205` and :issue:`9548`) + +* :class:`bz2.BZ2File` is now as fast or faster than the Python2 version for + most cases. :class:`lzma.LZMAFile` has also been optimized. (Contributed by + Serhiy Storchaka and Nadeem Vawda in :issue:`16034`.) + +* :func:`random.getrandbits` is 20%-40% faster for small integers (the most + common use case). (Contributed by Serhiy Storchaka in :issue:`16674`). + +* By taking advantage of the new storage format for strings, pickling of + strings is now significantly faster. (Contributed by Victor Stinner and + Antoine Pitrou in :issue:`15596`.) + +* A performance issue in :meth:`io.FileIO.readall` has been solved. This + particularly affects Windows, and significantly speeds up the case of piping + significant amounts of data through :mod:`subprocess`. (Contributed + by Richard Oudkerk in :issue:`15758`.) + +* :func:`html.escape` is now 10x faster. (Contributed by Matt Bryant in + :issue:`18020`.) + +* On Windows, the native ``VirtualAlloc`` is now used instead of the CRT + ``malloc`` in ``obmalloc``. Artificial benchmarks show about a 3% memory + savings. + +* :func:`os.urandom` now uses a lazily-opened persistent file descriptor + so as to avoid using many file descriptors when run in parallel from + multiple threads. (Contributed by Antoine Pitrou in :issue:`18756`.) + + +.. _deprecated-3.4: + +Deprecated +========== + +This section covers various APIs and other features that have been deprecated +in Python 3.4, and will be removed in Python 3.5 or later. In most (but not +all) cases, using the deprecated APIs will produce a :exc:`DeprecationWarning` +when the interpreter is run with deprecation warnings enabled (for example, by +using ``-Wd``). + + +Deprecations in the Python API +------------------------------ + +* As mentioned in :ref:`whatsnew-pep-451`, a number of :mod:`importilb` + methods and functions are deprecated: :meth:`importlib.find_loader` is + replaced by :func:`importlib.util.find_spec`; + :meth:`importlib.machinery.PathFinder.find_module` is replaced by + :meth:`importlib.machinery.PathFinder.find_spec`; + :meth:`importlib.abc.MetaPathFinder.find_module` is replaced by + :meth:`importlib.abc.MetaPathFinder.find_spec`; + :meth:`importlib.abc.PathEntryFinder.find_loader` and + :meth:`~importlib.abc.PathEntryFinder.find_module` are replaced by + :meth:`importlib.abc.PathEntryFinder.find_spec`; all of the ``xxxLoader`` ABC + ``load_module`` methods (:meth:`importlib.abc.Loader.load_module`, + :meth:`importlib.abc.InspectLoader.load_module`, + :meth:`importlib.abc.FileLoader.load_module`, + :meth:`importlib.abc.SourceLoader.load_module`) should no longer be + implemented, instead loaders should implement an + ``exec_module`` method + (:meth:`importlib.abc.Loader.exec_module`, + :meth:`importlib.abc.InspectLoader.exec_module` + :meth:`importlib.abc.SourceLoader.exec_module`) and let the import system + take care of the rest; and + :meth:`importlib.abc.Loader.module_repr`, + :meth:`importlib.util.module_for_loader`, :meth:`importlib.util.set_loader`, + and :meth:`importlib.util.set_package` are no longer needed because their + functions are now handled automatically by the import system. + +* The :mod:`imp` module is pending deprecation. To keep compatibility with + Python 2/3 code bases, the module's removal is currently not scheduled. + +* The :mod:`formatter` module is pending deprecation and is slated for removal + in Python 3.6. + +* ``MD5`` as the default *digestmod* for the :func:`hmac.new` function is + deprecated. Python 3.6 will require an explicit digest name or constructor as + *digestmod* argument. + +* The internal ``Netrc`` class in the :mod:`ftplib` module has been documented + as deprecated in its docstring for quite some time. It now emits a + :exc:`DeprecationWarning` and will be removed completely in Python 3.5. + +* The undocumented *endtime* argument to :meth:`subprocess.Popen.wait` should + not have been exposed and is hopefully not in use; it is deprecated and + will mostly likely be removed in Python 3.5. + +* The *strict* argument of :class:`~html.parser.HTMLParser` is deprecated. + +* The :mod:`plistlib` :func:`~plistlib.readPlist`, + :func:`~plistlib.writePlist`, :func:`~plistlib.readPlistFromBytes`, and + :func:`~plistlib.writePlistToBytes` functions are deprecated in favor of the + corresponding new functions :func:`~plistlib.load`, :func:`~plistlib.dump`, + :func:`~plistlib.loads`, and :func:`~plistlib.dumps`. :func:`~plistlib.Data` + is deprecated in favor of just using the :class:`bytes` constructor. + +* The :mod:`sysconfig` key ``SO`` is deprecated, it has been replaced by + ``EXT_SUFFIX``. + +* The ``U`` mode accepted by various ``open`` functions is deprecated. + In Python3 it does not do anything useful, and should be replaced by + appropriate uses of :class:`io.TextIOWrapper` (if needed) and its *newline* + argument. + +* The *parser* argument of :func:`xml.etree.ElementTree.iterparse` has + been deprecated, as has the *html* argument of + :func:`~xml.etree.ElementTree.XMLParser`. To prepare for the removal of the + latter, all arguments to ``XMLParser`` should be passed by keyword. + + +Deprecated Features +------------------- + +* Running :ref:`idle` with the ``-n`` flag (no subprocess) is deprecated. + However, the feature will not be removed until :issue:`18823` is resolved. + +* The site module adding a "site-python" directory to sys.path, if it + exists, is deprecated (:issue:`19375`). + + + +Removed +======= + + +Operating Systems No Longer Supported +------------------------------------- + +Support for the following operating systems has been removed from the source +and build tools: + +* OS/2 (:issue:`16135`). +* Windows 2000 (changeset e52df05b496a). +* Windows systems where ``COMSPEC`` points to ``command.com`` (:issue:`14470`). +* VMS (:issue:`16136`). + + +API and Feature Removals +------------------------ + +The following obsolete and previously deprecated APIs and features have been +removed: + +* The unmaintained ``Misc/TextMate`` and ``Misc/vim`` directories have been + removed (see the `devguide <http://docs.python.org/devguide>`_ + for suggestions on what to use instead). + +* The ``SO`` makefile macro is removed (it was replaced by the + ``SHLIB_SUFFIX`` and ``EXT_SUFFIX`` macros) (:issue:`16754`). + +* The ``PyThreadState.tick_counter`` field has been removed; its value has + been meaningless since Python 3.2, when the "new GIL" was introduced + (:issue:`19199`). + +* ``PyLoader`` and ``PyPycLoader`` have been removed from :mod:`importlib`. + (Contributed by Taras Lyapun in :issue:`15641`.) + +* The *strict* argument to :class:`~http.client.HTTPConnection` and + :class:`~http.client.HTTPSConnection` has been removed. HTTP 0.9-style + "Simple Responses" are no longer supported. + +* The deprecated :mod:`urllib.request.Request` getter and setter methods + ``add_data``, ``has_data``, ``get_data``, ``get_type``, ``get_host``, + ``get_selector``, ``set_proxy``, ``get_origin_req_host``, and + ``is_unverifiable`` have been removed (use direct attribute access instead). + +* Support for loading the deprecated ``TYPE_INT64`` has been removed from + :mod:`marshal`. (Contributed by Dan Riti in :issue:`15480`.) + +* :class:`inspect.Signature`: positional-only parameters are now required + to have a valid name. + +* :meth:`object.__format__` no longer accepts non-empty format strings, it now + raises a :exc:`TypeError` instead. Using a non-empty string has been + deprecated since Python 3.2. This change has been made to prevent a + situation where previously working (but incorrect) code would start failing + if an object gained a __format__ method, which means that your code may now + raise a :exc:`TypeError` if you are using an ``'s'`` format code with objects + that do not have a __format__ method that handles it. See :issue:`7994` for + background. + +* :meth:`difflib.SequenceMatcher.isbjunk` and + :meth:`difflib.SequenceMatcher.isbpopular` were deprecated in 3.2, and have + now been removed: use ``x in sm.bjunk`` and + ``x in sm.bpopular``, where *sm* is a :class:`~difflib.SequenceMatcher` object + (:issue:`13248`). + + +Code Cleanups +------------- + +* The unused and undocumented internal ``Scanner`` class has been removed from + the :mod:`pydoc` module. + +* The private and effectively unused ``_gestalt`` module has been removed, + along with the private :mod:`platform` functions ``_mac_ver_lookup``, + ``_mac_ver_gstalt``, and ``_bcd2str``, which would only have ever been called + on badly broken OSX systems (see :issue:`18393`). + +* The hardcoded copies of certain :mod:`stat` constants that were included in + the :mod:`tarfile` module namespace have been removed. + + + +Porting to Python 3.4 +===================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + + +Changes in 'python' Command Behavior +------------------------------------ + +* In a posix shell, setting the :envvar:`PATH` environment variable to + an empty value is equivalent to not setting it at all. However, setting + :envvar:`PYTHONPATH` to an empty value was *not* equivalent to not setting it + at all: setting :envvar:`PYTHONPATH` to an empty value was equivalent to + setting it to ``.``, which leads to confusion when reasoning by analogy to + how :envvar:`PATH` works. The behavior now conforms to the posix convention + for :envvar:`PATH`. + +* The [X refs, Y blocks] output of a debug (``--with-pydebug``) build of the + CPython interpreter is now off by default. It can be re-enabled using the + ``-X showrefcount`` option. (Contributed by Ezio Melotti in :issue:`17323`.) + +* The python command and most stdlib scripts (as well as :mod:`argparse`) now + output ``--version`` information to ``stdout`` instead of ``stderr`` (for + issue list see :ref:`other-improvements-3.4` above). + + +Changes in the Python API +------------------------- + +* The ABCs defined in :mod:`importlib.abc` now either raise the appropriate + exception or return a default value instead of raising + :exc:`NotImplementedError` blindly. This will only affect code calling + :func:`super` and falling through all the way to the ABCs. For compatibility, + catch both :exc:`NotImplementedError` or the appropriate exception as needed. + +* The module type now initializes the :attr:`__package__` and :attr:`__loader__` + attributes to ``None`` by default. To determine if these attributes were set + in a backwards-compatible fashion, use e.g. + ``getattr(module, '__loader__', None) is not None``. (:issue:`17115`.) + +* :meth:`importlib.util.module_for_loader` now sets ``__loader__`` and + ``__package__`` unconditionally to properly support reloading. If this is not + desired then you will need to set these attributes manually. You can use + :func:`importlib.util.module_to_load` for module management. + +* Import now resets relevant attributes (e.g. ``__name__``, ``__loader__``, + ``__package__``, ``__file__``, ``__cached__``) unconditionally when reloading. + Note that this restores a pre-3.3 behavior in that it means a module is + re-found when re-loaded (:issue:`19413`). + +* Frozen packages no longer set ``__path__`` to a list containing the package + name, they now set it to an empty list. The previous behavior could cause + the import system to do the wrong thing on submodule imports if there was + also a directory with the same name as the frozen package. The correct way + to determine if a module is a package or not is to use ``hasattr(module, + '__path__')`` (:issue:`18065`). + +* Frozen modules no longer define a ``__file__`` attribute. It's semantically + incorrect for frozen modules to set the attribute as they are not loaded from + any explicit location. If you must know that a module comes from frozen code + then you can see if the module's ``__spec__.location`` is set to ``'frozen'``, + check if the loader is a subclass of + :class:`importlib.machinery.FrozenImporter`, + or if Python 2 compatibility is necessary you can use :func:`imp.is_frozen`. + +* :func:`py_compile.compile` now raises :exc:`FileExistsError` if the file path + it would write to is a symlink or a non-regular file. This is to act as a + warning that import will overwrite those files with a regular file regardless + of what type of file path they were originally. + +* :meth:`importlib.abc.SourceLoader.get_source` no longer raises + :exc:`ImportError` when the source code being loaded triggers a + :exc:`SyntaxError` or :exc:`UnicodeDecodeError`. As :exc:`ImportError` is + meant to be raised only when source code cannot be found but it should, it was + felt to be over-reaching/overloading of that meaning when the source code is + found but improperly structured. If you were catching ImportError before and + wish to continue to ignore syntax or decoding issues, catch all three + exceptions now. + +* :func:`functools.update_wrapper` and :func:`functools.wraps` now correctly + set the ``__wrapped__`` attribute to the function being wrapped, even if + that function also had its ``__wrapped__`` attribute set. This means + ``__wrapped__`` attributes now correctly link a stack of decorated + functions rather than every ``__wrapped__`` attribute in the chain + referring to the innermost function. Introspection libraries that + assumed the previous behaviour was intentional can use + :func:`inspect.unwrap` to access the first function in the chain that has + no ``__wrapped__`` attribute. + +* :func:`inspect.getfullargspec` has been reimplemented on top of + :func:`inspect.signature` and hence handles a much wider variety of callable + objects than it did in the past. It is expected that additional builtin and + extension module callables will gain signature metadata over the course of + the Python 3.4 series. Code that assumes that + :func:`inspect.getfullargspec` will fail on non-Python callables may need + to be adjusted accordingly. + +* :class:`importlib.machinery.PathFinder` now passes on the current working + directory to objects in :data:`sys.path_hooks` for the empty string. This + results in :data:`sys.path_importer_cache` never containing ``''``, thus + iterating through :data:`sys.path_importer_cache` based on :data:`sys.path` + will not find all keys. A module's ``__file__`` when imported in the current + working directory will also now have an absolute path, including when using + ``-m`` with the interpreter (except for ``__main__.__file__`` when a script + has been executed directly using a relative path) (Contributed by Brett + Cannon in :issue:`18416`). is specified on the command-line) + (:issue:`18416`). + +* The removal of the *strict* argument to :class:`~http.client.HTTPConnection` + and :class:`~http.client.HTTPSConnection` changes the meaning of the + remaining arguments if you are specifying them positionally rather than by + keyword. If you've been paying attention to deprecation warnings your code + should already be specifying any additional arguments via keywords. + +* Strings between ``from __future__ import ...`` statements now *always* raise + a :exc:`SyntaxError`. Previously if there was no leading docstring, an + interstitial string would sometimes be ignored. This brings CPython into + compliance with the language spec; Jython and PyPy already were. + (:issue:`17434`). + +* :meth:`ssl.SSLSocket.getpeercert` and :meth:`ssl.SSLSocket.do_handshake` + now raise an :exc:`OSError` with ``ENOTCONN`` when the ``SSLSocket`` is not + connected, instead of the previous behavior of raising an + :exc:`AttributeError`. In addition, :meth:`~ssl.SSLSocket.getpeercert` + will raise a :exc:`ValueError` if the handshake has not yet been done. + +* :func:`base64.b32decode` now raises a :exc:`binascii.Error` when the + input string contains non-b32-alphabet characters, instead of a + :exc:`TypeError`. This particular :exc:`TypeError` was missed when the other + :exc:`TypeError`\ s were converted. (Contributed by Serhiy Storchaka in + :issue:`18011`.) Note: this change was also inadvertently applied in Python + 3.3.3. + +* The :attr:`~cgi.FieldStorage.file` attribute is now automatically closed when + the creating :class:`cgi.FieldStorage` instance is garbage collected. If you + were pulling the file object out separately from the :class:`cgi.FieldStorage` + instance and not keeping the instance alive, then you should either store the + entire :class:`cgi.FieldStorage` instance or read the contents of the file + before the :class:`cgi.FieldStorage` instance is garbage collected. + +* Calling ``read`` or ``write`` on a closed SSL socket now raises an + informative :exc:`ValueError` rather than the previous more mysterious + :exc:`AttributeError` (:issue:`9177`). + +* :meth:`slice.indices` no longer produces an :exc:`OverflowError` for huge + values. As a consequence of this fix, :meth:`slice.indices` now raises a + :exc:`ValueError` if given a negative length; previously it returned nonsense + values (:issue:`14794`). + +* The :class:`complex` constructor, unlike the :mod:`cmath` functions, was + incorrectly accepting :class:`float` values if an object's ``__complex__`` + special method returned one. This now raises a :exc:`TypeError`. + (:issue:`16290`.) + +* The :class:`int` constructor in 3.2 and 3.3 erroneously accepts :class:`float` + values for the *base* parameter. It is unlikely anyone was doing this, but + if so, it will now raise a :exc:`TypeError` (:issue:`16772`). + +* Defaults for keyword-only arguments are now evaluated *after* defaults for + regular keyword arguments, instead of before. Hopefully no one wrote any + code that depends on the previous buggy behavior (:issue:`16967`). + +* Stale thread states are now cleared after :func:`~os.fork`. This may cause + some system resources to be released that previously were incorrectly kept + perpetually alive (for example, database connections kept in thread-local + storage). (:issue:`17094`.) + +* Parameter names in ``__annotations__`` dicts are now mangled properly, + similarly to ``__kwdefaults__``. (Contributed by Yury Selivanov in + :issue:`20625`). + +* :attr:`hashlib.hash.name` now always returns the identifier in lower case. + Previously some builtin hashes had uppercase names, but now that it is a + formal public interface the naming has been made consistent (:issue:`18532`). + +* Because :mod:`unittest.TestSuite` now drops references to tests after they + are run, test harnesses that re-use a :class:`~unittest.TestSuite` to re-run + a set of tests may fail. Test suites should not be re-used in this fashion + since it means state is retained between test runs, breaking the test + isolation that :mod:`unittest` is designed to provide. However, if the lack + of isolation is considered acceptable, the old behavior can be restored by + creating a :mod:`~unittest.TestSuite` subclass that defines a + ``_removeTestAtIndex`` method that does nothing (see + :meth:`.TestSuite.__iter__`) (:issue:`11798`). + +* :mod:`unittest` now uses :mod:`argparse` for command line parsing. There are + certain invalid command forms that used to work that are no longer allowed; + in theory this should not cause backward compatibility issues since the + disallowed command forms didn't make any sense and are unlikely to be in use. + +* The :func:`re.split`, :func:`re.findall`, and :func:`re.sub` functions, and + the :meth:`~re.match.group` and :meth:`~re.match.groups` methods of + ``match`` objects now always return a *bytes* object when the string + to be matched is a :term:`bytes-like object`. Previously the return type + matched the input type, so if your code was depending on the return value + being, say, a ``bytearray``, you will need to change your code. + +* :mod:`audioop` functions now raise an error immediately if passed string + input, instead of failing randomly later on (:issue:`16685`). + +* The new *convert_charrefs* argument to :class:`~html.parser.HTMLParser` + currently defaults to ``False`` for backward compatibility, but will + eventually be changed to default to ``True``. It is recommended that you add + this keyword, with the appropriate value, to any + :class:`~html.parser.HTMLParser` calls in your code (:issue:`13633`). + +* Since the *digestmod* argument to the :func:`hmac.new` function will in the + future have no default, all calls to :func:`hmac.new` should be changed to + explicitly specify a *digestmod* (:issue:`17276`). + +* Calling :func:`sysconfig.get_config_var` with the ``SO`` key, or looking + ``SO`` up in the results of a call to :func:`sysconfig.get_config_vars` + is deprecated. This key should be replaced by ``EXT_SUFFIX`` or + ``SHLIB_SUFFIX``, depending on the context (:issue:`19555`). + +* Any calls to ``open`` functions that specify ``U`` should be modified. + ``U`` is ineffective in Python3 and will eventually raise an error if used. + Depending on the function, the equivalent of its old Python2 behavior can be + achieved using either a *newline* argument, or if necessary by wrapping the + stream in :mod:`~io.TextIOWrapper` to use its *newline* argument + (:issue:`15204`). + +* If you use :ref:`pyvenv <scripts-pyvenv>` in a script and desire that pip + *not* be installed, you must add ``--without-pip`` to your command + invocation. + +* The default behavior of :func:`json.dump` and :func:`json.dumps` when + an indent is specified has changed: it no longer produces trailing + spaces after the item separating commas at the ends of lines. This + will matter only if you have tests that are doing white-space-sensitive + comparisons of such output (:issue:`16333`). + +* :mod:`doctest` now looks for doctests in extension module ``__doc__`` + strings, so if your doctest test discovery includes extension modules that + have things that look like doctests in them you may see test failures you've + never seen before when running your tests (:issue:`3158`). + +* The :mod:`collections.abc` module has been slightly refactored as + part of the Python startup improvements. As a consequence of this, it is no + longer the case that importing :mod:`collections` automatically imports + :mod:`collections.abc`. If your program depended on the (undocumented) + implicit import, you will need to add an explicit ``import collections.abc`` + (:issue:`20784`). + + +Changes in the C API +-------------------- + +* :c:func:`PyEval_EvalFrameEx`, :c:func:`PyObject_Repr`, and + :c:func:`PyObject_Str`, along with some other internal C APIs, now include + a debugging assertion that ensures they are not used in situations where + they may silently discard a currently active exception. In cases where + discarding the active exception is expected and desired (for example, + because it has already been saved locally with :c:func:`PyErr_Fetch` or + is being deliberately replaced with a different exception), an explicit + :c:func:`PyErr_Clear` call will be needed to avoid triggering the + assertion when invoking these operations (directly or indirectly) and + running against a version of Python that is compiled with assertions + enabled. + +* :c:func:`PyErr_SetImportError` now sets :exc:`TypeError` when its **msg** + argument is not set. Previously only ``NULL`` was returned with no exception + set. + +* The result of the :c:data:`PyOS_ReadlineFunctionPointer` callback must + now be a string allocated by :c:func:`PyMem_RawMalloc` or + :c:func:`PyMem_RawRealloc`, or *NULL* if an error occurred, instead of a + string allocated by :c:func:`PyMem_Malloc` or :c:func:`PyMem_Realloc` + (:issue:`16742`) + +* :c:func:`PyThread_set_key_value` now always set the value. In Python + 3.3, the function did nothing if the key already exists (if the current + value is a non-NULL pointer). + +* The ``f_tstate`` (thread state) field of the :c:type:`PyFrameObject` + structure has been removed to fix a bug: see :issue:`14432` for the + rationale. diff --git a/Doc/whatsnew/index.rst b/Doc/whatsnew/index.rst index bc1206b..29902e4 100644 --- a/Doc/whatsnew/index.rst +++ b/Doc/whatsnew/index.rst @@ -11,6 +11,7 @@ anyone wishing to stay up-to-date after a new release. .. toctree:: :maxdepth: 2 + 3.4.rst 3.3.rst 3.2.rst 3.1.rst |
