summaryrefslogtreecommitdiffstats
path: root/Demo/xmlrpc
ModeNameSize
-rw-r--r--xmlrpc_handler.py2850logstatsplain
-rw-r--r--xmlrpcserver.py2348logstatsplain
>
Diffstat (limited to 'Doc')
-rw-r--r--Doc/ACKS.txt2
-rw-r--r--Doc/c-api/arg.rst29
-rw-r--r--Doc/c-api/buffer.rst493
-rw-r--r--Doc/c-api/code.rst6
-rw-r--r--Doc/c-api/conversion.rst4
-rw-r--r--Doc/c-api/datetime.rst25
-rw-r--r--Doc/c-api/dict.rst15
-rw-r--r--Doc/c-api/exceptions.rst252
-rw-r--r--Doc/c-api/function.rst10
-rw-r--r--Doc/c-api/import.rst64
-rw-r--r--Doc/c-api/index.rst1
-rw-r--r--Doc/c-api/init.rst2
-rw-r--r--Doc/c-api/list.rst7
-rw-r--r--Doc/c-api/long.rst14
-rw-r--r--Doc/c-api/memoryview.rst31
-rw-r--r--Doc/c-api/module.rst56
-rw-r--r--Doc/c-api/object.rst29
-rw-r--r--Doc/c-api/set.rst7
-rw-r--r--Doc/c-api/stable.rst39
-rw-r--r--Doc/c-api/type.rst25
-rw-r--r--Doc/c-api/typeobj.rst102
-rw-r--r--Doc/c-api/unicode.rst874
-rw-r--r--Doc/c-api/veryhigh.rst6
-rw-r--r--Doc/conf.py2
-rw-r--r--Doc/data/refcounts.dat5
-rw-r--r--Doc/distutils/apiref.rst13
-rw-r--r--Doc/extending/extending.rst5
-rw-r--r--Doc/extending/newtypes.rst72
-rw-r--r--Doc/faq/design.rst20
-rw-r--r--Doc/faq/extending.rst31
-rw-r--r--Doc/faq/library.rst2
-rw-r--r--Doc/glossary.rst46
-rw-r--r--Doc/howto/descriptor.rst12
-rw-r--r--Doc/howto/index.rst1
-rw-r--r--Doc/howto/ipaddress.rst299
-rw-r--r--Doc/howto/logging-cookbook.rst29
-rw-r--r--Doc/howto/sockets.rst36
-rw-r--r--Doc/howto/urllib2.rst7
-rw-r--r--Doc/includes/mp_benchmarks.py6
-rw-r--r--Doc/includes/noddy2.c50
-rw-r--r--Doc/includes/noddy3.c118
-rw-r--r--Doc/includes/noddy4.c48
-rw-r--r--Doc/library/_thread.rst5
-rw-r--r--Doc/library/abc.rst82
-rw-r--r--Doc/library/archiving.rst6
-rw-r--r--Doc/library/argparse.rst31
-rw-r--r--Doc/library/array.rst68
-rw-r--r--Doc/library/ast.rst3
-rw-r--r--Doc/library/asynchat.rst3
-rw-r--r--Doc/library/asyncore.rst13
-rw-r--r--Doc/library/atexit.rst8
-rw-r--r--Doc/library/base64.rst11
-rw-r--r--Doc/library/binascii.rst9
-rw-r--r--Doc/library/bz2.rst246
-rw-r--r--Doc/library/chunk.rst5
-rw-r--r--Doc/library/cmd.rst4
-rw-r--r--Doc/library/codecs.rst34
-rw-r--r--Doc/library/collections.abc.rst182
-rw-r--r--Doc/library/collections.rst1157
-rw-r--r--Doc/library/concurrent.futures.rst19
-rw-r--r--Doc/library/contextlib.rst347
-rw-r--r--Doc/library/copyreg.rst8
-rw-r--r--Doc/library/crypt.rst115
-rw-r--r--Doc/library/csv.rst18
-rw-r--r--Doc/library/ctypes.rst115
-rw-r--r--Doc/library/curses.rst31
-rw-r--r--Doc/library/datatypes.rst1
-rw-r--r--Doc/library/datetime.rst125
-rw-r--r--Doc/library/debug.rst3
-rw-r--r--Doc/library/decimal.rst179
-rw-r--r--Doc/library/depgraph-output.pngbin0 -> 24719 bytes
-rw-r--r--Doc/library/development.rst3
-rw-r--r--Doc/library/difflib.rst4
-rw-r--r--Doc/library/dis.rst19
-rw-r--r--Doc/library/distutils.rst12
-rw-r--r--Doc/library/email.errors.rst21
-rw-r--r--Doc/library/email.generator.rst65
-rw-r--r--Doc/library/email.headerregistry.rst452
-rw-r--r--Doc/library/email.message.rst8
-rw-r--r--Doc/library/email.mime.rst6
-rw-r--r--Doc/library/email.parser.rst79
-rw-r--r--Doc/library/email.policy.rst497
-rw-r--r--Doc/library/email.rst2
-rw-r--r--Doc/library/email.util.rst57
-rw-r--r--Doc/library/exceptions.rst247
-rw-r--r--Doc/library/faulthandler.rst136
-rw-r--r--Doc/library/fcntl.rst9
-rw-r--r--Doc/library/fileinput.rst9
-rw-r--r--Doc/library/ftplib.rst65
-rw-r--r--Doc/library/functions.rst87
-rw-r--r--Doc/library/functools.rst18
-rw-r--r--Doc/library/gc.rst39
-rw-r--r--Doc/library/gettext.rst7
-rw-r--r--Doc/library/gzip.rst51
-rw-r--r--Doc/library/hmac.rst35
-rw-r--r--Doc/library/html.entities.rst23
-rw-r--r--Doc/library/html.parser.rst24
-rw-r--r--Doc/library/http.client.rst21
-rw-r--r--Doc/library/http.cookiejar.rst11
-rw-r--r--Doc/library/http.cookies.rst5
-rw-r--r--Doc/library/http.server.rst59
-rw-r--r--Doc/library/imaplib.rst27
-rw-r--r--Doc/library/imp.rst96
-rw-r--r--Doc/library/importlib.rst336
-rw-r--r--Doc/library/inspect.rst69
-rw-r--r--Doc/library/internet.rst1
-rw-r--r--Doc/library/io.rst78
-rw-r--r--Doc/library/ipaddress.rst251
-rw-r--r--Doc/library/itertools.rst48
-rw-r--r--Doc/library/locale.rst7
-rw-r--r--Doc/library/logging.handlers.rst109
-rw-r--r--Doc/library/logging.rst41
-rw-r--r--Doc/library/lzma.rst382
-rw-r--r--Doc/library/math.rst13
-rw-r--r--Doc/library/mmap.rst12
-rw-r--r--Doc/library/msvcrt.rst15
-rw-r--r--Doc/library/multiprocessing.rst301
-rw-r--r--Doc/library/nntplib.rst16
-rw-r--r--Doc/library/numeric.rst6
-rw-r--r--Doc/library/os.path.rst3
-rw-r--r--Doc/library/os.rst1511
-rw-r--r--Doc/library/ossaudiodev.rst14
-rw-r--r--Doc/library/pdb.rst5
-rw-r--r--Doc/library/pickle.rst61
-rw-r--r--Doc/library/pkgutil.rst8
-rw-r--r--Doc/library/platform.rst4
-rw-r--r--Doc/library/random.rst6
-rw-r--r--Doc/library/re.rst20
-rw-r--r--Doc/library/readline.rst4
-rw-r--r--Doc/library/resource.rst8
-rw-r--r--Doc/library/sched.rst66
-rw-r--r--Doc/library/select.rst113
-rw-r--r--Doc/library/shlex.rst37
-rw-r--r--Doc/library/shutil.rst231
-rw-r--r--Doc/library/signal.rst169
-rw-r--r--Doc/library/site.rst24
-rw-r--r--Doc/library/smtpd.rst20
-rw-r--r--Doc/library/smtplib.rst65
-rw-r--r--Doc/library/socket.rst414
-rw-r--r--Doc/library/socketserver.rst16
-rw-r--r--Doc/library/sqlite3.rst20
-rw-r--r--Doc/library/ssl.rst269
-rw-r--r--Doc/library/stat.rst11
-rw-r--r--Doc/library/stdtypes.rst435
-rw-r--r--Doc/library/struct.rst21
-rw-r--r--Doc/library/subprocess.rst151
-rw-r--r--Doc/library/sys.rst231
-rw-r--r--Doc/library/syslog.rst10
-rw-r--r--Doc/library/tarfile.rst44
-rw-r--r--Doc/library/telnetlib.rst6
-rw-r--r--Doc/library/tempfile.rst2
-rw-r--r--Doc/library/test.rst167
-rw-r--r--Doc/library/textwrap.rst44
-rw-r--r--Doc/library/threading.rst83
-rw-r--r--Doc/library/time.rst227
-rw-r--r--Doc/library/timeit.rst30
-rw-r--r--Doc/library/tkinter.rst64
-rw-r--r--Doc/library/tokenize.rst108
-rw-r--r--Doc/library/types.rst151
-rw-r--r--Doc/library/unicodedata.rst13
-rw-r--r--Doc/library/unittest.mock-examples.rst1246
-rw-r--r--Doc/library/unittest.mock.rst2225
-rw-r--r--Doc/library/unittest.rst80
-rw-r--r--Doc/library/urllib.error.rst12
-rw-r--r--Doc/library/urllib.request.rst172
-rw-r--r--Doc/library/venv.rst199
-rw-r--r--Doc/library/warnings.rst3
-rw-r--r--Doc/library/webbrowser.rst97
-rw-r--r--Doc/library/winreg.rst51
-rw-r--r--Doc/library/xml.etree.elementtree.rst368
-rw-r--r--Doc/library/xml.sax.utils.rst2
-rw-r--r--Doc/library/xmlrpc.client.rst53
-rw-r--r--Doc/library/xmlrpc.server.rst28
-rw-r--r--Doc/library/zipfile.rst36
-rw-r--r--Doc/library/zipimport.rst5
-rw-r--r--Doc/library/zlib.rst84
-rw-r--r--Doc/license.rst2
-rw-r--r--Doc/reference/datamodel.rst203
-rw-r--r--Doc/reference/expressions.rst33
-rw-r--r--Doc/reference/lexical_analysis.rst37
-rw-r--r--Doc/reference/simple_stmts.rst30
-rw-r--r--Doc/tools/sphinxext/indexsidebar.html2
-rw-r--r--Doc/tools/sphinxext/patchlevel.py6
-rw-r--r--Doc/tools/sphinxext/pyspecific.py7
-rw-r--r--Doc/tools/sphinxext/susp-ignored.csv323
-rw-r--r--Doc/tutorial/classes.rst5
-rw-r--r--Doc/tutorial/datastructures.rst24
-rw-r--r--Doc/tutorial/interpreter.rst14
-rw-r--r--Doc/tutorial/stdlib.rst2
-rw-r--r--Doc/tutorial/stdlib2.rst6
-rw-r--r--Doc/using/cmdline.rst28
-rw-r--r--Doc/using/index.rst2
-rw-r--r--Doc/using/scripts.rst65
-rw-r--r--Doc/using/windows.rst268
-rw-r--r--Doc/whatsnew/3.2.rst8
-rw-r--r--Doc/whatsnew/3.3.rst1759
-rw-r--r--Doc/whatsnew/index.rst1
197 files changed, 18456 insertions, 3518 deletions
diff --git a/Doc/ACKS.txt b/Doc/ACKS.txt
index 70063c0..b64c650 100644
--- a/Doc/ACKS.txt
+++ b/Doc/ACKS.txt
@@ -62,6 +62,7 @@ docs@python.org), and we'll be glad to correct the problem.
* Stefan Franke
* Jim Fulton
* Peter Funk
+ * Ethan Furman
* Lele Gaifax
* Matthew Gallagher
* Gabriel Genellina
@@ -210,6 +211,7 @@ docs@python.org), and we'll be glad to correct the problem.
* David Turner
* Sandro Tosi
* Ville Vainio
+ * Nadeem Vawda
* Martijn Vries
* Charles G. Waldman
* Greg Ward
diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst
index d4dda7c..b28aa5f 100644
--- a/Doc/c-api/arg.rst
+++ b/Doc/c-api/arg.rst
@@ -146,7 +146,7 @@ Unless otherwise stated, buffers are not NUL-terminated.
Like ``u#``, but the Python object may also be ``None``, in which case the
:c:type:`Py_UNICODE` pointer is set to *NULL*.
-``U`` (:class:`str`) [PyUnicodeObject \*]
+``U`` (:class:`str`) [PyObject \*]
Requires that the Python object is a Unicode object, without attempting
any conversion. Raises :exc:`TypeError` if the object is not a Unicode
object. The C variable may also be declared as :c:type:`PyObject\*`.
@@ -260,9 +260,12 @@ Numbers
``n`` (:class:`int`) [Py_ssize_t]
Convert a Python integer to a C :c:type:`Py_ssize_t`.
-``c`` (:class:`bytes` of length 1) [char]
- Convert a Python byte, represented as a :class:`bytes` object of length 1,
- to a C :c:type:`char`.
+``c`` (:class:`bytes` or :class:`bytearray` of length 1) [char]
+ Convert a Python byte, represented as a :class:`bytes` or
+ :class:`bytearray` object of length 1, to a C :c:type:`char`.
+
+ .. versionchanged:: 3.3
+ Allow :class:`bytearray` objects.
``C`` (:class:`str` of length 1) [int]
Convert a Python character, represented as a :class:`str` object of
@@ -315,6 +318,15 @@ Other objects
.. versionchanged:: 3.1
``Py_CLEANUP_SUPPORTED`` was added.
+``p`` (:class:`bool`) [int]
+ Tests the value passed in for truth (a boolean **p**\redicate) and converts
+ the result to its equivalent C true/false integer value.
+ Sets the int to 1 if the expression was true and 0 if it was false.
+ This accepts any valid Python value. See :ref:`truth` for more
+ information about how Python tests values for truth.
+
+ .. versionadded:: 3.3
+
``(items)`` (:class:`tuple`) [*matching-items*]
The object must be a Python sequence whose length is the number of format units
in *items*. The C arguments must correspond to the individual format units in
@@ -336,6 +348,15 @@ inside nested parentheses. They are:
:c:func:`PyArg_ParseTuple` does not touch the contents of the corresponding C
variable(s).
+``$``
+ :c:func:`PyArg_ParseTupleAndKeywords` only:
+ Indicates that the remaining arguments in the Python argument list are
+ keyword-only. Currently, all keyword-only arguments must also be optional
+ arguments, so ``|`` must always be specified before ``$`` in the format
+ string.
+
+ .. versionadded:: 3.3
+
``:``
The list of format units ends here; the string after the colon is used as the
function name in error messages (the "associated value" of the exception that
diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst
index d98ece3..d636935 100644
--- a/Doc/c-api/buffer.rst
+++ b/Doc/c-api/buffer.rst
@@ -7,6 +7,7 @@ Buffer Protocol
.. sectionauthor:: Greg Stein <gstein@lyra.org>
.. sectionauthor:: Benjamin Peterson
+.. sectionauthor:: Stefan Krah
.. index::
@@ -20,7 +21,7 @@ as image processing or numeric analysis.
While each of these types have their own semantics, they share the common
characteristic of being backed by a possibly large memory buffer. It is
-then desireable, in some situations, to access that buffer directly and
+then desirable, in some situations, to access that buffer directly and
without intermediate copying.
Python provides such a facility at the C level in the form of the *buffer
@@ -60,8 +61,10 @@ isn't needed anymore. Failure to do so could lead to various issues such as
resource leaks.
-The buffer structure
-====================
+.. _buffer-structure:
+
+Buffer structure
+================
Buffer structures (or simply "buffers") are useful as a way to expose the
binary data from another object to the Python programmer. They can also be
@@ -78,249 +81,411 @@ allows them to be created and copied very simply. When a generic wrapper
around a buffer is needed, a :ref:`memoryview <memoryview-objects>` object
can be created.
+For short instructions how to write an exporting object, see
+:ref:`Buffer Object Structures <buffer-structs>`. For obtaining
+a buffer, see :c:func:`PyObject_GetBuffer`.
.. c:type:: Py_buffer
- .. c:member:: void *buf
+ .. c:member:: void \*obj
+
+ A new reference to the exporting object. The reference is owned by
+ the consumer and automatically decremented and set to *NULL* by
+ :c:func:`PyBuffer_Release`. The field is the equivalent of the return
+ value of any standard C-API function.
+
+ As a special case, for *temporary* buffers that are wrapped by
+ :c:func:`PyMemoryView_FromBuffer` or :c:func:`PyBuffer_FillInfo`
+ this field is *NULL*. In general, exporting objects MUST NOT
+ use this scheme.
- A pointer to the start of the memory for the object.
+ .. c:member:: void \*buf
+
+ A pointer to the start of the logical structure described by the buffer
+ fields. This can be any location within the underlying physical memory
+ block of the exporter. For example, with negative :c:member:`~Py_buffer.strides`
+ the value may point to the end of the memory block.
+
+ For contiguous arrays, the value points to the beginning of the memory
+ block.
.. c:member:: Py_ssize_t len
- :noindex:
- The total length of the memory in bytes.
+ ``product(shape) * itemsize``. For contiguous arrays, this is the length
+ of the underlying memory block. For non-contiguous arrays, it is the length
+ that the logical structure would have if it were copied to a contiguous
+ representation.
+
+ Accessing ``((char *)buf)[0] up to ((char *)buf)[len-1]`` is only valid
+ if the buffer has been obtained by a request that guarantees contiguity. In
+ most cases such a request will be :c:macro:`PyBUF_SIMPLE` or :c:macro:`PyBUF_WRITABLE`.
.. c:member:: int readonly
- An indicator of whether the buffer is read only.
+ An indicator of whether the buffer is read-only. This field is controlled
+ by the :c:macro:`PyBUF_WRITABLE` flag.
+
+ .. c:member:: Py_ssize_t itemsize
+
+ Item size in bytes of a single element. Same as the value of :func:`struct.calcsize`
+ called on non-NULL :c:member:`~Py_buffer.format` values.
+
+ Important exception: If a consumer requests a buffer without the
+ :c:macro:`PyBUF_FORMAT` flag, :c:member:`~Py_Buffer.format` will
+ be set to *NULL*, but :c:member:`~Py_buffer.itemsize` still has
+ the value for the original format.
+
+ If :c:member:`~Py_Buffer.shape` is present, the equality
+ ``product(shape) * itemsize == len`` still holds and the consumer
+ can use :c:member:`~Py_buffer.itemsize` to navigate the buffer.
+
+ If :c:member:`~Py_Buffer.shape` is *NULL* as a result of a :c:macro:`PyBUF_SIMPLE`
+ or a :c:macro:`PyBUF_WRITABLE` request, the consumer must disregard
+ :c:member:`~Py_buffer.itemsize` and assume ``itemsize == 1``.
- .. c:member:: const char *format
- :noindex:
+ .. c:member:: const char \*format
- A *NULL* terminated string in :mod:`struct` module style syntax giving
- the contents of the elements available through the buffer. If this is
- *NULL*, ``"B"`` (unsigned bytes) is assumed.
+ A *NUL* terminated string in :mod:`struct` module style syntax describing
+ the contents of a single item. If this is *NULL*, ``"B"`` (unsigned bytes)
+ is assumed.
+
+ This field is controlled by the :c:macro:`PyBUF_FORMAT` flag.
.. c:member:: int ndim
- The number of dimensions the memory represents as a multi-dimensional
- array. If it is 0, :c:data:`strides` and :c:data:`suboffsets` must be
- *NULL*.
-
- .. c:member:: Py_ssize_t *shape
-
- An array of :c:type:`Py_ssize_t`\s the length of :c:data:`ndim` giving the
- shape of the memory as a multi-dimensional array. Note that
- ``((*shape)[0] * ... * (*shape)[ndims-1])*itemsize`` should be equal to
- :c:data:`len`.
-
- .. c:member:: Py_ssize_t *strides
-
- An array of :c:type:`Py_ssize_t`\s the length of :c:data:`ndim` giving the
- number of bytes to skip to get to a new element in each dimension.
-
- .. c:member:: Py_ssize_t *suboffsets
-
- An array of :c:type:`Py_ssize_t`\s the length of :c:data:`ndim`. If these
- suboffset numbers are greater than or equal to 0, then the value stored
- along the indicated dimension is a pointer and the suboffset value
- dictates how many bytes to add to the pointer after de-referencing. A
- suboffset value that it negative indicates that no de-referencing should
- occur (striding in a contiguous memory block).
-
- Here is a function that returns a pointer to the element in an N-D array
- pointed to by an N-dimensional index when there are both non-NULL strides
- and suboffsets::
-
- void *get_item_pointer(int ndim, void *buf, Py_ssize_t *strides,
- Py_ssize_t *suboffsets, Py_ssize_t *indices) {
- char *pointer = (char*)buf;
- int i;
- for (i = 0; i < ndim; i++) {
- pointer += strides[i] * indices[i];
- if (suboffsets[i] >=0 ) {
- pointer = *((char**)pointer) + suboffsets[i];
- }
- }
- return (void*)pointer;
- }
+ The number of dimensions the memory represents as an n-dimensional array.
+ If it is 0, :c:member:`~Py_Buffer.buf` points to a single item representing
+ a scalar. In this case, :c:member:`~Py_buffer.shape`, :c:member:`~Py_buffer.strides`
+ and :c:member:`~Py_buffer.suboffsets` MUST be *NULL*.
+ The macro :c:macro:`PyBUF_MAX_NDIM` limits the maximum number of dimensions
+ to 64. Exporters MUST respect this limit, consumers of multi-dimensional
+ buffers SHOULD be able to handle up to :c:macro:`PyBUF_MAX_NDIM` dimensions.
- .. c:member:: Py_ssize_t itemsize
+ .. c:member:: Py_ssize_t \*shape
+
+ An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim`
+ indicating the shape of the memory as an n-dimensional array. Note that
+ ``shape[0] * ... * shape[ndim-1] * itemsize`` MUST be equal to
+ :c:member:`~Py_buffer.len`.
+
+ Shape values are restricted to ``shape[n] >= 0``. The case
+ ``shape[n] == 0`` requires special attention. See `complex arrays`_
+ for further information.
+
+ The shape array is read-only for the consumer.
+
+ .. c:member:: Py_ssize_t \*strides
+
+ An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim`
+ giving the number of bytes to skip to get to a new element in each
+ dimension.
+
+ Stride values can be any integer. For regular arrays, strides are
+ usually positive, but a consumer MUST be able to handle the case
+ ``strides[n] <= 0``. See `complex arrays`_ for further information.
+
+ The strides array is read-only for the consumer.
+
+ .. c:member:: Py_ssize_t \*suboffsets
+
+ An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim`.
+ If ``suboffsets[n] >= 0``, the values stored along the nth dimension are
+ pointers and the suboffset value dictates how many bytes to add to each
+ pointer after de-referencing. A suboffset value that is negative
+ indicates that no de-referencing should occur (striding in a contiguous
+ memory block).
- This is a storage for the itemsize (in bytes) of each element of the
- shared memory. It is technically un-necessary as it can be obtained
- using :c:func:`PyBuffer_SizeFromFormat`, however an exporter may know
- this information without parsing the format string and it is necessary
- to know the itemsize for proper interpretation of striding. Therefore,
- storing it is more convenient and faster.
+ This type of array representation is used by the Python Imaging Library
+ (PIL). See `complex arrays`_ for further information how to access elements
+ of such an array.
- .. c:member:: void *internal
+ The suboffsets array is read-only for the consumer.
+
+ .. c:member:: void \*internal
This is for use internally by the exporting object. For example, this
might be re-cast as an integer by the exporter and used to store flags
about whether or not the shape, strides, and suboffsets arrays must be
- freed when the buffer is released. The consumer should never alter this
+ freed when the buffer is released. The consumer MUST NOT alter this
value.
+.. _buffer-request-types:
-Buffer-related functions
-========================
+Buffer request types
+====================
+Buffers are usually obtained by sending a buffer request to an exporting
+object via :c:func:`PyObject_GetBuffer`. Since the complexity of the logical
+structure of the memory can vary drastically, the consumer uses the *flags*
+argument to specify the exact buffer type it can handle.
-.. c:function:: int PyObject_CheckBuffer(PyObject *obj)
+All :c:data:`Py_buffer` fields are unambiguously defined by the request
+type.
+
+request-independent fields
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+The following fields are not influenced by *flags* and must always be filled in
+with the correct values: :c:member:`~Py_buffer.obj`, :c:member:`~Py_buffer.buf`,
+:c:member:`~Py_buffer.len`, :c:member:`~Py_buffer.itemsize`, :c:member:`~Py_buffer.ndim`.
- Return 1 if *obj* supports the buffer interface otherwise 0. When 1 is
- returned, it doesn't guarantee that :c:func:`PyObject_GetBuffer` will
- succeed.
+readonly, format
+~~~~~~~~~~~~~~~~
-.. c:function:: int PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags)
+ .. c:macro:: PyBUF_WRITABLE
- Export a view over some internal data from the target object *obj*.
- *obj* must not be NULL, and *view* must point to an existing
- :c:type:`Py_buffer` structure allocated by the caller (most uses of
- this function will simply declare a local variable of type
- :c:type:`Py_buffer`). The *flags* argument is a bit field indicating
- what kind of buffer is requested. The buffer interface allows
- for complicated memory layout possibilities; however, some callers
- won't want to handle all the complexity and instead request a simple
- view of the target object (using :c:macro:`PyBUF_SIMPLE` for a read-only
- view and :c:macro:`PyBUF_WRITABLE` for a read-write view).
+ Controls the :c:member:`~Py_buffer.readonly` field. If set, the exporter
+ MUST provide a writable buffer or else report failure. Otherwise, the
+ exporter MAY provide either a read-only or writable buffer, but the choice
+ MUST be consistent for all consumers.
- Some exporters may not be able to share memory in every possible way and
- may need to raise errors to signal to some consumers that something is
- just not possible. These errors should be a :exc:`BufferError` unless
- there is another error that is actually causing the problem. The
- exporter can use flags information to simplify how much of the
- :c:data:`Py_buffer` structure is filled in with non-default values and/or
- raise an error if the object can't support a simpler view of its memory.
+ .. c:macro:: PyBUF_FORMAT
- On success, 0 is returned and the *view* structure is filled with useful
- values. On error, -1 is returned and an exception is raised; the *view*
- is left in an undefined state.
+ Controls the :c:member:`~Py_buffer.format` field. If set, this field MUST
+ be filled in correctly. Otherwise, this field MUST be *NULL*.
- The following are the possible values to the *flags* arguments.
- .. c:macro:: PyBUF_SIMPLE
+:c:macro:`PyBUF_WRITABLE` can be \|'d to any of the flags in the next section.
+Since :c:macro:`PyBUF_SIMPLE` is defined as 0, :c:macro:`PyBUF_WRITABLE`
+can be used as a stand-alone flag to request a simple writable buffer.
- This is the default flag. The returned buffer exposes a read-only
- memory area. The format of data is assumed to be raw unsigned bytes,
- without any particular structure. This is a "stand-alone" flag
- constant. It never needs to be '|'d to the others. The exporter will
- raise an error if it cannot provide such a contiguous buffer of bytes.
+:c:macro:`PyBUF_FORMAT` can be \|'d to any of the flags except :c:macro:`PyBUF_SIMPLE`.
+The latter already implies format ``B`` (unsigned bytes).
- .. c:macro:: PyBUF_WRITABLE
- Like :c:macro:`PyBUF_SIMPLE`, but the returned buffer is writable. If
- the exporter doesn't support writable buffers, an error is raised.
+shape, strides, suboffsets
+~~~~~~~~~~~~~~~~~~~~~~~~~~
- .. c:macro:: PyBUF_STRIDES
+The flags that control the logical structure of the memory are listed
+in decreasing order of complexity. Note that each flag contains all bits
+of the flags below it.
- This implies :c:macro:`PyBUF_ND`. The returned buffer must provide
- strides information (i.e. the strides cannot be NULL). This would be
- used when the consumer can handle strided, discontiguous arrays.
- Handling strides automatically assumes you can handle shape. The
- exporter can raise an error if a strided representation of the data is
- not possible (i.e. without the suboffsets).
- .. c:macro:: PyBUF_ND
++-----------------------------+-------+---------+------------+
+| Request | shape | strides | suboffsets |
++=============================+=======+=========+============+
+| .. c:macro:: PyBUF_INDIRECT | yes | yes | if needed |
++-----------------------------+-------+---------+------------+
+| .. c:macro:: PyBUF_STRIDES | yes | yes | NULL |
++-----------------------------+-------+---------+------------+
+| .. c:macro:: PyBUF_ND | yes | NULL | NULL |
++-----------------------------+-------+---------+------------+
+| .. c:macro:: PyBUF_SIMPLE | NULL | NULL | NULL |
++-----------------------------+-------+---------+------------+
- The returned buffer must provide shape information. The memory will be
- assumed C-style contiguous (last dimension varies the fastest). The
- exporter may raise an error if it cannot provide this kind of
- contiguous buffer. If this is not given then shape will be *NULL*.
- .. c:macro:: PyBUF_C_CONTIGUOUS
- PyBUF_F_CONTIGUOUS
- PyBUF_ANY_CONTIGUOUS
+contiguity requests
+~~~~~~~~~~~~~~~~~~~
- These flags indicate that the contiguity returned buffer must be
- respectively, C-contiguous (last dimension varies the fastest), Fortran
- contiguous (first dimension varies the fastest) or either one. All of
- these flags imply :c:macro:`PyBUF_STRIDES` and guarantee that the
- strides buffer info structure will be filled in correctly.
+C or Fortran contiguity can be explicitly requested, with and without stride
+information. Without stride information, the buffer must be C-contiguous.
- .. c:macro:: PyBUF_INDIRECT
++-----------------------------------+-------+---------+------------+--------+
+| Request | shape | strides | suboffsets | contig |
++===================================+=======+=========+============+========+
+| .. c:macro:: PyBUF_C_CONTIGUOUS | yes | yes | NULL | C |
++-----------------------------------+-------+---------+------------+--------+
+| .. c:macro:: PyBUF_F_CONTIGUOUS | yes | yes | NULL | F |
++-----------------------------------+-------+---------+------------+--------+
+| .. c:macro:: PyBUF_ANY_CONTIGUOUS | yes | yes | NULL | C or F |
++-----------------------------------+-------+---------+------------+--------+
+| .. c:macro:: PyBUF_ND | yes | NULL | NULL | C |
++-----------------------------------+-------+---------+------------+--------+
- This flag indicates the returned buffer must have suboffsets
- information (which can be NULL if no suboffsets are needed). This can
- be used when the consumer can handle indirect array referencing implied
- by these suboffsets. This implies :c:macro:`PyBUF_STRIDES`.
- .. c:macro:: PyBUF_FORMAT
+compound requests
+~~~~~~~~~~~~~~~~~
- The returned buffer must have true format information if this flag is
- provided. This would be used when the consumer is going to be checking
- for what 'kind' of data is actually stored. An exporter should always
- be able to provide this information if requested. If format is not
- explicitly requested then the format must be returned as *NULL* (which
- means ``'B'``, or unsigned bytes).
+All possible requests are fully defined by some combination of the flags in
+the previous section. For convenience, the buffer protocol provides frequently
+used combinations as single flags.
- .. c:macro:: PyBUF_STRIDED
+In the following table *U* stands for undefined contiguity. The consumer would
+have to call :c:func:`PyBuffer_IsContiguous` to determine contiguity.
- This is equivalent to ``(PyBUF_STRIDES | PyBUF_WRITABLE)``.
- .. c:macro:: PyBUF_STRIDED_RO
- This is equivalent to ``(PyBUF_STRIDES)``.
++-------------------------------+-------+---------+------------+--------+----------+--------+
+| Request | shape | strides | suboffsets | contig | readonly | format |
++===============================+=======+=========+============+========+==========+========+
+| .. c:macro:: PyBUF_FULL | yes | yes | if needed | U | 0 | yes |
++-------------------------------+-------+---------+------------+--------+----------+--------+
+| .. c:macro:: PyBUF_FULL_RO | yes | yes | if needed | U | 1 or 0 | yes |
++-------------------------------+-------+---------+------------+--------+----------+--------+
+| .. c:macro:: PyBUF_RECORDS | yes | yes | NULL | U | 0 | yes |
++-------------------------------+-------+---------+------------+--------+----------+--------+
+| .. c:macro:: PyBUF_RECORDS_RO | yes | yes | NULL | U | 1 or 0 | yes |
++-------------------------------+-------+---------+------------+--------+----------+--------+
+| .. c:macro:: PyBUF_STRIDED | yes | yes | NULL | U | 0 | NULL |
++-------------------------------+-------+---------+------------+--------+----------+--------+
+| .. c:macro:: PyBUF_STRIDED_RO | yes | yes | NULL | U | 1 or 0 | NULL |
++-------------------------------+-------+---------+------------+--------+----------+--------+
+| .. c:macro:: PyBUF_CONTIG | yes | NULL | NULL | C | 0 | NULL |
++-------------------------------+-------+---------+------------+--------+----------+--------+
+| .. c:macro:: PyBUF_CONTIG_RO | yes | NULL | NULL | C | 1 or 0 | NULL |
++-------------------------------+-------+---------+------------+--------+----------+--------+
- .. c:macro:: PyBUF_RECORDS
- This is equivalent to ``(PyBUF_STRIDES | PyBUF_FORMAT |
- PyBUF_WRITABLE)``.
+Complex arrays
+==============
- .. c:macro:: PyBUF_RECORDS_RO
+NumPy-style: shape and strides
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The logical structure of NumPy-style arrays is defined by :c:member:`~Py_buffer.itemsize`,
+:c:member:`~Py_buffer.ndim`, :c:member:`~Py_buffer.shape` and :c:member:`~Py_buffer.strides`.
+
+If ``ndim == 0``, the memory location pointed to by :c:member:`~Py_buffer.buf` is
+interpreted as a scalar of size :c:member:`~Py_buffer.itemsize`. In that case,
+both :c:member:`~Py_buffer.shape` and :c:member:`~Py_buffer.strides` are *NULL*.
+
+If :c:member:`~Py_buffer.strides` is *NULL*, the array is interpreted as
+a standard n-dimensional C-array. Otherwise, the consumer must access an
+n-dimensional array as follows:
+
+ ``ptr = (char *)buf + indices[0] * strides[0] + ... + indices[n-1] * strides[n-1]``
+ ``item = *((typeof(item) *)ptr);``
+
+
+As noted above, :c:member:`~Py_buffer.buf` can point to any location within
+the actual memory block. An exporter can check the validity of a buffer with
+this function:
+
+.. code-block:: python
+
+ def verify_structure(memlen, itemsize, ndim, shape, strides, offset):
+ """Verify that the parameters represent a valid array within
+ the bounds of the allocated memory:
+ char *mem: start of the physical memory block
+ memlen: length of the physical memory block
+ offset: (char *)buf - mem
+ """
+ if offset % itemsize:
+ return False
+ if offset < 0 or offset+itemsize > memlen:
+ return False
+ if any(v % itemsize for v in strides):
+ return False
+
+ if ndim <= 0:
+ return ndim == 0 and not shape and not strides
+ if 0 in shape:
+ return True
+
+ imin = sum(strides[j]*(shape[j]-1) for j in range(ndim)
+ if strides[j] <= 0)
+ imax = sum(strides[j]*(shape[j]-1) for j in range(ndim)
+ if strides[j] > 0)
+
+ return 0 <= offset+imin and offset+imax+itemsize <= memlen
+
+
+PIL-style: shape, strides and suboffsets
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In addition to the regular items, PIL-style arrays can contain pointers
+that must be followed in order to get to the next element in a dimension.
+For example, the regular three-dimensional C-array ``char v[2][2][3]`` can
+also be viewed as an array of 2 pointers to 2 two-dimensional arrays:
+``char (*v[2])[2][3]``. In suboffsets representation, those two pointers
+can be embedded at the start of :c:member:`~Py_buffer.buf`, pointing
+to two ``char x[2][3]`` arrays that can be located anywhere in memory.
+
+
+Here is a function that returns a pointer to the element in an N-D array
+pointed to by an N-dimensional index when there are both non-NULL strides
+and suboffsets::
+
+ void *get_item_pointer(int ndim, void *buf, Py_ssize_t *strides,
+ Py_ssize_t *suboffsets, Py_ssize_t *indices) {
+ char *pointer = (char*)buf;
+ int i;
+ for (i = 0; i < ndim; i++) {
+ pointer += strides[i] * indices[i];
+ if (suboffsets[i] >=0 ) {
+ pointer = *((char**)pointer) + suboffsets[i];
+ }
+ }
+ return (void*)pointer;
+ }
- This is equivalent to ``(PyBUF_STRIDES | PyBUF_FORMAT)``.
- .. c:macro:: PyBUF_FULL
+Buffer-related functions
+========================
- This is equivalent to ``(PyBUF_INDIRECT | PyBUF_FORMAT |
- PyBUF_WRITABLE)``.
+.. c:function:: int PyObject_CheckBuffer(PyObject *obj)
- .. c:macro:: PyBUF_FULL_RO
+ Return 1 if *obj* supports the buffer interface otherwise 0. When 1 is
+ returned, it doesn't guarantee that :c:func:`PyObject_GetBuffer` will
+ succeed.
- This is equivalent to ``(PyBUF_INDIRECT | PyBUF_FORMAT)``.
- .. c:macro:: PyBUF_CONTIG
+.. c:function:: int PyObject_GetBuffer(PyObject *exporter, Py_buffer *view, int flags)
- This is equivalent to ``(PyBUF_ND | PyBUF_WRITABLE)``.
+ Send a request to *exporter* to fill in *view* as specified by *flags*.
+ If the exporter cannot provide a buffer of the exact type, it MUST raise
+ :c:data:`PyExc_BufferError`, set :c:member:`view->obj` to *NULL* and
+ return -1.
- .. c:macro:: PyBUF_CONTIG_RO
+ On success, fill in *view*, set :c:member:`view->obj` to a new reference
+ to *exporter* and return 0. In the case of chained buffer providers
+ that redirect requests to a single object, :c:member:`view->obj` MAY
+ refer to this object instead of *exporter* (See :ref:`Buffer Object Structures <buffer-structs>`).
- This is equivalent to ``(PyBUF_ND)``.
+ Successful calls to :c:func:`PyObject_GetBuffer` must be paired with calls
+ to :c:func:`PyBuffer_Release`, similar to :c:func:`malloc` and :c:func:`free`.
+ Thus, after the consumer is done with the buffer, :c:func:`PyBuffer_Release`
+ must be called exactly once.
.. c:function:: void PyBuffer_Release(Py_buffer *view)
- Release the buffer *view*. This should be called when the buffer is no
- longer being used as it may free memory from it.
+ Release the buffer *view* and decrement the reference count for
+ :c:member:`view->obj`. This function MUST be called when the buffer
+ is no longer being used, otherwise reference leaks may occur.
+
+ It is an error to call this function on a buffer that was not obtained via
+ :c:func:`PyObject_GetBuffer`.
.. c:function:: Py_ssize_t PyBuffer_SizeFromFormat(const char *)
- Return the implied :c:data:`~Py_buffer.itemsize` from the struct-stype
- :c:data:`~Py_buffer.format`.
+ Return the implied :c:data:`~Py_buffer.itemsize` from :c:data:`~Py_buffer.format`.
+ This function is not yet implemented.
-.. c:function:: int PyBuffer_IsContiguous(Py_buffer *view, char fortran)
+.. c:function:: int PyBuffer_IsContiguous(Py_buffer *view, char order)
- Return 1 if the memory defined by the *view* is C-style (*fortran* is
- ``'C'``) or Fortran-style (*fortran* is ``'F'``) contiguous or either one
- (*fortran* is ``'A'``). Return 0 otherwise.
+ Return 1 if the memory defined by the *view* is C-style (*order* is
+ ``'C'``) or Fortran-style (*order* is ``'F'``) contiguous or either one
+ (*order* is ``'A'``). Return 0 otherwise.
-.. c:function:: void PyBuffer_FillContiguousStrides(int ndim, Py_ssize_t *shape, Py_ssize_t *strides, Py_ssize_t itemsize, char fortran)
+.. c:function:: void PyBuffer_FillContiguousStrides(int ndim, Py_ssize_t *shape, Py_ssize_t *strides, Py_ssize_t itemsize, char order)
Fill the *strides* array with byte-strides of a contiguous (C-style if
- *fortran* is ``'C'`` or Fortran-style if *fortran* is ``'F'``) array of the
+ *order* is ``'C'`` or Fortran-style if *order* is ``'F'``) array of the
given shape with the given number of bytes per element.
-.. c:function:: int PyBuffer_FillInfo(Py_buffer *view, PyObject *obj, void *buf, Py_ssize_t len, int readonly, int infoflags)
+.. c:function:: int PyBuffer_FillInfo(Py_buffer *view, PyObject *exporter, void *buf, Py_ssize_t len, int readonly, int flags)
+
+ Handle buffer requests for an exporter that wants to expose *buf* of size *len*
+ with writability set according to *readonly*. *buf* is interpreted as a sequence
+ of unsigned bytes.
+
+ The *flags* argument indicates the request type. This function always fills in
+ *view* as specified by flags, unless *buf* has been designated as read-only
+ and :c:macro:`PyBUF_WRITABLE` is set in *flags*.
+
+ On success, set :c:member:`view->obj` to a new reference to *exporter* and
+ return 0. Otherwise, raise :c:data:`PyExc_BufferError`, set
+ :c:member:`view->obj` to *NULL* and return -1;
+
+ If this function is used as part of a :ref:`getbufferproc <buffer-structs>`,
+ *exporter* MUST be set to the exporting object. Otherwise, *exporter* MUST
+ be NULL.
+
- Fill in a buffer-info structure, *view*, correctly for an exporter that can
- only share a contiguous chunk of memory of "unsigned bytes" of the given
- length. Return 0 on success and -1 (with raising an error) on error.
diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst
index 6932bb1..57e8072 100644
--- a/Doc/c-api/code.rst
+++ b/Doc/c-api/code.rst
@@ -31,11 +31,11 @@ bound into a function.
Return true if *co* is a :class:`code` object
-.. c:function:: int PyCode_GetNumFree(PyObject *co)
+.. c:function:: int PyCode_GetNumFree(PyCodeObject *co)
Return the number of free variables in *co*.
-.. c:function:: PyCodeObject *PyCode_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab)
+.. c:function:: PyCodeObject* PyCode_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab)
Return a new code object. If you need a dummy code object to
create a frame, use :c:func:`PyCode_NewEmpty` instead. Calling
@@ -43,7 +43,7 @@ bound into a function.
version since the definition of the bytecode changes often.
-.. c:function:: int PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
+.. c:function:: PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
Return a new empty code object with the specified filename,
function name, and first line number. It is illegal to
diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst
index dfc0a3a..9578f98 100644
--- a/Doc/c-api/conversion.rst
+++ b/Doc/c-api/conversion.rst
@@ -119,13 +119,13 @@ The following functions provide locale-independent string to number conversions.
.. versionadded:: 3.1
-.. c:function:: char* PyOS_stricmp(char *s1, char *s2)
+.. c:function:: int PyOS_stricmp(char *s1, char *s2)
Case insensitive comparison of strings. The function works almost
identically to :c:func:`strcmp` except that it ignores the case.
-.. c:function:: char* PyOS_strnicmp(char *s1, char *s2, Py_ssize_t size)
+.. c:function:: int PyOS_strnicmp(char *s1, char *s2, Py_ssize_t size)
Case insensitive comparison of strings. The function works almost
identically to :c:func:`strncmp` except that it ignores the case.
diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst
index fcd1395..39542bd 100644
--- a/Doc/c-api/datetime.rst
+++ b/Doc/c-api/datetime.rst
@@ -170,6 +170,31 @@ and the type is not checked:
Return the microsecond, as an int from 0 through 999999.
+Macros to extract fields from time delta objects. The argument must be an
+instance of :c:data:`PyDateTime_Delta`, including subclasses. The argument must
+not be *NULL*, and the type is not checked:
+
+.. c:function:: int PyDateTime_DELTA_GET_DAYS(PyDateTime_Delta *o)
+
+ Return the number of days, as an int from -999999999 to 999999999.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: int PyDateTime_DELTA_GET_SECONDS(PyDateTime_Delta *o)
+
+ Return the number of seconds, as an int from 0 through 86399.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: int PyDateTime_DELTA_GET_MICROSECOND(PyDateTime_Delta *o)
+
+ Return the number of microseconds, as an int from 0 through 999999.
+
+ .. versionadded:: 3.3
+
+
Macros for the convenience of modules implementing the DB API:
.. c:function:: PyObject* PyDateTime_FromTimestamp(PyObject *args)
diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst
index 6df84e0..6bacc32 100644
--- a/Doc/c-api/dict.rst
+++ b/Doc/c-api/dict.rst
@@ -36,11 +36,11 @@ Dictionary Objects
Return a new empty dictionary, or *NULL* on failure.
-.. c:function:: PyObject* PyDictProxy_New(PyObject *dict)
+.. c:function:: PyObject* PyDictProxy_New(PyObject *mapping)
- Return a proxy object for a mapping which enforces read-only behavior.
- This is normally used to create a proxy to prevent modification of the
- dictionary for non-dynamic class types.
+ Return a :class:`types.MappingProxyType` object for a mapping which
+ enforces read-only behavior. This is normally used to create a view to
+ prevent modification of the dictionary for non-dynamic class types.
.. c:function:: void PyDict_Clear(PyObject *p)
@@ -209,3 +209,10 @@ Dictionary Objects
for key, value in seq2:
if override or key not in a:
a[key] = value
+
+
+.. c:function:: int PyDict_ClearFreeList()
+
+ Clear the free list. Return the total number of freed items.
+
+ .. versionadded:: 3.3
diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst
index 6f13c80..9a66b7f 100644
--- a/Doc/c-api/exceptions.rst
+++ b/Doc/c-api/exceptions.rst
@@ -129,6 +129,41 @@ in various ways. There is a separate error indicator for each thread.
exception state.
+.. c:function:: void PyErr_GetExcInfo(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback)
+
+ Retrieve the exception info, as known from ``sys.exc_info()``. This refers
+ to an exception that was already caught, not to an exception that was
+ freshly raised. Returns new references for the three objects, any of which
+ may be *NULL*. Does not modify the exception info state.
+
+ .. note::
+
+ This function is not normally used by code that wants to handle exceptions.
+ Rather, it can be used when code needs to save and restore the exception
+ state temporarily. Use :c:func:`PyErr_SetExcInfo` to restore or clear the
+ exception state.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: void PyErr_SetExcInfo(PyObject *type, PyObject *value, PyObject *traceback)
+
+ Set the exception info, as known from ``sys.exc_info()``. This refers
+ to an exception that was already caught, not to an exception that was
+ freshly raised. This function steals the references of the arguments.
+ To clear the exception state, pass *NULL* for all three arguments.
+ For general rules about the three arguments, see :c:func:`PyErr_Restore`.
+
+ .. note::
+
+ This function is not normally used by code that wants to handle exceptions.
+ Rather, it can be used when code needs to save and restore the exception
+ state temporarily. Use :c:func:`PyErr_GetExcInfo` to read the exception
+ state.
+
+ .. versionadded:: 3.3
+
+
.. c:function:: void PyErr_SetString(PyObject *type, const char *message)
This is the most common way to set the error indicator. The first argument
@@ -230,6 +265,16 @@ in various ways. There is a separate error indicator for each thread.
parameter specifying the exception type to be raised. Availability: Windows.
+.. c:function:: PyObject* PyErr_SetImportError(PyObject *msg, PyObject *name, PyObject *path)
+
+ This is a convenience function to raise :exc:`ImportError`. *msg* will be
+ set as the exception's message string. *name* and *path*, both of which can
+ be ``NULL``, will be set as the :exc:`ImportError`'s respective ``name``
+ and ``path`` attributes.
+
+ .. versionadded:: 3.3
+
+
.. c:function:: void PyErr_SyntaxLocationEx(char *filename, int lineno, int col_offset)
Set file, line, and offset information for the current exception. If the
@@ -238,7 +283,7 @@ in various ways. There is a separate error indicator for each thread.
is a :exc:`SyntaxError`. *filename* is decoded from the filesystem encoding
(:func:`sys.getfilesystemencoding`).
-.. versionadded:: 3.2
+ .. versionadded:: 3.2
.. c:function:: void PyErr_SyntaxLocation(char *filename, int lineno)
@@ -311,6 +356,7 @@ in various ways. There is a separate error indicator for each thread.
.. versionadded:: 3.2
+
.. c:function:: int PyErr_CheckSignals()
.. index::
@@ -421,17 +467,18 @@ Exception Objects
.. c:function:: PyObject* PyException_GetCause(PyObject *ex)
- Return the cause (another exception instance set by ``raise ... from ...``)
- associated with the exception as a new reference, as accessible from Python
- through :attr:`__cause__`. If there is no cause associated, this returns
- *NULL*.
+ Return the cause (either an exception instance, or :const:`None`,
+ set by ``raise ... from ...``) associated with the exception as a new
+ reference, as accessible from Python through :attr:`__cause__`.
.. c:function:: void PyException_SetCause(PyObject *ex, PyObject *ctx)
Set the cause associated with the exception to *ctx*. Use *NULL* to clear
- it. There is no type check to make sure that *ctx* is an exception instance.
- This steals a reference to *ctx*.
+ it. There is no type check to make sure that *ctx* is either an exception
+ instance or :const:`None`. This steals a reference to *ctx*.
+
+ :attr:`__suppress_context__` is implicitly set to ``True`` by this function.
.. _unicodeexceptions:
@@ -525,7 +572,7 @@ recursion depth automatically).
Marks a point where a recursive C-level call is about to be performed.
- If :const:`USE_STACKCHECK` is defined, this function checks if the the OS
+ If :const:`USE_STACKCHECK` is defined, this function checks if the OS
stack overflowed using :c:func:`PyOS_CheckStack`. In this is the case, it
sets a :exc:`MemoryError` and returns a nonzero value.
@@ -582,65 +629,116 @@ All standard Python exceptions are available as global variables whose names are
:c:type:`PyObject\*`; they are all class objects. For completeness, here are all
the variables:
-+-------------------------------------+----------------------------+----------+
-| C Name | Python Name | Notes |
-+=====================================+============================+==========+
-| :c:data:`PyExc_BaseException` | :exc:`BaseException` | \(1) |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_Exception` | :exc:`Exception` | \(1) |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | \(1) |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_EOFError` | :exc:`EOFError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_EnvironmentError` | :exc:`EnvironmentError` | \(1) |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_IOError` | :exc:`IOError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_ImportError` | :exc:`ImportError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_IndexError` | :exc:`IndexError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_KeyError` | :exc:`KeyError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_NameError` | :exc:`NameError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_NotImplementedError` | :exc:`NotImplementedError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_OSError` | :exc:`OSError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_OverflowError` | :exc:`OverflowError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | \(2) |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_SystemError` | :exc:`SystemError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_TypeError` | :exc:`TypeError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_ValueError` | :exc:`ValueError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_WindowsError` | :exc:`WindowsError` | \(3) |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | |
-+-------------------------------------+----------------------------+----------+
++-----------------------------------------+---------------------------------+----------+
+| C Name | Python Name | Notes |
++=========================================+=================================+==========+
+| :c:data:`PyExc_BaseException` | :exc:`BaseException` | \(1) |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_Exception` | :exc:`Exception` | \(1) |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | \(1) |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_BlockingIOError` | :exc:`BlockingIOError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_EOFError` | :exc:`EOFError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ImportError` | :exc:`ImportError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_IndexError` | :exc:`IndexError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_InterruptedError` | :exc:`InterruptedError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_IsADirectoryError` | :exc:`IsADirectoryError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_KeyError` | :exc:`KeyError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_NameError` | :exc:`NameError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_NotImplementedError` | :exc:`NotImplementedError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_OSError` | :exc:`OSError` | \(1) |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_OverflowError` | :exc:`OverflowError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_PermissionError` | :exc:`PermissionError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | \(2) |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_SystemError` | :exc:`SystemError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_TypeError` | :exc:`TypeError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ValueError` | :exc:`ValueError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | |
++-----------------------------------------+---------------------------------+----------+
+
+.. versionadded:: 3.3
+ :c:data:`PyExc_BlockingIOError`, :c:data:`PyExc_BrokenPipeError`,
+ :c:data:`PyExc_ChildProcessError`, :c:data:`PyExc_ConnectionError`,
+ :c:data:`PyExc_ConnectionAbortedError`, :c:data:`PyExc_ConnectionRefusedError`,
+ :c:data:`PyExc_ConnectionResetError`, :c:data:`PyExc_FileExistsError`,
+ :c:data:`PyExc_FileNotFoundError`, :c:data:`PyExc_InterruptedError`,
+ :c:data:`PyExc_IsADirectoryError`, :c:data:`PyExc_NotADirectoryError`,
+ :c:data:`PyExc_PermissionError`, :c:data:`PyExc_ProcessLookupError`
+ and :c:data:`PyExc_TimeoutError` were introduced following :pep:`3151`.
+
+
+These are compatibility aliases to :c:data:`PyExc_OSError`:
+
++-------------------------------------+----------+
+| C Name | Notes |
++=====================================+==========+
+| :c:data:`PyExc_EnvironmentError` | |
++-------------------------------------+----------+
+| :c:data:`PyExc_IOError` | |
++-------------------------------------+----------+
+| :c:data:`PyExc_WindowsError` | \(3) |
++-------------------------------------+----------+
+
+.. versionchanged:: 3.3
+ These aliases used to be separate exception types.
+
.. index::
single: PyExc_BaseException
@@ -649,28 +747,42 @@ the variables:
single: PyExc_LookupError
single: PyExc_AssertionError
single: PyExc_AttributeError
+ single: PyExc_BlockingIOError
+ single: PyExc_BrokenPipeError
+ single: PyExc_ConnectionError
+ single: PyExc_ConnectionAbortedError
+ single: PyExc_ConnectionRefusedError
+ single: PyExc_ConnectionResetError
single: PyExc_EOFError
- single: PyExc_EnvironmentError
+ single: PyExc_FileExistsError
+ single: PyExc_FileNotFoundError
single: PyExc_FloatingPointError
- single: PyExc_IOError
single: PyExc_ImportError
single: PyExc_IndexError
+ single: PyExc_InterruptedError
+ single: PyExc_IsADirectoryError
single: PyExc_KeyError
single: PyExc_KeyboardInterrupt
single: PyExc_MemoryError
single: PyExc_NameError
+ single: PyExc_NotADirectoryError
single: PyExc_NotImplementedError
single: PyExc_OSError
single: PyExc_OverflowError
+ single: PyExc_PermissionError
+ single: PyExc_ProcessLookupError
single: PyExc_ReferenceError
single: PyExc_RuntimeError
single: PyExc_SyntaxError
single: PyExc_SystemError
single: PyExc_SystemExit
+ single: PyExc_TimeoutError
single: PyExc_TypeError
single: PyExc_ValueError
- single: PyExc_WindowsError
single: PyExc_ZeroDivisionError
+ single: PyExc_EnvironmentError
+ single: PyExc_IOError
+ single: PyExc_WindowsError
Notes:
diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst
index 31805fd..ad98322 100644
--- a/Doc/c-api/function.rst
+++ b/Doc/c-api/function.rst
@@ -38,6 +38,16 @@ There are a few functions specific to Python functions.
object, the argument defaults and closure are set to *NULL*.
+.. c:function:: PyObject* PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname)
+
+ As :c:func:`PyFunction_New`, but also allows to set the function object's
+ ``__qualname__`` attribute. *qualname* should be a unicode object or NULL;
+ if NULL, the ``__qualname__`` attribute is set to the same value as its
+ ``__name__`` attribute.
+
+ .. versionadded:: 3.3
+
+
.. c:function:: PyObject* PyFunction_GetCode(PyObject *op)
Return the code object associated with the function object *op*.
diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst
index cf48363..6de3fb0 100644
--- a/Doc/c-api/import.rst
+++ b/Doc/c-api/import.rst
@@ -30,13 +30,13 @@ Importing Modules
.. c:function:: PyObject* PyImport_ImportModuleNoBlock(const char *name)
- This version of :c:func:`PyImport_ImportModule` does not block. It's intended
- to be used in C functions that import other modules to execute a function.
- The import may block if another thread holds the import lock. The function
- :c:func:`PyImport_ImportModuleNoBlock` never blocks. It first tries to fetch
- the module from sys.modules and falls back to :c:func:`PyImport_ImportModule`
- unless the lock is held, in which case the function will raise an
- :exc:`ImportError`.
+ This function is a deprecated alias of :c:func:`PyImport_ImportModule`.
+
+ .. versionchanged:: 3.3
+ This function used to fail immediately when the import lock was held
+ by another thread. In Python 3.3 though, the locking scheme switched
+ to per-module locks for most purposes, so this function's special
+ behaviour isn't needed anymore.
.. c:function:: PyObject* PyImport_ImportModuleEx(char *name, PyObject *globals, PyObject *locals, PyObject *fromlist)
@@ -57,7 +57,7 @@ Importing Modules
:c:func:`PyImport_ImportModule`.
-.. c:function:: PyObject* PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level)
+.. c:function:: PyObject* PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level)
Import a module. This is best described by referring to the built-in Python
function :func:`__import__`, as the standard :func:`__import__` function calls
@@ -68,6 +68,13 @@ Importing Modules
the return value when a submodule of a package was requested is normally the
top-level package, unless a non-empty *fromlist* was given.
+ .. versionadded:: 3.3
+
+
+.. c:function:: PyObject* PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level)
+
+ Similar to :c:func:`PyImport_ImportModuleLevelObject`, but the name is an
+ UTF-8 encoded string instead of a Unicode object.
.. c:function:: PyObject* PyImport_Import(PyObject *name)
@@ -86,7 +93,7 @@ Importing Modules
an exception set on failure (the module still exists in this case).
-.. c:function:: PyObject* PyImport_AddModule(const char *name)
+.. c:function:: PyObject* PyImport_AddModuleObject(PyObject *name)
Return the module object corresponding to a module name. The *name* argument
may be of the form ``package.module``. First check the modules dictionary if
@@ -100,6 +107,14 @@ Importing Modules
or one of its variants to import a module. Package structures implied by a
dotted name for *name* are not created if not already present.
+ .. versionadded:: 3.3
+
+
+.. c:function:: PyObject* PyImport_AddModule(const char *name)
+
+ Similar to :c:func:`PyImport_AddModuleObject`, but the name is a UTF-8
+ encoded string instead of a Unicode object.
+
.. c:function:: PyObject* PyImport_ExecCodeModule(char *name, PyObject *co)
@@ -136,14 +151,23 @@ Importing Modules
See also :c:func:`PyImport_ExecCodeModuleWithPathnames`.
-.. c:function:: PyObject* PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, char *cpathname)
+.. c:function:: PyObject* PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, PyObject *cpathname)
Like :c:func:`PyImport_ExecCodeModuleEx`, but the :attr:`__cached__`
attribute of the module object is set to *cpathname* if it is
non-``NULL``. Of the three functions, this is the preferred one to use.
+ .. versionadded:: 3.3
+
+
+.. c:function:: PyObject* PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, char *cpathname)
+
+ Like :c:func:`PyImport_ExecCodeModuleObject`, but *name*, *pathname* and
+ *cpathname* are UTF-8 encoded strings.
+
.. versionadded:: 3.2
+
.. c:function:: long PyImport_GetMagicNumber()
Return the magic number for Python bytecode files (a.k.a. :file:`.pyc` and
@@ -200,7 +224,7 @@ Importing Modules
For internal use only.
-.. c:function:: int PyImport_ImportFrozenModule(char *name)
+.. c:function:: int PyImport_ImportFrozenModuleObject(PyObject *name)
Load a frozen module named *name*. Return ``1`` for success, ``0`` if the
module is not found, and ``-1`` with an exception set if the initialization
@@ -208,6 +232,14 @@ Importing Modules
:c:func:`PyImport_ImportModule`. (Note the misnomer --- this function would
reload the module if it was already imported.)
+ .. versionadded:: 3.3
+
+
+.. c:function:: int PyImport_ImportFrozenModule(char *name)
+
+ Similar to :c:func:`PyImport_ImportFrozenModuleObject`, but the name is a
+ UTF-8 encoded string instead of a Unicode object.
+
.. c:type:: struct _frozen
@@ -247,13 +279,13 @@ Importing Modules
Structure describing a single entry in the list of built-in modules. Each of
these structures gives the name and initialization function for a module built
- into the interpreter. Programs which embed Python may use an array of these
- structures in conjunction with :c:func:`PyImport_ExtendInittab` to provide
- additional built-in modules. The structure is defined in
- :file:`Include/import.h` as::
+ into the interpreter. The name is an ASCII encoded string. Programs which
+ embed Python may use an array of these structures in conjunction with
+ :c:func:`PyImport_ExtendInittab` to provide additional built-in modules.
+ The structure is defined in :file:`Include/import.h` as::
struct _inittab {
- char *name;
+ char *name; /* ASCII encoded string */
PyObject* (*initfunc)(void);
};
diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst
index 12a1ec7..a0cb6e5 100644
--- a/Doc/c-api/index.rst
+++ b/Doc/c-api/index.rst
@@ -25,3 +25,4 @@ document the API functions in detail.
init.rst
memory.rst
objimpl.rst
+ stable.rst
diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst
index 7507e3b..95ff4ee 100644
--- a/Doc/c-api/init.rst
+++ b/Doc/c-api/init.rst
@@ -646,7 +646,7 @@ with sub-interpreters:
:c:func:`PyGILState_Release` on the same thread.
-.. c:function:: PyThreadState PyGILState_GetThisThreadState()
+.. c:function:: PyThreadState* PyGILState_GetThisThreadState()
Get the current thread state for this thread. May return ``NULL`` if no
GILState API has been used on the current thread. Note that the main thread
diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst
index feb9015..5b263a7 100644
--- a/Doc/c-api/list.rst
+++ b/Doc/c-api/list.rst
@@ -142,3 +142,10 @@ List Objects
Return a new tuple object containing the contents of *list*; equivalent to
``tuple(list)``.
+
+
+.. c:function:: int PyList_ClearFreeList()
+
+ Clear the free list. Return the total number of freed items.
+
+ .. versionadded:: 3.3
diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst
index e2f58ad..365abd7 100644
--- a/Doc/c-api/long.rst
+++ b/Doc/c-api/long.rst
@@ -100,6 +100,20 @@ All integers are implemented as "long" integer objects of arbitrary size.
string is first encoded to a byte string using :c:func:`PyUnicode_EncodeDecimal`
and then converted using :c:func:`PyLong_FromString`.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyLong_FromUnicodeObject`.
+
+
+.. c:function:: PyObject* PyLong_FromUnicodeObject(PyObject *u, int base)
+
+ Convert a sequence of Unicode digits in the string *u* to a Python integer
+ value. The Unicode string is first encoded to a byte string using
+ :c:func:`PyUnicode_EncodeDecimal` and then converted using
+ :c:func:`PyLong_FromString`.
+
+ .. versionadded:: 3.3
+
.. c:function:: PyObject* PyLong_FromVoidPtr(void *p)
diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst
index 6b49cdf..5e50977 100644
--- a/Doc/c-api/memoryview.rst
+++ b/Doc/c-api/memoryview.rst
@@ -17,16 +17,21 @@ any other object.
Create a memoryview object from an object that provides the buffer interface.
If *obj* supports writable buffer exports, the memoryview object will be
- readable and writable, otherwise it will be read-only.
+ read/write, otherwise it may be either read-only or read/write at the
+ discretion of the exporter.
+.. c:function:: PyObject *PyMemoryView_FromMemory(char *mem, Py_ssize_t size, int flags)
+
+ Create a memoryview object using *mem* as the underlying buffer.
+ *flags* can be one of :c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE`.
+
+ .. versionadded:: 3.3
.. c:function:: PyObject *PyMemoryView_FromBuffer(Py_buffer *view)
Create a memoryview object wrapping the given buffer structure *view*.
- The memoryview object then owns the buffer represented by *view*, which
- means you shouldn't try to call :c:func:`PyBuffer_Release` yourself: it
- will be done on deallocation of the memoryview object.
-
+ For simple byte buffers, :c:func:`PyMemoryView_FromMemory` is the preferred
+ function.
.. c:function:: PyObject *PyMemoryView_GetContiguous(PyObject *obj, int buffertype, char order)
@@ -43,10 +48,16 @@ any other object.
currently allowed to create subclasses of :class:`memoryview`.
-.. c:function:: Py_buffer *PyMemoryView_GET_BUFFER(PyObject *obj)
+.. c:function:: Py_buffer *PyMemoryView_GET_BUFFER(PyObject *mview)
+
+ Return a pointer to the memoryview's private copy of the exporter's buffer.
+ *mview* **must** be a memoryview instance; this macro doesn't check its type,
+ you must do it yourself or you will risk crashes.
+
+.. c:function:: Py_buffer *PyMemoryView_GET_BASE(PyObject *mview)
- Return a pointer to the buffer structure wrapped by the given
- memoryview object. The object **must** be a memoryview instance;
- this macro doesn't check its type, you must do it yourself or you
- will risk crashes.
+ Return either a pointer to the exporting object that the memoryview is based
+ on or *NULL* if the memoryview has been created by one of the functions
+ :c:func:`PyMemoryView_FromMemory` or :c:func:`PyMemoryView_FromBuffer`.
+ *mview* **must** be a memoryview instance.
diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst
index ffd68e3..3be7fe3 100644
--- a/Doc/c-api/module.rst
+++ b/Doc/c-api/module.rst
@@ -29,7 +29,7 @@ There are only a few functions special to module objects.
:c:data:`PyModule_Type`.
-.. c:function:: PyObject* PyModule_New(const char *name)
+.. c:function:: PyObject* PyModule_NewObject(PyObject *name)
.. index::
single: __name__ (module attribute)
@@ -40,6 +40,14 @@ There are only a few functions special to module objects.
Only the module's :attr:`__doc__` and :attr:`__name__` attributes are filled in;
the caller is responsible for providing a :attr:`__file__` attribute.
+ .. versionadded:: 3.3
+
+
+.. c:function:: PyObject* PyModule_New(const char *name)
+
+ Similar to :c:func:`PyImport_NewObject`, but the name is an UTF-8 encoded
+ string instead of a Unicode object.
+
.. c:function:: PyObject* PyModule_GetDict(PyObject *module)
@@ -52,7 +60,7 @@ There are only a few functions special to module objects.
manipulate a module's :attr:`__dict__`.
-.. c:function:: char* PyModule_GetName(PyObject *module)
+.. c:function:: PyObject* PyModule_GetNameObject(PyObject *module)
.. index::
single: __name__ (module attribute)
@@ -61,15 +69,13 @@ There are only a few functions special to module objects.
Return *module*'s :attr:`__name__` value. If the module does not provide one,
or if it is not a string, :exc:`SystemError` is raised and *NULL* is returned.
+ .. versionadded:: 3.3
-.. c:function:: char* PyModule_GetFilename(PyObject *module)
- Similar to :c:func:`PyModule_GetFilenameObject` but return the filename
- encoded to 'utf-8'.
+.. c:function:: char* PyModule_GetName(PyObject *module)
- .. deprecated:: 3.2
- :c:func:`PyModule_GetFilename` raises :c:type:`UnicodeEncodeError` on
- unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead.
+ Similar to :c:func:`PyModule_GetNameObject` but return the name encoded to
+ ``'utf-8'``.
.. c:function:: PyObject* PyModule_GetFilenameObject(PyObject *module)
@@ -81,11 +87,21 @@ There are only a few functions special to module objects.
Return the name of the file from which *module* was loaded using *module*'s
:attr:`__file__` attribute. If this is not defined, or if it is not a
unicode string, raise :exc:`SystemError` and return *NULL*; otherwise return
- a reference to a :c:type:`PyUnicodeObject`.
+ a reference to a Unicode object.
.. versionadded:: 3.2
+.. c:function:: char* PyModule_GetFilename(PyObject *module)
+
+ Similar to :c:func:`PyModule_GetFilenameObject` but return the filename
+ encoded to 'utf-8'.
+
+ .. deprecated:: 3.2
+ :c:func:`PyModule_GetFilename` raises :c:type:`UnicodeEncodeError` on
+ unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead.
+
+
.. c:function:: void* PyModule_GetState(PyObject *module)
Return the "state" of the module, that is, a pointer to the block of memory
@@ -97,8 +113,28 @@ There are only a few functions special to module objects.
Return a pointer to the :c:type:`PyModuleDef` struct from which the module was
created, or *NULL* if the module wasn't created with
- :c:func:`PyModule_Create`.
+ :c:func:`PyModule_Create`.i
+
+.. c:function:: PyObject* PyState_FindModule(PyModuleDef *def)
+
+ Returns the module object that was created from *def* for the current interpreter.
+ This method requires that the module object has been attached to the interpreter state with
+ :c:func:`PyState_AddModule` beforehand. In case the corresponding module object is not
+ found or has not been attached to the interpreter state yet, it returns NULL.
+
+.. c:function:: int PyState_AddModule(PyModuleDef *def, PyObject *module)
+
+ Attaches the module object passed to the function to the interpreter state. This allows
+ the module object to be accessible via
+ :c:func:`PyState_FindModule`.
+
+ .. versionadded:: 3.3
+
+.. c:function:: int PyState_RemoveModule(PyModuleDef *def, PyObject *module)
+
+ Removes the module object created from *def* from the interpreter state.
+ .. versionadded:: 3.3
Initializing C modules
^^^^^^^^^^^^^^^^^^^^^^
diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst
index d0d45ad..d895547 100644
--- a/Doc/c-api/object.rst
+++ b/Doc/c-api/object.rst
@@ -6,6 +6,19 @@ Object Protocol
===============
+.. c:var:: PyObject* Py_NotImplemented
+
+ The ``NotImplemented`` singleton, used to signal that an operation is
+ not implemented for the given type combination.
+
+
+.. c:macro:: Py_RETURN_NOTIMPLEMENTED
+
+ Properly handle returning :c:data:`Py_NotImplemented` from within a C
+ function (that is, increment the reference count of NotImplemented and
+ return it).
+
+
.. c:function:: int PyObject_Print(PyObject *o, FILE *fp, int flags)
Print an object *o*, on file *fp*. Returns ``-1`` on error. The flags argument
@@ -88,6 +101,22 @@ Object Protocol
This is the equivalent of the Python statement ``del o.attr_name``.
+.. c:function:: PyObject* PyType_GenericGetDict(PyObject *o, void *context)
+
+ A generic implementation for the getter of a ``__dict__`` descriptor. It
+ creates the dictionary if necessary.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: int PyType_GenericSetDict(PyObject *o, void *context)
+
+ A generic implementation for the setter of a ``__dict__`` descriptor. This
+ implementation does not allow the dictionary to be deleted.
+
+ .. versionadded:: 3.3
+
+
.. c:function:: PyObject* PyObject_RichCompare(PyObject *o1, PyObject *o2, int opid)
Compare the values of *o1* and *o2* using the operation specified by *opid*,
diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst
index 66b47c4..5f0ef90 100644
--- a/Doc/c-api/set.rst
+++ b/Doc/c-api/set.rst
@@ -157,3 +157,10 @@ subtypes but not for instances of :class:`frozenset` or its subtypes.
.. c:function:: int PySet_Clear(PyObject *set)
Empty an existing set of all elements.
+
+
+.. c:function:: int PySet_ClearFreeList()
+
+ Clear the free list. Return the total number of freed items.
+
+ .. versionadded:: 3.3
diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst
new file mode 100644
index 0000000..2688c1c
--- /dev/null
+++ b/Doc/c-api/stable.rst
@@ -0,0 +1,39 @@
+.. highlightlang:: c
+
+.. _stable:
+
+**********************************
+Stable Appliction Binary Interface
+**********************************
+
+Traditionally, the C API of Python will change with every release.
+Most changes will be source-compatible, typically by only adding API,
+rather than changing existing API or removing API (although some
+interfaces do get removed after being deprecated first).
+
+Unfortunately, the API compatibility does not extend to binary
+compatibility (the ABI). The reason is primarily the evolution of
+struct definitions, where addition of a new field, or changing
+the type of a field, might not break the API, but can break the ABI.
+As a consequence, extension modules need to be recompiled for
+every Python release (although an exception is possible on Unix
+when none of the affected interfaces are used). In addition, on
+Windows, extension modules link with a specific pythonXY.dll and
+need to be recompiled to link with a newer one.
+
+Since Python 3.2, a subset of the API has been declared to guarantee
+a stable ABI. Extension modules wishing to use this API need to define
+Py_LIMITED_API. A number of interpreter details then become hidden
+from the extension module; in return, a module is built that works
+on any 3.x version (x>=2) without recompilation. In some cases, the
+stable ABI needs to be extended with new functions. Extensions modules
+wishing to use these new APIs need to set Py_LIMITED_API to the
+PY_VERSION_HEX value of the minimum Python version they want to
+support (e.g. 0x03030000 for Python 3.3). Such modules will work
+on all subsequent Python releases, but fail to load (because of
+missing symbols) on the older releases.
+
+As of Python 3.2, the set of functions available to the limited API
+is documented in PEP 384.
+
+.. XXX copy exact list here? Into each functions definition?
diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst
index b3386ea..44c731a 100644
--- a/Doc/c-api/type.rst
+++ b/Doc/c-api/type.rst
@@ -51,13 +51,13 @@ Type Objects
modification of the attributes or base classes of the type.
-.. c:function:: int PyType_HasFeature(PyObject *o, int feature)
+.. c:function:: int PyType_HasFeature(PyTypeObject *o, int feature)
Return true if the type object *o* sets the feature *feature*. Type features
are denoted by single bit flags.
-.. c:function:: int PyType_IS_GC(PyObject *o)
+.. c:function:: int PyType_IS_GC(PyTypeObject *o)
Return true if the type object includes support for the cycle detector; this
tests the type flag :const:`Py_TPFLAGS_HAVE_GC`.
@@ -70,13 +70,14 @@ Type Objects
.. c:function:: PyObject* PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems)
- XXX: Document.
-
+ Generic handler for the :attr:`tp_alloc` slot of a type object. Use
+ Python's default memory allocation mechanism to allocate a new instance and
+ initialize all its contents to *NULL*.
.. c:function:: PyObject* PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds)
- XXX: Document.
-
+ Generic handler for the :attr:`tp_new` slot of a type object. Create a
+ new instance using the type's :attr:`tp_alloc` slot.
.. c:function:: int PyType_Ready(PyTypeObject *type)
@@ -84,3 +85,15 @@ Type Objects
their initialization. This function is responsible for adding inherited slots
from a type's base class. Return ``0`` on success, or return ``-1`` and sets an
exception on error.
+
+.. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec)
+
+ Creates and returns a heap type object from the *spec* passed to the function.
+
+.. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
+
+ Creates and returns a heap type object from the *spec*. In addition to that,
+ the created heap type contains all types contained by the *bases* tuple as base
+ types. This allows the caller to reference other heap types as base types.
+
+ .. versionadded:: 3.3
diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst
index 68ca9ad..ea1a0ad 100644
--- a/Doc/c-api/typeobj.rst
+++ b/Doc/c-api/typeobj.rst
@@ -1198,46 +1198,88 @@ Buffer Object Structures
.. sectionauthor:: Greg J. Stein <greg@lyra.org>
.. sectionauthor:: Benjamin Peterson
+.. sectionauthor:: Stefan Krah
+.. c:type:: PyBufferProcs
-The :ref:`buffer interface <bufferobjects>` exports a model where an object can expose its internal
-data.
+ This structure holds pointers to the functions required by the
+ :ref:`Buffer protocol <bufferobjects>`. The protocol defines how
+ an exporter object can expose its internal data to consumer objects.
-If an object does not export the buffer interface, then its :attr:`tp_as_buffer`
-member in the :c:type:`PyTypeObject` structure should be *NULL*. Otherwise, the
-:attr:`tp_as_buffer` will point to a :c:type:`PyBufferProcs` structure.
+.. c:member:: getbufferproc PyBufferProcs.bf_getbuffer
+ The signature of this function is::
-.. c:type:: PyBufferProcs
+ int (PyObject *exporter, Py_buffer *view, int flags);
+
+ Handle a request to *exporter* to fill in *view* as specified by *flags*.
+ Except for point (3), an implementation of this function MUST take these
+ steps:
+
+ (1) Check if the request can be met. If not, raise :c:data:`PyExc_BufferError`,
+ set :c:data:`view->obj` to *NULL* and return -1.
+
+ (2) Fill in the requested fields.
+
+ (3) Increment an internal counter for the number of exports.
+
+ (4) Set :c:data:`view->obj` to *exporter* and increment :c:data:`view->obj`.
+
+ (5) Return 0.
+
+ If *exporter* is part of a chain or tree of buffer providers, two main
+ schemes can be used:
+
+ * Re-export: Each member of the tree acts as the exporting object and
+ sets :c:data:`view->obj` to a new reference to itself.
+
+ * Redirect: The buffer request is redirected to the root object of the
+ tree. Here, :c:data:`view->obj` will be a new reference to the root
+ object.
+
+ The individual fields of *view* are described in section
+ :ref:`Buffer structure <buffer-structure>`, the rules how an exporter
+ must react to specific requests are in section
+ :ref:`Buffer request types <buffer-request-types>`.
+
+ All memory pointed to in the :c:type:`Py_buffer` structure belongs to
+ the exporter and must remain valid until there are no consumers left.
+ :c:member:`~Py_buffer.format`, :c:member:`~Py_buffer.shape`,
+ :c:member:`~Py_buffer.strides`, :c:member:`~Py_buffer.suboffsets`
+ and :c:member:`~Py_buffer.internal`
+ are read-only for the consumer.
+
+ :c:func:`PyBuffer_FillInfo` provides an easy way of exposing a simple
+ bytes buffer while dealing correctly with all request types.
+
+ :c:func:`PyObject_GetBuffer` is the interface for the consumer that
+ wraps this function.
+
+.. c:member:: releasebufferproc PyBufferProcs.bf_releasebuffer
+
+ The signature of this function is::
+
+ void (PyObject *exporter, Py_buffer *view);
- Structure used to hold the function pointers which define an implementation of
- the buffer protocol.
+ Handle a request to release the resources of the buffer. If no resources
+ need to be released, :c:member:`PyBufferProcs.bf_releasebuffer` may be
+ *NULL*. Otherwise, a standard implementation of this function will take
+ these optional steps:
- .. c:member:: getbufferproc bf_getbuffer
+ (1) Decrement an internal counter for the number of exports.
- This should fill a :c:type:`Py_buffer` with the necessary data for
- exporting the type. The signature of :data:`getbufferproc` is ``int
- (PyObject *obj, Py_buffer *view, int flags)``. *obj* is the object to
- export, *view* is the :c:type:`Py_buffer` struct to fill, and *flags* gives
- the conditions the caller wants the memory under. (See
- :c:func:`PyObject_GetBuffer` for all flags.) :c:member:`bf_getbuffer` is
- responsible for filling *view* with the appropriate information.
- (:c:func:`PyBuffer_FillView` can be used in simple cases.) See
- :c:type:`Py_buffer`\s docs for what needs to be filled in.
+ (2) If the counter is 0, free all memory associated with *view*.
+ The exporter MUST use the :c:member:`~Py_buffer.internal` field to keep
+ track of buffer-specific resources. This field is guaranteed to remain
+ constant, while a consumer MAY pass a copy of the original buffer as the
+ *view* argument.
- .. c:member:: releasebufferproc bf_releasebuffer
- This should release the resources of the buffer. The signature of
- :c:data:`releasebufferproc` is ``void (PyObject *obj, Py_buffer *view)``.
- If the :c:data:`bf_releasebuffer` function is not provided (i.e. it is
- *NULL*), then it does not ever need to be called.
+ This function MUST NOT decrement :c:data:`view->obj`, since that is
+ done automatically in :c:func:`PyBuffer_Release` (this scheme is
+ useful for breaking reference cycles).
- The exporter of the buffer interface must make sure that any memory
- pointed to in the :c:type:`Py_buffer` structure remains valid until
- releasebuffer is called. Exporters will need to define a
- :c:data:`bf_releasebuffer` function if they can re-allocate their memory,
- strides, shape, suboffsets, or format variables which they might share
- through the struct bufferinfo.
- See :c:func:`PyBuffer_Release`.
+ :c:func:`PyBuffer_Release` is the interface for the consumer that
+ wraps this function.
diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst
index 3500654..fb7ba67 100644
--- a/Doc/c-api/unicode.rst
+++ b/Doc/c-api/unicode.rst
@@ -6,38 +6,72 @@ Unicode Objects and Codecs
--------------------------
.. sectionauthor:: Marc-Andre Lemburg <mal@lemburg.com>
+.. sectionauthor:: Georg Brandl <georg@python.org>
Unicode Objects
^^^^^^^^^^^^^^^
+Since the implementation of :pep:`393` in Python 3.3, Unicode objects internally
+use a variety of representations, in order to allow handling the complete range
+of Unicode characters while staying memory efficient. There are special cases
+for strings where all code points are below 128, 256, or 65536; otherwise, code
+points must be below 1114112 (which is the full Unicode range).
+
+:c:type:`Py_UNICODE*` and UTF-8 representations are created on demand and cached
+in the Unicode object. The :c:type:`Py_UNICODE*` representation is deprecated
+and inefficient; it should be avoided in performance- or memory-sensitive
+situations.
+
+Due to the transition between the old APIs and the new APIs, unicode objects
+can internally be in two states depending on how they were created:
+
+* "canonical" unicode objects are all objects created by a non-deprecated
+ unicode API. They use the most efficient representation allowed by the
+ implementation.
+
+* "legacy" unicode objects have been created through one of the deprecated
+ APIs (typically :c:func:`PyUnicode_FromUnicode`) and only bear the
+ :c:type:`Py_UNICODE*` representation; you will have to call
+ :c:func:`PyUnicode_READY` on them before calling any other API.
+
+
Unicode Type
""""""""""""
These are the basic Unicode object types used for the Unicode implementation in
Python:
+.. c:type:: Py_UCS4
+ Py_UCS2
+ Py_UCS1
+
+ These types are typedefs for unsigned integer types wide enough to contain
+ characters of 32 bits, 16 bits and 8 bits, respectively. When dealing with
+ single Unicode characters, use :c:type:`Py_UCS4`.
+
+ .. versionadded:: 3.3
+
.. c:type:: Py_UNICODE
- This type represents the storage type which is used by Python internally as
- basis for holding Unicode ordinals. Python's default builds use a 16-bit type
- for :c:type:`Py_UNICODE` and store Unicode values internally as UCS2. It is also
- possible to build a UCS4 version of Python (most recent Linux distributions come
- with UCS4 builds of Python). These builds then use a 32-bit type for
- :c:type:`Py_UNICODE` and store Unicode data internally as UCS4. On platforms
- where :c:type:`wchar_t` is available and compatible with the chosen Python
- Unicode build variant, :c:type:`Py_UNICODE` is a typedef alias for
- :c:type:`wchar_t` to enhance native platform compatibility. On all other
- platforms, :c:type:`Py_UNICODE` is a typedef alias for either :c:type:`unsigned
- short` (UCS2) or :c:type:`unsigned long` (UCS4).
+ This is a typedef of :c:type:`wchar_t`, which is a 16-bit type or 32-bit type
+ depending on the platform.
-Note that UCS2 and UCS4 Python builds are not binary compatible. Please keep
-this in mind when writing extensions or interfaces.
+ .. versionchanged:: 3.3
+ In previous versions, this was a 16-bit type or a 32-bit type depending on
+ whether you selected a "narrow" or "wide" Unicode version of Python at
+ build time.
-.. c:type:: PyUnicodeObject
+.. c:type:: PyASCIIObject
+ PyCompactUnicodeObject
+ PyUnicodeObject
- This subtype of :c:type:`PyObject` represents a Python Unicode object.
+ These subtypes of :c:type:`PyObject` represent a Python Unicode object. In
+ almost all cases, they shouldn't be used directly, since all API functions
+ that deal with Unicode objects take and return :c:type:`PyObject` pointers.
+
+ .. versionadded:: 3.3
.. c:var:: PyTypeObject PyUnicode_Type
@@ -45,10 +79,10 @@ this in mind when writing extensions or interfaces.
This instance of :c:type:`PyTypeObject` represents the Python Unicode type. It
is exposed to Python code as ``str``.
+
The following APIs are really C macros and can be used to do fast checks and to
access internal read-only data of Unicode objects:
-
.. c:function:: int PyUnicode_Check(PyObject *o)
Return true if the object *o* is a Unicode object or an instance of a Unicode
@@ -61,28 +95,106 @@ access internal read-only data of Unicode objects:
subtype.
-.. c:function:: Py_ssize_t PyUnicode_GET_SIZE(PyObject *o)
+.. c:function:: int PyUnicode_READY(PyObject *o)
- Return the size of the object. *o* has to be a :c:type:`PyUnicodeObject` (not
- checked).
+ Ensure the string object *o* is in the "canonical" representation. This is
+ required before using any of the access macros described below.
+ .. XXX expand on when it is not required
-.. c:function:: Py_ssize_t PyUnicode_GET_DATA_SIZE(PyObject *o)
+ Returns 0 on success and -1 with an exception set on failure, which in
+ particular happens if memory allocation fails.
- Return the size of the object's internal buffer in bytes. *o* has to be a
- :c:type:`PyUnicodeObject` (not checked).
+ .. versionadded:: 3.3
-.. c:function:: Py_UNICODE* PyUnicode_AS_UNICODE(PyObject *o)
+.. c:function:: Py_ssize_t PyUnicode_GET_LENGTH(PyObject *o)
+
+ Return the length of the Unicode string, in code points. *o* has to be a
+ Unicode object in the "canonical" representation (not checked).
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: Py_UCS1* PyUnicode_1BYTE_DATA(PyObject *o)
+ Py_UCS2* PyUnicode_2BYTE_DATA(PyObject *o)
+ Py_UCS4* PyUnicode_4BYTE_DATA(PyObject *o)
+
+ Return a pointer to the canonical representation cast to UCS1, UCS2 or UCS4
+ integer types for direct character access. No checks are performed if the
+ canonical representation has the correct character size; use
+ :c:func:`PyUnicode_KIND` to select the right macro. Make sure
+ :c:func:`PyUnicode_READY` has been called before accessing this.
+
+ .. versionadded:: 3.3
+
+
+.. c:macro:: PyUnicode_WCHAR_KIND
+ PyUnicode_1BYTE_KIND
+ PyUnicode_2BYTE_KIND
+ PyUnicode_4BYTE_KIND
+
+ Return values of the :c:func:`PyUnicode_KIND` macro.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: int PyUnicode_KIND(PyObject *o)
+
+ Return one of the PyUnicode kind constants (see above) that indicate how many
+ bytes per character this Unicode object uses to store its data. *o* has to
+ be a Unicode object in the "canonical" representation (not checked).
+
+ .. XXX document "0" return value?
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: void* PyUnicode_DATA(PyObject *o)
+
+ Return a void pointer to the raw unicode buffer. *o* has to be a Unicode
+ object in the "canonical" representation (not checked).
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: void PyUnicode_WRITE(int kind, void *data, Py_ssize_t index, \
+ Py_UCS4 value)
+
+ Write into a canonical representation *data* (as obtained with
+ :c:func:`PyUnicode_DATA`). This macro does not do any sanity checks and is
+ intended for usage in loops. The caller should cache the *kind* value and
+ *data* pointer as obtained from other macro calls. *index* is the index in
+ the string (starts at 0) and *value* is the new code point value which should
+ be written to that location.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: Py_UCS4 PyUnicode_READ(int kind, void *data, Py_ssize_t index)
+
+ Read a code point from a canonical representation *data* (as obtained with
+ :c:func:`PyUnicode_DATA`). No checks or ready calls are performed.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: Py_UCS4 PyUnicode_READ_CHAR(PyObject *o, Py_ssize_t index)
+
+ Read a character from a Unicode object *o*, which must be in the "canonical"
+ representation. This is less efficient than :c:func:`PyUnicode_READ` if you
+ do multiple consecutive reads.
+
+ .. versionadded:: 3.3
- Return a pointer to the internal :c:type:`Py_UNICODE` buffer of the object. *o*
- has to be a :c:type:`PyUnicodeObject` (not checked).
+.. c:function:: PyUnicode_MAX_CHAR_VALUE(PyObject *o)
-.. c:function:: const char* PyUnicode_AS_DATA(PyObject *o)
+ Return the maximum code point that is suitable for creating another string
+ based on *o*, which must be in the "canonical" representation. This is
+ always an approximation but more efficient than iterating over the string.
- Return a pointer to the internal buffer of the object. *o* has to be a
- :c:type:`PyUnicodeObject` (not checked).
+ .. versionadded:: 3.3
.. c:function:: int PyUnicode_ClearFreeList()
@@ -90,6 +202,46 @@ access internal read-only data of Unicode objects:
Clear the free list. Return the total number of freed items.
+.. c:function:: Py_ssize_t PyUnicode_GET_SIZE(PyObject *o)
+
+ Return the size of the deprecated :c:type:`Py_UNICODE` representation, in
+ code units (this includes surrogate pairs as 2 units). *o* has to be a
+ Unicode object (not checked).
+
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style Unicode API, please migrate to using
+ :c:func:`PyUnicode_GET_LENGTH`.
+
+
+.. c:function:: Py_ssize_t PyUnicode_GET_DATA_SIZE(PyObject *o)
+
+ Return the size of the deprecated :c:type:`Py_UNICODE` representation in
+ bytes. *o* has to be a Unicode object (not checked).
+
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style Unicode API, please migrate to using
+ :c:func:`PyUnicode_GET_LENGTH`.
+
+
+.. c:function:: Py_UNICODE* PyUnicode_AS_UNICODE(PyObject *o)
+ const char* PyUnicode_AS_DATA(PyObject *o)
+
+ Return a pointer to a :c:type:`Py_UNICODE` representation of the object. The
+ ``AS_DATA`` form casts the pointer to :c:type:`const char *`. *o* has to be
+ a Unicode object (not checked).
+
+ .. versionchanged:: 3.3
+ This macro is now inefficient -- because in many cases the
+ :c:type:`Py_UNICODE` representation does not exist and needs to be created
+ -- and can fail (return *NULL* with an exception set). Try to port the
+ code to use the new :c:func:`PyUnicode_nBYTE_DATA` macros or use
+ :c:func:`PyUnicode_WRITE` or :c:func:`PyUnicode_READ`.
+
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style Unicode API, please migrate to using the
+ :c:func:`PyUnicode_nBYTE_DATA` family of macros.
+
+
Unicode Character Properties
""""""""""""""""""""""""""""
@@ -166,16 +318,25 @@ These APIs can be used for fast direct character conversions:
Return the character *ch* converted to lower case.
+ .. deprecated:: 3.3
+ This function uses simple case mappings.
+
.. c:function:: Py_UNICODE Py_UNICODE_TOUPPER(Py_UNICODE ch)
Return the character *ch* converted to upper case.
+ .. deprecated:: 3.3
+ This function uses simple case mappings.
+
.. c:function:: Py_UNICODE Py_UNICODE_TOTITLE(Py_UNICODE ch)
Return the character *ch* converted to title case.
+ .. deprecated:: 3.3
+ This function uses simple case mappings.
+
.. c:function:: int Py_UNICODE_TODECIMAL(Py_UNICODE ch)
@@ -195,31 +356,66 @@ These APIs can be used for fast direct character conversions:
possible. This macro does not raise exceptions.
-Plain Py_UNICODE
-""""""""""""""""
+These APIs can be used to work with surrogates:
+
+.. c:macro:: Py_UNICODE_IS_SURROGATE(ch)
+
+ Check if *ch* is a surrogate (``0xD800 <= ch <= 0xDFFF``).
+
+.. c:macro:: Py_UNICODE_IS_HIGH_SURROGATE(ch)
+
+ Check if *ch* is an high surrogate (``0xD800 <= ch <= 0xDBFF``).
+
+.. c:macro:: Py_UNICODE_IS_LOW_SURROGATE(ch)
+
+ Check if *ch* is a low surrogate (``0xDC00 <= ch <= 0xDFFF``).
+
+.. c:macro:: Py_UNICODE_JOIN_SURROGATES(high, low)
+
+ Join two surrogate characters and return a single Py_UCS4 value.
+ *high* and *low* are respectively the leading and trailing surrogates in a
+ surrogate pair.
+
+
+Creating and accessing Unicode strings
+""""""""""""""""""""""""""""""""""""""
To create Unicode objects and access their basic sequence properties, use these
APIs:
+.. c:function:: PyObject* PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar)
-.. c:function:: PyObject* PyUnicode_FromUnicode(const Py_UNICODE *u, Py_ssize_t size)
+ Create a new Unicode object. *maxchar* should be the true maximum code point
+ to be placed in the string. As an approximation, it can be rounded up to the
+ nearest value in the sequence 127, 255, 65535, 1114111.
- Create a Unicode object from the Py_UNICODE buffer *u* of the given size. *u*
- may be *NULL* which causes the contents to be undefined. It is the user's
- responsibility to fill in the needed data. The buffer is copied into the new
- object. If the buffer is not *NULL*, the return value might be a shared object.
- Therefore, modification of the resulting Unicode object is only allowed when *u*
- is *NULL*.
+ This is the recommended way to allocate a new Unicode object. Objects
+ created using this function are not resizable.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: PyObject* PyUnicode_FromKindAndData(int kind, const void *buffer, \
+ Py_ssize_t size)
+
+ Create a new Unicode object with the given *kind* (possible values are
+ :c:macro:`PyUnicode_1BYTE_KIND` etc., as returned by
+ :c:func:`PyUnicode_KIND`). The *buffer* must point to an array of *size*
+ units of 1, 2 or 4 bytes per character, as given by the kind.
+
+ .. versionadded:: 3.3
.. c:function:: PyObject* PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size)
- Create a Unicode object from the char buffer *u*. The bytes will be interpreted
- as being UTF-8 encoded. *u* may also be *NULL* which
- causes the contents to be undefined. It is the user's responsibility to fill in
- the needed data. The buffer is copied into the new object. If the buffer is not
- *NULL*, the return value might be a shared object. Therefore, modification of
- the resulting Unicode object is only allowed when *u* is *NULL*.
+ Create a Unicode object from the char buffer *u*. The bytes will be
+ interpreted as being UTF-8 encoded. The buffer is copied into the new
+ object. If the buffer is not *NULL*, the return value might be a shared
+ object, i.e. modification of the data is not allowed.
+
+ If *u* is *NULL*, this function behaves like :c:func:`PyUnicode_FromUnicode`
+ with the buffer set to *NULL*. This usage is deprecated in favor of
+ :c:func:`PyUnicode_New`.
.. c:function:: PyObject *PyUnicode_FromString(const char *u)
@@ -260,18 +456,27 @@ APIs:
| :attr:`%ld` | long | Exactly equivalent to |
| | | ``printf("%ld")``. |
+-------------------+---------------------+--------------------------------+
+ | :attr:`%li` | long | Exactly equivalent to |
+ | | | ``printf("%li")``. |
+ +-------------------+---------------------+--------------------------------+
| :attr:`%lu` | unsigned long | Exactly equivalent to |
| | | ``printf("%lu")``. |
+-------------------+---------------------+--------------------------------+
| :attr:`%lld` | long long | Exactly equivalent to |
| | | ``printf("%lld")``. |
+-------------------+---------------------+--------------------------------+
+ | :attr:`%lli` | long long | Exactly equivalent to |
+ | | | ``printf("%lli")``. |
+ +-------------------+---------------------+--------------------------------+
| :attr:`%llu` | unsigned long long | Exactly equivalent to |
| | | ``printf("%llu")``. |
+-------------------+---------------------+--------------------------------+
| :attr:`%zd` | Py_ssize_t | Exactly equivalent to |
| | | ``printf("%zd")``. |
+-------------------+---------------------+--------------------------------+
+ | :attr:`%zi` | Py_ssize_t | Exactly equivalent to |
+ | | | ``printf("%zi")``. |
+ +-------------------+---------------------+--------------------------------+
| :attr:`%zu` | size_t | Exactly equivalent to |
| | | ``printf("%zu")``. |
+-------------------+---------------------+--------------------------------+
@@ -322,27 +527,178 @@ APIs:
.. versionchanged:: 3.2
Support for ``"%lld"`` and ``"%llu"`` added.
+ .. versionchanged:: 3.3
+ Support for ``"%li"``, ``"%lli"`` and ``"%zi"`` added.
+
.. c:function:: PyObject* PyUnicode_FromFormatV(const char *format, va_list vargs)
Identical to :c:func:`PyUnicode_FromFormat` except that it takes exactly two
arguments.
+
+.. c:function:: PyObject* PyUnicode_FromEncodedObject(PyObject *obj, \
+ const char *encoding, const char *errors)
+
+ Coerce an encoded object *obj* to an Unicode object and return a reference with
+ incremented refcount.
+
+ :class:`bytes`, :class:`bytearray` and other char buffer compatible objects
+ are decoded according to the given *encoding* and using the error handling
+ defined by *errors*. Both can be *NULL* to have the interface use the default
+ values (see the next section for details).
+
+ All other objects, including Unicode objects, cause a :exc:`TypeError` to be
+ set.
+
+ The API returns *NULL* if there was an error. The caller is responsible for
+ decref'ing the returned objects.
+
+
+.. c:function:: Py_ssize_t PyUnicode_GetLength(PyObject *unicode)
+
+ Return the length of the Unicode object, in code points.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: int PyUnicode_CopyCharacters(PyObject *to, Py_ssize_t to_start, \
+ PyObject *to, Py_ssize_t from_start, Py_ssize_t how_many)
+
+ Copy characters from one Unicode object into another. This function performs
+ character conversion when necessary and falls back to :c:func:`memcpy` if
+ possible. Returns ``-1`` and sets an exception on error, otherwise returns
+ ``0``.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: Py_ssize_t PyUnicode_Fill(PyObject *unicode, Py_ssize_t start, \
+ Py_ssize_t length, Py_UCS4 fill_char)
+
+ Fill a string with a character: write *fill_char* into
+ ``unicode[start:start+length]``.
+
+ Fail if *fill_char* is bigger than the string maximum character, or if the
+ string has more than 1 reference.
+
+ Return the number of written character, or return ``-1`` and raise an
+ exception on error.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: int PyUnicode_WriteChar(PyObject *unicode, Py_ssize_t index, \
+ Py_UCS4 character)
+
+ Write a character to a string. The string must have been created through
+ :c:func:`PyUnicode_New`. Since Unicode strings are supposed to be immutable,
+ the string must not be shared, or have been hashed yet.
+
+ This function checks that *unicode* is a Unicode object, that the index is
+ not out of bounds, and that the object can be modified safely (i.e. that it
+ its reference count is one), in contrast to the macro version
+ :c:func:`PyUnicode_WRITE_CHAR`.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: Py_UCS4 PyUnicode_ReadChar(PyObject *unicode, Py_ssize_t index)
+
+ Read a character from a string. This function checks that *unicode* is a
+ Unicode object and the index is not out of bounds, in contrast to the macro
+ version :c:func:`PyUnicode_READ_CHAR`.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: PyObject* PyUnicode_Substring(PyObject *str, Py_ssize_t start, \
+ Py_ssize_t end)
+
+ Return a substring of *str*, from character index *start* (included) to
+ character index *end* (excluded). Negative indices are not supported.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: Py_UCS4* PyUnicode_AsUCS4(PyObject *u, Py_UCS4 *buffer, \
+ Py_ssize_t buflen, int copy_null)
+
+ Copy the string *u* into a UCS4 buffer, including a null character, if
+ *copy_null* is set. Returns *NULL* and sets an exception on error (in
+ particular, a :exc:`ValueError` if *buflen* is smaller than the length of
+ *u*). *buffer* is returned on success.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: Py_UCS4* PyUnicode_AsUCS4Copy(PyObject *u)
+
+ Copy the string *u* into a new UCS4 buffer that is allocated using
+ :c:func:`PyMem_Malloc`. If this fails, *NULL* is returned with a
+ :exc:`MemoryError` set.
+
+ .. versionadded:: 3.3
+
+
+Deprecated Py_UNICODE APIs
+""""""""""""""""""""""""""
+
+.. deprecated-removed:: 3.3 4.0
+
+These API functions are deprecated with the implementation of :pep:`393`.
+Extension modules can continue using them, as they will not be removed in Python
+3.x, but need to be aware that their use can now cause performance and memory hits.
+
+
+.. c:function:: PyObject* PyUnicode_FromUnicode(const Py_UNICODE *u, Py_ssize_t size)
+
+ Create a Unicode object from the Py_UNICODE buffer *u* of the given size. *u*
+ may be *NULL* which causes the contents to be undefined. It is the user's
+ responsibility to fill in the needed data. The buffer is copied into the new
+ object.
+
+ If the buffer is not *NULL*, the return value might be a shared object.
+ Therefore, modification of the resulting Unicode object is only allowed when
+ *u* is *NULL*.
+
+ If the buffer is *NULL*, :c:func:`PyUnicode_READY` must be called once the
+ string content has been filled before using any of the access macros such as
+ :c:func:`PyUnicode_KIND`.
+
+ Please migrate to using :c:func:`PyUnicode_FromKindAndData` or
+ :c:func:`PyUnicode_New`.
+
+
+.. c:function:: Py_UNICODE* PyUnicode_AsUnicode(PyObject *unicode)
+
+ Return a read-only pointer to the Unicode object's internal
+ :c:type:`Py_UNICODE` buffer, or *NULL* on error. This will create the
+ :c:type:`Py_UNICODE*` representation of the object if it is not yet
+ available. Note that the resulting :c:type:`Py_UNICODE` string may contain
+ embedded null characters, which would cause the string to be truncated when
+ used in most C functions.
+
+ Please migrate to using :c:func:`PyUnicode_AsUCS4`,
+ :c:func:`PyUnicode_Substring`, :c:func:`PyUnicode_ReadChar` or similar new
+ APIs.
+
+
.. c:function:: PyObject* PyUnicode_TransformDecimalToASCII(Py_UNICODE *s, Py_ssize_t size)
Create a Unicode object by replacing all decimal digits in
:c:type:`Py_UNICODE` buffer of the given *size* by ASCII digits 0--9
- according to their decimal value. Return *NULL* if an exception
- occurs.
+ according to their decimal value. Return *NULL* if an exception occurs.
-.. c:function:: Py_UNICODE* PyUnicode_AsUnicode(PyObject *unicode)
+.. c:function:: Py_UNICODE* PyUnicode_AsUnicodeAndSize(PyObject *unicode, Py_ssize_t *size)
- Return a read-only pointer to the Unicode object's internal
- :c:type:`Py_UNICODE` buffer, *NULL* if *unicode* is not a Unicode object.
- Note that the resulting :c:type:`Py_UNICODE*` string may contain embedded
- null characters, which would cause the string to be truncated when used in
- most C functions.
+ Like :c:func:`PyUnicode_AsUnicode`, but also saves the :c:func:`Py_UNICODE`
+ array length in *size*. Note that the resulting :c:type:`Py_UNICODE*` string
+ may contain embedded null characters, which would cause the string to be
+ truncated when used in most C functions.
+
+ .. versionadded:: 3.3
.. c:function:: Py_UNICODE* PyUnicode_AsUnicodeCopy(PyObject *unicode)
@@ -350,44 +706,76 @@ APIs:
Create a copy of a Unicode string ending with a nul character. Return *NULL*
and raise a :exc:`MemoryError` exception on memory allocation failure,
otherwise return a new allocated buffer (use :c:func:`PyMem_Free` to free
- the buffer). Note that the resulting :c:type:`Py_UNICODE*` string may contain
- embedded null characters, which would cause the string to be truncated when
- used in most C functions.
+ the buffer). Note that the resulting :c:type:`Py_UNICODE*` string may
+ contain embedded null characters, which would cause the string to be
+ truncated when used in most C functions.
.. versionadded:: 3.2
+ Please migrate to using :c:func:`PyUnicode_AsUCS4Copy` or similar new APIs.
+
.. c:function:: Py_ssize_t PyUnicode_GetSize(PyObject *unicode)
- Return the length of the Unicode object.
+ Return the size of the deprecated :c:type:`Py_UNICODE` representation, in
+ code units (this includes surrogate pairs as 2 units).
+ Please migrate to using :c:func:`PyUnicode_GetLength`.
-.. c:function:: PyObject* PyUnicode_FromEncodedObject(PyObject *obj, const char *encoding, const char *errors)
- Coerce an encoded object *obj* to an Unicode object and return a reference with
- incremented refcount.
+.. c:function:: PyObject* PyUnicode_FromObject(PyObject *obj)
- :class:`bytes`, :class:`bytearray` and other char buffer compatible objects
- are decoded according to the given *encoding* and using the error handling
- defined by *errors*. Both can be *NULL* to have the interface use the default
- values (see the next section for details).
+ Shortcut for ``PyUnicode_FromEncodedObject(obj, NULL, "strict")`` which is used
+ throughout the interpreter whenever coercion to Unicode is needed.
- All other objects, including Unicode objects, cause a :exc:`TypeError` to be
- set.
- The API returns *NULL* if there was an error. The caller is responsible for
- decref'ing the returned objects.
+Locale Encoding
+"""""""""""""""
+The current locale encoding can be used to decode text from the operating
+system.
-.. c:function:: PyObject* PyUnicode_FromObject(PyObject *obj)
+.. c:function:: PyObject* PyUnicode_DecodeLocaleAndSize(const char *str, Py_ssize_t len, int surrogateescape)
+
+ Decode a string from the current locale encoding. The decoder is strict if
+ *surrogateescape* is equal to zero, otherwise it uses the
+ ``'surrogateescape'`` error handler (:pep:`383`) to escape undecodable
+ bytes. If a byte sequence can be decoded as a surrogate character and
+ *surrogateescape* is not equal to zero, the byte sequence is escaped using
+ the ``'surrogateescape'`` error handler instead of being decoded. *str*
+ must end with a null character but cannot contain embedded null characters.
+
+ .. seealso::
+
+ Use :c:func:`PyUnicode_DecodeFSDefaultAndSize` to decode a string from
+ :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at
+ Python startup).
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: PyObject* PyUnicode_DecodeLocale(const char *str, int surrogateescape)
+
+ Similar to :c:func:`PyUnicode_DecodeLocaleAndSize`, but compute the string
+ length using :c:func:`strlen`.
+
+ .. versionadded:: 3.3
- Shortcut for ``PyUnicode_FromEncodedObject(obj, NULL, "strict")`` which is used
- throughout the interpreter whenever coercion to Unicode is needed.
-If the platform supports :c:type:`wchar_t` and provides a header file wchar.h,
-Python can interface directly to this type using the following functions.
-Support is optimized if Python's own :c:type:`Py_UNICODE` type is identical to
-the system's :c:type:`wchar_t`.
+.. c:function:: PyObject* PyUnicode_EncodeLocale(PyObject *unicode, int surrogateescape)
+
+ Encode a Unicode object to the current locale encoding. The encoder is
+ strict if *surrogateescape* is equal to zero, otherwise it uses the
+ ``'surrogateescape'`` error handler (:pep:`383`). Return a :class:`bytes`
+ object. *str* cannot contain embedded null characters.
+
+ .. seealso::
+
+ Use :c:func:`PyUnicode_EncodeFSDefault` to encode a string to
+ :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at
+ Python startup).
+
+ .. versionadded:: 3.3
File System Encoding
@@ -430,6 +818,13 @@ used, passing :c:func:`PyUnicode_FSDecoder` as the conversion function:
If :c:data:`Py_FileSystemDefaultEncoding` is not set, fall back to the
locale encoding.
+ .. seealso::
+
+ :c:data:`Py_FileSystemDefaultEncoding` is initialized at startup from the
+ locale encoding and cannot be modified later. If you need to decode a
+ string from the current locale encoding, use
+ :c:func:`PyUnicode_DecodeLocaleAndSize`.
+
.. versionchanged:: 3.2
Use ``'strict'`` error handler on Windows.
@@ -458,6 +853,13 @@ used, passing :c:func:`PyUnicode_FSDecoder` as the conversion function:
If :c:data:`Py_FileSystemDefaultEncoding` is not set, fall back to the
locale encoding.
+ .. seealso::
+
+ :c:data:`Py_FileSystemDefaultEncoding` is initialized at startup from the
+ locale encoding and cannot be modified later. If you need to encode a
+ string to the current locale encoding, use
+ :c:func:`PyUnicode_EncodeLocale`.
+
.. versionadded:: 3.2
@@ -479,9 +881,9 @@ wchar_t Support
Copy the Unicode object contents into the :c:type:`wchar_t` buffer *w*. At most
*size* :c:type:`wchar_t` characters are copied (excluding a possibly trailing
0-termination character). Return the number of :c:type:`wchar_t` characters
- copied or -1 in case of an error. Note that the resulting :c:type:`wchar_t`
+ copied or -1 in case of an error. Note that the resulting :c:type:`wchar_t*`
string may or may not be 0-terminated. It is the responsibility of the caller
- to make sure that the :c:type:`wchar_t` string is 0-terminated in case this is
+ to make sure that the :c:type:`wchar_t*` string is 0-terminated in case this is
required by the application. Also, note that the :c:type:`wchar_t*` string
might contain null characters, which would cause the string to be truncated
when used with most C functions.
@@ -497,12 +899,32 @@ wchar_t Support
Returns a buffer allocated by :c:func:`PyMem_Alloc` (use
:c:func:`PyMem_Free` to free it) on success. On error, returns *NULL*,
*\*size* is undefined and raises a :exc:`MemoryError`. Note that the
- resulting :c:type:`wchar_t*` string might contain null characters, which
+ resulting :c:type:`wchar_t` string might contain null characters, which
would cause the string to be truncated when used with most C functions.
.. versionadded:: 3.2
+UCS4 Support
+""""""""""""
+
+.. versionadded:: 3.3
+
+.. XXX are these meant to be public?
+
+.. c:function:: size_t Py_UCS4_strlen(const Py_UCS4 *u)
+ Py_UCS4* Py_UCS4_strcpy(Py_UCS4 *s1, const Py_UCS4 *s2)
+ Py_UCS4* Py_UCS4_strncpy(Py_UCS4 *s1, const Py_UCS4 *s2, size_t n)
+ Py_UCS4* Py_UCS4_strcat(Py_UCS4 *s1, const Py_UCS4 *s2)
+ int Py_UCS4_strcmp(const Py_UCS4 *s1, const Py_UCS4 *s2)
+ int Py_UCS4_strncmp(const Py_UCS4 *s1, const Py_UCS4 *s2, size_t n)
+ Py_UCS4* Py_UCS4_strchr(const Py_UCS4 *s, Py_UCS4 c)
+ Py_UCS4* Py_UCS4_strrchr(const Py_UCS4 *s, Py_UCS4 c)
+
+ These utility functions work on strings of :c:type:`Py_UCS4` characters and
+ otherwise behave like the C standard library functions with the same name.
+
+
.. _builtincodecs:
Built-in Codecs
@@ -537,7 +959,8 @@ Generic Codecs
These are the generic codec APIs:
-.. c:function:: PyObject* PyUnicode_Decode(const char *s, Py_ssize_t size, const char *encoding, const char *errors)
+.. c:function:: PyObject* PyUnicode_Decode(const char *s, Py_ssize_t size, \
+ const char *encoding, const char *errors)
Create a Unicode object by decoding *size* bytes of the encoded string *s*.
*encoding* and *errors* have the same meaning as the parameters of the same name
@@ -546,7 +969,18 @@ These are the generic codec APIs:
the codec.
-.. c:function:: PyObject* PyUnicode_Encode(const Py_UNICODE *s, Py_ssize_t size, const char *encoding, const char *errors)
+.. c:function:: PyObject* PyUnicode_AsEncodedString(PyObject *unicode, \
+ const char *encoding, const char *errors)
+
+ Encode a Unicode object and return the result as Python bytes object.
+ *encoding* and *errors* have the same meaning as the parameters of the same
+ name in the Unicode :meth:`encode` method. The codec to be used is looked up
+ using the Python codec registry. Return *NULL* if an exception was raised by
+ the codec.
+
+
+.. c:function:: PyObject* PyUnicode_Encode(const Py_UNICODE *s, Py_ssize_t size, \
+ const char *encoding, const char *errors)
Encode the :c:type:`Py_UNICODE` buffer *s* of the given *size* and return a Python
bytes object. *encoding* and *errors* have the same meaning as the
@@ -554,14 +988,9 @@ These are the generic codec APIs:
to be used is looked up using the Python codec registry. Return *NULL* if an
exception was raised by the codec.
-
-.. c:function:: PyObject* PyUnicode_AsEncodedString(PyObject *unicode, const char *encoding, const char *errors)
-
- Encode a Unicode object and return the result as Python bytes object.
- *encoding* and *errors* have the same meaning as the parameters of the same
- name in the Unicode :meth:`encode` method. The codec to be used is looked up
- using the Python codec registry. Return *NULL* if an exception was raised by
- the codec.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsEncodedString`.
UTF-8 Codecs
@@ -576,7 +1005,8 @@ These are the UTF-8 codec APIs:
*s*. Return *NULL* if an exception was raised by the codec.
-.. c:function:: PyObject* PyUnicode_DecodeUTF8Stateful(const char *s, Py_ssize_t size, const char *errors, Py_ssize_t *consumed)
+.. c:function:: PyObject* PyUnicode_DecodeUTF8Stateful(const char *s, Py_ssize_t size, \
+ const char *errors, Py_ssize_t *consumed)
If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeUTF8`. If
*consumed* is not *NULL*, trailing incomplete UTF-8 byte sequences will not be
@@ -584,18 +1014,45 @@ These are the UTF-8 codec APIs:
that have been decoded will be stored in *consumed*.
+.. c:function:: PyObject* PyUnicode_AsUTF8String(PyObject *unicode)
+
+ Encode a Unicode object using UTF-8 and return the result as Python bytes
+ object. Error handling is "strict". Return *NULL* if an exception was
+ raised by the codec.
+
+
+.. c:function:: char* PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *size)
+
+ Return a pointer to the default encoding (UTF-8) of the Unicode object, and
+ store the size of the encoded representation (in bytes) in *size*. *size*
+ can be *NULL*, in this case no size will be stored.
+
+ In the case of an error, *NULL* is returned with an exception set and no
+ *size* is stored.
+
+ This caches the UTF-8 representation of the string in the Unicode object, and
+ subsequent calls will return a pointer to the same buffer. The caller is not
+ responsible for deallocating the buffer.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: char* PyUnicode_AsUTF8(PyObject *unicode)
+
+ As :c:func:`PyUnicode_AsUTF8AndSize`, but does not store the size.
+
+ .. versionadded:: 3.3
+
+
.. c:function:: PyObject* PyUnicode_EncodeUTF8(const Py_UNICODE *s, Py_ssize_t size, const char *errors)
Encode the :c:type:`Py_UNICODE` buffer *s* of the given *size* using UTF-8 and
return a Python bytes object. Return *NULL* if an exception was raised by
the codec.
-
-.. c:function:: PyObject* PyUnicode_AsUTF8String(PyObject *unicode)
-
- Encode a Unicode object using UTF-8 and return the result as Python bytes
- object. Error handling is "strict". Return *NULL* if an exception was
- raised by the codec.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsUTF8String` or :c:func:`PyUnicode_AsUTF8AndSize`.
UTF-32 Codecs
@@ -604,7 +1061,8 @@ UTF-32 Codecs
These are the UTF-32 codec APIs:
-.. c:function:: PyObject* PyUnicode_DecodeUTF32(const char *s, Py_ssize_t size, const char *errors, int *byteorder)
+.. c:function:: PyObject* PyUnicode_DecodeUTF32(const char *s, Py_ssize_t size, \
+ const char *errors, int *byteorder)
Decode *size* bytes from a UTF-32 encoded buffer string and return the
corresponding Unicode object. *errors* (if non-*NULL*) defines the error
@@ -632,7 +1090,8 @@ These are the UTF-32 codec APIs:
Return *NULL* if an exception was raised by the codec.
-.. c:function:: PyObject* PyUnicode_DecodeUTF32Stateful(const char *s, Py_ssize_t size, const char *errors, int *byteorder, Py_ssize_t *consumed)
+.. c:function:: PyObject* PyUnicode_DecodeUTF32Stateful(const char *s, Py_ssize_t size, \
+ const char *errors, int *byteorder, Py_ssize_t *consumed)
If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeUTF32`. If
*consumed* is not *NULL*, :c:func:`PyUnicode_DecodeUTF32Stateful` will not treat
@@ -641,7 +1100,15 @@ These are the UTF-32 codec APIs:
that have been decoded will be stored in *consumed*.
-.. c:function:: PyObject* PyUnicode_EncodeUTF32(const Py_UNICODE *s, Py_ssize_t size, const char *errors, int byteorder)
+.. c:function:: PyObject* PyUnicode_AsUTF32String(PyObject *unicode)
+
+ Return a Python byte string using the UTF-32 encoding in native byte
+ order. The string always starts with a BOM mark. Error handling is "strict".
+ Return *NULL* if an exception was raised by the codec.
+
+
+.. c:function:: PyObject* PyUnicode_EncodeUTF32(const Py_UNICODE *s, Py_ssize_t size, \
+ const char *errors, int byteorder)
Return a Python bytes object holding the UTF-32 encoded value of the Unicode
data in *s*. Output is written according to the following byte order::
@@ -658,12 +1125,9 @@ These are the UTF-32 codec APIs:
Return *NULL* if an exception was raised by the codec.
-
-.. c:function:: PyObject* PyUnicode_AsUTF32String(PyObject *unicode)
-
- Return a Python byte string using the UTF-32 encoding in native byte
- order. The string always starts with a BOM mark. Error handling is "strict".
- Return *NULL* if an exception was raised by the codec.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsUTF32String`.
UTF-16 Codecs
@@ -672,7 +1136,8 @@ UTF-16 Codecs
These are the UTF-16 codec APIs:
-.. c:function:: PyObject* PyUnicode_DecodeUTF16(const char *s, Py_ssize_t size, const char *errors, int *byteorder)
+.. c:function:: PyObject* PyUnicode_DecodeUTF16(const char *s, Py_ssize_t size, \
+ const char *errors, int *byteorder)
Decode *size* bytes from a UTF-16 encoded buffer string and return the
corresponding Unicode object. *errors* (if non-*NULL*) defines the error
@@ -699,7 +1164,8 @@ These are the UTF-16 codec APIs:
Return *NULL* if an exception was raised by the codec.
-.. c:function:: PyObject* PyUnicode_DecodeUTF16Stateful(const char *s, Py_ssize_t size, const char *errors, int *byteorder, Py_ssize_t *consumed)
+.. c:function:: PyObject* PyUnicode_DecodeUTF16Stateful(const char *s, Py_ssize_t size, \
+ const char *errors, int *byteorder, Py_ssize_t *consumed)
If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeUTF16`. If
*consumed* is not *NULL*, :c:func:`PyUnicode_DecodeUTF16Stateful` will not treat
@@ -708,7 +1174,15 @@ These are the UTF-16 codec APIs:
number of bytes that have been decoded will be stored in *consumed*.
-.. c:function:: PyObject* PyUnicode_EncodeUTF16(const Py_UNICODE *s, Py_ssize_t size, const char *errors, int byteorder)
+.. c:function:: PyObject* PyUnicode_AsUTF16String(PyObject *unicode)
+
+ Return a Python byte string using the UTF-16 encoding in native byte
+ order. The string always starts with a BOM mark. Error handling is "strict".
+ Return *NULL* if an exception was raised by the codec.
+
+
+.. c:function:: PyObject* PyUnicode_EncodeUTF16(const Py_UNICODE *s, Py_ssize_t size, \
+ const char *errors, int byteorder)
Return a Python bytes object holding the UTF-16 encoded value of the Unicode
data in *s*. Output is written according to the following byte order::
@@ -726,12 +1200,9 @@ These are the UTF-16 codec APIs:
Return *NULL* if an exception was raised by the codec.
-
-.. c:function:: PyObject* PyUnicode_AsUTF16String(PyObject *unicode)
-
- Return a Python byte string using the UTF-16 encoding in native byte
- order. The string always starts with a BOM mark. Error handling is "strict".
- Return *NULL* if an exception was raised by the codec.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsUTF16String`.
UTF-7 Codecs
@@ -746,7 +1217,8 @@ These are the UTF-7 codec APIs:
*s*. Return *NULL* if an exception was raised by the codec.
-.. c:function:: PyObject* PyUnicode_DecodeUTF7Stateful(const char *s, Py_ssize_t size, const char *errors, Py_ssize_t *consumed)
+.. c:function:: PyObject* PyUnicode_DecodeUTF7Stateful(const char *s, Py_ssize_t size, \
+ const char *errors, Py_ssize_t *consumed)
If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeUTF7`. If
*consumed* is not *NULL*, trailing incomplete UTF-7 base-64 sections will not
@@ -754,7 +1226,8 @@ These are the UTF-7 codec APIs:
bytes that have been decoded will be stored in *consumed*.
-.. c:function:: PyObject* PyUnicode_EncodeUTF7(const Py_UNICODE *s, Py_ssize_t size, int base64SetO, int base64WhiteSpace, const char *errors)
+.. c:function:: PyObject* PyUnicode_EncodeUTF7(const Py_UNICODE *s, Py_ssize_t size, \
+ int base64SetO, int base64WhiteSpace, const char *errors)
Encode the :c:type:`Py_UNICODE` buffer of the given size using UTF-7 and
return a Python bytes object. Return *NULL* if an exception was raised by
@@ -765,6 +1238,11 @@ These are the UTF-7 codec APIs:
nonzero, whitespace will be encoded in base-64. Both are set to zero for the
Python "utf-7" codec.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API.
+
+ .. XXX replace with what?
+
Unicode-Escape Codecs
"""""""""""""""""""""
@@ -772,24 +1250,29 @@ Unicode-Escape Codecs
These are the "Unicode Escape" codec APIs:
-.. c:function:: PyObject* PyUnicode_DecodeUnicodeEscape(const char *s, Py_ssize_t size, const char *errors)
+.. c:function:: PyObject* PyUnicode_DecodeUnicodeEscape(const char *s, \
+ Py_ssize_t size, const char *errors)
Create a Unicode object by decoding *size* bytes of the Unicode-Escape encoded
string *s*. Return *NULL* if an exception was raised by the codec.
+.. c:function:: PyObject* PyUnicode_AsUnicodeEscapeString(PyObject *unicode)
+
+ Encode a Unicode object using Unicode-Escape and return the result as Python
+ string object. Error handling is "strict". Return *NULL* if an exception was
+ raised by the codec.
+
+
.. c:function:: PyObject* PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s, Py_ssize_t size)
Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Unicode-Escape and
return a Python string object. Return *NULL* if an exception was raised by the
codec.
-
-.. c:function:: PyObject* PyUnicode_AsUnicodeEscapeString(PyObject *unicode)
-
- Encode a Unicode object using Unicode-Escape and return the result as Python
- string object. Error handling is "strict". Return *NULL* if an exception was
- raised by the codec.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsUnicodeEscapeString`.
Raw-Unicode-Escape Codecs
@@ -798,19 +1281,13 @@ Raw-Unicode-Escape Codecs
These are the "Raw Unicode Escape" codec APIs:
-.. c:function:: PyObject* PyUnicode_DecodeRawUnicodeEscape(const char *s, Py_ssize_t size, const char *errors)
+.. c:function:: PyObject* PyUnicode_DecodeRawUnicodeEscape(const char *s, \
+ Py_ssize_t size, const char *errors)
Create a Unicode object by decoding *size* bytes of the Raw-Unicode-Escape
encoded string *s*. Return *NULL* if an exception was raised by the codec.
-.. c:function:: PyObject* PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s, Py_ssize_t size, const char *errors)
-
- Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Raw-Unicode-Escape
- and return a Python string object. Return *NULL* if an exception was raised by
- the codec.
-
-
.. c:function:: PyObject* PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode)
Encode a Unicode object using Raw-Unicode-Escape and return the result as
@@ -818,6 +1295,18 @@ These are the "Raw Unicode Escape" codec APIs:
was raised by the codec.
+.. c:function:: PyObject* PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s, \
+ Py_ssize_t size, const char *errors)
+
+ Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Raw-Unicode-Escape
+ and return a Python string object. Return *NULL* if an exception was raised by
+ the codec.
+
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsRawUnicodeEscapeString`.
+
+
Latin-1 Codecs
""""""""""""""
@@ -831,18 +1320,22 @@ ordinals and only these are accepted by the codecs during encoding.
*s*. Return *NULL* if an exception was raised by the codec.
+.. c:function:: PyObject* PyUnicode_AsLatin1String(PyObject *unicode)
+
+ Encode a Unicode object using Latin-1 and return the result as Python bytes
+ object. Error handling is "strict". Return *NULL* if an exception was
+ raised by the codec.
+
+
.. c:function:: PyObject* PyUnicode_EncodeLatin1(const Py_UNICODE *s, Py_ssize_t size, const char *errors)
Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Latin-1 and
return a Python bytes object. Return *NULL* if an exception was raised by
the codec.
-
-.. c:function:: PyObject* PyUnicode_AsLatin1String(PyObject *unicode)
-
- Encode a Unicode object using Latin-1 and return the result as Python bytes
- object. Error handling is "strict". Return *NULL* if an exception was
- raised by the codec.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsLatin1String`.
ASCII Codecs
@@ -858,18 +1351,22 @@ codes generate errors.
*s*. Return *NULL* if an exception was raised by the codec.
+.. c:function:: PyObject* PyUnicode_AsASCIIString(PyObject *unicode)
+
+ Encode a Unicode object using ASCII and return the result as Python bytes
+ object. Error handling is "strict". Return *NULL* if an exception was
+ raised by the codec.
+
+
.. c:function:: PyObject* PyUnicode_EncodeASCII(const Py_UNICODE *s, Py_ssize_t size, const char *errors)
Encode the :c:type:`Py_UNICODE` buffer of the given *size* using ASCII and
return a Python bytes object. Return *NULL* if an exception was raised by
the codec.
-
-.. c:function:: PyObject* PyUnicode_AsASCIIString(PyObject *unicode)
-
- Encode a Unicode object using ASCII and return the result as Python bytes
- object. Error handling is "strict". Return *NULL* if an exception was
- raised by the codec.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsASCIIString`.
Character Map Codecs
@@ -898,7 +1395,8 @@ characters to different code points.
These are the mapping codec APIs:
-.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *s, Py_ssize_t size, PyObject *mapping, const char *errors)
+.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *s, Py_ssize_t size, \
+ PyObject *mapping, const char *errors)
Create a Unicode object by decoding *size* bytes of the encoded string *s* using
the given *mapping* object. Return *NULL* if an exception was raised by the
@@ -908,13 +1406,6 @@ These are the mapping codec APIs:
treated as "undefined mapping".
-.. c:function:: PyObject* PyUnicode_EncodeCharmap(const Py_UNICODE *s, Py_ssize_t size, PyObject *mapping, const char *errors)
-
- Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given
- *mapping* object and return a Python string object. Return *NULL* if an
- exception was raised by the codec.
-
-
.. c:function:: PyObject* PyUnicode_AsCharmapString(PyObject *unicode, PyObject *mapping)
Encode a Unicode object using the given *mapping* object and return the result
@@ -924,7 +1415,8 @@ These are the mapping codec APIs:
The following codec API is special in that maps Unicode to Unicode.
-.. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, PyObject *table, const char *errors)
+.. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, \
+ PyObject *table, const char *errors)
Translate a :c:type:`Py_UNICODE` buffer of the given *size* by applying a
character mapping *table* to it and return the resulting Unicode object. Return
@@ -937,6 +1429,22 @@ The following codec API is special in that maps Unicode to Unicode.
and sequences work well. Unmapped character ordinals (ones which cause a
:exc:`LookupError`) are left untouched and are copied as-is.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API.
+
+ .. XXX replace with what?
+
+
+.. c:function:: PyObject* PyUnicode_EncodeCharmap(const Py_UNICODE *s, Py_ssize_t size, \
+ PyObject *mapping, const char *errors)
+
+ Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given
+ *mapping* object and return a Python string object. Return *NULL* if an
+ exception was raised by the codec.
+
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsCharmapString`.
MBCS codecs for Windows
@@ -953,7 +1461,8 @@ the user settings on the machine running the codec.
Return *NULL* if an exception was raised by the codec.
-.. c:function:: PyObject* PyUnicode_DecodeMBCSStateful(const char *s, int size, const char *errors, int *consumed)
+.. c:function:: PyObject* PyUnicode_DecodeMBCSStateful(const char *s, int size, \
+ const char *errors, int *consumed)
If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeMBCS`. If
*consumed* is not *NULL*, :c:func:`PyUnicode_DecodeMBCSStateful` will not decode
@@ -961,18 +1470,31 @@ the user settings on the machine running the codec.
in *consumed*.
+.. c:function:: PyObject* PyUnicode_AsMBCSString(PyObject *unicode)
+
+ Encode a Unicode object using MBCS and return the result as Python bytes
+ object. Error handling is "strict". Return *NULL* if an exception was
+ raised by the codec.
+
+
+.. c:function:: PyObject* PyUnicode_EncodeCodePage(int code_page, PyObject *unicode, const char *errors)
+
+ Encode the Unicode object using the specified code page and return a Python
+ bytes object. Return *NULL* if an exception was raised by the codec. Use
+ :c:data:`CP_ACP` code page to get the MBCS encoder.
+
+ .. versionadded:: 3.3
+
+
.. c:function:: PyObject* PyUnicode_EncodeMBCS(const Py_UNICODE *s, Py_ssize_t size, const char *errors)
Encode the :c:type:`Py_UNICODE` buffer of the given *size* using MBCS and return
a Python bytes object. Return *NULL* if an exception was raised by the
codec.
-
-.. c:function:: PyObject* PyUnicode_AsMBCSString(PyObject *unicode)
-
- Encode a Unicode object using MBCS and return the result as Python bytes
- object. Error handling is "strict". Return *NULL* if an exception was
- raised by the codec.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsMBCSString` or :c:func:`PyUnicode_EncodeCodePage`.
Methods & Slots
@@ -1011,7 +1533,8 @@ They all return *NULL* or ``-1`` if an exception occurs.
characters are not included in the resulting strings.
-.. c:function:: PyObject* PyUnicode_Translate(PyObject *str, PyObject *table, const char *errors)
+.. c:function:: PyObject* PyUnicode_Translate(PyObject *str, PyObject *table, \
+ const char *errors)
Translate a string by applying a character mapping table to it and return the
resulting Unicode object.
@@ -1033,14 +1556,16 @@ They all return *NULL* or ``-1`` if an exception occurs.
Unicode string.
-.. c:function:: int PyUnicode_Tailmatch(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end, int direction)
+.. c:function:: int PyUnicode_Tailmatch(PyObject *str, PyObject *substr, \
+ Py_ssize_t start, Py_ssize_t end, int direction)
Return 1 if *substr* matches ``str[start:end]`` at the given tail end
(*direction* == -1 means to do a prefix match, *direction* == 1 a suffix match),
0 otherwise. Return ``-1`` if an error occurred.
-.. c:function:: Py_ssize_t PyUnicode_Find(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end, int direction)
+.. c:function:: Py_ssize_t PyUnicode_Find(PyObject *str, PyObject *substr, \
+ Py_ssize_t start, Py_ssize_t end, int direction)
Return the first position of *substr* in ``str[start:end]`` using the given
*direction* (*direction* == 1 means to do a forward search, *direction* == -1 a
@@ -1049,13 +1574,27 @@ They all return *NULL* or ``-1`` if an exception occurs.
occurred and an exception has been set.
-.. c:function:: Py_ssize_t PyUnicode_Count(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end)
+.. c:function:: Py_ssize_t PyUnicode_FindChar(PyObject *str, Py_UCS4 ch, \
+ Py_ssize_t start, Py_ssize_t end, int direction)
+
+ Return the first position of the character *ch* in ``str[start:end]`` using
+ the given *direction* (*direction* == 1 means to do a forward search,
+ *direction* == -1 a backward search). The return value is the index of the
+ first match; a value of ``-1`` indicates that no match was found, and ``-2``
+ indicates that an error occurred and an exception has been set.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: Py_ssize_t PyUnicode_Count(PyObject *str, PyObject *substr, \
+ Py_ssize_t start, Py_ssize_t end)
Return the number of non-overlapping occurrences of *substr* in
``str[start:end]``. Return ``-1`` if an error occurred.
-.. c:function:: PyObject* PyUnicode_Replace(PyObject *str, PyObject *substr, PyObject *replstr, Py_ssize_t maxcount)
+.. c:function:: PyObject* PyUnicode_Replace(PyObject *str, PyObject *substr, \
+ PyObject *replstr, Py_ssize_t maxcount)
Replace at most *maxcount* occurrences of *substr* in *str* with *replstr* and
return the resulting Unicode object. *maxcount* == -1 means replace all
@@ -1076,7 +1615,7 @@ They all return *NULL* or ``-1`` if an exception occurs.
ISO-8859-1 if it contains non-ASCII characters".
-.. c:function:: int PyUnicode_RichCompare(PyObject *left, PyObject *right, int op)
+.. c:function:: PyObject* PyUnicode_RichCompare(PyObject *left, PyObject *right, int op)
Rich compare two unicode strings and return one of the following:
@@ -1103,8 +1642,8 @@ They all return *NULL* or ``-1`` if an exception occurs.
Check whether *element* is contained in *container* and return true or false
accordingly.
- *element* has to coerce to a one element Unicode string. ``-1`` is returned if
- there was an error.
+ *element* has to coerce to a one element Unicode string. ``-1`` is returned
+ if there was an error.
.. c:function:: void PyUnicode_InternInPlace(PyObject **string)
@@ -1123,7 +1662,6 @@ They all return *NULL* or ``-1`` if an exception occurs.
.. c:function:: PyObject* PyUnicode_InternFromString(const char *v)
A combination of :c:func:`PyUnicode_FromString` and
- :c:func:`PyUnicode_InternInPlace`, returning either a new unicode string object
- that has been interned, or a new ("owned") reference to an earlier interned
- string object with the same value.
-
+ :c:func:`PyUnicode_InternInPlace`, returning either a new unicode string
+ object that has been interned, or a new ("owned") reference to an earlier
+ interned string object with the same value.
diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst
index 41cdd6b..499eb3e 100644
--- a/Doc/c-api/veryhigh.rst
+++ b/Doc/c-api/veryhigh.rst
@@ -95,12 +95,6 @@ the same library that the Python runtime is using.
leaving *closeit* set to ``0`` and *flags* set to *NULL*.
-.. c:function:: int PyRun_SimpleFileFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)
-
- This is a simplified interface to :c:func:`PyRun_SimpleFileExFlags` below,
- leaving *closeit* set to ``0``.
-
-
.. c:function:: int PyRun_SimpleFileEx(FILE *fp, const char *filename, int closeit)
This is a simplified interface to :c:func:`PyRun_SimpleFileExFlags` below,
diff --git a/Doc/conf.py b/Doc/conf.py
index 555f281..6b085e0 100644
--- a/Doc/conf.py
+++ b/Doc/conf.py
@@ -91,7 +91,7 @@ html_additional_pages = {
}
# Output an OpenSearch description file.
-html_use_opensearch = 'http://docs.python.org/3.2'
+html_use_opensearch = 'http://docs.python.org/' + version
# Additional static files.
html_static_path = ['tools/sphinxext/static']
diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat
index c7d7bd1..a1004ad 100644
--- a/Doc/data/refcounts.dat
+++ b/Doc/data/refcounts.dat
@@ -465,6 +465,11 @@ PyFunction_New:PyObject*::+1:
PyFunction_New:PyObject*:code:+1:
PyFunction_New:PyObject*:globals:+1:
+PyFunction_NewWithQualName:PyObject*::+1:
+PyFunction_NewWithQualName:PyObject*:code:+1:
+PyFunction_NewWithQualName:PyObject*:globals:+1:
+PyFunction_NewWithQualName:PyObject*:qualname:+1:
+
PyFunction_SetClosure:int:::
PyFunction_SetClosure:PyObject*:op:0:
PyFunction_SetClosure:PyObject*:closure:+1:
diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst
index e15dc76..71702e5 100644
--- a/Doc/distutils/apiref.rst
+++ b/Doc/distutils/apiref.rst
@@ -160,7 +160,7 @@ the full reference.
.. class:: Extension
The Extension class describes a single C or C++extension module in a setup
- script. It accepts the following keyword arguments in its constructor
+ script. It accepts the following keyword arguments in its constructor:
+------------------------+--------------------------------+---------------------------+
| argument name | value | type |
@@ -1157,12 +1157,11 @@ other utility module.
.. function:: grok_environment_error(exc[, prefix='error: '])
- Generate a useful error message from an :exc:`EnvironmentError` (:exc:`IOError`
- or :exc:`OSError`) exception object. Handles Python 1.5.1 and later styles,
- and does what it can to deal with exception objects that don't have a filename
- (which happens when the error is due to a two-file operation, such as
- :func:`rename` or :func:`link`). Returns the error message as a string
- prefixed with *prefix*.
+ Generate a useful error message from an :exc:`OSError` exception object.
+ Handles Python 1.5.1 and later styles, and does what it can to deal with
+ exception objects that don't have a filename (which happens when the error
+ is due to a two-file operation, such as :func:`rename` or :func:`link`).
+ Returns the error message as a string prefixed with *prefix*.
.. function:: split_quoted(s)
diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst
index c4ced1a..3261580 100644
--- a/Doc/extending/extending.rst
+++ b/Doc/extending/extending.rst
@@ -321,7 +321,7 @@ parameters to be passed in as a tuple acceptable for parsing via
The :const:`METH_KEYWORDS` bit may be set in the third field if keyword
arguments should be passed to the function. In this case, the C function should
-accept a third ``PyObject \*`` parameter which will be a dictionary of keywords.
+accept a third ``PyObject *`` parameter which will be a dictionary of keywords.
Use :c:func:`PyArg_ParseTupleAndKeywords` to parse the arguments to such a
function.
@@ -384,9 +384,6 @@ optionally followed by an import of the module::
imports it. */
PyImport_ImportModule("spam");
-An example may be found in the file :file:`Demo/embed/demo.c` in the Python
-source distribution.
-
.. note::
Removing entries from ``sys.modules`` or importing compiled modules into
diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst
index 8f18253..b90d35c 100644
--- a/Doc/extending/newtypes.rst
+++ b/Doc/extending/newtypes.rst
@@ -26,11 +26,12 @@ The Basics
==========
The Python runtime sees all Python objects as variables of type
-:c:type:`PyObject\*`. A :c:type:`PyObject` is not a very magnificent object - it
-just contains the refcount and a pointer to the object's "type object". This is
-where the action is; the type object determines which (C) functions get called
-when, for instance, an attribute gets looked up on an object or it is multiplied
-by another object. These C functions are called "type methods".
+:c:type:`PyObject\*`, which serves as a "base type" for all Python objects.
+:c:type:`PyObject` itself only contains the refcount and a pointer to the
+object's "type object". This is where the action is; the type object determines
+which (C) functions get called when, for instance, an attribute gets looked
+up on an object or it is multiplied by another object. These C functions
+are called "type methods".
So, if you want to define a new object type, you need to create a new type
object.
@@ -50,15 +51,15 @@ The first bit that will be new is::
PyObject_HEAD
} noddy_NoddyObject;
-This is what a Noddy object will contain---in this case, nothing more than every
-Python object contains, namely a refcount and a pointer to a type object. These
-are the fields the ``PyObject_HEAD`` macro brings in. The reason for the macro
-is to standardize the layout and to enable special debugging fields in debug
-builds. Note that there is no semicolon after the ``PyObject_HEAD`` macro; one
-is included in the macro definition. Be wary of adding one by accident; it's
-easy to do from habit, and your compiler might not complain, but someone else's
-probably will! (On Windows, MSVC is known to call this an error and refuse to
-compile the code.)
+This is what a Noddy object will contain---in this case, nothing more than what
+every Python object contains---a refcount and a pointer to a type object.
+These are the fields the ``PyObject_HEAD`` macro brings in. The reason for the
+macro is to standardize the layout and to enable special debugging fields in
+debug builds. Note that there is no semicolon after the ``PyObject_HEAD``
+macro; one is included in the macro definition. Be wary of adding one by
+accident; it's easy to do from habit, and your compiler might not complain,
+but someone else's probably will! (On Windows, MSVC is known to call this an
+error and refuse to compile the code.)
For contrast, let's take a look at the corresponding definition for standard
Python floats::
@@ -224,7 +225,7 @@ doesn't do anything. It can't even be subclassed.
Adding data and methods to the Basic example
--------------------------------------------
-Let's expend the basic example to add some data and methods. Let's also make
+Let's extend the basic example to add some data and methods. Let's also make
the type usable as a base class. We'll create a new module, :mod:`noddy2` that
adds these capabilities:
@@ -288,18 +289,16 @@ strings, so we provide a new method::
self = (Noddy *)type->tp_alloc(type, 0);
if (self != NULL) {
self->first = PyString_FromString("");
- if (self->first == NULL)
- {
+ if (self->first == NULL) {
Py_DECREF(self);
return NULL;
- }
+ }
self->last = PyString_FromString("");
- if (self->last == NULL)
- {
+ if (self->last == NULL) {
Py_DECREF(self);
return NULL;
- }
+ }
self->number = 0;
}
@@ -327,8 +326,8 @@ any arguments passed when the type was called, and that returns the new object
created. New methods always accept positional and keyword arguments, but they
often ignore the arguments, leaving the argument handling to initializer
methods. Note that if the type supports subclassing, the type passed may not be
-the type being defined. The new method calls the tp_alloc slot to allocate
-memory. We don't fill the :attr:`tp_alloc` slot ourselves. Rather
+the type being defined. The new method calls the :attr:`tp_alloc` slot to
+allocate memory. We don't fill the :attr:`tp_alloc` slot ourselves. Rather
:c:func:`PyType_Ready` fills it for us by inheriting it from our base class,
which is :class:`object` by default. Most types use the default allocation.
@@ -445,15 +444,6 @@ concatenation of the first and last names. ::
static PyObject *
Noddy_name(Noddy* self)
{
- static PyObject *format = NULL;
- PyObject *args, *result;
-
- if (format == NULL) {
- format = PyString_FromString("%s %s");
- if (format == NULL)
- return NULL;
- }
-
if (self->first == NULL) {
PyErr_SetString(PyExc_AttributeError, "first");
return NULL;
@@ -464,20 +454,13 @@ concatenation of the first and last names. ::
return NULL;
}
- args = Py_BuildValue("OO", self->first, self->last);
- if (args == NULL)
- return NULL;
-
- result = PyString_Format(format, args);
- Py_DECREF(args);
-
- return result;
+ return PyUnicode_FromFormat("%S %S", self->first, self->last);
}
The method is implemented as a C function that takes a :class:`Noddy` (or
:class:`Noddy` subclass) instance as the first argument. Methods always take an
instance as the first argument. Methods often take positional and keyword
-arguments as well, but in this cased we don't take any and don't need to accept
+arguments as well, but in this case we don't take any and don't need to accept
a positional argument tuple or keyword argument dictionary. This method is
equivalent to the Python method::
@@ -1124,9 +1107,6 @@ needed for methods inherited from a base type. One additional entry is needed
at the end; it is a sentinel that marks the end of the array. The
:attr:`ml_name` field of the sentinel must be *NULL*.
-XXX Need to refer to some unified discussion of the structure fields, shared
-with the next section.
-
The second table is used to define attributes which map directly to data stored
in the instance. A variety of primitive C types are supported, and access may
be read-only or read-write. The structures in the table are defined as::
@@ -1146,8 +1126,6 @@ type which will be able to extract a value from the instance structure. The
convert Python values to and from C values. The :attr:`flags` field is used to
store flags which control how the attribute can be accessed.
-XXX Need to move some of this to a shared section!
-
The following flag constants are defined in :file:`structmember.h`; they may be
combined using bitwise-OR.
@@ -1372,7 +1350,7 @@ Here is a desultory example of the implementation of the call function. ::
return result;
}
-XXX some fields need to be added here... ::
+::
/* Iterators */
getiterfunc tp_iter;
diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst
index 7c5116d..6b8a8fd 100644
--- a/Doc/faq/design.rst
+++ b/Doc/faq/design.rst
@@ -515,14 +515,16 @@ far) under most circumstances, and the implementation is simpler.
Dictionaries work by computing a hash code for each key stored in the dictionary
using the :func:`hash` built-in function. The hash code varies widely depending
-on the key; for example, "Python" hashes to -539294296 while "python", a string
-that differs by a single bit, hashes to 1142331976. The hash code is then used
-to calculate a location in an internal array where the value will be stored.
-Assuming that you're storing keys that all have different hash values, this
-means that dictionaries take constant time -- O(1), in computer science notation
--- to retrieve a key. It also means that no sorted order of the keys is
-maintained, and traversing the array as the ``.keys()`` and ``.items()`` do will
-output the dictionary's content in some arbitrary jumbled order.
+on the key and a per-process seed; for example, "Python" could hash to
+-539294296 while "python", a string that differs by a single bit, could hash
+to 1142331976. The hash code is then used to calculate a location in an
+internal array where the value will be stored. Assuming that you're storing
+keys that all have different hash values, this means that dictionaries take
+constant time -- O(1), in computer science notation -- to retrieve a key. It
+also means that no sorted order of the keys is maintained, and traversing the
+array as the ``.keys()`` and ``.items()`` do will output the dictionary's
+content in some arbitrary jumbled order that can change with every invocation of
+a program.
Why must dictionary keys be immutable?
@@ -634,7 +636,7 @@ construction of large programs.
Python 2.6 adds an :mod:`abc` module that lets you define Abstract Base Classes
(ABCs). You can then use :func:`isinstance` and :func:`issubclass` to check
whether an instance or a class implements a particular ABC. The
-:mod:`collections` module defines a set of useful ABCs such as
+:mod:`collections.abc` module defines a set of useful ABCs such as
:class:`Iterable`, :class:`Container`, and :class:`MutableMapping`.
For Python, many of the advantages of interface specifications can be obtained
diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst
index 7c684a0..fa245c7 100644
--- a/Doc/faq/extending.rst
+++ b/Doc/faq/extending.rst
@@ -445,34 +445,3 @@ In Python 2.2, you can inherit from built-in classes such as :class:`int`,
The Boost Python Library (BPL, http://www.boost.org/libs/python/doc/index.html)
provides a way of doing this from C++ (i.e. you can inherit from an extension
class written in C++ using the BPL).
-
-
-When importing module X, why do I get "undefined symbol: PyUnicodeUCS2*"?
--------------------------------------------------------------------------
-
-You are using a version of Python that uses a 4-byte representation for Unicode
-characters, but some C extension module you are importing was compiled using a
-Python that uses a 2-byte representation for Unicode characters (the default).
-
-If instead the name of the undefined symbol starts with ``PyUnicodeUCS4``, the
-problem is the reverse: Python was built using 2-byte Unicode characters, and
-the extension module was compiled using a Python with 4-byte Unicode characters.
-
-This can easily occur when using pre-built extension packages. RedHat Linux
-7.x, in particular, provided a "python2" binary that is compiled with 4-byte
-Unicode. This only causes the link failure if the extension uses any of the
-``PyUnicode_*()`` functions. It is also a problem if an extension uses any of
-the Unicode-related format specifiers for :c:func:`Py_BuildValue` (or similar) or
-parameter specifications for :c:func:`PyArg_ParseTuple`.
-
-You can check the size of the Unicode character a Python interpreter is using by
-checking the value of sys.maxunicode:
-
- >>> import sys
- >>> if sys.maxunicode > 65535:
- ... print('UCS4 build')
- ... else:
- ... print('UCS2 build')
-
-The only way to solve this problem is to use extension modules compiled with a
-Python binary built using the same size for Unicode characters.
diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst
index 7385c59..cab2d7b 100644
--- a/Doc/faq/library.rst
+++ b/Doc/faq/library.rst
@@ -351,7 +351,7 @@ When run, this will produce the following output:
Worker <Thread(worker 1, started 130283832797456)> running with argument 5
...
-Consult the module's documentation for more details; the :class:`~queue.Queue``
+Consult the module's documentation for more details; the :class:`~queue.Queue`
class provides a featureful interface.
diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index bf1234d..cb647a3 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -34,7 +34,7 @@ Glossary
subclasses, which are classes that don't inherit from a class but are
still recognized by :func:`isinstance` and :func:`issubclass`; see the
:mod:`abc` module documentation. Python comes with many built-in ABCs for
- data structures (in the :mod:`collections` module), numbers (in the
+ data structures (in the :mod:`collections.abc` module), numbers (in the
:mod:`numbers` module), streams (in the :mod:`io` module), import finders
and loaders (in the :mod:`importlib.abc` module). You can create your own
ABCs with the :mod:`abc` module.
@@ -434,8 +434,8 @@ Glossary
mapping
A container object that supports arbitrary key lookups and implements the
- methods specified in the :class:`~collections.Mapping` or
- :class:`~collections.MutableMapping`
+ methods specified in the :class:`~collections.abc.Mapping` or
+ :class:`~collections.abc.MutableMapping`
:ref:`abstract base classes <collections-abstract-base-classes>`. Examples
include :class:`dict`, :class:`collections.defaultdict`,
:class:`collections.OrderedDict` and :class:`collections.Counter`.
@@ -523,6 +523,20 @@ Glossary
definition), or pass several arguments as a list to a function. See
:term:`argument`.
+ provisional package
+ A provisional package is one which has been deliberately excluded from the
+ standard library's backwards compatibility guarantees. While major
+ changes to such packages are not expected, as long as they are marked
+ provisional, backwards incompatible changes (up to and including removal
+ of the package) may occur if deemed necessary by core developers. Such
+ changes will not be made gratuitously -- they will occur only if serious
+ flaws are uncovered that were missed prior to the inclusion of the
+ package.
+
+ This process allows the standard library to continue to evolve over time,
+ without locking in problematic design errors for extended periods of time.
+ See :pep:`411` for more details.
+
Python 3000
Nickname for the Python 3.x release line (coined long ago when the release
of version 3 was something in the distant future.) This is also
@@ -544,6 +558,24 @@ Glossary
for piece in food:
print(piece)
+ qualified name
+ A dotted name showing the "path" from a module's global scope to a
+ class, function or method defined in that module, as defined in
+ :pep:`3155`. For top-level functions and classes, the qualified name
+ is the same as the object's name::
+
+ >>> class C:
+ ... class D:
+ ... def meth(self):
+ ... pass
+ ...
+ >>> C.__qualname__
+ 'C'
+ >>> C.D.__qualname__
+ 'C.D'
+ >>> C.D.meth.__qualname__
+ 'C.D.meth'
+
reference count
The number of references to an object. When the reference count of an
object drops to zero, it is deallocated. Reference counting is
@@ -586,6 +618,14 @@ Glossary
an :term:`expression` or a one of several constructs with a keyword, such
as :keyword:`if`, :keyword:`while` or :keyword:`for`.
+ struct sequence
+ A tuple with named elements. Struct sequences expose an interface similar
+ to :term:`named tuple` in that elements can either be accessed either by
+ index or as an attribute. However, they do not have any of the named tuple
+ methods like :meth:`~collections.somenamedtuple._make` or
+ :meth:`~collections.somenamedtuple._asdict`. Examples of struct sequences
+ include :data:`sys.float_info` and the return value of :func:`os.stat`.
+
triple-quoted string
A string which is bound by three instances of either a quotation mark
(") or an apostrophe ('). While they don't provide any functionality
diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst
index 1616f67..0b513f9 100644
--- a/Doc/howto/descriptor.rst
+++ b/Doc/howto/descriptor.rst
@@ -36,9 +36,7 @@ continuing through the base classes of ``type(a)`` excluding metaclasses. If the
looked-up value is an object defining one of the descriptor methods, then Python
may override the default behavior and invoke the descriptor method instead.
Where this occurs in the precedence chain depends on which descriptor methods
-were defined. Note that descriptors are only invoked for new style objects or
-classes (a class is new style if it inherits from :class:`object` or
-:class:`type`).
+were defined.
Descriptors are a powerful, general purpose protocol. They are the mechanism
behind properties, methods, static methods, class methods, and :func:`super()`.
@@ -89,8 +87,6 @@ of ``obj``. If ``d`` defines the method :meth:`__get__`, then ``d.__get__(obj)`
is invoked according to the precedence rules listed below.
The details of invocation depend on whether ``obj`` is an object or a class.
-Either way, descriptors only work for new style objects and classes. A class is
-new style if it is a subclass of :class:`object`.
For objects, the machinery is in :meth:`object.__getattribute__` which
transforms ``b.x`` into ``type(b).__dict__['x'].__get__(b, type(b))``. The
@@ -115,7 +111,6 @@ The important points to remember are:
* descriptors are invoked by the :meth:`__getattribute__` method
* overriding :meth:`__getattribute__` prevents automatic descriptor calls
-* :meth:`__getattribute__` is only available with new style classes and objects
* :meth:`object.__getattribute__` and :meth:`type.__getattribute__` make
different calls to :meth:`__get__`.
* data descriptors always override instance dictionaries.
@@ -128,10 +123,7 @@ and then returns ``A.__dict__['m'].__get__(obj, A)``. If not a descriptor,
``m`` is returned unchanged. If not in the dictionary, ``m`` reverts to a
search using :meth:`object.__getattribute__`.
-Note, in Python 2.2, ``super(B, obj).m()`` would only invoke :meth:`__get__` if
-``m`` was a data descriptor. In Python 2.3, non-data descriptors also get
-invoked unless an old-style class is involved. The implementation details are
-in :c:func:`super_getattro()` in
+The implementation details are in :c:func:`super_getattro()` in
`Objects/typeobject.c <http://svn.python.org/view/python/trunk/Objects/typeobject.c?view=markup>`_
and a pure Python equivalent can be found in `Guido's Tutorial`_.
diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst
index a11d3da..f44e8c0 100644
--- a/Doc/howto/index.rst
+++ b/Doc/howto/index.rst
@@ -28,4 +28,5 @@ Currently, the HOWTOs are:
urllib2.rst
webservers.rst
argparse.rst
+ ipaddress.rst
diff --git a/Doc/howto/ipaddress.rst b/Doc/howto/ipaddress.rst
new file mode 100644
index 0000000..4d9ca08
--- /dev/null
+++ b/Doc/howto/ipaddress.rst
@@ -0,0 +1,299 @@
+.. _ipaddress-howto:
+
+***************************************
+An Introduction to the ipaddress module
+***************************************
+
+:author: Peter Moody
+:author: Nick Coghlan
+
+.. topic:: Overview
+
+ This document aims to provide a gentle introduction to :mod:`ipaddress`
+ module. It is aimed primarily at users that aren't already familiar with
+ IP networking terminology, but may also be useful to network engineers
+ wanting an overview of how the ipaddress module represents IP network
+ addressing concepts.
+
+
+Creating Address/Network/Interface objects
+==========================================
+
+Since :mod:`ipaddress` is a module for inspecting and manipulating IP address,
+the first thing you'll want to do is create some objects. You can use
+:mod:`ipaddress` to create objects from strings and integers.
+
+
+A Note on IP Versions
+---------------------
+
+For readers that aren't particularly familiar with IP addressing, it's
+important to know that the Internet Protocol is currently in the process
+of moving from version 4 of the protocol to version 6. This transition is
+occurring largely because version 4 of the protocol doesn't provide enough
+addresses to handle the needs of the whole world, especially given the
+increasing number of devices with direct connections to the internet.
+
+Explaining the details of the differences between the two versions of the
+protocol is beyond the scope of this introduction, but readers need to at
+least be aware that these two versions exist, and it will sometimes be
+necessary to force the use of one version or the other.
+
+
+IP Host Addresses
+-----------------
+
+Addresses, often referred to as "host addresses" are the most basic unit
+when working with IP addressing. The simplest way to create addresses is
+to use the :func:`ipaddress.ip_address` factory function, which automatically determines
+whether to create an IPv4 or IPv6 address based on the passed in value::
+
+ >>> ipaddress.ip_address('192.0.2.1')
+ IPv4Address('192.0.2.1')
+ >>> ipaddress.ip_address('2001:DB8::1')
+ IPv6Address('2001:db8::1')
+
+Addresses can also be created directly from integers. Values that will
+fit within 32 bits are assumed to be IPv4 addresses::
+
+ >>> ipaddress.ip_address(3221225985)
+ IPv4Address('192.0.2.1')
+ >>> ipaddress.ip_address(42540766411282592856903984951653826561)
+ IPv6Address('2001:db8::1')
+
+To force the use of IPv4 or IPv6 addresses, the relevant classes can be
+invoked directly. This is particularly useful to force creation of IPv6
+addresses for small integers::
+
+ >>> ipaddress.ip_address(1)
+ IPv4Address('0.0.0.1')
+ >>> ipaddress.IPv4Address(1)
+ IPv4Address('0.0.0.1')
+ >>> ipaddress.IPv6Address(1)
+ IPv6Address('::1')
+
+
+Defining Networks
+-----------------
+
+Host addresses are usually grouped together into IP networks, so
+:mod:`ipaddress` provides a way to create, inspect and manipulate network
+definitions. IP network objects are constructed from strings that define the
+range of host addresses that are part of that network. The simplest form
+for that information is a "network address/network prefix" pair, where the
+prefix defines the number of leading bits that are compared to determine
+whether or not an address is part of the network and the network address
+defines the expected value of those bits.
+
+As for addresses, a factory function is provided that determines the correct
+IP version automatically::
+
+ >>> ipaddress.ip_network('192.0.2.0/24')
+ IPv4Network('192.0.2.0/24')
+ >>> ipaddress.ip_network('2001:db8::0/96')
+ IPv6Network('2001:db8::/96')
+
+Network objects cannot have any host bits set. The practical effect of this
+is that ``192.0.2.1/24`` does not describe a network. Such definitions are
+referred to as interface objects since the ip-on-a-network notation is
+commonly used to describe network interfaces of a computer on a given network
+and are described further in the next section.
+
+By default, attempting to create a network object with host bits set will
+result in :exc:`ValueError` being raised. To request that the
+additional bits instead be coerced to zero, the flag ``strict=False`` can
+be passed to the constructor::
+
+ >>> ipaddress.ip_network('192.0.2.1/24')
+ Traceback (most recent call last):
+ ...
+ ValueError: 192.0.2.1/24 has host bits set
+ >>> ipaddress.ip_network('192.0.2.1/24', strict=False)
+ IPv4Network('192.0.2.0/24')
+
+While the string form offers significantly more flexibility, networks can
+also be defined with integers, just like host addresses. In this case, the
+network is considered to contain only the single address identified by the
+integer, so the network prefix includes the entire network address::
+
+ >>> ipaddress.ip_network(3221225984)
+ IPv4Network('192.0.2.0/32')
+ >>> ipaddress.ip_network(42540766411282592856903984951653826560)
+ IPv6Network('2001:db8::/128')
+
+Creation of a particular kind of network can be forced by calling the
+class constructor directly instead of using the factory function.
+
+
+Host Interfaces
+---------------
+
+As mentioned just above, if you need to describe an address on a particular
+network, neither the address nor the network classes are sufficient.
+Notation like ``192.0.2.1/24`` is commonly used network engineers and the
+people who write tools for firewalls and routers as shorthand for "the host
+``192.0.2.1`` on the network ``192.0.2.0/24``", Accordingly, :mod:`ipaddress`
+provides a set of hybrid classes that associate an address with a particular
+network. The interface for creation is identical to that for defining network
+objects, except that the address portion isn't constrained to being a network
+address.
+
+ >>> ipaddress.ip_interface('192.0.2.1/24')
+ IPv4Interface('192.0.2.1/24')
+ >>> ipaddress.ip_network('2001:db8::1/96')
+ IPv6Interface('2001:db8::1/96')
+
+Integer inputs are accepted (as with networks), and use of a particular IP
+version can be forced by calling the relevant constructor directly.
+
+
+Inspecting Address/Network/Interface Objects
+============================================
+
+You've gone to the trouble of creating an IPv(4|6)(Address|Network|Interface)
+object, so you probably want to get information about it. :mod:`ipaddress`
+tries to make doing this easy and intuitive.
+
+Extracting the IP version::
+
+ >>> addr4 = ipaddress.ip_address('192.0.2.1')
+ >>> addr6 = ipaddress.ip_address('2001:db8::1')
+ >>> addr6.version
+ 6
+ >>> addr4.version
+ 4
+
+Obtaining the network from an interface::
+
+ >>> host4 = ipaddress.ip_interface('192.0.2.1/24')
+ >>> host4.network
+ IPv4Network('192.0.2.0/24')
+ >>> host6 = ipaddress.ip_interface('2001:db8::1/96')
+ >>> host6.network
+ IPv6Network('2001:db8::/96')
+
+Finding out how many individual addresses are in a network::
+
+ >>> net4 = ipaddress.ip_network('192.0.2.0/24')
+ >>> net4.numhosts
+ 256
+ >>> net6 = ipaddress.ip_network('2001:db8::0/96')
+ >>> net6.numhosts
+ 4294967296
+
+Iterating through the 'usable' addresses on a network::
+
+ >>> net4 = ipaddress.ip_network('192.0.2.0/24')
+ >>> for x in net4.iterhosts():
+ print(x)
+ 192.0.2.1
+ 192.0.2.2
+ 192.0.2.3
+ 192.0.2.4
+ <snip>
+ 192.0.2.252
+ 192.0.2.253
+ 192.0.2.254
+
+
+Obtaining the netmask (i.e. set bits corresponding to the network prefix) or
+the hostmask (any bits that are not part of the netmask):
+
+ >>> net4 = ipaddress.ip_network('192.0.2.0/24')
+ >>> net4.netmask
+ IPv4Address('255.255.255.0')
+ >>> net4.hostmask
+ IPv4Address('0.0.0.255')
+ >>> net6 = ipaddress.ip_network('2001:db8::0/96')
+ >>> net6.netmask
+ IPv6Address('ffff:ffff:ffff:ffff:ffff:ffff::')
+ >>> net6.hostmask
+ IPv6Address('::ffff:ffff')
+
+
+Exploding or compressing the address::
+
+ >>> net6.exploded
+ '2001:0000:0000:0000:0000:0000:0000:0000/96'
+ >>> addr6.exploded
+ '2001:0000:0000:0000:0000:0000:0000:0001'
+
+
+Networks as lists of Addresses
+==============================
+
+It's sometimes useful to treat networks as lists. This means it is possible
+to index them like this::
+
+ >>> net4[1]
+ IPv4Address('192.0.2.1')
+ >>> net4[-1]
+ IPv4Address('192.0.2.255')
+ >>> net6[1]
+ IPv6Address('2001::1')
+ >>> net6[-1]
+ IPv6Address('2001::ffff:ffff')
+
+
+It also means that network objects lend themselves to using the list
+membership test syntax like this::
+
+ if address in network:
+ # do something
+
+Containment testing is done efficiently based on the network prefix::
+
+ >>> addr4 = ipaddress.ip_address('192.0.2.1')
+ >>> addr4 in ipaddress.ip_network('192.0.2.0/24')
+ True
+ >>> addr4 in ipaddress.ip_network('192.0.3.0/24')
+ False
+
+
+Comparisons
+===========
+
+:mod:`ipaddress` provides some simple, hopefully intuitive ways to compare
+objects, where it makes sense::
+
+ >>> ipaddress.ip_address('192.0.2.1') < ipaddress.ip_address('192.0.2.2')
+ True
+
+A :exc:`TypeError` exception is raised if you try to compare objects of
+different versions or different types.
+
+
+Using IP Addresses with other modules
+=====================================
+
+Other modules that use IP addresses (such as :mod:`socket`) usually won't
+accept objects from this module directly. Instead, they must be coerced to
+an integer or string that the other module will accept::
+
+ >>> addr4 = ipaddress.ip_address('192.0.2.1')
+ >>> str(addr4)
+ '192.0.2.1'
+ >>> int(addr4)
+ 3221225985
+
+
+Exceptions raised by :mod:`ipaddress`
+=====================================
+
+When creating address/network/interface objects using the version-agnostic
+factory functions, any errors will be reported as :exc:`ValueError`.
+
+For some use cases, it desirable to know whether it is the address or the
+netmask which is incorrect. To support these use cases, the class
+constructors actually raise the :exc:`ValueError` subclasses
+:exc:`ipaddress.AddressValueError` and :exc:`ipaddress.NetmaskValueError`
+to indicate exactly which part of the definition failed to parse correctly.
+
+Both of the module specific exceptions have :exc:`ValueError` as their
+parent class, so if you're not concerned with the particular type of error,
+you can still write code like the following::
+
+ try:
+ ipaddress.IPv4Address(address)
+ except ValueError:
+ print('address/netmask is invalid:', address)
diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst
index 59864a9..bb9c166 100644
--- a/Doc/howto/logging-cookbook.rst
+++ b/Doc/howto/logging-cookbook.rst
@@ -1316,6 +1316,33 @@ For more information about this configuration, you can see the `relevant
section <https://docs.djangoproject.com/en/1.3/topics/logging/#configuring-logging>`_
of the Django documentation.
+.. _cookbook-rotator-namer:
+
+Using a rotator and namer to customise log rotation processing
+--------------------------------------------------------------
+
+An example of how you can define a namer and rotator is given in the following
+snippet, which shows zlib-based compression of the log file::
+
+ def namer(name):
+ return name + ".gz"
+
+ def rotator(source, dest):
+ with open(source, "rb") as sf:
+ data = sf.read()
+ compressed = zlib.compress(data, 9)
+ with open(dest, "wb") as df:
+ df.write(compressed)
+ os.remove(source)
+
+ rh = logging.handlers.RotatingFileHandler(...)
+ rh.rotator = rotator
+ rh.namer = namer
+
+These are not "true" .gz files, as they are bare compressed data, with no
+"container" such as you’d find in an actual gzip file. This snippet is just
+for illustration purposes.
+
A more elaborate multiprocessing example
----------------------------------------
@@ -1572,7 +1599,7 @@ UTF-8, then you need to do the following:
'ASCII section\ufeffUnicode section'
- The Unicode code point ``'\feff```, when encoded using UTF-8, will be
+ The Unicode code point ``'\feff'``, when encoded using UTF-8, will be
encoded as a UTF-8 BOM -- the byte-string ``b'\xef\xbb\xbf'``.
#. Replace the ASCII section with whatever placeholders you like, but make sure
diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst
index 279bb3e..ca6528b 100644
--- a/Doc/howto/sockets.rst
+++ b/Doc/howto/sockets.rst
@@ -25,8 +25,8 @@ It's not really a tutorial - you'll still have work to do in getting things
working. It doesn't cover the fine points (and there are a lot of them), but I
hope it will give you enough background to begin using them decently.
-I'm only going to talk about INET sockets, but they account for at least 99% of
-the sockets in use. And I'll only talk about STREAM sockets - unless you really
+I'm only going to talk about INET (i.e. IPv4) sockets, but they account for at least 99% of
+the sockets in use. And I'll only talk about STREAM (i.e. TCP) sockets - unless you really
know what you're doing (in which case this HOWTO isn't for you!), you'll get
better behavior and performance from a STREAM socket than anything else. I will
try to clear up the mystery of what a socket is, as well as some hints on how to
@@ -208,10 +208,10 @@ length message::
totalsent = totalsent + sent
def myreceive(self):
- msg = ''
+ msg = b''
while len(msg) < MSGLEN:
chunk = self.sock.recv(MSGLEN-len(msg))
- if chunk == '':
+ if chunk == b'':
raise RuntimeError("socket connection broken")
msg = msg + chunk
return msg
@@ -371,12 +371,6 @@ have created a new socket to ``connect`` to someone else, put it in the
potential_writers list. If it shows up in the writable list, you have a decent
chance that it has connected.
-One very nasty problem with ``select``: if somewhere in those input lists of
-sockets is one which has died a nasty death, the ``select`` will fail. You then
-need to loop through every single damn socket in all those lists and do a
-``select([sock],[],[],0)`` until you find the bad one. That timeout of 0 means
-it won't take long, but it's ugly.
-
Actually, ``select`` can be handy even with blocking sockets. It's one way of
determining whether you will block - the socket returns as readable when there's
something in the buffers. However, this still doesn't help with the problem of
@@ -386,26 +380,6 @@ determining whether the other end is done, or just busy with something else.
files. Don't try this on Windows. On Windows, ``select`` works with sockets
only. Also note that in C, many of the more advanced socket options are done
differently on Windows. In fact, on Windows I usually use threads (which work
-very, very well) with my sockets. Face it, if you want any kind of performance,
-your code will look very different on Windows than on Unix.
-
-
-Performance
------------
+very, very well) with my sockets.
-There's no question that the fastest sockets code uses non-blocking sockets and
-select to multiplex them. You can put together something that will saturate a
-LAN connection without putting any strain on the CPU.
-
-The trouble is that an app written this way can't do much of anything else -
-it needs to be ready to shuffle bytes around at all times. Assuming that your
-app is actually supposed to do something more than that, threading is the
-optimal solution, (and using non-blocking sockets will be faster than using
-blocking sockets).
-
-Finally, remember that even though blocking sockets are somewhat slower than
-non-blocking, in many cases they are the "right" solution. After all, if your
-app is driven by the data it receives over a socket, there's not much sense in
-complicating the logic just so your app can wait on ``select`` instead of
-``recv``.
diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst
index fade4a3..20dca20 100644
--- a/Doc/howto/urllib2.rst
+++ b/Doc/howto/urllib2.rst
@@ -56,6 +56,13 @@ The simplest way to use urllib.request is as follows::
response = urllib.request.urlopen('http://python.org/')
html = response.read()
+If you wish to retrieve a resource via URL and store it in a temporary location,
+you can do so via the :func:`urlretrieve` function::
+
+ import urllib.request
+ local_filename, headers = urllib.request.urlretrieve('http://python.org/')
+ html = open(local_filename)
+
Many uses of urllib will be that simple (note that instead of an 'http:' URL we
could have used an URL starting with 'ftp:', 'file:', etc.). However, it's the
purpose of this tutorial to explain the more complicated cases, concentrating on
diff --git a/Doc/includes/mp_benchmarks.py b/Doc/includes/mp_benchmarks.py
index acdf642..3763ea9 100644
--- a/Doc/includes/mp_benchmarks.py
+++ b/Doc/includes/mp_benchmarks.py
@@ -6,16 +6,12 @@
#
import time
-import sys
import multiprocessing
import threading
import queue
import gc
-if sys.platform == 'win32':
- _timer = time.clock
-else:
- _timer = time.time
+_timer = time.perf_counter
delta = 1
diff --git a/Doc/includes/noddy2.c b/Doc/includes/noddy2.c
index 9b8eafb..9641558 100644
--- a/Doc/includes/noddy2.c
+++ b/Doc/includes/noddy2.c
@@ -24,18 +24,16 @@ Noddy_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
self = (Noddy *)type->tp_alloc(type, 0);
if (self != NULL) {
self->first = PyUnicode_FromString("");
- if (self->first == NULL)
- {
+ if (self->first == NULL) {
Py_DECREF(self);
return NULL;
- }
-
+ }
+
self->last = PyUnicode_FromString("");
- if (self->last == NULL)
- {
+ if (self->last == NULL) {
Py_DECREF(self);
return NULL;
- }
+ }
self->number = 0;
}
@@ -50,10 +48,10 @@ Noddy_init(Noddy *self, PyObject *args, PyObject *kwds)
static char *kwlist[] = {"first", "last", "number", NULL};
- if (! PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist,
- &first, &last,
+ if (! PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist,
+ &first, &last,
&self->number))
- return -1;
+ return -1;
if (first) {
tmp = self->first;
@@ -86,15 +84,6 @@ static PyMemberDef Noddy_members[] = {
static PyObject *
Noddy_name(Noddy* self)
{
- static PyObject *format = NULL;
- PyObject *args, *result;
-
- if (format == NULL) {
- format = PyUnicode_FromString("%s %s");
- if (format == NULL)
- return NULL;
- }
-
if (self->first == NULL) {
PyErr_SetString(PyExc_AttributeError, "first");
return NULL;
@@ -105,14 +94,7 @@ Noddy_name(Noddy* self)
return NULL;
}
- args = Py_BuildValue("OO", self->first, self->last);
- if (args == NULL)
- return NULL;
-
- result = PyUnicode_Format(format, args);
- Py_DECREF(args);
-
- return result;
+ return PyUnicode_FromFormat("%S %S", self->first, self->last);
}
static PyMethodDef Noddy_methods[] = {
@@ -145,12 +127,12 @@ static PyTypeObject NoddyType = {
Py_TPFLAGS_DEFAULT |
Py_TPFLAGS_BASETYPE, /* tp_flags */
"Noddy objects", /* tp_doc */
- 0, /* tp_traverse */
- 0, /* tp_clear */
- 0, /* tp_richcompare */
- 0, /* tp_weaklistoffset */
- 0, /* tp_iter */
- 0, /* tp_iternext */
+ 0, /* tp_traverse */
+ 0, /* tp_clear */
+ 0, /* tp_richcompare */
+ 0, /* tp_weaklistoffset */
+ 0, /* tp_iter */
+ 0, /* tp_iternext */
Noddy_methods, /* tp_methods */
Noddy_members, /* tp_members */
0, /* tp_getset */
@@ -173,7 +155,7 @@ static PyModuleDef noddy2module = {
};
PyMODINIT_FUNC
-PyInit_noddy2(void)
+PyInit_noddy2(void)
{
PyObject* m;
diff --git a/Doc/includes/noddy3.c b/Doc/includes/noddy3.c
index 89f3a77..8a5a753 100644
--- a/Doc/includes/noddy3.c
+++ b/Doc/includes/noddy3.c
@@ -24,18 +24,16 @@ Noddy_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
self = (Noddy *)type->tp_alloc(type, 0);
if (self != NULL) {
self->first = PyUnicode_FromString("");
- if (self->first == NULL)
- {
+ if (self->first == NULL) {
Py_DECREF(self);
return NULL;
- }
-
+ }
+
self->last = PyUnicode_FromString("");
- if (self->last == NULL)
- {
+ if (self->last == NULL) {
Py_DECREF(self);
return NULL;
- }
+ }
self->number = 0;
}
@@ -50,10 +48,10 @@ Noddy_init(Noddy *self, PyObject *args, PyObject *kwds)
static char *kwlist[] = {"first", "last", "number", NULL};
- if (! PyArg_ParseTupleAndKeywords(args, kwds, "|SSi", kwlist,
- &first, &last,
+ if (! PyArg_ParseTupleAndKeywords(args, kwds, "|SSi", kwlist,
+ &first, &last,
&self->number))
- return -1;
+ return -1;
if (first) {
tmp = self->first;
@@ -88,22 +86,22 @@ Noddy_getfirst(Noddy *self, void *closure)
static int
Noddy_setfirst(Noddy *self, PyObject *value, void *closure)
{
- if (value == NULL) {
- PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute");
- return -1;
- }
-
- if (! PyUnicode_Check(value)) {
- PyErr_SetString(PyExc_TypeError,
- "The first attribute value must be a string");
- return -1;
- }
-
- Py_DECREF(self->first);
- Py_INCREF(value);
- self->first = value;
-
- return 0;
+ if (value == NULL) {
+ PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute");
+ return -1;
+ }
+
+ if (! PyUnicode_Check(value)) {
+ PyErr_SetString(PyExc_TypeError,
+ "The first attribute value must be a string");
+ return -1;
+ }
+
+ Py_DECREF(self->first);
+ Py_INCREF(value);
+ self->first = value;
+
+ return 0;
}
static PyObject *
@@ -116,30 +114,30 @@ Noddy_getlast(Noddy *self, void *closure)
static int
Noddy_setlast(Noddy *self, PyObject *value, void *closure)
{
- if (value == NULL) {
- PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute");
- return -1;
- }
-
- if (! PyUnicode_Check(value)) {
- PyErr_SetString(PyExc_TypeError,
- "The last attribute value must be a string");
- return -1;
- }
-
- Py_DECREF(self->last);
- Py_INCREF(value);
- self->last = value;
-
- return 0;
+ if (value == NULL) {
+ PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute");
+ return -1;
+ }
+
+ if (! PyUnicode_Check(value)) {
+ PyErr_SetString(PyExc_TypeError,
+ "The last attribute value must be a string");
+ return -1;
+ }
+
+ Py_DECREF(self->last);
+ Py_INCREF(value);
+ self->last = value;
+
+ return 0;
}
static PyGetSetDef Noddy_getseters[] = {
- {"first",
+ {"first",
(getter)Noddy_getfirst, (setter)Noddy_setfirst,
"first name",
NULL},
- {"last",
+ {"last",
(getter)Noddy_getlast, (setter)Noddy_setlast,
"last name",
NULL},
@@ -149,23 +147,7 @@ static PyGetSetDef Noddy_getseters[] = {
static PyObject *
Noddy_name(Noddy* self)
{
- static PyObject *format = NULL;
- PyObject *args, *result;
-
- if (format == NULL) {
- format = PyUnicode_FromString("%s %s");
- if (format == NULL)
- return NULL;
- }
-
- args = Py_BuildValue("OO", self->first, self->last);
- if (args == NULL)
- return NULL;
-
- result = PyUnicode_Format(format, args);
- Py_DECREF(args);
-
- return result;
+ return PyUnicode_FromFormat("%S %S", self->first, self->last);
}
static PyMethodDef Noddy_methods[] = {
@@ -198,12 +180,12 @@ static PyTypeObject NoddyType = {
Py_TPFLAGS_DEFAULT |
Py_TPFLAGS_BASETYPE, /* tp_flags */
"Noddy objects", /* tp_doc */
- 0, /* tp_traverse */
- 0, /* tp_clear */
- 0, /* tp_richcompare */
- 0, /* tp_weaklistoffset */
- 0, /* tp_iter */
- 0, /* tp_iternext */
+ 0, /* tp_traverse */
+ 0, /* tp_clear */
+ 0, /* tp_richcompare */
+ 0, /* tp_weaklistoffset */
+ 0, /* tp_iter */
+ 0, /* tp_iternext */
Noddy_methods, /* tp_methods */
Noddy_members, /* tp_members */
Noddy_getseters, /* tp_getset */
@@ -226,7 +208,7 @@ static PyModuleDef noddy3module = {
};
PyMODINIT_FUNC
-PyInit_noddy3(void)
+PyInit_noddy3(void)
{
PyObject* m;
diff --git a/Doc/includes/noddy4.c b/Doc/includes/noddy4.c
index 6a96fac..eb9622a 100644
--- a/Doc/includes/noddy4.c
+++ b/Doc/includes/noddy4.c
@@ -27,7 +27,7 @@ Noddy_traverse(Noddy *self, visitproc visit, void *arg)
return 0;
}
-static int
+static int
Noddy_clear(Noddy *self)
{
PyObject *tmp;
@@ -58,18 +58,16 @@ Noddy_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
self = (Noddy *)type->tp_alloc(type, 0);
if (self != NULL) {
self->first = PyUnicode_FromString("");
- if (self->first == NULL)
- {
+ if (self->first == NULL) {
Py_DECREF(self);
return NULL;
- }
-
+ }
+
self->last = PyUnicode_FromString("");
- if (self->last == NULL)
- {
+ if (self->last == NULL) {
Py_DECREF(self);
return NULL;
- }
+ }
self->number = 0;
}
@@ -84,10 +82,10 @@ Noddy_init(Noddy *self, PyObject *args, PyObject *kwds)
static char *kwlist[] = {"first", "last", "number", NULL};
- if (! PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist,
- &first, &last,
+ if (! PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist,
+ &first, &last,
&self->number))
- return -1;
+ return -1;
if (first) {
tmp = self->first;
@@ -120,15 +118,6 @@ static PyMemberDef Noddy_members[] = {
static PyObject *
Noddy_name(Noddy* self)
{
- static PyObject *format = NULL;
- PyObject *args, *result;
-
- if (format == NULL) {
- format = PyUnicode_FromString("%s %s");
- if (format == NULL)
- return NULL;
- }
-
if (self->first == NULL) {
PyErr_SetString(PyExc_AttributeError, "first");
return NULL;
@@ -139,14 +128,7 @@ Noddy_name(Noddy* self)
return NULL;
}
- args = Py_BuildValue("OO", self->first, self->last);
- if (args == NULL)
- return NULL;
-
- result = PyUnicode_Format(format, args);
- Py_DECREF(args);
-
- return result;
+ return PyUnicode_FromFormat("%S %S", self->first, self->last);
}
static PyMethodDef Noddy_methods[] = {
@@ -182,10 +164,10 @@ static PyTypeObject NoddyType = {
"Noddy objects", /* tp_doc */
(traverseproc)Noddy_traverse, /* tp_traverse */
(inquiry)Noddy_clear, /* tp_clear */
- 0, /* tp_richcompare */
- 0, /* tp_weaklistoffset */
- 0, /* tp_iter */
- 0, /* tp_iternext */
+ 0, /* tp_richcompare */
+ 0, /* tp_weaklistoffset */
+ 0, /* tp_iter */
+ 0, /* tp_iternext */
Noddy_methods, /* tp_methods */
Noddy_members, /* tp_members */
0, /* tp_getset */
@@ -208,7 +190,7 @@ static PyModuleDef noddy4module = {
};
PyMODINIT_FUNC
-PyInit_noddy4(void)
+PyInit_noddy4(void)
{
PyObject* m;
diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst
index 369e9cd..751c3e8 100644
--- a/Doc/library/_thread.rst
+++ b/Doc/library/_thread.rst
@@ -35,6 +35,9 @@ It defines the following constants and functions:
Raised on thread-specific errors.
+ .. versionchanged:: 3.3
+ This is now a synonym of the built-in :exc:`RuntimeError`.
+
.. data:: LockType
@@ -91,7 +94,7 @@ It defines the following constants and functions:
*size* argument specifies the stack size to be used for subsequently created
threads, and must be 0 (use platform or configured default) or a positive
integer value of at least 32,768 (32kB). If changing the thread stack size is
- unsupported, a :exc:`ThreadError` is raised. If the specified stack size is
+ unsupported, a :exc:`RuntimeError` is raised. If the specified stack size is
invalid, a :exc:`ValueError` is raised and the stack size is unmodified. 32kB
is currently the minimum supported stack size value to guarantee sufficient
stack space for the interpreter itself. Note that some platforms may have
diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst
index 1048b24..dca6993 100644
--- a/Doc/library/abc.rst
+++ b/Doc/library/abc.rst
@@ -18,7 +18,7 @@ regarding a type hierarchy for numbers based on ABCs.)
The :mod:`collections` module has some concrete classes that derive from
ABCs; these can, of course, be further derived. In addition the
-:mod:`collections` module has some ABCs that can be used to test whether
+:mod:`collections.abc` submodule has some ABCs that can be used to test whether
a class or instance provides a particular interface, for example, is it
hashable or a mapping.
@@ -55,6 +55,9 @@ This module provides the following class:
assert issubclass(tuple, MyABC)
assert isinstance((), MyABC)
+ .. versionchanged:: 3.3
+ Returns the registered subclass, to allow usage as a class decorator.
+
You can also override this method in an abstract base class:
.. method:: __subclasshook__(subclass)
@@ -124,19 +127,18 @@ This module provides the following class:
available as a method of ``Foo``, so it is provided separately.
-It also provides the following decorators:
+The :mod:`abc` module also provides the following decorators:
.. decorator:: abstractmethod(function)
A decorator indicating abstract methods.
- Using this decorator requires that the class's metaclass is :class:`ABCMeta` or
- is derived from it.
- A class that has a metaclass derived from :class:`ABCMeta`
- cannot be instantiated unless all of its abstract methods and
- properties are overridden.
- The abstract methods can be called using any of the normal 'super' call
- mechanisms.
+ Using this decorator requires that the class's metaclass is :class:`ABCMeta`
+ or is derived from it. A class that has a metaclass derived from
+ :class:`ABCMeta` cannot be instantiated unless all of its abstract methods
+ and properties are overridden. The abstract methods can be called using any
+ of the normal 'super' call mechanisms. :func:`abstractmethod` may be used
+ to declare abstract methods for properties and descriptors.
Dynamically adding abstract methods to a class, or attempting to modify the
abstraction status of a method or class once it is created, are not
@@ -144,12 +146,52 @@ It also provides the following decorators:
regular inheritance; "virtual subclasses" registered with the ABC's
:meth:`register` method are not affected.
- Usage::
+ When :func:`abstractmethod` is applied in combination with other method
+ descriptors, it should be applied as the innermost decorator, as shown in
+ the following usage examples::
class C(metaclass=ABCMeta):
@abstractmethod
def my_abstract_method(self, ...):
...
+ @classmethod
+ @abstractmethod
+ def my_abstract_classmethod(cls, ...):
+ ...
+ @staticmethod
+ @abstractmethod
+ def my_abstract_staticmethod(...):
+ ...
+
+ @property
+ @abstractmethod
+ def my_abstract_property(self):
+ ...
+ @my_abstract_property.setter
+ @abstractmethod
+ def my_abstract_property(self, val):
+ ...
+
+ @abstractmethod
+ def _get_x(self):
+ ...
+ @abstractmethod
+ def _set_x(self, val):
+ ...
+ x = property(_get_x, _set_x)
+
+ In order to correctly interoperate with the abstract base class machinery,
+ the descriptor must identify itself as abstract using
+ :attr:`__isabstractmethod__`. In general, this attribute should be ``True``
+ if any of the methods used to compose the descriptor are abstract. For
+ example, Python's built-in property does the equivalent of::
+
+ class Descriptor:
+ ...
+ @property
+ def __isabstractmethod__(self):
+ return any(getattr(f, '__isabstractmethod__', False) for
+ f in (self._fget, self._fset, self._fdel))
.. note::
@@ -174,6 +216,8 @@ It also provides the following decorators:
...
.. versionadded:: 3.2
+ .. deprecated:: 3.3
+ Use :class:`classmethod` with :func:`abstractmethod` instead.
.. decorator:: abstractstaticmethod(function)
@@ -189,18 +233,19 @@ It also provides the following decorators:
...
.. versionadded:: 3.2
+ .. deprecated:: 3.3
+ Use :class:`staticmethod` with :func:`abstractmethod` instead.
-.. function:: abstractproperty(fget=None, fset=None, fdel=None, doc=None)
+.. decorator:: abstractproperty(fget=None, fset=None, fdel=None, doc=None)
A subclass of the built-in :func:`property`, indicating an abstract property.
- Using this function requires that the class's metaclass is :class:`ABCMeta` or
- is derived from it.
- A class that has a metaclass derived from :class:`ABCMeta` cannot be
- instantiated unless all of its abstract methods and properties are overridden.
- The abstract properties can be called using any of the normal
- 'super' call mechanisms.
+ Using this function requires that the class's metaclass is :class:`ABCMeta`
+ or is derived from it. A class that has a metaclass derived from
+ :class:`ABCMeta` cannot be instantiated unless all of its abstract methods
+ and properties are overridden. The abstract properties can be called using
+ any of the normal 'super' call mechanisms.
Usage::
@@ -217,6 +262,9 @@ It also provides the following decorators:
def setx(self, value): ...
x = abstractproperty(getx, setx)
+ .. deprecated:: 3.3
+ Use :class:`property` with :func:`abstractmethod` instead
+
.. rubric:: Footnotes
diff --git a/Doc/library/archiving.rst b/Doc/library/archiving.rst
index 75d137c..c928494 100644
--- a/Doc/library/archiving.rst
+++ b/Doc/library/archiving.rst
@@ -5,8 +5,9 @@ Data Compression and Archiving
******************************
The modules described in this chapter support data compression with the zlib,
-gzip, and bzip2 algorithms, and the creation of ZIP- and tar-format archives.
-See also :ref:`archiving-operations` provided by the :mod:`shutil` module.
+gzip, bzip2 and lzma algorithms, and the creation of ZIP- and tar-format
+archives. See also :ref:`archiving-operations` provided by the :mod:`shutil`
+module.
.. toctree::
@@ -14,5 +15,6 @@ See also :ref:`archiving-operations` provided by the :mod:`shutil` module.
zlib.rst
gzip.rst
bz2.rst
+ lzma.rst
zipfile.rst
tarfile.rst
diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst
index 186e12d..8e7fa5e 100644
--- a/Doc/library/argparse.rst
+++ b/Doc/library/argparse.rst
@@ -368,16 +368,16 @@ formatter_class
^^^^^^^^^^^^^^^
:class:`ArgumentParser` objects allow the help formatting to be customized by
-specifying an alternate formatting class. Currently, there are three such
+specifying an alternate formatting class. Currently, there are four such
classes:
.. class:: RawDescriptionHelpFormatter
RawTextHelpFormatter
ArgumentDefaultsHelpFormatter
+ MetavarTypeHelpFormatter
-The first two allow more control over how textual descriptions are displayed,
-while the last automatically adds information about argument default values.
-
+:class:`RawDescriptionHelpFormatter` and :class:`RawTextHelpFormatter` give
+more control over how textual descriptions are displayed.
By default, :class:`ArgumentParser` objects line-wrap the description_ and
epilog_ texts in command-line help messages::
@@ -430,8 +430,8 @@ should not be line-wrapped::
:class:`RawTextHelpFormatter` maintains whitespace for all sorts of help text,
including argument descriptions.
-The other formatter class available, :class:`ArgumentDefaultsHelpFormatter`,
-will add information about the default value of each of the arguments::
+:class:`ArgumentDefaultsHelpFormatter` automatically adds information about
+default values to each of the argument help messages::
>>> parser = argparse.ArgumentParser(
... prog='PROG',
@@ -448,6 +448,25 @@ will add information about the default value of each of the arguments::
-h, --help show this help message and exit
--foo FOO FOO! (default: 42)
+:class:`MetavarTypeHelpFormatter` uses the name of the type_ argument for each
+argument as the display name for its values (rather than using the dest_
+as the regular formatter does)::
+
+ >>> parser = argparse.ArgumentParser(
+ ... prog='PROG',
+ ... formatter_class=argparse.MetavarTypeHelpFormatter)
+ >>> parser.add_argument('--foo', type=int)
+ >>> parser.add_argument('bar', type=float)
+ >>> parser.print_help()
+ usage: PROG [-h] [--foo int] float
+
+ positional arguments:
+ float
+
+ optional arguments:
+ -h, --help show this help message and exit
+ --foo int
+
conflict_handler
^^^^^^^^^^^^^^^^
diff --git a/Doc/library/array.rst b/Doc/library/array.rst
index d563cce..3e275a2 100644
--- a/Doc/library/array.rst
+++ b/Doc/library/array.rst
@@ -14,36 +14,44 @@ them is constrained. The type is specified at object creation time by using a
:dfn:`type code`, which is a single character. The following type codes are
defined:
-+-----------+----------------+-------------------+-----------------------+
-| Type code | C Type | Python Type | Minimum size in bytes |
-+===========+================+===================+=======================+
-| ``'b'`` | signed char | int | 1 |
-+-----------+----------------+-------------------+-----------------------+
-| ``'B'`` | unsigned char | int | 1 |
-+-----------+----------------+-------------------+-----------------------+
-| ``'u'`` | Py_UNICODE | Unicode character | 2 (see note) |
-+-----------+----------------+-------------------+-----------------------+
-| ``'h'`` | signed short | int | 2 |
-+-----------+----------------+-------------------+-----------------------+
-| ``'H'`` | unsigned short | int | 2 |
-+-----------+----------------+-------------------+-----------------------+
-| ``'i'`` | signed int | int | 2 |
-+-----------+----------------+-------------------+-----------------------+
-| ``'I'`` | unsigned int | int | 2 |
-+-----------+----------------+-------------------+-----------------------+
-| ``'l'`` | signed long | int | 4 |
-+-----------+----------------+-------------------+-----------------------+
-| ``'L'`` | unsigned long | int | 4 |
-+-----------+----------------+-------------------+-----------------------+
-| ``'f'`` | float | float | 4 |
-+-----------+----------------+-------------------+-----------------------+
-| ``'d'`` | double | float | 8 |
-+-----------+----------------+-------------------+-----------------------+
-
-.. note::
-
- The ``'u'`` typecode corresponds to Python's unicode character. On narrow
- Unicode builds this is 2-bytes, on wide builds this is 4-bytes.
++-----------+--------------------+-------------------+-----------------------+-------+
+| Type code | C Type | Python Type | Minimum size in bytes | Notes |
++===========+====================+===================+=======================+=======+
+| ``'b'`` | signed char | int | 1 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'B'`` | unsigned char | int | 1 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'u'`` | Py_UCS4 | Unicode character | 4 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'h'`` | signed short | int | 2 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'H'`` | unsigned short | int | 2 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'i'`` | signed int | int | 2 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'I'`` | unsigned int | int | 2 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'l'`` | signed long | int | 4 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'L'`` | unsigned long | int | 4 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'q'`` | signed long long | int | 8 | \(1) |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'Q'`` | unsigned long long | int | 8 | \(1) |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'f'`` | float | float | 4 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'d'`` | double | float | 8 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+
+Notes:
+
+(1)
+ The ``'q'`` and ``'Q'`` type codes are available only if
+ the platform C compiler used to build Python supports C :c:type:`long long`,
+ or, on Windows, :c:type:`__int64`.
+
+ .. versionadded:: 3.3
The actual representation of values is determined by the machine architecture
(strictly speaking, by the C implementation). The actual size can be accessed
diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst
index e2c0b6d..16de3ca 100644
--- a/Doc/library/ast.rst
+++ b/Doc/library/ast.rst
@@ -96,9 +96,6 @@ Node classes
Abstract Grammar
----------------
-The module defines a string constant ``__version__`` which is the decimal
-Subversion revision number of the file shown below.
-
The abstract grammar is currently defined as follows:
.. literalinclude:: ../../Parser/Python.asdl
diff --git a/Doc/library/asynchat.rst b/Doc/library/asynchat.rst
index 75b3cda..55c61d7 100644
--- a/Doc/library/asynchat.rst
+++ b/Doc/library/asynchat.rst
@@ -197,6 +197,9 @@ The :meth:`handle_request` method is called once all relevant input has been
marshalled, after setting the channel terminator to ``None`` to ensure that
any extraneous data sent by the web client are ignored. ::
+
+ import asynchat
+
class http_request_handler(asynchat.async_chat):
def __init__(self, sock, addr, sessions, log):
diff --git a/Doc/library/asyncore.rst b/Doc/library/asyncore.rst
index 619b7bb..7cacca1 100644
--- a/Doc/library/asyncore.rst
+++ b/Doc/library/asyncore.rst
@@ -184,12 +184,15 @@ any that have been added to the map during asynchronous service) is closed.
Most of these are nearly identical to their socket partners.
- .. method:: create_socket(family, type)
+ .. method:: create_socket(family=socket.AF_INET, type=socket.SOCK_STREAM)
This is identical to the creation of a normal socket, and will use the
same options for creation. Refer to the :mod:`socket` documentation for
information on creating sockets.
+ .. versionchanged:: 3.3
+ *family* and *type* arguments can be omitted.
+
.. method:: connect(address)
@@ -274,13 +277,13 @@ asyncore Example basic HTTP client
Here is a very basic HTTP client that uses the :class:`dispatcher` class to
implement its socket handling::
- import asyncore, socket
+ import asyncore
class HTTPClient(asyncore.dispatcher):
def __init__(self, host, path):
asyncore.dispatcher.__init__(self)
- self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+ self.create_socket()
self.connect( (host, 80) )
self.buffer = bytes('GET %s HTTP/1.0\r\nHost: %s\r\n\r\n' %
(path, host), 'ascii')
@@ -314,7 +317,6 @@ Here is a basic echo server that uses the :class:`dispatcher` class to accept
connections and dispatches the incoming connections to a handler::
import asyncore
- import socket
class EchoHandler(asyncore.dispatcher_with_send):
@@ -327,7 +329,7 @@ connections and dispatches the incoming connections to a handler::
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
- self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+ self.create_socket()
self.set_reuse_addr()
self.bind((host, port))
self.listen(5)
@@ -338,4 +340,3 @@ connections and dispatches the incoming connections to a handler::
server = EchoServer('localhost', 8080)
asyncore.loop()
-
diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst
index 7c76bab..3d5c014 100644
--- a/Doc/library/atexit.rst
+++ b/Doc/library/atexit.rst
@@ -67,8 +67,9 @@ automatically when the program terminates without relying on the application
making an explicit call into this module at termination. ::
try:
- _count = int(open("/tmp/counter").read())
- except IOError:
+ with open("/tmp/counter") as infile:
+ _count = int(infile.read())
+ except FileNotFoundError:
_count = 0
def incrcounter(n):
@@ -76,7 +77,8 @@ making an explicit call into this module at termination. ::
_count = _count + n
def savecounter():
- open("/tmp/counter", "w").write("%d" % _count)
+ with open("/tmp/counter", "w") as outfile:
+ outfile.write("%d" % _count)
import atexit
atexit.register(savecounter)
diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst
index 06f3ab1..afbedce 100644
--- a/Doc/library/base64.rst
+++ b/Doc/library/base64.rst
@@ -18,9 +18,14 @@ POST request. The encoding algorithm is not the same as the
There are two interfaces provided by this module. The modern interface
supports encoding and decoding ASCII byte string objects using all three
-alphabets. The legacy interface provides for encoding and decoding to and from
-file-like objects as well as byte strings, but only using the Base64 standard
-alphabet.
+alphabets. Additionally, the decoding functions of the modern interface also
+accept Unicode strings containing only ASCII characters. The legacy interface
+provides for encoding and decoding to and from file-like objects as well as
+byte strings, but only using the Base64 standard alphabet.
+
+.. versionchanged:: 3.3
+ ASCII-only Unicode strings are now accepted by the decoding functions of
+ the modern interface.
The modern interface provides:
diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst
index 2aa3702..baf430d 100644
--- a/Doc/library/binascii.rst
+++ b/Doc/library/binascii.rst
@@ -20,8 +20,13 @@ higher-level modules.
.. note::
- Encoding and decoding functions do not accept Unicode strings. Only bytestring
- and bytearray objects can be processed.
+ ``a2b_*`` functions accept Unicode strings containing only ASCII characters.
+ Other functions only accept bytes and bytes-compatible objects (such as
+ bytearray objects and other objects implementing the buffer API).
+
+ .. versionchanged:: 3.3
+ ASCII-only unicode strings are now accepted by the ``a2b_*`` functions.
+
The :mod:`binascii` module defines the following functions:
diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst
index d13f6e0..d06a39a 100644
--- a/Doc/library/bz2.rst
+++ b/Doc/library/bz2.rst
@@ -1,198 +1,202 @@
-:mod:`bz2` --- Compression compatible with :program:`bzip2`
-===========================================================
+:mod:`bz2` --- Support for :program:`bzip2` compression
+=======================================================
.. module:: bz2
- :synopsis: Interface to compression and decompression routines
- compatible with bzip2.
+ :synopsis: Interfaces for bzip2 compression and decompression.
.. moduleauthor:: Gustavo Niemeyer <niemeyer@conectiva.com>
+.. moduleauthor:: Nadeem Vawda <nadeem.vawda@gmail.com>
.. sectionauthor:: Gustavo Niemeyer <niemeyer@conectiva.com>
+.. sectionauthor:: Nadeem Vawda <nadeem.vawda@gmail.com>
-This module provides a comprehensive interface for the bz2 compression library.
-It implements a complete file interface, one-shot (de)compression functions, and
-types for sequential (de)compression.
+This module provides a comprehensive interface for compressing and
+decompressing data using the bzip2 compression algorithm.
-Here is a summary of the features offered by the bz2 module:
+The :mod:`bz2` module contains:
-* :class:`BZ2File` class implements a complete file interface, including
- :meth:`~BZ2File.readline`, :meth:`~BZ2File.readlines`,
- :meth:`~BZ2File.writelines`, :meth:`~BZ2File.seek`, etc;
+* The :func:`.open` function and :class:`BZ2File` class for reading and
+ writing compressed files.
+* The :class:`BZ2Compressor` and :class:`BZ2Decompressor` classes for
+ incremental (de)compression.
+* The :func:`compress` and :func:`decompress` functions for one-shot
+ (de)compression.
-* :class:`BZ2File` class implements emulated :meth:`~BZ2File.seek` support;
+All of the classes in this module may safely be accessed from multiple threads.
-* :class:`BZ2File` class implements universal newline support;
-* :class:`BZ2File` class offers an optimized line iteration using a readahead
- algorithm;
+(De)compression of files
+------------------------
-* Sequential (de)compression supported by :class:`BZ2Compressor` and
- :class:`BZ2Decompressor` classes;
+.. function:: open(filename, mode='r', compresslevel=9, encoding=None, errors=None, newline=None)
-* One-shot (de)compression supported by :func:`compress` and :func:`decompress`
- functions;
+ Open a bzip2-compressed file in binary or text mode, returning a :term:`file
+ object`.
-* Thread safety uses individual locking mechanism.
+ As with the constructor for :class:`BZ2File`, the *filename* argument can be
+ an actual filename (a :class:`str` or :class:`bytes` object), or an existing
+ file object to read from or write to.
+ The *mode* argument can be any of ``'r'``, ``'rb'``, ``'w'``, ``'wb'``,
+ ``'a'``, or ``'ab'`` for binary mode, or ``'rt'``, ``'wt'``, or ``'at'`` for
+ text mode. The default is ``'rb'``.
-(De)compression of files
-------------------------
+ The *compresslevel* argument is an integer from 1 to 9, as for the
+ :class:`BZ2File` constructor.
-Handling of compressed files is offered by the :class:`BZ2File` class.
+ For binary mode, this function is equivalent to the :class:`BZ2File`
+ constructor: ``BZ2File(filename, mode, compresslevel=compresslevel)``. In
+ this case, the *encoding*, *errors* and *newline* arguments must not be
+ provided.
+ For text mode, a :class:`BZ2File` object is created, and wrapped in an
+ :class:`io.TextIOWrapper` instance with the specified encoding, error
+ handling behavior, and line ending(s).
-.. class:: BZ2File(filename, mode='r', buffering=0, compresslevel=9)
+ .. versionadded:: 3.3
- Open a bz2 file. Mode can be either ``'r'`` or ``'w'``, for reading (default)
- or writing. When opened for writing, the file will be created if it doesn't
- exist, and truncated otherwise. If *buffering* is given, ``0`` means
- unbuffered, and larger numbers specify the buffer size; the default is
- ``0``. If *compresslevel* is given, it must be a number between ``1`` and
- ``9``; the default is ``9``. Add a ``'U'`` to mode to open the file for input
- with universal newline support. Any line ending in the input file will be
- seen as a ``'\n'`` in Python. Also, a file so opened gains the attribute
- :attr:`newlines`; the value for this attribute is one of ``None`` (no newline
- read yet), ``'\r'``, ``'\n'``, ``'\r\n'`` or a tuple containing all the
- newline types seen. Universal newlines are available only when
- reading. Instances support iteration in the same way as normal :class:`file`
- instances.
- :class:`BZ2File` supports the :keyword:`with` statement.
-
- .. versionchanged:: 3.1
- Support for the :keyword:`with` statement was added.
+.. class:: BZ2File(filename, mode='r', buffering=None, compresslevel=9)
+ Open a bzip2-compressed file in binary mode.
- .. note::
+ If *filename* is a :class:`str` or :class:`bytes` object, open the named file
+ directly. Otherwise, *filename* should be a :term:`file object`, which will
+ be used to read or write the compressed data.
- This class does not support input files containing multiple streams (such
- as those produced by the :program:`pbzip2` tool). When reading such an
- input file, only the first stream will be accessible. If you require
- support for multi-stream files, consider using the third-party
- :mod:`bz2file` module (available from
- `PyPI <http://pypi.python.org/pypi/bz2file>`_). This module provides a
- backport of Python 3.3's :class:`BZ2File` class, which does support
- multi-stream files.
+ The *mode* argument can be either ``'r'`` for reading (default), ``'w'`` for
+ overwriting, or ``'a'`` for appending. These can equivalently be given as
+ ``'rb'``, ``'wb'``, and ``'ab'`` respectively.
+ If *filename* is a file object (rather than an actual file name), a mode of
+ ``'w'`` does not truncate the file, and is instead equivalent to ``'a'``.
- .. method:: close()
+ The *buffering* argument is ignored. Its use is deprecated.
- Close the file. Sets data attribute :attr:`closed` to true. A closed file
- cannot be used for further I/O operations. :meth:`close` may be called
- more than once without error.
+ If *mode* is ``'w'`` or ``'a'``, *compresslevel* can be a number between
+ ``1`` and ``9`` specifying the level of compression: ``1`` produces the
+ least compression, and ``9`` (default) produces the most compression.
+ If *mode* is ``'r'``, the input file may be the concatenation of multiple
+ compressed streams.
- .. method:: read([size])
+ :class:`BZ2File` provides all of the members specified by the
+ :class:`io.BufferedIOBase`, except for :meth:`detach` and :meth:`truncate`.
+ Iteration and the :keyword:`with` statement are supported.
- Read at most *size* uncompressed bytes, returned as a byte string. If the
- *size* argument is negative or omitted, read until EOF is reached.
+ :class:`BZ2File` also provides the following method:
+ .. method:: peek([n])
- .. method:: readline([size])
+ Return buffered data without advancing the file position. At least one
+ byte of data will be returned (unless at EOF). The exact number of bytes
+ returned is unspecified.
- Return the next line from the file, as a byte string, retaining newline.
- A non-negative *size* argument limits the maximum number of bytes to
- return (an incomplete line may be returned then). Return an empty byte
- string at EOF.
+ .. versionadded:: 3.3
+ .. versionchanged:: 3.1
+ Support for the :keyword:`with` statement was added.
- .. method:: readlines([size])
-
- Return a list of lines read. The optional *size* argument, if given, is an
- approximate bound on the total number of bytes in the lines returned.
+ .. versionchanged:: 3.3
+ The :meth:`fileno`, :meth:`readable`, :meth:`seekable`, :meth:`writable`,
+ :meth:`read1` and :meth:`readinto` methods were added.
+ .. versionchanged:: 3.3
+ Support was added for *filename* being a :term:`file object` instead of an
+ actual filename.
- .. method:: seek(offset[, whence])
+ .. versionchanged:: 3.3
+ The ``'a'`` (append) mode was added, along with support for reading
+ multi-stream files.
- Move to new file position. Argument *offset* is a byte count. Optional
- argument *whence* defaults to ``os.SEEK_SET`` or ``0`` (offset from start
- of file; offset should be ``>= 0``); other values are ``os.SEEK_CUR`` or
- ``1`` (move relative to current position; offset can be positive or
- negative), and ``os.SEEK_END`` or ``2`` (move relative to end of file;
- offset is usually negative, although many platforms allow seeking beyond
- the end of a file).
- Note that seeking of bz2 files is emulated, and depending on the
- parameters the operation may be extremely slow.
+Incremental (de)compression
+---------------------------
+.. class:: BZ2Compressor(compresslevel=9)
- .. method:: tell()
+ Create a new compressor object. This object may be used to compress data
+ incrementally. For one-shot compression, use the :func:`compress` function
+ instead.
- Return the current file position, an integer.
+ *compresslevel*, if given, must be a number between ``1`` and ``9``. The
+ default is ``9``.
+ .. method:: compress(data)
- .. method:: write(data)
+ Provide data to the compressor object. Returns a chunk of compressed data
+ if possible, or an empty byte string otherwise.
- Write the byte string *data* to file. Note that due to buffering,
- :meth:`close` may be needed before the file on disk reflects the data
- written.
+ When you have finished providing data to the compressor, call the
+ :meth:`flush` method to finish the compression process.
- .. method:: writelines(sequence_of_byte_strings)
+ .. method:: flush()
- Write the sequence of byte strings to the file. Note that newlines are not
- added. The sequence can be any iterable object producing byte strings.
- This is equivalent to calling write() for each byte string.
+ Finish the compression process. Returns the compressed data left in
+ internal buffers.
+ The compressor object may not be used after this method has been called.
-Sequential (de)compression
---------------------------
-Sequential compression and decompression is done using the classes
-:class:`BZ2Compressor` and :class:`BZ2Decompressor`.
+.. class:: BZ2Decompressor()
+ Create a new decompressor object. This object may be used to decompress data
+ incrementally. For one-shot compression, use the :func:`decompress` function
+ instead.
-.. class:: BZ2Compressor(compresslevel=9)
+ .. note::
+ This class does not transparently handle inputs containing multiple
+ compressed streams, unlike :func:`decompress` and :class:`BZ2File`. If
+ you need to decompress a multi-stream input with :class:`BZ2Decompressor`,
+ you must use a new decompressor for each stream.
- Create a new compressor object. This object may be used to compress data
- sequentially. If you want to compress data in one shot, use the
- :func:`compress` function instead. The *compresslevel* parameter, if given,
- must be a number between ``1`` and ``9``; the default is ``9``.
+ .. method:: decompress(data)
- .. method:: compress(data)
+ Provide data to the decompressor object. Returns a chunk of decompressed
+ data if possible, or an empty byte string otherwise.
- Provide more data to the compressor object. It will return chunks of
- compressed data whenever possible. When you've finished providing data to
- compress, call the :meth:`flush` method to finish the compression process,
- and return what is left in internal buffers.
+ Attempting to decompress data after the end of the current stream is
+ reached raises an :exc:`EOFError`. If any data is found after the end of
+ the stream, it is ignored and saved in the :attr:`unused_data` attribute.
- .. method:: flush()
+ .. attribute:: eof
- Finish the compression process and return what is left in internal
- buffers. You must not use the compressor object after calling this method.
+ True if the end-of-stream marker has been reached.
+ .. versionadded:: 3.3
-.. class:: BZ2Decompressor()
- Create a new decompressor object. This object may be used to decompress data
- sequentially. If you want to decompress data in one shot, use the
- :func:`decompress` function instead.
+ .. attribute:: unused_data
- .. method:: decompress(data)
+ Data found after the end of the compressed stream.
- Provide more data to the decompressor object. It will return chunks of
- decompressed data whenever possible. If you try to decompress data after
- the end of stream is found, :exc:`EOFError` will be raised. If any data
- was found after the end of stream, it'll be ignored and saved in
- :attr:`unused_data` attribute.
+ If this attribute is accessed before the end of the stream has been
+ reached, its value will be ``b''``.
One-shot (de)compression
------------------------
-One-shot compression and decompression is provided through the :func:`compress`
-and :func:`decompress` functions.
+.. function:: compress(data, compresslevel=9)
+ Compress *data*.
-.. function:: compress(data, compresslevel=9)
+ *compresslevel*, if given, must be a number between ``1`` and ``9``. The
+ default is ``9``.
- Compress *data* in one shot. If you want to compress data sequentially, use
- an instance of :class:`BZ2Compressor` instead. The *compresslevel* parameter,
- if given, must be a number between ``1`` and ``9``; the default is ``9``.
+ For incremental compression, use a :class:`BZ2Compressor` instead.
.. function:: decompress(data)
- Decompress *data* in one shot. If you want to decompress data sequentially,
- use an instance of :class:`BZ2Decompressor` instead.
+ Decompress *data*.
+
+ If *data* is the concatenation of multiple compressed streams, decompress
+ all of the streams.
+
+ For incremental decompression, use a :class:`BZ2Decompressor` instead.
+
+ .. versionchanged:: 3.3
+ Support for multi-stream inputs was added.
diff --git a/Doc/library/chunk.rst b/Doc/library/chunk.rst
index d3558a4..c1ba497 100644
--- a/Doc/library/chunk.rst
+++ b/Doc/library/chunk.rst
@@ -84,8 +84,9 @@ instance will fail with a :exc:`EOFError` exception.
Close and skip to the end of the chunk. This does not close the
underlying file.
- The remaining methods will raise :exc:`IOError` if called after the
- :meth:`close` method has been called.
+ The remaining methods will raise :exc:`OSError` if called after the
+ :meth:`close` method has been called. Before Python 3.3, they used to
+ raise :exc:`IOError`, now an alias of :exc:`OSError`.
.. method:: isatty()
diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst
index fd7f453..0c43bb8 100644
--- a/Doc/library/cmd.rst
+++ b/Doc/library/cmd.rst
@@ -285,8 +285,8 @@ immediate playback::
def do_playback(self, arg):
'Playback commands from a file: PLAYBACK rose.cmd'
self.close()
- cmds = open(arg).read().splitlines()
- self.cmdqueue.extend(cmds)
+ with open(arg) as f:
+ self.cmdqueue.extend(f.read().splitlines())
def precmd(self, line):
line = line.lower()
if self.file and 'playback' not in line:
diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst
index 7747794..a9fae95 100644
--- a/Doc/library/codecs.rst
+++ b/Doc/library/codecs.rst
@@ -458,7 +458,8 @@ define in order to be compatible with the Python codec registry.
.. method:: reset()
- Reset the encoder to the initial state.
+ Reset the encoder to the initial state. The output is discarded: call
+ ``.encode('', final=True)`` to reset the encoder and to get the output.
.. method:: IncrementalEncoder.getstate()
@@ -786,11 +787,9 @@ methods and attributes from the underlying stream.
Encodings and Unicode
---------------------
-Strings are stored internally as sequences of codepoints (to be precise
-as :c:type:`Py_UNICODE` arrays). Depending on the way Python is compiled (either
-via ``--without-wide-unicode`` or ``--with-wide-unicode``, with the
-former being the default) :c:type:`Py_UNICODE` is either a 16-bit or 32-bit data
-type. Once a string object is used outside of CPU and memory, CPU endianness
+Strings are stored internally as sequences of codepoints in range ``0 - 10FFFF``
+(see :pep:`393` for more details about the implementation).
+Once a string object is used outside of CPU and memory, CPU endianness
and how these arrays are stored as bytes become an issue. Transforming a
string object into a sequence of bytes is called encoding and recreating the
string object from the sequence of bytes is known as decoding. There are many
@@ -901,6 +900,15 @@ is meant to be exhaustive. Notice that spelling alternatives that only differ in
case or use a hyphen instead of an underscore are also valid aliases; therefore,
e.g. ``'utf-8'`` is a valid alias for the ``'utf_8'`` codec.
+.. impl-detail::
+
+ Some common encodings can bypass the codecs lookup machinery to
+ improve performance. These optimization opportunities are only
+ recognized by CPython for a limited set of aliases: utf-8, utf8,
+ latin-1, latin1, iso-8859-1, mbcs (Windows only), ascii, utf-16,
+ and utf-32. Using alternative spellings for these encodings may
+ result in slower execution.
+
Many of the character sets support the same languages. They vary in individual
characters (e.g. whether the EURO SIGN is supported or not), and in the
assignment of characters to code positions. For the European languages in
@@ -1003,6 +1011,11 @@ particular, the following variants typically exist:
+-----------------+--------------------------------+--------------------------------+
| cp1258 | windows-1258 | Vietnamese |
+-----------------+--------------------------------+--------------------------------+
+| cp65001 | | Windows only: Windows UTF-8 |
+| | | (``CP_UTF8``) |
+| | | |
+| | | .. versionadded:: 3.3 |
++-----------------+--------------------------------+--------------------------------+
| euc_jp | eucjp, ujis, u-jis | Japanese |
+-----------------+--------------------------------+--------------------------------+
| euc_jis_2004 | jisx0213, eucjis2004 | Japanese |
@@ -1160,6 +1173,8 @@ particular, the following variants typically exist:
| unicode_internal | | Return the internal |
| | | representation of the |
| | | operand |
+| | | |
+| | | .. deprecated:: 3.3 |
+--------------------+---------+---------------------------+
The following codecs provide bytes-to-bytes mappings.
@@ -1272,12 +1287,13 @@ functions can be used directly if desired.
.. module:: encodings.mbcs
:synopsis: Windows ANSI codepage
-Encode operand according to the ANSI codepage (CP_ACP). This codec only
-supports ``'strict'`` and ``'replace'`` error handlers to encode, and
-``'strict'`` and ``'ignore'`` error handlers to decode.
+Encode operand according to the ANSI codepage (CP_ACP).
Availability: Windows only.
+.. versionchanged:: 3.3
+ Support any error handler.
+
.. versionchanged:: 3.2
Before 3.2, the *errors* argument was ignored; ``'replace'`` was always used
to encode, and ``'ignore'`` to decode.
diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst
new file mode 100644
index 0000000..9873489
--- /dev/null
+++ b/Doc/library/collections.abc.rst
@@ -0,0 +1,182 @@
+:mod:`collections.abc` --- Abstract Base Classes for Containers
+===============================================================
+
+.. module:: collections.abc
+ :synopsis: Abstract base classes for containers
+.. moduleauthor:: Raymond Hettinger <python at rcn.com>
+.. sectionauthor:: Raymond Hettinger <python at rcn.com>
+
+.. versionadded:: 3.3
+ Formerly, this module was part of the :mod:`collections` module.
+
+.. testsetup:: *
+
+ from collections import *
+ import itertools
+ __name__ = '<doctest>'
+
+**Source code:** :source:`Lib/collections/abc.py`
+
+--------------
+
+This module provides :term:`abstract base classes <abstract base class>` that
+can be used to test whether a class provides a particular interface; for
+example, whether it is hashable or whether it is a mapping.
+
+
+.. _collections-abstract-base-classes:
+
+Collections Abstract Base Classes
+---------------------------------
+
+The collections module offers the following :term:`ABCs <abstract base class>`:
+
+========================= ===================== ====================== ====================================================
+ABC Inherits from Abstract Methods Mixin Methods
+========================= ===================== ====================== ====================================================
+:class:`Container` ``__contains__``
+:class:`Hashable` ``__hash__``
+:class:`Iterable` ``__iter__``
+:class:`Iterator` :class:`Iterable` ``__next__`` ``__iter__``
+:class:`Sized` ``__len__``
+:class:`Callable` ``__call__``
+
+:class:`Sequence` :class:`Sized`, ``__getitem__`` ``__contains__``, ``__iter__``, ``__reversed__``,
+ :class:`Iterable`, ``index``, and ``count``
+ :class:`Container`
+
+:class:`MutableSequence` :class:`Sequence` ``__setitem__``, Inherited :class:`Sequence` methods and
+ ``__delitem__``, ``append``, ``reverse``, ``extend``, ``pop``,
+ ``insert`` ``remove``, ``clear``, and ``__iadd__``
+
+:class:`Set` :class:`Sized`, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``,
+ :class:`Iterable`, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``,
+ :class:`Container` ``__sub__``, ``__xor__``, and ``isdisjoint``
+
+:class:`MutableSet` :class:`Set` ``add``, Inherited :class:`Set` methods and
+ ``discard`` ``clear``, ``pop``, ``remove``, ``__ior__``,
+ ``__iand__``, ``__ixor__``, and ``__isub__``
+
+:class:`Mapping` :class:`Sized`, ``__getitem__`` ``__contains__``, ``keys``, ``items``, ``values``,
+ :class:`Iterable`, ``get``, ``__eq__``, and ``__ne__``
+ :class:`Container`
+
+:class:`MutableMapping` :class:`Mapping` ``__setitem__``, Inherited :class:`Mapping` methods and
+ ``__delitem__`` ``pop``, ``popitem``, ``clear``, ``update``,
+ and ``setdefault``
+
+
+:class:`MappingView` :class:`Sized` ``__len__``
+:class:`ItemsView` :class:`MappingView`, ``__contains__``,
+ :class:`Set` ``__iter__``
+:class:`KeysView` :class:`MappingView`, ``__contains__``,
+ :class:`Set` ``__iter__``
+:class:`ValuesView` :class:`MappingView` ``__contains__``, ``__iter__``
+========================= ===================== ====================== ====================================================
+
+
+.. class:: Container
+ Hashable
+ Sized
+ Callable
+
+ ABCs for classes that provide respectively the methods :meth:`__contains__`,
+ :meth:`__hash__`, :meth:`__len__`, and :meth:`__call__`.
+
+.. class:: Iterable
+
+ ABC for classes that provide the :meth:`__iter__` method.
+ See also the definition of :term:`iterable`.
+
+.. class:: Iterator
+
+ ABC for classes that provide the :meth:`__iter__` and :meth:`next` methods.
+ See also the definition of :term:`iterator`.
+
+.. class:: Sequence
+ MutableSequence
+
+ ABCs for read-only and mutable :term:`sequences <sequence>`.
+
+.. class:: Set
+ MutableSet
+
+ ABCs for read-only and mutable sets.
+
+.. class:: Mapping
+ MutableMapping
+
+ ABCs for read-only and mutable :term:`mappings <mapping>`.
+
+.. class:: MappingView
+ ItemsView
+ KeysView
+ ValuesView
+
+ ABCs for mapping, items, keys, and values :term:`views <view>`.
+
+
+These ABCs allow us to ask classes or instances if they provide
+particular functionality, for example::
+
+ size = None
+ if isinstance(myvar, collections.Sized):
+ size = len(myvar)
+
+Several of the ABCs are also useful as mixins that make it easier to develop
+classes supporting container APIs. For example, to write a class supporting
+the full :class:`Set` API, it only necessary to supply the three underlying
+abstract methods: :meth:`__contains__`, :meth:`__iter__`, and :meth:`__len__`.
+The ABC supplies the remaining methods such as :meth:`__and__` and
+:meth:`isdisjoint` ::
+
+ class ListBasedSet(collections.Set):
+ ''' Alternate set implementation favoring space over speed
+ and not requiring the set elements to be hashable. '''
+ def __init__(self, iterable):
+ self.elements = lst = []
+ for value in iterable:
+ if value not in lst:
+ lst.append(value)
+ def __iter__(self):
+ return iter(self.elements)
+ def __contains__(self, value):
+ return value in self.elements
+ def __len__(self):
+ return len(self.elements)
+
+ s1 = ListBasedSet('abcdef')
+ s2 = ListBasedSet('defghi')
+ overlap = s1 & s2 # The __and__() method is supported automatically
+
+Notes on using :class:`Set` and :class:`MutableSet` as a mixin:
+
+(1)
+ Since some set operations create new sets, the default mixin methods need
+ a way to create new instances from an iterable. The class constructor is
+ assumed to have a signature in the form ``ClassName(iterable)``.
+ That assumption is factored-out to an internal classmethod called
+ :meth:`_from_iterable` which calls ``cls(iterable)`` to produce a new set.
+ If the :class:`Set` mixin is being used in a class with a different
+ constructor signature, you will need to override :meth:`_from_iterable`
+ with a classmethod that can construct new instances from
+ an iterable argument.
+
+(2)
+ To override the comparisons (presumably for speed, as the
+ semantics are fixed), redefine :meth:`__le__` and
+ then the other operations will automatically follow suit.
+
+(3)
+ The :class:`Set` mixin provides a :meth:`_hash` method to compute a hash value
+ for the set; however, :meth:`__hash__` is not defined because not all sets
+ are hashable or immutable. To add set hashabilty using mixins,
+ inherit from both :meth:`Set` and :meth:`Hashable`, then define
+ ``__hash__ = Set._hash``.
+
+.. seealso::
+
+ * `OrderedSet recipe <http://code.activestate.com/recipes/576694/>`_ for an
+ example built on :class:`MutableSet`.
+
+ * For more about ABCs, see the :mod:`abc` module and :pep:`3119`.
diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst
index e90b25e..d538c55 100644
--- a/Doc/library/collections.rst
+++ b/Doc/library/collections.rst
@@ -2,17 +2,17 @@
==========================================
.. module:: collections
- :synopsis: Container datatypes
+ :synopsis: Container datatypes
.. moduleauthor:: Raymond Hettinger <python@rcn.com>
.. sectionauthor:: Raymond Hettinger <python@rcn.com>
.. testsetup:: *
- from collections import *
- import itertools
- __name__ = '<doctest>'
+ from collections import *
+ import itertools
+ __name__ = '<doctest>'
-**Source code:** :source:`Lib/collections.py` and :source:`Lib/_abcoll.py`
+**Source code:** :source:`Lib/collections/__init__.py`
--------------
@@ -23,6 +23,7 @@ Python's general purpose built-in containers, :class:`dict`, :class:`list`,
===================== ====================================================================
:func:`namedtuple` factory function for creating tuple subclasses with named fields
:class:`deque` list-like container with fast appends and pops on either end
+:class:`ChainMap` dict-like class for creating a single view of multiple mappings
:class:`Counter` dict subclass for counting hashable objects
:class:`OrderedDict` dict subclass that remembers the order entries were added
:class:`defaultdict` dict subclass that calls a factory function to supply missing values
@@ -31,10 +32,160 @@ Python's general purpose built-in containers, :class:`dict`, :class:`list`,
:class:`UserString` wrapper around string objects for easier string subclassing
===================== ====================================================================
-In addition to the concrete container classes, the collections module provides
-:ref:`abstract base classes <collections-abstract-base-classes>` that can be
-used to test whether a class provides a particular interface, for example,
-whether it is hashable or a mapping.
+.. versionchanged:: 3.3
+ Moved :ref:`collections-abstract-base-classes` to the :mod:`collections.abc` module.
+ For backwards compatibility, they continue to be visible in this module
+ as well.
+
+
+:class:`ChainMap` objects
+-------------------------
+
+.. versionadded:: 3.3
+
+A :class:`ChainMap` class is provided for quickly linking a number of mappings
+so they can be treated as a single unit. It is often much faster than creating
+a new dictionary and running multiple :meth:`~dict.update` calls.
+
+The class can be used to simulate nested scopes and is useful in templating.
+
+.. class:: ChainMap(*maps)
+
+ A :class:`ChainMap` groups multiple dicts or other mappings together to
+ create a single, updateable view. If no *maps* are specified, a single empty
+ dictionary is provided so that a new chain always has at least one mapping.
+
+ The underlying mappings are stored in a list. That list is public and can
+ accessed or updated using the *maps* attribute. There is no other state.
+
+ Lookups search the underlying mappings successively until a key is found. In
+ contrast, writes, updates, and deletions only operate on the first mapping.
+
+ A :class:`ChainMap` incorporates the underlying mappings by reference. So, if
+ one of the underlying mappings gets updated, those changes will be reflected
+ in :class:`ChainMap`.
+
+ All of the usual dictionary methods are supported. In addition, there is a
+ *maps* attribute, a method for creating new subcontexts, and a property for
+ accessing all but the first mapping:
+
+ .. attribute:: maps
+
+ A user updateable list of mappings. The list is ordered from
+ first-searched to last-searched. It is the only stored state and can
+ be modified to change which mappings are searched. The list should
+ always contain at least one mapping.
+
+ .. method:: new_child()
+
+ Returns a new :class:`ChainMap` containing a new :class:`dict` followed by
+ all of the maps in the current instance. A call to ``d.new_child()`` is
+ equivalent to: ``ChainMap({}, *d.maps)``. This method is used for
+ creating subcontexts that can be updated without altering values in any
+ of the parent mappings.
+
+ .. method:: parents()
+
+ Returns a new :class:`ChainMap` containing all of the maps in the current
+ instance except the first one. This is useful for skipping the first map
+ in the search. The use-cases are similar to those for the
+ :keyword:`nonlocal` keyword used in :term:`nested scopes <nested scope>`.
+ The use-cases also parallel those for the builtin :func:`super` function.
+ A reference to ``d.parents`` is equivalent to: ``ChainMap(*d.maps[1:])``.
+
+
+.. seealso::
+
+ * The `MultiContext class
+ <http://svn.enthought.com/svn/enthought/CodeTools/trunk/enthought/contexts/multi_context.py>`_
+ in the Enthought `CodeTools package
+ <https://github.com/enthought/codetools>`_ has options to support
+ writing to any mapping in the chain.
+
+ * Django's `Context class
+ <http://code.djangoproject.com/browser/django/trunk/django/template/context.py>`_
+ for templating is a read-only chain of mappings. It also features
+ pushing and popping of contexts similar to the
+ :meth:`~collections.ChainMap.new_child` method and the
+ :meth:`~collections.ChainMap.parents` property.
+
+ * The `Nested Contexts recipe
+ <http://code.activestate.com/recipes/577434/>`_ has options to control
+ whether writes and other mutations apply only to the first mapping or to
+ any mapping in the chain.
+
+ * A `greatly simplified read-only version of Chainmap
+ <http://code.activestate.com/recipes/305268/>`_.
+
+
+:class:`ChainMap` Examples and Recipes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This section shows various approaches to working with chained maps.
+
+
+Example of simulating Python's internal lookup chain::
+
+ import builtins
+ pylookup = ChainMap(locals(), globals(), vars(builtins))
+
+Example of letting user specified values take precedence over environment
+variables which in turn take precedence over default values::
+
+ import os, argparse
+ defaults = {'color': 'red', 'user': guest}
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-u', '--user')
+ parser.add_argument('-c', '--color')
+ user_specified = vars(parser.parse_args())
+ combined = ChainMap(user_specified, os.environ, defaults)
+
+Example patterns for using the :class:`ChainMap` class to simulate nested
+contexts::
+
+ c = ChainMap() # Create root context
+ d = c.new_child() # Create nested child context
+ e = c.new_child() # Child of c, independent from d
+ e.maps[0] # Current context dictionary -- like Python's locals()
+ e.maps[-1] # Root context -- like Python's globals()
+ e.parents # Enclosing context chain -- like Python's nonlocals
+
+ d['x'] # Get first key in the chain of contexts
+ d['x'] = 1 # Set value in current context
+ del['x'] # Delete from current context
+ list(d) # All nested values
+ k in d # Check all nested values
+ len(d) # Number of nested values
+ d.items() # All nested items
+ dict(d) # Flatten into a regular dictionary
+
+The :class:`ChainMap` class only makes updates (writes and deletions) to the
+first mapping in the chain while lookups will search the full chain. However,
+if deep writes and deletions are desired, it is easy to make a subclass that
+updates keys found deeper in the chain::
+
+ class DeepChainMap(ChainMap):
+ 'Variant of ChainMap that allows direct updates to inner scopes'
+
+ def __setitem__(self, key, value):
+ for mapping in self.maps:
+ if key in mapping:
+ mapping[key] = value
+ return
+ self.maps[0][key] = value
+
+ def __delitem__(self, key):
+ for mapping in self.maps:
+ if key in mapping:
+ del mapping[key]
+ return
+ raise KeyError(key)
+
+ >>> d = DeepChainMap({'zebra': 'black'}, {'elephant' : 'blue'}, {'lion' : 'yellow'})
+ >>> d['lion'] = 'orange' # update an existing key two levels down
+ >>> d['snake'] = 'red' # new keys get added to the topmost dict
+ >>> del d['elephant'] # remove an existing key one level down
+ DeepChainMap({'zebra': 'black', 'snake': 'red'}, {}, {'lion': 'orange'})
:class:`Counter` objects
@@ -59,85 +210,85 @@ For example::
.. class:: Counter([iterable-or-mapping])
- A :class:`Counter` is a :class:`dict` subclass for counting hashable objects.
- It is an unordered collection where elements are stored as dictionary keys
- and their counts are stored as dictionary values. Counts are allowed to be
- any integer value including zero or negative counts. The :class:`Counter`
- class is similar to bags or multisets in other languages.
+ A :class:`Counter` is a :class:`dict` subclass for counting hashable objects.
+ It is an unordered collection where elements are stored as dictionary keys
+ and their counts are stored as dictionary values. Counts are allowed to be
+ any integer value including zero or negative counts. The :class:`Counter`
+ class is similar to bags or multisets in other languages.
- Elements are counted from an *iterable* or initialized from another
- *mapping* (or counter):
+ Elements are counted from an *iterable* or initialized from another
+ *mapping* (or counter):
>>> c = Counter() # a new, empty counter
>>> c = Counter('gallahad') # a new counter from an iterable
>>> c = Counter({'red': 4, 'blue': 2}) # a new counter from a mapping
>>> c = Counter(cats=4, dogs=8) # a new counter from keyword args
- Counter objects have a dictionary interface except that they return a zero
- count for missing items instead of raising a :exc:`KeyError`:
+ Counter objects have a dictionary interface except that they return a zero
+ count for missing items instead of raising a :exc:`KeyError`:
>>> c = Counter(['eggs', 'ham'])
>>> c['bacon'] # count of a missing element is zero
0
- Setting a count to zero does not remove an element from a counter.
- Use ``del`` to remove it entirely:
+ Setting a count to zero does not remove an element from a counter.
+ Use ``del`` to remove it entirely:
>>> c['sausage'] = 0 # counter entry with a zero count
>>> del c['sausage'] # del actually removes the entry
- .. versionadded:: 3.1
+ .. versionadded:: 3.1
- Counter objects support three methods beyond those available for all
- dictionaries:
+ Counter objects support three methods beyond those available for all
+ dictionaries:
- .. method:: elements()
+ .. method:: elements()
- Return an iterator over elements repeating each as many times as its
- count. Elements are returned in arbitrary order. If an element's count
- is less than one, :meth:`elements` will ignore it.
+ Return an iterator over elements repeating each as many times as its
+ count. Elements are returned in arbitrary order. If an element's count
+ is less than one, :meth:`elements` will ignore it.
>>> c = Counter(a=4, b=2, c=0, d=-2)
>>> list(c.elements())
['a', 'a', 'a', 'a', 'b', 'b']
- .. method:: most_common([n])
+ .. method:: most_common([n])
- Return a list of the *n* most common elements and their counts from the
- most common to the least. If *n* is not specified, :func:`most_common`
- returns *all* elements in the counter. Elements with equal counts are
- ordered arbitrarily:
+ Return a list of the *n* most common elements and their counts from the
+ most common to the least. If *n* is not specified, :func:`most_common`
+ returns *all* elements in the counter. Elements with equal counts are
+ ordered arbitrarily:
>>> Counter('abracadabra').most_common(3)
[('a', 5), ('r', 2), ('b', 2)]
- .. method:: subtract([iterable-or-mapping])
+ .. method:: subtract([iterable-or-mapping])
- Elements are subtracted from an *iterable* or from another *mapping*
- (or counter). Like :meth:`dict.update` but subtracts counts instead
- of replacing them. Both inputs and outputs may be zero or negative.
+ Elements are subtracted from an *iterable* or from another *mapping*
+ (or counter). Like :meth:`dict.update` but subtracts counts instead
+ of replacing them. Both inputs and outputs may be zero or negative.
>>> c = Counter(a=4, b=2, c=0, d=-2)
>>> d = Counter(a=1, b=2, c=3, d=4)
>>> c.subtract(d)
Counter({'a': 3, 'b': 0, 'c': -3, 'd': -6})
- .. versionadded:: 3.2
+ .. versionadded:: 3.2
- The usual dictionary methods are available for :class:`Counter` objects
- except for two which work differently for counters.
+ The usual dictionary methods are available for :class:`Counter` objects
+ except for two which work differently for counters.
- .. method:: fromkeys(iterable)
+ .. method:: fromkeys(iterable)
- This class method is not implemented for :class:`Counter` objects.
+ This class method is not implemented for :class:`Counter` objects.
- .. method:: update([iterable-or-mapping])
+ .. method:: update([iterable-or-mapping])
- Elements are counted from an *iterable* or added-in from another
- *mapping* (or counter). Like :meth:`dict.update` but adds counts
- instead of replacing them. Also, the *iterable* is expected to be a
- sequence of elements, not a sequence of ``(key, value)`` pairs.
+ Elements are counted from an *iterable* or added-in from another
+ *mapping* (or counter). Like :meth:`dict.update` but adds counts
+ instead of replacing them. Also, the *iterable* is expected to be a
+ sequence of elements, not a sequence of ``(key, value)`` pairs.
Common patterns for working with :class:`Counter` objects::
@@ -149,7 +300,7 @@ Common patterns for working with :class:`Counter` objects::
c.items() # convert to a list of (elem, cnt) pairs
Counter(dict(list_of_pairs)) # convert from a list of (elem, cnt) pairs
c.most_common()[:-n:-1] # n least common elements
- c += Counter() # remove zero and negative counts
+ +c # remove zero and negative counts
Several mathematical operations are provided for combining :class:`Counter`
objects to produce multisets (counters that have counts greater than zero).
@@ -169,32 +320,44 @@ counts, but the output will exclude results with counts of zero or less.
>>> c | d # union: max(c[x], d[x])
Counter({'a': 3, 'b': 2})
+Unary addition and substraction are shortcuts for adding an empty counter
+or subtracting from an empty counter.
+
+ >>> c = Counter(a=2, b=-4)
+ >>> +c
+ Counter({'a': 2})
+ >>> -c
+ Counter({'b': 4})
+
+.. versionadded:: 3.3
+ Added support for unary plus, unary minus, and in-place multiset operations.
+
.. note::
- Counters were primarily designed to work with positive integers to represent
- running counts; however, care was taken to not unnecessarily preclude use
- cases needing other types or negative values. To help with those use cases,
- this section documents the minimum range and type restrictions.
+ Counters were primarily designed to work with positive integers to represent
+ running counts; however, care was taken to not unnecessarily preclude use
+ cases needing other types or negative values. To help with those use cases,
+ this section documents the minimum range and type restrictions.
- * The :class:`Counter` class itself is a dictionary subclass with no
- restrictions on its keys and values. The values are intended to be numbers
- representing counts, but you *could* store anything in the value field.
+ * The :class:`Counter` class itself is a dictionary subclass with no
+ restrictions on its keys and values. The values are intended to be numbers
+ representing counts, but you *could* store anything in the value field.
- * The :meth:`most_common` method requires only that the values be orderable.
+ * The :meth:`most_common` method requires only that the values be orderable.
- * For in-place operations such as ``c[key] += 1``, the value type need only
- support addition and subtraction. So fractions, floats, and decimals would
- work and negative values are supported. The same is also true for
- :meth:`update` and :meth:`subtract` which allow negative and zero values
- for both inputs and outputs.
+ * For in-place operations such as ``c[key] += 1``, the value type need only
+ support addition and subtraction. So fractions, floats, and decimals would
+ work and negative values are supported. The same is also true for
+ :meth:`update` and :meth:`subtract` which allow negative and zero values
+ for both inputs and outputs.
- * The multiset methods are designed only for use cases with positive values.
- The inputs may be negative or zero, but only outputs with positive values
- are created. There are no type restrictions, but the value type needs to
- support addition, subtraction, and comparison.
+ * The multiset methods are designed only for use cases with positive values.
+ The inputs may be negative or zero, but only outputs with positive values
+ are created. There are no type restrictions, but the value type needs to
+ support addition, subtraction, and comparison.
- * The :meth:`elements` method requires integer counts. It ignores zero and
- negative counts.
+ * The :meth:`elements` method requires integer counts. It ignores zero and
+ negative counts.
.. seealso::
@@ -217,7 +380,7 @@ counts, but the output will exclude results with counts of zero or less.
* To enumerate all distinct multisets of a given size over a given set of
elements, see :func:`itertools.combinations_with_replacement`.
- map(Counter, combinations_with_replacement('ABC', 2)) --> AA AB AC BB BC CC
+ map(Counter, combinations_with_replacement('ABC', 2)) --> AA AB AC BB BC CC
:class:`deque` objects
@@ -225,105 +388,105 @@ counts, but the output will exclude results with counts of zero or less.
.. class:: deque([iterable, [maxlen]])
- Returns a new deque object initialized left-to-right (using :meth:`append`) with
- data from *iterable*. If *iterable* is not specified, the new deque is empty.
+ Returns a new deque object initialized left-to-right (using :meth:`append`) with
+ data from *iterable*. If *iterable* is not specified, the new deque is empty.
- Deques are a generalization of stacks and queues (the name is pronounced "deck"
- and is short for "double-ended queue"). Deques support thread-safe, memory
- efficient appends and pops from either side of the deque with approximately the
- same O(1) performance in either direction.
+ Deques are a generalization of stacks and queues (the name is pronounced "deck"
+ and is short for "double-ended queue"). Deques support thread-safe, memory
+ efficient appends and pops from either side of the deque with approximately the
+ same O(1) performance in either direction.
- Though :class:`list` objects support similar operations, they are optimized for
- fast fixed-length operations and incur O(n) memory movement costs for
- ``pop(0)`` and ``insert(0, v)`` operations which change both the size and
- position of the underlying data representation.
+ Though :class:`list` objects support similar operations, they are optimized for
+ fast fixed-length operations and incur O(n) memory movement costs for
+ ``pop(0)`` and ``insert(0, v)`` operations which change both the size and
+ position of the underlying data representation.
- If *maxlen* is not specified or is *None*, deques may grow to an
- arbitrary length. Otherwise, the deque is bounded to the specified maximum
- length. Once a bounded length deque is full, when new items are added, a
- corresponding number of items are discarded from the opposite end. Bounded
- length deques provide functionality similar to the ``tail`` filter in
- Unix. They are also useful for tracking transactions and other pools of data
- where only the most recent activity is of interest.
+ If *maxlen* is not specified or is *None*, deques may grow to an
+ arbitrary length. Otherwise, the deque is bounded to the specified maximum
+ length. Once a bounded length deque is full, when new items are added, a
+ corresponding number of items are discarded from the opposite end. Bounded
+ length deques provide functionality similar to the ``tail`` filter in
+ Unix. They are also useful for tracking transactions and other pools of data
+ where only the most recent activity is of interest.
- Deque objects support the following methods:
+ Deque objects support the following methods:
- .. method:: append(x)
+ .. method:: append(x)
- Add *x* to the right side of the deque.
+ Add *x* to the right side of the deque.
- .. method:: appendleft(x)
+ .. method:: appendleft(x)
- Add *x* to the left side of the deque.
+ Add *x* to the left side of the deque.
- .. method:: clear()
+ .. method:: clear()
- Remove all elements from the deque leaving it with length 0.
+ Remove all elements from the deque leaving it with length 0.
- .. method:: count(x)
+ .. method:: count(x)
- Count the number of deque elements equal to *x*.
+ Count the number of deque elements equal to *x*.
- .. versionadded:: 3.2
+ .. versionadded:: 3.2
- .. method:: extend(iterable)
+ .. method:: extend(iterable)
- Extend the right side of the deque by appending elements from the iterable
- argument.
+ Extend the right side of the deque by appending elements from the iterable
+ argument.
- .. method:: extendleft(iterable)
+ .. method:: extendleft(iterable)
- Extend the left side of the deque by appending elements from *iterable*.
- Note, the series of left appends results in reversing the order of
- elements in the iterable argument.
+ Extend the left side of the deque by appending elements from *iterable*.
+ Note, the series of left appends results in reversing the order of
+ elements in the iterable argument.
- .. method:: pop()
+ .. method:: pop()
- Remove and return an element from the right side of the deque. If no
- elements are present, raises an :exc:`IndexError`.
+ Remove and return an element from the right side of the deque. If no
+ elements are present, raises an :exc:`IndexError`.
- .. method:: popleft()
+ .. method:: popleft()
- Remove and return an element from the left side of the deque. If no
- elements are present, raises an :exc:`IndexError`.
+ Remove and return an element from the left side of the deque. If no
+ elements are present, raises an :exc:`IndexError`.
- .. method:: remove(value)
+ .. method:: remove(value)
- Removed the first occurrence of *value*. If not found, raises a
- :exc:`ValueError`.
+ Removed the first occurrence of *value*. If not found, raises a
+ :exc:`ValueError`.
- .. method:: reverse()
+ .. method:: reverse()
- Reverse the elements of the deque in-place and then return ``None``.
+ Reverse the elements of the deque in-place and then return ``None``.
- .. versionadded:: 3.2
+ .. versionadded:: 3.2
- .. method:: rotate(n)
+ .. method:: rotate(n)
- Rotate the deque *n* steps to the right. If *n* is negative, rotate to
- the left. Rotating one step to the right is equivalent to:
- ``d.appendleft(d.pop())``.
+ Rotate the deque *n* steps to the right. If *n* is negative, rotate to
+ the left. Rotating one step to the right is equivalent to:
+ ``d.appendleft(d.pop())``.
- Deque objects also provide one read-only attribute:
+ Deque objects also provide one read-only attribute:
- .. attribute:: maxlen
+ .. attribute:: maxlen
- Maximum size of a deque or *None* if unbounded.
+ Maximum size of a deque or *None* if unbounded.
- .. versionadded:: 3.1
+ .. versionadded:: 3.1
In addition to the above, deques support iteration, pickling, ``len(d)``,
@@ -336,56 +499,56 @@ Example:
.. doctest::
- >>> from collections import deque
- >>> d = deque('ghi') # make a new deque with three items
- >>> for elem in d: # iterate over the deque's elements
- ... print(elem.upper())
- G
- H
- I
-
- >>> d.append('j') # add a new entry to the right side
- >>> d.appendleft('f') # add a new entry to the left side
- >>> d # show the representation of the deque
- deque(['f', 'g', 'h', 'i', 'j'])
-
- >>> d.pop() # return and remove the rightmost item
- 'j'
- >>> d.popleft() # return and remove the leftmost item
- 'f'
- >>> list(d) # list the contents of the deque
- ['g', 'h', 'i']
- >>> d[0] # peek at leftmost item
- 'g'
- >>> d[-1] # peek at rightmost item
- 'i'
-
- >>> list(reversed(d)) # list the contents of a deque in reverse
- ['i', 'h', 'g']
- >>> 'h' in d # search the deque
- True
- >>> d.extend('jkl') # add multiple elements at once
- >>> d
- deque(['g', 'h', 'i', 'j', 'k', 'l'])
- >>> d.rotate(1) # right rotation
- >>> d
- deque(['l', 'g', 'h', 'i', 'j', 'k'])
- >>> d.rotate(-1) # left rotation
- >>> d
- deque(['g', 'h', 'i', 'j', 'k', 'l'])
-
- >>> deque(reversed(d)) # make a new deque in reverse order
- deque(['l', 'k', 'j', 'i', 'h', 'g'])
- >>> d.clear() # empty the deque
- >>> d.pop() # cannot pop from an empty deque
- Traceback (most recent call last):
- File "<pyshell#6>", line 1, in -toplevel-
- d.pop()
- IndexError: pop from an empty deque
-
- >>> d.extendleft('abc') # extendleft() reverses the input order
- >>> d
- deque(['c', 'b', 'a'])
+ >>> from collections import deque
+ >>> d = deque('ghi') # make a new deque with three items
+ >>> for elem in d: # iterate over the deque's elements
+ ... print(elem.upper())
+ G
+ H
+ I
+
+ >>> d.append('j') # add a new entry to the right side
+ >>> d.appendleft('f') # add a new entry to the left side
+ >>> d # show the representation of the deque
+ deque(['f', 'g', 'h', 'i', 'j'])
+
+ >>> d.pop() # return and remove the rightmost item
+ 'j'
+ >>> d.popleft() # return and remove the leftmost item
+ 'f'
+ >>> list(d) # list the contents of the deque
+ ['g', 'h', 'i']
+ >>> d[0] # peek at leftmost item
+ 'g'
+ >>> d[-1] # peek at rightmost item
+ 'i'
+
+ >>> list(reversed(d)) # list the contents of a deque in reverse
+ ['i', 'h', 'g']
+ >>> 'h' in d # search the deque
+ True
+ >>> d.extend('jkl') # add multiple elements at once
+ >>> d
+ deque(['g', 'h', 'i', 'j', 'k', 'l'])
+ >>> d.rotate(1) # right rotation
+ >>> d
+ deque(['l', 'g', 'h', 'i', 'j', 'k'])
+ >>> d.rotate(-1) # left rotation
+ >>> d
+ deque(['g', 'h', 'i', 'j', 'k', 'l'])
+
+ >>> deque(reversed(d)) # make a new deque in reverse order
+ deque(['l', 'k', 'j', 'i', 'h', 'g'])
+ >>> d.clear() # empty the deque
+ >>> d.pop() # cannot pop from an empty deque
+ Traceback (most recent call last):
+ File "<pyshell#6>", line 1, in -toplevel-
+ d.pop()
+ IndexError: pop from an empty deque
+
+ >>> d.extendleft('abc') # extendleft() reverses the input order
+ >>> d
+ deque(['c', 'b', 'a'])
:class:`deque` Recipes
@@ -396,9 +559,10 @@ This section shows various approaches to working with deques.
Bounded length deques provide functionality similar to the ``tail`` filter
in Unix::
- def tail(filename, n=10):
- 'Return the last n lines of a file'
- return deque(open(filename), n)
+ def tail(filename, n=10):
+ 'Return the last n lines of a file'
+ with open(filename) as f:
+ return deque(f, n)
Another approach to using deques is to maintain a sequence of recently
added elements by appending to the right and popping to the left::
@@ -419,10 +583,10 @@ The :meth:`rotate` method provides a way to implement :class:`deque` slicing and
deletion. For example, a pure Python implementation of ``del d[n]`` relies on
the :meth:`rotate` method to position elements to be popped::
- def delete_nth(d, n):
- d.rotate(-n)
- d.popleft()
- d.rotate(n)
+ def delete_nth(d, n):
+ d.rotate(-n)
+ d.popleft()
+ d.rotate(n)
To implement :class:`deque` slicing, use a similar approach applying
:meth:`rotate` to bring a target element to the left side of the deque. Remove
@@ -438,50 +602,50 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``,
.. class:: defaultdict([default_factory[, ...]])
- Returns a new dictionary-like object. :class:`defaultdict` is a subclass of the
- built-in :class:`dict` class. It overrides one method and adds one writable
- instance variable. The remaining functionality is the same as for the
- :class:`dict` class and is not documented here.
+ Returns a new dictionary-like object. :class:`defaultdict` is a subclass of the
+ built-in :class:`dict` class. It overrides one method and adds one writable
+ instance variable. The remaining functionality is the same as for the
+ :class:`dict` class and is not documented here.
- The first argument provides the initial value for the :attr:`default_factory`
- attribute; it defaults to ``None``. All remaining arguments are treated the same
- as if they were passed to the :class:`dict` constructor, including keyword
- arguments.
+ The first argument provides the initial value for the :attr:`default_factory`
+ attribute; it defaults to ``None``. All remaining arguments are treated the same
+ as if they were passed to the :class:`dict` constructor, including keyword
+ arguments.
- :class:`defaultdict` objects support the following method in addition to the
- standard :class:`dict` operations:
+ :class:`defaultdict` objects support the following method in addition to the
+ standard :class:`dict` operations:
- .. method:: __missing__(key)
+ .. method:: __missing__(key)
- If the :attr:`default_factory` attribute is ``None``, this raises a
- :exc:`KeyError` exception with the *key* as argument.
+ If the :attr:`default_factory` attribute is ``None``, this raises a
+ :exc:`KeyError` exception with the *key* as argument.
- If :attr:`default_factory` is not ``None``, it is called without arguments
- to provide a default value for the given *key*, this value is inserted in
- the dictionary for the *key*, and returned.
+ If :attr:`default_factory` is not ``None``, it is called without arguments
+ to provide a default value for the given *key*, this value is inserted in
+ the dictionary for the *key*, and returned.
- If calling :attr:`default_factory` raises an exception this exception is
- propagated unchanged.
+ If calling :attr:`default_factory` raises an exception this exception is
+ propagated unchanged.
- This method is called by the :meth:`__getitem__` method of the
- :class:`dict` class when the requested key is not found; whatever it
- returns or raises is then returned or raised by :meth:`__getitem__`.
+ This method is called by the :meth:`__getitem__` method of the
+ :class:`dict` class when the requested key is not found; whatever it
+ returns or raises is then returned or raised by :meth:`__getitem__`.
- Note that :meth:`__missing__` is *not* called for any operations besides
- :meth:`__getitem__`. This means that :meth:`get` will, like normal
- dictionaries, return ``None`` as a default rather than using
- :attr:`default_factory`.
+ Note that :meth:`__missing__` is *not* called for any operations besides
+ :meth:`__getitem__`. This means that :meth:`get` will, like normal
+ dictionaries, return ``None`` as a default rather than using
+ :attr:`default_factory`.
- :class:`defaultdict` objects support the following instance variable:
+ :class:`defaultdict` objects support the following instance variable:
- .. attribute:: default_factory
+ .. attribute:: default_factory
- This attribute is used by the :meth:`__missing__` method; it is
- initialized from the first argument to the constructor, if present, or to
- ``None``, if absent.
+ This attribute is used by the :meth:`__missing__` method; it is
+ initialized from the first argument to the constructor, if present, or to
+ ``None``, if absent.
:class:`defaultdict` Examples
@@ -490,13 +654,13 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``,
Using :class:`list` as the :attr:`default_factory`, it is easy to group a
sequence of key-value pairs into a dictionary of lists:
- >>> s = [('yellow', 1), ('blue', 2), ('yellow', 3), ('blue', 4), ('red', 1)]
- >>> d = defaultdict(list)
- >>> for k, v in s:
- ... d[k].append(v)
- ...
- >>> list(d.items())
- [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])]
+ >>> s = [('yellow', 1), ('blue', 2), ('yellow', 3), ('blue', 4), ('red', 1)]
+ >>> d = defaultdict(list)
+ >>> for k, v in s:
+ ... d[k].append(v)
+ ...
+ >>> list(d.items())
+ [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])]
When each key is encountered for the first time, it is not already in the
mapping; so an entry is automatically created using the :attr:`default_factory`
@@ -506,24 +670,24 @@ again, the look-up proceeds normally (returning the list for that key) and the
:meth:`list.append` operation adds another value to the list. This technique is
simpler and faster than an equivalent technique using :meth:`dict.setdefault`:
- >>> d = {}
- >>> for k, v in s:
- ... d.setdefault(k, []).append(v)
- ...
- >>> list(d.items())
- [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])]
+ >>> d = {}
+ >>> for k, v in s:
+ ... d.setdefault(k, []).append(v)
+ ...
+ >>> list(d.items())
+ [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])]
Setting the :attr:`default_factory` to :class:`int` makes the
:class:`defaultdict` useful for counting (like a bag or multiset in other
languages):
- >>> s = 'mississippi'
- >>> d = defaultdict(int)
- >>> for k in s:
- ... d[k] += 1
- ...
- >>> list(d.items())
- [('i', 4), ('p', 2), ('s', 4), ('m', 1)]
+ >>> s = 'mississippi'
+ >>> d = defaultdict(int)
+ >>> for k in s:
+ ... d[k] += 1
+ ...
+ >>> list(d.items())
+ [('i', 4), ('p', 2), ('s', 4), ('m', 1)]
When a letter is first encountered, it is missing from the mapping, so the
:attr:`default_factory` function calls :func:`int` to supply a default count of
@@ -534,23 +698,23 @@ constant functions. A faster and more flexible way to create constant functions
is to use a lambda function which can supply any constant value (not just
zero):
- >>> def constant_factory(value):
- ... return lambda: value
- >>> d = defaultdict(constant_factory('<missing>'))
- >>> d.update(name='John', action='ran')
- >>> '%(name)s %(action)s to %(object)s' % d
- 'John ran to <missing>'
+ >>> def constant_factory(value):
+ ... return lambda: value
+ >>> d = defaultdict(constant_factory('<missing>'))
+ >>> d.update(name='John', action='ran')
+ >>> '%(name)s %(action)s to %(object)s' % d
+ 'John ran to <missing>'
Setting the :attr:`default_factory` to :class:`set` makes the
:class:`defaultdict` useful for building a dictionary of sets:
- >>> s = [('red', 1), ('blue', 2), ('red', 3), ('blue', 4), ('red', 1), ('blue', 4)]
- >>> d = defaultdict(set)
- >>> for k, v in s:
- ... d[k].add(v)
- ...
- >>> list(d.items())
- [('blue', set([2, 4])), ('red', set([1, 3]))]
+ >>> s = [('red', 1), ('blue', 2), ('red', 3), ('blue', 4), ('red', 1), ('blue', 4)]
+ >>> d = defaultdict(set)
+ >>> for k, v in s:
+ ... d[k].add(v)
+ ...
+ >>> list(d.items())
+ [('blue', {2, 4}), ('red', {1, 3})]
:func:`namedtuple` Factory Function for Tuples with Named Fields
@@ -562,168 +726,131 @@ they add the ability to access fields by name instead of position index.
.. function:: namedtuple(typename, field_names, verbose=False, rename=False)
- Returns a new tuple subclass named *typename*. The new subclass is used to
- create tuple-like objects that have fields accessible by attribute lookup as
- well as being indexable and iterable. Instances of the subclass also have a
- helpful docstring (with typename and field_names) and a helpful :meth:`__repr__`
- method which lists the tuple contents in a ``name=value`` format.
+ Returns a new tuple subclass named *typename*. The new subclass is used to
+ create tuple-like objects that have fields accessible by attribute lookup as
+ well as being indexable and iterable. Instances of the subclass also have a
+ helpful docstring (with typename and field_names) and a helpful :meth:`__repr__`
+ method which lists the tuple contents in a ``name=value`` format.
- The *field_names* are a single string with each fieldname separated by whitespace
- and/or commas, for example ``'x y'`` or ``'x, y'``. Alternatively, *field_names*
- can be a sequence of strings such as ``['x', 'y']``.
+ The *field_names* are a single string with each fieldname separated by whitespace
+ and/or commas, for example ``'x y'`` or ``'x, y'``. Alternatively, *field_names*
+ can be a sequence of strings such as ``['x', 'y']``.
- Any valid Python identifier may be used for a fieldname except for names
- starting with an underscore. Valid identifiers consist of letters, digits,
- and underscores but do not start with a digit or underscore and cannot be
- a :mod:`keyword` such as *class*, *for*, *return*, *global*, *pass*,
- or *raise*.
+ Any valid Python identifier may be used for a fieldname except for names
+ starting with an underscore. Valid identifiers consist of letters, digits,
+ and underscores but do not start with a digit or underscore and cannot be
+ a :mod:`keyword` such as *class*, *for*, *return*, *global*, *pass*,
+ or *raise*.
- If *rename* is true, invalid fieldnames are automatically replaced
- with positional names. For example, ``['abc', 'def', 'ghi', 'abc']`` is
- converted to ``['abc', '_1', 'ghi', '_3']``, eliminating the keyword
- ``def`` and the duplicate fieldname ``abc``.
+ If *rename* is true, invalid fieldnames are automatically replaced
+ with positional names. For example, ``['abc', 'def', 'ghi', 'abc']`` is
+ converted to ``['abc', '_1', 'ghi', '_3']``, eliminating the keyword
+ ``def`` and the duplicate fieldname ``abc``.
- If *verbose* is true, the class definition is printed just before being built.
+ If *verbose* is true, the class definition is printed after it is
+ built. This option is outdated; instead, it is simpler to print the
+ :attr:`_source` attribute.
- Named tuple instances do not have per-instance dictionaries, so they are
- lightweight and require no more memory than regular tuples.
+ Named tuple instances do not have per-instance dictionaries, so they are
+ lightweight and require no more memory than regular tuples.
- .. versionchanged:: 3.1
- Added support for *rename*.
+ .. versionchanged:: 3.1
+ Added support for *rename*.
.. doctest::
- :options: +NORMALIZE_WHITESPACE
-
- >>> # Basic example
- >>> Point = namedtuple('Point', ['x', 'y'])
- >>> p = Point(x=10, y=11)
-
- >>> # Example using the verbose option to print the class definition
- >>> Point = namedtuple('Point', 'x y', verbose=True)
- class Point(tuple):
- 'Point(x, y)'
- <BLANKLINE>
- __slots__ = ()
- <BLANKLINE>
- _fields = ('x', 'y')
- <BLANKLINE>
- def __new__(_cls, x, y):
- 'Create a new instance of Point(x, y)'
- return _tuple.__new__(_cls, (x, y))
- <BLANKLINE>
- @classmethod
- def _make(cls, iterable, new=tuple.__new__, len=len):
- 'Make a new Point object from a sequence or iterable'
- result = new(cls, iterable)
- if len(result) != 2:
- raise TypeError('Expected 2 arguments, got %d' % len(result))
- return result
- <BLANKLINE>
- def __repr__(self):
- 'Return a nicely formatted representation string'
- return self.__class__.__name__ + '(x=%r, y=%r)' % self
- <BLANKLINE>
- def _asdict(self):
- 'Return a new OrderedDict which maps field names to their values'
- return OrderedDict(zip(self._fields, self))
- <BLANKLINE>
- __dict__ = property(_asdict)
- <BLANKLINE>
- def _replace(_self, **kwds):
- 'Return a new Point object replacing specified fields with new values'
- result = _self._make(map(kwds.pop, ('x', 'y'), _self))
- if kwds:
- raise ValueError('Got unexpected field names: %r' % list(kwds.keys()))
- return result
- <BLANKLINE>
- def __getnewargs__(self):
- 'Return self as a plain tuple. Used by copy and pickle.'
- return tuple(self)
- <BLANKLINE>
- x = _property(_itemgetter(0), doc='Alias for field number 0')
- y = _property(_itemgetter(1), doc='Alias for field number 1')
-
- >>> p = Point(11, y=22) # instantiate with positional or keyword arguments
- >>> p[0] + p[1] # indexable like the plain tuple (11, 22)
- 33
- >>> x, y = p # unpack like a regular tuple
- >>> x, y
- (11, 22)
- >>> p.x + p.y # fields also accessible by name
- 33
- >>> p # readable __repr__ with a name=value style
- Point(x=11, y=22)
+ :options: +NORMALIZE_WHITESPACE
+
+ >>> # Basic example
+ >>> Point = namedtuple('Point', ['x', 'y'])
+ >>> p = Point(11, y=22) # instantiate with positional or keyword arguments
+ >>> p[0] + p[1] # indexable like the plain tuple (11, 22)
+ 33
+ >>> x, y = p # unpack like a regular tuple
+ >>> x, y
+ (11, 22)
+ >>> p.x + p.y # fields also accessible by name
+ 33
+ >>> p # readable __repr__ with a name=value style
+ Point(x=11, y=22)
Named tuples are especially useful for assigning field names to result tuples returned
by the :mod:`csv` or :mod:`sqlite3` modules::
- EmployeeRecord = namedtuple('EmployeeRecord', 'name, age, title, department, paygrade')
+ EmployeeRecord = namedtuple('EmployeeRecord', 'name, age, title, department, paygrade')
- import csv
- for emp in map(EmployeeRecord._make, csv.reader(open("employees.csv", "rb"))):
- print(emp.name, emp.title)
+ import csv
+ for emp in map(EmployeeRecord._make, csv.reader(open("employees.csv", "rb"))):
+ print(emp.name, emp.title)
- import sqlite3
- conn = sqlite3.connect('/companydata')
- cursor = conn.cursor()
- cursor.execute('SELECT name, age, title, department, paygrade FROM employees')
- for emp in map(EmployeeRecord._make, cursor.fetchall()):
- print(emp.name, emp.title)
+ import sqlite3
+ conn = sqlite3.connect('/companydata')
+ cursor = conn.cursor()
+ cursor.execute('SELECT name, age, title, department, paygrade FROM employees')
+ for emp in map(EmployeeRecord._make, cursor.fetchall()):
+ print(emp.name, emp.title)
In addition to the methods inherited from tuples, named tuples support
-three additional methods and one attribute. To prevent conflicts with
+three additional methods and two attributes. To prevent conflicts with
field names, the method and attribute names start with an underscore.
.. classmethod:: somenamedtuple._make(iterable)
- Class method that makes a new instance from an existing sequence or iterable.
+ Class method that makes a new instance from an existing sequence or iterable.
-.. doctest::
+ .. doctest::
- >>> t = [11, 22]
- >>> Point._make(t)
- Point(x=11, y=22)
+ >>> t = [11, 22]
+ >>> Point._make(t)
+ Point(x=11, y=22)
.. method:: somenamedtuple._asdict()
- Return a new :class:`OrderedDict` which maps field names to their corresponding
- values::
+ Return a new :class:`OrderedDict` which maps field names to their corresponding
+ values. Note, this method is no longer needed now that the same effect can
+ be achieved by using the built-in :func:`vars` function::
- >>> p._asdict()
- OrderedDict([('x', 11), ('y', 22)])
+ >>> vars(p)
+ OrderedDict([('x', 11), ('y', 22)])
- .. versionchanged:: 3.1
- Returns an :class:`OrderedDict` instead of a regular :class:`dict`.
+ .. versionchanged:: 3.1
+ Returns an :class:`OrderedDict` instead of a regular :class:`dict`.
.. method:: somenamedtuple._replace(kwargs)
- Return a new instance of the named tuple replacing specified fields with new
- values:
+ Return a new instance of the named tuple replacing specified fields with new
+ values::
-::
+ >>> p = Point(x=11, y=22)
+ >>> p._replace(x=33)
+ Point(x=33, y=22)
- >>> p = Point(x=11, y=22)
- >>> p._replace(x=33)
- Point(x=33, y=22)
+ >>> for partnum, record in inventory.items():
+ ... inventory[partnum] = record._replace(price=newprices[partnum], timestamp=time.now())
- >>> for partnum, record in inventory.items():
- ... inventory[partnum] = record._replace(price=newprices[partnum], timestamp=time.now())
+.. attribute:: somenamedtuple._source
+
+ A string with the pure Python source code used to create the named
+ tuple class. The source makes the named tuple self-documenting.
+ It can be printed, executed using :func:`exec`, or saved to a file
+ and imported.
+
+ .. versionadded:: 3.3
.. attribute:: somenamedtuple._fields
- Tuple of strings listing the field names. Useful for introspection
- and for creating new named tuple types from existing named tuples.
+ Tuple of strings listing the field names. Useful for introspection
+ and for creating new named tuple types from existing named tuples.
-.. doctest::
+ .. doctest::
- >>> p._fields # view the field names
- ('x', 'y')
+ >>> p._fields # view the field names
+ ('x', 'y')
- >>> Color = namedtuple('Color', 'red green blue')
- >>> Pixel = namedtuple('Pixel', Point._fields + Color._fields)
- >>> Pixel(11, 22, 128, 255, 0)
- Pixel(x=11, y=22, red=128, green=255, blue=0)
+ >>> Color = namedtuple('Color', 'red green blue')
+ >>> Pixel = namedtuple('Pixel', Point._fields + Color._fields)
+ >>> Pixel(11, 22, 128, 255, 0)
+ Pixel(x=11, y=22, red=128, green=255, blue=0)
To retrieve a field whose name is stored in a string, use the :func:`getattr`
function:
@@ -734,31 +861,30 @@ function:
To convert a dictionary to a named tuple, use the double-star-operator
(as described in :ref:`tut-unpacking-arguments`):
- >>> d = {'x': 11, 'y': 22}
- >>> Point(**d)
- Point(x=11, y=22)
+ >>> d = {'x': 11, 'y': 22}
+ >>> Point(**d)
+ Point(x=11, y=22)
Since a named tuple is a regular Python class, it is easy to add or change
functionality with a subclass. Here is how to add a calculated field and
a fixed-width print format:
>>> class Point(namedtuple('Point', 'x y')):
- __slots__ = ()
- @property
- def hypot(self):
- return (self.x ** 2 + self.y ** 2) ** 0.5
- def __str__(self):
- return 'Point: x=%6.3f y=%6.3f hypot=%6.3f' % (self.x, self.y, self.hypot)
+ __slots__ = ()
+ @property
+ def hypot(self):
+ return (self.x ** 2 + self.y ** 2) ** 0.5
+ def __str__(self):
+ return 'Point: x=%6.3f y=%6.3f hypot=%6.3f' % (self.x, self.y, self.hypot)
>>> for p in Point(3, 4), Point(14, 5/7):
- print(p)
+ print(p)
Point: x= 3.000 y= 4.000 hypot= 5.000
Point: x=14.000 y= 0.714 hypot=14.018
The subclass shown above sets ``__slots__`` to an empty tuple. This helps
keep memory requirements low by preventing the creation of instance dictionaries.
-
Subclassing is not useful for adding new, stored fields. Instead, simply
create a new named tuple type from the :attr:`_fields` attribute:
@@ -770,6 +896,7 @@ customize a prototype instance:
>>> Account = namedtuple('Account', 'owner balance transaction_count')
>>> default_account = Account('<owner name>', 0.0, 0)
>>> johns_account = default_account._replace(owner='John')
+ >>> janes_account = default_account._replace(owner='Jane')
Enumerated constants can be implemented with named tuples, but it is simpler
and more efficient to use a simple class declaration:
@@ -778,19 +905,19 @@ and more efficient to use a simple class declaration:
>>> Status.open, Status.pending, Status.closed
(0, 1, 2)
>>> class Status:
- open, pending, closed = range(3)
+ open, pending, closed = range(3)
.. seealso::
- * `Named tuple recipe <http://code.activestate.com/recipes/500261/>`_
- adapted for Python 2.4.
+ * `Named tuple recipe <http://code.activestate.com/recipes/500261/>`_
+ adapted for Python 2.4.
- * `Recipe for named tuple abstract base class with a metaclass mix-in
- <http://code.activestate.com/recipes/577629-namedtupleabc-abstract-base-class-mix-in-for-named/>`_
- by Jan Kaliszewski. Besides providing an :term:`abstract base class` for
- named tuples, it also supports an alternate :term:`metaclass`-based
- constructor that is convenient for use cases where named tuples are being
- subclassed.
+ * `Recipe for named tuple abstract base class with a metaclass mix-in
+ <http://code.activestate.com/recipes/577629-namedtupleabc-abstract-base-class-mix-in-for-named/>`_
+ by Jan Kaliszewski. Besides providing an :term:`abstract base class` for
+ named tuples, it also supports an alternate :term:`metaclass`-based
+ constructor that is convenient for use cases where named tuples are being
+ subclassed.
:class:`OrderedDict` objects
@@ -802,36 +929,36 @@ the items are returned in the order their keys were first added.
.. class:: OrderedDict([items])
- Return an instance of a dict subclass, supporting the usual :class:`dict`
- methods. An *OrderedDict* is a dict that remembers the order that keys
- were first inserted. If a new entry overwrites an existing entry, the
- original insertion position is left unchanged. Deleting an entry and
- reinserting it will move it to the end.
+ Return an instance of a dict subclass, supporting the usual :class:`dict`
+ methods. An *OrderedDict* is a dict that remembers the order that keys
+ were first inserted. If a new entry overwrites an existing entry, the
+ original insertion position is left unchanged. Deleting an entry and
+ reinserting it will move it to the end.
- .. versionadded:: 3.1
+ .. versionadded:: 3.1
- .. method:: popitem(last=True)
+ .. method:: popitem(last=True)
- The :meth:`popitem` method for ordered dictionaries returns and removes a
- (key, value) pair. The pairs are returned in LIFO order if *last* is true
- or FIFO order if false.
+ The :meth:`popitem` method for ordered dictionaries returns and removes a
+ (key, value) pair. The pairs are returned in LIFO order if *last* is true
+ or FIFO order if false.
- .. method:: move_to_end(key, last=True)
+ .. method:: move_to_end(key, last=True)
- Move an existing *key* to either end of an ordered dictionary. The item
- is moved to the right end if *last* is true (the default) or to the
- beginning if *last* is false. Raises :exc:`KeyError` if the *key* does
- not exist::
+ Move an existing *key* to either end of an ordered dictionary. The item
+ is moved to the right end if *last* is true (the default) or to the
+ beginning if *last* is false. Raises :exc:`KeyError` if the *key* does
+ not exist::
- >>> d = OrderedDict.fromkeys('abcde')
- >>> d.move_to_end('b')
- >>> ''.join(d.keys())
- 'acdeb'
- >>> d.move_to_end('b', last=False)
- >>> ''.join(d.keys())
- 'bacde'
+ >>> d = OrderedDict.fromkeys('abcde')
+ >>> d.move_to_end('b')
+ >>> ''.join(d.keys())
+ 'acdeb'
+ >>> d.move_to_end('b', last=False)
+ >>> ''.join(d.keys())
+ 'bacde'
- .. versionadded:: 3.2
+ .. versionadded:: 3.2
In addition to the usual mapping methods, ordered dictionaries also support
reverse iteration using :func:`reversed`.
@@ -849,8 +976,8 @@ semantics pass-in keyword arguments using a regular unordered dictionary.
.. seealso::
- `Equivalent OrderedDict recipe <http://code.activestate.com/recipes/576693/>`_
- that runs on Python 2.4 or later.
+ `Equivalent OrderedDict recipe <http://code.activestate.com/recipes/576693/>`_
+ that runs on Python 2.4 or later.
:class:`OrderedDict` Examples and Recipes
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -893,7 +1020,7 @@ original insertion position is changed and moved to the end::
An ordered dictionary can be combined with the :class:`Counter` class
so that the counter remembers the order elements are first encountered::
- class OrderedCounter(Counter, OrderedDict):
+ class OrderedCounter(Counter, OrderedDict):
'Counter that remembers the order elements are first encountered'
def __repr__(self):
@@ -914,19 +1041,19 @@ attribute.
.. class:: UserDict([initialdata])
- Class that simulates a dictionary. The instance's contents are kept in a
- regular dictionary, which is accessible via the :attr:`data` attribute of
- :class:`UserDict` instances. If *initialdata* is provided, :attr:`data` is
- initialized with its contents; note that a reference to *initialdata* will not
- be kept, allowing it be used for other purposes.
+ Class that simulates a dictionary. The instance's contents are kept in a
+ regular dictionary, which is accessible via the :attr:`data` attribute of
+ :class:`UserDict` instances. If *initialdata* is provided, :attr:`data` is
+ initialized with its contents; note that a reference to *initialdata* will not
+ be kept, allowing it be used for other purposes.
- In addition to supporting the methods and operations of mappings,
- :class:`UserDict` instances provide the following attribute:
+ In addition to supporting the methods and operations of mappings,
+ :class:`UserDict` instances provide the following attribute:
- .. attribute:: data
+ .. attribute:: data
- A real dictionary used to store the contents of the :class:`UserDict`
- class.
+ A real dictionary used to store the contents of the :class:`UserDict`
+ class.
@@ -944,19 +1071,19 @@ to work with because the underlying list is accessible as an attribute.
.. class:: UserList([list])
- Class that simulates a list. The instance's contents are kept in a regular
- list, which is accessible via the :attr:`data` attribute of :class:`UserList`
- instances. The instance's contents are initially set to a copy of *list*,
- defaulting to the empty list ``[]``. *list* can be any iterable, for
- example a real Python list or a :class:`UserList` object.
+ Class that simulates a list. The instance's contents are kept in a regular
+ list, which is accessible via the :attr:`data` attribute of :class:`UserList`
+ instances. The instance's contents are initially set to a copy of *list*,
+ defaulting to the empty list ``[]``. *list* can be any iterable, for
+ example a real Python list or a :class:`UserList` object.
- In addition to supporting the methods and operations of mutable sequences,
- :class:`UserList` instances provide the following attribute:
+ In addition to supporting the methods and operations of mutable sequences,
+ :class:`UserList` instances provide the following attribute:
- .. attribute:: data
+ .. attribute:: data
- A real :class:`list` object used to store the contents of the
- :class:`UserList` class.
+ A real :class:`list` object used to store the contents of the
+ :class:`UserList` class.
**Subclassing requirements:** Subclasses of :class:`UserList` are expect to
offer a constructor which can be called with either no arguments or one
@@ -981,168 +1108,10 @@ attribute.
.. class:: UserString([sequence])
- Class that simulates a string or a Unicode string object. The instance's
- content is kept in a regular string object, which is accessible via the
- :attr:`data` attribute of :class:`UserString` instances. The instance's
- contents are initially set to a copy of *sequence*. The *sequence* can
- be an instance of :class:`bytes`, :class:`str`, :class:`UserString` (or a
- subclass) or an arbitrary sequence which can be converted into a string using
- the built-in :func:`str` function.
-
-
-.. _collections-abstract-base-classes:
-
-ABCs - abstract base classes
-----------------------------
-
-The collections module offers the following :term:`ABCs <abstract base class>`:
-
-========================= ===================== ====================== ====================================================
-ABC Inherits from Abstract Methods Mixin Methods
-========================= ===================== ====================== ====================================================
-:class:`Container` ``__contains__``
-:class:`Hashable` ``__hash__``
-:class:`Iterable` ``__iter__``
-:class:`Iterator` :class:`Iterable` ``__next__`` ``__iter__``
-:class:`Sized` ``__len__``
-:class:`Callable` ``__call__``
-
-:class:`Sequence` :class:`Sized`, ``__getitem__`` ``__contains__``, ``__iter__``, ``__reversed__``,
- :class:`Iterable`, ``index``, and ``count``
- :class:`Container`
-
-:class:`MutableSequence` :class:`Sequence` ``__setitem__``, Inherited :class:`Sequence` methods and
- ``__delitem__``, ``append``, ``reverse``, ``extend``, ``pop``,
- ``insert`` ``remove``, and ``__iadd__``
-
-:class:`Set` :class:`Sized`, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``,
- :class:`Iterable`, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``,
- :class:`Container` ``__sub__``, ``__xor__``, and ``isdisjoint``
-
-:class:`MutableSet` :class:`Set` ``add``, Inherited :class:`Set` methods and
- ``discard`` ``clear``, ``pop``, ``remove``, ``__ior__``,
- ``__iand__``, ``__ixor__``, and ``__isub__``
-
-:class:`Mapping` :class:`Sized`, ``__getitem__`` ``__contains__``, ``keys``, ``items``, ``values``,
- :class:`Iterable`, ``get``, ``__eq__``, and ``__ne__``
- :class:`Container`
-
-:class:`MutableMapping` :class:`Mapping` ``__setitem__``, Inherited :class:`Mapping` methods and
- ``__delitem__`` ``pop``, ``popitem``, ``clear``, ``update``,
- and ``setdefault``
-
-
-:class:`MappingView` :class:`Sized` ``__len__``
-:class:`ItemsView` :class:`MappingView`, ``__contains__``,
- :class:`Set` ``__iter__``
-:class:`KeysView` :class:`MappingView`, ``__contains__``,
- :class:`Set` ``__iter__``
-:class:`ValuesView` :class:`MappingView` ``__contains__``, ``__iter__``
-========================= ===================== ====================== ====================================================
-
-
-.. class:: Container
- Hashable
- Sized
- Callable
-
- ABCs for classes that provide respectively the methods :meth:`__contains__`,
- :meth:`__hash__`, :meth:`__len__`, and :meth:`__call__`.
-
-.. class:: Iterable
-
- ABC for classes that provide the :meth:`__iter__` method.
- See also the definition of :term:`iterable`.
-
-.. class:: Iterator
-
- ABC for classes that provide the :meth:`__iter__` and :meth:`next` methods.
- See also the definition of :term:`iterator`.
-
-.. class:: Sequence
- MutableSequence
-
- ABCs for read-only and mutable :term:`sequences <sequence>`.
-
-.. class:: Set
- MutableSet
-
- ABCs for read-only and mutable sets.
-
-.. class:: Mapping
- MutableMapping
-
- ABCs for read-only and mutable :term:`mappings <mapping>`.
-
-.. class:: MappingView
- ItemsView
- KeysView
- ValuesView
-
- ABCs for mapping, items, keys, and values :term:`views <view>`.
-
-
-These ABCs allow us to ask classes or instances if they provide
-particular functionality, for example::
-
- size = None
- if isinstance(myvar, collections.Sized):
- size = len(myvar)
-
-Several of the ABCs are also useful as mixins that make it easier to develop
-classes supporting container APIs. For example, to write a class supporting
-the full :class:`Set` API, it only necessary to supply the three underlying
-abstract methods: :meth:`__contains__`, :meth:`__iter__`, and :meth:`__len__`.
-The ABC supplies the remaining methods such as :meth:`__and__` and
-:meth:`isdisjoint` ::
-
- class ListBasedSet(collections.Set):
- ''' Alternate set implementation favoring space over speed
- and not requiring the set elements to be hashable. '''
- def __init__(self, iterable):
- self.elements = lst = []
- for value in iterable:
- if value not in lst:
- lst.append(value)
- def __iter__(self):
- return iter(self.elements)
- def __contains__(self, value):
- return value in self.elements
- def __len__(self):
- return len(self.elements)
-
- s1 = ListBasedSet('abcdef')
- s2 = ListBasedSet('defghi')
- overlap = s1 & s2 # The __and__() method is supported automatically
-
-Notes on using :class:`Set` and :class:`MutableSet` as a mixin:
-
-(1)
- Since some set operations create new sets, the default mixin methods need
- a way to create new instances from an iterable. The class constructor is
- assumed to have a signature in the form ``ClassName(iterable)``.
- That assumption is factored-out to an internal classmethod called
- :meth:`_from_iterable` which calls ``cls(iterable)`` to produce a new set.
- If the :class:`Set` mixin is being used in a class with a different
- constructor signature, you will need to override :meth:`_from_iterable`
- with a classmethod that can construct new instances from
- an iterable argument.
-
-(2)
- To override the comparisons (presumably for speed, as the
- semantics are fixed), redefine :meth:`__le__` and
- then the other operations will automatically follow suit.
-
-(3)
- The :class:`Set` mixin provides a :meth:`_hash` method to compute a hash value
- for the set; however, :meth:`__hash__` is not defined because not all sets
- are hashable or immutable. To add set hashabilty using mixins,
- inherit from both :meth:`Set` and :meth:`Hashable`, then define
- ``__hash__ = Set._hash``.
-
-.. seealso::
-
- * `OrderedSet recipe <http://code.activestate.com/recipes/576694/>`_ for an
- example built on :class:`MutableSet`.
-
- * For more about ABCs, see the :mod:`abc` module and :pep:`3119`.
+ Class that simulates a string or a Unicode string object. The instance's
+ content is kept in a regular string object, which is accessible via the
+ :attr:`data` attribute of :class:`UserString` instances. The instance's
+ contents are initially set to a copy of *sequence*. The *sequence* can
+ be an instance of :class:`bytes`, :class:`str`, :class:`UserString` (or a
+ subclass) or an arbitrary sequence which can be converted into a string using
+ the built-in :func:`str` function.
diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst
index 29ffc0d..a88f10c 100644
--- a/Doc/library/concurrent.futures.rst
+++ b/Doc/library/concurrent.futures.rst
@@ -169,6 +169,12 @@ to a :class:`ProcessPoolExecutor` will result in deadlock.
of at most *max_workers* processes. If *max_workers* is ``None`` or not
given, it will default to the number of processors on the machine.
+ .. versionchanged:: 3.3
+ When one of the worker processes terminates abruptly, a
+ :exc:`BrokenProcessPool` error is now raised. Previously, behaviour
+ was undefined but operations on the executor or its futures would often
+ freeze or deadlock.
+
.. _processpoolexecutor-example:
@@ -369,3 +375,16 @@ Module Functions
:pep:`3148` -- futures - execute computations asynchronously
The proposal which described this feature for inclusion in the Python
standard library.
+
+
+Exception classes
+-----------------
+
+.. exception:: BrokenProcessPool
+
+ Derived from :exc:`RuntimeError`, this exception class is raised when
+ one of the workers of a :class:`ProcessPoolExecutor` has terminated
+ in a non-clean fashion (for example, if it was killed from the outside).
+
+ .. versionadded:: 3.3
+
diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst
index e8dc17f..ed9ebb8 100644
--- a/Doc/library/contextlib.rst
+++ b/Doc/library/contextlib.rst
@@ -12,8 +12,11 @@ This module provides utilities for common tasks involving the :keyword:`with`
statement. For more information see also :ref:`typecontextmanager` and
:ref:`context-managers`.
-Functions provided:
+Utilities
+---------
+
+Functions and classes provided:
.. decorator:: contextmanager
@@ -168,6 +171,348 @@ Functions provided:
.. versionadded:: 3.2
+.. class:: ExitStack()
+
+ A context manager that is designed to make it easy to programmatically
+ combine other context managers and cleanup functions, especially those
+ that are optional or otherwise driven by input data.
+
+ For example, a set of files may easily be handled in a single with
+ statement as follows::
+
+ with ExitStack() as stack:
+ files = [stack.enter_context(open(fname)) for fname in filenames]
+ # All opened files will automatically be closed at the end of
+ # the with statement, even if attempts to open files later
+ # in the list throw an exception
+
+ Each instance maintains a stack of registered callbacks that are called in
+ reverse order when the instance is closed (either explicitly or implicitly
+ at the end of a :keyword:`with` statement). Note that callbacks are *not*
+ invoked implicitly when the context stack instance is garbage collected.
+
+ This stack model is used so that context managers that acquire their
+ resources in their ``__init__`` method (such as file objects) can be
+ handled correctly.
+
+ Since registered callbacks are invoked in the reverse order of
+ registration, this ends up behaving as if multiple nested :keyword:`with`
+ statements had been used with the registered set of callbacks. This even
+ extends to exception handling - if an inner callback suppresses or replaces
+ an exception, then outer callbacks will be passed arguments based on that
+ updated state.
+
+ This is a relatively low level API that takes care of the details of
+ correctly unwinding the stack of exit callbacks. It provides a suitable
+ foundation for higher level context managers that manipulate the exit
+ stack in application specific ways.
+
+ .. versionadded:: 3.3
+
+ .. method:: enter_context(cm)
+
+ Enters a new context manager and adds its :meth:`__exit__` method to
+ the callback stack. The return value is the result of the context
+ manager's own :meth:`__enter__` method.
+
+ These context managers may suppress exceptions just as they normally
+ would if used directly as part of a :keyword:`with` statement.
+
+ .. method:: push(exit)
+
+ Adds a context manager's :meth:`__exit__` method to the callback stack.
+
+ As ``__enter__`` is *not* invoked, this method can be used to cover
+ part of an :meth:`__enter__` implementation with a context manager's own
+ :meth:`__exit__` method.
+
+ If passed an object that is not a context manager, this method assumes
+ it is a callback with the same signature as a context manager's
+ :meth:`__exit__` method and adds it directly to the callback stack.
+
+ By returning true values, these callbacks can suppress exceptions the
+ same way context manager :meth:`__exit__` methods can.
+
+ The passed in object is returned from the function, allowing this
+ method to be used as a function decorator.
+
+ .. method:: callback(callback, *args, **kwds)
+
+ Accepts an arbitrary callback function and arguments and adds it to
+ the callback stack.
+
+ Unlike the other methods, callbacks added this way cannot suppress
+ exceptions (as they are never passed the exception details).
+
+ The passed in callback is returned from the function, allowing this
+ method to be used as a function decorator.
+
+ .. method:: pop_all()
+
+ Transfers the callback stack to a fresh :class:`ExitStack` instance
+ and returns it. No callbacks are invoked by this operation - instead,
+ they will now be invoked when the new stack is closed (either
+ explicitly or implicitly at the end of a :keyword:`with` statement).
+
+ For example, a group of files can be opened as an "all or nothing"
+ operation as follows::
+
+ with ExitStack() as stack:
+ files = [stack.enter_context(open(fname)) for fname in filenames]
+ close_files = stack.pop_all().close
+ # If opening any file fails, all previously opened files will be
+ # closed automatically. If all files are opened successfully,
+ # they will remain open even after the with statement ends.
+ # close_files() can then be invoked explicitly to close them all
+
+ .. method:: close()
+
+ Immediately unwinds the callback stack, invoking callbacks in the
+ reverse order of registration. For any context managers and exit
+ callbacks registered, the arguments passed in will indicate that no
+ exception occurred.
+
+
+Examples and Recipes
+--------------------
+
+This section describes some examples and recipes for making effective use of
+the tools provided by :mod:`contextlib`.
+
+
+Supporting a variable number of context managers
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The primary use case for :class:`ExitStack` is the one given in the class
+documentation: supporting a variable number of context managers and other
+cleanup operations in a single :keyword:`with` statement. The variability
+may come from the number of context managers needed being driven by user
+input (such as opening a user specified collection of files), or from
+some of the context managers being optional::
+
+ with ExitStack() as stack:
+ for resource in resources:
+ stack.enter_context(resource)
+ if need_special resource:
+ special = acquire_special_resource()
+ stack.callback(release_special_resource, special)
+ # Perform operations that use the acquired resources
+
+As shown, :class:`ExitStack` also makes it quite easy to use :keyword:`with`
+statements to manage arbitrary resources that don't natively support the
+context management protocol.
+
+
+Simplifying support for single optional context managers
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In the specific case of a single optional context manager, :class:`ExitStack`
+instances can be used as a "do nothing" context manager, allowing a context
+manager to easily be omitted without affecting the overall structure of
+the source code::
+
+ def debug_trace(details):
+ if __debug__:
+ return TraceContext(details)
+ # Don't do anything special with the context in release mode
+ return ExitStack()
+
+ with debug_trace():
+ # Suite is traced in debug mode, but runs normally otherwise
+
+
+Catching exceptions from ``__enter__`` methods
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+It is occasionally desirable to catch exceptions from an ``__enter__``
+method implementation, *without* inadvertently catching exceptions from
+the :keyword:`with` statement body or the context manager's ``__exit__``
+method. By using :class:`ExitStack` the steps in the context management
+protocol can be separated slightly in order to allow this::
+
+ stack = ExitStack()
+ try:
+ x = stack.enter_context(cm)
+ except Exception:
+ # handle __enter__ exception
+ else:
+ with stack:
+ # Handle normal case
+
+Actually needing to do this is likely to indicate that the underlying API
+should be providing a direct resource management interface for use with
+:keyword:`try`/:keyword:`except`/:keyword:`finally` statements, but not
+all APIs are well designed in that regard. When a context manager is the
+only resource management API provided, then :class:`ExitStack` can make it
+easier to handle various situations that can't be handled directly in a
+:keyword:`with` statement.
+
+
+Cleaning up in an ``__enter__`` implementation
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+As noted in the documentation of :meth:`ExitStack.push`, this
+method can be useful in cleaning up an already allocated resource if later
+steps in the :meth:`__enter__` implementation fail.
+
+Here's an example of doing this for a context manager that accepts resource
+acquisition and release functions, along with an optional validation function,
+and maps them to the context management protocol::
+
+ from contextlib import contextmanager, ExitStack
+
+ class ResourceManager(object):
+
+ def __init__(self, acquire_resource, release_resource, check_resource_ok=None):
+ self.acquire_resource = acquire_resource
+ self.release_resource = release_resource
+ if check_resource_ok is None:
+ def check_resource_ok(resource):
+ return True
+ self.check_resource_ok = check_resource_ok
+
+ @contextmanager
+ def _cleanup_on_error(self):
+ with ExitStack() as stack:
+ stack.push(self)
+ yield
+ # The validation check passed and didn't raise an exception
+ # Accordingly, we want to keep the resource, and pass it
+ # back to our caller
+ stack.pop_all()
+
+ def __enter__(self):
+ resource = self.acquire_resource()
+ with self._cleanup_on_error():
+ if not self.check_resource_ok(resource):
+ msg = "Failed validation for {!r}"
+ raise RuntimeError(msg.format(resource))
+ return resource
+
+ def __exit__(self, *exc_details):
+ # We don't need to duplicate any of our resource release logic
+ self.release_resource()
+
+
+Replacing any use of ``try-finally`` and flag variables
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A pattern you will sometimes see is a ``try-finally`` statement with a flag
+variable to indicate whether or not the body of the ``finally`` clause should
+be executed. In its simplest form (that can't already be handled just by
+using an ``except`` clause instead), it looks something like this::
+
+ cleanup_needed = True
+ try:
+ result = perform_operation()
+ if result:
+ cleanup_needed = False
+ finally:
+ if cleanup_needed:
+ cleanup_resources()
+
+As with any ``try`` statement based code, this can cause problems for
+development and review, because the setup code and the cleanup code can end
+up being separated by arbitrarily long sections of code.
+
+:class:`ExitStack` makes it possible to instead register a callback for
+execution at the end of a ``with`` statement, and then later decide to skip
+executing that callback::
+
+ from contextlib import ExitStack
+
+ with ExitStack() as stack:
+ stack.callback(cleanup_resources)
+ result = perform_operation()
+ if result:
+ stack.pop_all()
+
+This allows the intended cleanup up behaviour to be made explicit up front,
+rather than requiring a separate flag variable.
+
+If a particular application uses this pattern a lot, it can be simplified
+even further by means of a small helper class::
+
+ from contextlib import ExitStack
+
+ class Callback(ExitStack):
+ def __init__(self, callback, *args, **kwds):
+ super(Callback, self).__init__()
+ self.callback(callback, *args, **kwds)
+
+ def cancel(self):
+ self.pop_all()
+
+ with Callback(cleanup_resources) as cb:
+ result = perform_operation()
+ if result:
+ cb.cancel()
+
+If the resource cleanup isn't already neatly bundled into a standalone
+function, then it is still possible to use the decorator form of
+:meth:`ExitStack.callback` to declare the resource cleanup in
+advance::
+
+ from contextlib import ExitStack
+
+ with ExitStack() as stack:
+ @stack.callback
+ def cleanup_resources():
+ ...
+ result = perform_operation()
+ if result:
+ stack.pop_all()
+
+Due to the way the decorator protocol works, a callback function
+declared this way cannot take any parameters. Instead, any resources to
+be released must be accessed as closure variables
+
+
+Using a context manager as a function decorator
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+:class:`ContextDecorator` makes it possible to use a context manager in
+both an ordinary ``with`` statement and also as a function decorator.
+
+For example, it is sometimes useful to wrap functions or groups of statements
+with a logger that can track the time of entry and time of exit. Rather than
+writing both a function decorator and a context manager for the task,
+inheriting from :class:`ContextDecorator` provides both capabilities in a
+single definition::
+
+ from contextlib import ContextDecorator
+ import logging
+
+ logging.basicConfig(level=logging.INFO)
+
+ class track_entry_and_exit(ContextDecorator):
+ def __init__(self, name):
+ self.name = name
+
+ def __enter__(self):
+ logging.info('Entering: {}'.format(name))
+
+ def __exit__(self, exc_type, exc, exc_tb):
+ logging.info('Exiting: {}'.format(name))
+
+Instances of this class can be used as both a context manager::
+
+ with track_entry_and_exit('widget loader'):
+ print('Some time consuming activity goes here')
+ load_widget()
+
+And also as a function decorator::
+
+ @track_entry_and_exit('widget loader')
+ def activity():
+ print('Some time consuming activity goes here')
+ load_widget()
+
+Note that there is one additional limitation when using context managers
+as function decorators: there's no way to access the return value of
+:meth:`__enter__`. If that value is needed, then it is still necessary to use
+an explicit ``with`` statement.
+
.. seealso::
:pep:`0343` - The "with" statement
diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst
index a2d316e..41061e5 100644
--- a/Doc/library/copyreg.rst
+++ b/Doc/library/copyreg.rst
@@ -32,6 +32,8 @@ Such constructors may be factory functions or class instances.
returned by *function* at pickling time. :exc:`TypeError` will be raised if
*object* is a class or *constructor* is not callable.
- See the :mod:`pickle` module for more details on the interface expected of
- *function* and *constructor*.
-
+ See the :mod:`pickle` module for more details on the interface
+ expected of *function* and *constructor*. Note that the
+ :attr:`~pickle.Pickler.dispatch_table` attribute of a pickler
+ object or subclass of :class:`pickle.Pickler` can also be used for
+ declaring reduction functions.
diff --git a/Doc/library/crypt.rst b/Doc/library/crypt.rst
index 0be571e..1ba2ed3 100644
--- a/Doc/library/crypt.rst
+++ b/Doc/library/crypt.rst
@@ -15,9 +15,9 @@
This module implements an interface to the :manpage:`crypt(3)` routine, which is
a one-way hash function based upon a modified DES algorithm; see the Unix man
-page for further details. Possible uses include allowing Python scripts to
-accept typed passwords from the user, or attempting to crack Unix passwords with
-a dictionary.
+page for further details. Possible uses include storing hashed passwords
+so you can check passwords without storing the actual password, or attempting
+to crack Unix passwords with a dictionary.
.. index:: single: crypt(3)
@@ -26,15 +26,74 @@ the :manpage:`crypt(3)` routine in the running system. Therefore, any
extensions available on the current implementation will also be available on
this module.
+Hashing Methods
+---------------
-.. function:: crypt(word, salt)
+.. versionadded:: 3.3
+
+The :mod:`crypt` module defines the list of hashing methods (not all methods
+are available on all platforms):
+
+.. data:: METHOD_SHA512
+
+ A Modular Crypt Format method with 16 character salt and 86 character
+ hash. This is the strongest method.
+
+.. data:: METHOD_SHA256
+
+ Another Modular Crypt Format method with 16 character salt and 43
+ character hash.
+
+.. data:: METHOD_MD5
+
+ Another Modular Crypt Format method with 8 character salt and 22
+ character hash.
+
+.. data:: METHOD_CRYPT
+
+ The traditional method with a 2 character salt and 13 characters of
+ hash. This is the weakest method.
+
+
+Module Attributes
+-----------------
+
+.. versionadded:: 3.3
+
+.. attribute:: methods
+
+ A list of available password hashing algorithms, as
+ ``crypt.METHOD_*`` objects. This list is sorted from strongest to
+ weakest, and is guaranteed to have at least ``crypt.METHOD_CRYPT``.
+
+
+Module Functions
+----------------
+
+The :mod:`crypt` module defines the following functions:
+
+.. function:: crypt(word, salt=None)
*word* will usually be a user's password as typed at a prompt or in a graphical
- interface. *salt* is usually a random two-character string which will be used
- to perturb the DES algorithm in one of 4096 ways. The characters in *salt* must
- be in the set ``[./a-zA-Z0-9]``. Returns the hashed password as a string, which
- will be composed of characters from the same alphabet as the salt (the first two
- characters represent the salt itself).
+ interface. The optional *salt* is either a string as returned from
+ :func:`mksalt`, one of the ``crypt.METHOD_*`` values (though not all
+ may be available on all platforms), or a full encrypted password
+ including salt, as returned by this function. If *salt* is not
+ provided, the strongest method will be used (as returned by
+ :func:`methods`.
+
+ Checking a password is usually done by passing the plain-text password
+ as *word* and the full results of a previous :func:`crypt` call,
+ which should be the same as the results of this call.
+
+ *salt* (either a random 2 or 16 character string, possibly prefixed with
+ ``$digit$`` to indicate the method) which will be used to perturb the
+ encryption algorithm. The characters in *salt* must be in the set
+ ``[./a-zA-Z0-9]``, with the exception of Modular Crypt Format which
+ prefixes a ``$digit$``.
+
+ Returns the hashed password as a string, which will be composed of
+ characters from the same alphabet as the salt.
.. index:: single: crypt(3)
@@ -42,18 +101,48 @@ this module.
different sizes in the *salt*, it is recommended to use the full crypted
password as salt when checking for a password.
+ .. versionchanged:: 3.3
+ Accept ``crypt.METHOD_*`` values in addition to strings for *salt*.
+
+
+.. function:: mksalt(method=None)
+
+ Return a randomly generated salt of the specified method. If no
+ *method* is given, the strongest method available as returned by
+ :func:`methods` is used.
+
+ The return value is a string either of 2 characters in length for
+ ``crypt.METHOD_CRYPT``, or 19 characters starting with ``$digit$`` and
+ 16 random characters from the set ``[./a-zA-Z0-9]``, suitable for
+ passing as the *salt* argument to :func:`crypt`.
+
+ .. versionadded:: 3.3
+
+Examples
+--------
+
A simple example illustrating typical use::
- import crypt, getpass, pwd
+ import pwd
+ import crypt
+ import getpass
def login():
- username = input('Python login:')
+ username = input('Python login: ')
cryptedpasswd = pwd.getpwnam(username)[1]
if cryptedpasswd:
if cryptedpasswd == 'x' or cryptedpasswd == '*':
- raise "Sorry, currently no support for shadow passwords"
+ raise ValueError('no support for shadow passwords')
cleartext = getpass.getpass()
return crypt.crypt(cleartext, cryptedpasswd) == cryptedpasswd
else:
- return 1
+ return True
+
+To generate a hash of a password using the strongest available method and
+check it against the original::
+
+ import crypt
+ hashed = crypt.crypt(plaintext)
+ if hashed != crypt.crypt(plaintext, hashed):
+ raise ValueError("hashed version doesn't validate against original")
diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst
index edbe726..ec0dfcc 100644
--- a/Doc/library/csv.rst
+++ b/Doc/library/csv.rst
@@ -11,15 +11,15 @@
pair: data; tabular
The so-called CSV (Comma Separated Values) format is the most common import and
-export format for spreadsheets and databases. There is no "CSV standard", so
-the format is operationally defined by the many applications which read and
-write it. The lack of a standard means that subtle differences often exist in
-the data produced and consumed by different applications. These differences can
-make it annoying to process CSV files from multiple sources. Still, while the
-delimiters and quoting characters vary, the overall format is similar enough
-that it is possible to write a single module which can efficiently manipulate
-such data, hiding the details of reading and writing the data from the
-programmer.
+export format for spreadsheets and databases. CSV format was used for many
+years prior to attempts to describe the format in a standardized way in
+:rfc:`4180`. The lack of a well-defined standard means that subtle differences
+often exist in the data produced and consumed by different applications. These
+differences can make it annoying to process CSV files from multiple sources.
+Still, while the delimiters and quoting characters vary, the overall format is
+similar enough that it is possible to write a single module which can
+efficiently manipulate such data, hiding the details of reading and writing the
+data from the programmer.
The :mod:`csv` module implements classes to read and write tabular data in CSV
format. It allows programmers to say, "write this data in the format preferred
diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst
index df39c28..8a2b196 100644
--- a/Doc/library/ctypes.rst
+++ b/Doc/library/ctypes.rst
@@ -39,9 +39,14 @@ loads libraries which export functions using the standard ``cdecl`` calling
convention, while *windll* libraries call functions using the ``stdcall``
calling convention. *oledll* also uses the ``stdcall`` calling convention, and
assumes the functions return a Windows :c:type:`HRESULT` error code. The error
-code is used to automatically raise a :class:`WindowsError` exception when the
+code is used to automatically raise a :class:`OSError` exception when the
function call fails.
+.. versionchanged:: 3.3
+ Windows errors used to raise :exc:`WindowsError`, which is now an alias
+ of :exc:`OSError`.
+
+
Here are some examples for Windows. Note that ``msvcrt`` is the MS standard C
library containing most standard C functions, and uses the cdecl calling
convention::
@@ -189,7 +194,7 @@ argument values::
>>> windll.kernel32.GetModuleHandleA(32) # doctest: +WINDOWS
Traceback (most recent call last):
File "<stdin>", line 1, in ?
- WindowsError: exception: access violation reading 0x00000020
+ OSError: exception: access violation reading 0x00000020
>>>
There are, however, enough ways to crash Python with :mod:`ctypes`, so you
@@ -491,7 +496,7 @@ useful to check for error return values and automatically raise an exception::
Traceback (most recent call last):
File "<stdin>", line 1, in ?
File "<stdin>", line 3, in ValidHandle
- WindowsError: [Errno 126] The specified module could not be found.
+ OSError: [Errno 126] The specified module could not be found.
>>>
``WinError`` is a function which will call Windows ``FormatMessage()`` api to
@@ -921,21 +926,21 @@ Callback functions
:mod:`ctypes` allows to create C callable function pointers from Python callables.
These are sometimes called *callback functions*.
-First, you must create a class for the callback function, the class knows the
+First, you must create a class for the callback function. The class knows the
calling convention, the return type, and the number and types of arguments this
function will receive.
-The CFUNCTYPE factory function creates types for callback functions using the
-normal cdecl calling convention, and, on Windows, the WINFUNCTYPE factory
-function creates types for callback functions using the stdcall calling
-convention.
+The :func:`CFUNCTYPE` factory function creates types for callback functions
+using the ``cdecl`` calling convention. On Windows, the :func:`WINFUNCTYPE`
+factory function creates types for callback functions using the ``stdcall``
+calling convention.
Both of these factory functions are called with the result type as first
argument, and the callback functions expected argument types as the remaining
arguments.
I will present an example here which uses the standard C library's
-:c:func:`qsort` function, this is used to sort items with the help of a callback
+:c:func:`qsort` function, that is used to sort items with the help of a callback
function. :c:func:`qsort` will be used to sort an array of integers::
>>> IntArray5 = c_int * 5
@@ -948,7 +953,7 @@ function. :c:func:`qsort` will be used to sort an array of integers::
items in the data array, the size of one item, and a pointer to the comparison
function, the callback. The callback will then be called with two pointers to
items, and it must return a negative integer if the first item is smaller than
-the second, a zero if they are equal, and a positive integer else.
+the second, a zero if they are equal, and a positive integer otherwise.
So our callback function receives pointers to integers, and must return an
integer. First we create the ``type`` for the callback function::
@@ -956,36 +961,8 @@ integer. First we create the ``type`` for the callback function::
>>> CMPFUNC = CFUNCTYPE(c_int, POINTER(c_int), POINTER(c_int))
>>>
-For the first implementation of the callback function, we simply print the
-arguments we get, and return 0 (incremental development ;-)::
-
- >>> def py_cmp_func(a, b):
- ... print("py_cmp_func", a, b)
- ... return 0
- ...
- >>>
-
-Create the C callable callback::
-
- >>> cmp_func = CMPFUNC(py_cmp_func)
- >>>
-
-And we're ready to go::
-
- >>> qsort(ia, len(ia), sizeof(c_int), cmp_func) # doctest: +WINDOWS
- py_cmp_func <ctypes.LP_c_long object at 0x00...> <ctypes.LP_c_long object at 0x00...>
- py_cmp_func <ctypes.LP_c_long object at 0x00...> <ctypes.LP_c_long object at 0x00...>
- py_cmp_func <ctypes.LP_c_long object at 0x00...> <ctypes.LP_c_long object at 0x00...>
- py_cmp_func <ctypes.LP_c_long object at 0x00...> <ctypes.LP_c_long object at 0x00...>
- py_cmp_func <ctypes.LP_c_long object at 0x00...> <ctypes.LP_c_long object at 0x00...>
- py_cmp_func <ctypes.LP_c_long object at 0x00...> <ctypes.LP_c_long object at 0x00...>
- py_cmp_func <ctypes.LP_c_long object at 0x00...> <ctypes.LP_c_long object at 0x00...>
- py_cmp_func <ctypes.LP_c_long object at 0x00...> <ctypes.LP_c_long object at 0x00...>
- py_cmp_func <ctypes.LP_c_long object at 0x00...> <ctypes.LP_c_long object at 0x00...>
- py_cmp_func <ctypes.LP_c_long object at 0x00...> <ctypes.LP_c_long object at 0x00...>
- >>>
-
-We know how to access the contents of a pointer, so lets redefine our callback::
+To get started, here is a simple callback that shows the values it gets
+passed::
>>> def py_cmp_func(a, b):
... print("py_cmp_func", a[0], b[0])
@@ -994,23 +971,7 @@ We know how to access the contents of a pointer, so lets redefine our callback::
>>> cmp_func = CMPFUNC(py_cmp_func)
>>>
-Here is what we get on Windows::
-
- >>> qsort(ia, len(ia), sizeof(c_int), cmp_func) # doctest: +WINDOWS
- py_cmp_func 7 1
- py_cmp_func 33 1
- py_cmp_func 99 1
- py_cmp_func 5 1
- py_cmp_func 7 5
- py_cmp_func 33 5
- py_cmp_func 99 5
- py_cmp_func 7 99
- py_cmp_func 33 99
- py_cmp_func 7 33
- >>>
-
-It is funny to see that on linux the sort function seems to work much more
-efficiently, it is doing less comparisons::
+The result::
>>> qsort(ia, len(ia), sizeof(c_int), cmp_func) # doctest: +LINUX
py_cmp_func 5 1
@@ -1020,32 +981,13 @@ efficiently, it is doing less comparisons::
py_cmp_func 1 7
>>>
-Ah, we're nearly done! The last step is to actually compare the two items and
-return a useful result::
+Now we can actually compare the two items and return a useful result::
>>> def py_cmp_func(a, b):
... print("py_cmp_func", a[0], b[0])
... return a[0] - b[0]
...
>>>
-
-Final run on Windows::
-
- >>> qsort(ia, len(ia), sizeof(c_int), CMPFUNC(py_cmp_func)) # doctest: +WINDOWS
- py_cmp_func 33 7
- py_cmp_func 99 33
- py_cmp_func 5 99
- py_cmp_func 1 99
- py_cmp_func 33 7
- py_cmp_func 1 33
- py_cmp_func 5 33
- py_cmp_func 5 7
- py_cmp_func 1 7
- py_cmp_func 5 1
- >>>
-
-and on Linux::
-
>>> qsort(ia, len(ia), sizeof(c_int), CMPFUNC(py_cmp_func)) # doctest: +LINUX
py_cmp_func 5 1
py_cmp_func 33 99
@@ -1054,9 +996,6 @@ and on Linux::
py_cmp_func 5 7
>>>
-It is quite interesting to see that the Windows :func:`qsort` function needs
-more comparisons than the linux version!
-
As we can easily check, our array is sorted now::
>>> for i in ia: print(i, end=" ")
@@ -1066,9 +1005,9 @@ As we can easily check, our array is sorted now::
**Important note for callback functions:**
-Make sure you keep references to CFUNCTYPE objects as long as they are used from
-C code. :mod:`ctypes` doesn't, and if you don't, they may be garbage collected,
-crashing your program when a callback is made.
+Make sure you keep references to :func:`CFUNCTYPE` objects as long as they are
+used from C code. :mod:`ctypes` doesn't, and if you don't, they may be garbage
+collected, crashing your program when a callback is made.
.. _ctypes-accessing-values-exported-from-dlls:
@@ -1345,7 +1284,10 @@ way is to instantiate one of the following classes:
assumed to return the windows specific :class:`HRESULT` code. :class:`HRESULT`
values contain information specifying whether the function call failed or
succeeded, together with additional error code. If the return value signals a
- failure, an :class:`WindowsError` is automatically raised.
+ failure, an :class:`OSError` is automatically raised.
+
+ .. versionchanged:: 3.3
+ :exc:`WindowsError` used to be raised.
.. class:: WinDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False)
@@ -1962,11 +1904,14 @@ Utility functions
.. function:: WinError(code=None, descr=None)
Windows only: this function is probably the worst-named thing in ctypes. It
- creates an instance of WindowsError. If *code* is not specified,
+ creates an instance of OSError. If *code* is not specified,
``GetLastError`` is called to determine the error code. If *descr* is not
specified, :func:`FormatError` is called to get a textual description of the
error.
+ .. versionchanged:: 3.3
+ An instance of :exc:`WindowsError` used to be created.
+
.. function:: wstring_at(address, size=-1)
diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst
index f31b9c5..ff3a793 100644
--- a/Doc/library/curses.rst
+++ b/Doc/library/curses.rst
@@ -598,6 +598,17 @@ The module :mod:`curses` defines the following functions:
Only one *ch* can be pushed before :meth:`getch` is called.
+.. function:: unget_wch(ch)
+
+ Push *ch* so the next :meth:`get_wch` will return it.
+
+ .. note::
+
+ Only one *ch* can be pushed before :meth:`get_wch` is called.
+
+ .. versionadded:: 3.3
+
+
.. function:: ungetmouse(id, x, y, z, bstate)
Push a :const:`KEY_MOUSE` event onto the input queue, associating the given
@@ -642,7 +653,7 @@ Window Objects
--------------
Window objects, as returned by :func:`initscr` and :func:`newwin` above, have
-the following methods:
+the following methods and attributes:
.. method:: window.addch([y, x,] ch[, attr])
@@ -823,6 +834,16 @@ the following methods:
event.
+.. attribute:: window.encoding
+
+ Encoding used to encode method arguments (Unicode strings and characters).
+ The encoding attribute is inherited from by parent window when a subwindow
+ is created, for example with :meth:`window.subwin`. By default, the locale
+ encoding is used (see :func:`locale.getpreferredencoding`).
+
+ .. versionadded:: 3.3
+
+
.. method:: window.erase()
Clear the window.
@@ -846,6 +867,14 @@ the following methods:
until a key is pressed.
+.. method:: window.get_wch([y, x])
+
+ Get a wide character. Like :meth:`getch`, but the integer returned is the
+ Unicode code point for the key pressed, so it can be passed to :func:`chr`.
+
+ .. versionadded:: 3.3
+
+
.. method:: window.getkey([y, x])
Get a character, returning a string instead of an integer, as :meth:`getch`
diff --git a/Doc/library/datatypes.rst b/Doc/library/datatypes.rst
index 6b4a71a..8e33c1f 100644
--- a/Doc/library/datatypes.rst
+++ b/Doc/library/datatypes.rst
@@ -21,6 +21,7 @@ The following modules are documented in this chapter:
datetime.rst
calendar.rst
collections.rst
+ collections.abc.rst
heapq.rst
bisect.rst
array.rst
diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst
index f7e8762..c0cc2f1 100644
--- a/Doc/library/datetime.rst
+++ b/Doc/library/datetime.rst
@@ -404,12 +404,19 @@ Other constructors, all class methods:
.. classmethod:: date.fromtimestamp(timestamp)
Return the local date corresponding to the POSIX timestamp, such as is returned
- by :func:`time.time`. This may raise :exc:`ValueError`, if the timestamp is out
- of the range of values supported by the platform C :c:func:`localtime` function.
+ by :func:`time.time`. This may raise :exc:`OverflowError`, if the timestamp is out
+ of the range of values supported by the platform C :c:func:`localtime` function,
+ and :exc:`OSError` on :c:func:`localtime` failure.
It's common for this to be restricted to years from 1970 through 2038. Note
that on non-POSIX systems that include leap seconds in their notion of a
timestamp, leap seconds are ignored by :meth:`fromtimestamp`.
+ .. versionchanged:: 3.3
+ Raise :exc:`OverflowError` instead of :exc:`ValueError` if the timestamp
+ is out of the range of values supported by the platform C
+ :c:func:`localtime` function. Raise :exc:`OSError` instead of
+ :exc:`ValueError` on :c:func:`localtime` failure.
+
.. classmethod:: date.fromordinal(ordinal)
@@ -713,23 +720,44 @@ Other constructors, all class methods:
equivalent to
``tz.fromutc(datetime.utcfromtimestamp(timestamp).replace(tzinfo=tz))``.
- :meth:`fromtimestamp` may raise :exc:`ValueError`, if the timestamp is out of
+ :meth:`fromtimestamp` may raise :exc:`OverflowError`, if the timestamp is out of
the range of values supported by the platform C :c:func:`localtime` or
- :c:func:`gmtime` functions. It's common for this to be restricted to years in
+ :c:func:`gmtime` functions, and :exc:`OSError` on :c:func:`localtime` or
+ :c:func:`gmtime` failure.
+ It's common for this to be restricted to years in
1970 through 2038. Note that on non-POSIX systems that include leap seconds in
their notion of a timestamp, leap seconds are ignored by :meth:`fromtimestamp`,
and then it's possible to have two timestamps differing by a second that yield
identical :class:`.datetime` objects. See also :meth:`utcfromtimestamp`.
+ .. versionchanged:: 3.3
+ Raise :exc:`OverflowError` instead of :exc:`ValueError` if the timestamp
+ is out of the range of values supported by the platform C
+ :c:func:`localtime` or :c:func:`gmtime` functions. Raise :exc:`OSError`
+ instead of :exc:`ValueError` on :c:func:`localtime` or :c:func:`gmtime`
+ failure.
+
.. classmethod:: datetime.utcfromtimestamp(timestamp)
Return the UTC :class:`.datetime` corresponding to the POSIX timestamp, with
- :attr:`tzinfo` ``None``. This may raise :exc:`ValueError`, if the timestamp is
- out of the range of values supported by the platform C :c:func:`gmtime` function.
+ :attr:`tzinfo` ``None``. This may raise :exc:`OverflowError`, if the timestamp is
+ out of the range of values supported by the platform C :c:func:`gmtime` function,
+ and :exc:`OSError` on :c:func:`gmtime` failure.
It's common for this to be restricted to years in 1970 through 2038. See also
:meth:`fromtimestamp`.
+ On the POSIX compliant platforms, ``utcfromtimestamp(timestamp)``
+ is equivalent to the following expression::
+
+ datetime(1970, 1, 1) + timedelta(seconds=timestamp)
+
+ .. versionchanged:: 3.3
+ Raise :exc:`OverflowError` instead of :exc:`ValueError` if the timestamp
+ is out of the range of values supported by the platform C
+ :c:func:`gmtime` function. Raise :exc:`OSError` instead of
+ :exc:`ValueError` on :c:func:`gmtime` failure.
+
.. classmethod:: datetime.fromordinal(ordinal)
@@ -873,13 +901,20 @@ Supported operations:
*datetime1* is considered less than *datetime2* when *datetime1* precedes
*datetime2* in time.
- If one comparand is naive and the other is aware, :exc:`TypeError` is raised.
+ If one comparand is naive and the other is aware, :exc:`TypeError`
+ is raised if an order comparison is attempted. For equality
+ comparisons, naive instances are never equal to aware instances.
+
If both comparands are aware, and have the same :attr:`tzinfo` attribute, the
common :attr:`tzinfo` attribute is ignored and the base datetimes are
compared. If both comparands are aware and have different :attr:`tzinfo`
attributes, the comparands are first adjusted by subtracting their UTC
offsets (obtained from ``self.utcoffset()``).
+ .. versionchanged:: 3.3
+ Equality comparisons between naive and aware :class:`datetime`
+ instances don't raise :exc:`TypeError`.
+
.. note::
In order to stop comparison from falling back to the default scheme of comparing
@@ -922,17 +957,22 @@ Instance methods:
datetime with no conversion of date and time data.
-.. method:: datetime.astimezone(tz)
+.. method:: datetime.astimezone(tz=None)
- Return a :class:`.datetime` object with new :attr:`tzinfo` attribute *tz*,
+ Return a :class:`datetime` object with new :attr:`tzinfo` attribute *tz*,
adjusting the date and time data so the result is the same UTC time as
*self*, but in *tz*'s local time.
- *tz* must be an instance of a :class:`tzinfo` subclass, and its
+ If provided, *tz* must be an instance of a :class:`tzinfo` subclass, and its
:meth:`utcoffset` and :meth:`dst` methods must not return ``None``. *self* must
be aware (``self.tzinfo`` must not be ``None``, and ``self.utcoffset()`` must
not return ``None``).
+ If called without arguments (or with ``tz=None``) the system local
+ timezone is assumed. The ``tzinfo`` attribute of the converted
+ datetime instance will be set to an instance of :class:`timezone`
+ with the zone name and offset obtained from the OS.
+
If ``self.tzinfo`` is *tz*, ``self.astimezone(tz)`` is equal to *self*: no
adjustment of date or time data is performed. Else the result is local
time in time zone *tz*, representing the same UTC time as *self*: after
@@ -1015,6 +1055,39 @@ Instance methods:
Return the proleptic Gregorian ordinal of the date. The same as
``self.date().toordinal()``.
+.. method:: datetime.timestamp()
+
+ Return POSIX timestamp corresponding to the :class:`datetime`
+ instance. The return value is a :class:`float` similar to that
+ returned by :func:`time.time`.
+
+ Naive :class:`datetime` instances are assumed to represent local
+ time and this method relies on the platform C :c:func:`mktime`
+ function to perform the conversion. Since :class:`datetime`
+ supports wider range of values than :c:func:`mktime` on many
+ platforms, this method may raise :exc:`OverflowError` for times far
+ in the past or far in the future.
+
+ For aware :class:`datetime` instances, the return value is computed
+ as::
+
+ (dt - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds()
+
+ .. versionadded:: 3.3
+
+ .. note::
+
+ There is no method to obtain the POSIX timestamp directly from a
+ naive :class:`datetime` instance representing UTC time. If your
+ application uses this convention and your system timezone is not
+ set to UTC, you can obtain the POSIX timestamp by supplying
+ ``tzinfo=timezone.utc``::
+
+ timestamp = dt.replace(tzinfo=timezone.utc).timestamp()
+
+ or by calculating the timestamp directly::
+
+ timestamp = (dt - datetime(1970, 1, 1)) / timedelta(seconds=1)
.. method:: datetime.weekday()
@@ -1255,7 +1328,10 @@ Supported operations:
* comparison of :class:`.time` to :class:`.time`, where *a* is considered less
than *b* when *a* precedes *b* in time. If one comparand is naive and the other
- is aware, :exc:`TypeError` is raised. If both comparands are aware, and have
+ is aware, :exc:`TypeError` is raised if an order comparison is attempted. For equality
+ comparisons, naive instances are never equal to aware instances.
+
+ If both comparands are aware, and have
the same :attr:`tzinfo` attribute, the common :attr:`tzinfo` attribute is
ignored and the base times are compared. If both comparands are aware and
have different :attr:`tzinfo` attributes, the comparands are first adjusted by
@@ -1265,6 +1341,10 @@ Supported operations:
different type, :exc:`TypeError` is raised unless the comparison is ``==`` or
``!=``. The latter cases return :const:`False` or :const:`True`, respectively.
+ .. versionchanged:: 3.3
+ Equality comparisons between naive and aware :class:`time` instances
+ don't raise :exc:`TypeError`.
+
* hash, use as dict key
* efficient pickling
@@ -1588,11 +1668,12 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)).
:class:`timezone` Objects
--------------------------
-A :class:`timezone` object represents a timezone that is defined by a
-fixed offset from UTC. Note that objects of this class cannot be used
-to represent timezone information in the locations where different
-offsets are used in different days of the year or where historical
-changes have been made to civil time.
+The :class:`timezone` class is a subclass of :class:`tzinfo`, each
+instance of which represents a timezone defined by a fixed offset from
+UTC. Note that objects of this class cannot be used to represent
+timezone information in the locations where different offsets are used
+in different days of the year or where historical changes have been
+made to civil time.
.. class:: timezone(offset[, name])
@@ -1761,8 +1842,7 @@ format codes.
| | decimal number [00,99]. | |
+-----------+--------------------------------+-------+
| ``%Y`` | Year with century as a decimal | \(5) |
-| | number [0001,9999] (strptime), | |
-| | [1000,9999] (strftime). | |
+| | number [0001,9999]. | |
+-----------+--------------------------------+-------+
| ``%z`` | UTC offset in the form +HHMM | \(6) |
| | or -HHMM (empty string if the | |
@@ -1796,10 +1876,7 @@ Notes:
calculations when the day of the week and the year are specified.
(5)
- For technical reasons, :meth:`strftime` method does not support
- dates before year 1000: ``t.strftime(format)`` will raise a
- :exc:`ValueError` when ``t.year < 1000`` even if ``format`` does
- not contain ``%Y`` directive. The :meth:`strptime` method can
+ The :meth:`strptime` method can
parse years in the full [1, 9999] range, but years < 1000 must be
zero-filled to 4-digit width.
@@ -1807,6 +1884,10 @@ Notes:
In previous versions, :meth:`strftime` method was restricted to
years >= 1900.
+ .. versionchanged:: 3.3
+ In version 3.2, :meth:`strftime` method was restricted to
+ years >= 1000.
+
(6)
For example, if :meth:`utcoffset` returns ``timedelta(hours=-3, minutes=-30)``,
``%z`` is replaced with the string ``'-0330'``.
diff --git a/Doc/library/debug.rst b/Doc/library/debug.rst
index b2ee4fa..c69fb1c 100644
--- a/Doc/library/debug.rst
+++ b/Doc/library/debug.rst
@@ -10,7 +10,8 @@ allowing you to identify bottlenecks in your programs.
.. toctree::
bdb.rst
+ faulthandler.rst
pdb.rst
profile.rst
timeit.rst
- trace.rst \ No newline at end of file
+ trace.rst
diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst
index ef8b43f..3fa006b 100644
--- a/Doc/library/decimal.rst
+++ b/Doc/library/decimal.rst
@@ -9,6 +9,7 @@
.. moduleauthor:: Raymond Hettinger <python at rcn.com>
.. moduleauthor:: Aahz <aahz at pobox.com>
.. moduleauthor:: Tim Peters <tim.one at comcast.net>
+.. moduleauthor:: Stefan Krah <skrah at bytereef.org>
.. sectionauthor:: Raymond D. Hettinger <python at rcn.com>
.. import modules for testing inline doctests with the Sphinx doctest builder
@@ -20,8 +21,9 @@
# make sure each group gets a fresh context
setcontext(Context())
-The :mod:`decimal` module provides support for decimal floating point
-arithmetic. It offers several advantages over the :class:`float` datatype:
+The :mod:`decimal` module provides support for fast correctly-rounded
+decimal floating point arithmetic. It offers several advantages over the
+:class:`float` datatype:
* Decimal "is based on a floating-point model which was designed with people
in mind, and necessarily has a paramount guiding principle -- computers must
@@ -92,7 +94,7 @@ computation. Depending on the needs of the application, signals may be ignored,
considered as informational, or treated as exceptions. The signals in the
decimal module are: :const:`Clamped`, :const:`InvalidOperation`,
:const:`DivisionByZero`, :const:`Inexact`, :const:`Rounded`, :const:`Subnormal`,
-:const:`Overflow`, and :const:`Underflow`.
+:const:`Overflow`, :const:`Underflow` and :const:`FloatOperation`.
For each signal there is a flag and a trap enabler. When a signal is
encountered, its flag is set to one, then, if the trap enabler is
@@ -122,7 +124,7 @@ precision, rounding, or enabled traps::
>>> from decimal import *
>>> getcontext()
- Context(prec=28, rounding=ROUND_HALF_EVEN, Emin=-999999999, Emax=999999999,
+ Context(prec=28, rounding=ROUND_HALF_EVEN, Emin=-999999, Emax=999999,
capitals=1, clamp=0, flags=[], traps=[Overflow, DivisionByZero,
InvalidOperation])
@@ -132,7 +134,7 @@ Decimal instances can be constructed from integers, strings, floats, or tuples.
Construction from an integer or a float performs an exact conversion of the
value of that integer or float. Decimal numbers include special values such as
:const:`NaN` which stands for "Not a number", positive and negative
-:const:`Infinity`, and :const:`-0`.
+:const:`Infinity`, and :const:`-0`::
>>> getcontext().prec = 28
>>> Decimal(10)
@@ -152,6 +154,25 @@ value of that integer or float. Decimal numbers include special values such as
>>> Decimal('-Infinity')
Decimal('-Infinity')
+If the :exc:`FloatOperation` signal is trapped, accidental mixing of
+decimals and floats in constructors or ordering comparisons raises
+an exception::
+
+ >>> c = getcontext()
+ >>> c.traps[FloatOperation] = True
+ >>> Decimal(3.14)
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ decimal.FloatOperation: [<class 'decimal.FloatOperation'>]
+ >>> Decimal('3.5') < 3.7
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ decimal.FloatOperation: [<class 'decimal.FloatOperation'>]
+ >>> Decimal('3.5') == 3.5
+ True
+
+.. versionadded:: 3.3
+
The significance of a new Decimal is determined solely by the number of digits
input. Context precision and rounding only come into play during arithmetic
operations.
@@ -169,6 +190,16 @@ operations.
>>> Decimal('3.1415926535') + Decimal('2.7182818285')
Decimal('5.85988')
+If the internal limits of the C version are exceeded, constructing
+a decimal raises :class:`InvalidOperation`::
+
+ >>> Decimal("1e9999999999999999999")
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ decimal.InvalidOperation: [<class 'decimal.InvalidOperation'>]
+
+.. versionchanged:: 3.3
+
Decimals interact well with much of the rest of Python. Here is a small decimal
floating point flying circus:
@@ -244,7 +275,7 @@ enabled:
Decimal('0.142857142857142857142857142857142857142857142857142857142857')
>>> ExtendedContext
- Context(prec=9, rounding=ROUND_HALF_EVEN, Emin=-999999999, Emax=999999999,
+ Context(prec=9, rounding=ROUND_HALF_EVEN, Emin=-999999, Emax=999999,
capitals=1, clamp=0, flags=[], traps=[])
>>> setcontext(ExtendedContext)
>>> Decimal(1) / Decimal(7)
@@ -269,7 +300,7 @@ using the :meth:`clear_flags` method. ::
>>> Decimal(355) / Decimal(113)
Decimal('3.14159292')
>>> getcontext()
- Context(prec=9, rounding=ROUND_HALF_EVEN, Emin=-999999999, Emax=999999999,
+ Context(prec=9, rounding=ROUND_HALF_EVEN, Emin=-999999, Emax=999999,
capitals=1, clamp=0, flags=[Inexact, Rounded], traps=[])
The *flags* entry shows that the rational approximation to :const:`Pi` was
@@ -358,6 +389,10 @@ Decimal objects
The argument to the constructor is now permitted to be a :class:`float`
instance.
+ .. versionchanged:: 3.3
+ :class:`float` arguments raise an exception if the :exc:`FloatOperation`
+ trap is set. By default the trap is off.
+
Decimal floating point objects share many properties with the other built-in
numeric types such as :class:`float` and :class:`int`. All of the usual math
operations and special methods apply. Likewise, decimal objects can be
@@ -880,39 +915,33 @@ described below. In addition, the module provides three pre-made contexts:
In single threaded environments, it is preferable to not use this context at
all. Instead, simply create contexts explicitly as described below.
- The default values are precision=28, rounding=ROUND_HALF_EVEN, and enabled traps
- for Overflow, InvalidOperation, and DivisionByZero.
+ The default values are :attr:`prec`\ =\ :const:`28`,
+ :attr:`rounding`\ =\ :const:`ROUND_HALF_EVEN`,
+ and enabled traps for :class:`Overflow`, :class:`InvalidOperation`, and
+ :class:`DivisionByZero`.
In addition to the three supplied contexts, new contexts can be created with the
:class:`Context` constructor.
-.. class:: Context(prec=None, rounding=None, traps=None, flags=None, Emin=None, Emax=None, capitals=None, clamp=None)
+.. class:: Context(prec=None, rounding=None, Emin=None, Emax=None, capitals=None, clamp=None, flags=None, traps=None)
Creates a new context. If a field is not specified or is :const:`None`, the
default values are copied from the :const:`DefaultContext`. If the *flags*
field is not specified or is :const:`None`, all flags are cleared.
- The *prec* field is a positive integer that sets the precision for arithmetic
- operations in the context.
-
- The *rounding* option is one of:
+ *prec* is an integer in the range [:const:`1`, :const:`MAX_PREC`] that sets
+ the precision for arithmetic operations in the context.
- * :const:`ROUND_CEILING` (towards :const:`Infinity`),
- * :const:`ROUND_DOWN` (towards zero),
- * :const:`ROUND_FLOOR` (towards :const:`-Infinity`),
- * :const:`ROUND_HALF_DOWN` (to nearest with ties going towards zero),
- * :const:`ROUND_HALF_EVEN` (to nearest with ties going to nearest even integer),
- * :const:`ROUND_HALF_UP` (to nearest with ties going away from zero), or
- * :const:`ROUND_UP` (away from zero).
- * :const:`ROUND_05UP` (away from zero if last digit after rounding towards zero
- would have been 0 or 5; otherwise towards zero)
+ The *rounding* option is one of the constants listed in the section
+ `Rounding Modes`_.
The *traps* and *flags* fields list any signals to be set. Generally, new
contexts should only set traps and leave the flags clear.
The *Emin* and *Emax* fields are integers specifying the outer limits allowable
- for exponents.
+ for exponents. *Emin* must be in the range [:const:`MIN_EMIN`, :const:`0`],
+ *Emax* in the range [:const:`0`, :const:`MAX_EMAX`].
The *capitals* field is either :const:`0` or :const:`1` (the default). If set to
:const:`1`, exponents are printed with a capital :const:`E`; otherwise, a
@@ -951,6 +980,12 @@ In addition to the three supplied contexts, new contexts can be created with the
Resets all of the flags to :const:`0`.
+ .. method:: clear_traps()
+
+ Resets all of the traps to :const:`0`.
+
+ .. versionadded:: 3.3
+
.. method:: copy()
Return a duplicate of the context.
@@ -1250,8 +1285,13 @@ In addition to the three supplied contexts, new contexts can be created with the
With two arguments, compute ``x**y``. If ``x`` is negative then ``y``
must be integral. The result will be inexact unless ``y`` is integral and
the result is finite and can be expressed exactly in 'precision' digits.
- The result should always be correctly rounded, using the rounding mode of
- the current thread's context.
+ The rounding mode of the context is used. Results are always correctly-rounded
+ in the Python version.
+
+ .. versionchanged:: 3.3
+ The C module computes :meth:`power` in terms of the correctly-rounded
+ :meth:`exp` and :meth:`ln` functions. The result is well-defined but
+ only "almost always correctly-rounded".
With three arguments, compute ``(x**y) % modulo``. For the three argument
form, the following restrictions on the arguments hold:
@@ -1339,6 +1379,69 @@ In addition to the three supplied contexts, new contexts can be created with the
.. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+.. _decimal-rounding-modes:
+
+Constants
+---------
+
+The constants in this section are only relevant for the C module. They
+are also included in the pure Python version for compatibility.
+
++---------------------+---------------------+-------------------------------+
+| | 32-bit | 64-bit |
++=====================+=====================+===============================+
+| .. data:: MAX_PREC | :const:`425000000` | :const:`999999999999999999` |
++---------------------+---------------------+-------------------------------+
+| .. data:: MAX_EMAX | :const:`425000000` | :const:`999999999999999999` |
++---------------------+---------------------+-------------------------------+
+| .. data:: MIN_EMIN | :const:`-425000000` | :const:`-999999999999999999` |
++---------------------+---------------------+-------------------------------+
+| .. data:: MIN_ETINY | :const:`-849999999` | :const:`-1999999999999999997` |
++---------------------+---------------------+-------------------------------+
+
+
+.. data:: HAVE_THREADS
+
+ The default value is True. If Python is compiled without threads, the
+ C version automatically disables the expensive thread local context
+ machinery. In this case, the value is False.
+
+Rounding modes
+--------------
+
+.. data:: ROUND_CEILING
+
+ Round towards :const:`Infinity`.
+
+.. data:: ROUND_DOWN
+
+ Round towards zero.
+
+.. data:: ROUND_FLOOR
+
+ Round towards :const:`-Infinity`.
+
+.. data:: ROUND_HALF_DOWN
+
+ Round to nearest with ties going towards zero.
+
+.. data:: ROUND_HALF_EVEN
+
+ Round to nearest with ties going to nearest even integer.
+
+.. data:: ROUND_HALF_UP
+
+ Round to nearest with ties going away from zero.
+
+.. data:: ROUND_UP
+
+ Round away from zero.
+
+.. data:: ROUND_05UP
+
+ Round away from zero if last digit after rounding towards zero would have
+ been 0 or 5; otherwise round towards zero.
+
.. _decimal-signals:
@@ -1403,7 +1506,6 @@ condition.
Infinity / Infinity
x % 0
Infinity % x
- x._rescale( non-integer )
sqrt(-x) and x > 0
0 ** 0
x ** (non-integer)
@@ -1446,6 +1548,23 @@ condition.
Occurs when a subnormal result is pushed to zero by rounding. :class:`Inexact`
and :class:`Subnormal` are also signaled.
+
+.. class:: FloatOperation
+
+ Enable stricter semantics for mixing floats and Decimals.
+
+ If the signal is not trapped (default), mixing floats and Decimals is
+ permitted in the :class:`~decimal.Decimal` constructor,
+ :meth:`~decimal.Context.create_decimal` and all comparison operators.
+ Both conversion and comparisons are exact. Any occurrence of a mixed
+ operation is silently recorded by setting :exc:`FloatOperation` in the
+ context flags. Explicit conversions with :meth:`~decimal.Decimal.from_float`
+ or :meth:`~decimal.Context.create_decimal_from_float` do not set the flag.
+
+ Otherwise (the signal is trapped), only equality comparisons and explicit
+ conversions are silent. All other mixed operations raise :exc:`FloatOperation`.
+
+
The following table summarizes the hierarchy of signals::
exceptions.ArithmeticError(exceptions.Exception)
@@ -1458,10 +1577,12 @@ The following table summarizes the hierarchy of signals::
InvalidOperation
Rounded
Subnormal
+ FloatOperation(DecimalException, exceptions.TypeError)
.. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
.. _decimal-notes:
Floating Point Notes
@@ -1571,7 +1692,7 @@ normalized floating point representations, it is not immediately obvious that
the following calculation returns a value equal to zero:
>>> 1 / Decimal('Infinity')
- Decimal('0E-1000000026')
+ Decimal('0E-1000026')
.. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -1583,7 +1704,7 @@ Working with threads
The :func:`getcontext` function accesses a different :class:`Context` object for
each thread. Having separate thread contexts means that threads may make
-changes (such as ``getcontext.prec=10``) without interfering with other threads.
+changes (such as ``getcontext().prec=10``) without interfering with other threads.
Likewise, the :func:`setcontext` function automatically assigns its target to
the current thread.
diff --git a/Doc/library/depgraph-output.png b/Doc/library/depgraph-output.png
new file mode 100644
index 0000000..960bb1b
--- /dev/null
+++ b/Doc/library/depgraph-output.png
Binary files differ
diff --git a/Doc/library/development.rst b/Doc/library/development.rst
index c822e08..2368769 100644
--- a/Doc/library/development.rst
+++ b/Doc/library/development.rst
@@ -19,5 +19,8 @@ The list of modules described in this chapter is:
pydoc.rst
doctest.rst
unittest.rst
+ unittest.mock.rst
+ unittest.mock-examples.rst
2to3.rst
test.rst
+ venv.rst
diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst
index bdc37b3..836e240 100644
--- a/Doc/library/difflib.rst
+++ b/Doc/library/difflib.rst
@@ -752,8 +752,8 @@ It is also contained in the Python source distribution, as
# we're passing these as arguments to the diff function
fromdate = time.ctime(os.stat(fromfile).st_mtime)
todate = time.ctime(os.stat(tofile).st_mtime)
- fromlines = open(fromfile, 'U').readlines()
- tolines = open(tofile, 'U').readlines()
+ with open(fromlines) as fromf, open(tofile) as tof:
+ fromlines, tolines = list(fromf), list(tof)
if options.u:
diff = difflib.unified_diff(fromlines, tolines, fromfile, tofile,
diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst
index 79cc583..6d08a0c 100644
--- a/Doc/library/dis.rst
+++ b/Doc/library/dis.rst
@@ -171,11 +171,6 @@ The Python compiler currently generates the following bytecode instructions.
**General instructions**
-.. opcode:: STOP_CODE
-
- Indicates end-of-code to the compiler, not used by the interpreter.
-
-
.. opcode:: NOP
Do nothing code. Used as a placeholder by the bytecode optimizer.
@@ -436,6 +431,13 @@ the stack so that it is available for further iterations of the loop.
Pops ``TOS`` and yields it from a :term:`generator`.
+.. opcode:: YIELD_FROM
+
+ Pops ``TOS`` and delegates to it as a subiterator from a :term:`generator`.
+
+ .. versionadded:: 3.3
+
+
.. opcode:: IMPORT_STAR
Loads all symbols not starting with ``'_'`` directly from the module TOS to the
@@ -752,9 +754,10 @@ the more significant byte last.
.. opcode:: MAKE_FUNCTION (argc)
- Pushes a new function object on the stack. TOS is the code associated with the
- function. The function object is defined to have *argc* default parameters,
- which are found below TOS.
+ Pushes a new function object on the stack. TOS is the
+ :term:`qualified name` of the function; TOS1 is the code associated with
+ the function. The function object is defined to have *argc* default parameters,
+ which are found below TOS1.
.. opcode:: MAKE_CLOSURE (argc)
diff --git a/Doc/library/distutils.rst b/Doc/library/distutils.rst
index 238b79d..11a2949 100644
--- a/Doc/library/distutils.rst
+++ b/Doc/library/distutils.rst
@@ -12,18 +12,14 @@ additional modules into a Python installation. The new modules may be either
100%-pure Python, or may be extension modules written in C, or may be
collections of Python packages which include modules coded in both Python and C.
-This package is discussed in two separate chapters:
+User documentation and API reference are provided in another document:
.. seealso::
:ref:`distutils-index`
The manual for developers and packagers of Python modules. This describes
how to prepare :mod:`distutils`\ -based packages so that they may be
- easily installed into an existing Python installation.
-
- :ref:`install-index`
- An "administrators" manual which includes information on installing
- modules into an existing Python installation. You do not need to be a
- Python programmer to read this manual.
-
+ easily installed into an existing Python installation. If also contains
+ instructions for end-users wanting to install a distutils-based package,
+ :ref:`install-index`.
diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst
index d8f330f..9f0a9e2 100644
--- a/Doc/library/email.errors.rst
+++ b/Doc/library/email.errors.rst
@@ -73,17 +73,38 @@ this class is *not* an exception!
* :class:`StartBoundaryNotFoundDefect` -- The start boundary claimed in the
:mailheader:`Content-Type` header was never found.
+* :class:`CloseBoundaryNotFoundDefect` -- A start boundary was found, but
+ no corresponding close boundary was ever found.
+
+ .. versionadded:: 3.3
+
* :class:`FirstHeaderLineIsContinuationDefect` -- The message had a continuation
line as its first header line.
* :class:`MisplacedEnvelopeHeaderDefect` - A "Unix From" header was found in the
middle of a header block.
+* :class:`MissingHeaderBodySeparatorDefect` - A line was found while parsing
+ headers that had no leading white space but contained no ':'. Parsing
+ continues assuming that the line represents the first line of the body.
+
+ .. versionadded:: 3.3
+
* :class:`MalformedHeaderDefect` -- A header was found that was missing a colon,
or was otherwise malformed.
+ .. deprecated:: 3.3
+ This defect has not been used for several Python versions.
+
* :class:`MultipartInvariantViolationDefect` -- A message claimed to be a
:mimetype:`multipart`, but no subparts were found. Note that when a message has
this defect, its :meth:`is_multipart` method may return false even though its
content type claims to be :mimetype:`multipart`.
+* :class:`InvalidBase64PaddingDefect` -- When decoding a block of base64
+ enocded bytes, the padding was not correct. Enough padding is added to
+ perform the decode, but the resulting decoded bytes may be invalid.
+
+* :class:`InvalidBase64CharactersDefect` -- When decoding a block of base64
+ enocded bytes, characters outside the base64 alphebet were encountered.
+ The characters are ignored, but the resulting decoded bytes may be invalid.
diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst
index 88c62a2..c172acb 100644
--- a/Doc/library/email.generator.rst
+++ b/Doc/library/email.generator.rst
@@ -32,7 +32,7 @@ Here are the public methods of the :class:`Generator` class, imported from the
:mod:`email.generator` module:
-.. class:: Generator(outfp, mangle_from_=True, maxheaderlen=78)
+.. class:: Generator(outfp, mangle_from_=True, maxheaderlen=78, *, policy=None)
The constructor for the :class:`Generator` class takes a :term:`file-like object`
called *outfp* for an argument. *outfp* must support the :meth:`write` method
@@ -53,10 +53,17 @@ Here are the public methods of the :class:`Generator` class, imported from the
:class:`~email.header.Header` class. Set to zero to disable header wrapping.
The default is 78, as recommended (but not required) by :rfc:`2822`.
+ The *policy* keyword specifies a :mod:`~email.policy` object that controls a
+ number of aspects of the generator's operation. If no *policy* is specified,
+ then the *policy* attached to the message object passed to :attr:`flatten`
+ is used.
+
+ .. versionchanged:: 3.3 Added the *policy* keyword.
+
The other public :class:`Generator` methods are:
- .. method:: flatten(msg, unixfrom=False, linesep='\\n')
+ .. method:: flatten(msg, unixfrom=False, linesep=None)
Print the textual representation of the message object structure rooted at
*msg* to the output file specified when the :class:`Generator` instance
@@ -72,19 +79,20 @@ Here are the public methods of the :class:`Generator` class, imported from the
Note that for subparts, no envelope header is ever printed.
Optional *linesep* specifies the line separator character used to
- terminate lines in the output. It defaults to ``\n`` because that is
- the most useful value for Python application code (other library packages
- expect ``\n`` separated lines). ``linesep=\r\n`` can be used to
- generate output with RFC-compliant line separators.
+ terminate lines in the output. If specified it overrides the value
+ specified by the *msg*\'s or ``Generator``\'s ``policy``.
- Messages parsed with a Bytes parser that have a
- :mailheader:`Content-Transfer-Encoding` of 8bit will be converted to a
- use a 7bit Content-Transfer-Encoding. Non-ASCII bytes in the headers
- will be :rfc:`2047` encoded with a charset of `unknown-8bit`.
+ Because strings cannot represent non-ASCII bytes, if the policy that
+ applies when ``flatten`` is run has :attr:`~email.policy.Policy.cte_type`
+ set to ``8bit``, ``Generator`` will operate as if it were set to
+ ``7bit``. This means that messages parsed with a Bytes parser that have
+ a :mailheader:`Content-Transfer-Encoding` of ``8bit`` will be converted
+ to a use a ``7bit`` Content-Transfer-Encoding. Non-ASCII bytes in the
+ headers will be :rfc:`2047` encoded with a charset of ``unknown-8bit``.
.. versionchanged:: 3.2
- Added support for re-encoding 8bit message bodies, and the *linesep*
- argument.
+ Added support for re-encoding ``8bit`` message bodies, and the
+ *linesep* argument.
.. method:: clone(fp)
@@ -103,7 +111,8 @@ As a convenience, see the :class:`~email.message.Message` methods
formatted string representation of a message object. For more detail, see
:mod:`email.message`.
-.. class:: BytesGenerator(outfp, mangle_from_=True, maxheaderlen=78)
+.. class:: BytesGenerator(outfp, mangle_from_=True, maxheaderlen=78, *, \
+ policy=policy.default)
The constructor for the :class:`BytesGenerator` class takes a binary
:term:`file-like object` called *outfp* for an argument. *outfp* must
@@ -125,19 +134,31 @@ formatted string representation of a message object. For more detail, see
wrapping. The default is 78, as recommended (but not required) by
:rfc:`2822`.
+ The *policy* keyword specifies a :mod:`~email.policy` object that controls a
+ number of aspects of the generator's operation. The default policy
+ maintains backward compatibility.
+
+ .. versionchanged:: 3.3 Added the *policy* keyword.
+
The other public :class:`BytesGenerator` methods are:
- .. method:: flatten(msg, unixfrom=False, linesep='\n')
+ .. method:: flatten(msg, unixfrom=False, linesep=None)
Print the textual representation of the message object structure rooted
at *msg* to the output file specified when the :class:`BytesGenerator`
instance was created. Subparts are visited depth-first and the resulting
- text will be properly MIME encoded. If the input that created the *msg*
- contained bytes with the high bit set and those bytes have not been
- modified, they will be copied faithfully to the output, even if doing so
- is not strictly RFC compliant. (To produce strictly RFC compliant
- output, use the :class:`Generator` class.)
+ text will be properly MIME encoded. If the :mod:`~email.policy` option
+ :attr:`~email.policy.Policy.cte_type` is ``8bit`` (the default),
+ then any bytes with the high bit set in the original parsed message that
+ have not been modified will be copied faithfully to the output. If
+ ``cte_type`` is ``7bit``, the bytes will be converted as needed
+ using an ASCII-compatible Content-Transfer-Encoding. In particular,
+ RFC-invalid non-ASCII bytes in headers will be encoded using the MIME
+ ``unknown-8bit`` character set, thus rendering them RFC-compliant.
+
+ .. XXX: There should be a complementary option that just does the RFC
+ compliance transformation but leaves CTE 8bit parts alone.
Messages parsed with a Bytes parser that have a
:mailheader:`Content-Transfer-Encoding` of 8bit will be reconstructed
@@ -152,10 +173,8 @@ formatted string representation of a message object. For more detail, see
Note that for subparts, no envelope header is ever printed.
Optional *linesep* specifies the line separator character used to
- terminate lines in the output. It defaults to ``\n`` because that is
- the most useful value for Python application code (other library packages
- expect ``\n`` separated lines). ``linesep=\r\n`` can be used to
- generate output with RFC-compliant line separators.
+ terminate lines in the output. If specified it overrides the value
+ specified by the ``Generator``\ 's ``policy``.
.. method:: clone(fp)
diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst
new file mode 100644
index 0000000..c884159
--- /dev/null
+++ b/Doc/library/email.headerregistry.rst
@@ -0,0 +1,452 @@
+:mod:`email.headerregistry`: Custom Header Objects
+--------------------------------------------------
+
+.. module:: email.headerregistry
+ :synopsis: Automatic Parsing of headers based on the field name
+
+.. moduleauthor:: R. David Murray <rdmurray@bitdance.com>
+.. sectionauthor:: R. David Murray <rdmurray@bitdance.com>
+
+
+.. note::
+
+ The headerregistry module has been included in the standard library on a
+ :term:`provisional basis <provisional package>`. Backwards incompatible
+ changes (up to and including removal of the module) may occur if deemed
+ necessary by the core developers.
+
+.. versionadded:: 3.3
+ as a :term:`provisional module <provisional package>`.
+
+Headers are represented by customized subclasses of :class:`str`. The
+particular class used to represent a given header is determined by the
+:attr:`~email.policy.EmailPolicy.header_factory` of the :mod:`~email.policy` in
+effect when the headers are created. This section documents the particular
+``header_factory`` implemented by the email package for handling :RFC:`5322`
+compliant email messages, which not only provides customized header objects for
+various header types, but also provides an extension mechanism for applications
+to add their own custom header types.
+
+When using any of the policy objects derived from
+:data:`~email.policy.EmailPolicy`, all headers are produced by
+:class:`.HeaderRegistry` and have :class:`.BaseHeader` as their last base
+class. Each header class has an additional base class that is determined by
+the type of the header. For example, many headers have the class
+:class:`.UnstructuredHeader` as their other base class. The specialized second
+class for a header is determined by the name of the header, using a lookup
+table stored in the :class:`.HeaderRegistry`. All of this is managed
+transparently for the typical application program, but interfaces are provided
+for modifying the default behavior for use by more complex applications.
+
+The sections below first document the header base classes and their attributes,
+followed by the API for modifying the behavior of :class:`.HeaderRegistry`, and
+finally the support classes used to represent the data parsed from structured
+headers.
+
+
+.. class:: BaseHeader(name, value)
+
+ *name* and *value* are passed to ``BaseHeader`` from the
+ :attr:`~email.policy.EmailPolicy.header_factory` call. The string value of
+ any header object is the *value* fully decoded to unicode.
+
+ This base class defines the following read-only properties:
+
+
+ .. attribute:: name
+
+ The name of the header (the portion of the field before the ':'). This
+ is exactly the value passed in the :attr:`~EmailPolicy.header_factory`
+ call for *name*; that is, case is preserved.
+
+
+ .. attribute:: defects
+
+ A tuple of :exc:`~email.errors.HeaderDefect` instances reporting any
+ RFC compliance problems found during parsing. The email package tries to
+ be complete about detecting compliance issues. See the :mod:`errors`
+ module for a discussion of the types of defects that may be reported.
+
+
+ .. attribute:: max_count
+
+ The maximum number of headers of this type that can have the same
+ ``name``. A value of ``None`` means unlimited. The ``BaseHeader`` value
+ for this attribute is ``None``; it is expected that specialized header
+ classes will override this value as needed.
+
+ ``BaseHeader`` also provides the following method, which is called by the
+ email library code and should not in general be called by application
+ programs:
+
+ .. method:: fold(*, policy)
+
+ Return a string containing :attr:`~email.policy.Policy.linesep`
+ characters as required to correctly fold the header according
+ to *policy*. A :attr:`~email.policy.Policy.cte_type` of
+ ``8bit`` will be treated as if it were ``7bit``, since strings
+ may not contain binary data.
+
+
+ ``BaseHeader`` by itself cannot be used to create a header object. It
+ defines a protocol that each specialized header cooperates with in order to
+ produce the header object. Specifically, ``BaseHeader`` requires that
+ the specialized class provide a :func:`classmethod` named ``parse``. This
+ method is called as follows::
+
+ parse(string, kwds)
+
+ ``kwds`` is a dictionary containing one pre-initialized key, ``defects``.
+ ``defects`` is an empty list. The parse method should append any detected
+ defects to this list. On return, the ``kwds`` dictionary *must* contain
+ values for at least the keys ``decoded`` and ``defects``. ``decoded``
+ should be the string value for the header (that is, the header value fully
+ decoded to unicode). The parse method should assume that *string* may
+ contain transport encoded parts, but should correctly handle all valid
+ unicode characters as well so that it can parse un-encoded header values.
+
+ ``BaseHeader``'s ``__new__`` then creates the header instance, and calls its
+ ``init`` method. The specialized class only needs to provide an ``init``
+ method if it wishes to set additional attributes beyond those provided by
+ ``BaseHeader`` itself. Such an ``init`` method should look like this::
+
+ def init(self, *args, **kw):
+ self._myattr = kw.pop('myattr')
+ super().init(*args, **kw)
+
+ That is, anything extra that the specialized class puts in to the ``kwds``
+ dictionary should be removed and handled, and the remaining contents of
+ ``kw`` (and ``args``) passed to the ``BaseHeader`` ``init`` method.
+
+
+.. class:: UnstructuredHeader
+
+ An "unstructured" header is the default type of header in :rfc:`5322`.
+ Any header that does not have a specified syntax is treated as
+ unstructured. The classic example of an unstructured header is the
+ :mailheader:`Subject` header.
+
+ In :rfc:`5322`, an unstructured header is a run of arbitrary text in the
+ ASCII character set. :rfc:`2047`, however, has an :rfc:`5322` compatible
+ mechanism for encoding non-ASCII text as ASCII characters within a header
+ value. When a *value* containing encoded words is passed to the
+ constructor, the ``UnstructuredHeader`` parser converts such encoded words
+ back in to the original unicode, following the :rfc:`2047` rules for
+ unstructured text. The parser uses heuristics to attempt to decode certain
+ non-compliant encoded words. Defects are registered in such cases, as well
+ as defects for issues such as invalid characters within the encoded words or
+ the non-encoded text.
+
+ This header type provides no additional attributes.
+
+
+.. class:: DateHeader
+
+ :rfc:`5322` specifies a very specific format for dates within email headers.
+ The ``DateHeader`` parser recognizes that date format, as well as
+ recognizing a number of variant forms that are sometimes found "in the
+ wild".
+
+ This header type provides the following additional attributes:
+
+ .. attribute:: datetime
+
+ If the header value can be recognized as a valid date of one form or
+ another, this attribute will contain a :class:`~datetime.datetime`
+ instance representing that date. If the timezone of the input date is
+ specified as ``-0000`` (indicating it is in UTC but contains no
+ information about the source timezone), then :attr:`.datetime` will be a
+ naive :class:`~datetime.datetime`. If a specific timezone offset is
+ found (including `+0000`), then :attr:`.datetime` will contain an aware
+ ``datetime`` that uses :class:`datetime.timezone` to record the timezone
+ offset.
+
+ The ``decoded`` value of the header is determined by formatting the
+ ``datetime`` according to the :rfc:`5322` rules; that is, it is set to::
+
+ email.utils.format_datetime(self.datetime)
+
+ When creating a ``DateHeader``, *value* may be
+ :class:`~datetime.datetime` instance. This means, for example, that
+ the following code is valid and does what one would expect::
+
+ msg['Date'] = datetime(2011, 7, 15, 21)
+
+ Because this is a naive ``datetime`` it will be interpreted as a UTC
+ timestamp, and the resulting value will have a timezone of ``-0000``. Much
+ more useful is to use the :func:`~email.utils.localtime` function from the
+ :mod:`~email.utils` module::
+
+ msg['Date'] = utils.localtime()
+
+ This example sets the date header to the current time and date using
+ the current timezone offset.
+
+
+.. class:: AddressHeader
+
+ Address headers are one of the most complex structured header types.
+ The ``AddressHeader`` class provides a generic interface to any address
+ header.
+
+ This header type provides the following additional attributes:
+
+
+ .. attribute:: groups
+
+ A tuple of :class:`.Group` objects encoding the
+ addresses and groups found in the header value. Addresses that are
+ not part of a group are represented in this list as single-address
+ ``Groups`` whose :attr:`~.Group.display_name` is ``None``.
+
+
+ .. attribute:: addresses
+
+ A tuple of :class:`.Address` objects encoding all
+ of the individual addresses from the header value. If the header value
+ contains any groups, the individual addresses from the group are included
+ in the list at the point where the group occurs in the value (that is,
+ the list of addresses is "flattened" into a one dimensional list).
+
+ The ``decoded`` value of the header will have all encoded words decoded to
+ unicode. :class:`~encodings.idna` encoded domain names are also decoded to unicode. The
+ ``decoded`` value is set by :attr:`~str.join`\ ing the :class:`str` value of
+ the elements of the ``groups`` attribute with ``', '``.
+
+ A list of :class:`.Address` and :class:`.Group` objects in any combination
+ may be used to set the value of an address header. ``Group`` objects whose
+ ``display_name`` is ``None`` will be interpreted as single addresses, which
+ allows an address list to be copied with groups intact by using the list
+ obtained ``groups`` attribute of the source header.
+
+
+.. class:: SingleAddressHeader
+
+ A subclass of :class:`.AddressHeader` that adds one
+ additional attribute:
+
+
+ .. attribute:: address
+
+ The single address encoded by the header value. If the header value
+ actually contains more than one address (which would be a violation of
+ the RFC under the default :mod:`policy`), accessing this attribute will
+ result in a :exc:`ValueError`.
+
+
+Many of the above classes also have a ``Unique`` variant (for example,
+``UniqueUnstructuredHeader``). The only difference is that in the ``Unique``
+variant, :attr:`~.BaseHeader.max_count` is set to 1.
+
+
+.. class:: MIMEVersionHeader
+
+ There is really only one valid value for the :mailheader:`MIME-Version`
+ header, and that is ``1.0``. For future proofing, this header class
+ supports other valid version numbers. If a version number has a valid value
+ per :rfc:`2045`, then the header object will have non-``None`` values for
+ the following attributes:
+
+ .. attribute:: version
+
+ The version number as a string, with any whitespace and/or comments
+ removed.
+
+ .. attribute:: major
+
+ The major version number as an integer
+
+ .. attribute:: minor
+
+ The minor version number as an integer
+
+
+.. class:: ParameterizedMIMEHeader
+
+ MOME headers all start with the prefix 'Content-'. Each specific header has
+ a certain value, described under the class for that header. Some can
+ also take a list of supplemental parameters, which have a common format.
+ This class serves as a base for all the MIME headers that take parameters.
+
+ .. attribute:: params
+
+ A dictionary mapping parameter names to parameter values.
+
+
+.. class:: ContentTypeHeader
+
+ A :class:`ParameterizedMIMEHheader` class that handles the
+ :mailheader:`Content-Type` header.
+
+ .. attribute:: content_type
+
+ The content type string, in the form ``maintype/subtype``.
+
+ .. attribute:: maintype
+
+ .. attribute:: subtype
+
+
+.. class:: ContentDispositionHeader
+
+ A :class:`ParameterizedMIMEHheader` class that handles the
+ :mailheader:`Content-Disposition` header.
+
+ .. attribute:: content-disposition
+
+ ``inline`` and ``attachment`` are the only valid values in common use.
+
+
+.. class:: ContentTransferEncoding
+
+ Handles the :mailheader:`Content-Transfer-Encoding` header.
+
+ .. attribute:: cte
+
+ Valid values are ``7bit``, ``8bit``, ``base64``, and
+ ``quoted-printable``. See :rfc:`2045` for more information.
+
+
+
+.. class:: HeaderRegistry(base_class=BaseHeader, \
+ default_class=UnstructuredHeader, \
+ use_default_map=True)
+
+ This is the factory used by :class:`~email.policy.EmailPolicy` by default.
+ ``HeaderRegistry`` builds the class used to create a header instance
+ dynamically, using *base_class* and a specialized class retrieved from a
+ registry that it holds. When a given header name does not appear in the
+ registry, the class specified by *default_class* is used as the specialized
+ class. When *use_default_map* is ``True`` (the default), the standard
+ mapping of header names to classes is copied in to the registry during
+ initialization. *base_class* is always the last class in the generated
+ class's ``__bases__`` list.
+
+ The default mappings are:
+
+ :subject: UniqueUnstructuredHeader
+ :date: UniqueDateHeader
+ :resent-date: DateHeader
+ :orig-date: UniqueDateHeader
+ :sender: UniqueSingleAddressHeader
+ :resent-sender: SingleAddressHeader
+ :to: UniqueAddressHeader
+ :resent-to: AddressHeader
+ :cc: UniqueAddressHeader
+ :resent-cc: AddressHeader
+ :from: UniqueAddressHeader
+ :resent-from: AddressHeader
+ :reply-to: UniqueAddressHeader
+
+ ``HeaderRegistry`` has the following methods:
+
+
+ .. method:: map_to_type(self, name, cls)
+
+ *name* is the name of the header to be mapped. It will be converted to
+ lower case in the registry. *cls* is the specialized class to be used,
+ along with *base_class*, to create the class used to instantiate headers
+ that match *name*.
+
+
+ .. method:: __getitem__(name)
+
+ Construct and return a class to handle creating a *name* header.
+
+
+ .. method:: __call__(name, value)
+
+ Retrieves the specialized header associated with *name* from the
+ registry (using *default_class* if *name* does not appear in the
+ registry) and composes it with *base_class* to produce a class,
+ calls the constructed class's constructor, passing it the same
+ argument list, and finally returns the class instance created thereby.
+
+
+The following classes are the classes used to represent data parsed from
+structured headers and can, in general, be used by an application program to
+construct structured values to assign to specific headers.
+
+
+.. class:: Address(display_name='', username='', domain='', addr_spec=None)
+
+ The class used to represent an email address. The general form of an
+ address is::
+
+ [display_name] <username@domain>
+
+ or::
+
+ username@domain
+
+ where each part must conform to specific syntax rules spelled out in
+ :rfc:`5322`.
+
+ As a convenience *addr_spec* can be specified instead of *username* and
+ *domain*, in which case *username* and *domain* will be parsed from the
+ *addr_spec*. An *addr_spec* must be a properly RFC quoted string; if it is
+ not ``Address`` will raise an error. Unicode characters are allowed and
+ will be property encoded when serialized. However, per the RFCs, unicode is
+ *not* allowed in the username portion of the address.
+
+ .. attribute:: display_name
+
+ The display name portion of the address, if any, with all quoting
+ removed. If the address does not have a display name, this attribute
+ will be an empty string.
+
+ .. attribute:: username
+
+ The ``username`` portion of the address, with all quoting removed.
+
+ .. attribute:: domain
+
+ The ``domain`` portion of the address.
+
+ .. attribute:: addr_spec
+
+ The ``username@domain`` portion of the address, correctly quoted
+ for use as a bare address (the second form shown above). This
+ attribute is not mutable.
+
+ .. method:: __str__()
+
+ The ``str`` value of the object is the address quoted according to
+ :rfc:`5322` rules, but with no Content Transfer Encoding of any non-ASCII
+ characters.
+
+ To support SMTP (:rfc:`5321`), ``Address`` handles one special case: if
+ ``username`` and ``domain`` are both the empty string (or ``None``), then
+ the string value of the ``Address`` is ``<>``.
+
+
+.. class:: Group(display_name=None, addresses=None)
+
+ The class used to represent an address group. The general form of an
+ address group is::
+
+ display_name: [address-list];
+
+ As a convenience for processing lists of addresses that consist of a mixture
+ of groups and single addresses, a ``Group`` may also be used to represent
+ single addresses that are not part of a group by setting *display_name* to
+ ``None`` and providing a list of the single address as *addresses*.
+
+ .. attribute:: display_name
+
+ The ``display_name`` of the group. If it is ``None`` and there is
+ exactly one ``Address`` in ``addresses``, then the ``Group`` represents a
+ single address that is not in a group.
+
+ .. attribute:: addresses
+
+ A possibly empty tuple of :class:`.Address` objects representing the
+ addresses in the group.
+
+ .. method:: __str__()
+
+ The ``str`` value of a ``Group`` is formatted according to :rfc:`5322`,
+ but with no Content Transfer Encoding of any non-ASCII characters. If
+ ``display_name`` is none and there is a single ``Address`` in the
+ ``addresses`` list, the ``str`` value will be the same as the ``str`` of
+ that single ``Address``.
diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst
index f685e54..59ab47d 100644
--- a/Doc/library/email.message.rst
+++ b/Doc/library/email.message.rst
@@ -111,10 +111,14 @@ Here are the methods of the :class:`Message` class:
header. When ``True`` and the message is not a multipart, the payload will
be decoded if this header's value is ``quoted-printable`` or ``base64``.
If some other encoding is used, or :mailheader:`Content-Transfer-Encoding`
- header is missing, or if the payload has bogus base64 data, the payload is
+ header is missing, the payload is
returned as-is (undecoded). In all cases the returned value is binary
data. If the message is a multipart and the *decode* flag is ``True``,
- then ``None`` is returned.
+ then ``None`` is returned. If the payload is base64 and it was not
+ perfectly formed (missing padding, characters outside the base64
+ alphabet), then an appropriate defect will be added to the message's
+ defect property (:class:`~email.errors.InvalidBase64PaddingDefect` or
+ :class:`~email.errors.InvalidBase64CharactersDefect`, respectively).
When *decode* is ``False`` (the default) the body is returned as a string
without decoding the :mailheader:`Content-Transfer-Encoding`. However,
diff --git a/Doc/library/email.mime.rst b/Doc/library/email.mime.rst
index ae340f7..db5584c 100644
--- a/Doc/library/email.mime.rst
+++ b/Doc/library/email.mime.rst
@@ -175,7 +175,7 @@ Here are the classes:
.. currentmodule:: email.mime.text
-.. class:: MIMEText(_text, _subtype='plain', _charset='us-ascii')
+.. class:: MIMEText(_text, _subtype='plain', _charset=None)
Module: :mod:`email.mime.text`
@@ -185,5 +185,5 @@ Here are the classes:
minor type and defaults to :mimetype:`plain`. *_charset* is the character
set of the text and is passed as a parameter to the
:class:`~email.mime.nonmultipart.MIMENonMultipart` constructor; it defaults
- to ``us-ascii``. No guessing or encoding is performed on the text data.
-
+ to ``us-ascii`` if the string contains only ``ascii`` codepoints, and
+ ``utf-8`` otherwise.
diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst
index 49a59c0..54e5521 100644
--- a/Doc/library/email.parser.rst
+++ b/Doc/library/email.parser.rst
@@ -58,12 +58,18 @@ list of defects that it can find.
Here is the API for the :class:`FeedParser`:
-.. class:: FeedParser(_factory=email.message.Message)
+.. class:: FeedParser(_factory=email.message.Message, *, policy=policy.default)
Create a :class:`FeedParser` instance. Optional *_factory* is a no-argument
callable that will be called whenever a new message object is needed. It
defaults to the :class:`email.message.Message` class.
+ The *policy* keyword specifies a :mod:`~email.policy` object that controls a
+ number of aspects of the parser's operation. The default policy maintains
+ backward compatibility.
+
+ .. versionchanged:: 3.3 Added the *policy* keyword.
+
.. method:: feed(data)
Feed the :class:`FeedParser` some more data. *data* should be a string
@@ -94,15 +100,18 @@ Parser class API
The :class:`Parser` class, imported from the :mod:`email.parser` module,
provides an API that can be used to parse a message when the complete contents
of the message are available in a string or file. The :mod:`email.parser`
-module also provides a second class, called :class:`HeaderParser` which can be
-used if you're only interested in the headers of the message.
-:class:`HeaderParser` can be much faster in these situations, since it does not
-attempt to parse the message body, instead setting the payload to the raw body
-as a string. :class:`HeaderParser` has the same API as the :class:`Parser`
-class.
+module also provides header-only parsers, called :class:`HeaderParser` and
+:class:`BytesHeaderParser`, which can be used if you're only interested in the
+headers of the message. :class:`HeaderParser` and :class:`BytesHeaderParser`
+can be much faster in these situations, since they do not attempt to parse the
+message body, instead setting the payload to the raw body as a string. They
+have the same API as the :class:`Parser` and :class:`BytesParser` classes.
+.. versionadded:: 3.3
+ The BytesHeaderParser class.
-.. class:: Parser(_class=email.message.Message, strict=None)
+
+.. class:: Parser(_class=email.message.Message, *, policy=policy.default)
The constructor for the :class:`Parser` class takes an optional argument
*_class*. This must be a callable factory (such as a function or a class), and
@@ -110,13 +119,13 @@ class.
:class:`~email.message.Message` (see :mod:`email.message`). The factory will
be called without arguments.
- The optional *strict* flag is ignored.
+ The *policy* keyword specifies a :mod:`~email.policy` object that controls a
+ number of aspects of the parser's operation. The default policy maintains
+ backward compatibility.
- .. deprecated:: 2.4
- Because the :class:`Parser` class is a backward compatible API wrapper
- around the new-in-Python 2.4 :class:`FeedParser`, *all* parsing is
- effectively non-strict. You should simply stop passing a *strict* flag to
- the :class:`Parser` constructor.
+ .. versionchanged:: 3.3
+ Removed the *strict* argument that was deprecated in 2.4. Added the
+ *policy* keyword.
The other public :class:`Parser` methods are:
@@ -147,12 +156,18 @@ class.
Optional *headersonly* is as with the :meth:`parse` method.
-.. class:: BytesParser(_class=email.message.Message, strict=None)
+.. class:: BytesParser(_class=email.message.Message, *, policy=policy.default)
This class is exactly parallel to :class:`Parser`, but handles bytes input.
The *_class* and *strict* arguments are interpreted in the same way as for
- the :class:`Parser` constructor. *strict* is supported only to make porting
- code easier; it is deprecated.
+ the :class:`Parser` constructor.
+
+ The *policy* keyword specifies a :mod:`~email.policy` object that
+ controls a number of aspects of the parser's operation. The default
+ policy maintains backward compatibility.
+
+ .. versionchanged:: 3.3
+ Removed the *strict* argument. Added the *policy* keyword.
.. method:: parse(fp, headeronly=False)
@@ -190,34 +205,48 @@ in the top-level :mod:`email` package namespace.
.. currentmodule:: email
-.. function:: message_from_string(s, _class=email.message.Message, strict=None)
+.. function:: message_from_string(s, _class=email.message.Message, *, \
+ policy=policy.default)
Return a message object structure from a string. This is exactly equivalent to
- ``Parser().parsestr(s)``. Optional *_class* and *strict* are interpreted as
+ ``Parser().parsestr(s)``. *_class* and *policy* are interpreted as
with the :class:`Parser` class constructor.
-.. function:: message_from_bytes(s, _class=email.message.Message, strict=None)
+ .. versionchanged:: 3.3
+ Removed the *strict* argument. Added the *policy* keyword.
+
+.. function:: message_from_bytes(s, _class=email.message.Message, *, \
+ policy=policy.default)
Return a message object structure from a byte string. This is exactly
equivalent to ``BytesParser().parsebytes(s)``. Optional *_class* and
*strict* are interpreted as with the :class:`Parser` class constructor.
.. versionadded:: 3.2
+ .. versionchanged:: 3.3
+ Removed the *strict* argument. Added the *policy* keyword.
-.. function:: message_from_file(fp, _class=email.message.Message, strict=None)
+.. function:: message_from_file(fp, _class=email.message.Message, *, \
+ policy=policy.default)
Return a message object structure tree from an open :term:`file object`.
- This is exactly equivalent to ``Parser().parse(fp)``. Optional *_class*
- and *strict* are interpreted as with the :class:`Parser` class constructor.
+ This is exactly equivalent to ``Parser().parse(fp)``. *_class*
+ and *policy* are interpreted as with the :class:`Parser` class constructor.
+
+ .. versionchanged::
+ Removed the *strict* argument. Added the *policy* keyword.
-.. function:: message_from_binary_file(fp, _class=email.message.Message, strict=None)
+.. function:: message_from_binary_file(fp, _class=email.message.Message, *, \
+ policy=policy.default)
Return a message object structure tree from an open binary :term:`file
object`. This is exactly equivalent to ``BytesParser().parse(fp)``.
- Optional *_class* and *strict* are interpreted as with the :class:`Parser`
+ *_class* and *policy* are interpreted as with the :class:`Parser`
class constructor.
.. versionadded:: 3.2
+ .. versionchanged:: 3.3
+ Removed the *strict* argument. Added the *policy* keyword.
Here's an example of how you might use this at an interactive Python prompt::
diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst
new file mode 100644
index 0000000..31b13c1
--- /dev/null
+++ b/Doc/library/email.policy.rst
@@ -0,0 +1,497 @@
+:mod:`email.policy`: Policy Objects
+-----------------------------------
+
+.. module:: email.policy
+ :synopsis: Controlling the parsing and generating of messages
+
+.. moduleauthor:: R. David Murray <rdmurray@bitdance.com>
+.. sectionauthor:: R. David Murray <rdmurray@bitdance.com>
+
+.. versionadded:: 3.3
+
+
+The :mod:`email` package's prime focus is the handling of email messages as
+described by the various email and MIME RFCs. However, the general format of
+email messages (a block of header fields each consisting of a name followed by
+a colon followed by a value, the whole block followed by a blank line and an
+arbitrary 'body'), is a format that has found utility outside of the realm of
+email. Some of these uses conform fairly closely to the main RFCs, some do
+not. And even when working with email, there are times when it is desirable to
+break strict compliance with the RFCs.
+
+Policy objects give the email package the flexibility to handle all these
+disparate use cases.
+
+A :class:`Policy` object encapsulates a set of attributes and methods that
+control the behavior of various components of the email package during use.
+:class:`Policy` instances can be passed to various classes and methods in the
+email package to alter the default behavior. The settable values and their
+defaults are described below.
+
+There is a default policy used by all classes in the email package. This
+policy is named :class:`Compat32`, with a corresponding pre-defined instance
+named :const:`compat32`. It provides for complete backward compatibility (in
+some cases, including bug compatibility) with the pre-Python3.3 version of the
+email package.
+
+The first part of this documentation covers the features of :class:`Policy`, an
+:term:`abstract base class` that defines the features that are common to all
+policy objects, including :const:`compat32`. This includes certain hook
+methods that are called internally by the email package, which a custom policy
+could override to obtain different behavior.
+
+When a :class:`~email.message.Message` object is created, it acquires a policy.
+By default this will be :const:`compat32`, but a different policy can be
+specified. If the ``Message`` is created by a :mod:`~email.parser`, a policy
+passed to the parser will be the policy used by the ``Message`` it creates. If
+the ``Message`` is created by the program, then the policy can be specified
+when it is created. When a ``Message`` is passed to a :mod:`~email.generator`,
+the generator uses the policy from the ``Message`` by default, but you can also
+pass a specific policy to the generator that will override the one stored on
+the ``Message`` object.
+
+:class:`Policy` instances are immutable, but they can be cloned, accepting the
+same keyword arguments as the class constructor and returning a new
+:class:`Policy` instance that is a copy of the original but with the specified
+attributes values changed.
+
+As an example, the following code could be used to read an email message from a
+file on disk and pass it to the system ``sendmail`` program on a Unix system::
+
+ >>> from email import msg_from_binary_file
+ >>> from email.generator import BytesGenerator
+ >>> from subprocess import Popen, PIPE
+ >>> with open('mymsg.txt', 'b') as f:
+ ... msg = msg_from_binary_file(f)
+ >>> p = Popen(['sendmail', msg['To'][0].address], stdin=PIPE)
+ >>> g = BytesGenerator(p.stdin, policy=msg.policy.clone(linesep='\r\n'))
+ >>> g.flatten(msg)
+ >>> p.stdin.close()
+ >>> rc = p.wait()
+
+Here we are telling :class:`~email.generator.BytesGenerator` to use the RFC
+correct line separator characters when creating the binary string to feed into
+``sendmail's`` ``stdin``, where the default policy would use ``\n`` line
+separators.
+
+Some email package methods accept a *policy* keyword argument, allowing the
+policy to be overridden for that method. For example, the following code uses
+the :meth:`~email.message.Message.as_string` method of the *msg* object from
+the previous example and writes the message to a file using the native line
+separators for the platform on which it is running::
+
+ >>> import os
+ >>> with open('converted.txt', 'wb') as f:
+ ... f.write(msg.as_string(policy=msg.policy.clone(linesep=os.linesep))
+
+Policy objects can also be combined using the addition operator, producing a
+policy object whose settings are a combination of the non-default values of the
+summed objects::
+
+ >>> compat_SMTP = email.policy.clone(linesep='\r\n')
+ >>> compat_strict = email.policy.clone(raise_on_defect=True)
+ >>> compat_strict_SMTP = compat_SMTP + compat_strict
+
+This operation is not commutative; that is, the order in which the objects are
+added matters. To illustrate::
+
+ >>> policy100 = compat32.clone(max_line_length=100)
+ >>> policy80 = compat32.clone(max_line_length=80)
+ >>> apolicy = policy100 + Policy80
+ >>> apolicy.max_line_length
+ 80
+ >>> apolicy = policy80 + policy100
+ >>> apolicy.max_line_length
+ 100
+
+
+.. class:: Policy(**kw)
+
+ This is the :term:`abstract base class` for all policy classes. It provides
+ default implementations for a couple of trivial methods, as well as the
+ implementation of the immutability property, the :meth:`clone` method, and
+ the constructor semantics.
+
+ The constructor of a policy class can be passed various keyword arguments.
+ The arguments that may be specified are any non-method properties on this
+ class, plus any additional non-method properties on the concrete class. A
+ value specified in the constructor will override the default value for the
+ corresponding attribute.
+
+ This class defines the following properties, and thus values for the
+ following may be passed in the constructor of any policy class:
+
+ .. attribute:: max_line_length
+
+ The maximum length of any line in the serialized output, not counting the
+ end of line character(s). Default is 78, per :rfc:`5322`. A value of
+ ``0`` or :const:`None` indicates that no line wrapping should be
+ done at all.
+
+ .. attribute:: linesep
+
+ The string to be used to terminate lines in serialized output. The
+ default is ``\n`` because that's the internal end-of-line discipline used
+ by Python, though ``\r\n`` is required by the RFCs.
+
+ .. attribute:: cte_type
+
+ Controls the type of Content Transfer Encodings that may be or are
+ required to be used. The possible values are:
+
+ ======== ===============================================================
+ ``7bit`` all data must be "7 bit clean" (ASCII-only). This means that
+ where necessary data will be encoded using either
+ quoted-printable or base64 encoding.
+
+ ``8bit`` data is not constrained to be 7 bit clean. Data in headers is
+ still required to be ASCII-only and so will be encoded (see
+ 'binary_fold' below for an exception), but body parts may use
+ the ``8bit`` CTE.
+ ======== ===============================================================
+
+ A ``cte_type`` value of ``8bit`` only works with ``BytesGenerator``, not
+ ``Generator``, because strings cannot contain binary data. If a
+ ``Generator`` is operating under a policy that specifies
+ ``cte_type=8bit``, it will act as if ``cte_type`` is ``7bit``.
+
+ .. attribute:: raise_on_defect
+
+ If :const:`True`, any defects encountered will be raised as errors. If
+ :const:`False` (the default), defects will be passed to the
+ :meth:`register_defect` method.
+
+ The following :class:`Policy` method is intended to be called by code using
+ the email library to create policy instances with custom settings:
+
+ .. method:: clone(**kw)
+
+ Return a new :class:`Policy` instance whose attributes have the same
+ values as the current instance, except where those attributes are
+ given new values by the keyword arguments.
+
+ The remaining :class:`Policy` methods are called by the email package code,
+ and are not intended to be called by an application using the email package.
+ A custom policy must implement all of these methods.
+
+ .. method:: handle_defect(obj, defect)
+
+ Handle a *defect* found on *obj*. When the email package calls this
+ method, *defect* will always be a subclass of
+ :class:`~email.errors.Defect`.
+
+ The default implementation checks the :attr:`raise_on_defect` flag. If
+ it is ``True``, *defect* is raised as an exception. If it is ``False``
+ (the default), *obj* and *defect* are passed to :meth:`register_defect`.
+
+ .. method:: register_defect(obj, defect)
+
+ Register a *defect* on *obj*. In the email package, *defect* will always
+ be a subclass of :class:`~email.errors.Defect`.
+
+ The default implementation calls the ``append`` method of the ``defects``
+ attribute of *obj*. When the email package calls :attr:`handle_defect`,
+ *obj* will normally have a ``defects`` attribute that has an ``append``
+ method. Custom object types used with the email package (for example,
+ custom ``Message`` objects) should also provide such an attribute,
+ otherwise defects in parsed messages will raise unexpected errors.
+
+ .. method:: header_max_count(name)
+
+ Return the maximum allowed number of headers named *name*.
+
+ Called when a header is added to a :class:`~email.message.Message`
+ object. If the returned value is not ``0`` or ``None``, and there are
+ already a number of headers with the name *name* equal to the value
+ returned, a :exc:`ValueError` is raised.
+
+ Because the default behavior of ``Message.__setitem__`` is to append the
+ value to the list of headers, it is easy to create duplicate headers
+ without realizing it. This method allows certain headers to be limited
+ in the number of instances of that header that may be added to a
+ ``Message`` programmatically. (The limit is not observed by the parser,
+ which will faithfully produce as many headers as exist in the message
+ being parsed.)
+
+ The default implementation returns ``None`` for all header names.
+
+ .. method:: header_source_parse(sourcelines)
+
+ The email package calls this method with a list of strings, each string
+ ending with the line separation characters found in the source being
+ parsed. The first line includes the field header name and separator.
+ All whitespace in the source is preserved. The method should return the
+ ``(name, value)`` tuple that is to be stored in the ``Message`` to
+ represent the parsed header.
+
+ If an implementation wishes to retain compatibility with the existing
+ email package policies, *name* should be the case preserved name (all
+ characters up to the '``:``' separator), while *value* should be the
+ unfolded value (all line separator characters removed, but whitespace
+ kept intact), stripped of leading whitespace.
+
+ *sourcelines* may contain surrogateescaped binary data.
+
+ There is no default implementation
+
+ .. method:: header_store_parse(name, value)
+
+ The email package calls this method with the name and value provided by
+ the application program when the application program is modifying a
+ ``Message`` programmatically (as opposed to a ``Message`` created by a
+ parser). The method should return the ``(name, value)`` tuple that is to
+ be stored in the ``Message`` to represent the header.
+
+ If an implementation wishes to retain compatibility with the existing
+ email package policies, the *name* and *value* should be strings or
+ string subclasses that do not change the content of the passed in
+ arguments.
+
+ There is no default implementation
+
+ .. method:: header_fetch_parse(name, value)
+
+ The email package calls this method with the *name* and *value* currently
+ stored in the ``Message`` when that header is requested by the
+ application program, and whatever the method returns is what is passed
+ back to the application as the value of the header being retrieved.
+ Note that there may be more than one header with the same name stored in
+ the ``Message``; the method is passed the specific name and value of the
+ header destined to be returned to the application.
+
+ *value* may contain surrogateescaped binary data. There should be no
+ surrogateescaped binary data in the value returned by the method.
+
+ There is no default implementation
+
+ .. method:: fold(name, value)
+
+ The email package calls this method with the *name* and *value* currently
+ stored in the ``Message`` for a given header. The method should return a
+ string that represents that header "folded" correctly (according to the
+ policy settings) by composing the *name* with the *value* and inserting
+ :attr:`linesep` characters at the appropriate places. See :rfc:`5322`
+ for a discussion of the rules for folding email headers.
+
+ *value* may contain surrogateescaped binary data. There should be no
+ surrogateescaped binary data in the string returned by the method.
+
+ .. method:: fold_binary(name, value)
+
+ The same as :meth:`fold`, except that the returned value should be a
+ bytes object rather than a string.
+
+ *value* may contain surrogateescaped binary data. These could be
+ converted back into binary data in the returned bytes object.
+
+
+.. class:: Compat32(**kw)
+
+ This concrete :class:`Policy` is the backward compatibility policy. It
+ replicates the behavior of the email package in Python 3.2. The
+ :mod:`policy` module also defines an instance of this class,
+ :const:`compat32`, that is used as the default policy. Thus the default
+ behavior of the email package is to maintain compatibility with Python 3.2.
+
+ The class provides the following concrete implementations of the
+ abstract methods of :class:`Policy`:
+
+ .. method:: header_source_parse(sourcelines)
+
+ The name is parsed as everything up to the '``:``' and returned
+ unmodified. The value is determined by stripping leading whitespace off
+ the remainder of the first line, joining all subsequent lines together,
+ and stripping any trailing carriage return or linefeed characters.
+
+ .. method:: header_store_parse(name, value)
+
+ The name and value are returned unmodified.
+
+ .. method:: header_fetch_parse(name, value)
+
+ If the value contains binary data, it is converted into a
+ :class:`~email.header.Header` object using the ``unknown-8bit`` charset.
+ Otherwise it is returned unmodified.
+
+ .. method:: fold(name, value)
+
+ Headers are folded using the :class:`~email.header.Header` folding
+ algorithm, which preserves existing line breaks in the value, and wraps
+ each resulting line to the ``max_line_length``. Non-ASCII binary data are
+ CTE encoded using the ``unknown-8bit`` charset.
+
+ .. method:: fold_binary(name, value)
+
+ Headers are folded using the :class:`~email.header.Header` folding
+ algorithm, which preserves existing line breaks in the value, and wraps
+ each resulting line to the ``max_line_length``. If ``cte_type`` is
+ ``7bit``, non-ascii binary data is CTE encoded using the ``unknown-8bit``
+ charset. Otherwise the original source header is used, with its existing
+ line breaks and and any (RFC invalid) binary data it may contain.
+
+
+.. note::
+
+ The documentation below describes new policies that are included in the
+ standard library on a :term:`provisional basis <provisional package>`.
+ Backwards incompatible changes (up to and including removal of the feature)
+ may occur if deemed necessary by the core developers.
+
+
+.. class:: EmailPolicy(**kw)
+
+ This concrete :class:`Policy` provides behavior that is intended to be fully
+ compliant with the current email RFCs. These include (but are not limited
+ to) :rfc:`5322`, :rfc:`2047`, and the current MIME RFCs.
+
+ This policy adds new header parsing and folding algorithms. Instead of
+ simple strings, headers are custom objects with custom attributes depending
+ on the type of the field. The parsing and folding algorithm fully implement
+ :rfc:`2047` and :rfc:`5322`.
+
+ In addition to the settable attributes listed above that apply to all
+ policies, this policy adds the following additional attributes:
+
+ .. attribute:: refold_source
+
+ If the value for a header in the ``Message`` object originated from a
+ :mod:`~email.parser` (as opposed to being set by a program), this
+ attribute indicates whether or not a generator should refold that value
+ when transforming the message back into stream form. The possible values
+ are:
+
+ ======== ===============================================================
+ ``none`` all source values use original folding
+
+ ``long`` source values that have any line that is longer than
+ ``max_line_length`` will be refolded
+
+ ``all`` all values are refolded.
+ ======== ===============================================================
+
+ The default is ``long``.
+
+ .. attribute:: header_factory
+
+ A callable that takes two arguments, ``name`` and ``value``, where
+ ``name`` is a header field name and ``value`` is an unfolded header field
+ value, and returns a string subclass that represents that header. A
+ default ``header_factory`` (see :mod:`~email.headerregistry`) is provided
+ that understands some of the :RFC:`5322` header field types. (Currently
+ address fields and date fields have special treatment, while all other
+ fields are treated as unstructured. This list will be completed before
+ the extension is marked stable.)
+
+ The class provides the following concrete implementations of the abstract
+ methods of :class:`Policy`:
+
+ .. method:: header_max_count(name)
+
+ Returns the value of the
+ :attr:`~email.headerregistry.BaseHeader.max_count` attribute of the
+ specialized class used to represent the header with the given name.
+
+ .. method:: header_source_parse(sourcelines)
+
+ The implementation of this method is the same as that for the
+ :class:`Compat32` policy.
+
+ .. method:: header_store_parse(name, value)
+
+ The name is returned unchanged. If the input value has a ``name``
+ attribute and it matches *name* ignoring case, the value is returned
+ unchanged. Otherwise the *name* and *value* are passed to
+ ``header_factory``, and the resulting custom header object is returned as
+ the value. In this case a ``ValueError`` is raised if the input value
+ contains CR or LF characters.
+
+ .. method:: header_fetch_parse(name, value)
+
+ If the value has a ``name`` attribute, it is returned to unmodified.
+ Otherwise the *name*, and the *value* with any CR or LF characters
+ removed, are passed to the ``header_factory``, and the resulting custom
+ header object is returned. Any surrogateescaped bytes get turned into
+ the unicode unknown-character glyph.
+
+ .. method:: fold(name, value)
+
+ Header folding is controlled by the :attr:`refold_source` policy setting.
+ A value is considered to be a 'source value' if and only if it does not
+ have a ``name`` attribute (having a ``name`` attribute means it is a
+ header object of some sort). If a source value needs to be refolded
+ according to the policy, it is converted into a custom header object by
+ passing the *name* and the *value* with any CR and LF characters removed
+ to the ``header_factory``. Folding of a custom header object is done by
+ calling its ``fold`` method with the current policy.
+
+ Source values are split into lines using :meth:`~str.splitlines`. If
+ the value is not to be refolded, the lines are rejoined using the
+ ``linesep`` from the policy and returned. The exception is lines
+ containing non-ascii binary data. In that case the value is refolded
+ regardless of the ``refold_source`` setting, which causes the binary data
+ to be CTE encoded using the ``unknown-8bit`` charset.
+
+ .. method:: fold_binary(name, value)
+
+ The same as :meth:`fold` if :attr:`cte_type` is ``7bit``, except that
+ the returned value is bytes.
+
+ If :attr:`cte_type` is ``8bit``, non-ASCII binary data is converted back
+ into bytes. Headers with binary data are not refolded, regardless of the
+ ``refold_header`` setting, since there is no way to know whether the
+ binary data consists of single byte characters or multibyte characters.
+
+The following instances of :class:`EmailPolicy` provide defaults suitable for
+specific application domains. Note that in the future the behavior of these
+instances (in particular the ``HTTP`` instance) may be adjusted to conform even
+more closely to the RFCs relevant to their domains.
+
+.. data:: default
+
+ An instance of ``EmailPolicy`` with all defaults unchanged. This policy
+ uses the standard Python ``\n`` line endings rather than the RFC-correct
+ ``\r\n``.
+
+.. data:: SMTP
+
+ Suitable for serializing messages in conformance with the email RFCs.
+ Like ``default``, but with ``linesep`` set to ``\r\n``, which is RFC
+ compliant.
+
+.. data:: HTTP
+
+ Suitable for serializing headers with for use in HTTP traffic. Like
+ ``SMTP`` except that ``max_line_length`` is set to ``None`` (unlimited).
+
+.. data:: strict
+
+ Convenience instance. The same as ``default`` except that
+ ``raise_on_defect`` is set to ``True``. This allows any policy to be made
+ strict by writing::
+
+ somepolicy + policy.strict
+
+With all of these :class:`EmailPolicies <.EmailPolicy>`, the effective API of
+the email package is changed from the Python 3.2 API in the following ways:
+
+ * Setting a header on a :class:`~email.message.Message` results in that
+ header being parsed and a custom header object created.
+
+ * Fetching a header value from a :class:`~email.message.Message` results
+ in that header being parsed and a custom header object created and
+ returned.
+
+ * Any custom header object, or any header that is refolded due to the
+ policy settings, is folded using an algorithm that fully implements the
+ RFC folding algorithms, including knowing where encoded words are required
+ and allowed.
+
+From the application view, this means that any header obtained through the
+:class:`~email.message.Message` is a custom header object with custom
+attributes, whose string value is the fully decoded unicode value of the
+header. Likewise, a header may be assigned a new value, or a new header
+created, using a unicode string, and the policy will take care of converting
+the unicode string into the correct RFC encoded form.
+
+The custom header objects and their attributes are described in
+:mod:`~email.headerregistry`.
diff --git a/Doc/library/email.rst b/Doc/library/email.rst
index 4530b95..ef5354f 100644
--- a/Doc/library/email.rst
+++ b/Doc/library/email.rst
@@ -51,6 +51,8 @@ Contents of the :mod:`email` package documentation:
email.message.rst
email.parser.rst
email.generator.rst
+ email.policy.rst
+ email.headerregistry.rst
email.mime.rst
email.header.rst
email.charset.rst
diff --git a/Doc/library/email.util.rst b/Doc/library/email.util.rst
index 11bf3b2..bad0b24 100644
--- a/Doc/library/email.util.rst
+++ b/Doc/library/email.util.rst
@@ -29,13 +29,21 @@ There are several useful utilities provided in the :mod:`email.utils` module:
fails, in which case a 2-tuple of ``('', '')`` is returned.
-.. function:: formataddr(pair)
+.. function:: formataddr(pair, charset='utf-8')
The inverse of :meth:`parseaddr`, this takes a 2-tuple of the form ``(realname,
email_address)`` and returns the string value suitable for a :mailheader:`To` or
:mailheader:`Cc` header. If the first element of *pair* is false, then the
second element is returned unmodified.
+ Optional *charset* is the character set that will be used in the :rfc:`2047`
+ encoding of the ``realname`` if the ``realname`` contains non-ASCII
+ characters. Can be an instance of :class:`str` or a
+ :class:`~email.charset.Charset`. Defaults to ``utf-8``.
+
+ .. versionchanged:: 3.3
+ Added the *charset* option.
+
.. function:: getaddresses(fieldvalues)
@@ -74,6 +82,20 @@ There are several useful utilities provided in the :mod:`email.utils` module:
indexes 6, 7, and 8 of the result tuple are not usable.
+.. function:: parsedate_to_datetime(date)
+
+ The inverse of :func:`format_datetime`. Performs the same function as
+ :func:`parsedate`, but on success returns a :mod:`~datetime.datetime`. If
+ the input date has a timezone of ``-0000``, the ``datetime`` will be a naive
+ ``datetime``, and if the date is conforming to the RFCs it will represent a
+ time in UTC but with no indication of the actual source timezone of the
+ message the date comes from. If the input date has any other valid timezone
+ offset, the ``datetime`` will be an aware ``datetime`` with the
+ corresponding a :class:`~datetime.timezone` :class:`~datetime.tzinfo`.
+
+ .. versionadded:: 3.3
+
+
.. function:: mktime_tz(tuple)
Turn a 10-tuple as returned by :func:`parsedate_tz` into a UTC timestamp. It
@@ -105,6 +127,36 @@ There are several useful utilities provided in the :mod:`email.utils` module:
``False``. The default is ``False``.
+.. function:: format_datetime(dt, usegmt=False)
+
+ Like ``formatdate``, but the input is a :mod:`datetime` instance. If it is
+ a naive datetime, it is assumed to be "UTC with no information about the
+ source timezone", and the conventional ``-0000`` is used for the timezone.
+ If it is an aware ``datetime``, then the numeric timezone offset is used.
+ If it is an aware timezone with offset zero, then *usegmt* may be set to
+ ``True``, in which case the string ``GMT`` is used instead of the numeric
+ timezone offset. This provides a way to generate standards conformant HTTP
+ date headers.
+
+ .. versionadded:: 3.3
+
+
+.. function:: localtime(dt=None)
+
+ Return local time as an aware datetime object. If called without
+ arguments, return current time. Otherwise *dt* argument should be a
+ :class:`~datetime.datetime` instance, and it is converted to the local time
+ zone according to the system time zone database. If *dt* is naive (that
+ is, ``dt.tzinfo`` is ``None``), it is assumed to be in local time. In this
+ case, a positive or zero value for *isdst* causes ``localtime`` to presume
+ initially that summer time (for example, Daylight Saving Time) is or is not
+ (respectively) in effect for the specified time. A negative value for
+ *isdst* causes the ``localtime`` to attempt to divine whether summer time
+ is in effect for the specified time.
+
+ .. versionadded:: 3.3
+
+
.. function:: make_msgid(idstring=None, domain=None)
Returns a string suitable for an :rfc:`2822`\ -compliant
@@ -115,7 +167,8 @@ There are several useful utilities provided in the :mod:`email.utils` module:
may be useful certain cases, such as a constructing distributed system that
uses a consistent domain name across multiple hosts.
- .. versionchanged:: 3.2 domain keyword added
+ .. versionchanged:: 3.2
+ Added the *domain* keyword.
.. function:: decode_rfc2231(s)
diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst
index ca3ad3e..6ba58d4 100644
--- a/Doc/library/exceptions.rst
+++ b/Doc/library/exceptions.rst
@@ -34,6 +34,27 @@ programmers are encouraged to at least derive new exceptions from the
defining exceptions is available in the Python Tutorial under
:ref:`tut-userexceptions`.
+When raising (or re-raising) an exception in an :keyword:`except` clause
+:attr:`__context__` is automatically set to the last exception caught; if the
+new exception is not handled the traceback that is eventually displayed will
+include the originating exception(s) and the final exception.
+
+This implicit exception chain can be made explicit by using :keyword:`from` with
+:keyword:`raise`. The single argument to :keyword:`from` must be an exception
+or ``None``. It will be set as :attr:`__cause__` on the raised exception.
+Setting :attr:`__cause__` implicitly sets the :attr:`__suppress_context__` to
+``True``. If :attr:`__cause__` is an exception, it will be displayed. If
+:attr:`__cause__` is present or :attr:`__suppress_context__` has a true value,
+:attr:`__context__` will not be displayed.
+
+In either case, the default exception handling code will not display any of the
+remaining links in the :attr:`__context__` chain if :attr:`__cause__` has been
+set.
+
+
+Base classes
+------------
+
The following exceptions are used mostly as base classes for other exceptions.
.. exception:: BaseException
@@ -90,27 +111,8 @@ The following exceptions are used mostly as base classes for other exceptions.
can be raised directly by :func:`codecs.lookup`.
-.. exception:: EnvironmentError
-
- The base class for exceptions that can occur outside the Python system:
- :exc:`IOError`, :exc:`OSError`. When exceptions of this type are created with a
- 2-tuple, the first item is available on the instance's :attr:`errno` attribute
- (it is assumed to be an error number), and the second item is available on the
- :attr:`strerror` attribute (it is usually the associated error message). The
- tuple itself is also available on the :attr:`args` attribute.
-
- When an :exc:`EnvironmentError` exception is instantiated with a 3-tuple, the
- first two items are available as above, while the third item is available on the
- :attr:`filename` attribute. However, for backwards compatibility, the
- :attr:`args` attribute contains only a 2-tuple of the first two constructor
- arguments.
-
- The :attr:`filename` attribute is ``None`` when this exception is created with
- other than 3 arguments. The :attr:`errno` and :attr:`strerror` attributes are
- also ``None`` when the instance was created with other than 2 or 3 arguments.
- In this last case, :attr:`args` contains the verbatim constructor arguments as a
- tuple.
-
+Concrete exceptions
+-------------------
The following exceptions are the exceptions that are usually raised.
@@ -151,21 +153,19 @@ The following exceptions are the exceptions that are usually raised.
it is technically not an error.
-.. exception:: IOError
-
- Raised when an I/O operation (such as the built-in :func:`print` or
- :func:`open` functions or a method of a :term:`file object`) fails for an
- I/O-related reason, e.g., "file not found" or "disk full".
-
- This class is derived from :exc:`EnvironmentError`. See the discussion above
- for more information on exception instance attributes.
-
-
.. exception:: ImportError
Raised when an :keyword:`import` statement fails to find the module definition
or when a ``from ... import`` fails to find a name that is to be imported.
+ The :attr:`name` and :attr:`path` attributes can be set using keyword-only
+ arguments to the constructor. When set they represent the name of the module
+ that was attempted to be imported and the path to any file which triggered
+ the exception, respectively.
+
+ .. versionchanged:: 3.3
+ Added the :attr:`name` and :attr:`path` attributes.
+
.. exception:: IndexError
@@ -221,17 +221,30 @@ The following exceptions are the exceptions that are usually raised.
.. index:: module: errno
- This exception is derived from :exc:`EnvironmentError`. It is raised when a
- function returns a system-related error (not for illegal argument types or
- other incidental errors). The :attr:`errno` attribute is a numeric error
- code from :c:data:`errno`, and the :attr:`strerror` attribute is the
- corresponding string, as would be printed by the C function :c:func:`perror`.
- See the module :mod:`errno`, which contains names for the error codes defined
- by the underlying operating system.
+ This exception is raised when a system function returns a system-related
+ error, including I/O failures such as "file not found" or "disk full"
+ (not for illegal argument types or other incidental errors). Often a
+ subclass of :exc:`OSError` will actually be raised as described in
+ `OS exceptions`_ below. The :attr:`errno` attribute is a numeric error
+ code from the C variable :c:data:`errno`.
+
+ Under Windows, the :attr:`winerror` attribute gives you the native
+ Windows error code. The :attr:`errno` attribute is then an approximate
+ translation, in POSIX terms, of that native error code.
+
+ Under all platforms, the :attr:`strerror` attribute is the corresponding
+ error message as provided by the operating system (as formatted by the C
+ functions :c:func:`perror` under POSIX, and :c:func:`FormatMessage`
+ Windows).
- For exceptions that involve a file system path (such as :func:`chdir` or
- :func:`unlink`), the exception instance will contain a third attribute,
- :attr:`filename`, which is the file name passed to the function.
+ For exceptions that involve a file system path (such as :func:`open` or
+ :func:`os.unlink`), the exception instance will contain an additional
+ attribute, :attr:`filename`, which is the file name passed to the function.
+
+ .. versionchanged:: 3.3
+ :exc:`EnvironmentError`, :exc:`IOError`, :exc:`WindowsError`,
+ :exc:`VMSError`, :exc:`socket.error`, :exc:`select.error` and
+ :exc:`mmap.error` have been merged into :exc:`OSError`.
.. exception:: OverflowError
@@ -262,8 +275,20 @@ The following exceptions are the exceptions that are usually raised.
.. exception:: StopIteration
Raised by built-in function :func:`next` and an :term:`iterator`\'s
- :meth:`__next__` method to signal that there are no further values.
+ :meth:`__next__` method to signal that there are no further items to be
+ produced by the iterator.
+
+ The exception object has a single attribute :attr:`value`, which is
+ given as an argument when constructing the exception, and defaults
+ to :const:`None`.
+
+ When a generator function returns, a new :exc:`StopIteration` instance is
+ raised, and the value returned by the function is used as the
+ :attr:`value` parameter to the constructor of the exception.
+ .. versionchanged:: 3.3
+ Added ``value`` attribute and the ability for generator functions to
+ use it to return a value.
.. exception:: SyntaxError
@@ -372,27 +397,141 @@ The following exceptions are the exceptions that are usually raised.
more precise exception such as :exc:`IndexError`.
-.. exception:: VMSError
+.. exception:: ZeroDivisionError
- Only available on VMS. Raised when a VMS-specific error occurs.
+ Raised when the second argument of a division or modulo operation is zero. The
+ associated value is a string indicating the type of the operands and the
+ operation.
+The following exceptions are kept for compatibility with previous versions;
+starting from Python 3.3, they are aliases of :exc:`OSError`.
+
+.. exception:: EnvironmentError
+
+.. exception:: IOError
+
+.. exception:: VMSError
+
+ Only available on VMS.
+
.. exception:: WindowsError
- Raised when a Windows-specific error occurs or when the error number does not
- correspond to an :c:data:`errno` value. The :attr:`winerror` and
- :attr:`strerror` values are created from the return values of the
- :c:func:`GetLastError` and :c:func:`FormatMessage` functions from the Windows
- Platform API. The :attr:`errno` value maps the :attr:`winerror` value to
- corresponding ``errno.h`` values. This is a subclass of :exc:`OSError`.
+ Only available on Windows.
-.. exception:: ZeroDivisionError
+OS exceptions
+^^^^^^^^^^^^^
+
+The following exceptions are subclasses of :exc:`OSError`, they get raised
+depending on the system error code.
+
+.. exception:: BlockingIOError
+
+ Raised when an operation would block on an object (e.g. socket) set
+ for non-blocking operation.
+ Corresponds to :c:data:`errno` ``EAGAIN``, ``EALREADY``,
+ ``EWOULDBLOCK`` and ``EINPROGRESS``.
+
+ In addition to those of :exc:`OSError`, :exc:`BlockingIOError` can have
+ one more attribute:
+
+ .. attribute:: characters_written
+
+ An integer containing the number of characters written to the stream
+ before it blocked. This attribute is available when using the
+ buffered I/O classes from the :mod:`io` module.
+
+.. exception:: ChildProcessError
+
+ Raised when an operation on a child process failed.
+ Corresponds to :c:data:`errno` ``ECHILD``.
+
+.. exception:: ConnectionError
+
+ A base class for connection-related issues. Subclasses are
+ :exc:`BrokenPipeError`, :exc:`ConnectionAbortedError`,
+ :exc:`ConnectionRefusedError` and :exc:`ConnectionResetError`.
+
+ .. exception:: BrokenPipeError
+
+ A subclass of :exc:`ConnectionError`, raised when trying to write on a
+ pipe while the other end has been closed, or trying to write on a socket
+ which has been shutdown for writing.
+ Corresponds to :c:data:`errno` ``EPIPE`` and ``ESHUTDOWN``.
+
+ .. exception:: ConnectionAbortedError
+
+ A subclass of :exc:`ConnectionError`, raised when a connection attempt
+ is aborted by the peer.
+ Corresponds to :c:data:`errno` ``ECONNABORTED``.
+
+ .. exception:: ConnectionRefusedError
+
+ A subclass of :exc:`ConnectionError`, raised when a connection attempt
+ is refused by the peer.
+ Corresponds to :c:data:`errno` ``ECONNREFUSED``.
+
+ .. exception:: ConnectionResetError
+
+ A subclass of :exc:`ConnectionError`, raised when a connection is
+ reset by the peer.
+ Corresponds to :c:data:`errno` ``ECONNRESET``.
+
+.. exception:: FileExistsError
+
+ Raised when trying to create a file or directory which already exists.
+ Corresponds to :c:data:`errno` ``EEXIST``.
+
+.. exception:: FileNotFoundError
+
+ Raised when a file or directory is requested but doesn't exist.
+ Corresponds to :c:data:`errno` ``ENOENT``.
+
+.. exception:: InterruptedError
+
+ Raised when a system call is interrupted by an incoming signal.
+ Corresponds to :c:data:`errno` ``EEINTR``.
+
+.. exception:: IsADirectoryError
+
+ Raised when a file operation (such as :func:`os.remove`) is requested
+ on a directory.
+ Corresponds to :c:data:`errno` ``EISDIR``.
+
+.. exception:: NotADirectoryError
+
+ Raised when a directory operation (such as :func:`os.listdir`) is requested
+ on something which is not a directory.
+ Corresponds to :c:data:`errno` ``ENOTDIR``.
+
+.. exception:: PermissionError
+
+ Raised when trying to run an operation without the adequate access
+ rights - for example filesystem permissions.
+ Corresponds to :c:data:`errno` ``EACCES`` and ``EPERM``.
+
+.. exception:: ProcessLookupError
+
+ Raised when a given process doesn't exist.
+ Corresponds to :c:data:`errno` ``ESRCH``.
+
+.. exception:: TimeoutError
+
+ Raised when a system function timed out at the system level.
+ Corresponds to :c:data:`errno` ``ETIMEDOUT``.
+
+.. versionadded:: 3.3
+ All the above :exc:`OSError` subclasses were added.
+
+
+.. seealso::
+
+ :pep:`3151` - Reworking the OS and IO exception hierarchy
- Raised when the second argument of a division or modulo operation is zero. The
- associated value is a string indicating the type of the operands and the
- operation.
+Warnings
+--------
The following exceptions are used as warning categories; see the :mod:`warnings`
module for more information.
diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst
new file mode 100644
index 0000000..c9b9546
--- /dev/null
+++ b/Doc/library/faulthandler.rst
@@ -0,0 +1,136 @@
+:mod:`faulthandler` --- Dump the Python traceback
+=================================================
+
+.. module:: faulthandler
+ :synopsis: Dump the Python traceback.
+
+This module contains functions to dump Python tracebacks explicitly, on a fault,
+after a timeout, or on a user signal. Call :func:`faulthandler.enable` to
+install fault handlers for the :const:`SIGSEGV`, :const:`SIGFPE`,
+:const:`SIGABRT`, :const:`SIGBUS`, and :const:`SIGILL` signals. You can also
+enable them at startup by setting the :envvar:`PYTHONFAULTHANDLER` environment
+variable or by using :option:`-X` ``faulthandler`` command line option.
+
+The fault handler is compatible with system fault handlers like Apport or the
+Windows fault handler. The module uses an alternative stack for signal handlers
+if the :c:func:`sigaltstack` function is available. This allows it to dump the
+traceback even on a stack overflow.
+
+The fault handler is called on catastrophic cases and therefore can only use
+signal-safe functions (e.g. it cannot allocate memory on the heap). Because of
+this limitation traceback dumping is minimal compared to normal Python
+tracebacks:
+
+* Only ASCII is supported. The ``backslashreplace`` error handler is used on
+ encoding.
+* Each string is limited to 100 characters.
+* Only the filename, the function name and the line number are
+ displayed. (no source code)
+* It is limited to 100 frames and 100 threads.
+
+By default, the Python traceback is written to :data:`sys.stderr`. To see
+tracebacks, applications must be run in the terminal. A log file can
+alternatively be passed to :func:`faulthandler.enable`.
+
+The module is implemented in C, so tracebacks can be dumped on a crash or when
+Python is deadlocked.
+
+.. versionadded:: 3.3
+
+
+Dump the traceback
+------------------
+
+.. function:: dump_traceback(file=sys.stderr, all_threads=True)
+
+ Dump the tracebacks of all threads into *file*. If *all_threads* is
+ ``False``, dump only the current thread.
+
+
+Fault handler state
+-------------------
+
+.. function:: enable(file=sys.stderr, all_threads=True)
+
+ Enable the fault handler: install handlers for the :const:`SIGSEGV`,
+ :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and :const:`SIGILL`
+ signals to dump the Python traceback. If *all_threads* is ``True``,
+ produce tracebacks for every running thread. Otherwise, dump only the current
+ thread.
+
+.. function:: disable()
+
+ Disable the fault handler: uninstall the signal handlers installed by
+ :func:`enable`.
+
+.. function:: is_enabled()
+
+ Check if the fault handler is enabled.
+
+
+Dump the tracebacks after a timeout
+-----------------------------------
+
+.. function:: dump_tracebacks_later(timeout, repeat=False, file=sys.stderr, exit=False)
+
+ Dump the tracebacks of all threads, after a timeout of *timeout* seconds, or
+ every *timeout* seconds if *repeat* is ``True``. If *exit* is ``True``, call
+ :c:func:`_exit` with status=1 after dumping the tracebacks. (Note
+ :c:func:`_exit` exits the process immediately, which means it doesn't do any
+ cleanup like flushing file buffers.) If the function is called twice, the new
+ call replaces previous parameters and resets the timeout. The timer has a
+ sub-second resolution.
+
+ This function is implemented using a watchdog thread and therefore is not
+ available if Python is compiled with threads disabled.
+
+.. function:: cancel_dump_tracebacks_later()
+
+ Cancel the last call to :func:`dump_tracebacks_later`.
+
+
+Dump the traceback on a user signal
+-----------------------------------
+
+.. function:: register(signum, file=sys.stderr, all_threads=True, chain=False)
+
+ Register a user signal: install a handler for the *signum* signal to dump
+ the traceback of all threads, or of the current thread if *all_threads* is
+ ``False``, into *file*. Call the previous handler if chain is ``True``.
+
+ Not available on Windows.
+
+.. function:: unregister(signum)
+
+ Unregister a user signal: uninstall the handler of the *signum* signal
+ installed by :func:`register`. Return ``True`` if the signal was registered,
+ ``False`` otherwise.
+
+ Not available on Windows.
+
+
+File descriptor issue
+---------------------
+
+:func:`enable`, :func:`dump_tracebacks_later` and :func:`register` keep the
+file descriptor of their *file* argument. If the file is closed and its file
+descriptor is reused by a new file, or if :func:`os.dup2` is used to replace
+the file descriptor, the traceback will be written into a different file. Call
+these functions again each time that the file is replaced.
+
+
+Example
+-------
+
+Example of a segmentation fault on Linux: ::
+
+ $ python -q -X faulthandler
+ >>> import ctypes
+ >>> ctypes.string_at(0)
+ Fatal Python error: Segmentation fault
+
+ Current thread 0x00007fb899f39700:
+ File "/home/python/cpython/Lib/ctypes/__init__.py", line 486 in string_at
+ File "<stdin>", line 1 in <module>
+ Segmentation fault
+
diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst
index 6192400..9a9cdc1 100644
--- a/Doc/library/fcntl.rst
+++ b/Doc/library/fcntl.rst
@@ -19,6 +19,11 @@ argument. This can be an integer file descriptor, such as returned by
``sys.stdin.fileno()``, or a :class:`io.IOBase` object, such as ``sys.stdin``
itself, which provides a :meth:`fileno` that returns a genuine file descriptor.
+.. versionchanged:: 3.3
+ Operations in this module used to raise a :exc:`IOError` where they now
+ raise a :exc:`OSError`.
+
+
The module defines the following functions:
@@ -40,7 +45,7 @@ The module defines the following functions:
larger than 1024 bytes, this is most likely to result in a segmentation
violation or a more subtle data corruption.
- If the :c:func:`fcntl` fails, an :exc:`IOError` is raised.
+ If the :c:func:`fcntl` fails, an :exc:`OSError` is raised.
.. function:: ioctl(fd, op[, arg[, mutate_flag]])
@@ -107,7 +112,7 @@ The module defines the following functions:
When *operation* is :const:`LOCK_SH` or :const:`LOCK_EX`, it can also be
bitwise ORed with :const:`LOCK_NB` to avoid blocking on lock acquisition.
If :const:`LOCK_NB` is used and the lock cannot be acquired, an
- :exc:`IOError` will be raised and the exception will have an *errno*
+ :exc:`OSError` will be raised and the exception will have an *errno*
attribute set to :const:`EACCES` or :const:`EAGAIN` (depending on the
operating system; for portability, check for both values). On at least some
systems, :const:`LOCK_EX` can only be used if the file descriptor refers to a
diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst
index ac44311..f8ec436 100644
--- a/Doc/library/fileinput.rst
+++ b/Doc/library/fileinput.rst
@@ -28,7 +28,10 @@ as the first argument to :func:`.input`. A single file name is also allowed.
All files are opened in text mode by default, but you can override this by
specifying the *mode* parameter in the call to :func:`.input` or
:class:`FileInput`. If an I/O error occurs during opening or reading a file,
-:exc:`IOError` is raised.
+:exc:`OSError` is raised.
+
+.. versionchanged:: 3.3
+ :exc:`IOError` used to be raised; it is now an alias of :exc:`OSError`.
If ``sys.stdin`` is used more than once, the second and further use will return
no lines, except perhaps for interactive use, or if it has been explicitly reset
@@ -168,10 +171,6 @@ and the backup file remains around; by default, the extension is ``'.bak'`` and
it is deleted when the output file is closed. In-place filtering is disabled
when standard input is read.
-.. note::
-
- The current implementation does not work for MS-DOS 8+3 filesystems.
-
The two following opening hooks are provided by this module:
diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst
index 5bbef4f..3cc295a 100644
--- a/Doc/library/ftplib.rst
+++ b/Doc/library/ftplib.rst
@@ -40,7 +40,7 @@ Here's a sample session using the :mod:`ftplib` module::
The module defines the following items:
-.. class:: FTP(host='', user='', passwd='', acct=''[, timeout])
+.. class:: FTP(host='', user='', passwd='', acct='', timeout=None, source_address=None)
Return a new instance of the :class:`FTP` class. When *host* is given, the
method call ``connect(host)`` is made. When *user* is given, additionally
@@ -48,7 +48,8 @@ The module defines the following items:
*acct* default to the empty string when not given). The optional *timeout*
parameter specifies a timeout in seconds for blocking operations like the
connection attempt (if is not specified, the global default timeout setting
- will be used).
+ will be used). *source_address* is a 2-tuple ``(host, port)`` for the socket
+ to bind to as its source address before connecting.
:class:`FTP` class supports the :keyword:`with` statement. Here is a sample
on how using it:
@@ -68,8 +69,11 @@ The module defines the following items:
.. versionchanged:: 3.2
Support for the :keyword:`with` statement was added.
+ .. versionchanged:: 3.3
+ *source_address* parameter was added.
-.. class:: FTP_TLS(host='', user='', passwd='', acct='', [keyfile[, certfile[, context[, timeout]]]])
+
+.. class:: FTP_TLS(host='', user='', passwd='', acct='', keyfile=None, certfile=None, context=None, timeout=None, source_address=None)
A :class:`FTP` subclass which adds TLS support to FTP as described in
:rfc:`4217`.
@@ -80,10 +84,15 @@ The module defines the following items:
private key and certificate chain file name for the SSL connection.
*context* parameter is a :class:`ssl.SSLContext` object which allows
bundling SSL configuration options, certificates and private keys into a
- single (potentially long-lived) structure.
+ single (potentially long-lived) structure. *source_address* is a 2-tuple
+ ``(host, port)`` for the socket to bind to as its source address before
+ connecting.
.. versionadded:: 3.2
+ .. versionchanged:: 3.3
+ *source_address* parameter was added.
+
Here's a sample session using the :class:`FTP_TLS` class:
>>> from ftplib import FTP_TLS
@@ -135,8 +144,7 @@ The module defines the following items:
The set of all exceptions (as a tuple) that methods of :class:`FTP`
instances may raise as a result of problems with the FTP connection (as
opposed to programming errors made by the caller). This set includes the
- four exceptions listed above as well as :exc:`socket.error` and
- :exc:`IOError`.
+ four exceptions listed above as well as :exc:`OSError`.
.. seealso::
@@ -174,7 +182,7 @@ followed by ``lines`` for the text version or ``binary`` for the binary version.
debugging output, logging each line sent and received on the control connection.
-.. method:: FTP.connect(host='', port=0[, timeout])
+.. method:: FTP.connect(host='', port=0, timeout=None, source_address=None)
Connect to the given host and port. The default port number is ``21``, as
specified by the FTP protocol specification. It is rarely needed to specify a
@@ -182,10 +190,14 @@ followed by ``lines`` for the text version or ``binary`` for the binary version.
instance; it should not be called at all if a host was given when the instance
was created. All other methods can only be used after a connection has been
made.
-
The optional *timeout* parameter specifies a timeout in seconds for the
connection attempt. If no *timeout* is passed, the global default timeout
setting will be used.
+ *source_address* is a 2-tuple ``(host, port)`` for the socket to bind to as
+ its source address before connecting.
+
+ .. versionchanged:: 3.3
+ *source_address* parameter was added.
.. method:: FTP.getwelcome()
@@ -241,13 +253,12 @@ followed by ``lines`` for the text version or ``binary`` for the binary version.
Retrieve a file or directory listing in ASCII transfer mode. *cmd* should be
an appropriate ``RETR`` command (see :meth:`retrbinary`) or a command such as
- ``LIST``, ``NLST`` or ``MLSD`` (usually just the string ``'LIST'``).
+ ``LIST`` or ``NLST`` (usually just the string ``'LIST'``).
``LIST`` retrieves a list of files and information about those files.
- ``NLST`` retrieves a list of file names. On some servers, ``MLSD`` retrieves
- a machine readable list of files and information about those files. The
- *callback* function is called for each line with a string argument containing
- the line with the trailing CRLF stripped. The default *callback* prints the
- line to ``sys.stdout``.
+ ``NLST`` retrieves a list of file names.
+ The *callback* function is called for each line with a string argument
+ containing the line with the trailing CRLF stripped. The default *callback*
+ prints the line to ``sys.stdout``.
.. method:: FTP.set_pasv(boolean)
@@ -307,6 +318,20 @@ followed by ``lines`` for the text version or ``binary`` for the binary version.
in :meth:`transfercmd`.
+.. method:: FTP.mlsd(path="", facts=[])
+
+ List a directory in a standardized format by using MLSD command
+ (:rfc:`3659`). If *path* is omitted the current directory is assumed.
+ *facts* is a list of strings representing the type of information desired
+ (e.g. ``["type", "size", "perm"]``). Return a generator object yielding a
+ tuple of two elements for every file found in path. First element is the
+ file name, the second one is a dictionary containing facts about the file
+ name. Content of this dictionary might be limited by the *facts* argument
+ but server is not guaranteed to return all requested facts.
+
+ .. versionadded:: 3.3
+
+
.. method:: FTP.nlst(argument[, ...])
Return a list of file names as returned by the ``NLST`` command. The
@@ -314,6 +339,8 @@ followed by ``lines`` for the text version or ``binary`` for the binary version.
directory). Multiple arguments can be used to pass non-standard options to
the ``NLST`` command.
+ .. deprecated:: 3.3 use :meth:`mlsd` instead.
+
.. method:: FTP.dir(argument[, ...])
@@ -324,6 +351,8 @@ followed by ``lines`` for the text version or ``binary`` for the binary version.
as a *callback* function as for :meth:`retrlines`; the default prints to
``sys.stdout``. This method returns ``None``.
+ .. deprecated:: 3.3 use :meth:`mlsd` instead.
+
.. method:: FTP.rename(fromname, toname)
@@ -396,6 +425,14 @@ FTP_TLS Objects
Set up secure control connection by using TLS or SSL, depending on what specified in :meth:`ssl_version` attribute.
+.. method:: FTP_TLS.ccc()
+
+ Revert control channel back to plaintext. This can be useful to take
+ advantage of firewalls that know how to handle NAT with non-secure FTP
+ without opening fixed ports.
+
+ .. versionadded:: 3.3
+
.. method:: FTP_TLS.prot_p()
Set up secure data connection.
diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index 3fcd694..5254299 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -152,10 +152,6 @@ are always available. They are listed here in alphabetical order.
1,114,111 (0x10FFFF in base 16). :exc:`ValueError` will be raised if *i* is
outside that range.
- Note that on narrow Unicode builds, the result is a string of
- length two for *i* greater than 65,535 (0xFFFF in hexadecimal).
-
-
.. function:: classmethod(function)
@@ -787,10 +783,10 @@ are always available. They are listed here in alphabetical order.
:meth:`__index__` method that returns an integer.
-.. function:: open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True)
+.. function:: open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None)
Open *file* and return a corresponding stream. If the file cannot be opened,
- an :exc:`IOError` is raised.
+ an :exc:`OSError` is raised.
*file* is either a string or bytes object giving the pathname (absolute or
relative to the current working directory) of the file to be opened or
@@ -801,17 +797,20 @@ are always available. They are listed here in alphabetical order.
*mode* is an optional string that specifies the mode in which the file is
opened. It defaults to ``'r'`` which means open for reading in text mode.
Other common values are ``'w'`` for writing (truncating the file if it
- already exists), and ``'a'`` for appending (which on *some* Unix systems,
- means that *all* writes append to the end of the file regardless of the
- current seek position). In text mode, if *encoding* is not specified the
- encoding used is platform dependent. (For reading and writing raw bytes use
- binary mode and leave *encoding* unspecified.) The available modes are:
+ already exists), ``'x'`` for exclusive creation and ``'a'`` for appending
+ (which on *some* Unix systems, means that *all* writes append to the end of
+ the file regardless of the current seek position). In text mode, if
+ *encoding* is not specified the encoding used is platform dependent:
+ ``locale.getpreferredencoding(False)`` is called to get the current locale
+ encoding. (For reading and writing raw bytes use binary mode and leave
+ *encoding* unspecified.) The available modes are:
========= ===============================================================
Character Meaning
--------- ---------------------------------------------------------------
``'r'`` open for reading (default)
``'w'`` open for writing, truncating the file first
+ ``'x'`` open for exclusive creation, failing if the file already exists
``'a'`` open for writing, appending to the end of the file if it exists
``'b'`` binary mode
``'t'`` text mode (default)
@@ -894,6 +893,16 @@ are always available. They are listed here in alphabetical order.
closed. If a filename is given *closefd* has no effect and must be ``True``
(the default).
+ A custom opener can be used by passing a callable as *opener*. The underlying
+ file descriptor for the file object is then obtained by calling *opener* with
+ (*file*, *flags*). *opener* must return an open file descriptor (passing
+ :mod:`os.open` as *opener* results in functionality similar to passing
+ ``None``).
+
+ .. versionchanged:: 3.3
+ The *opener* parameter was added.
+ The ``'x'`` mode was added.
+
The type of file object returned by the :func:`open` function depends on the
mode. When :func:`open` is used to open a file in a text mode (``'w'``,
``'r'``, ``'wt'``, ``'rt'``, etc.), it returns a subclass of
@@ -919,6 +928,11 @@ are always available. They are listed here in alphabetical order.
(where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`,
and :mod:`shutil`.
+ .. versionchanged:: 3.3
+ :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`.
+ :exc:`FileExistsError` is now raised if the file opened in exclusive
+ creation mode (``'x'``) already exists.
+
.. XXX works for bytes too, but should it?
.. function:: ord(c)
@@ -928,9 +942,6 @@ are always available. They are listed here in alphabetical order.
point of that character. For example, ``ord('a')`` returns the integer ``97``
and ``ord('\u2020')`` returns ``8224``. This is the inverse of :func:`chr`.
- On wide Unicode builds, if the argument length is not one, a
- :exc:`TypeError` will be raised. On narrow Unicode builds, strings
- of length two are accepted when they form a UTF-16 surrogate pair.
.. function:: pow(x, y[, z])
@@ -948,7 +959,7 @@ are always available. They are listed here in alphabetical order.
must be of integer types, and *y* must be non-negative.
-.. function:: print([object, ...], *, sep=' ', end='\\n', file=sys.stdout)
+.. function:: print([object, ...], *, sep=' ', end='\\n', file=sys.stdout, flush=False)
Print *object*\(s) to the stream *file*, separated by *sep* and followed by
*end*. *sep*, *end* and *file*, if present, must be given as keyword
@@ -961,9 +972,12 @@ are always available. They are listed here in alphabetical order.
*end*.
The *file* argument must be an object with a ``write(string)`` method; if it
- is not present or ``None``, :data:`sys.stdout` will be used. Output buffering
- is determined by *file*. Use ``file.flush()`` to ensure, for instance,
- immediate appearance on a screen.
+ is not present or ``None``, :data:`sys.stdout` will be used. Whether output
+ is buffered is usually determined by *file*, but if the *flush* keyword
+ argument is true, the stream is forcibly flushed.
+
+ .. versionchanged:: 3.3
+ Added the *flush* keyword argument.
.. function:: property(fget=None, fset=None, fdel=None, doc=None)
@@ -1046,7 +1060,9 @@ are always available. They are listed here in alphabetical order.
...]``. If *step* is positive, the last element is the largest ``start + i *
step`` less than *stop*; if *step* is negative, the last element is the
smallest ``start + i * step`` greater than *stop*. *step* must not be zero
- (or else :exc:`ValueError` is raised). Example:
+ (or else :exc:`ValueError` is raised). Range objects have read-only data
+ attributes :attr:`start`, :attr:`stop` and :attr:`step` which return the
+ argument values (or their default). Example:
>>> list(range(10))
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
@@ -1083,6 +1099,13 @@ are always available. They are listed here in alphabetical order.
>>> r[-1]
18
+ Testing range objects for equality with ``==`` and ``!=`` compares
+ them as sequences. That is, two range objects are considered equal if
+ they represent the same sequence of values. (Note that two range
+ objects that compare equal might have different :attr:`start`,
+ :attr:`stop` and :attr:`step` attributes, for example ``range(0) ==
+ range(2, 1, 3)`` or ``range(0, 3, 2) == range(0, 4, 2)``.)
+
Ranges containing absolute values larger than :data:`sys.maxsize` are permitted
but some features (such as :func:`len`) will raise :exc:`OverflowError`.
@@ -1092,6 +1115,14 @@ are always available. They are listed here in alphabetical order.
Test integers for membership in constant time instead of iterating
through all items.
+ .. versionchanged:: 3.3
+ Define '==' and '!=' to compare range objects based on the
+ sequence of values they define (instead of comparing based on
+ object identity).
+
+ .. versionadded:: 3.3
+ The :attr:`start`, :attr:`stop` and :attr:`step` attributes.
+
.. function:: repr(object)
@@ -1299,10 +1330,12 @@ are always available. They are listed here in alphabetical order.
Accordingly, :func:`super` is undefined for implicit lookups using statements or
operators such as ``super()[name]``.
- Also note that :func:`super` is not limited to use inside methods. The two
- argument form specifies the arguments exactly and makes the appropriate
- references. The zero argument form automatically searches the stack frame
- for the class (``__class__``) and the first argument.
+ Also note that, aside from the zero argument form, :func:`super` is not
+ limited to use inside methods. The two argument form specifies the
+ arguments exactly and makes the appropriate references. The zero
+ argument form only works inside a class definition, as the compiler fills
+ in the necessary details to correctly retrieve the class being defined,
+ as well as accessing the current instance for ordinary methods.
For practical suggestions on how to design cooperative classes using
:func:`super`, see `guide to using super()
@@ -1422,8 +1455,9 @@ are always available. They are listed here in alphabetical order.
replaced (by importing the :mod:`builtins` module and assigning to
``builtins.__import__``) in order to change semantics of the
:keyword:`import` statement, but nowadays it is usually simpler to use import
- hooks (see :pep:`302`). Direct use of :func:`__import__` is rare, except in
- cases where you want to import a module whose name is only known at runtime.
+ hooks (see :pep:`302`) to attain the same goals. Direct use of
+ :func:`__import__` is entirely discouraged in favor of
+ :func:`importlib.import_module`.
The function imports the module *name*, potentially using the given *globals*
and *locals* to determine how to interpret the name in a package context.
@@ -1435,7 +1469,8 @@ are always available. They are listed here in alphabetical order.
*level* specifies whether to use absolute or relative imports. ``0`` (the
default) means only perform absolute imports. Positive values for
*level* indicate the number of parent directories to search relative to the
- directory of the module calling :func:`__import__`.
+ directory of the module calling :func:`__import__` (see :pep:`328` for the
+ details).
When the *name* variable is of the form ``package.module``, normally, the
top-level package (the name up till the first dot) is returned, *not* the
diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst
index 04743d3..f5c6608 100644
--- a/Doc/library/functools.rst
+++ b/Doc/library/functools.rst
@@ -40,7 +40,7 @@ The :mod:`functools` module defines the following functions:
.. versionadded:: 3.2
-.. decorator:: lru_cache(maxsize=100)
+.. decorator:: lru_cache(maxsize=128, typed=False)
Decorator to wrap a function with a memoizing callable that saves up to the
*maxsize* most recent calls. It can save time when an expensive or I/O bound
@@ -49,8 +49,13 @@ The :mod:`functools` module defines the following functions:
Since a dictionary is used to cache results, the positional and keyword
arguments to the function must be hashable.
- If *maxsize* is set to None, the LRU feature is disabled and the cache
- can grow without bound.
+ If *maxsize* is set to None, the LRU feature is disabled and the cache can
+ grow without bound. The LRU feature performs best when *maxsize* is a
+ power-of-two.
+
+ If *typed* is set to True, function arguments of different types will be
+ cached separately. For example, ``f(3)`` and ``f(3.0)`` will be treated
+ as distinct calls with distinct results.
To help measure the effectiveness of the cache and tune the *maxsize*
parameter, the wrapped function is instrumented with a :func:`cache_info`
@@ -67,8 +72,8 @@ The :mod:`functools` module defines the following functions:
An `LRU (least recently used) cache
<http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used>`_ works
- best when more recent calls are the best predictors of upcoming calls (for
- example, the most popular articles on a news server tend to change daily).
+ best when the most recent calls are the best predictors of upcoming calls (for
+ example, the most popular articles on a news server tend to change each day).
The cache's size limit assures that the cache does not grow without bound on
long-running processes such as web servers.
@@ -111,6 +116,9 @@ The :mod:`functools` module defines the following functions:
.. versionadded:: 3.2
+ .. versionchanged:: 3.3
+ Added the *typed* option.
+
.. decorator:: total_ordering
Given a class defining one or more rich comparison ordering methods, this
diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst
index 0281bb7..3d76411 100644
--- a/Doc/library/gc.rst
+++ b/Doc/library/gc.rst
@@ -153,8 +153,8 @@ The :mod:`gc` module provides the following functions:
.. versionadded:: 3.1
-The following variable is provided for read-only access (you can mutate its
-value but should not rebind it):
+The following variables are provided for read-only access (you can mutate the
+values but should not rebind them):
.. data:: garbage
@@ -183,6 +183,41 @@ value but should not rebind it):
:const:`DEBUG_UNCOLLECTABLE` is set, in addition all uncollectable objects
are printed.
+.. data:: callbacks
+
+ A list of callbacks that will be invoked by the garbage collector before and
+ after collection. The callbacks will be called with two arguments,
+ *phase* and *info*.
+
+ *phase* can one of two values:
+
+ "start": The garbage collection is about to start.
+
+ "stop": The garbage collection has finished.
+
+ *info* provides more information for the callback. The following
+ keys are currently defined:
+
+ "generation": The oldest generation being collected.
+
+ "collected": When *phase* is "stop", the number of objects
+ successfully collected.
+
+ "uncollectable": when *phase* is "stop", the number of objects
+ that could not be collected and were put in :data:`garbage`.
+
+ Applications can add their own callbacks to this list. The primary
+ use cases are:
+
+ Gathering statistics about garbage collection, such as how often
+ various generations are collected, and how long the collection
+ takes.
+
+ Allowing applications to identify and clear their own uncollectable
+ types when they appear in :data:`garbage`.
+
+ .. versionadded:: 3.3
+
The following constants are provided for use with :func:`set_debug`:
diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst
index 0fa022c..825311b 100644
--- a/Doc/library/gettext.rst
+++ b/Doc/library/gettext.rst
@@ -185,10 +185,13 @@ class can also install themselves in the built-in namespace as the function
translation object from the cache; the actual instance data is still shared with
the cache.
- If no :file:`.mo` file is found, this function raises :exc:`IOError` if
+ If no :file:`.mo` file is found, this function raises :exc:`OSError` if
*fallback* is false (which is the default), and returns a
:class:`NullTranslations` instance if *fallback* is true.
+ .. versionchanged:: 3.3
+ :exc:`IOError` used to be raised instead of :exc:`OSError`.
+
.. function:: install(domain, localedir=None, codeset=None, names=None)
@@ -342,7 +345,7 @@ The entire set of key/value pairs are placed into a dictionary and set as the
If the :file:`.mo` file's magic number is invalid, or if other problems occur
while reading the file, instantiating a :class:`GNUTranslations` class can raise
-:exc:`IOError`.
+:exc:`OSError`.
The following methods are overridden from the base class implementation:
diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst
index 9422ea9..50d0462 100644
--- a/Doc/library/gzip.rst
+++ b/Doc/library/gzip.rst
@@ -13,9 +13,11 @@ like the GNU programs :program:`gzip` and :program:`gunzip` would.
The data compression is provided by the :mod:`zlib` module.
-The :mod:`gzip` module provides the :class:`GzipFile` class. The :class:`GzipFile`
-class reads and writes :program:`gzip`\ -format files, automatically compressing
-or decompressing the data so that it looks like an ordinary :term:`file object`.
+The :mod:`gzip` module provides the :class:`GzipFile` class, as well as the
+:func:`.open`, :func:`compress` and :func:`decompress` convenience functions.
+The :class:`GzipFile` class reads and writes :program:`gzip`\ -format files,
+automatically compressing or decompressing the data so that it looks like an
+ordinary :term:`file object`.
Note that additional file formats which can be decompressed by the
:program:`gzip` and :program:`gunzip` programs, such as those produced by
@@ -24,6 +26,34 @@ Note that additional file formats which can be decompressed by the
The module defines the following items:
+.. function:: open(filename, mode='rb', compresslevel=9, encoding=None, errors=None, newline=None)
+
+ Open a gzip-compressed file in binary or text mode, returning a :term:`file
+ object`.
+
+ The *filename* argument can be an actual filename (a :class:`str` or
+ :class:`bytes` object), or an existing file object to read from or write to.
+
+ The *mode* argument can be any of ``'r'``, ``'rb'``, ``'a'``, ``'ab'``,
+ ``'w'``, or ``'wb'`` for binary mode, or ``'rt'``, ``'at'``, or ``'wt'`` for
+ text mode. The default is ``'rb'``.
+
+ The *compresslevel* argument is an integer from 1 to 9, as for the
+ :class:`GzipFile` constructor.
+
+ For binary mode, this function is equivalent to the :class:`GzipFile`
+ constructor: ``GzipFile(filename, mode, compresslevel)``. In this case, the
+ *encoding*, *errors* and *newline* arguments must not be provided.
+
+ For text mode, a :class:`GzipFile` object is created, and wrapped in an
+ :class:`io.TextIOWrapper` instance with the specified encoding, error
+ handling behavior, and line ending(s).
+
+ .. versionchanged:: 3.3
+ Added support for *filename* being a file object, support for text mode,
+ and the *encoding*, *errors* and *newline* arguments.
+
+
.. class:: GzipFile(filename=None, mode=None, compresslevel=9, fileobj=None, mtime=None)
Constructor for the :class:`GzipFile` class, which simulates most of the
@@ -46,9 +76,9 @@ The module defines the following items:
or ``'wb'``, depending on whether the file will be read or written. The default
is the mode of *fileobj* if discernible; otherwise, the default is ``'rb'``.
- Note that the file is always opened in binary mode; text mode is not
- supported. If you need to read a compressed file in text mode, wrap your
- :class:`GzipFile` with an :class:`io.TextIOWrapper`.
+ Note that the file is always opened in binary mode. To open a compressed file
+ in text mode, use :func:`.open` (or wrap your :class:`GzipFile` with an
+ :class:`io.TextIOWrapper`).
The *compresslevel* argument is an integer from ``1`` to ``9`` controlling the
level of compression; ``1`` is fastest and produces the least compression, and
@@ -71,7 +101,7 @@ The module defines the following items:
:class:`GzipFile` supports the :class:`io.BufferedIOBase` interface,
including iteration and the :keyword:`with` statement. Only the
- :meth:`read1` and :meth:`truncate` methods aren't implemented.
+ :meth:`truncate` method isn't implemented.
:class:`GzipFile` also provides the following method:
@@ -93,12 +123,9 @@ The module defines the following items:
.. versionchanged:: 3.2
Support for unseekable files was added.
+ .. versionchanged:: 3.3
+ The :meth:`io.BufferedIOBase.read1` method is now implemented.
-.. function:: open(filename, mode='rb', compresslevel=9)
-
- This is a shorthand for ``GzipFile(filename,`` ``mode,`` ``compresslevel)``.
- The *filename* argument is required; *mode* defaults to ``'rb'`` and
- *compresslevel* defaults to ``9``.
.. function:: compress(data, compresslevel=9)
diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst
index eff2724..38f1687 100644
--- a/Doc/library/hmac.rst
+++ b/Doc/library/hmac.rst
@@ -38,6 +38,13 @@ An HMAC object has the following methods:
given to the constructor. It may contain non-ASCII bytes, including NUL
bytes.
+ .. warning::
+
+ When comparing the output of :meth:`digest` to an externally-supplied
+ digest during a verification routine, it is recommended to use the
+ :func:`compare_digest` function instead of the ``==`` operator
+ to reduce the vulnerability to timing attacks.
+
.. method:: HMAC.hexdigest()
@@ -45,6 +52,13 @@ An HMAC object has the following methods:
length containing only hexadecimal digits. This may be used to exchange the
value safely in email or other non-binary environments.
+ .. warning::
+
+ When comparing the output of :meth:`hexdigest` to an externally-supplied
+ digest during a verification routine, it is recommended to use the
+ :func:`compare_digest` function instead of the ``==`` operator
+ to reduce the vulnerability to timing attacks.
+
.. method:: HMAC.copy()
@@ -52,6 +66,27 @@ An HMAC object has the following methods:
compute the digests of strings that share a common initial substring.
+This module also provides the following helper function:
+
+.. function:: compare_digest(a, b)
+
+ Return ``a == b``. This function uses an approach designed to prevent
+ timing analysis by avoiding content-based short circuiting behaviour,
+ making it appropriate for cryptography. *a* and *b* must both be of the
+ same type: either :class:`str` (ASCII only, as e.g. returned by
+ :meth:`HMAC.hexdigest`), or any type that supports the buffer protocol
+ (e.g. :class:`bytes`).
+
+ .. note::
+
+ If *a* and *b* are of different lengths, or if an error occurs,
+ a timing attack may be able to reveal information about the types
+ and lengths of *a* and *b*, but not their values.
+
+
+ .. versionadded:: 3.3
+
+
.. seealso::
Module :mod:`hashlib`
diff --git a/Doc/library/html.entities.rst b/Doc/library/html.entities.rst
index b8b4aa8..65ce817 100644
--- a/Doc/library/html.entities.rst
+++ b/Doc/library/html.entities.rst
@@ -9,11 +9,19 @@
--------------
-This module defines three dictionaries, ``name2codepoint``, ``codepoint2name``,
-and ``entitydefs``. ``entitydefs`` is used to provide the :attr:`entitydefs`
-attribute of the :class:`html.parser.HTMLParser` class. The definition provided
-here contains all the entities defined by XHTML 1.0 that can be handled using
-simple textual substitution in the Latin-1 character set (ISO-8859-1).
+This module defines four dictionaries, :data:`html5`,
+:data:`name2codepoint`, :data:`codepoint2name`, and :data:`entitydefs`.
+
+
+.. data:: html5
+
+ A dictionary that maps HTML5 named character references [#]_ to the
+ equivalent Unicode character(s), e.g. ``html5['gt;'] == '>'``.
+ Note that the trailing semicolon is included in the name (e.g. ``'gt;'``),
+ however some of the names are accepted by the standard even without the
+ semicolon: in this case the name is present with and without the ``';'``.
+
+ .. versionadded:: 3.3
.. data:: entitydefs
@@ -30,3 +38,8 @@ simple textual substitution in the Latin-1 character set (ISO-8859-1).
.. data:: codepoint2name
A dictionary that maps Unicode codepoints to HTML entity names.
+
+
+.. rubric:: Footnotes
+
+.. [#] See http://www.w3.org/TR/html5/named-character-references.html
diff --git a/Doc/library/html.parser.rst b/Doc/library/html.parser.rst
index f3c36ec..e4154ef 100644
--- a/Doc/library/html.parser.rst
+++ b/Doc/library/html.parser.rst
@@ -16,13 +16,14 @@
This module defines a class :class:`HTMLParser` which serves as the basis for
parsing text files formatted in HTML (HyperText Mark-up Language) and XHTML.
-.. class:: HTMLParser(strict=True)
+.. class:: HTMLParser(strict=False)
- Create a parser instance. If *strict* is ``True`` (the default), invalid
- HTML results in :exc:`~html.parser.HTMLParseError` exceptions [#]_. If
- *strict* is ``False``, the parser uses heuristics to make a best guess at
- the intention of any invalid HTML it encounters, similar to the way most
- browsers do. Using ``strict=False`` is advised.
+ Create a parser instance. If *strict* is ``False`` (the default), the parser
+ will accept and parse invalid markup. If *strict* is ``True`` the parser
+ will raise an :exc:`~html.parser.HTMLParseError` exception instead [#]_ when
+ it's not able to parse the markup.
+ The use of ``strict=True`` is discouraged and the *strict* argument is
+ deprecated.
An :class:`.HTMLParser` instance is fed HTML data and calls handler methods
when start tags, end tags, text, comments, and other markup elements are
@@ -32,7 +33,12 @@ parsing text files formatted in HTML (HyperText Mark-up Language) and XHTML.
This parser does not check that end tags match start tags or call the end-tag
handler for elements which are closed implicitly by closing an outer element.
- .. versionchanged:: 3.2 *strict* keyword added
+ .. versionchanged:: 3.2
+ *strict* keyword added.
+
+ .. deprecated-removed:: 3.3 3.5
+ The *strict* argument and the strict mode have been deprecated.
+ The parser is now able to accept and parse invalid markup too.
An exception is defined as well:
@@ -46,6 +52,10 @@ An exception is defined as well:
detected, and :attr:`offset` is the number of characters into the line at
which the construct starts.
+ .. deprecated-removed:: 3.3 3.5
+ This exception has been deprecated because it's never raised by the parser
+ (when the default non-strict mode is used).
+
Example HTML Parser Application
-------------------------------
diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst
index 52fbe57..efb4c41 100644
--- a/Doc/library/http.client.rst
+++ b/Doc/library/http.client.rst
@@ -339,6 +339,15 @@ and also the following constants for integer status codes:
| :const:`UPGRADE_REQUIRED` | ``426`` | HTTP Upgrade to TLS, |
| | | :rfc:`2817`, Section 6 |
+------------------------------------------+---------+-----------------------------------------------------------------------+
+| :const:`PRECONDITION_REQUIRED` | ``428`` | Additional HTTP Status Codes, |
+| | | :rfc:`6585`, Section 3 |
++------------------------------------------+---------+-----------------------------------------------------------------------+
+| :const:`TOO_MANY_REQUESTS` | ``429`` | Additional HTTP Status Codes, |
+| | | :rfc:`6585`, Section 4 |
++------------------------------------------+---------+-----------------------------------------------------------------------+
+| :const:`REQUEST_HEADER_FIELDS_TOO_LARGE` | ``431`` | Additional HTTP Status Codes, |
+| | | :rfc:`6585`, Section 5 |
++------------------------------------------+---------+-----------------------------------------------------------------------+
| :const:`INTERNAL_SERVER_ERROR` | ``500`` | HTTP/1.1, `RFC 2616, Section |
| | | 10.5.1 |
| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.5.1>`_ |
@@ -369,6 +378,12 @@ and also the following constants for integer status codes:
| :const:`NOT_EXTENDED` | ``510`` | An HTTP Extension Framework, |
| | | :rfc:`2774`, Section 7 |
+------------------------------------------+---------+-----------------------------------------------------------------------+
+| :const:`NETWORK_AUTHENTICATION_REQUIRED` | ``511`` | Additional HTTP Status Codes, |
+| | | :rfc:`6585`, Section 6 |
++------------------------------------------+---------+-----------------------------------------------------------------------+
+
+.. versionchanged:: 3.3
+ Added codes ``428``, ``429``, ``431`` and ``511`` from :rfc:`6585`.
.. data:: responses
@@ -502,6 +517,12 @@ statement.
Reads and returns the response body, or up to the next *amt* bytes.
+.. method:: HTTPResponse.readinto(b)
+
+ Reads up to the next len(b) bytes of the response body into the buffer *b*.
+ Returns the number of bytes read.
+
+ .. versionadded:: 3.3
.. method:: HTTPResponse.getheader(name, default=None)
diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst
index cc8f251..40f24f2 100644
--- a/Doc/library/http.cookiejar.rst
+++ b/Doc/library/http.cookiejar.rst
@@ -40,7 +40,11 @@ The module defines the following exception:
.. exception:: LoadError
Instances of :class:`FileCookieJar` raise this exception on failure to load
- cookies from a file. :exc:`LoadError` is a subclass of :exc:`IOError`.
+ cookies from a file. :exc:`LoadError` is a subclass of :exc:`OSError`.
+
+ .. versionchanged:: 3.3
+ LoadError was made a subclass of :exc:`OSError` instead of
+ :exc:`IOError`.
The following classes are provided:
@@ -257,9 +261,12 @@ contained :class:`Cookie` objects.
Arguments are as for :meth:`save`.
The named file must be in the format understood by the class, or
- :exc:`LoadError` will be raised. Also, :exc:`IOError` may be raised, for
+ :exc:`LoadError` will be raised. Also, :exc:`OSError` may be raised, for
example if the file does not exist.
+ .. versionchanged:: 3.3
+ :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`.
+
.. method:: FileCookieJar.revert(filename=None, ignore_discard=False, ignore_expires=False)
diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst
index 5ae3fd4..646f2e8 100644
--- a/Doc/library/http.cookies.rst
+++ b/Doc/library/http.cookies.rst