summaryrefslogtreecommitdiffstats
path: root/Include/longintrepr.h
blob: 2a911c24c3616e067fc02add186f0feaaecac87e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
#ifndef Py_LONGINTREPR_H
#define Py_LONGINTREPR_H
#ifdef __cplusplus
extern "C" {
#endif

/***********************************************************
Copyright 1991-1995 by Stichting Mathematisch Centrum, Amsterdam,
The Netherlands.

                        All Rights Reserved

Permission to use, copy, modify, and distribute this software and its 
documentation for any purpose and without fee is hereby granted, 
provided that the above copyright notice appear in all copies and that
both that copyright notice and this permission notice appear in 
supporting documentation, and that the names of Stichting Mathematisch
Centrum or CWI not be used in advertising or publicity pertaining to
distribution of the software without specific, written prior permission.

STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO
THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE
FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

******************************************************************/

/* This is published for the benefit of "friend" marshal.c only. */

/* Parameters of the long integer representation.
   These shouldn't have to be changed as C should guarantee that a short
   contains at least 16 bits, but it's made changeable any way.
   Note: 'digit' should be able to hold 2*MASK+1, and 'twodigits'
   should be able to hold the intermediate results in 'mul'
   (at most MASK << SHIFT).
   Also, x_sub assumes that 'digit' is an unsigned type, and overflow
   is handled by taking the result mod 2**N for some N > SHIFT.
   And, at some places it is assumed that MASK fits in an int, as well. */

typedef unsigned short digit;
typedef unsigned int wdigit; /* digit widened to parameter size */
typedef unsigned long twodigits;
typedef long stwodigits; /* signed variant of twodigits */

#define SHIFT	15
#define BASE	((digit)1 << SHIFT)
#define MASK	((int)(BASE - 1))

/* Long integer representation.
   The absolute value of a number is equal to
   	SUM(for i=0 through abs(ob_size)-1) ob_digit[i] * 2**(SHIFT*i)
   Negative numbers are represented with ob_size < 0;
   zero is represented by ob_size == 0.
   In a normalized number, ob_digit[abs(ob_size)-1] (the most significant
   digit) is never zero.  Also, in all cases, for all valid i,
   	0 <= ob_digit[i] <= MASK.
   The allocation fuction takes care of allocating extra memory
   so that ob_digit[0] ... ob_digit[abs(ob_size)-1] are actually available. */

struct _longobject {
	PyObject_HEAD
	int ob_size;
	digit ob_digit[1];
};

PyLongObject *_PyLong_New Py_PROTO((int));

#ifdef __cplusplus
}
#endif
#endif /* !Py_LONGINTREPR_H */
iff/Doc/documenting/style.rst?h=v3.8.0b3&id=8c481b6212d2158bef06eea1f1ed32a015cfc486&id2=8c482ee95564f04df01fc856f313ff8c98846718'>Doc/documenting/style.rst7
-rw-r--r--Doc/extending/newtypes.rst10
-rw-r--r--Doc/faq/design.rst2
-rw-r--r--Doc/faq/extending.rst31
-rw-r--r--Doc/glossary.rst14
-rw-r--r--Doc/howto/sockets.rst42
-rw-r--r--Doc/includes/noddy2.c34
-rw-r--r--Doc/includes/noddy3.c102
-rw-r--r--Doc/includes/noddy4.c32
-rw-r--r--Doc/install/index.rst1088
-rw-r--r--Doc/install/install.rst1117
-rw-r--r--Doc/install/pysetup-config.rst44
-rw-r--r--Doc/install/pysetup-servers.rst61
-rw-r--r--Doc/install/pysetup.rst164
-rw-r--r--Doc/library/_thread.rst3
-rw-r--r--Doc/library/abc.rst7
-rw-r--r--Doc/library/argparse.rst31
-rw-r--r--Doc/library/array.rst68
-rw-r--r--Doc/library/ast.rst3
-rw-r--r--Doc/library/asyncore.rst8
-rw-r--r--Doc/library/atexit.rst8
-rw-r--r--Doc/library/bz2.rst219
-rw-r--r--Doc/library/chunk.rst5
-rw-r--r--Doc/library/cmd.rst4
-rw-r--r--Doc/library/codecs.rst32
-rw-r--r--Doc/library/collections.abc.rst182
-rw-r--r--Doc/library/collections.rst365
-rw-r--r--Doc/library/concurrent.futures.rst19
-rw-r--r--Doc/library/crypt.rst115
-rw-r--r--Doc/library/csv.rst18
-rw-r--r--Doc/library/ctypes.rst21
-rw-r--r--Doc/library/curses.rst19
-rw-r--r--Doc/library/datatypes.rst1
-rw-r--r--Doc/library/datetime.rst39
-rw-r--r--Doc/library/debug.rst3
-rw-r--r--Doc/library/depgraph-output.pngbin0 -> 24719 bytes-rw-r--r--Doc/library/difflib.rst4
-rw-r--r--Doc/library/dis.rst5
-rw-r--r--Doc/library/distutils.rst20
-rw-r--r--Doc/library/email.generator.rst55
-rw-r--r--Doc/library/email.parser.rst78
-rw-r--r--Doc/library/email.policy.rst182
-rw-r--r--Doc/library/email.rst1
-rw-r--r--Doc/library/email.util.rst37
-rw-r--r--Doc/library/exceptions.rst209
-rw-r--r--Doc/library/faulthandler.rst136
-rw-r--r--Doc/library/fcntl.rst9
-rw-r--r--Doc/library/fileinput.rst9
-rw-r--r--Doc/library/ftplib.rst65
-rw-r--r--Doc/library/functions.rst35
-rw-r--r--Doc/library/functools.rst13
-rw-r--r--Doc/library/gettext.rst7
-rw-r--r--Doc/library/gzip.rst5
-rw-r--r--Doc/library/http.cookiejar.rst11
-rw-r--r--Doc/library/http.server.rst45
-rw-r--r--Doc/library/imaplib.rst11
-rw-r--r--Doc/library/inspect.rst12
-rw-r--r--Doc/library/io.rst55
-rw-r--r--Doc/library/itertools.rst48
-rw-r--r--Doc/library/logging.handlers.rst18
-rw-r--r--Doc/library/logging.rst41
-rw-r--r--Doc/library/math.rst13
-rw-r--r--Doc/library/mmap.rst12
-rw-r--r--Doc/library/msvcrt.rst15
-rw-r--r--Doc/library/multiprocessing.rst40
-rw-r--r--Doc/library/nntplib.rst13
-rw-r--r--Doc/library/os.rst882
-rw-r--r--Doc/library/ossaudiodev.rst14
-rw-r--r--Doc/library/packaging-misc.rst27
-rw-r--r--Doc/library/packaging.command.rst111
-rw-r--r--Doc/library/packaging.compiler.rst681
-rw-r--r--Doc/library/packaging.database.rst324
-rw-r--r--Doc/library/packaging.depgraph.rst199
-rw-r--r--Doc/library/packaging.dist.rst108
-rw-r--r--Doc/library/packaging.fancy_getopt.rst75
-rw-r--r--Doc/library/packaging.install.rst112
-rw-r--r--Doc/library/packaging.metadata.rst122
-rw-r--r--Doc/library/packaging.pypi.dist.rst114
-rw-r--r--Doc/library/packaging.pypi.rst74
-rw-r--r--Doc/library/packaging.pypi.simple.rst218
-rw-r--r--Doc/library/packaging.pypi.xmlrpc.rst143
-rw-r--r--Doc/library/packaging.rst75
-rw-r--r--Doc/library/packaging.tests.pypi_server.rst105
-rw-r--r--Doc/library/packaging.util.rst154
-rw-r--r--Doc/library/packaging.version.rst104
-rw-r--r--Doc/library/platform.rst4
-rw-r--r--Doc/library/python.rst1
-rw-r--r--Doc/library/random.rst6
-rw-r--r--Doc/library/re.rst9
-rw-r--r--Doc/library/readline.rst4
-rw-r--r--Doc/library/select.rst12
-rw-r--r--Doc/library/shlex.rst36
-rw-r--r--Doc/library/shutil.rst28
-rw-r--r--Doc/library/signal.rst165
-rw-r--r--Doc/library/site.rst26
-rw-r--r--Doc/library/smtplib.rst62
-rw-r--r--Doc/library/socket.rst343
-rw-r--r--Doc/library/socketserver.rst15
-rw-r--r--Doc/library/sqlite3.rst16
-rw-r--r--Doc/library/ssl.rst136
-rw-r--r--Doc/library/stdtypes.rst29
-rw-r--r--Doc/library/struct.rst21
-rw-r--r--Doc/library/subprocess.rst117
-rw-r--r--Doc/library/sys.rst136
-rw-r--r--Doc/library/tarfile.rst6
-rw-r--r--Doc/library/telnetlib.rst6
-rw-r--r--Doc/library/tempfile.rst2
-rw-r--r--Doc/library/test.rst124
-rw-r--r--Doc/library/threading.rst56
-rw-r--r--Doc/library/time.rst96
-rw-r--r--Doc/library/tokenize.rst57
-rw-r--r--Doc/library/unicodedata.rst9
-rw-r--r--Doc/library/unittest.rst50
-rw-r--r--Doc/library/urllib.error.rst12
-rw-r--r--Doc/library/urllib.request.rst63
-rw-r--r--Doc/library/warnings.rst3
-rw-r--r--Doc/library/winreg.rst48
-rw-r--r--Doc/library/zipimport.rst5
-rw-r--r--Doc/library/zlib.rst41
-rw-r--r--Doc/license.rst2
-rw-r--r--Doc/packaging/builtdist.rst302
-rw-r--r--Doc/packaging/commandhooks.rst47
-rw-r--r--Doc/packaging/commandref.rst374
-rw-r--r--Doc/packaging/configfile.rst125
-rw-r--r--Doc/packaging/examples.rst334
-rw-r--r--Doc/packaging/extending.rst95
-rw-r--r--Doc/packaging/index.rst45
-rw-r--r--Doc/packaging/introduction.rst193
-rw-r--r--Doc/packaging/packageindex.rst104
-rw-r--r--Doc/packaging/setupcfg.rst890
-rw-r--r--Doc/packaging/setupscript.rst693
-rw-r--r--Doc/packaging/sourcedist.rst266
-rw-r--r--Doc/packaging/tutorial.rst112
-rw-r--r--Doc/packaging/uploading.rst80
-rw-r--r--Doc/reference/datamodel.rst23
-rw-r--r--Doc/reference/lexical_analysis.rst17
-rw-r--r--Doc/tools/sphinxext/indexcontent.html8
-rw-r--r--Doc/tools/sphinxext/indexsidebar.html2
-rw-r--r--Doc/tools/sphinxext/pyspecific.py2
-rw-r--r--Doc/tools/sphinxext/susp-ignored.csv119
-rw-r--r--Doc/tutorial/classes.rst4
-rw-r--r--Doc/tutorial/datastructures.rst24
-rw-r--r--Doc/tutorial/interpreter.rst14
-rw-r--r--Doc/tutorial/stdlib.rst2
-rw-r--r--Doc/tutorial/stdlib2.rst6
-rw-r--r--Doc/using/cmdline.rst16
-rw-r--r--Doc/whatsnew/3.2.rst8
-rw-r--r--Doc/whatsnew/3.3.rst597
-rw-r--r--Doc/whatsnew/index.rst1
-rw-r--r--Include/Python-ast.h46
-rw-r--r--Include/Python.h38
-rw-r--r--Include/abstract.h12
-rw-r--r--Include/asdl.h1
-rw-r--r--Include/ast.h1
-rw-r--r--Include/code.h6
-rw-r--r--Include/codecs.h2
-rw-r--r--Include/compile.h14
-rw-r--r--Include/complexobject.h5
-rw-r--r--Include/floatobject.h17
-rw-r--r--Include/import.h29
-rw-r--r--Include/longobject.h16
-rw-r--r--Include/moduleobject.h4
-rw-r--r--Include/object.h8
-rw-r--r--Include/opcode.h1
-rw-r--r--Include/parsetok.h9
-rw-r--r--Include/patchlevel.h12
-rw-r--r--Include/pydebug.h3
-rw-r--r--Include/pyerrors.h62
-rw-r--r--Include/pymacro.h55
-rw-r--r--Include/pymath.h6
-rw-r--r--Include/pyport.h2
-rw-r--r--Include/pystate.h10
-rw-r--r--Include/pythonrun.h3
-rw-r--r--Include/pythread.h4
-rw-r--r--Include/symtable.h11
-rw-r--r--Include/traceback.h38
-rw-r--r--Include/ucnhash.h6
-rw-r--r--Include/unicodeobject.h927
-rw-r--r--LICENSE1
-rw-r--r--Lib/_dummy_thread.py6
-rw-r--r--Lib/_pyio.py103
-rw-r--r--Lib/abc.py8
-rw-r--r--Lib/argparse.py50
-rw-r--r--Lib/ast.py1
-rw-r--r--Lib/asynchat.py2
-rw-r--r--Lib/asyncore.py15
-rw-r--r--Lib/binhex.py1
-rw-r--r--Lib/bz2.py413
-rwxr-xr-xLib/cgi.py11
-rw-r--r--Lib/cgitb.py1
-rw-r--r--Lib/codecs.py14
-rw-r--r--Lib/collections/__init__.py (renamed from Lib/collections.py)98
-rw-r--r--Lib/collections/abc.py (renamed from Lib/_abcoll.py)37
-rw-r--r--Lib/concurrent/futures/_base.py22
-rw-r--r--Lib/concurrent/futures/process.py141
-rw-r--r--Lib/configparser.py3
-rw-r--r--Lib/contextlib.py1
-rw-r--r--Lib/copy.py77
-rw-r--r--Lib/crypt.py62
-rw-r--r--Lib/ctypes/test/test_callbacks.py2
-rw-r--r--Lib/ctypes/test/test_memfunctions.py2
-rw-r--r--Lib/ctypes/test/test_parameters.py9
-rw-r--r--Lib/ctypes/test/test_python_api.py3
-rw-r--r--Lib/ctypes/test/test_refcounts.py3
-rw-r--r--Lib/ctypes/test/test_stringptr.py2
-rw-r--r--Lib/ctypes/util.py46
-rw-r--r--Lib/curses/__init__.py46
-rw-r--r--Lib/curses/wrapper.py50
-rw-r--r--Lib/datetime.py4
-rw-r--r--Lib/decimal.py143
-rw-r--r--Lib/difflib.py20
-rw-r--r--Lib/distutils/__init__.py2
-rw-r--r--Lib/distutils/command/bdist_wininst.py6
-rw-r--r--Lib/distutils/command/build_ext.py3
-rw-r--r--Lib/distutils/command/build_scripts.py7
-rw-r--r--Lib/distutils/tests/test_bdist_rpm.py9
-rw-r--r--Lib/doctest.py8
-rw-r--r--Lib/email/_parseaddr.py27
-rw-r--r--Lib/email/errors.py5
-rw-r--r--Lib/email/feedparser.py31
-rw-r--r--Lib/email/generator.py66
-rw-r--r--Lib/email/parser.py43
-rw-r--r--Lib/email/policy.py174
-rw-r--r--Lib/email/utils.py74
-rw-r--r--Lib/encodings/cp65001.py40
-rw-r--r--Lib/fnmatch.py8
-rw-r--r--Lib/ftplib.py82
-rw-r--r--Lib/functools.py27
-rw-r--r--Lib/getopt.py19
-rw-r--r--Lib/getpass.py4
-rw-r--r--Lib/gzip.py34
-rw-r--r--Lib/http/client.py4
-rw-r--r--Lib/http/server.py50
-rw-r--r--Lib/idlelib/configHandler.py3
-rw-r--r--Lib/idlelib/idlever.py2
-rw-r--r--Lib/idlelib/rpc.py4
-rw-r--r--Lib/imaplib.py33
-rw-r--r--Lib/importlib/__init__.py3
-rw-r--r--Lib/importlib/_bootstrap.py62
-rw-r--r--Lib/importlib/test/__main__.py7
-rw-r--r--Lib/importlib/test/regrtest.py7
-rw-r--r--Lib/inspect.py127
-rw-r--r--Lib/json/decoder.py3
-rw-r--r--Lib/lib2to3/__main__.py4
-rw-r--r--Lib/lib2to3/fixer_base.py4
-rw-r--r--Lib/lib2to3/refactor.py4
-rw-r--r--Lib/logging/__init__.py153
-rw-r--r--Lib/logging/config.py6
-rw-r--r--Lib/logging/handlers.py72
-rw-r--r--Lib/mailbox.py7
-rw-r--r--Lib/mailcap.py6
-rw-r--r--Lib/mimetypes.py1
-rw-r--r--Lib/modulefinder.py16
-rw-r--r--Lib/multiprocessing/connection.py388
-rw-r--r--Lib/multiprocessing/dummy/__init__.py13
-rw-r--r--Lib/multiprocessing/forking.py40
-rw-r--r--Lib/multiprocessing/process.py20
-rw-r--r--Lib/multiprocessing/queues.py19
-rw-r--r--Lib/multiprocessing/reduction.py34
-rw-r--r--Lib/multiprocessing/util.py23
-rw-r--r--Lib/nntplib.py14
-rw-r--r--Lib/opcode.py1
-rw-r--r--Lib/optparse.py31
-rw-r--r--Lib/os.py2
-rw-r--r--Lib/packaging/__init__.py17
-rw-r--r--Lib/packaging/_trove.py559
-rw-r--r--Lib/packaging/command/__init__.py63
-rw-r--r--Lib/packaging/command/bdist.py141
-rw-r--r--Lib/packaging/command/bdist_dumb.py139
-rw-r--r--Lib/packaging/command/bdist_msi.py744
-rw-r--r--Lib/packaging/command/bdist_wininst.py346
-rw-r--r--Lib/packaging/command/build.py151
-rw-r--r--Lib/packaging/command/build_clib.py197
-rw-r--r--Lib/packaging/command/build_ext.py648
-rw-r--r--Lib/packaging/command/build_py.py409
-rw-r--r--Lib/packaging/command/build_scripts.py154
-rw-r--r--Lib/packaging/command/check.py88
-rw-r--r--Lib/packaging/command/clean.py76
-rw-r--r--Lib/packaging/command/cmd.py441
-rw-r--r--Lib/packaging/command/command_template35
-rw-r--r--Lib/packaging/command/config.py349
-rw-r--r--Lib/packaging/command/install_data.py79
-rw-r--r--Lib/packaging/command/install_dist.py607
-rw-r--r--Lib/packaging/command/install_distinfo.py144
-rw-r--r--Lib/packaging/command/install_headers.py43
-rw-r--r--Lib/packaging/command/install_lib.py208
-rw-r--r--Lib/packaging/command/install_scripts.py59
-rw-r--r--Lib/packaging/command/register.py263
-rw-r--r--Lib/packaging/command/sdist.py348
-rw-r--r--Lib/packaging/command/test.py81
-rw-r--r--Lib/packaging/command/upload.py168
-rw-r--r--Lib/packaging/command/upload_docs.py131
-rw-r--r--Lib/packaging/command/wininst-10.0-amd64.exebin0 -> 222208 bytes-rw-r--r--Lib/packaging/command/wininst-10.0.exebin0 -> 190464 bytes-rw-r--r--Lib/packaging/command/wininst-6.0.exebin0 -> 61440 bytes-rw-r--r--Lib/packaging/command/wininst-7.1.exebin0 -> 65536 bytes-rw-r--r--Lib/packaging/command/wininst-8.0.exebin0 -> 61440 bytes-rw-r--r--Lib/packaging/command/wininst-9.0-amd64.exebin0 -> 223744 bytes-rw-r--r--Lib/packaging/command/wininst-9.0.exebin0 -> 196096 bytes-rw-r--r--Lib/packaging/compat.py51
-rw-r--r--Lib/packaging/compiler/__init__.py279
-rw-r--r--Lib/packaging/compiler/bcppcompiler.py355
-rw-r--r--Lib/packaging/compiler/ccompiler.py864
-rw-r--r--Lib/packaging/compiler/cygwinccompiler.py352
-rw-r--r--Lib/packaging/compiler/extension.py121
-rw-r--r--Lib/packaging/compiler/msvc9compiler.py720
-rw-r--r--Lib/packaging/compiler/msvccompiler.py635
-rw-r--r--Lib/packaging/compiler/unixccompiler.py339
-rw-r--r--Lib/packaging/config.py380
-rw-r--r--Lib/packaging/create.py681
-rw-r--r--Lib/packaging/database.py650
-rw-r--r--Lib/packaging/depgraph.py273
-rw-r--r--Lib/packaging/dist.py768
-rw-r--r--Lib/packaging/errors.py138
-rw-r--r--Lib/packaging/fancy_getopt.py388
-rw-r--r--Lib/packaging/install.py538
-rw-r--r--Lib/packaging/manifest.py374
-rw-r--r--Lib/packaging/markers.py187
-rw-r--r--Lib/packaging/metadata.py568
-rw-r--r--Lib/packaging/pypi/__init__.py9
-rw-r--r--Lib/packaging/pypi/base.py48
-rw-r--r--Lib/packaging/pypi/dist.py544
-rw-r--r--Lib/packaging/pypi/errors.py39
-rw-r--r--Lib/packaging/pypi/mirrors.py52
-rw-r--r--Lib/packaging/pypi/simple.py462
-rw-r--r--Lib/packaging/pypi/wrapper.py99
-rw-r--r--Lib/packaging/pypi/xmlrpc.py200
-rw-r--r--Lib/packaging/run.py666
-rw-r--r--Lib/packaging/tests/LONG_DESC.txt44
-rw-r--r--Lib/packaging/tests/PKG-INFO57
-rw-r--r--Lib/packaging/tests/SETUPTOOLS-PKG-INFO182
-rw-r--r--Lib/packaging/tests/SETUPTOOLS-PKG-INFO2183
-rw-r--r--Lib/packaging/tests/__init__.py130
-rw-r--r--Lib/packaging/tests/__main__.py24
-rw-r--r--Lib/packaging/tests/fake_dists/babar-0.1.dist-info/INSTALLER (renamed from Lib/email/test/__init__.py)0
-rw-r--r--Lib/packaging/tests/fake_dists/babar-0.1.dist-info/METADATA4
-rw-r--r--Lib/packaging/tests/fake_dists/babar-0.1.dist-info/RECORD0
-rw-r--r--Lib/packaging/tests/fake_dists/babar-0.1.dist-info/REQUESTED0
-rw-r--r--Lib/packaging/tests/fake_dists/babar-0.1.dist-info/RESOURCES2
-rw-r--r--Lib/packaging/tests/fake_dists/babar.cfg1
-rw-r--r--Lib/packaging/tests/fake_dists/babar.png0
-rw-r--r--Lib/packaging/tests/fake_dists/bacon-0.1.egg-info/PKG-INFO6
-rw-r--r--Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/PKG-INFO18
-rw-r--r--Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/SOURCES.txt0
-rw-r--r--Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/dependency_links.txt1
-rw-r--r--Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/entry_points.txt3
-rw-r--r--Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/not-zip-safe1
-rw-r--r--Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/requires.txt6
-rw-r--r--Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/top_level.txt0
-rw-r--r--Lib/packaging/tests/fake_dists/cheese-2.0.2.egg-info5
-rw-r--r--Lib/packaging/tests/fake_dists/choxie-2.0.0.9.dist-info/INSTALLER0
-rw-r--r--Lib/packaging/tests/fake_dists/choxie-2.0.0.9.dist-info/METADATA9
-rw-r--r--Lib/packaging/tests/fake_dists/choxie-2.0.0.9.dist-info/RECORD0
-rw-r--r--Lib/packaging/tests/fake_dists/choxie-2.0.0.9.dist-info/REQUESTED0
-rw-r--r--Lib/packaging/tests/fake_dists/choxie-2.0.0.9/choxie/__init__.py1
-rw-r--r--Lib/packaging/tests/fake_dists/choxie-2.0.0.9/choxie/chocolate.py10
-rw-r--r--Lib/packaging/tests/fake_dists/choxie-2.0.0.9/truffles.py5
-rw-r--r--Lib/packaging/tests/fake_dists/coconuts-aster-10.3.egg-info/PKG-INFO5
-rw-r--r--Lib/packaging/tests/fake_dists/grammar-1.0a4.dist-info/INSTALLER0
-rw-r--r--Lib/packaging/tests/fake_dists/grammar-1.0a4.dist-info/METADATA5
-rw-r--r--Lib/packaging/tests/fake_dists/grammar-1.0a4.dist-info/RECORD0
-rw-r--r--Lib/packaging/tests/fake_dists/grammar-1.0a4.dist-info/REQUESTED0
-rw-r--r--Lib/packaging/tests/fake_dists/grammar-1.0a4/grammar/__init__.py1
-rw-r--r--Lib/packaging/tests/fake_dists/grammar-1.0a4/grammar/utils.py8
-rw-r--r--Lib/packaging/tests/fake_dists/nut-funkyversion.egg-info3
-rw-r--r--Lib/packaging/tests/fake_dists/strawberry-0.6.eggbin0 -> 1402 bytes-rw-r--r--Lib/packaging/tests/fake_dists/towel_stuff-0.1.dist-info/INSTALLER0
-rw-r--r--Lib/packaging/tests/fake_dists/towel_stuff-0.1.dist-info/METADATA7
-rw-r--r--Lib/packaging/tests/fake_dists/towel_stuff-0.1.dist-info/RECORD0
-rw-r--r--Lib/packaging/tests/fake_dists/towel_stuff-0.1.dist-info/REQUESTED0
-rw-r--r--Lib/packaging/tests/fake_dists/towel_stuff-0.1/towel_stuff/__init__.py18
-rw-r--r--Lib/packaging/tests/fake_dists/truffles-5.0.egg-info3
-rw-r--r--Lib/packaging/tests/fixer/__init__.py0
-rw-r--r--Lib/packaging/tests/fixer/fix_idioms.py134
-rw-r--r--Lib/packaging/tests/pypi_server.py449
-rw-r--r--Lib/packaging/tests/pypi_test_server.py59
-rw-r--r--Lib/packaging/tests/pypiserver/downloads_with_md5/packages/source/f/foobar/foobar-0.1.tar.gzbin0 -> 110 bytes-rw-r--r--Lib/packaging/tests/pypiserver/downloads_with_md5/simple/badmd5/badmd5-0.1.tar.gz0
-rw-r--r--Lib/packaging/tests/pypiserver/downloads_with_md5/simple/badmd5/index.html3
-rw-r--r--Lib/packaging/tests/pypiserver/downloads_with_md5/simple/foobar/index.html3
-rw-r--r--Lib/packaging/tests/pypiserver/downloads_with_md5/simple/index.html2
-rw-r--r--Lib/packaging/tests/pypiserver/foo_bar_baz/simple/bar/index.html6
-rw-r--r--Lib/packaging/tests/pypiserver/foo_bar_baz/simple/baz/index.html6
-rw-r--r--Lib/packaging/tests/pypiserver/foo_bar_baz/simple/foo/index.html6
-rw-r--r--Lib/packaging/tests/pypiserver/foo_bar_baz/simple/index.html3
-rw-r--r--Lib/packaging/tests/pypiserver/project_list/simple/index.html5
-rw-r--r--Lib/packaging/tests/pypiserver/test_found_links/simple/foobar/index.html6
-rw-r--r--Lib/packaging/tests/pypiserver/test_found_links/simple/index.html1
-rw-r--r--Lib/packaging/tests/pypiserver/test_pypi_server/external/index.html1
-rw-r--r--Lib/packaging/tests/pypiserver/test_pypi_server/simple/index.html1
-rw-r--r--Lib/packaging/tests/pypiserver/with_externals/external/external.html3
-rw-r--r--Lib/packaging/tests/pypiserver/with_externals/simple/foobar/index.html4
-rw-r--r--Lib/packaging/tests/pypiserver/with_externals/simple/index.html1
-rw-r--r--Lib/packaging/tests/pypiserver/with_norel_links/external/homepage.html7
-rw-r--r--Lib/packaging/tests/pypiserver/with_norel_links/external/nonrel.html1
-rw-r--r--Lib/packaging/tests/pypiserver/with_norel_links/simple/foobar/index.html6
-rw-r--r--Lib/packaging/tests/pypiserver/with_norel_links/simple/index.html1
-rw-r--r--Lib/packaging/tests/pypiserver/with_real_externals/simple/foobar/index.html4
-rw-r--r--Lib/packaging/tests/pypiserver/with_real_externals/simple/index.html1
-rw-r--r--Lib/packaging/tests/support.py363
-rw-r--r--Lib/packaging/tests/test_ccompiler.py15
-rw-r--r--Lib/packaging/tests/test_command_bdist.py58
-rw-r--r--Lib/packaging/tests/test_command_bdist_dumb.py91
-rw-r--r--Lib/packaging/tests/test_command_bdist_msi.py25
-rw-r--r--Lib/packaging/tests/test_command_bdist_wininst.py32
-rw-r--r--Lib/packaging/tests/test_command_build.py55
-rw-r--r--Lib/packaging/tests/test_command_build_clib.py141
-rw-r--r--Lib/packaging/tests/test_command_build_ext.py409
-rw-r--r--Lib/packaging/tests/test_command_build_py.py146
-rw-r--r--Lib/packaging/tests/test_command_build_scripts.py109
-rw-r--r--Lib/packaging/tests/test_command_check.py161
-rw-r--r--Lib/packaging/tests/test_command_clean.py48
-rw-r--r--Lib/packaging/tests/test_command_cmd.py102
-rw-r--r--Lib/packaging/tests/test_command_config.py76
-rw-r--r--Lib/packaging/tests/test_command_install_data.py144
-rw-r--r--Lib/packaging/tests/test_command_install_dist.py241
-rw-r--r--Lib/packaging/tests/test_command_install_distinfo.py252
-rw-r--r--Lib/packaging/tests/test_command_install_headers.py38
-rw-r--r--Lib/packaging/tests/test_command_install_lib.py110
-rw-r--r--Lib/packaging/tests/test_command_install_scripts.py75
-rw-r--r--Lib/packaging/tests/test_command_register.py271
-rw-r--r--Lib/packaging/tests/test_command_sdist.py393
-rw-r--r--Lib/packaging/tests/test_command_test.py224
-rw-r--r--Lib/packaging/tests/test_command_upload.py159
-rw-r--r--Lib/packaging/tests/test_command_upload_docs.py186
-rw-r--r--Lib/packaging/tests/test_compiler.py66
-rw-r--r--Lib/packaging/tests/test_config.py504
-rw-r--r--Lib/packaging/tests/test_create.py245
-rw-r--r--Lib/packaging/tests/test_cygwinccompiler.py88
-rw-r--r--Lib/packaging/tests/test_database.py679
-rw-r--r--Lib/packaging/tests/test_depgraph.py310
-rw-r--r--Lib/packaging/tests/test_dist.py266
-rw-r--r--Lib/packaging/tests/test_extension.py15
-rw-r--r--Lib/packaging/tests/test_install.py391
-rw-r--r--Lib/packaging/tests/test_manifest.py271
-rw-r--r--Lib/packaging/tests/test_markers.py71
-rw-r--r--Lib/packaging/tests/test_metadata.py454
-rw-r--r--Lib/packaging/tests/test_mixin2to3.py81
-rw-r--r--Lib/packaging/tests/test_msvc9compiler.py140
-rw-r--r--Lib/packaging/tests/test_pypi_dist.py287
-rw-r--r--Lib/packaging/tests/test_pypi_server.py88
-rw-r--r--Lib/packaging/tests/test_pypi_simple.py353
-rw-r--r--Lib/packaging/tests/test_pypi_xmlrpc.py101
-rw-r--r--Lib/packaging/tests/test_run.py77
-rw-r--r--Lib/packaging/tests/test_support.py78
-rw-r--r--Lib/packaging/tests/test_uninstall.py134
-rw-r--r--Lib/packaging/tests/test_unixccompiler.py132
-rw-r--r--Lib/packaging/tests/test_util.py1016
-rw-r--r--Lib/packaging/tests/test_version.py262
-rw-r--r--Lib/packaging/util.py1486
-rw-r--r--Lib/packaging/version.py449
-rw-r--r--Lib/pickle.py4
-rw-r--r--Lib/pickletools.py5
-rw-r--r--Lib/pipes.py23
-rw-r--r--Lib/pkgutil.py8
-rw-r--r--Lib/plat-linux/CDROM.py (renamed from Lib/plat-linux2/CDROM.py)0
-rw-r--r--Lib/plat-linux/DLFCN.py (renamed from Lib/plat-linux2/DLFCN.py)0
-rw-r--r--Lib/plat-linux/IN.py (renamed from Lib/plat-linux2/IN.py)0
-rw-r--r--Lib/plat-linux/TYPES.py (renamed from Lib/plat-linux2/TYPES.py)0
-rwxr-xr-xLib/plat-linux/regen (renamed from Lib/plat-linux2/regen)0
-rwxr-xr-xLib/platform.py110
-rw-r--r--Lib/plistlib.py10
-rw-r--r--Lib/poplib.py17
-rw-r--r--Lib/pstats.py1
-rwxr-xr-xLib/pydoc.py289
-rw-r--r--Lib/pydoc_data/topics.py20
-rw-r--r--Lib/random.py2
-rw-r--r--Lib/re.py15
-rw-r--r--Lib/shlex.py20
-rw-r--r--Lib/shutil.py85
-rw-r--r--Lib/site.py17
-rwxr-xr-xLib/smtpd.py2
-rw-r--r--Lib/smtplib.py152
-rw-r--r--Lib/socket.py11
-rw-r--r--Lib/socketserver.py20
-rw-r--r--Lib/sqlite3/test/hooks.py52
-rw-r--r--Lib/sqlite3/test/regression.py22
-rw-r--r--Lib/sqlite3/test/types.py2
-rw-r--r--Lib/sre_compile.py4
-rw-r--r--Lib/sre_parse.py2
-rw-r--r--Lib/ssl.py50
-rw-r--r--Lib/string.py24
-rw-r--r--Lib/subprocess.py685
-rw-r--r--Lib/sysconfig.cfg111
-rw-r--r--Lib/sysconfig.py308
-rwxr-xr-xLib/tabnanny.py3
-rw-r--r--Lib/tarfile.py26
-rw-r--r--Lib/tempfile.py47
-rw-r--r--Lib/test/buffer_tests.py8
-rw-r--r--Lib/test/crashers/README4
-rw-r--r--Lib/test/crashers/compiler_recursion.py12
-rw-r--r--Lib/test/datetimetester.py18
-rw-r--r--Lib/test/decimaltestdata/extra.decTest13
-rw-r--r--Lib/test/exception_hierarchy.txt21
-rw-r--r--Lib/test/fork_wait.py10
-rw-r--r--Lib/test/future_test1.py (renamed from Lib/test/test_future1.py)0
-rw-r--r--Lib/test/future_test2.py (renamed from Lib/test/test_future2.py)0
-rw-r--r--Lib/test/json_tests/test_dump.py19
-rw-r--r--Lib/test/json_tests/test_scanstring.py11
-rw-r--r--Lib/test/keycert.passwd.pem33
-rw-r--r--Lib/test/list_tests.py41
-rw-r--r--Lib/test/lock_tests.py6
-rw-r--r--Lib/test/mailcap.txt39
-rw-r--r--Lib/test/math_testcases.txt114
-rw-r--r--Lib/test/mock_socket.py3
-rw-r--r--Lib/test/pickletester.py39
-rwxr-xr-xLib/test/regrtest.py262
-rw-r--r--Lib/test/script_helper.py5
-rw-r--r--Lib/test/ssl_key.passwd.pem18
-rw-r--r--Lib/test/string_tests.py49
-rw-r--r--Lib/test/support.py314
-rw-r--r--Lib/test/test_abc.py24
-rw-r--r--Lib/test/test_abstract_numbers.py2
-rw-r--r--Lib/test/test_argparse.py94
-rwxr-xr-xLib/test/test_array.py32
-rw-r--r--Lib/test/test_ast.py419
-rw-r--r--Lib/test/test_asyncore.py178
-rw-r--r--Lib/test/test_bigmem.py178
-rw-r--r--Lib/test/test_bool.py10
-rw-r--r--Lib/test/test_builtin.py44
-rw-r--r--Lib/test/test_bytes.py175
-rw-r--r--Lib/test/test_bz2.py353
-rw-r--r--Lib/test/test_calendar.py2
-rw-r--r--Lib/test/test_capi.py5
-rw-r--r--Lib/test/test_cgi.py8
-rw-r--r--Lib/test/test_cgitb.py55
-rw-r--r--Lib/test/test_cmd_line.py10
-rw-r--r--Lib/test/test_codeccallbacks.py81
-rw-r--r--Lib/test/test_codecencodings_cn.py21
-rw-r--r--Lib/test/test_codecencodings_hk.py4
-rw-r--r--Lib/test/test_codecencodings_jp.py96
-rw-r--r--Lib/test/test_codecencodings_kr.py25
-rw-r--r--Lib/test/test_codecencodings_tw.py4
-rw-r--r--Lib/test/test_codecmaps_tw.py3
-rw-r--r--Lib/test/test_codecs.py355
-rw-r--r--Lib/test/test_collections.py96
-rw-r--r--Lib/test/test_compile.py9
-rw-r--r--Lib/test/test_concurrent_futures.py62
-rw-r--r--Lib/test/test_configparser.py (renamed from Lib/test/test_cfgparser.py)0
-rw-r--r--Lib/test/test_copy.py187
-rw-r--r--Lib/test/test_cprofile.py23
-rw-r--r--Lib/test/test_crashers.py38
-rw-r--r--Lib/test/test_crypt.py19
-rw-r--r--Lib/test/test_curses.py23
-rw-r--r--Lib/test/test_dbm.py4
-rw-r--r--Lib/test/test_decimal.py26
-rw-r--r--Lib/test/test_descr.py77
-rw-r--r--Lib/test/test_descrtut.py3
-rw-r--r--Lib/test/test_dis.py106
-rw-r--r--Lib/test/test_doctest.py444
-rw-r--r--Lib/test/test_dummy_thread.py4
-rw-r--r--Lib/test/test_email.py14
-rw-r--r--Lib/test/test_email/__init__.py45
-rw-r--r--Lib/test/test_email/__main__.py3
-rw-r--r--Lib/test/test_email/data/PyBanner048.gif (renamed from Lib/email/test/data/PyBanner048.gif)bin954 -> 954 bytes-rw-r--r--Lib/test/test_email/data/audiotest.au (renamed from Lib/email/test/data/audiotest.au)bin28144 -> 28144 bytes-rw-r--r--Lib/test/test_email/data/msg_01.txt (renamed from Lib/email/test/data/msg_01.txt)0
-rw-r--r--Lib/test/test_email/data/msg_02.txt (renamed from Lib/email/test/data/msg_02.txt)0
-rw-r--r--Lib/test/test_email/data/msg_03.txt (renamed from Lib/email/test/data/msg_03.txt)0
-rw-r--r--Lib/test/test_email/data/msg_04.txt (renamed from Lib/email/test/data/msg_04.txt)0
-rw-r--r--Lib/test/test_email/data/msg_05.txt (renamed from Lib/email/test/data/msg_05.txt)0
-rw-r--r--Lib/test/test_email/data/msg_06.txt (renamed from Lib/email/test/data/msg_06.txt)0
-rw-r--r--Lib/test/test_email/data/msg_07.txt (renamed from Lib/email/test/data/msg_07.txt)0
-rw-r--r--Lib/test/test_email/data/msg_08.txt (renamed from Lib/email/test/data/msg_08.txt)0
-rw-r--r--Lib/test/test_email/data/msg_09.txt (renamed from Lib/email/test/data/msg_09.txt)0
-rw-r--r--Lib/test/test_email/data/msg_10.txt (renamed from Lib/email/test/data/msg_10.txt)0
-rw-r--r--Lib/test/test_email/data/msg_11.txt (renamed from Lib/email/test/data/msg_11.txt)0
-rw-r--r--Lib/test/test_email/data/msg_12.txt (renamed from Lib/email/test/data/msg_12.txt)0
-rw-r--r--Lib/test/test_email/data/msg_12a.txt (renamed from Lib/email/test/data/msg_12a.txt)0
-rw-r--r--Lib/test/test_email/data/msg_13.txt (renamed from Lib/email/test/data/msg_13.txt)0
-rw-r--r--Lib/test/test_email/data/msg_14.txt (renamed from Lib/email/test/data/msg_14.txt)0
-rw-r--r--Lib/test/test_email/data/msg_15.txt (renamed from Lib/email/test/data/msg_15.txt)0
-rw-r--r--Lib/test/test_email/data/msg_16.txt (renamed from Lib/email/test/data/msg_16.txt)0
-rw-r--r--Lib/test/test_email/data/msg_17.txt (renamed from Lib/email/test/data/msg_17.txt)0
-rw-r--r--Lib/test/test_email/data/msg_18.txt (renamed from Lib/email/test/data/msg_18.txt)0
-rw-r--r--Lib/test/test_email/data/msg_19.txt (renamed from Lib/email/test/data/msg_19.txt)0
-rw-r--r--Lib/test/test_email/data/msg_20.txt (renamed from Lib/email/test/data/msg_20.txt)0
-rw-r--r--Lib/test/test_email/data/msg_21.txt (renamed from Lib/email/test/data/msg_21.txt)0
-rw-r--r--Lib/test/test_email/data/msg_22.txt (renamed from Lib/email/test/data/msg_22.txt)0
-rw-r--r--Lib/test/test_email/data/msg_23.txt (renamed from Lib/email/test/data/msg_23.txt)0
-rw-r--r--Lib/test/test_email/data/msg_24.txt (renamed from Lib/email/test/data/msg_24.txt)0
-rw-r--r--Lib/test/test_email/data/msg_25.txt (renamed from Lib/email/test/data/msg_25.txt)0
-rw-r--r--Lib/test/test_email/data/msg_26.txt (renamed from Lib/email/test/data/msg_26.txt)0
-rw-r--r--Lib/test/test_email/data/msg_27.txt (renamed from Lib/email/test/data/msg_27.txt)0
-rw-r--r--Lib/test/test_email/data/msg_28.txt (renamed from Lib/email/test/data/msg_28.txt)0
-rw-r--r--Lib/test/test_email/data/msg_29.txt (renamed from Lib/email/test/data/msg_29.txt)0
-rw-r--r--Lib/test/test_email/data/msg_30.txt (renamed from Lib/email/test/data/msg_30.txt)0
-rw-r--r--Lib/test/test_email/data/msg_31.txt (renamed from Lib/email/test/data/msg_31.txt)0
-rw-r--r--Lib/test/test_email/data/msg_32.txt (renamed from Lib/email/test/data/msg_32.txt)0
-rw-r--r--Lib/test/test_email/data/msg_33.txt (renamed from Lib/email/test/data/msg_33.txt)0
-rw-r--r--Lib/test/test_email/data/msg_34.txt (renamed from Lib/email/test/data/msg_34.txt)0
-rw-r--r--Lib/test/test_email/data/msg_35.txt (renamed from Lib/email/test/data/msg_35.txt)0
-rw-r--r--Lib/test/test_email/data/msg_36.txt (renamed from Lib/email/test/data/msg_36.txt)0
-rw-r--r--Lib/test/test_email/data/msg_37.txt (renamed from Lib/email/test/data/msg_37.txt)0
-rw-r--r--Lib/test/test_email/data/msg_38.txt (renamed from Lib/email/test/data/msg_38.txt)0
-rw-r--r--Lib/test/test_email/data/msg_39.txt (renamed from Lib/email/test/data/msg_39.txt)0
-rw-r--r--Lib/test/test_email/data/msg_40.txt (renamed from Lib/email/test/data/msg_40.txt)0
-rw-r--r--Lib/test/test_email/data/msg_41.txt (renamed from Lib/email/test/data/msg_41.txt)0
-rw-r--r--Lib/test/test_email/data/msg_42.txt (renamed from Lib/email/test/data/msg_42.txt)0
-rw-r--r--Lib/test/test_email/data/msg_43.txt (renamed from Lib/email/test/data/msg_43.txt)0
-rw-r--r--Lib/test/test_email/data/msg_44.txt (renamed from Lib/email/test/data/msg_44.txt)0
-rw-r--r--Lib/test/test_email/data/msg_45.txt (renamed from Lib/email/test/data/msg_45.txt)0
-rw-r--r--Lib/test/test_email/data/msg_46.txt (renamed from Lib/email/test/data/msg_46.txt)0
-rw-r--r--Lib/test/test_email/test_asian_codecs.py (renamed from Lib/email/test/test_email_codecs.py)15
-rw-r--r--Lib/test/test_email/test_email.py (renamed from Lib/email/test/test_email.py)329
-rw-r--r--Lib/test/test_email/test_generator.py136
-rw-r--r--Lib/test/test_email/test_policy.py148
-rw-r--r--Lib/test/test_email/test_utils.py45
-rw-r--r--Lib/test/test_email/torture_test.py (renamed from Lib/email/test/test_email_torture.py)0
-rw-r--r--Lib/test/test_exceptions.py48
-rw-r--r--Lib/test/test_extcall.py87
-rw-r--r--Lib/test/test_faulthandler.py554
-rw-r--r--Lib/test/test_fileinput.py640
-rw-r--r--Lib/test/test_float.py2
-rw-r--r--Lib/test/test_ftplib.py162
-rw-r--r--Lib/test/test_functools.py81
-rw-r--r--Lib/test/test_future.py12
-rw-r--r--Lib/test/test_gc.py6
-rw-r--r--Lib/test/test_genexps.py8
-rw-r--r--Lib/test/test_getargs2.py9
-rw-r--r--Lib/test/test_glob.py6
-rw-r--r--Lib/test/test_grammar.py91
-rw-r--r--Lib/test/test_gzip.py25
-rw-r--r--Lib/test/test_hash.py3
-rw-r--r--Lib/test/test_http_cookiejar.py11
-rw-r--r--Lib/test/test_httplib.py8
-rw-r--r--Lib/test/test_httpservers.py73
-rw-r--r--Lib/test/test_imaplib.py55
-rw-r--r--Lib/test/test_imp.py13
-rw-r--r--Lib/test/test_import.py6
-rw-r--r--Lib/test/test_importhooks.py9
-rw-r--r--Lib/test/test_inspect.py2
-rw-r--r--Lib/test/test_io.py51
-rw-r--r--Lib/test/test_itertools.py33
-rw-r--r--Lib/test/test_keywordonlyarg.py2
-rw-r--r--Lib/test/test_lib2to3.py4
-rw-r--r--Lib/test/test_locale.py2
-rw-r--r--Lib/test/test_logging.py1619
-rw-r--r--Lib/test/test_long.py80
-rw-r--r--Lib/test/test_mailbox.py9
-rw-r--r--Lib/test/test_mailcap.py221
-rw-r--r--Lib/test/test_marshal.py17
-rw-r--r--Lib/test/test_math.py43
-rw-r--r--Lib/test/test_metaclass.py8
-rw-r--r--Lib/test/test_minidom.py81
-rw-r--r--Lib/test/test_mmap.py32
-rw-r--r--Lib/test/test_modulefinder.py48
-rw-r--r--Lib/test/test_multibytecodec.py16
-rw-r--r--Lib/test/test_multibytecodec_support.py15
-rw-r--r--Lib/test/test_multiprocessing.py61
-rw-r--r--Lib/test/test_nntplib.py27
-rw-r--r--Lib/test/test_optparse.py4
-rw-r--r--Lib/test/test_os.py402
-rw-r--r--Lib/test/test_ossaudiodev.py16
-rw-r--r--Lib/test/test_osx_env.py3
-rw-r--r--Lib/test/test_packaging.py5
-rw-r--r--Lib/test/test_parser.py8
-rw-r--r--Lib/test/test_pdb.py3
-rw-r--r--Lib/test/test_peepholer.py47
-rw-r--r--Lib/test/test_pep277.py42
-rw-r--r--Lib/test/test_pep292.py33
-rw-r--r--Lib/test/test_pep3120.py4
-rw-r--r--Lib/test/test_pep3131.py7
-rw-r--r--Lib/test/test_pep3151.py151
-rw-r--r--Lib/test/test_pipes.py15
-rw-r--r--Lib/test/test_pkgimport.py4
-rw-r--r--Lib/test/test_platform.py43
-rw-r--r--Lib/test/test_poplib.py15
-rw-r--r--Lib/test/test_posix.py575
-rw-r--r--Lib/test/test_pulldom.py347
-rw-r--r--Lib/test/test_pydoc.py12
-rw-r--r--Lib/test/test_raise.py7
-rw-r--r--Lib/test/test_range.py52
-rw-r--r--Lib/test/test_re.py11
-rw-r--r--Lib/test/test_reprlib.py25
-rw-r--r--Lib/test/test_richcmp.py1
-rw-r--r--Lib/test/test_runpy.py14
-rw-r--r--Lib/test/test_sax.py4
-rw-r--r--Lib/test/test_scope.py19
-rw-r--r--Lib/test/test_select.py22
-rw-r--r--Lib/test/test_shelve.py6
-rw-r--r--Lib/test/test_shlex.py21
-rw-r--r--Lib/test/test_shutil.py258
-rw-r--r--Lib/test/test_signal.py471
-rw-r--r--Lib/test/test_smtplib.py60
-rw-r--r--Lib/test/test_smtpnet.py42
-rw-r--r--Lib/test/test_socket.py2439
-rw-r--r--Lib/test/test_ssl.py239
-rw-r--r--Lib/test/test_string.py68
-rw-r--r--Lib/test/test_strlit.py2
-rw-r--r--Lib/test/test_struct.py66
-rw-r--r--Lib/test/test_subprocess.py136
-rw-r--r--Lib/test/test_super.py10
-rw-r--r--Lib/test/test_support.py188
-rw-r--r--Lib/test/test_sys.py54
-rw-r--r--Lib/test/test_sys_settrace.py10
-rw-r--r--Lib/test/test_sysconfig.py83
-rw-r--r--Lib/test/test_tarfile.py16
-rw-r--r--Lib/test/test_telnetlib.py1
-rw-r--r--Lib/test/test_tempfile.py2
-rw-r--r--Lib/test/test_threaded_import.py13
-rw-r--r--Lib/test/test_threading.py19
-rw-r--r--Lib/test/test_threadsignals.py6
-rw-r--r--Lib/test/test_time.py245
-rw-r--r--Lib/test/test_tokenize.py2
-rw-r--r--Lib/test/test_trace.py11
-rw-r--r--Lib/test/test_ucn.py88
-rw-r--r--Lib/test/test_unicode.py160
-rw-r--r--Lib/test/test_unicode_file.py5
-rw-r--r--Lib/test/test_unicodedata.py12
-rw-r--r--Lib/test/test_urllib.py25
-rw-r--r--Lib/test/test_urllib2.py42
-rw-r--r--Lib/test/test_userlist.py6
-rwxr-xr-xLib/test/test_userstring.py4
-rw-r--r--Lib/test/test_uuid.py4
-rw-r--r--Lib/test/test_wait3.py7
-rw-r--r--Lib/test/test_warnings.py14
-rw-r--r--Lib/test/test_xml_etree.py14
-rw-r--r--Lib/test/test_xmlrpc_net.py4
-rw-r--r--Lib/test/test_zipfile.py76
-rw-r--r--Lib/test/test_zipfile64.py20
-rw-r--r--Lib/test/test_zipimport.py10
-rw-r--r--Lib/test/test_zipimport_support.py5
-rw-r--r--Lib/test/test_zlib.py32
-rw-r--r--Lib/test/threaded_import_hangers.py13
-rw-r--r--Lib/textwrap.py2
-rw-r--r--Lib/threading.py75
-rw-r--r--Lib/timeit.py8
-rw-r--r--Lib/tkinter/__init__.py7
-rw-r--r--Lib/tkinter/filedialog.py2
-rw-r--r--Lib/tokenize.py114
-rw-r--r--Lib/turtle.py1
-rw-r--r--Lib/unittest/case.py131
-rw-r--r--Lib/unittest/main.py1
-rw-r--r--Lib/unittest/result.py1
-rw-r--r--Lib/unittest/test/_test_warnings.py1
-rw-r--r--Lib/unittest/test/test_assertions.py73
-rw-r--r--Lib/unittest/test/test_case.py63
-rw-r--r--Lib/unittest/test/test_loader.py4
-rw-r--r--Lib/urllib/error.py3
-rw-r--r--Lib/urllib/request.py80
-rw-r--r--Lib/urllib/response.py7
-rw-r--r--Lib/wsgiref.egg-info8
-rw-r--r--Lib/xdrlib.py6
-rw-r--r--Lib/xml/parsers/expat.py2
-rw-r--r--Lib/xmlrpc/client.py77
-rw-r--r--Lib/xmlrpc/server.py9
-rw-r--r--Makefile.pre.in339
-rw-r--r--Misc/ACKS71
-rw-r--r--Misc/NEWS1607
-rw-r--r--Misc/README1
-rw-r--r--Misc/RPM/python-3.3.spec (renamed from Misc/RPM/python-3.2.spec)4
-rw-r--r--Misc/python.man17
-rw-r--r--Misc/svnmap.txt72546
-rw-r--r--Modules/Setup.dist5
-rw-r--r--Modules/_bisectmodule.c8
-rw-r--r--Modules/_bz2module.c590
-rw-r--r--Modules/_codecsmodule.c79
-rw-r--r--Modules/_collectionsmodule.c26
-rw-r--r--Modules/_cryptmodule.c (renamed from Modules/cryptmodule.c)4
-rw-r--r--Modules/_csv.c147
-rw-r--r--Modules/_ctypes/_ctypes.c10
-rw-r--r--Modules/_ctypes/callproc.c18
-rw-r--r--Modules/_ctypes/cfield.c69
-rw-r--r--Modules/_ctypes/stgdict.c12
-rw-r--r--Modules/_cursesmodule.c134
-rw-r--r--Modules/_datetimemodule.c195
-rw-r--r--Modules/_dbmmodule.c12
-rw-r--r--Modules/_elementtree.c112
-rw-r--r--Modules/_functoolsmodule.c172
-rw-r--r--Modules/_hashopenssl.c8
-rw-r--r--Modules/_io/_iomodule.c122
-rw-r--r--Modules/_io/_iomodule.h12
-rw-r--r--Modules/_io/bufferedio.c206
-rw-r--r--Modules/_io/bytesio.c2
-rw-r--r--Modules/_io/fileio.c110
-rw-r--r--Modules/_io/iobase.c36
-rw-r--r--Modules/_io/stringio.c75
-rw-r--r--Modules/_io/textio.c609
-rw-r--r--Modules/_json.c436
-rw-r--r--Modules/_localemodule.c41
-rw-r--r--Modules/_lsprof.c45
-rw-r--r--Modules/_multiprocessing/connection.h527
-rw-r--r--Modules/_multiprocessing/multiprocessing.c163
-rw-r--r--Modules/_multiprocessing/multiprocessing.h79
-rw-r--r--Modules/_multiprocessing/pipe_connection.c149
-rw-r--r--Modules/_multiprocessing/semaphore.c4
-rw-r--r--Modules/_multiprocessing/socket_connection.c202
-rw-r--r--Modules/_multiprocessing/win32_functions.c536
-rw-r--r--Modules/_pickle.c235
-rw-r--r--Modules/_posixsubprocess.c15
-rw-r--r--Modules/_sqlite/connection.c99
-rw-r--r--Modules/_sqlite/cursor.c40
-rw-r--r--Modules/_sqlite/cursor.h1
-rw-r--r--Modules/_sqlite/microprotocols.c8
-rw-r--r--Modules/_sqlite/module.c3
-rw-r--r--Modules/_sqlite/row.c10
-rw-r--r--Modules/_sqlite/statement.c4
-rw-r--r--Modules/_sre.c616
-rw-r--r--Modules/_ssl.c371
-rw-r--r--Modules/_struct.c90
-rw-r--r--Modules/_testcapimodule.c93
-rw-r--r--Modules/_threadmodule.c22
-rw-r--r--Modules/_tkinter.c83
-rw-r--r--Modules/arraymodule.c222
-rw-r--r--Modules/audioop.c4
-rw-r--r--Modules/binascii.c8
-rw-r--r--Modules/bz2module.c2169
-rw-r--r--Modules/cjkcodecs/_codecs_cn.c14
-rw-r--r--Modules/cjkcodecs/_codecs_hk.c2
-rw-r--r--Modules/cjkcodecs/_codecs_iso2022.c2
-rw-r--r--Modules/cjkcodecs/_codecs_jp.c34
-rw-r--r--Modules/cjkcodecs/_codecs_kr.c18
-rw-r--r--Modules/cjkcodecs/_codecs_tw.c4
-rw-r--r--Modules/cjkcodecs/multibytecodec.c22
-rw-r--r--Modules/errnomodule.c55
-rw-r--r--Modules/faulthandler.c1329
-rw-r--r--Modules/fcntlmodule.c5
-rw-r--r--Modules/fpectlmodule.c11
-rw-r--r--Modules/gcmodule.c8
-rw-r--r--Modules/getbuildinfo.c18
-rw-r--r--Modules/getpath.c16
-rw-r--r--Modules/itertoolsmodule.c30
-rw-r--r--Modules/main.c5
-rw-r--r--Modules/mathmodule.c75
-rw-r--r--Modules/md5module.c20
-rw-r--r--Modules/mmapmodule.c62
-rw-r--r--Modules/nismodule.c2
-rw-r--r--Modules/operator.c27
-rw-r--r--Modules/ossaudiodev.c101
-rw-r--r--Modules/parsermodule.c13
-rw-r--r--Modules/posixmodule.c3537
-rw-r--r--Modules/pyexpat.c43
-rw-r--r--Modules/readline.c13
-rw-r--r--Modules/selectmodule.c34
-rw-r--r--Modules/sha1module.c20
-rw-r--r--Modules/sha256module.c18
-rw-r--r--Modules/sha512module.c18
-rw-r--r--Modules/signalmodule.c408
-rw-r--r--Modules/socketmodule.c1182
-rw-r--r--Modules/socketmodule.h16
-rw-r--r--Modules/sre.h4
-rw-r--r--Modules/syslogmodule.c14
-rw-r--r--Modules/termios.c5
-rw-r--r--Modules/testcapi_long.h26
-rw-r--r--Modules/timemodule.c238
-rw-r--r--Modules/unicodedata.c301
-rw-r--r--Modules/unicodename_db.h35013
-rw-r--r--Modules/xxlimited.c3
-rw-r--r--Modules/zipimport.c503
-rw-r--r--Modules/zlibmodule.c27
-rw-r--r--Objects/abstract.c123
-rw-r--r--Objects/bytearrayobject.c261
-rw-r--r--Objects/bytesobject.c216
-rw-r--r--Objects/classobject.c14
-rw-r--r--Objects/codeobject.c116
-rw-r--r--Objects/complexobject.c36
-rw-r--r--Objects/descrobject.c20
-rw-r--r--Objects/dictobject.c49
-rw-r--r--Objects/exceptions.c569
-rw-r--r--Objects/fileobject.c34
-rw-r--r--Objects/floatobject.c32
-rw-r--r--Objects/genobject.c6
-rw-r--r--Objects/listobject.c41
-rw-r--r--Objects/longobject.c144
-rw-r--r--Objects/memoryobject.c3
-rw-r--r--Objects/methodobject.c3
-rw-r--r--Objects/moduleobject.c106
-rw-r--r--Objects/object.c387
-rw-r--r--Objects/obmalloc.c31
-rw-r--r--Objects/rangeobject.c135
-rw-r--r--Objects/setobject.c124
-rw-r--r--Objects/sliceobject.c35
-rw-r--r--Objects/stringlib/asciilib.h35
-rw-r--r--Objects/stringlib/count.h9
-rw-r--r--Objects/stringlib/eq.h23
-rw-r--r--Objects/stringlib/fastsearch.h76
-rw-r--r--Objects/stringlib/find.h82
-rw-r--r--Objects/stringlib/find_max_char.h136
-rw-r--r--Objects/stringlib/formatter.h1518
-rw-r--r--Objects/stringlib/localeutil.h27
-rw-r--r--Objects/stringlib/partition.h12
-rw-r--r--Objects/stringlib/split.h26
-rw-r--r--Objects/stringlib/stringdefs.h3
-rw-r--r--Objects/stringlib/ucs1lib.h36
-rw-r--r--Objects/stringlib/ucs2lib.h35
-rw-r--r--Objects/stringlib/ucs4lib.h35
-rw-r--r--Objects/stringlib/undef.h11
-rw-r--r--Objects/stringlib/unicode_format.h (renamed from Objects/stringlib/string_format.h)401
-rw-r--r--Objects/stringlib/unicodedefs.h3
-rw-r--r--Objects/tupleobject.c17
-rw-r--r--Objects/typeobject.c392
-rw-r--r--Objects/typeslots.inc2
-rw-r--r--Objects/typeslots.py2
-rw-r--r--Objects/unicodeobject.c9429
-rw-r--r--Objects/weakrefobject.c61
-rw-r--r--PC/VC6/_multiprocessing.dsp8
-rw-r--r--PC/VC6/pythoncore.dsp4
-rw-r--r--PC/VC6/readme.txt4
-rw-r--r--PC/VS7.1/pythoncore.vcproj24
-rw-r--r--PC/VS7.1/readme.txt4
-rw-r--r--PC/VS8.0/_multiprocessing.vcproj12
-rw-r--r--PC/VS8.0/build_ssl.bat4
-rw-r--r--PC/VS8.0/kill_python.c2
-rw-r--r--PC/VS8.0/pyproject.vsprops2
-rw-r--r--PC/_msi.c8
-rw-r--r--PC/_subprocess.c62
-rw-r--r--PC/config.c2
-rw-r--r--PC/example_nt/example.vcproj4
-rw-r--r--PC/frozen_dllmain.c4
-rw-r--r--PC/getpathp.c6
-rw-r--r--PC/import_nt.c121
-rwxr-xr-xPC/msvcrtmodule.c8
-rw-r--r--PC/os2emx/Makefile2
-rw-r--r--PC/os2emx/README.os2emx2
-rw-r--r--PC/os2emx/pyconfig.h2
-rw-r--r--PC/os2emx/python33.def (renamed from PC/os2emx/python27.def)250
-rw-r--r--PC/pyconfig.h11
-rw-r--r--PC/python3.def1374
-rw-r--r--PC/python3.mak12
-rw-r--r--PC/python33gen.py (renamed from PC/python32gen.py)9
-rw-r--r--PC/python33stub.def (renamed from PC/python32stub.def)2
-rw-r--r--PC/winreg.c8
-rw-r--r--PCbuild/_bz2.vcproj (renamed from PCbuild/bz2.vcproj)4
-rw-r--r--PCbuild/_multiprocessing.vcproj12
-rw-r--r--PCbuild/build_ssl.bat4
-rw-r--r--PCbuild/kill_python.c2
-rw-r--r--PCbuild/pcbuild.sln2
-rw-r--r--PCbuild/pyproject.vsprops2
-rw-r--r--PCbuild/pythoncore.vcproj8
-rw-r--r--PCbuild/readme.txt8
-rw-r--r--Parser/Python.asdl14
-rw-r--r--Parser/asdl.py25
-rwxr-xr-xParser/asdl_c.py59
-rw-r--r--Parser/intrcheck.c174
-rw-r--r--Parser/parsetok.c37
-rw-r--r--Parser/parsetok_pgen.c2
-rw-r--r--Parser/pgenmain.c2
-rw-r--r--Parser/tokenizer.c79
-rw-r--r--Parser/tokenizer.h9
-rw-r--r--Python/Python-ast.c1174
-rw-r--r--Python/_warnings.c67
-rw-r--r--Python/ast.c614
-rw-r--r--Python/bltinmodule.c89
-rw-r--r--Python/ceval.c673
-rw-r--r--Python/codecs.c217
-rw-r--r--Python/compile.c179
-rw-r--r--Python/dtoa.c4
-rw-r--r--Python/dynload_aix.c5
-rw-r--r--Python/dynload_dl.c16
-rw-r--r--Python/dynload_hpux.c2
-rw-r--r--Python/dynload_next.c4
-rw-r--r--Python/dynload_os2.c2
-rw-r--r--Python/dynload_shlib.c2
-rw-r--r--Python/dynload_win.c48
-rw-r--r--Python/errors.c70
-rw-r--r--Python/formatter_unicode.c1473
-rw-r--r--Python/getargs.c128
-rw-r--r--Python/import.c2502
-rw-r--r--Python/importdl.c84
-rw-r--r--Python/importdl.h2
-rwxr-xr-xPython/makeopcodetargets.py3
-rw-r--r--Python/marshal.c50
-rw-r--r--Python/modsupport.c16
-rw-r--r--Python/peephole.c189
-rw-r--r--Python/pystrtod.c2
-rw-r--r--Python/pythonrun.c168
-rw-r--r--Python/symtable.c158
-rw-r--r--Python/sysmodule.c102
-rw-r--r--Python/thread.c126
-rw-r--r--Python/thread_cthread.h112
-rw-r--r--Python/thread_lwp.h113
-rw-r--r--Python/thread_pth.h3
-rw-r--r--Python/thread_pthread.h10
-rw-r--r--Python/thread_sgi.h259
-rw-r--r--Python/thread_solaris.h130
-rw-r--r--Python/thread_wince.h136
-rw-r--r--Python/traceback.c295
-rw-r--r--README30
-rw-r--r--Tools/gdb/libpython.py45
-rw-r--r--Tools/msi/msi.py11
-rw-r--r--Tools/msi/uuids.py60
-rwxr-xr-xTools/pybench/pybench.py1
-rw-r--r--Tools/scripts/README3
-rwxr-xr-xTools/scripts/findnocoding.py4
-rwxr-xr-xTools/scripts/patchcheck.py30
-rwxr-xr-xTools/scripts/pysetup34
-rwxr-xr-xTools/scripts/pysource.py2
-rwxr-xr-xTools/scripts/reindent.py18
-rwxr-xr-xTools/scripts/run_tests.py47
-rw-r--r--Tools/unicode/comparecodecs.py2
-rw-r--r--Tools/unicode/makeunicodedata.py271
-rw-r--r--Tools/unittestgui/unittestgui.py1
-rwxr-xr-xconfigure704
-rw-r--r--configure.in297
-rw-r--r--pyconfig.h.in202
-rw-r--r--setup.py56
1011 files changed, 184255 insertions, 42692 deletions
diff --git a/.hgeol b/.hgeol
index ed13171..afb1e6b 100644
--- a/.hgeol
+++ b/.hgeol
@@ -28,8 +28,9 @@
Lib/email/test/data/msg_26.txt = BIN
Lib/test/cjkencodings/* = BIN
-Lib/test/sndhdrdata/sndhdr.* = BIN
Lib/test/decimaltestdata/*.decTest = BIN
+Lib/test/sndhdrdata/sndhdr.* = BIN
+Lib/test/test_email/data/msg_26.txt = BIN
# All other files (which presumably are human-editable) are "native".
# This must be the last rule!
diff --git a/.hgignore b/.hgignore
index 24df3b9..0fd8562 100644
--- a/.hgignore
+++ b/.hgignore
@@ -5,7 +5,8 @@ Makefile$
Makefile.pre$
TAGS$
autom4te.cache$
-build/
+^build/
+^Doc/build/
buildno$
config.cache
config.log
@@ -32,6 +33,7 @@ Modules/config.c
Modules/ld_so_aix$
Parser/pgen$
Parser/pgen.stamp$
+PCbuild/amd64/
^core
^python-gdb.py
^python.exe-gdb.py
@@ -47,6 +49,7 @@ libpython*.so*
*.pyd
*.cover
*~
+Lib/_sysconfigdata.py
Lib/lib2to3/*.pickle
Lib/test/data/*
Misc/*.wpu
@@ -62,7 +65,10 @@ PCbuild/*.exp
PCbuild/*.o
PCbuild/*.ncb
PCbuild/*.bsc
+PCbuild/*.user
+PCbuild/*.suo
PCbuild/Win32-temp-*
+PCbuild/x64-temp-*
__pycache__
Modules/_testembed
.coverage
diff --git a/Doc/ACKS.txt b/Doc/ACKS.txt
index 81f7c46..164c2c1 100644
--- a/Doc/ACKS.txt
+++ b/Doc/ACKS.txt
@@ -209,6 +209,7 @@ docs@python.org), and we'll be glad to correct the problem.
* David Turner
* Sandro Tosi
* Ville Vainio
+ * Nadeem Vawda
* Martijn Vries
* Charles G. Waldman
* Greg Ward
diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst
index d4dda7c..a171ac7 100644
--- a/Doc/c-api/arg.rst
+++ b/Doc/c-api/arg.rst
@@ -260,9 +260,11 @@ Numbers
``n`` (:class:`int`) [Py_ssize_t]
Convert a Python integer to a C :c:type:`Py_ssize_t`.
-``c`` (:class:`bytes` of length 1) [char]
- Convert a Python byte, represented as a :class:`bytes` object of length 1,
- to a C :c:type:`char`.
+``c`` (:class:`bytes` or :class:`bytearray` of length 1) [char]
+ Convert a Python byte, represented as a :class:`bytes` or
+ :class:`bytearray` object of length 1, to a C :c:type:`char`.
+
+ .. versionchanged:: 3.3 Allow :class:`bytearray` objects
``C`` (:class:`str` of length 1) [int]
Convert a Python character, represented as a :class:`str` object of
diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst
index 6f13c80..c7252ed 100644
--- a/Doc/c-api/exceptions.rst
+++ b/Doc/c-api/exceptions.rst
@@ -525,7 +525,7 @@ recursion depth automatically).
Marks a point where a recursive C-level call is about to be performed.
- If :const:`USE_STACKCHECK` is defined, this function checks if the the OS
+ If :const:`USE_STACKCHECK` is defined, this function checks if the OS
stack overflowed using :c:func:`PyOS_CheckStack`. In this is the case, it
sets a :exc:`MemoryError` and returns a nonzero value.
@@ -582,65 +582,116 @@ All standard Python exceptions are available as global variables whose names are
:c:type:`PyObject\*`; they are all class objects. For completeness, here are all
the variables:
-+-------------------------------------+----------------------------+----------+
-| C Name | Python Name | Notes |
-+=====================================+============================+==========+
-| :c:data:`PyExc_BaseException` | :exc:`BaseException` | \(1) |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_Exception` | :exc:`Exception` | \(1) |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | \(1) |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_EOFError` | :exc:`EOFError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_EnvironmentError` | :exc:`EnvironmentError` | \(1) |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_IOError` | :exc:`IOError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_ImportError` | :exc:`ImportError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_IndexError` | :exc:`IndexError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_KeyError` | :exc:`KeyError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_NameError` | :exc:`NameError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_NotImplementedError` | :exc:`NotImplementedError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_OSError` | :exc:`OSError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_OverflowError` | :exc:`OverflowError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | \(2) |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_SystemError` | :exc:`SystemError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_TypeError` | :exc:`TypeError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_ValueError` | :exc:`ValueError` | |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_WindowsError` | :exc:`WindowsError` | \(3) |
-+-------------------------------------+----------------------------+----------+
-| :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | |
-+-------------------------------------+----------------------------+----------+
++-----------------------------------------+---------------------------------+----------+
+| C Name | Python Name | Notes |
++=========================================+=================================+==========+
+| :c:data:`PyExc_BaseException` | :exc:`BaseException` | \(1) |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_Exception` | :exc:`Exception` | \(1) |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | \(1) |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_BlockingIOError` | :exc:`BlockingIOError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_EOFError` | :exc:`EOFError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ImportError` | :exc:`ImportError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_IndexError` | :exc:`IndexError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_InterruptedError` | :exc:`InterruptedError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_IsADirectoryError` | :exc:`IsADirectoryError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_KeyError` | :exc:`KeyError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_NameError` | :exc:`NameError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_NotImplementedError` | :exc:`NotImplementedError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_OSError` | :exc:`OSError` | \(1) |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_OverflowError` | :exc:`OverflowError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_PermissionError` | :exc:`PermissionError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | \(2) |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_SystemError` | :exc:`SystemError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_TypeError` | :exc:`TypeError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ValueError` | :exc:`ValueError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | |
++-----------------------------------------+---------------------------------+----------+
+
+.. versionadded:: 3.3
+ :c:data:`PyExc_BlockingIOError`, :c:data:`PyExc_BrokenPipeError`,
+ :c:data:`PyExc_ChildProcessError`, :c:data:`PyExc_ConnectionError`,
+ :c:data:`PyExc_ConnectionAbortedError`, :c:data:`PyExc_ConnectionRefusedError`,
+ :c:data:`PyExc_ConnectionResetError`, :c:data:`PyExc_FileExistsError`,
+ :c:data:`PyExc_FileNotFoundError`, :c:data:`PyExc_InterruptedError`,
+ :c:data:`PyExc_IsADirectoryError`, :c:data:`PyExc_NotADirectoryError`,
+ :c:data:`PyExc_PermissionError`, :c:data:`PyExc_ProcessLookupError`
+ and :c:data:`PyExc_TimeoutError` were introduced following :pep:`3151`.
+
+
+These are compatibility aliases to :c:data:`PyExc_OSError`:
+
++-------------------------------------+----------+
+| C Name | Notes |
++=====================================+==========+
+| :c:data:`PyExc_EnvironmentError` | |
++-------------------------------------+----------+
+| :c:data:`PyExc_IOError` | |
++-------------------------------------+----------+
+| :c:data:`PyExc_WindowsError` | \(3) |
++-------------------------------------+----------+
+
+.. versionchanged:: 3.3
+ These aliases used to be separate exception types.
+
.. index::
single: PyExc_BaseException
@@ -649,28 +700,42 @@ the variables:
single: PyExc_LookupError
single: PyExc_AssertionError
single: PyExc_AttributeError
+ single: PyExc_BlockingIOError
+ single: PyExc_BrokenPipeError
+ single: PyExc_ConnectionError
+ single: PyExc_ConnectionAbortedError
+ single: PyExc_ConnectionRefusedError
+ single: PyExc_ConnectionResetError
single: PyExc_EOFError
- single: PyExc_EnvironmentError
+ single: PyExc_FileExistsError
+ single: PyExc_FileNotFoundError
single: PyExc_FloatingPointError
- single: PyExc_IOError
single: PyExc_ImportError
single: PyExc_IndexError
+ single: PyExc_InterruptedError
+ single: PyExc_IsADirectoryError
single: PyExc_KeyError
single: PyExc_KeyboardInterrupt
single: PyExc_MemoryError
single: PyExc_NameError
+ single: PyExc_NotADirectoryError
single: PyExc_NotImplementedError
single: PyExc_OSError
single: PyExc_OverflowError
+ single: PyExc_PermissionError
+ single: PyExc_ProcessLookupError
single: PyExc_ReferenceError
single: PyExc_RuntimeError
single: PyExc_SyntaxError
single: PyExc_SystemError
single: PyExc_SystemExit
+ single: PyExc_TimeoutError
single: PyExc_TypeError
single: PyExc_ValueError
- single: PyExc_WindowsError
single: PyExc_ZeroDivisionError
+ single: PyExc_EnvironmentError
+ single: PyExc_IOError
+ single: PyExc_WindowsError
Notes:
diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst
index cf48363..b168751 100644
--- a/Doc/c-api/import.rst
+++ b/Doc/c-api/import.rst
@@ -57,7 +57,7 @@ Importing Modules
:c:func:`PyImport_ImportModule`.
-.. c:function:: PyObject* PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level)
+.. c:function:: PyObject* PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level)
Import a module. This is best described by referring to the built-in Python
function :func:`__import__`, as the standard :func:`__import__` function calls
@@ -68,6 +68,13 @@ Importing Modules
the return value when a submodule of a package was requested is normally the
top-level package, unless a non-empty *fromlist* was given.
+ .. versionadded:: 3.3
+
+
+.. c:function:: PyObject* PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level)
+
+ Similar to :c:func:`PyImport_ImportModuleLevelObject`, but the name is an
+ UTF-8 encoded string instead of a Unicode object.
.. c:function:: PyObject* PyImport_Import(PyObject *name)
@@ -86,7 +93,7 @@ Importing Modules
an exception set on failure (the module still exists in this case).
-.. c:function:: PyObject* PyImport_AddModule(const char *name)
+.. c:function:: PyObject* PyImport_AddModuleObject(PyObject *name)
Return the module object corresponding to a module name. The *name* argument
may be of the form ``package.module``. First check the modules dictionary if
@@ -100,6 +107,14 @@ Importing Modules
or one of its variants to import a module. Package structures implied by a
dotted name for *name* are not created if not already present.
+ .. versionadded:: 3.3
+
+
+.. c:function:: PyObject* PyImport_AddModule(const char *name)
+
+ Similar to :c:func:`PyImport_AddModuleObject`, but the name is a UTF-8
+ encoded string instead of a Unicode object.
+
.. c:function:: PyObject* PyImport_ExecCodeModule(char *name, PyObject *co)
@@ -136,14 +151,23 @@ Importing Modules
See also :c:func:`PyImport_ExecCodeModuleWithPathnames`.
-.. c:function:: PyObject* PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, char *cpathname)
+.. c:function:: PyObject* PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, PyObject *cpathname)
Like :c:func:`PyImport_ExecCodeModuleEx`, but the :attr:`__cached__`
attribute of the module object is set to *cpathname* if it is
non-``NULL``. Of the three functions, this is the preferred one to use.
+ .. versionadded:: 3.3
+
+
+.. c:function:: PyObject* PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, char *cpathname)
+
+ Like :c:func:`PyImport_ExecCodeModuleObject`, but *name*, *pathname* and
+ *cpathname* are UTF-8 encoded strings.
+
.. versionadded:: 3.2
+
.. c:function:: long PyImport_GetMagicNumber()
Return the magic number for Python bytecode files (a.k.a. :file:`.pyc` and
@@ -200,7 +224,7 @@ Importing Modules
For internal use only.
-.. c:function:: int PyImport_ImportFrozenModule(char *name)
+.. c:function:: int PyImport_ImportFrozenModuleObject(PyObject *name)
Load a frozen module named *name*. Return ``1`` for success, ``0`` if the
module is not found, and ``-1`` with an exception set if the initialization
@@ -208,6 +232,14 @@ Importing Modules
:c:func:`PyImport_ImportModule`. (Note the misnomer --- this function would
reload the module if it was already imported.)
+ .. versionadded:: 3.3
+
+
+.. c:function:: int PyImport_ImportFrozenModule(char *name)
+
+ Similar to :c:func:`PyImport_ImportFrozenModuleObject`, but the name is a
+ UTF-8 encoded string instead of a Unicode object.
+
.. c:type:: struct _frozen
@@ -247,13 +279,13 @@ Importing Modules
Structure describing a single entry in the list of built-in modules. Each of
these structures gives the name and initialization function for a module built
- into the interpreter. Programs which embed Python may use an array of these
- structures in conjunction with :c:func:`PyImport_ExtendInittab` to provide
- additional built-in modules. The structure is defined in
- :file:`Include/import.h` as::
+ into the interpreter. The name is an ASCII encoded string. Programs which
+ embed Python may use an array of these structures in conjunction with
+ :c:func:`PyImport_ExtendInittab` to provide additional built-in modules.
+ The structure is defined in :file:`Include/import.h` as::
struct _inittab {
- char *name;
+ char *name; /* ASCII encoded string */
PyObject* (*initfunc)(void);
};
diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst
index b2295e0..4c295fa 100644
--- a/Doc/c-api/long.rst
+++ b/Doc/c-api/long.rst
@@ -100,6 +100,20 @@ All integers are implemented as "long" integer objects of arbitrary size.
string is first encoded to a byte string using :c:func:`PyUnicode_EncodeDecimal`
and then converted using :c:func:`PyLong_FromString`.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyLong_FromUnicodeObject`.
+
+
+.. c:function:: PyObject* PyLong_FromUnicodeObject(PyObject *u, int base)
+
+ Convert a sequence of Unicode digits in the string *u* to a Python integer
+ value. The Unicode string is first encoded to a byte string using
+ :c:func:`PyUnicode_EncodeDecimal` and then converted using
+ :c:func:`PyLong_FromString`.
+
+ .. versionadded:: 3.3
+
.. c:function:: PyObject* PyLong_FromVoidPtr(void *p)
diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst
index ffd68e3..32587be 100644
--- a/Doc/c-api/module.rst
+++ b/Doc/c-api/module.rst
@@ -29,7 +29,7 @@ There are only a few functions special to module objects.
:c:data:`PyModule_Type`.
-.. c:function:: PyObject* PyModule_New(const char *name)
+.. c:function:: PyObject* PyModule_NewObject(PyObject *name)
.. index::
single: __name__ (module attribute)
@@ -40,6 +40,14 @@ There are only a few functions special to module objects.
Only the module's :attr:`__doc__` and :attr:`__name__` attributes are filled in;
the caller is responsible for providing a :attr:`__file__` attribute.
+ .. versionadded:: 3.3
+
+
+.. c:function:: PyObject* PyModule_New(const char *name)
+
+ Similar to :c:func:`PyImport_NewObject`, but the name is an UTF-8 encoded
+ string instead of a Unicode object.
+
.. c:function:: PyObject* PyModule_GetDict(PyObject *module)
@@ -52,7 +60,7 @@ There are only a few functions special to module objects.
manipulate a module's :attr:`__dict__`.
-.. c:function:: char* PyModule_GetName(PyObject *module)
+.. c:function:: PyObject* PyModule_GetNameObject(PyObject *module)
.. index::
single: __name__ (module attribute)
@@ -61,15 +69,13 @@ There are only a few functions special to module objects.
Return *module*'s :attr:`__name__` value. If the module does not provide one,
or if it is not a string, :exc:`SystemError` is raised and *NULL* is returned.
+ .. versionadded:: 3.3
-.. c:function:: char* PyModule_GetFilename(PyObject *module)
- Similar to :c:func:`PyModule_GetFilenameObject` but return the filename
- encoded to 'utf-8'.
+.. c:function:: char* PyModule_GetName(PyObject *module)
- .. deprecated:: 3.2
- :c:func:`PyModule_GetFilename` raises :c:type:`UnicodeEncodeError` on
- unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead.
+ Similar to :c:func:`PyModule_GetNameObject` but return the name encoded to
+ ``'utf-8'``.
.. c:function:: PyObject* PyModule_GetFilenameObject(PyObject *module)
@@ -81,11 +87,21 @@ There are only a few functions special to module objects.
Return the name of the file from which *module* was loaded using *module*'s
:attr:`__file__` attribute. If this is not defined, or if it is not a
unicode string, raise :exc:`SystemError` and return *NULL*; otherwise return
- a reference to a :c:type:`PyUnicodeObject`.
+ a reference to a Unicode object.
.. versionadded:: 3.2
+.. c:function:: char* PyModule_GetFilename(PyObject *module)
+
+ Similar to :c:func:`PyModule_GetFilenameObject` but return the filename
+ encoded to 'utf-8'.
+
+ .. deprecated:: 3.2
+ :c:func:`PyModule_GetFilename` raises :c:type:`UnicodeEncodeError` on
+ unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead.
+
+
.. c:function:: void* PyModule_GetState(PyObject *module)
Return the "state" of the module, that is, a pointer to the block of memory
diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst
index d0d45ad..88ba5ac 100644
--- a/Doc/c-api/object.rst
+++ b/Doc/c-api/object.rst
@@ -6,6 +6,19 @@ Object Protocol
===============
+.. c:var:: PyObject* Py_NotImplemented
+
+ The ``NotImplemented`` singleton, used to signal that an operation is
+ not implemented for the given type combination.
+
+
+.. c:macro:: Py_RETURN_NOTIMPLEMENTED
+
+ Properly handle returning :c:data:`Py_NotImplemented` from within a C
+ function (that is, increment the reference count of NotImplemented and
+ return it).
+
+
.. c:function:: int PyObject_Print(PyObject *o, FILE *fp, int flags)
Print an object *o*, on file *fp*. Returns ``-1`` on error. The flags argument
diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst
index f48eb73..3332d31 100644
--- a/Doc/c-api/unicode.rst
+++ b/Doc/c-api/unicode.rst
@@ -6,38 +6,72 @@ Unicode Objects and Codecs
--------------------------
.. sectionauthor:: Marc-Andre Lemburg <mal@lemburg.com>
+.. sectionauthor:: Georg Brandl <georg@python.org>
Unicode Objects
^^^^^^^^^^^^^^^
+Since the implementation of :pep:`393` in Python 3.3, Unicode objects internally
+use a variety of representations, in order to allow handling the complete range
+of Unicode characters while staying memory efficient. There are special cases
+for strings where all code points are below 128, 256, or 65536; otherwise, code
+points must be below 1114112 (which is the full Unicode range).
+
+:c:type:`Py_UNICODE*` and UTF-8 representations are created on demand and cached
+in the Unicode object. The :c:type:`Py_UNICODE*` representation is deprecated
+and inefficient; it should be avoided in performance- or memory-sensitive
+situations.
+
+Due to the transition between the old APIs and the new APIs, unicode objects
+can internally be in two states depending on how they were created:
+
+* "canonical" unicode objects are all objects created by a non-deprecated
+ unicode API. They use the most efficient representation allowed by the
+ implementation.
+
+* "legacy" unicode objects have been created through one of the deprecated
+ APIs (typically :c:func:`PyUnicode_FromUnicode`) and only bear the
+ :c:type:`Py_UNICODE*` representation; you will have to call
+ :c:func:`PyUnicode_READY` on them before calling any other API.
+
+
Unicode Type
""""""""""""
These are the basic Unicode object types used for the Unicode implementation in
Python:
+.. c:type:: Py_UCS4
+ Py_UCS2
+ Py_UCS1
+
+ These types are typedefs for unsigned integer types wide enough to contain
+ characters of 32 bits, 16 bits and 8 bits, respectively. When dealing with
+ single Unicode characters, use :c:type:`Py_UCS4`.
+
+ .. versionadded:: 3.3
+
.. c:type:: Py_UNICODE
- This type represents the storage type which is used by Python internally as
- basis for holding Unicode ordinals. Python's default builds use a 16-bit type
- for :c:type:`Py_UNICODE` and store Unicode values internally as UCS2. It is also
- possible to build a UCS4 version of Python (most recent Linux distributions come
- with UCS4 builds of Python). These builds then use a 32-bit type for
- :c:type:`Py_UNICODE` and store Unicode data internally as UCS4. On platforms
- where :c:type:`wchar_t` is available and compatible with the chosen Python
- Unicode build variant, :c:type:`Py_UNICODE` is a typedef alias for
- :c:type:`wchar_t` to enhance native platform compatibility. On all other
- platforms, :c:type:`Py_UNICODE` is a typedef alias for either :c:type:`unsigned
- short` (UCS2) or :c:type:`unsigned long` (UCS4).
+ This is a typedef of :c:type:`wchar_t`, which is a 16-bit type or 32-bit type
+ depending on the platform.
-Note that UCS2 and UCS4 Python builds are not binary compatible. Please keep
-this in mind when writing extensions or interfaces.
+ .. versionchanged:: 3.3
+ In previous versions, this was a 16-bit type or a 32-bit type depending on
+ whether you selected a "narrow" or "wide" Unicode version of Python at
+ build time.
-.. c:type:: PyUnicodeObject
+.. c:type:: PyASCIIObject
+ PyCompactUnicodeObject
+ PyUnicodeObject
- This subtype of :c:type:`PyObject` represents a Python Unicode object.
+ These subtypes of :c:type:`PyObject` represent a Python Unicode object. In
+ almost all cases, they shouldn't be used directly, since all API functions
+ that deal with Unicode objects take and return :c:type:`PyObject` pointers.
+
+ .. versionadded:: 3.3
.. c:var:: PyTypeObject PyUnicode_Type
@@ -45,10 +79,10 @@ this in mind when writing extensions or interfaces.
This instance of :c:type:`PyTypeObject` represents the Python Unicode type. It
is exposed to Python code as ``str``.
+
The following APIs are really C macros and can be used to do fast checks and to
access internal read-only data of Unicode objects:
-
.. c:function:: int PyUnicode_Check(PyObject *o)
Return true if the object *o* is a Unicode object or an instance of a Unicode
@@ -61,28 +95,105 @@ access internal read-only data of Unicode objects:
subtype.
-.. c:function:: Py_ssize_t PyUnicode_GET_SIZE(PyObject *o)
+.. c:function:: int PyUnicode_READY(PyObject *o)
- Return the size of the object. *o* has to be a :c:type:`PyUnicodeObject` (not
- checked).
+ Ensure the string object *o* is in the "canonical" representation. This is
+ required before using any of the access macros described below.
+ .. XXX expand on when it is not required
-.. c:function:: Py_ssize_t PyUnicode_GET_DATA_SIZE(PyObject *o)
+ Returns 0 on success and -1 with an exception set on failure, which in
+ particular happens if memory allocation fails.
- Return the size of the object's internal buffer in bytes. *o* has to be a
- :c:type:`PyUnicodeObject` (not checked).
+ .. versionadded:: 3.3
-.. c:function:: Py_UNICODE* PyUnicode_AS_UNICODE(PyObject *o)
+.. c:function:: Py_ssize_t PyUnicode_GET_LENGTH(PyObject *o)
+
+ Return the length of the Unicode string, in code points. *o* has to be a
+ Unicode object in the "canonical" representation (not checked).
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: Py_UCS1* PyUnicode_1BYTE_DATA(PyObject *o)
+ Py_UCS2* PyUnicode_2BYTE_DATA(PyObject *o)
+ Py_UCS4* PyUnicode_4BYTE_DATA(PyObject *o)
+
+ Return a pointer to the canonical representation cast to UCS1, UCS2 or UCS4
+ integer types for direct character access. No checks are performed if the
+ canonical representation has the correct character size; use
+ :c:func:`PyUnicode_KIND` to select the right macro. Make sure
+ :c:func:`PyUnicode_READY` has been called before accessing this.
+
+ .. versionadded:: 3.3
+
+
+.. c:macro:: PyUnicode_1BYTE_KIND
+ PyUnicode_2BYTE_KIND
+ PyUnicode_4BYTE_KIND
+
+ Return values of the :c:func:`PyUnicode_KIND` macro.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: int PyUnicode_KIND(PyObject *o)
+
+ Return one of the PyUnicode kind constants (see above) that indicate how many
+ bytes per character this Unicode object uses to store its data. *o* has to
+ be a Unicode object in the "canonical" representation (not checked).
+
+ .. XXX document "0" return value?
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: void* PyUnicode_DATA(PyObject *o)
+
+ Return a void pointer to the raw unicode buffer. *o* has to be a Unicode
+ object in the "canonical" representation (not checked).
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: void PyUnicode_WRITE(int kind, void *data, Py_ssize_t index, \
+ Py_UCS4 value)
+
+ Write into a canonical representation *data* (as obtained with
+ :c:func:`PyUnicode_DATA`). This macro does not do any sanity checks and is
+ intended for usage in loops. The caller should cache the *kind* value and
+ *data* pointer as obtained from other macro calls. *index* is the index in
+ the string (starts at 0) and *value* is the new code point value which should
+ be written to that location.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: Py_UCS4 PyUnicode_READ(int kind, void *data, Py_ssize_t index)
+
+ Read a code point from a canonical representation *data* (as obtained with
+ :c:func:`PyUnicode_DATA`). No checks or ready calls are performed.
- Return a pointer to the internal :c:type:`Py_UNICODE` buffer of the object. *o*
- has to be a :c:type:`PyUnicodeObject` (not checked).
+ .. versionadded:: 3.3
-.. c:function:: const char* PyUnicode_AS_DATA(PyObject *o)
+.. c:function:: Py_UCS4 PyUnicode_READ_CHAR(PyObject *o, Py_ssize_t index)
- Return a pointer to the internal buffer of the object. *o* has to be a
- :c:type:`PyUnicodeObject` (not checked).
+ Read a character from a Unicode object *o*, which must be in the "canonical"
+ representation. This is less efficient than :c:func:`PyUnicode_READ` if you
+ do multiple consecutive reads.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: PyUnicode_MAX_CHAR_VALUE(PyObject *o)
+
+ Return the maximum code point that is suitable for creating another string
+ based on *o*, which must be in the "canonical" representation. This is
+ always an approximation but more efficient than iterating over the string.
+
+ .. versionadded:: 3.3
.. c:function:: int PyUnicode_ClearFreeList()
@@ -90,6 +201,46 @@ access internal read-only data of Unicode objects:
Clear the free list. Return the total number of freed items.
+.. c:function:: Py_ssize_t PyUnicode_GET_SIZE(PyObject *o)
+
+ Return the size of the deprecated :c:type:`Py_UNICODE` representation, in
+ code units (this includes surrogate pairs as 2 units). *o* has to be a
+ Unicode object (not checked).
+
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style Unicode API, please migrate to using
+ :c:func:`PyUnicode_GET_LENGTH`.
+
+
+.. c:function:: Py_ssize_t PyUnicode_GET_DATA_SIZE(PyObject *o)
+
+ Return the size of the deprecated :c:type:`Py_UNICODE` representation in
+ bytes. *o* has to be a Unicode object (not checked).
+
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style Unicode API, please migrate to using
+ :c:func:`PyUnicode_GET_LENGTH`.
+
+
+.. c:function:: Py_UNICODE* PyUnicode_AS_UNICODE(PyObject *o)
+ const char* PyUnicode_AS_DATA(PyObject *o)
+
+ Return a pointer to a :c:type:`Py_UNICODE` representation of the object. The
+ ``AS_DATA`` form casts the pointer to :c:type:`const char *`. *o* has to be
+ a Unicode object (not checked).
+
+ .. versionchanged:: 3.3
+ This macro is now inefficient -- because in many cases the
+ :c:type:`Py_UNICODE` representation does not exist and needs to be created
+ -- and can fail (return *NULL* with an exception set). Try to port the
+ code to use the new :c:func:`PyUnicode_nBYTE_DATA` macros or use
+ :c:func:`PyUnicode_WRITE` or :c:func:`PyUnicode_READ`.
+
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style Unicode API, please migrate to using the
+ :c:func:`PyUnicode_nBYTE_DATA` family of macros.
+
+
Unicode Character Properties
""""""""""""""""""""""""""""
@@ -195,31 +346,66 @@ These APIs can be used for fast direct character conversions:
possible. This macro does not raise exceptions.
-Plain Py_UNICODE
-""""""""""""""""
+These APIs can be used to work with surrogates:
+
+.. c:macro:: Py_UNICODE_IS_SURROGATE(ch)
+
+ Check if *ch* is a surrogate (``0xD800 <= ch <= 0xDFFF``).
+
+.. c:macro:: Py_UNICODE_IS_HIGH_SURROGATE(ch)
+
+ Check if *ch* is an high surrogate (``0xD800 <= ch <= 0xDBFF``).
+
+.. c:macro:: Py_UNICODE_IS_LOW_SURROGATE(ch)
+
+ Check if *ch* is a low surrogate (``0xDC00 <= ch <= 0xDFFF``).
+
+.. c:macro:: Py_UNICODE_JOIN_SURROGATES(high, low)
+
+ Join two surrogate characters and return a single Py_UCS4 value.
+ *high* and *low* are respectively the leading and trailing surrogates in a
+ surrogate pair.
+
+
+Creating and accessing Unicode strings
+""""""""""""""""""""""""""""""""""""""
To create Unicode objects and access their basic sequence properties, use these
APIs:
+.. c:function:: PyObject* PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar)
-.. c:function:: PyObject* PyUnicode_FromUnicode(const Py_UNICODE *u, Py_ssize_t size)
+ Create a new Unicode object. *maxchar* should be the true maximum code point
+ to be placed in the string. As an approximation, it can be rounded up to the
+ nearest value in the sequence 127, 255, 65535, 1114111.
- Create a Unicode object from the Py_UNICODE buffer *u* of the given size. *u*
- may be *NULL* which causes the contents to be undefined. It is the user's
- responsibility to fill in the needed data. The buffer is copied into the new
- object. If the buffer is not *NULL*, the return value might be a shared object.
- Therefore, modification of the resulting Unicode object is only allowed when *u*
- is *NULL*.
+ This is the recommended way to allocate a new Unicode object. Objects
+ created using this function are not resizable.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: PyObject* PyUnicode_FromKindAndData(int kind, const void *buffer, \
+ Py_ssize_t size)
+
+ Create a new Unicode object with the given *kind* (possible values are
+ :c:macro:`PyUnicode_1BYTE_KIND` etc., as returned by
+ :c:func:`PyUnicode_KIND`). The *buffer* must point to an array of *size*
+ units of 1, 2 or 4 bytes per character, as given by the kind.
+
+ .. versionadded:: 3.3
.. c:function:: PyObject* PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size)
- Create a Unicode object from the char buffer *u*. The bytes will be interpreted
- as being UTF-8 encoded. *u* may also be *NULL* which
- causes the contents to be undefined. It is the user's responsibility to fill in
- the needed data. The buffer is copied into the new object. If the buffer is not
- *NULL*, the return value might be a shared object. Therefore, modification of
- the resulting Unicode object is only allowed when *u* is *NULL*.
+ Create a Unicode object from the char buffer *u*. The bytes will be
+ interpreted as being UTF-8 encoded. The buffer is copied into the new
+ object. If the buffer is not *NULL*, the return value might be a shared
+ object, i.e. modification of the data is not allowed.
+
+ If *u* is *NULL*, this function behaves like :c:func:`PyUnicode_FromUnicode`
+ with the buffer set to *NULL*. This usage is deprecated in favor of
+ :c:func:`PyUnicode_New`.
.. c:function:: PyObject *PyUnicode_FromString(const char *u)
@@ -260,18 +446,27 @@ APIs:
| :attr:`%ld` | long | Exactly equivalent to |
| | | ``printf("%ld")``. |
+-------------------+---------------------+--------------------------------+
+ | :attr:`%li` | long | Exactly equivalent to |
+ | | | ``printf("%li")``. |
+ +-------------------+---------------------+--------------------------------+
| :attr:`%lu` | unsigned long | Exactly equivalent to |
| | | ``printf("%lu")``. |
+-------------------+---------------------+--------------------------------+
| :attr:`%lld` | long long | Exactly equivalent to |
| | | ``printf("%lld")``. |
+-------------------+---------------------+--------------------------------+
+ | :attr:`%lli` | long long | Exactly equivalent to |
+ | | | ``printf("%lli")``. |
+ +-------------------+---------------------+--------------------------------+
| :attr:`%llu` | unsigned long long | Exactly equivalent to |
| | | ``printf("%llu")``. |
+-------------------+---------------------+--------------------------------+
| :attr:`%zd` | Py_ssize_t | Exactly equivalent to |
| | | ``printf("%zd")``. |
+-------------------+---------------------+--------------------------------+
+ | :attr:`%zi` | Py_ssize_t | Exactly equivalent to |
+ | | | ``printf("%zi")``. |
+ +-------------------+---------------------+--------------------------------+
| :attr:`%zu` | size_t | Exactly equivalent to |
| | | ``printf("%zu")``. |
+-------------------+---------------------+--------------------------------+
@@ -322,24 +517,159 @@ APIs:
.. versionchanged:: 3.2
Support for ``"%lld"`` and ``"%llu"`` added.
+ .. versionchanged:: 3.3
+ Support for ``"%li"``, ``"%lli"`` and ``"%zi"`` added.
+
.. c:function:: PyObject* PyUnicode_FromFormatV(const char *format, va_list vargs)
Identical to :c:func:`PyUnicode_FromFormat` except that it takes exactly two
arguments.
+
+.. c:function:: PyObject* PyUnicode_FromEncodedObject(PyObject *obj, \
+ const char *encoding, const char *errors)
+
+ Coerce an encoded object *obj* to an Unicode object and return a reference with
+ incremented refcount.
+
+ :class:`bytes`, :class:`bytearray` and other char buffer compatible objects
+ are decoded according to the given *encoding* and using the error handling
+ defined by *errors*. Both can be *NULL* to have the interface use the default
+ values (see the next section for details).
+
+ All other objects, including Unicode objects, cause a :exc:`TypeError` to be
+ set.
+
+ The API returns *NULL* if there was an error. The caller is responsible for
+ decref'ing the returned objects.
+
+
+.. c:function:: Py_ssize_t PyUnicode_GetLength(PyObject *unicode)
+
+ Return the length of the Unicode object, in code points.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: int PyUnicode_CopyCharacters(PyObject *to, Py_ssize_t to_start, \
+ PyObject *to, Py_ssize_t from_start, Py_ssize_t how_many)
+
+ Copy characters from one Unicode object into another. This function performs
+ character conversion when necessary and falls back to :c:func:`memcpy` if
+ possible. Returns ``-1`` and sets an exception on error, otherwise returns
+ ``0``.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: int PyUnicode_WriteChar(PyObject *unicode, Py_ssize_t index, \
+ Py_UCS4 character)
+
+ Write a character to a string. The string must have been created through
+ :c:func:`PyUnicode_New`. Since Unicode strings are supposed to be immutable,
+ the string must not be shared, or have been hashed yet.
+
+ This function checks that *unicode* is a Unicode object, that the index is
+ not out of bounds, and that the object can be modified safely (i.e. that it
+ its reference count is one), in contrast to the macro version
+ :c:func:`PyUnicode_WRITE_CHAR`.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: Py_UCS4 PyUnicode_ReadChar(PyObject *unicode, Py_ssize_t index)
+
+ Read a character from a string. This function checks that *unicode* is a
+ Unicode object and the index is not out of bounds, in contrast to the macro
+ version :c:func:`PyUnicode_READ_CHAR`.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: PyObject* PyUnicode_Substring(PyObject *str, Py_ssize_t start, \
+ Py_ssize_t end)
+
+ Return a substring of *str*, from character index *start* (included) to
+ character index *end* (excluded). Negative indices are not supported.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: Py_UCS4* PyUnicode_AsUCS4(PyObject *u, Py_UCS4 *buffer, \
+ Py_ssize_t buflen, int copy_null)
+
+ Copy the string *u* into a UCS4 buffer, including a null character, if
+ *copy_null* is set. Returns *NULL* and sets an exception on error (in
+ particular, a :exc:`ValueError` if *buflen* is smaller than the length of
+ *u*). *buffer* is returned on success.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: Py_UCS4* PyUnicode_AsUCS4Copy(PyObject *u)
+
+ Copy the string *u* into a new UCS4 buffer that is allocated using
+ :c:func:`PyMem_Malloc`. If this fails, *NULL* is returned with a
+ :exc:`MemoryError` set.
+
+ .. versionadded:: 3.3
+
+
+Deprecated Py_UNICODE APIs
+""""""""""""""""""""""""""
+
+.. deprecated-removed:: 3.3 4.0
+
+These API functions are deprecated with the implementation of :pep:`393`.
+Extension modules can continue using them, as they will not be removed in Python
+3.x, but need to be aware that their use can now cause performance and memory hits.
+
+
+.. c:function:: PyObject* PyUnicode_FromUnicode(const Py_UNICODE *u, Py_ssize_t size)
+
+ Create a Unicode object from the Py_UNICODE buffer *u* of the given size. *u*
+ may be *NULL* which causes the contents to be undefined. It is the user's
+ responsibility to fill in the needed data. The buffer is copied into the new
+ object.
+
+ If the buffer is not *NULL*, the return value might be a shared object.
+ Therefore, modification of the resulting Unicode object is only allowed when
+ *u* is *NULL*.
+
+ If the buffer is *NULL*, :c:func:`PyUnicode_READY` must be called once the
+ string content has been filled before using any of the access macros such as
+ :c:func:`PyUnicode_KIND`.
+
+ Please migrate to using :c:func:`PyUnicode_FromKindAndData` or
+ :c:func:`PyUnicode_New`.
+
+
+.. c:function:: Py_UNICODE* PyUnicode_AsUnicode(PyObject *unicode)
+
+ Return a read-only pointer to the Unicode object's internal
+ :c:type:`Py_UNICODE` buffer, *NULL* if *unicode* is not a Unicode object.
+ This will create the :c:type:`Py_UNICODE` representation of the object if it
+ is not yet available.
+
+ Please migrate to using :c:func:`PyUnicode_AsUCS4`,
+ :c:func:`PyUnicode_Substring`, :c:func:`PyUnicode_ReadChar` or similar new
+ APIs.
+
+
.. c:function:: PyObject* PyUnicode_TransformDecimalToASCII(Py_UNICODE *s, Py_ssize_t size)
Create a Unicode object by replacing all decimal digits in
:c:type:`Py_UNICODE` buffer of the given *size* by ASCII digits 0--9
- according to their decimal value. Return *NULL* if an exception
- occurs.
+ according to their decimal value. Return *NULL* if an exception occurs.
-.. c:function:: Py_UNICODE* PyUnicode_AsUnicode(PyObject *unicode)
+.. c:function:: Py_UNICODE* PyUnicode_AsUnicodeAndSize(PyObject *unicode, Py_ssize_t *size)
- Return a read-only pointer to the Unicode object's internal :c:type:`Py_UNICODE`
- buffer, *NULL* if *unicode* is not a Unicode object.
+ Like :c:func:`PyUnicode_AsUnicode`, but also saves the :c:func:`Py_UNICODE`
+ array length in *size*.
+
+ .. versionadded:: 3.3
.. c:function:: Py_UNICODE* PyUnicode_AsUnicodeCopy(PyObject *unicode)
@@ -351,27 +681,15 @@ APIs:
.. versionadded:: 3.2
+ Please migrate to using :c:func:`PyUnicode_AsUCS4Copy` or similar new APIs.
-.. c:function:: Py_ssize_t PyUnicode_GetSize(PyObject *unicode)
-
- Return the length of the Unicode object.
-
-
-.. c:function:: PyObject* PyUnicode_FromEncodedObject(PyObject *obj, const char *encoding, const char *errors)
-
- Coerce an encoded object *obj* to an Unicode object and return a reference with
- incremented refcount.
- :class:`bytes`, :class:`bytearray` and other char buffer compatible objects
- are decoded according to the given *encoding* and using the error handling
- defined by *errors*. Both can be *NULL* to have the interface use the default
- values (see the next section for details).
+.. c:function:: Py_ssize_t PyUnicode_GetSize(PyObject *unicode)
- All other objects, including Unicode objects, cause a :exc:`TypeError` to be
- set.
+ Return the size of the deprecated :c:type:`Py_UNICODE` representation, in
+ code units (this includes surrogate pairs as 2 units).
- The API returns *NULL* if there was an error. The caller is responsible for
- decref'ing the returned objects.
+ Please migrate to using :c:func:`PyUnicode_GetLength`.
.. c:function:: PyObject* PyUnicode_FromObject(PyObject *obj)
@@ -379,11 +697,6 @@ APIs:
Shortcut for ``PyUnicode_FromEncodedObject(obj, NULL, "strict")`` which is used
throughout the interpreter whenever coercion to Unicode is needed.
-If the platform supports :c:type:`wchar_t` and provides a header file wchar.h,
-Python can interface directly to this type using the following functions.
-Support is optimized if Python's own :c:type:`Py_UNICODE` type is identical to
-the system's :c:type:`wchar_t`.
-
File System Encoding
""""""""""""""""""""
@@ -493,6 +806,26 @@ wchar_t Support
.. versionadded:: 3.2
+UCS4 Support
+""""""""""""
+
+.. versionadded:: 3.3
+
+.. XXX are these meant to be public?
+
+.. c:function:: size_t Py_UCS4_strlen(const Py_UCS4 *u)
+ Py_UCS4* Py_UCS4_strcpy(Py_UCS4 *s1, const Py_UCS4 *s2)
+ Py_UCS4* Py_UCS4_strncpy(Py_UCS4 *s1, const Py_UCS4 *s2, size_t n)
+ Py_UCS4* Py_UCS4_strcat(Py_UCS4 *s1, const Py_UCS4 *s2)
+ int Py_UCS4_strcmp(const Py_UCS4 *s1, const Py_UCS4 *s2)
+ int Py_UCS4_strncmp(const Py_UCS4 *s1, const Py_UCS4 *s2, size_t n)
+ Py_UCS4* Py_UCS4_strchr(const Py_UCS4 *s, Py_UCS4 c)
+ Py_UCS4* Py_UCS4_strrchr(const Py_UCS4 *s, Py_UCS4 c)
+
+ These utility functions work on strings of :c:type:`Py_UCS4` characters and
+ otherwise behave like the C standard library functions with the same name.
+
+
.. _builtincodecs:
Built-in Codecs
@@ -527,7 +860,8 @@ Generic Codecs
These are the generic codec APIs:
-.. c:function:: PyObject* PyUnicode_Decode(const char *s, Py_ssize_t size, const char *encoding, const char *errors)
+.. c:function:: PyObject* PyUnicode_Decode(const char *s, Py_ssize_t size, \
+ const char *encoding, const char *errors)
Create a Unicode object by decoding *size* bytes of the encoded string *s*.
*encoding* and *errors* have the same meaning as the parameters of the same name
@@ -536,7 +870,18 @@ These are the generic codec APIs:
the codec.
-.. c:function:: PyObject* PyUnicode_Encode(const Py_UNICODE *s, Py_ssize_t size, const char *encoding, const char *errors)
+.. c:function:: PyObject* PyUnicode_AsEncodedString(PyObject *unicode, \
+ const char *encoding, const char *errors)
+
+ Encode a Unicode object and return the result as Python bytes object.
+ *encoding* and *errors* have the same meaning as the parameters of the same
+ name in the Unicode :meth:`encode` method. The codec to be used is looked up
+ using the Python codec registry. Return *NULL* if an exception was raised by
+ the codec.
+
+
+.. c:function:: PyObject* PyUnicode_Encode(const Py_UNICODE *s, Py_ssize_t size, \
+ const char *encoding, const char *errors)
Encode the :c:type:`Py_UNICODE` buffer *s* of the given *size* and return a Python
bytes object. *encoding* and *errors* have the same meaning as the
@@ -544,14 +889,9 @@ These are the generic codec APIs:
to be used is looked up using the Python codec registry. Return *NULL* if an
exception was raised by the codec.
-
-.. c:function:: PyObject* PyUnicode_AsEncodedString(PyObject *unicode, const char *encoding, const char *errors)
-
- Encode a Unicode object and return the result as Python bytes object.
- *encoding* and *errors* have the same meaning as the parameters of the same
- name in the Unicode :meth:`encode` method. The codec to be used is looked up
- using the Python codec registry. Return *NULL* if an exception was raised by
- the codec.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsEncodedString`.
UTF-8 Codecs
@@ -566,7 +906,8 @@ These are the UTF-8 codec APIs:
*s*. Return *NULL* if an exception was raised by the codec.
-.. c:function:: PyObject* PyUnicode_DecodeUTF8Stateful(const char *s, Py_ssize_t size, const char *errors, Py_ssize_t *consumed)
+.. c:function:: PyObject* PyUnicode_DecodeUTF8Stateful(const char *s, Py_ssize_t size, \
+ const char *errors, Py_ssize_t *consumed)
If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeUTF8`. If
*consumed* is not *NULL*, trailing incomplete UTF-8 byte sequences will not be
@@ -574,18 +915,45 @@ These are the UTF-8 codec APIs:
that have been decoded will be stored in *consumed*.
+.. c:function:: PyObject* PyUnicode_AsUTF8String(PyObject *unicode)
+
+ Encode a Unicode object using UTF-8 and return the result as Python bytes
+ object. Error handling is "strict". Return *NULL* if an exception was
+ raised by the codec.
+
+
+.. c:function:: char* PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *size)
+
+ Return a pointer to the default encoding (UTF-8) of the Unicode object, and
+ store the size of the encoded representation (in bytes) in *size*. *size*
+ can be *NULL*, in this case no size will be stored.
+
+ In the case of an error, *NULL* is returned with an exception set and no
+ *size* is stored.
+
+ This caches the UTF-8 representation of the string in the Unicode object, and
+ subsequent calls will return a pointer to the same buffer. The caller is not
+ responsible for deallocating the buffer.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: char* PyUnicode_AsUTF8(PyObject *unicode)
+
+ As :c:func:`PyUnicode_AsUTF8AndSize`, but does not store the size.
+
+ .. versionadded:: 3.3
+
+
.. c:function:: PyObject* PyUnicode_EncodeUTF8(const Py_UNICODE *s, Py_ssize_t size, const char *errors)
Encode the :c:type:`Py_UNICODE` buffer *s* of the given *size* using UTF-8 and
return a Python bytes object. Return *NULL* if an exception was raised by
the codec.
-
-.. c:function:: PyObject* PyUnicode_AsUTF8String(PyObject *unicode)
-
- Encode a Unicode object using UTF-8 and return the result as Python bytes
- object. Error handling is "strict". Return *NULL* if an exception was
- raised by the codec.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsUTF8String` or :c:func:`PyUnicode_AsUTF8AndSize`.
UTF-32 Codecs
@@ -594,7 +962,8 @@ UTF-32 Codecs
These are the UTF-32 codec APIs:
-.. c:function:: PyObject* PyUnicode_DecodeUTF32(const char *s, Py_ssize_t size, const char *errors, int *byteorder)
+.. c:function:: PyObject* PyUnicode_DecodeUTF32(const char *s, Py_ssize_t size, \
+ const char *errors, int *byteorder)
Decode *size* bytes from a UTF-32 encoded buffer string and return the
corresponding Unicode object. *errors* (if non-*NULL*) defines the error
@@ -622,7 +991,8 @@ These are the UTF-32 codec APIs:
Return *NULL* if an exception was raised by the codec.
-.. c:function:: PyObject* PyUnicode_DecodeUTF32Stateful(const char *s, Py_ssize_t size, const char *errors, int *byteorder, Py_ssize_t *consumed)
+.. c:function:: PyObject* PyUnicode_DecodeUTF32Stateful(const char *s, Py_ssize_t size, \
+ const char *errors, int *byteorder, Py_ssize_t *consumed)
If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeUTF32`. If
*consumed* is not *NULL*, :c:func:`PyUnicode_DecodeUTF32Stateful` will not treat
@@ -631,7 +1001,15 @@ These are the UTF-32 codec APIs:
that have been decoded will be stored in *consumed*.
-.. c:function:: PyObject* PyUnicode_EncodeUTF32(const Py_UNICODE *s, Py_ssize_t size, const char *errors, int byteorder)
+.. c:function:: PyObject* PyUnicode_AsUTF32String(PyObject *unicode)
+
+ Return a Python byte string using the UTF-32 encoding in native byte
+ order. The string always starts with a BOM mark. Error handling is "strict".
+ Return *NULL* if an exception was raised by the codec.
+
+
+.. c:function:: PyObject* PyUnicode_EncodeUTF32(const Py_UNICODE *s, Py_ssize_t size, \
+ const char *errors, int byteorder)
Return a Python bytes object holding the UTF-32 encoded value of the Unicode
data in *s*. Output is written according to the following byte order::
@@ -648,12 +1026,9 @@ These are the UTF-32 codec APIs:
Return *NULL* if an exception was raised by the codec.
-
-.. c:function:: PyObject* PyUnicode_AsUTF32String(PyObject *unicode)
-
- Return a Python byte string using the UTF-32 encoding in native byte
- order. The string always starts with a BOM mark. Error handling is "strict".
- Return *NULL* if an exception was raised by the codec.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsUTF32String`.
UTF-16 Codecs
@@ -662,7 +1037,8 @@ UTF-16 Codecs
These are the UTF-16 codec APIs:
-.. c:function:: PyObject* PyUnicode_DecodeUTF16(const char *s, Py_ssize_t size, const char *errors, int *byteorder)
+.. c:function:: PyObject* PyUnicode_DecodeUTF16(const char *s, Py_ssize_t size, \
+ const char *errors, int *byteorder)
Decode *size* bytes from a UTF-16 encoded buffer string and return the
corresponding Unicode object. *errors* (if non-*NULL*) defines the error
@@ -689,7 +1065,8 @@ These are the UTF-16 codec APIs:
Return *NULL* if an exception was raised by the codec.
-.. c:function:: PyObject* PyUnicode_DecodeUTF16Stateful(const char *s, Py_ssize_t size, const char *errors, int *byteorder, Py_ssize_t *consumed)
+.. c:function:: PyObject* PyUnicode_DecodeUTF16Stateful(const char *s, Py_ssize_t size, \
+ const char *errors, int *byteorder, Py_ssize_t *consumed)
If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeUTF16`. If
*consumed* is not *NULL*, :c:func:`PyUnicode_DecodeUTF16Stateful` will not treat
@@ -698,7 +1075,15 @@ These are the UTF-16 codec APIs:
number of bytes that have been decoded will be stored in *consumed*.
-.. c:function:: PyObject* PyUnicode_EncodeUTF16(const Py_UNICODE *s, Py_ssize_t size, const char *errors, int byteorder)
+.. c:function:: PyObject* PyUnicode_AsUTF16String(PyObject *unicode)
+
+ Return a Python byte string using the UTF-16 encoding in native byte
+ order. The string always starts with a BOM mark. Error handling is "strict".
+ Return *NULL* if an exception was raised by the codec.
+
+
+.. c:function:: PyObject* PyUnicode_EncodeUTF16(const Py_UNICODE *s, Py_ssize_t size, \
+ const char *errors, int byteorder)
Return a Python bytes object holding the UTF-16 encoded value of the Unicode
data in *s*. Output is written according to the following byte order::
@@ -716,12 +1101,9 @@ These are the UTF-16 codec APIs:
Return *NULL* if an exception was raised by the codec.
-
-.. c:function:: PyObject* PyUnicode_AsUTF16String(PyObject *unicode)
-
- Return a Python byte string using the UTF-16 encoding in native byte
- order. The string always starts with a BOM mark. Error handling is "strict".
- Return *NULL* if an exception was raised by the codec.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsUTF16String`.
UTF-7 Codecs
@@ -736,7 +1118,8 @@ These are the UTF-7 codec APIs:
*s*. Return *NULL* if an exception was raised by the codec.
-.. c:function:: PyObject* PyUnicode_DecodeUTF7Stateful(const char *s, Py_ssize_t size, const char *errors, Py_ssize_t *consumed)
+.. c:function:: PyObject* PyUnicode_DecodeUTF7Stateful(const char *s, Py_ssize_t size, \
+ const char *errors, Py_ssize_t *consumed)
If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeUTF7`. If
*consumed* is not *NULL*, trailing incomplete UTF-7 base-64 sections will not
@@ -744,7 +1127,8 @@ These are the UTF-7 codec APIs:
bytes that have been decoded will be stored in *consumed*.
-.. c:function:: PyObject* PyUnicode_EncodeUTF7(const Py_UNICODE *s, Py_ssize_t size, int base64SetO, int base64WhiteSpace, const char *errors)
+.. c:function:: PyObject* PyUnicode_EncodeUTF7(const Py_UNICODE *s, Py_ssize_t size, \
+ int base64SetO, int base64WhiteSpace, const char *errors)
Encode the :c:type:`Py_UNICODE` buffer of the given size using UTF-7 and
return a Python bytes object. Return *NULL* if an exception was raised by
@@ -755,6 +1139,11 @@ These are the UTF-7 codec APIs:
nonzero, whitespace will be encoded in base-64. Both are set to zero for the
Python "utf-7" codec.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API.
+
+ .. XXX replace with what?
+
Unicode-Escape Codecs
"""""""""""""""""""""
@@ -762,24 +1151,29 @@ Unicode-Escape Codecs
These are the "Unicode Escape" codec APIs:
-.. c:function:: PyObject* PyUnicode_DecodeUnicodeEscape(const char *s, Py_ssize_t size, const char *errors)
+.. c:function:: PyObject* PyUnicode_DecodeUnicodeEscape(const char *s, \
+ Py_ssize_t size, const char *errors)
Create a Unicode object by decoding *size* bytes of the Unicode-Escape encoded
string *s*. Return *NULL* if an exception was raised by the codec.
+.. c:function:: PyObject* PyUnicode_AsUnicodeEscapeString(PyObject *unicode)
+
+ Encode a Unicode object using Unicode-Escape and return the result as Python
+ string object. Error handling is "strict". Return *NULL* if an exception was
+ raised by the codec.
+
+
.. c:function:: PyObject* PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s, Py_ssize_t size)
Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Unicode-Escape and
return a Python string object. Return *NULL* if an exception was raised by the
codec.
-
-.. c:function:: PyObject* PyUnicode_AsUnicodeEscapeString(PyObject *unicode)
-
- Encode a Unicode object using Unicode-Escape and return the result as Python
- string object. Error handling is "strict". Return *NULL* if an exception was
- raised by the codec.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsUnicodeEscapeString`.
Raw-Unicode-Escape Codecs
@@ -788,19 +1182,13 @@ Raw-Unicode-Escape Codecs
These are the "Raw Unicode Escape" codec APIs:
-.. c:function:: PyObject* PyUnicode_DecodeRawUnicodeEscape(const char *s, Py_ssize_t size, const char *errors)
+.. c:function:: PyObject* PyUnicode_DecodeRawUnicodeEscape(const char *s, \
+ Py_ssize_t size, const char *errors)
Create a Unicode object by decoding *size* bytes of the Raw-Unicode-Escape
encoded string *s*. Return *NULL* if an exception was raised by the codec.
-.. c:function:: PyObject* PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s, Py_ssize_t size, const char *errors)
-
- Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Raw-Unicode-Escape
- and return a Python string object. Return *NULL* if an exception was raised by
- the codec.
-
-
.. c:function:: PyObject* PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode)
Encode a Unicode object using Raw-Unicode-Escape and return the result as
@@ -808,6 +1196,18 @@ These are the "Raw Unicode Escape" codec APIs:
was raised by the codec.
+.. c:function:: PyObject* PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s, \
+ Py_ssize_t size, const char *errors)
+
+ Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Raw-Unicode-Escape
+ and return a Python string object. Return *NULL* if an exception was raised by
+ the codec.
+
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsRawUnicodeEscapeString`.
+
+
Latin-1 Codecs
""""""""""""""
@@ -821,18 +1221,22 @@ ordinals and only these are accepted by the codecs during encoding.
*s*. Return *NULL* if an exception was raised by the codec.
+.. c:function:: PyObject* PyUnicode_AsLatin1String(PyObject *unicode)
+
+ Encode a Unicode object using Latin-1 and return the result as Python bytes
+ object. Error handling is "strict". Return *NULL* if an exception was
+ raised by the codec.
+
+
.. c:function:: PyObject* PyUnicode_EncodeLatin1(const Py_UNICODE *s, Py_ssize_t size, const char *errors)
Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Latin-1 and
return a Python bytes object. Return *NULL* if an exception was raised by
the codec.
-
-.. c:function:: PyObject* PyUnicode_AsLatin1String(PyObject *unicode)
-
- Encode a Unicode object using Latin-1 and return the result as Python bytes
- object. Error handling is "strict". Return *NULL* if an exception was
- raised by the codec.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsLatin1String`.
ASCII Codecs
@@ -848,18 +1252,22 @@ codes generate errors.
*s*. Return *NULL* if an exception was raised by the codec.
+.. c:function:: PyObject* PyUnicode_AsASCIIString(PyObject *unicode)
+
+ Encode a Unicode object using ASCII and return the result as Python bytes
+ object. Error handling is "strict". Return *NULL* if an exception was
+ raised by the codec.
+
+
.. c:function:: PyObject* PyUnicode_EncodeASCII(const Py_UNICODE *s, Py_ssize_t size, const char *errors)
Encode the :c:type:`Py_UNICODE` buffer of the given *size* using ASCII and
return a Python bytes object. Return *NULL* if an exception was raised by
the codec.
-
-.. c:function:: PyObject* PyUnicode_AsASCIIString(PyObject *unicode)
-
- Encode a Unicode object using ASCII and return the result as Python bytes
- object. Error handling is "strict". Return *NULL* if an exception was
- raised by the codec.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsASCIIString`.
Character Map Codecs
@@ -888,7 +1296,8 @@ characters to different code points.
These are the mapping codec APIs:
-.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *s, Py_ssize_t size, PyObject *mapping, const char *errors)
+.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *s, Py_ssize_t size, \
+ PyObject *mapping, const char *errors)
Create a Unicode object by decoding *size* bytes of the encoded string *s* using
the given *mapping* object. Return *NULL* if an exception was raised by the
@@ -898,13 +1307,6 @@ These are the mapping codec APIs:
treated as "undefined mapping".
-.. c:function:: PyObject* PyUnicode_EncodeCharmap(const Py_UNICODE *s, Py_ssize_t size, PyObject *mapping, const char *errors)
-
- Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given
- *mapping* object and return a Python string object. Return *NULL* if an
- exception was raised by the codec.
-
-
.. c:function:: PyObject* PyUnicode_AsCharmapString(PyObject *unicode, PyObject *mapping)
Encode a Unicode object using the given *mapping* object and return the result
@@ -914,7 +1316,8 @@ These are the mapping codec APIs:
The following codec API is special in that maps Unicode to Unicode.
-.. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, PyObject *table, const char *errors)
+.. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, \
+ PyObject *table, const char *errors)
Translate a :c:type:`Py_UNICODE` buffer of the given *size* by applying a
character mapping *table* to it and return the resulting Unicode object. Return
@@ -927,6 +1330,22 @@ The following codec API is special in that maps Unicode to Unicode.
and sequences work well. Unmapped character ordinals (ones which cause a
:exc:`LookupError`) are left untouched and are copied as-is.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API.
+
+ .. XXX replace with what?
+
+
+.. c:function:: PyObject* PyUnicode_EncodeCharmap(const Py_UNICODE *s, Py_ssize_t size, \
+ PyObject *mapping, const char *errors)
+
+ Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given
+ *mapping* object and return a Python string object. Return *NULL* if an
+ exception was raised by the codec.
+
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsCharmapString`.
MBCS codecs for Windows
@@ -943,7 +1362,8 @@ the user settings on the machine running the codec.
Return *NULL* if an exception was raised by the codec.
-.. c:function:: PyObject* PyUnicode_DecodeMBCSStateful(const char *s, int size, const char *errors, int *consumed)
+.. c:function:: PyObject* PyUnicode_DecodeMBCSStateful(const char *s, int size, \
+ const char *errors, int *consumed)
If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeMBCS`. If
*consumed* is not *NULL*, :c:func:`PyUnicode_DecodeMBCSStateful` will not decode
@@ -951,18 +1371,22 @@ the user settings on the machine running the codec.
in *consumed*.
+.. c:function:: PyObject* PyUnicode_AsMBCSString(PyObject *unicode)
+
+ Encode a Unicode object using MBCS and return the result as Python bytes
+ object. Error handling is "strict". Return *NULL* if an exception was
+ raised by the codec.
+
+
.. c:function:: PyObject* PyUnicode_EncodeMBCS(const Py_UNICODE *s, Py_ssize_t size, const char *errors)
Encode the :c:type:`Py_UNICODE` buffer of the given *size* using MBCS and return
a Python bytes object. Return *NULL* if an exception was raised by the
codec.
-
-.. c:function:: PyObject* PyUnicode_AsMBCSString(PyObject *unicode)
-
- Encode a Unicode object using MBCS and return the result as Python bytes
- object. Error handling is "strict". Return *NULL* if an exception was
- raised by the codec.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsMBCSString`.
Methods & Slots
@@ -1001,7 +1425,8 @@ They all return *NULL* or ``-1`` if an exception occurs.
characters are not included in the resulting strings.
-.. c:function:: PyObject* PyUnicode_Translate(PyObject *str, PyObject *table, const char *errors)
+.. c:function:: PyObject* PyUnicode_Translate(PyObject *str, PyObject *table, \
+ const char *errors)
Translate a string by applying a character mapping table to it and return the
resulting Unicode object.
@@ -1023,14 +1448,16 @@ They all return *NULL* or ``-1`` if an exception occurs.
Unicode string.
-.. c:function:: int PyUnicode_Tailmatch(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end, int direction)
+.. c:function:: int PyUnicode_Tailmatch(PyObject *str, PyObject *substr, \
+ Py_ssize_t start, Py_ssize_t end, int direction)
Return 1 if *substr* matches ``str[start:end]`` at the given tail end
(*direction* == -1 means to do a prefix match, *direction* == 1 a suffix match),
0 otherwise. Return ``-1`` if an error occurred.
-.. c:function:: Py_ssize_t PyUnicode_Find(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end, int direction)
+.. c:function:: Py_ssize_t PyUnicode_Find(PyObject *str, PyObject *substr, \
+ Py_ssize_t start, Py_ssize_t end, int direction)
Return the first position of *substr* in ``str[start:end]`` using the given
*direction* (*direction* == 1 means to do a forward search, *direction* == -1 a
@@ -1039,13 +1466,27 @@ They all return *NULL* or ``-1`` if an exception occurs.
occurred and an exception has been set.
-.. c:function:: Py_ssize_t PyUnicode_Count(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end)
+.. c:function:: Py_ssize_t PyUnicode_FindChar(PyObject *str, Py_UCS4 ch, \
+ Py_ssize_t start, Py_ssize_t end, int direction)
+
+ Return the first position of the character *ch* in ``str[start:end]`` using
+ the given *direction* (*direction* == 1 means to do a forward search,
+ *direction* == -1 a backward search). The return value is the index of the
+ first match; a value of ``-1`` indicates that no match was found, and ``-2``
+ indicates that an error occurred and an exception has been set.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: Py_ssize_t PyUnicode_Count(PyObject *str, PyObject *substr, \
+ Py_ssize_t start, Py_ssize_t end)
Return the number of non-overlapping occurrences of *substr* in
``str[start:end]``. Return ``-1`` if an error occurred.
-.. c:function:: PyObject* PyUnicode_Replace(PyObject *str, PyObject *substr, PyObject *replstr, Py_ssize_t maxcount)
+.. c:function:: PyObject* PyUnicode_Replace(PyObject *str, PyObject *substr, \
+ PyObject *replstr, Py_ssize_t maxcount)
Replace at most *maxcount* occurrences of *substr* in *str* with *replstr* and
return the resulting Unicode object. *maxcount* == -1 means replace all
@@ -1093,8 +1534,8 @@ They all return *NULL* or ``-1`` if an exception occurs.
Check whether *element* is contained in *container* and return true or false
accordingly.
- *element* has to coerce to a one element Unicode string. ``-1`` is returned if
- there was an error.
+ *element* has to coerce to a one element Unicode string. ``-1`` is returned
+ if there was an error.
.. c:function:: void PyUnicode_InternInPlace(PyObject **string)
@@ -1113,7 +1554,6 @@ They all return *NULL* or ``-1`` if an exception occurs.
.. c:function:: PyObject* PyUnicode_InternFromString(const char *v)
A combination of :c:func:`PyUnicode_FromString` and
- :c:func:`PyUnicode_InternInPlace`, returning either a new unicode string object
- that has been interned, or a new ("owned") reference to an earlier interned
- string object with the same value.
-
+ :c:func:`PyUnicode_InternInPlace`, returning either a new unicode string
+ object that has been interned, or a new ("owned") reference to an earlier
+ interned string object with the same value.
diff --git a/Doc/contents.rst b/Doc/contents.rst
index e938fcd..e9d1771 100644
--- a/Doc/contents.rst
+++ b/Doc/contents.rst
@@ -11,7 +11,7 @@
library/index.rst
extending/index.rst
c-api/index.rst
- distutils/index.rst
+ packaging/index.rst
install/index.rst
documenting/index.rst
howto/index.rst
diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst
index b3def22..97f1bed 100644
--- a/Doc/distutils/apiref.rst
+++ b/Doc/distutils/apiref.rst
@@ -160,7 +160,7 @@ the full reference.
.. class:: Extension
The Extension class describes a single C or C++extension module in a setup
- script. It accepts the following keyword arguments in its constructor
+ script. It accepts the following keyword arguments in its constructor:
+------------------------+--------------------------------+---------------------------+
| argument name | value | type |
@@ -1162,12 +1162,11 @@ other utility module.
.. function:: grok_environment_error(exc[, prefix='error: '])
- Generate a useful error message from an :exc:`EnvironmentError` (:exc:`IOError`
- or :exc:`OSError`) exception object. Handles Python 1.5.1 and later styles,
- and does what it can to deal with exception objects that don't have a filename
- (which happens when the error is due to a two-file operation, such as
- :func:`rename` or :func:`link`). Returns the error message as a string
- prefixed with *prefix*.
+ Generate a useful error message from an :exc:`OSError` exception object.
+ Handles Python 1.5.1 and later styles, and does what it can to deal with
+ exception objects that don't have a filename (which happens when the error
+ is due to a two-file operation, such as :func:`rename` or :func:`link`).
+ Returns the error message as a string prefixed with *prefix*.
.. function:: split_quoted(s)
diff --git a/Doc/distutils/index.rst b/Doc/distutils/index.rst
index ace8280..c8dd9f4 100644
--- a/Doc/distutils/index.rst
+++ b/Doc/distutils/index.rst
@@ -14,9 +14,12 @@ the module developer's point of view, describing how to use the Distutils to
make Python modules and extensions easily available to a wider audience with
very little overhead for build/release/install mechanics.
+.. deprecated:: 3.3
+ :mod:`packaging` replaces Distutils. See :ref:`packaging-index` and
+ :ref:`packaging-install-index`.
+
.. toctree::
:maxdepth: 2
- :numbered:
introduction.rst
setupscript.rst
@@ -29,3 +32,10 @@ very little overhead for build/release/install mechanics.
extending.rst
commandref.rst
apiref.rst
+
+Another document describes how to install modules and extensions packaged
+following the above guidelines:
+
+.. toctree::
+
+ install.rst
diff --git a/Doc/distutils/install.rst b/Doc/distutils/install.rst
new file mode 100644
index 0000000..b20f1fb
--- /dev/null
+++ b/Doc/distutils/install.rst
@@ -0,0 +1,1086 @@
+.. highlightlang:: none
+
+.. _install-index:
+
+*****************************
+ Installing Python Modules
+*****************************
+
+:Author: Greg Ward
+:Release: |version|
+:Date: |today|
+
+.. TODO: Fill in XXX comments
+
+.. The audience for this document includes people who don't know anything
+ about Python and aren't about to learn the language just in order to
+ install and maintain it for their users, i.e. system administrators.
+ Thus, I have to be sure to explain the basics at some point:
+ sys.path and PYTHONPATH at least. Should probably give pointers to
+ other docs on "import site", PYTHONSTARTUP, PYTHONHOME, etc.
+
+ Finally, it might be useful to include all the material from my "Care
+ and Feeding of a Python Installation" talk in here somewhere. Yow!
+
+.. topic:: Abstract
+
+ This document describes the Python Distribution Utilities ("Distutils") from the
+ end-user's point-of-view, describing how to extend the capabilities of a
+ standard Python installation by building and installing third-party Python
+ modules and extensions.
+
+
+.. _inst-intro:
+
+Introduction
+============
+
+Although Python's extensive standard library covers many programming needs,
+there often comes a time when you need to add some new functionality to your
+Python installation in the form of third-party modules. This might be necessary
+to support your own programming, or to support an application that you want to
+use and that happens to be written in Python.
+
+In the past, there has been little support for adding third-party modules to an
+existing Python installation. With the introduction of the Python Distribution
+Utilities (Distutils for short) in Python 2.0, this changed.
+
+This document is aimed primarily at the people who need to install third-party
+Python modules: end-users and system administrators who just need to get some
+Python application running, and existing Python programmers who want to add some
+new goodies to their toolbox. You don't need to know Python to read this
+document; there will be some brief forays into using Python's interactive mode
+to explore your installation, but that's it. If you're looking for information
+on how to distribute your own Python modules so that others may use them, see
+the :ref:`distutils-index` manual.
+
+
+.. _inst-trivial-install:
+
+Best case: trivial installation
+-------------------------------
+
+In the best case, someone will have prepared a special version of the module
+distribution you want to install that is targeted specifically at your platform
+and is installed just like any other software on your platform. For example,
+the module developer might make an executable installer available for Windows
+users, an RPM package for users of RPM-based Linux systems (Red Hat, SuSE,
+Mandrake, and many others), a Debian package for users of Debian-based Linux
+systems, and so forth.
+
+In that case, you would download the installer appropriate to your platform and
+do the obvious thing with it: run it if it's an executable installer, ``rpm
+--install`` it if it's an RPM, etc. You don't need to run Python or a setup
+script, you don't need to compile anything---you might not even need to read any
+instructions (although it's always a good idea to do so anyway).
+
+Of course, things will not always be that easy. You might be interested in a
+module distribution that doesn't have an easy-to-use installer for your
+platform. In that case, you'll have to start with the source distribution
+released by the module's author/maintainer. Installing from a source
+distribution is not too hard, as long as the modules are packaged in the
+standard way. The bulk of this document is about building and installing
+modules from standard source distributions.
+
+
+.. _inst-new-standard:
+
+The new standard: Distutils
+---------------------------
+
+If you download a module source distribution, you can tell pretty quickly if it
+was packaged and distributed in the standard way, i.e. using the Distutils.
+First, the distribution's name and version number will be featured prominently
+in the name of the downloaded archive, e.g. :file:`foo-1.0.tar.gz` or
+:file:`widget-0.9.7.zip`. Next, the archive will unpack into a similarly-named
+directory: :file:`foo-1.0` or :file:`widget-0.9.7`. Additionally, the
+distribution will contain a setup script :file:`setup.py`, and a file named
+:file:`README.txt` or possibly just :file:`README`, which should explain that
+building and installing the module distribution is a simple matter of running
+one command from a terminal::
+
+ python setup.py install
+
+For Windows, this command should be run from a command prompt window
+(:menuselection:`Start --> Accessories`)::
+
+ setup.py install
+
+If all these things are true, then you already know how to build and install the
+modules you've just downloaded: Run the command above. Unless you need to
+install things in a non-standard way or customize the build process, you don't
+really need this manual. Or rather, the above command is everything you need to
+get out of this manual.
+
+
+.. _inst-standard-install:
+
+Standard Build and Install
+==========================
+
+As described in section :ref:`inst-new-standard`, building and installing a module
+distribution using the Distutils is usually one simple command to run from a
+terminal::
+
+ python setup.py install
+
+
+.. _inst-platform-variations:
+
+Platform variations
+-------------------
+
+You should always run the setup command from the distribution root directory,
+i.e. the top-level subdirectory that the module source distribution unpacks
+into. For example, if you've just downloaded a module source distribution
+:file:`foo-1.0.tar.gz` onto a Unix system, the normal thing to do is::
+
+ gunzip -c foo-1.0.tar.gz | tar xf - # unpacks into directory foo-1.0
+ cd foo-1.0
+ python setup.py install
+
+On Windows, you'd probably download :file:`foo-1.0.zip`. If you downloaded the
+archive file to :file:`C:\\Temp`, then it would unpack into
+:file:`C:\\Temp\\foo-1.0`; you can use either a archive manipulator with a
+graphical user interface (such as WinZip) or a command-line tool (such as
+:program:`unzip` or :program:`pkunzip`) to unpack the archive. Then, open a
+command prompt window and run::
+
+ cd c:\Temp\foo-1.0
+ python setup.py install
+
+
+.. _inst-splitting-up:
+
+Splitting the job up
+--------------------
+
+Running ``setup.py install`` builds and installs all modules in one run. If you
+prefer to work incrementally---especially useful if you want to customize the
+build process, or if things are going wrong---you can use the setup script to do
+one thing at a time. This is particularly helpful when the build and install
+will be done by different users---for example, you might want to build a module
+distribution and hand it off to a system administrator for installation (or do
+it yourself, with super-user privileges).
+
+For example, you can build everything in one step, and then install everything
+in a second step, by invoking the setup script twice::
+
+ python setup.py build
+ python setup.py install
+
+If you do this, you will notice that running the :command:`install` command
+first runs the :command:`build` command, which---in this case---quickly notices
+that it has nothing to do, since everything in the :file:`build` directory is
+up-to-date.
+
+You may not need this ability to break things down often if all you do is
+install modules downloaded off the 'net, but it's very handy for more advanced
+tasks. If you get into distributing your own Python modules and extensions,
+you'll run lots of individual Distutils commands on their own.
+
+
+.. _inst-how-build-works:
+
+How building works
+------------------
+
+As implied above, the :command:`build` command is responsible for putting the
+files to install into a *build directory*. By default, this is :file:`build`
+under the distribution root; if you're excessively concerned with speed, or want
+to keep the source tree pristine, you can change the build directory with the
+:option:`--build-base` option. For example::
+
+ python setup.py build --build-base=/tmp/pybuild/foo-1.0
+
+(Or you could do this permanently with a directive in your system or personal
+Distutils configuration file; see section :ref:`inst-config-files`.) Normally, this
+isn't necessary.
+
+The default layout for the build tree is as follows::
+
+ --- build/ --- lib/
+ or
+ --- build/ --- lib.<plat>/
+ temp.<plat>/
+
+where ``<plat>`` expands to a brief description of the current OS/hardware
+platform and Python version. The first form, with just a :file:`lib` directory,
+is used for "pure module distributions"---that is, module distributions that
+include only pure Python modules. If a module distribution contains any
+extensions (modules written in C/C++), then the second form, with two ``<plat>``
+directories, is used. In that case, the :file:`temp.{plat}` directory holds
+temporary files generated by the compile/link process that don't actually get
+installed. In either case, the :file:`lib` (or :file:`lib.{plat}`) directory
+contains all Python modules (pure Python and extensions) that will be installed.
+
+In the future, more directories will be added to handle Python scripts,
+documentation, binary executables, and whatever else is needed to handle the job
+of installing Python modules and applications.
+
+
+.. _inst-how-install-works:
+
+How installation works
+----------------------
+
+After the :command:`build` command runs (whether you run it explicitly, or the
+:command:`install` command does it for you), the work of the :command:`install`
+command is relatively simple: all it has to do is copy everything under
+:file:`build/lib` (or :file:`build/lib.{plat}`) to your chosen installation
+directory.
+
+If you don't choose an installation directory---i.e., if you just run ``setup.py
+install``\ ---then the :command:`install` command installs to the standard
+location for third-party Python modules. This location varies by platform and
+by how you built/installed Python itself. On Unix (and Mac OS X, which is also
+Unix-based), it also depends on whether the module distribution being installed
+is pure Python or contains extensions ("non-pure"):
+
++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+
+| Platform | Standard installation location | Default value | Notes |
++=================+=====================================================+==================================================+=======+
+| Unix (pure) | :file:`{prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) |
++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+
+| Unix (non-pure) | :file:`{exec-prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) |
++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+
+| Windows | :file:`{prefix}\\Lib\\site-packages` | :file:`C:\\Python{XY}\\Lib\\site-packages` | \(2) |
++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+
+
+Notes:
+
+(1)
+ Most Linux distributions include Python as a standard part of the system, so
+ :file:`{prefix}` and :file:`{exec-prefix}` are usually both :file:`/usr` on
+ Linux. If you build Python yourself on Linux (or any Unix-like system), the
+ default :file:`{prefix}` and :file:`{exec-prefix}` are :file:`/usr/local`.
+
+(2)
+ The default installation directory on Windows was :file:`C:\\Program
+ Files\\Python` under Python 1.6a1, 1.5.2, and earlier.
+
+:file:`{prefix}` and :file:`{exec-prefix}` stand for the directories that Python
+is installed to, and where it finds its libraries at run-time. They are always
+the same under Windows, and very often the same under Unix and Mac OS X. You
+can find out what your Python installation uses for :file:`{prefix}` and
+:file:`{exec-prefix}` by running Python in interactive mode and typing a few
+simple commands. Under Unix, just type ``python`` at the shell prompt. Under
+Windows, choose :menuselection:`Start --> Programs --> Python X.Y -->
+Python (command line)`. Once the interpreter is started, you type Python code
+at the prompt. For example, on my Linux system, I type the three Python
+statements shown below, and get the output as shown, to find out my
+:file:`{prefix}` and :file:`{exec-prefix}`::
+
+ Python 2.4 (#26, Aug 7 2004, 17:19:02)
+ Type "help", "copyright", "credits" or "license" for more information.
+ >>> import sys
+ >>> sys.prefix
+ '/usr'
+ >>> sys.exec_prefix
+ '/usr'
+
+A few other placeholders are used in this document: :file:`{X.Y}` stands for the
+version of Python, for example ``3.2``; :file:`{abiflags}` will be replaced by
+the value of :data:`sys.abiflags` or the empty string for platforms which don't
+define ABI flags; :file:`{distname}` will be replaced by the name of the module
+distribution being installed. Dots and capitalization are important in the
+paths; for example, a value that uses ``python3.2`` on UNIX will typically use
+``Python32`` on Windows.
+
+If you don't want to install modules to the standard location, or if you don't
+have permission to write there, then you need to read about alternate
+installations in section :ref:`inst-alt-install`. If you want to customize your
+installation directories more heavily, see section :ref:`inst-custom-install` on
+custom installations.
+
+
+.. _inst-alt-install:
+
+Alternate Installation
+======================
+
+Often, it is necessary or desirable to install modules to a location other than
+the standard location for third-party Python modules. For example, on a Unix
+system you might not have permission to write to the standard third-party module
+directory. Or you might wish to try out a module before making it a standard
+part of your local Python installation. This is especially true when upgrading
+a distribution already present: you want to make sure your existing base of
+scripts still works with the new version before actually upgrading.
+
+The Distutils :command:`install` command is designed to make installing module
+distributions to an alternate location simple and painless. The basic idea is
+that you supply a base directory for the installation, and the
+:command:`install` command picks a set of directories (called an *installation
+scheme*) under this base directory in which to install files. The details
+differ across platforms, so read whichever of the following sections applies to
+you.
+
+Note that the various alternate installation schemes are mutually exclusive: you
+can pass ``--user``, or ``--home``, or ``--prefix`` and ``--exec-prefix``, or
+``--install-base`` and ``--install-platbase``, but you can't mix from these
+groups.
+
+
+.. _inst-alt-install-user:
+
+Alternate installation: the user scheme
+---------------------------------------
+
+This scheme is designed to be the most convenient solution for users that don't
+have write permission to the global site-packages directory or don't want to
+install into it. It is enabled with a simple option::
+
+ python setup.py install --user
+
+Files will be installed into subdirectories of :data:`site.USER_BASE` (written
+as :file:`{userbase}` hereafter). This scheme installs pure Python modules and
+extension modules in the same location (also known as :data:`site.USER_SITE`).
+Here are the values for UNIX, including Mac OS X:
+
+=============== ===========================================================
+Type of file Installation directory
+=============== ===========================================================
+modules :file:`{userbase}/lib/python{X.Y}/site-packages`
+scripts :file:`{userbase}/bin`
+data :file:`{userbase}`
+C headers :file:`{userbase}/include/python{X.Y}{abiflags}/{distname}`
+=============== ===========================================================
+
+And here are the values used on Windows:
+
+=============== ===========================================================
+Type of file Installation directory
+=============== ===========================================================
+modules :file:`{userbase}\\Python{XY}\\site-packages`
+scripts :file:`{userbase}\\Scripts`
+data :file:`{userbase}`
+C headers :file:`{userbase}\\Python{XY}\\Include\\{distname}`
+=============== ===========================================================
+
+The advantage of using this scheme compared to the other ones described below is
+that the user site-packages directory is under normal conditions always included
+in :data:`sys.path` (see :mod:`site` for more information), which means that
+there is no additional step to perform after running the :file:`setup.py` script
+to finalize the installation.
+
+The :command:`build_ext` command also has a ``--user`` option to add
+:file:`{userbase}/include` to the compiler search path for header files and
+:file:`{userbase}/lib` to the compiler search path for libraries as well as to
+the runtime search path for shared C libraries (rpath).
+
+
+.. _inst-alt-install-home:
+
+Alternate installation: the home scheme
+---------------------------------------
+
+The idea behind the "home scheme" is that you build and maintain a personal
+stash of Python modules. This scheme's name is derived from the idea of a
+"home" directory on Unix, since it's not unusual for a Unix user to make their
+home directory have a layout similar to :file:`/usr/` or :file:`/usr/local/`.
+This scheme can be used by anyone, regardless of the operating system they
+are installing for.
+
+Installing a new module distribution is as simple as ::
+
+ python setup.py install --home=<dir>
+
+where you can supply any directory you like for the :option:`--home` option. On
+Unix, lazy typists can just type a tilde (``~``); the :command:`install` command
+will expand this to your home directory::
+
+ python setup.py install --home=~
+
+To make Python find the distributions installed with this scheme, you may have
+to :ref:`modify Python's search path <inst-search-path>` or edit
+:mod:`sitecustomize` (see :mod:`site`) to call :func:`site.addsitedir` or edit
+:data:`sys.path`.
+
+The :option:`--home` option defines the installation base directory. Files are
+installed to the following directories under the installation base as follows:
+
+=============== ===========================================================
+Type of file Installation directory
+=============== ===========================================================
+modules :file:`{home}/lib/python`
+scripts :file:`{home}/bin`
+data :file:`{home}`
+C headers :file:`{home}/include/python/{distname}`
+=============== ===========================================================
+
+(Mentally replace slashes with backslashes if you're on Windows.)
+
+
+.. _inst-alt-install-prefix-unix:
+
+Alternate installation: Unix (the prefix scheme)
+------------------------------------------------
+
+The "prefix scheme" is useful when you wish to use one Python installation to
+perform the build/install (i.e., to run the setup script), but install modules
+into the third-party module directory of a different Python installation (or
+something that looks like a different Python installation). If this sounds a
+trifle unusual, it is---that's why the user and home schemes come before. However,
+there are at least two known cases where the prefix scheme will be useful.
+
+First, consider that many Linux distributions put Python in :file:`/usr`, rather
+than the more traditional :file:`/usr/local`. This is entirely appropriate,
+since in those cases Python is part of "the system" rather than a local add-on.
+However, if you are installing Python modules from source, you probably want
+them to go in :file:`/usr/local/lib/python2.{X}` rather than
+:file:`/usr/lib/python2.{X}`. This can be done with ::
+
+ /usr/bin/python setup.py install --prefix=/usr/local
+
+Another possibility is a network filesystem where the name used to write to a
+remote directory is different from the name used to read it: for example, the
+Python interpreter accessed as :file:`/usr/local/bin/python` might search for
+modules in :file:`/usr/local/lib/python2.{X}`, but those modules would have to
+be installed to, say, :file:`/mnt/{@server}/export/lib/python2.{X}`. This could
+be done with ::
+
+ /usr/local/bin/python setup.py install --prefix=/mnt/@server/export
+
+In either case, the :option:`--prefix` option defines the installation base, and
+the :option:`--exec-prefix` option defines the platform-specific installation
+base, which is used for platform-specific files. (Currently, this just means
+non-pure module distributions, but could be expanded to C libraries, binary
+executables, etc.) If :option:`--exec-prefix` is not supplied, it defaults to
+:option:`--prefix`. Files are installed as follows:
+
+================= ==========================================================
+Type of file Installation directory
+================= ==========================================================
+Python modules :file:`{prefix}/lib/python{X.Y}/site-packages`
+extension modules :file:`{exec-prefix}/lib/python{X.Y}/site-packages`
+scripts :file:`{prefix}/bin`
+data :file:`{prefix}`
+C headers :file:`{prefix}/include/python{X.Y}{abiflags}/{distname}`
+================= ==========================================================
+
+There is no requirement that :option:`--prefix` or :option:`--exec-prefix`
+actually point to an alternate Python installation; if the directories listed
+above do not already exist, they are created at installation time.
+
+Incidentally, the real reason the prefix scheme is important is simply that a
+standard Unix installation uses the prefix scheme, but with :option:`--prefix`
+and :option:`--exec-prefix` supplied by Python itself as ``sys.prefix`` and
+``sys.exec_prefix``. Thus, you might think you'll never use the prefix scheme,
+but every time you run ``python setup.py install`` without any other options,
+you're using it.
+
+Note that installing extensions to an alternate Python installation has no
+effect on how those extensions are built: in particular, the Python header files
+(:file:`Python.h` and friends) installed with the Python interpreter used to run
+the setup script will be used in compiling extensions. It is your
+responsibility to ensure that the interpreter used to run extensions installed
+in this way is compatible with the interpreter used to build them. The best way
+to do this is to ensure that the two interpreters are the same version of Python
+(possibly different builds, or possibly copies of the same build). (Of course,
+if your :option:`--prefix` and :option:`--exec-prefix` don't even point to an
+alternate Python installation, this is immaterial.)
+
+
+.. _inst-alt-install-prefix-windows:
+
+Alternate installation: Windows (the prefix scheme)
+---------------------------------------------------
+
+Windows has no concept of a user's home directory, and since the standard Python
+installation under Windows is simpler than under Unix, the :option:`--prefix`
+option has traditionally been used to install additional packages in separate
+locations on Windows. ::
+
+ python setup.py install --prefix="\Temp\Python"
+
+to install modules to the :file:`\\Temp\\Python` directory on the current drive.
+
+The installation base is defined by the :option:`--prefix` option; the
+:option:`--exec-prefix` option is not supported under Windows, which means that
+pure Python modules and extension modules are installed into the same location.
+Files are installed as follows:
+
+=============== ==========================================================
+Type of file Installation directory
+=============== ==========================================================
+modules :file:`{prefix}\\Lib\\site-packages`
+scripts :file:`{prefix}\\Scripts`
+data :file:`{prefix}`
+C headers :file:`{prefix}\\Include\\{distname}`
+=============== ==========================================================
+
+
+.. _inst-custom-install:
+
+Custom Installation
+===================
+
+Sometimes, the alternate installation schemes described in section
+:ref:`inst-alt-install` just don't do what you want. You might want to tweak just
+one or two directories while keeping everything under the same base directory,
+or you might want to completely redefine the installation scheme. In either
+case, you're creating a *custom installation scheme*.
+
+To create a custom installation scheme, you start with one of the alternate
+schemes and override some of the installation directories used for the various
+types of files, using these options:
+
+====================== =======================
+Type of file Override option
+====================== =======================
+Python modules ``--install-purelib``
+extension modules ``--install-platlib``
+all modules ``--install-lib``
+scripts ``--install-scripts``
+data ``--install-data``
+C headers ``--install-headers``
+====================== =======================
+
+These override options can be relative, absolute,
+or explicitly defined in terms of one of the installation base directories.
+(There are two installation base directories, and they are normally the same---
+they only differ when you use the Unix "prefix scheme" and supply different
+``--prefix`` and ``--exec-prefix`` options; using ``--install-lib`` will
+override values computed or given for ``--install-purelib`` and
+``--install-platlib``, and is recommended for schemes that don't make a
+difference between Python and extension modules.)
+
+For example, say you're installing a module distribution to your home directory
+under Unix---but you want scripts to go in :file:`~/scripts` rather than
+:file:`~/bin`. As you might expect, you can override this directory with the
+:option:`--install-scripts` option; in this case, it makes most sense to supply
+a relative path, which will be interpreted relative to the installation base
+directory (your home directory, in this case)::
+
+ python setup.py install --home=~ --install-scripts=scripts
+
+Another Unix example: suppose your Python installation was built and installed
+with a prefix of :file:`/usr/local/python`, so under a standard installation
+scripts will wind up in :file:`/usr/local/python/bin`. If you want them in
+:file:`/usr/local/bin` instead, you would supply this absolute directory for the
+:option:`--install-scripts` option::
+
+ python setup.py install --install-scripts=/usr/local/bin
+
+(This performs an installation using the "prefix scheme," where the prefix is
+whatever your Python interpreter was installed with--- :file:`/usr/local/python`
+in this case.)
+
+If you maintain Python on Windows, you might want third-party modules to live in
+a subdirectory of :file:`{prefix}`, rather than right in :file:`{prefix}`
+itself. This is almost as easy as customizing the script installation directory
+---you just have to remember that there are two types of modules to worry about,
+Python and extension modules, which can conveniently be both controlled by one
+option::
+
+ python setup.py install --install-lib=Site
+
+The specified installation directory is relative to :file:`{prefix}`. Of
+course, you also have to ensure that this directory is in Python's module
+search path, such as by putting a :file:`.pth` file in a site directory (see
+:mod:`site`). See section :ref:`inst-search-path` to find out how to modify
+Python's search path.
+
+If you want to define an entire installation scheme, you just have to supply all
+of the installation directory options. The recommended way to do this is to
+supply relative paths; for example, if you want to maintain all Python
+module-related files under :file:`python` in your home directory, and you want a
+separate directory for each platform that you use your home directory from, you
+might define the following installation scheme::
+
+ python setup.py install --home=~ \
+ --install-purelib=python/lib \
+ --install-platlib=python/lib.$PLAT \
+ --install-scripts=python/scripts
+ --install-data=python/data
+
+or, equivalently, ::
+
+ python setup.py install --home=~/python \
+ --install-purelib=lib \
+ --install-platlib='lib.$PLAT' \
+ --install-scripts=scripts
+ --install-data=data
+
+``$PLAT`` is not (necessarily) an environment variable---it will be expanded by
+the Distutils as it parses your command line options, just as it does when
+parsing your configuration file(s).
+
+Obviously, specifying the entire installation scheme every time you install a
+new module distribution would be very tedious. Thus, you can put these options
+into your Distutils config file (see section :ref:`inst-config-files`)::
+
+ [install]
+ install-base=$HOME
+ install-purelib=python/lib
+ install-platlib=python/lib.$PLAT
+ install-scripts=python/scripts
+ install-data=python/data
+
+or, equivalently, ::
+
+ [install]
+ install-base=$HOME/python
+ install-purelib=lib
+ install-platlib=lib.$PLAT
+ install-scripts=scripts
+ install-data=data
+
+Note that these two are *not* equivalent if you supply a different installation
+base directory when you run the setup script. For example, ::
+
+ python setup.py install --install-base=/tmp
+
+would install pure modules to :file:`/tmp/python/lib` in the first case, and
+to :file:`/tmp/lib` in the second case. (For the second case, you probably
+want to supply an installation base of :file:`/tmp/python`.)
+
+You probably noticed the use of ``$HOME`` and ``$PLAT`` in the sample
+configuration file input. These are Distutils configuration variables, which
+bear a strong resemblance to environment variables. In fact, you can use
+environment variables in config files on platforms that have such a notion but
+the Distutils additionally define a few extra variables that may not be in your
+environment, such as ``$PLAT``. (And of course, on systems that don't have
+environment variables, such as Mac OS 9, the configuration variables supplied by
+the Distutils are the only ones you can use.) See section :ref:`inst-config-files`
+for details.
+
+.. XXX need some Windows examples---when would custom installation schemes be
+ needed on those platforms?
+
+
+.. XXX Move this to Doc/using
+
+.. _inst-search-path:
+
+Modifying Python's Search Path
+------------------------------
+
+When the Python interpreter executes an :keyword:`import` statement, it searches
+for both Python code and extension modules along a search path. A default value
+for the path is configured into the Python binary when the interpreter is built.
+You can determine the path by importing the :mod:`sys` module and printing the
+value of ``sys.path``. ::
+
+ $ python
+ Python 2.2 (#11, Oct 3 2002, 13:31:27)
+ [GCC 2.96 20000731 (Red Hat Linux 7.3 2.96-112)] on linux2
+ Type "help", "copyright", "credits" or "license" for more information.
+ >>> import sys
+ >>> sys.path
+ ['', '/usr/local/lib/python2.3', '/usr/local/lib/python2.3/plat-linux2',
+ '/usr/local/lib/python2.3/lib-tk', '/usr/local/lib/python2.3/lib-dynload',
+ '/usr/local/lib/python2.3/site-packages']
+ >>>
+
+The null string in ``sys.path`` represents the current working directory.
+
+The expected convention for locally installed packages is to put them in the
+:file:`{...}/site-packages/` directory, but you may want to install Python
+modules into some arbitrary directory. For example, your site may have a
+convention of keeping all software related to the web server under :file:`/www`.
+Add-on Python modules might then belong in :file:`/www/python`, and in order to
+import them, this directory must be added to ``sys.path``. There are several
+different ways to add the directory.
+
+The most convenient way is to add a path configuration file to a directory
+that's already on Python's path, usually to the :file:`.../site-packages/`
+directory. Path configuration files have an extension of :file:`.pth`, and each
+line must contain a single path that will be appended to ``sys.path``. (Because
+the new paths are appended to ``sys.path``, modules in the added directories
+will not override standard modules. This means you can't use this mechanism for
+installing fixed versions of standard modules.)
+
+Paths can be absolute or relative, in which case they're relative to the
+directory containing the :file:`.pth` file. See the documentation of
+the :mod:`site` module for more information.
+
+A slightly less convenient way is to edit the :file:`site.py` file in Python's
+standard library, and modify ``sys.path``. :file:`site.py` is automatically
+imported when the Python interpreter is executed, unless the :option:`-S` switch
+is supplied to suppress this behaviour. So you could simply edit
+:file:`site.py` and add two lines to it::
+
+ import sys
+ sys.path.append('/www/python/')
+
+However, if you reinstall the same major version of Python (perhaps when
+upgrading from 2.2 to 2.2.2, for example) :file:`site.py` will be overwritten by
+the stock version. You'd have to remember that it was modified and save a copy
+before doing the installation.
+
+There are two environment variables that can modify ``sys.path``.
+:envvar:`PYTHONHOME` sets an alternate value for the prefix of the Python
+installation. For example, if :envvar:`PYTHONHOME` is set to ``/www/python``,
+the search path will be set to ``['', '/www/python/lib/pythonX.Y/',
+'/www/python/lib/pythonX.Y/plat-linux2', ...]``.
+
+The :envvar:`PYTHONPATH` variable can be set to a list of paths that will be
+added to the beginning of ``sys.path``. For example, if :envvar:`PYTHONPATH` is
+set to ``/www/python:/opt/py``, the search path will begin with
+``['/www/python', '/opt/py']``. (Note that directories must exist in order to
+be added to ``sys.path``; the :mod:`site` module removes paths that don't
+exist.)
+
+Finally, ``sys.path`` is just a regular Python list, so any Python application
+can modify it by adding or removing entries.
+
+
+.. _inst-config-files:
+
+Distutils Configuration Files
+=============================
+
+As mentioned above, you can use Distutils configuration files to record personal
+or site preferences for any Distutils options. That is, any option to any
+command can be stored in one of two or three (depending on your platform)
+configuration files, which will be consulted before the command-line is parsed.
+This means that configuration files will override default values, and the
+command-line will in turn override configuration files. Furthermore, if
+multiple configuration files apply, values from "earlier" files are overridden
+by "later" files.
+
+
+.. _inst-config-filenames:
+
+Location and names of config files
+----------------------------------
+
+The names and locations of the configuration files vary slightly across
+platforms. On Unix and Mac OS X, the three configuration files (in the order
+they are processed) are:
+
++--------------+----------------------------------------------------------+-------+
+| Type of file | Location and filename | Notes |
++==============+==========================================================+=======+
+| system | :file:`{prefix}/lib/python{ver}/distutils/distutils.cfg` | \(1) |
++--------------+----------------------------------------------------------+-------+
+| personal | :file:`$HOME/.pydistutils.cfg` | \(2) |
++--------------+----------------------------------------------------------+-------+
+| local | :file:`setup.cfg` | \(3) |
++--------------+----------------------------------------------------------+-------+
+
+And on Windows, the configuration files are:
+
++--------------+-------------------------------------------------+-------+
+| Type of file | Location and filename | Notes |
++==============+=================================================+=======+
+| system | :file:`{prefix}\\Lib\\distutils\\distutils.cfg` | \(4) |
++--------------+-------------------------------------------------+-------+
+| personal | :file:`%HOME%\\pydistutils.cfg` | \(5) |
++--------------+-------------------------------------------------+-------+
+| local | :file:`setup.cfg` | \(3) |
++--------------+-------------------------------------------------+-------+
+
+On all platforms, the "personal" file can be temporarily disabled by
+passing the `--no-user-cfg` option.
+
+Notes:
+
+(1)
+ Strictly speaking, the system-wide configuration file lives in the directory
+ where the Distutils are installed; under Python 1.6 and later on Unix, this is
+ as shown. For Python 1.5.2, the Distutils will normally be installed to
+ :file:`{prefix}/lib/python1.5/site-packages/distutils`, so the system
+ configuration file should be put there under Python 1.5.2.
+
+(2)
+ On Unix, if the :envvar:`HOME` environment variable is not defined, the user's
+ home directory will be determined with the :func:`getpwuid` function from the
+ standard :mod:`pwd` module. This is done by the :func:`os.path.expanduser`
+ function used by Distutils.
+
+(3)
+ I.e., in the current directory (usually the location of the setup script).
+
+(4)
+ (See also note (1).) Under Python 1.6 and later, Python's default "installation
+ prefix" is :file:`C:\\Python`, so the system configuration file is normally
+ :file:`C:\\Python\\Lib\\distutils\\distutils.cfg`. Under Python 1.5.2, the
+ default prefix was :file:`C:\\Program Files\\Python`, and the Distutils were not
+ part of the standard library---so the system configuration file would be
+ :file:`C:\\Program Files\\Python\\distutils\\distutils.cfg` in a standard Python
+ 1.5.2 installation under Windows.
+
+(5)
+ On Windows, if the :envvar:`HOME` environment variable is not defined,
+ :envvar:`USERPROFILE` then :envvar:`HOMEDRIVE` and :envvar:`HOMEPATH` will
+ be tried. This is done by the :func:`os.path.expanduser` function used
+ by Distutils.
+
+
+.. _inst-config-syntax:
+
+Syntax of config files
+----------------------
+
+The Distutils configuration files all have the same syntax. The config files
+are grouped into sections. There is one section for each Distutils command,
+plus a ``global`` section for global options that affect every command. Each
+section consists of one option per line, specified as ``option=value``.
+
+For example, the following is a complete config file that just forces all
+commands to run quietly by default::
+
+ [global]
+ verbose=0
+
+If this is installed as the system config file, it will affect all processing of
+any Python module distribution by any user on the current system. If it is
+installed as your personal config file (on systems that support them), it will
+affect only module distributions processed by you. And if it is used as the
+:file:`setup.cfg` for a particular module distribution, it affects only that
+distribution.
+
+You could override the default "build base" directory and make the
+:command:`build\*` commands always forcibly rebuild all files with the
+following::
+
+ [build]
+ build-base=blib
+ force=1
+
+which corresponds to the command-line arguments ::
+
+ python setup.py build --build-base=blib --force
+
+except that including the :command:`build` command on the command-line means
+that command will be run. Including a particular command in config files has no
+such implication; it only means that if the command is run, the options in the
+config file will apply. (Or if other commands that derive values from it are
+run, they will use the values in the config file.)
+
+You can find out the complete list of options for any command using the
+:option:`--help` option, e.g.::
+
+ python setup.py build --help
+
+and you can find out the complete list of global options by using
+:option:`--help` without a command::
+
+ python setup.py --help
+
+See also the "Reference" section of the "Distributing Python Modules" manual.
+
+
+.. _inst-building-ext:
+
+Building Extensions: Tips and Tricks
+====================================
+
+Whenever possible, the Distutils try to use the configuration information made
+available by the Python interpreter used to run the :file:`setup.py` script.
+For example, the same compiler and linker flags used to compile Python will also
+be used for compiling extensions. Usually this will work well, but in
+complicated situations this might be inappropriate. This section discusses how
+to override the usual Distutils behaviour.
+
+
+.. _inst-tweak-flags:
+
+Tweaking compiler/linker flags
+------------------------------
+
+Compiling a Python extension written in C or C++ will sometimes require
+specifying custom flags for the compiler and linker in order to use a particular
+library or produce a special kind of object code. This is especially true if the
+extension hasn't been tested on your platform, or if you're trying to
+cross-compile Python.
+
+In the most general case, the extension author might have foreseen that
+compiling the extensions would be complicated, and provided a :file:`Setup` file
+for you to edit. This will likely only be done if the module distribution
+contains many separate extension modules, or if they often require elaborate
+sets of compiler flags in order to work.
+
+A :file:`Setup` file, if present, is parsed in order to get a list of extensions
+to build. Each line in a :file:`Setup` describes a single module. Lines have
+the following structure::
+
+ module ... [sourcefile ...] [cpparg ...] [library ...]
+
+
+Let's examine each of the fields in turn.
+
+* *module* is the name of the extension module to be built, and should be a
+ valid Python identifier. You can't just change this in order to rename a module
+ (edits to the source code would also be needed), so this should be left alone.
+
+* *sourcefile* is anything that's likely to be a source code file, at least
+ judging by the filename. Filenames ending in :file:`.c` are assumed to be
+ written in C, filenames ending in :file:`.C`, :file:`.cc`, and :file:`.c++` are
+ assumed to be C++, and filenames ending in :file:`.m` or :file:`.mm` are assumed
+ to be in Objective C.
+
+* *cpparg* is an argument for the C preprocessor, and is anything starting with
+ :option:`-I`, :option:`-D`, :option:`-U` or :option:`-C`.
+
+* *library* is anything ending in :file:`.a` or beginning with :option:`-l` or
+ :option:`-L`.
+
+If a particular platform requires a special library on your platform, you can
+add it by editing the :file:`Setup` file and running ``python setup.py build``.
+For example, if the module defined by the line ::
+
+ foo foomodule.c
+
+must be linked with the math library :file:`libm.a` on your platform, simply add
+:option:`-lm` to the line::
+
+ foo foomodule.c -lm
+
+Arbitrary switches intended for the compiler or the linker can be supplied with
+the :option:`-Xcompiler` *arg* and :option:`-Xlinker` *arg* options::
+
+ foo foomodule.c -Xcompiler -o32 -Xlinker -shared -lm
+
+The next option after :option:`-Xcompiler` and :option:`-Xlinker` will be
+appended to the proper command line, so in the above example the compiler will
+be passed the :option:`-o32` option, and the linker will be passed
+:option:`-shared`. If a compiler option requires an argument, you'll have to
+supply multiple :option:`-Xcompiler` options; for example, to pass ``-x c++``
+the :file:`Setup` file would have to contain ``-Xcompiler -x -Xcompiler c++``.
+
+Compiler flags can also be supplied through setting the :envvar:`CFLAGS`
+environment variable. If set, the contents of :envvar:`CFLAGS` will be added to
+the compiler flags specified in the :file:`Setup` file.
+
+
+.. _inst-non-ms-compilers:
+
+Using non-Microsoft compilers on Windows
+----------------------------------------
+
+.. sectionauthor:: Rene Liebscher <R.Liebscher@gmx.de>
+
+
+
+Borland/CodeGear C++
+^^^^^^^^^^^^^^^^^^^^
+
+This subsection describes the necessary steps to use Distutils with the Borland
+C++ compiler version 5.5. First you have to know that Borland's object file
+format (OMF) is different from the format used by the Python version you can
+download from the Python or ActiveState Web site. (Python is built with
+Microsoft Visual C++, which uses COFF as the object file format.) For this
+reason you have to convert Python's library :file:`python25.lib` into the
+Borland format. You can do this as follows:
+
+.. Should we mention that users have to create cfg-files for the compiler?
+.. see also http://community.borland.com/article/0,1410,21205,00.html
+
+::
+
+ coff2omf python25.lib python25_bcpp.lib
+
+The :file:`coff2omf` program comes with the Borland compiler. The file
+:file:`python25.lib` is in the :file:`Libs` directory of your Python
+installation. If your extension uses other libraries (zlib, ...) you have to
+convert them too.
+
+The converted files have to reside in the same directories as the normal
+libraries.
+
+How does Distutils manage to use these libraries with their changed names? If
+the extension needs a library (eg. :file:`foo`) Distutils checks first if it
+finds a library with suffix :file:`_bcpp` (eg. :file:`foo_bcpp.lib`) and then
+uses this library. In the case it doesn't find such a special library it uses
+the default name (:file:`foo.lib`.) [#]_
+
+To let Distutils compile your extension with Borland C++ you now have to type::
+
+ python setup.py build --compiler=bcpp
+
+If you want to use the Borland C++ compiler as the default, you could specify
+this in your personal or system-wide configuration file for Distutils (see
+section :ref:`inst-config-files`.)
+
+
+.. seealso::
+
+ `C++Builder Compiler <http://www.codegear.com/downloads/free/cppbuilder>`_
+ Information about the free C++ compiler from Borland, including links to the
+ download pages.
+
+ `Creating Python Extensions Using Borland's Free Compiler <http://www.cyberus.ca/~g_will/pyExtenDL.shtml>`_
+ Document describing how to use Borland's free command-line C++ compiler to build
+ Python.
+
+
+GNU C / Cygwin / MinGW
+^^^^^^^^^^^^^^^^^^^^^^
+
+This section describes the necessary steps to use Distutils with the GNU C/C++
+compilers in their Cygwin and MinGW distributions. [#]_ For a Python interpreter
+that was built with Cygwin, everything should work without any of these
+following steps.
+
+Not all extensions can be built with MinGW or Cygwin, but many can. Extensions
+most likely to not work are those that use C++ or depend on Microsoft Visual C
+extensions.
+
+To let Distutils compile your extension with Cygwin you have to type::
+
+ python setup.py build --compiler=cygwin
+
+and for Cygwin in no-cygwin mode [#]_ or for MinGW type::
+
+ python setup.py build --compiler=mingw32
+
+If you want to use any of these options/compilers as default, you should
+consider writing it in your personal or system-wide configuration file for
+Distutils (see section :ref:`inst-config-files`.)
+
+Older Versions of Python and MinGW
+""""""""""""""""""""""""""""""""""
+The following instructions only apply if you're using a version of Python
+inferior to 2.4.1 with a MinGW inferior to 3.0.0 (with
+binutils-2.13.90-20030111-1).
+
+These compilers require some special libraries. This task is more complex than
+for Borland's C++, because there is no program to convert the library. First
+you have to create a list of symbols which the Python DLL exports. (You can find
+a good program for this task at
+http://www.emmestech.com/software/pexports-0.43/download_pexports.html).
+
+.. I don't understand what the next line means. --amk
+.. (inclusive the references on data structures.)
+
+::
+
+ pexports python25.dll >python25.def
+
+The location of an installed :file:`python25.dll` will depend on the
+installation options and the version and language of Windows. In a "just for
+me" installation, it will appear in the root of the installation directory. In
+a shared installation, it will be located in the system directory.
+
+Then you can create from these information an import library for gcc. ::
+
+ /cygwin/bin/dlltool --dllname python25.dll --def python25.def --output-lib libpython25.a
+
+The resulting library has to be placed in the same directory as
+:file:`python25.lib`. (Should be the :file:`libs` directory under your Python
+installation directory.)
+
+If your extension uses other libraries (zlib,...) you might have to convert
+them too. The converted files have to reside in the same directories as the
+normal libraries do.
+
+
+.. seealso::
+
+ `Building Python modules on MS Windows platform with MinGW <http://www.zope.org/Members/als/tips/win32_mingw_modules>`_
+ Information about building the required libraries for the MinGW environment.
+
+
+.. rubric:: Footnotes
+
+.. [#] This also means you could replace all existing COFF-libraries with OMF-libraries
+ of the same name.
+
+.. [#] Check http://sources.redhat.com/cygwin/ and http://www.mingw.org/ for more
+ information
+
+.. [#] Then you have no POSIX emulation available, but you also don't need
+ :file:`cygwin1.dll`.
diff --git a/Doc/documenting/style.rst b/Doc/documenting/style.rst
index 2548cb0..71a52f2 100644
--- a/Doc/documenting/style.rst
+++ b/Doc/documenting/style.rst
@@ -14,9 +14,10 @@ document.
Use of whitespace
-----------------
-All reST files use an indentation of 3 spaces. The maximum line length is 80
-characters for normal text, but tables, deeply indented code samples and long
-links may extend beyond that.
+All reST files use an indentation of 3 spaces; no tabs are allowed. The
+maximum line length is 80 characters for normal text, but tables, deeply
+indented code samples and long links may extend beyond that. Code example
+bodies should use normal Python 4-space indentation.
Make generous use of blank lines where applicable; they help grouping things
together.
diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst
index 2ba01bc..6a65941 100644
--- a/Doc/extending/newtypes.rst
+++ b/Doc/extending/newtypes.rst
@@ -288,18 +288,16 @@ strings, so we provide a new method::
self = (Noddy *)type->tp_alloc(type, 0);
if (self != NULL) {
self->first = PyString_FromString("");
- if (self->first == NULL)
- {
+ if (self->first == NULL) {
Py_DECREF(self);
return NULL;
- }
+ }
self->last = PyString_FromString("");
- if (self->last == NULL)
- {
+ if (self->last == NULL) {
Py_DECREF(self);
return NULL;
- }
+ }
self->number = 0;
}
diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst
index 02417b1..1f3135a 100644
--- a/Doc/faq/design.rst
+++ b/Doc/faq/design.rst
@@ -667,7 +667,7 @@ construction of large programs.
Python 2.6 adds an :mod:`abc` module that lets you define Abstract Base Classes
(ABCs). You can then use :func:`isinstance` and :func:`issubclass` to check
whether an instance or a class implements a particular ABC. The
-:mod:`collections` module defines a set of useful ABCs such as
+:mod:`collections.abc` module defines a set of useful ABCs such as
:class:`Iterable`, :class:`Container`, and :class:`MutableMapping`.
For Python, many of the advantages of interface specifications can be obtained
diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst
index fa8e6e7..bbc322a 100644
--- a/Doc/faq/extending.rst
+++ b/Doc/faq/extending.rst
@@ -447,34 +447,3 @@ In Python 2.2, you can inherit from built-in classes such as :class:`int`,
The Boost Python Library (BPL, http://www.boost.org/libs/python/doc/index.html)
provides a way of doing this from C++ (i.e. you can inherit from an extension
class written in C++ using the BPL).
-
-
-When importing module X, why do I get "undefined symbol: PyUnicodeUCS2*"?
--------------------------------------------------------------------------
-
-You are using a version of Python that uses a 4-byte representation for Unicode
-characters, but some C extension module you are importing was compiled using a
-Python that uses a 2-byte representation for Unicode characters (the default).
-
-If instead the name of the undefined symbol starts with ``PyUnicodeUCS4``, the
-problem is the reverse: Python was built using 2-byte Unicode characters, and
-the extension module was compiled using a Python with 4-byte Unicode characters.
-
-This can easily occur when using pre-built extension packages. RedHat Linux
-7.x, in particular, provided a "python2" binary that is compiled with 4-byte
-Unicode. This only causes the link failure if the extension uses any of the
-``PyUnicode_*()`` functions. It is also a problem if an extension uses any of
-the Unicode-related format specifiers for :c:func:`Py_BuildValue` (or similar) or
-parameter specifications for :c:func:`PyArg_ParseTuple`.
-
-You can check the size of the Unicode character a Python interpreter is using by
-checking the value of sys.maxunicode:
-
- >>> import sys
- >>> if sys.maxunicode > 65535:
- ... print('UCS4 build')
- ... else:
- ... print('UCS2 build')
-
-The only way to solve this problem is to use extension modules compiled with a
-Python binary built using the same size for Unicode characters.
diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index 3b211ae..4f1795d 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -34,7 +34,7 @@ Glossary
subclasses, which are classes that don't inherit from a class but are
still recognized by :func:`isinstance` and :func:`issubclass`; see the
:mod:`abc` module documentation. Python comes with many built-in ABCs for
- data structures (in the :mod:`collections` module), numbers (in the
+ data structures (in the :mod:`collections.abc` module), numbers (in the
:mod:`numbers` module), streams (in the :mod:`io` module), import finders
and loaders (in the :mod:`importlib.abc` module). You can create your own
ABCs with the :mod:`abc` module.
@@ -434,8 +434,8 @@ Glossary
mapping
A container object that supports arbitrary key lookups and implements the
- methods specified in the :class:`~collections.Mapping` or
- :class:`~collections.MutableMapping`
+ methods specified in the :class:`~collections.abc.Mapping` or
+ :class:`~collections.abc.MutableMapping`
:ref:`abstract base classes <collections-abstract-base-classes>`. Examples
include :class:`dict`, :class:`collections.defaultdict`,
:class:`collections.OrderedDict` and :class:`collections.Counter`.
@@ -586,6 +586,14 @@ Glossary
an :term:`expression` or a one of several constructs with a keyword, such
as :keyword:`if`, :keyword:`while` or :keyword:`for`.
+ struct sequence
+ A tuple with named elements. Struct sequences expose an interface similiar
+ to :term:`named tuple` in that elements can either be accessed either by
+ index or as an attribute. However, they do not have any of the named tuple
+ methods like :meth:`~collections.somenamedtuple._make` or
+ :meth:`~collections.somenamedtuple._asdict`. Examples of struct sequences
+ include :data:`sys.float_info` and the return value of :func:`os.stat`.
+
triple-quoted string
A string which is bound by three instances of either a quotation mark
(") or an apostrophe ('). While they don't provide any functionality
diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst
index 04e9b98..324ea0a 100644
--- a/Doc/howto/sockets.rst
+++ b/Doc/howto/sockets.rst
@@ -23,8 +23,8 @@ It's not really a tutorial - you'll still have work to do in getting things
working. It doesn't cover the fine points (and there are a lot of them), but I
hope it will give you enough background to begin using them decently.
-I'm only going to talk about INET sockets, but they account for at least 99% of
-the sockets in use. And I'll only talk about STREAM sockets - unless you really
+I'm only going to talk about INET (i.e. IPv4) sockets, but they account for at least 99% of
+the sockets in use. And I'll only talk about STREAM (i.e. TCP) sockets - unless you really
know what you're doing (in which case this HOWTO isn't for you!), you'll get
better behavior and performance from a STREAM socket than anything else. I will
try to clear up the mystery of what a socket is, as well as some hints on how to
@@ -208,10 +208,10 @@ length message::
totalsent = totalsent + sent
def myreceive(self):
- msg = ''
+ msg = b''
while len(msg) < MSGLEN:
chunk = self.sock.recv(MSGLEN-len(msg))
- if chunk == '':
+ if chunk == b'':
raise RuntimeError("socket connection broken")
msg = msg + chunk
return msg
@@ -371,12 +371,6 @@ have created a new socket to ``connect`` to someone else, put it in the
potential_writers list. If it shows up in the writable list, you have a decent
chance that it has connected.
-One very nasty problem with ``select``: if somewhere in those input lists of
-sockets is one which has died a nasty death, the ``select`` will fail. You then
-need to loop through every single damn socket in all those lists and do a
-``select([sock],[],[],0)`` until you find the bad one. That timeout of 0 means
-it won't take long, but it's ugly.
-
Actually, ``select`` can be handy even with blocking sockets. It's one way of
determining whether you will block - the socket returns as readable when there's
something in the buffers. However, this still doesn't help with the problem of
@@ -386,32 +380,6 @@ determining whether the other end is done, or just busy with something else.
files. Don't try this on Windows. On Windows, ``select`` works with sockets
only. Also note that in C, many of the more advanced socket options are done
differently on Windows. In fact, on Windows I usually use threads (which work
-very, very well) with my sockets. Face it, if you want any kind of performance,
-your code will look very different on Windows than on Unix.
-
-
-Performance
------------
+very, very well) with my sockets.
-There's no question that the fastest sockets code uses non-blocking sockets and
-select to multiplex them. You can put together something that will saturate a
-LAN connection without putting any strain on the CPU. The trouble is that an app
-written this way can't do much of anything else - it needs to be ready to
-shuffle bytes around at all times.
-
-Assuming that your app is actually supposed to do something more than that,
-threading is the optimal solution, (and using non-blocking sockets will be
-faster than using blocking sockets). Unfortunately, threading support in Unixes
-varies both in API and quality. So the normal Unix solution is to fork a
-subprocess to deal with each connection. The overhead for this is significant
-(and don't do this on Windows - the overhead of process creation is enormous
-there). It also means that unless each subprocess is completely independent,
-you'll need to use another form of IPC, say a pipe, or shared memory and
-semaphores, to communicate between the parent and child processes.
-
-Finally, remember that even though blocking sockets are somewhat slower than
-non-blocking, in many cases they are the "right" solution. After all, if your
-app is driven by the data it receives over a socket, there's not much sense in
-complicating the logic just so your app can wait on ``select`` instead of
-``recv``.
diff --git a/Doc/includes/noddy2.c b/Doc/includes/noddy2.c
index 9b8eafb..c02d49b 100644
--- a/Doc/includes/noddy2.c
+++ b/Doc/includes/noddy2.c
@@ -24,18 +24,16 @@ Noddy_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
self = (Noddy *)type->tp_alloc(type, 0);
if (self != NULL) {
self->first = PyUnicode_FromString("");
- if (self->first == NULL)
- {
+ if (self->first == NULL) {
Py_DECREF(self);
return NULL;
- }
-
+ }
+
self->last = PyUnicode_FromString("");
- if (self->last == NULL)
- {
+ if (self->last == NULL) {
Py_DECREF(self);
return NULL;
- }
+ }
self->number = 0;
}
@@ -50,10 +48,10 @@ Noddy_init(Noddy *self, PyObject *args, PyObject *kwds)
static char *kwlist[] = {"first", "last", "number", NULL};
- if (! PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist,
- &first, &last,
+ if (! PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist,
+ &first, &last,
&self->number))
- return -1;
+ return -1;
if (first) {
tmp = self->first;
@@ -111,7 +109,7 @@ Noddy_name(Noddy* self)
result = PyUnicode_Format(format, args);
Py_DECREF(args);
-
+
return result;
}
@@ -145,12 +143,12 @@ static PyTypeObject NoddyType = {
Py_TPFLAGS_DEFAULT |
Py_TPFLAGS_BASETYPE, /* tp_flags */
"Noddy objects", /* tp_doc */
- 0, /* tp_traverse */
- 0, /* tp_clear */
- 0, /* tp_richcompare */
- 0, /* tp_weaklistoffset */
- 0, /* tp_iter */
- 0, /* tp_iternext */
+ 0, /* tp_traverse */
+ 0, /* tp_clear */
+ 0, /* tp_richcompare */
+ 0, /* tp_weaklistoffset */
+ 0, /* tp_iter */
+ 0, /* tp_iternext */
Noddy_methods, /* tp_methods */
Noddy_members, /* tp_members */
0, /* tp_getset */
@@ -173,7 +171,7 @@ static PyModuleDef noddy2module = {
};
PyMODINIT_FUNC
-PyInit_noddy2(void)
+PyInit_noddy2(void)
{
PyObject* m;
diff --git a/Doc/includes/noddy3.c b/Doc/includes/noddy3.c
index 89f3a77..002a0e1 100644
--- a/Doc/includes/noddy3.c
+++ b/Doc/includes/noddy3.c
@@ -24,18 +24,16 @@ Noddy_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
self = (Noddy *)type->tp_alloc(type, 0);
if (self != NULL) {
self->first = PyUnicode_FromString("");
- if (self->first == NULL)
- {
+ if (self->first == NULL) {
Py_DECREF(self);
return NULL;
- }
-
+ }
+
self->last = PyUnicode_FromString("");
- if (self->last == NULL)
- {
+ if (self->last == NULL) {
Py_DECREF(self);
return NULL;
- }
+ }
self->number = 0;
}
@@ -50,10 +48,10 @@ Noddy_init(Noddy *self, PyObject *args, PyObject *kwds)
static char *kwlist[] = {"first", "last", "number", NULL};
- if (! PyArg_ParseTupleAndKeywords(args, kwds, "|SSi", kwlist,
- &first, &last,
+ if (! PyArg_ParseTupleAndKeywords(args, kwds, "|SSi", kwlist,
+ &first, &last,
&self->number))
- return -1;
+ return -1;
if (first) {
tmp = self->first;
@@ -88,22 +86,22 @@ Noddy_getfirst(Noddy *self, void *closure)
static int
Noddy_setfirst(Noddy *self, PyObject *value, void *closure)
{
- if (value == NULL) {
- PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute");
- return -1;
- }
-
- if (! PyUnicode_Check(value)) {
- PyErr_SetString(PyExc_TypeError,
- "The first attribute value must be a string");
- return -1;
- }
-
- Py_DECREF(self->first);
- Py_INCREF(value);
- self->first = value;
-
- return 0;
+ if (value == NULL) {
+ PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute");
+ return -1;
+ }
+
+ if (! PyUnicode_Check(value)) {
+ PyErr_SetString(PyExc_TypeError,
+ "The first attribute value must be a string");
+ return -1;
+ }
+
+ Py_DECREF(self->first);
+ Py_INCREF(value);
+ self->first = value;
+
+ return 0;
}
static PyObject *
@@ -116,30 +114,30 @@ Noddy_getlast(Noddy *self, void *closure)
static int
Noddy_setlast(Noddy *self, PyObject *value, void *closure)
{
- if (value == NULL) {
- PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute");
- return -1;
- }
-
- if (! PyUnicode_Check(value)) {
- PyErr_SetString(PyExc_TypeError,
- "The last attribute value must be a string");
- return -1;
- }
-
- Py_DECREF(self->last);
- Py_INCREF(value);
- self->last = value;
-
- return 0;
+ if (value == NULL) {
+ PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute");
+ return -1;
+ }
+
+ if (! PyUnicode_Check(value)) {
+ PyErr_SetString(PyExc_TypeError,
+ "The last attribute value must be a string");
+ return -1;
+ }
+
+ Py_DECREF(self->last);
+ Py_INCREF(value);
+ self->last = value;
+
+ return 0;
}
static PyGetSetDef Noddy_getseters[] = {
- {"first",
+ {"first",
(getter)Noddy_getfirst, (setter)Noddy_setfirst,
"first name",
NULL},
- {"last",
+ {"last",
(getter)Noddy_getlast, (setter)Noddy_setlast,
"last name",
NULL},
@@ -164,7 +162,7 @@ Noddy_name(Noddy* self)
result = PyUnicode_Format(format, args);
Py_DECREF(args);
-
+
return result;
}
@@ -198,12 +196,12 @@ static PyTypeObject NoddyType = {
Py_TPFLAGS_DEFAULT |
Py_TPFLAGS_BASETYPE, /* tp_flags */
"Noddy objects", /* tp_doc */
- 0, /* tp_traverse */
- 0, /* tp_clear */
- 0, /* tp_richcompare */
- 0, /* tp_weaklistoffset */
- 0, /* tp_iter */
- 0, /* tp_iternext */
+ 0, /* tp_traverse */
+ 0, /* tp_clear */
+ 0, /* tp_richcompare */
+ 0, /* tp_weaklistoffset */
+ 0, /* tp_iter */
+ 0, /* tp_iternext */
Noddy_methods, /* tp_methods */
Noddy_members, /* tp_members */
Noddy_getseters, /* tp_getset */
@@ -226,7 +224,7 @@ static PyModuleDef noddy3module = {
};
PyMODINIT_FUNC
-PyInit_noddy3(void)
+PyInit_noddy3(void)
{
PyObject* m;
diff --git a/Doc/includes/noddy4.c b/Doc/includes/noddy4.c
index 6a96fac..c747682 100644
--- a/Doc/includes/noddy4.c
+++ b/Doc/includes/noddy4.c
@@ -27,7 +27,7 @@ Noddy_traverse(Noddy *self, visitproc visit, void *arg)
return 0;
}
-static int
+static int
Noddy_clear(Noddy *self)
{
PyObject *tmp;
@@ -58,18 +58,16 @@ Noddy_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
self = (Noddy *)type->tp_alloc(type, 0);
if (self != NULL) {
self->first = PyUnicode_FromString("");
- if (self->first == NULL)
- {
+ if (self->first == NULL) {
Py_DECREF(self);
return NULL;
- }
-
+ }
+
self->last = PyUnicode_FromString("");
- if (self->last == NULL)
- {
+ if (self->last == NULL) {
Py_DECREF(self);
return NULL;
- }
+ }
self->number = 0;
}
@@ -84,10 +82,10 @@ Noddy_init(Noddy *self, PyObject *args, PyObject *kwds)
static char *kwlist[] = {"first", "last", "number", NULL};
- if (! PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist,
- &first, &last,
+ if (! PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist,
+ &first, &last,
&self->number))
- return -1;
+ return -1;
if (first) {
tmp = self->first;
@@ -145,7 +143,7 @@ Noddy_name(Noddy* self)
result = PyUnicode_Format(format, args);
Py_DECREF(args);
-
+
return result;
}
@@ -182,10 +180,10 @@ static PyTypeObject NoddyType = {
"Noddy objects", /* tp_doc */
(traverseproc)Noddy_traverse, /* tp_traverse */
(inquiry)Noddy_clear, /* tp_clear */
- 0, /* tp_richcompare */
- 0, /* tp_weaklistoffset */
- 0, /* tp_iter */
- 0, /* tp_iternext */
+ 0, /* tp_richcompare */
+ 0, /* tp_weaklistoffset */
+ 0, /* tp_iter */
+ 0, /* tp_iternext */
Noddy_methods, /* tp_methods */
Noddy_members, /* tp_members */
0, /* tp_getset */
@@ -208,7 +206,7 @@ static PyModuleDef noddy4module = {
};
PyMODINIT_FUNC
-PyInit_noddy4(void)
+PyInit_noddy4(void)
{
PyObject* m;
diff --git a/Doc/install/index.rst b/Doc/install/index.rst
index b20f1fb..bb2e9c5 100644
--- a/Doc/install/index.rst
+++ b/Doc/install/index.rst
@@ -1,12 +1,10 @@
-.. highlightlang:: none
+.. _packaging-install-index:
-.. _install-index:
+******************************
+ Installing Python Projects
+******************************
-*****************************
- Installing Python Modules
-*****************************
-
-:Author: Greg Ward
+:Author: The Fellowship of the Packaging
:Release: |version|
:Date: |today|
@@ -16,1071 +14,43 @@
about Python and aren't about to learn the language just in order to
install and maintain it for their users, i.e. system administrators.
Thus, I have to be sure to explain the basics at some point:
- sys.path and PYTHONPATH at least. Should probably give pointers to
+ sys.path and PYTHONPATH at least. Should probably give pointers to
other docs on "import site", PYTHONSTARTUP, PYTHONHOME, etc.
Finally, it might be useful to include all the material from my "Care
- and Feeding of a Python Installation" talk in here somewhere. Yow!
+ and Feeding of a Python Installation" talk in here somewhere. Yow!
.. topic:: Abstract
- This document describes the Python Distribution Utilities ("Distutils") from the
- end-user's point-of-view, describing how to extend the capabilities of a
- standard Python installation by building and installing third-party Python
- modules and extensions.
-
-
-.. _inst-intro:
-
-Introduction
-============
-
-Although Python's extensive standard library covers many programming needs,
-there often comes a time when you need to add some new functionality to your
-Python installation in the form of third-party modules. This might be necessary
-to support your own programming, or to support an application that you want to
-use and that happens to be written in Python.
-
-In the past, there has been little support for adding third-party modules to an
-existing Python installation. With the introduction of the Python Distribution
-Utilities (Distutils for short) in Python 2.0, this changed.
-
-This document is aimed primarily at the people who need to install third-party
-Python modules: end-users and system administrators who just need to get some
-Python application running, and existing Python programmers who want to add some
-new goodies to their toolbox. You don't need to know Python to read this
-document; there will be some brief forays into using Python's interactive mode
-to explore your installation, but that's it. If you're looking for information
-on how to distribute your own Python modules so that others may use them, see
-the :ref:`distutils-index` manual.
-
-
-.. _inst-trivial-install:
-
-Best case: trivial installation
--------------------------------
-
-In the best case, someone will have prepared a special version of the module
-distribution you want to install that is targeted specifically at your platform
-and is installed just like any other software on your platform. For example,
-the module developer might make an executable installer available for Windows
-users, an RPM package for users of RPM-based Linux systems (Red Hat, SuSE,
-Mandrake, and many others), a Debian package for users of Debian-based Linux
-systems, and so forth.
-
-In that case, you would download the installer appropriate to your platform and
-do the obvious thing with it: run it if it's an executable installer, ``rpm
---install`` it if it's an RPM, etc. You don't need to run Python or a setup
-script, you don't need to compile anything---you might not even need to read any
-instructions (although it's always a good idea to do so anyway).
-
-Of course, things will not always be that easy. You might be interested in a
-module distribution that doesn't have an easy-to-use installer for your
-platform. In that case, you'll have to start with the source distribution
-released by the module's author/maintainer. Installing from a source
-distribution is not too hard, as long as the modules are packaged in the
-standard way. The bulk of this document is about building and installing
-modules from standard source distributions.
-
-
-.. _inst-new-standard:
-
-The new standard: Distutils
----------------------------
-
-If you download a module source distribution, you can tell pretty quickly if it
-was packaged and distributed in the standard way, i.e. using the Distutils.
-First, the distribution's name and version number will be featured prominently
-in the name of the downloaded archive, e.g. :file:`foo-1.0.tar.gz` or
-:file:`widget-0.9.7.zip`. Next, the archive will unpack into a similarly-named
-directory: :file:`foo-1.0` or :file:`widget-0.9.7`. Additionally, the
-distribution will contain a setup script :file:`setup.py`, and a file named
-:file:`README.txt` or possibly just :file:`README`, which should explain that
-building and installing the module distribution is a simple matter of running
-one command from a terminal::
-
- python setup.py install
-
-For Windows, this command should be run from a command prompt window
-(:menuselection:`Start --> Accessories`)::
-
- setup.py install
-
-If all these things are true, then you already know how to build and install the
-modules you've just downloaded: Run the command above. Unless you need to
-install things in a non-standard way or customize the build process, you don't
-really need this manual. Or rather, the above command is everything you need to
-get out of this manual.
-
-
-.. _inst-standard-install:
-
-Standard Build and Install
-==========================
-
-As described in section :ref:`inst-new-standard`, building and installing a module
-distribution using the Distutils is usually one simple command to run from a
-terminal::
-
- python setup.py install
-
-
-.. _inst-platform-variations:
-
-Platform variations
--------------------
-
-You should always run the setup command from the distribution root directory,
-i.e. the top-level subdirectory that the module source distribution unpacks
-into. For example, if you've just downloaded a module source distribution
-:file:`foo-1.0.tar.gz` onto a Unix system, the normal thing to do is::
-
- gunzip -c foo-1.0.tar.gz | tar xf - # unpacks into directory foo-1.0
- cd foo-1.0
- python setup.py install
-
-On Windows, you'd probably download :file:`foo-1.0.zip`. If you downloaded the
-archive file to :file:`C:\\Temp`, then it would unpack into
-:file:`C:\\Temp\\foo-1.0`; you can use either a archive manipulator with a
-graphical user interface (such as WinZip) or a command-line tool (such as
-:program:`unzip` or :program:`pkunzip`) to unpack the archive. Then, open a
-command prompt window and run::
-
- cd c:\Temp\foo-1.0
- python setup.py install
-
-
-.. _inst-splitting-up:
-
-Splitting the job up
---------------------
-
-Running ``setup.py install`` builds and installs all modules in one run. If you
-prefer to work incrementally---especially useful if you want to customize the
-build process, or if things are going wrong---you can use the setup script to do
-one thing at a time. This is particularly helpful when the build and install
-will be done by different users---for example, you might want to build a module
-distribution and hand it off to a system administrator for installation (or do
-it yourself, with super-user privileges).
-
-For example, you can build everything in one step, and then install everything
-in a second step, by invoking the setup script twice::
-
- python setup.py build
- python setup.py install
-
-If you do this, you will notice that running the :command:`install` command
-first runs the :command:`build` command, which---in this case---quickly notices
-that it has nothing to do, since everything in the :file:`build` directory is
-up-to-date.
-
-You may not need this ability to break things down often if all you do is
-install modules downloaded off the 'net, but it's very handy for more advanced
-tasks. If you get into distributing your own Python modules and extensions,
-you'll run lots of individual Distutils commands on their own.
-
-
-.. _inst-how-build-works:
-
-How building works
-------------------
-
-As implied above, the :command:`build` command is responsible for putting the
-files to install into a *build directory*. By default, this is :file:`build`
-under the distribution root; if you're excessively concerned with speed, or want
-to keep the source tree pristine, you can change the build directory with the
-:option:`--build-base` option. For example::
-
- python setup.py build --build-base=/tmp/pybuild/foo-1.0
-
-(Or you could do this permanently with a directive in your system or personal
-Distutils configuration file; see section :ref:`inst-config-files`.) Normally, this
-isn't necessary.
-
-The default layout for the build tree is as follows::
-
- --- build/ --- lib/
- or
- --- build/ --- lib.<plat>/
- temp.<plat>/
-
-where ``<plat>`` expands to a brief description of the current OS/hardware
-platform and Python version. The first form, with just a :file:`lib` directory,
-is used for "pure module distributions"---that is, module distributions that
-include only pure Python modules. If a module distribution contains any
-extensions (modules written in C/C++), then the second form, with two ``<plat>``
-directories, is used. In that case, the :file:`temp.{plat}` directory holds
-temporary files generated by the compile/link process that don't actually get
-installed. In either case, the :file:`lib` (or :file:`lib.{plat}`) directory
-contains all Python modules (pure Python and extensions) that will be installed.
-
-In the future, more directories will be added to handle Python scripts,
-documentation, binary executables, and whatever else is needed to handle the job
-of installing Python modules and applications.
-
-
-.. _inst-how-install-works:
-
-How installation works
-----------------------
-
-After the :command:`build` command runs (whether you run it explicitly, or the
-:command:`install` command does it for you), the work of the :command:`install`
-command is relatively simple: all it has to do is copy everything under
-:file:`build/lib` (or :file:`build/lib.{plat}`) to your chosen installation
-directory.
-
-If you don't choose an installation directory---i.e., if you just run ``setup.py
-install``\ ---then the :command:`install` command installs to the standard
-location for third-party Python modules. This location varies by platform and
-by how you built/installed Python itself. On Unix (and Mac OS X, which is also
-Unix-based), it also depends on whether the module distribution being installed
-is pure Python or contains extensions ("non-pure"):
-
-+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+
-| Platform | Standard installation location | Default value | Notes |
-+=================+=====================================================+==================================================+=======+
-| Unix (pure) | :file:`{prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) |
-+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+
-| Unix (non-pure) | :file:`{exec-prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) |
-+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+
-| Windows | :file:`{prefix}\\Lib\\site-packages` | :file:`C:\\Python{XY}\\Lib\\site-packages` | \(2) |
-+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+
-
-Notes:
-
-(1)
- Most Linux distributions include Python as a standard part of the system, so
- :file:`{prefix}` and :file:`{exec-prefix}` are usually both :file:`/usr` on
- Linux. If you build Python yourself on Linux (or any Unix-like system), the
- default :file:`{prefix}` and :file:`{exec-prefix}` are :file:`/usr/local`.
-
-(2)
- The default installation directory on Windows was :file:`C:\\Program
- Files\\Python` under Python 1.6a1, 1.5.2, and earlier.
-
-:file:`{prefix}` and :file:`{exec-prefix}` stand for the directories that Python
-is installed to, and where it finds its libraries at run-time. They are always
-the same under Windows, and very often the same under Unix and Mac OS X. You
-can find out what your Python installation uses for :file:`{prefix}` and
-:file:`{exec-prefix}` by running Python in interactive mode and typing a few
-simple commands. Under Unix, just type ``python`` at the shell prompt. Under
-Windows, choose :menuselection:`Start --> Programs --> Python X.Y -->
-Python (command line)`. Once the interpreter is started, you type Python code
-at the prompt. For example, on my Linux system, I type the three Python
-statements shown below, and get the output as shown, to find out my
-:file:`{prefix}` and :file:`{exec-prefix}`::
-
- Python 2.4 (#26, Aug 7 2004, 17:19:02)
- Type "help", "copyright", "credits" or "license" for more information.
- >>> import sys
- >>> sys.prefix
- '/usr'
- >>> sys.exec_prefix
- '/usr'
-
-A few other placeholders are used in this document: :file:`{X.Y}` stands for the
-version of Python, for example ``3.2``; :file:`{abiflags}` will be replaced by
-the value of :data:`sys.abiflags` or the empty string for platforms which don't
-define ABI flags; :file:`{distname}` will be replaced by the name of the module
-distribution being installed. Dots and capitalization are important in the
-paths; for example, a value that uses ``python3.2`` on UNIX will typically use
-``Python32`` on Windows.
-
-If you don't want to install modules to the standard location, or if you don't
-have permission to write there, then you need to read about alternate
-installations in section :ref:`inst-alt-install`. If you want to customize your
-installation directories more heavily, see section :ref:`inst-custom-install` on
-custom installations.
-
-
-.. _inst-alt-install:
-
-Alternate Installation
-======================
-
-Often, it is necessary or desirable to install modules to a location other than
-the standard location for third-party Python modules. For example, on a Unix
-system you might not have permission to write to the standard third-party module
-directory. Or you might wish to try out a module before making it a standard
-part of your local Python installation. This is especially true when upgrading
-a distribution already present: you want to make sure your existing base of
-scripts still works with the new version before actually upgrading.
-
-The Distutils :command:`install` command is designed to make installing module
-distributions to an alternate location simple and painless. The basic idea is
-that you supply a base directory for the installation, and the
-:command:`install` command picks a set of directories (called an *installation
-scheme*) under this base directory in which to install files. The details
-differ across platforms, so read whichever of the following sections applies to
-you.
-
-Note that the various alternate installation schemes are mutually exclusive: you
-can pass ``--user``, or ``--home``, or ``--prefix`` and ``--exec-prefix``, or
-``--install-base`` and ``--install-platbase``, but you can't mix from these
-groups.
-
-
-.. _inst-alt-install-user:
-
-Alternate installation: the user scheme
----------------------------------------
-
-This scheme is designed to be the most convenient solution for users that don't
-have write permission to the global site-packages directory or don't want to
-install into it. It is enabled with a simple option::
-
- python setup.py install --user
-
-Files will be installed into subdirectories of :data:`site.USER_BASE` (written
-as :file:`{userbase}` hereafter). This scheme installs pure Python modules and
-extension modules in the same location (also known as :data:`site.USER_SITE`).
-Here are the values for UNIX, including Mac OS X:
-
-=============== ===========================================================
-Type of file Installation directory
-=============== ===========================================================
-modules :file:`{userbase}/lib/python{X.Y}/site-packages`
-scripts :file:`{userbase}/bin`
-data :file:`{userbase}`
-C headers :file:`{userbase}/include/python{X.Y}{abiflags}/{distname}`
-=============== ===========================================================
-
-And here are the values used on Windows:
-
-=============== ===========================================================
-Type of file Installation directory
-=============== ===========================================================
-modules :file:`{userbase}\\Python{XY}\\site-packages`
-scripts :file:`{userbase}\\Scripts`
-data :file:`{userbase}`
-C headers :file:`{userbase}\\Python{XY}\\Include\\{distname}`
-=============== ===========================================================
-
-The advantage of using this scheme compared to the other ones described below is
-that the user site-packages directory is under normal conditions always included
-in :data:`sys.path` (see :mod:`site` for more information), which means that
-there is no additional step to perform after running the :file:`setup.py` script
-to finalize the installation.
-
-The :command:`build_ext` command also has a ``--user`` option to add
-:file:`{userbase}/include` to the compiler search path for header files and
-:file:`{userbase}/lib` to the compiler search path for libraries as well as to
-the runtime search path for shared C libraries (rpath).
-
-
-.. _inst-alt-install-home:
-
-Alternate installation: the home scheme
----------------------------------------
-
-The idea behind the "home scheme" is that you build and maintain a personal
-stash of Python modules. This scheme's name is derived from the idea of a
-"home" directory on Unix, since it's not unusual for a Unix user to make their
-home directory have a layout similar to :file:`/usr/` or :file:`/usr/local/`.
-This scheme can be used by anyone, regardless of the operating system they
-are installing for.
-
-Installing a new module distribution is as simple as ::
-
- python setup.py install --home=<dir>
-
-where you can supply any directory you like for the :option:`--home` option. On
-Unix, lazy typists can just type a tilde (``~``); the :command:`install` command
-will expand this to your home directory::
-
- python setup.py install --home=~
-
-To make Python find the distributions installed with this scheme, you may have
-to :ref:`modify Python's search path <inst-search-path>` or edit
-:mod:`sitecustomize` (see :mod:`site`) to call :func:`site.addsitedir` or edit
-:data:`sys.path`.
-
-The :option:`--home` option defines the installation base directory. Files are
-installed to the following directories under the installation base as follows:
-
-=============== ===========================================================
-Type of file Installation directory
-=============== ===========================================================
-modules :file:`{home}/lib/python`
-scripts :file:`{home}/bin`
-data :file:`{home}`
-C headers :file:`{home}/include/python/{distname}`
-=============== ===========================================================
-
-(Mentally replace slashes with backslashes if you're on Windows.)
-
-
-.. _inst-alt-install-prefix-unix:
-
-Alternate installation: Unix (the prefix scheme)
-------------------------------------------------
-
-The "prefix scheme" is useful when you wish to use one Python installation to
-perform the build/install (i.e., to run the setup script), but install modules
-into the third-party module directory of a different Python installation (or
-something that looks like a different Python installation). If this sounds a
-trifle unusual, it is---that's why the user and home schemes come before. However,
-there are at least two known cases where the prefix scheme will be useful.
-
-First, consider that many Linux distributions put Python in :file:`/usr`, rather
-than the more traditional :file:`/usr/local`. This is entirely appropriate,
-since in those cases Python is part of "the system" rather than a local add-on.
-However, if you are installing Python modules from source, you probably want
-them to go in :file:`/usr/local/lib/python2.{X}` rather than
-:file:`/usr/lib/python2.{X}`. This can be done with ::
-
- /usr/bin/python setup.py install --prefix=/usr/local
-
-Another possibility is a network filesystem where the name used to write to a
-remote directory is different from the name used to read it: for example, the
-Python interpreter accessed as :file:`/usr/local/bin/python` might search for
-modules in :file:`/usr/local/lib/python2.{X}`, but those modules would have to
-be installed to, say, :file:`/mnt/{@server}/export/lib/python2.{X}`. This could
-be done with ::
-
- /usr/local/bin/python setup.py install --prefix=/mnt/@server/export
-
-In either case, the :option:`--prefix` option defines the installation base, and
-the :option:`--exec-prefix` option defines the platform-specific installation
-base, which is used for platform-specific files. (Currently, this just means
-non-pure module distributions, but could be expanded to C libraries, binary
-executables, etc.) If :option:`--exec-prefix` is not supplied, it defaults to
-:option:`--prefix`. Files are installed as follows:
-
-================= ==========================================================
-Type of file Installation directory
-================= ==========================================================
-Python modules :file:`{prefix}/lib/python{X.Y}/site-packages`
-extension modules :file:`{exec-prefix}/lib/python{X.Y}/site-packages`
-scripts :file:`{prefix}/bin`
-data :file:`{prefix}`
-C headers :file:`{prefix}/include/python{X.Y}{abiflags}/{distname}`
-================= ==========================================================
-
-There is no requirement that :option:`--prefix` or :option:`--exec-prefix`
-actually point to an alternate Python installation; if the directories listed
-above do not already exist, they are created at installation time.
-
-Incidentally, the real reason the prefix scheme is important is simply that a
-standard Unix installation uses the prefix scheme, but with :option:`--prefix`
-and :option:`--exec-prefix` supplied by Python itself as ``sys.prefix`` and
-``sys.exec_prefix``. Thus, you might think you'll never use the prefix scheme,
-but every time you run ``python setup.py install`` without any other options,
-you're using it.
-
-Note that installing extensions to an alternate Python installation has no
-effect on how those extensions are built: in particular, the Python header files
-(:file:`Python.h` and friends) installed with the Python interpreter used to run
-the setup script will be used in compiling extensions. It is your
-responsibility to ensure that the interpreter used to run extensions installed
-in this way is compatible with the interpreter used to build them. The best way
-to do this is to ensure that the two interpreters are the same version of Python
-(possibly different builds, or possibly copies of the same build). (Of course,
-if your :option:`--prefix` and :option:`--exec-prefix` don't even point to an
-alternate Python installation, this is immaterial.)
-
-
-.. _inst-alt-install-prefix-windows:
-
-Alternate installation: Windows (the prefix scheme)
----------------------------------------------------
-
-Windows has no concept of a user's home directory, and since the standard Python
-installation under Windows is simpler than under Unix, the :option:`--prefix`
-option has traditionally been used to install additional packages in separate
-locations on Windows. ::
-
- python setup.py install --prefix="\Temp\Python"
-
-to install modules to the :file:`\\Temp\\Python` directory on the current drive.
-
-The installation base is defined by the :option:`--prefix` option; the
-:option:`--exec-prefix` option is not supported under Windows, which means that
-pure Python modules and extension modules are installed into the same location.
-Files are installed as follows:
-
-=============== ==========================================================
-Type of file Installation directory
-=============== ==========================================================
-modules :file:`{prefix}\\Lib\\site-packages`
-scripts :file:`{prefix}\\Scripts`
-data :file:`{prefix}`
-C headers :file:`{prefix}\\Include\\{distname}`
-=============== ==========================================================
-
-
-.. _inst-custom-install:
-
-Custom Installation
-===================
-
-Sometimes, the alternate installation schemes described in section
-:ref:`inst-alt-install` just don't do what you want. You might want to tweak just
-one or two directories while keeping everything under the same base directory,
-or you might want to completely redefine the installation scheme. In either
-case, you're creating a *custom installation scheme*.
-
-To create a custom installation scheme, you start with one of the alternate
-schemes and override some of the installation directories used for the various
-types of files, using these options:
-
-====================== =======================
-Type of file Override option
-====================== =======================
-Python modules ``--install-purelib``
-extension modules ``--install-platlib``
-all modules ``--install-lib``
-scripts ``--install-scripts``
-data ``--install-data``
-C headers ``--install-headers``
-====================== =======================
-
-These override options can be relative, absolute,
-or explicitly defined in terms of one of the installation base directories.
-(There are two installation base directories, and they are normally the same---
-they only differ when you use the Unix "prefix scheme" and supply different
-``--prefix`` and ``--exec-prefix`` options; using ``--install-lib`` will
-override values computed or given for ``--install-purelib`` and
-``--install-platlib``, and is recommended for schemes that don't make a
-difference between Python and extension modules.)
-
-For example, say you're installing a module distribution to your home directory
-under Unix---but you want scripts to go in :file:`~/scripts` rather than
-:file:`~/bin`. As you might expect, you can override this directory with the
-:option:`--install-scripts` option; in this case, it makes most sense to supply
-a relative path, which will be interpreted relative to the installation base
-directory (your home directory, in this case)::
-
- python setup.py install --home=~ --install-scripts=scripts
-
-Another Unix example: suppose your Python installation was built and installed
-with a prefix of :file:`/usr/local/python`, so under a standard installation
-scripts will wind up in :file:`/usr/local/python/bin`. If you want them in
-:file:`/usr/local/bin` instead, you would supply this absolute directory for the
-:option:`--install-scripts` option::
-
- python setup.py install --install-scripts=/usr/local/bin
-
-(This performs an installation using the "prefix scheme," where the prefix is
-whatever your Python interpreter was installed with--- :file:`/usr/local/python`
-in this case.)
-
-If you maintain Python on Windows, you might want third-party modules to live in
-a subdirectory of :file:`{prefix}`, rather than right in :file:`{prefix}`
-itself. This is almost as easy as customizing the script installation directory
----you just have to remember that there are two types of modules to worry about,
-Python and extension modules, which can conveniently be both controlled by one
-option::
-
- python setup.py install --install-lib=Site
-
-The specified installation directory is relative to :file:`{prefix}`. Of
-course, you also have to ensure that this directory is in Python's module
-search path, such as by putting a :file:`.pth` file in a site directory (see
-:mod:`site`). See section :ref:`inst-search-path` to find out how to modify
-Python's search path.
-
-If you want to define an entire installation scheme, you just have to supply all
-of the installation directory options. The recommended way to do this is to
-supply relative paths; for example, if you want to maintain all Python
-module-related files under :file:`python` in your home directory, and you want a
-separate directory for each platform that you use your home directory from, you
-might define the following installation scheme::
+ This document describes Packaging from the end-user's point of view: it
+ explains how to extend the functionality of a standard Python installation by
+ building and installing third-party Python modules and applications.
- python setup.py install --home=~ \
- --install-purelib=python/lib \
- --install-platlib=python/lib.$PLAT \
- --install-scripts=python/scripts
- --install-data=python/data
-or, equivalently, ::
+This guide is split into a simple overview followed by a longer presentation of
+the :program:`pysetup` script, the Python package management tool used to
+build, distribute, search for, install, remove and list Python distributions.
- python setup.py install --home=~/python \
- --install-purelib=lib \
- --install-platlib='lib.$PLAT' \
- --install-scripts=scripts
- --install-data=data
+.. TODO integrate install and pysetup instead of duplicating
-``$PLAT`` is not (necessarily) an environment variable---it will be expanded by
-the Distutils as it parses your command line options, just as it does when
-parsing your configuration file(s).
+.. toctree::
+ :maxdepth: 2
+ :numbered:
-Obviously, specifying the entire installation scheme every time you install a
-new module distribution would be very tedious. Thus, you can put these options
-into your Distutils config file (see section :ref:`inst-config-files`)::
-
- [install]
- install-base=$HOME
- install-purelib=python/lib
- install-platlib=python/lib.$PLAT
- install-scripts=python/scripts
- install-data=python/data
-
-or, equivalently, ::
-
- [install]
- install-base=$HOME/python
- install-purelib=lib
- install-platlib=lib.$PLAT
- install-scripts=scripts
- install-data=data
-
-Note that these two are *not* equivalent if you supply a different installation
-base directory when you run the setup script. For example, ::
-
- python setup.py install --install-base=/tmp
-
-would install pure modules to :file:`/tmp/python/lib` in the first case, and
-to :file:`/tmp/lib` in the second case. (For the second case, you probably
-want to supply an installation base of :file:`/tmp/python`.)
-
-You probably noticed the use of ``$HOME`` and ``$PLAT`` in the sample
-configuration file input. These are Distutils configuration variables, which
-bear a strong resemblance to environment variables. In fact, you can use
-environment variables in config files on platforms that have such a notion but
-the Distutils additionally define a few extra variables that may not be in your
-environment, such as ``$PLAT``. (And of course, on systems that don't have
-environment variables, such as Mac OS 9, the configuration variables supplied by
-the Distutils are the only ones you can use.) See section :ref:`inst-config-files`
-for details.
-
-.. XXX need some Windows examples---when would custom installation schemes be
- needed on those platforms?
-
-
-.. XXX Move this to Doc/using
-
-.. _inst-search-path:
-
-Modifying Python's Search Path
-------------------------------
-
-When the Python interpreter executes an :keyword:`import` statement, it searches
-for both Python code and extension modules along a search path. A default value
-for the path is configured into the Python binary when the interpreter is built.
-You can determine the path by importing the :mod:`sys` module and printing the
-value of ``sys.path``. ::
-
- $ python
- Python 2.2 (#11, Oct 3 2002, 13:31:27)
- [GCC 2.96 20000731 (Red Hat Linux 7.3 2.96-112)] on linux2
- Type "help", "copyright", "credits" or "license" for more information.
- >>> import sys
- >>> sys.path
- ['', '/usr/local/lib/python2.3', '/usr/local/lib/python2.3/plat-linux2',
- '/usr/local/lib/python2.3/lib-tk', '/usr/local/lib/python2.3/lib-dynload',
- '/usr/local/lib/python2.3/site-packages']
- >>>
-
-The null string in ``sys.path`` represents the current working directory.
-
-The expected convention for locally installed packages is to put them in the
-:file:`{...}/site-packages/` directory, but you may want to install Python
-modules into some arbitrary directory. For example, your site may have a
-convention of keeping all software related to the web server under :file:`/www`.
-Add-on Python modules might then belong in :file:`/www/python`, and in order to
-import them, this directory must be added to ``sys.path``. There are several
-different ways to add the directory.
-
-The most convenient way is to add a path configuration file to a directory
-that's already on Python's path, usually to the :file:`.../site-packages/`
-directory. Path configuration files have an extension of :file:`.pth`, and each
-line must contain a single path that will be appended to ``sys.path``. (Because
-the new paths are appended to ``sys.path``, modules in the added directories
-will not override standard modules. This means you can't use this mechanism for
-installing fixed versions of standard modules.)
-
-Paths can be absolute or relative, in which case they're relative to the
-directory containing the :file:`.pth` file. See the documentation of
-the :mod:`site` module for more information.
-
-A slightly less convenient way is to edit the :file:`site.py` file in Python's
-standard library, and modify ``sys.path``. :file:`site.py` is automatically
-imported when the Python interpreter is executed, unless the :option:`-S` switch
-is supplied to suppress this behaviour. So you could simply edit
-:file:`site.py` and add two lines to it::
-
- import sys
- sys.path.append('/www/python/')
-
-However, if you reinstall the same major version of Python (perhaps when
-upgrading from 2.2 to 2.2.2, for example) :file:`site.py` will be overwritten by
-the stock version. You'd have to remember that it was modified and save a copy
-before doing the installation.
-
-There are two environment variables that can modify ``sys.path``.
-:envvar:`PYTHONHOME` sets an alternate value for the prefix of the Python
-installation. For example, if :envvar:`PYTHONHOME` is set to ``/www/python``,
-the search path will be set to ``['', '/www/python/lib/pythonX.Y/',
-'/www/python/lib/pythonX.Y/plat-linux2', ...]``.
-
-The :envvar:`PYTHONPATH` variable can be set to a list of paths that will be
-added to the beginning of ``sys.path``. For example, if :envvar:`PYTHONPATH` is
-set to ``/www/python:/opt/py``, the search path will begin with
-``['/www/python', '/opt/py']``. (Note that directories must exist in order to
-be added to ``sys.path``; the :mod:`site` module removes paths that don't
-exist.)
-
-Finally, ``sys.path`` is just a regular Python list, so any Python application
-can modify it by adding or removing entries.
-
-
-.. _inst-config-files:
-
-Distutils Configuration Files
-=============================
-
-As mentioned above, you can use Distutils configuration files to record personal
-or site preferences for any Distutils options. That is, any option to any
-command can be stored in one of two or three (depending on your platform)
-configuration files, which will be consulted before the command-line is parsed.
-This means that configuration files will override default values, and the
-command-line will in turn override configuration files. Furthermore, if
-multiple configuration files apply, values from "earlier" files are overridden
-by "later" files.
-
-
-.. _inst-config-filenames:
-
-Location and names of config files
-----------------------------------
-
-The names and locations of the configuration files vary slightly across
-platforms. On Unix and Mac OS X, the three configuration files (in the order
-they are processed) are:
-
-+--------------+----------------------------------------------------------+-------+
-| Type of file | Location and filename | Notes |
-+==============+==========================================================+=======+
-| system | :file:`{prefix}/lib/python{ver}/distutils/distutils.cfg` | \(1) |
-+--------------+----------------------------------------------------------+-------+
-| personal | :file:`$HOME/.pydistutils.cfg` | \(2) |
-+--------------+----------------------------------------------------------+-------+
-| local | :file:`setup.cfg` | \(3) |
-+--------------+----------------------------------------------------------+-------+
-
-And on Windows, the configuration files are:
-
-+--------------+-------------------------------------------------+-------+
-| Type of file | Location and filename | Notes |
-+==============+=================================================+=======+
-| system | :file:`{prefix}\\Lib\\distutils\\distutils.cfg` | \(4) |
-+--------------+-------------------------------------------------+-------+
-| personal | :file:`%HOME%\\pydistutils.cfg` | \(5) |
-+--------------+-------------------------------------------------+-------+
-| local | :file:`setup.cfg` | \(3) |
-+--------------+-------------------------------------------------+-------+
-
-On all platforms, the "personal" file can be temporarily disabled by
-passing the `--no-user-cfg` option.
-
-Notes:
-
-(1)
- Strictly speaking, the system-wide configuration file lives in the directory
- where the Distutils are installed; under Python 1.6 and later on Unix, this is
- as shown. For Python 1.5.2, the Distutils will normally be installed to
- :file:`{prefix}/lib/python1.5/site-packages/distutils`, so the system
- configuration file should be put there under Python 1.5.2.
-
-(2)
- On Unix, if the :envvar:`HOME` environment variable is not defined, the user's
- home directory will be determined with the :func:`getpwuid` function from the
- standard :mod:`pwd` module. This is done by the :func:`os.path.expanduser`
- function used by Distutils.
-
-(3)
- I.e., in the current directory (usually the location of the setup script).
-
-(4)
- (See also note (1).) Under Python 1.6 and later, Python's default "installation
- prefix" is :file:`C:\\Python`, so the system configuration file is normally
- :file:`C:\\Python\\Lib\\distutils\\distutils.cfg`. Under Python 1.5.2, the
- default prefix was :file:`C:\\Program Files\\Python`, and the Distutils were not
- part of the standard library---so the system configuration file would be
- :file:`C:\\Program Files\\Python\\distutils\\distutils.cfg` in a standard Python
- 1.5.2 installation under Windows.
-
-(5)
- On Windows, if the :envvar:`HOME` environment variable is not defined,
- :envvar:`USERPROFILE` then :envvar:`HOMEDRIVE` and :envvar:`HOMEPATH` will
- be tried. This is done by the :func:`os.path.expanduser` function used
- by Distutils.
-
-
-.. _inst-config-syntax:
-
-Syntax of config files
-----------------------
-
-The Distutils configuration files all have the same syntax. The config files
-are grouped into sections. There is one section for each Distutils command,
-plus a ``global`` section for global options that affect every command. Each
-section consists of one option per line, specified as ``option=value``.
-
-For example, the following is a complete config file that just forces all
-commands to run quietly by default::
-
- [global]
- verbose=0
-
-If this is installed as the system config file, it will affect all processing of
-any Python module distribution by any user on the current system. If it is
-installed as your personal config file (on systems that support them), it will
-affect only module distributions processed by you. And if it is used as the
-:file:`setup.cfg` for a particular module distribution, it affects only that
-distribution.
-
-You could override the default "build base" directory and make the
-:command:`build\*` commands always forcibly rebuild all files with the
-following::
-
- [build]
- build-base=blib
- force=1
-
-which corresponds to the command-line arguments ::
-
- python setup.py build --build-base=blib --force
-
-except that including the :command:`build` command on the command-line means
-that command will be run. Including a particular command in config files has no
-such implication; it only means that if the command is run, the options in the
-config file will apply. (Or if other commands that derive values from it are
-run, they will use the values in the config file.)
-
-You can find out the complete list of options for any command using the
-:option:`--help` option, e.g.::
-
- python setup.py build --help
-
-and you can find out the complete list of global options by using
-:option:`--help` without a command::
-
- python setup.py --help
-
-See also the "Reference" section of the "Distributing Python Modules" manual.
-
-
-.. _inst-building-ext:
-
-Building Extensions: Tips and Tricks
-====================================
-
-Whenever possible, the Distutils try to use the configuration information made
-available by the Python interpreter used to run the :file:`setup.py` script.
-For example, the same compiler and linker flags used to compile Python will also
-be used for compiling extensions. Usually this will work well, but in
-complicated situations this might be inappropriate. This section discusses how
-to override the usual Distutils behaviour.
-
-
-.. _inst-tweak-flags:
-
-Tweaking compiler/linker flags
-------------------------------
-
-Compiling a Python extension written in C or C++ will sometimes require
-specifying custom flags for the compiler and linker in order to use a particular
-library or produce a special kind of object code. This is especially true if the
-extension hasn't been tested on your platform, or if you're trying to
-cross-compile Python.
-
-In the most general case, the extension author might have foreseen that
-compiling the extensions would be complicated, and provided a :file:`Setup` file
-for you to edit. This will likely only be done if the module distribution
-contains many separate extension modules, or if they often require elaborate
-sets of compiler flags in order to work.
-
-A :file:`Setup` file, if present, is parsed in order to get a list of extensions
-to build. Each line in a :file:`Setup` describes a single module. Lines have
-the following structure::
-
- module ... [sourcefile ...] [cpparg ...] [library ...]
-
-
-Let's examine each of the fields in turn.
-
-* *module* is the name of the extension module to be built, and should be a
- valid Python identifier. You can't just change this in order to rename a module
- (edits to the source code would also be needed), so this should be left alone.
-
-* *sourcefile* is anything that's likely to be a source code file, at least
- judging by the filename. Filenames ending in :file:`.c` are assumed to be
- written in C, filenames ending in :file:`.C`, :file:`.cc`, and :file:`.c++` are
- assumed to be C++, and filenames ending in :file:`.m` or :file:`.mm` are assumed
- to be in Objective C.
-
-* *cpparg* is an argument for the C preprocessor, and is anything starting with
- :option:`-I`, :option:`-D`, :option:`-U` or :option:`-C`.
-
-* *library* is anything ending in :file:`.a` or beginning with :option:`-l` or
- :option:`-L`.
-
-If a particular platform requires a special library on your platform, you can
-add it by editing the :file:`Setup` file and running ``python setup.py build``.
-For example, if the module defined by the line ::
-
- foo foomodule.c
-
-must be linked with the math library :file:`libm.a` on your platform, simply add
-:option:`-lm` to the line::
-
- foo foomodule.c -lm
-
-Arbitrary switches intended for the compiler or the linker can be supplied with
-the :option:`-Xcompiler` *arg* and :option:`-Xlinker` *arg* options::
-
- foo foomodule.c -Xcompiler -o32 -Xlinker -shared -lm
-
-The next option after :option:`-Xcompiler` and :option:`-Xlinker` will be
-appended to the proper command line, so in the above example the compiler will
-be passed the :option:`-o32` option, and the linker will be passed
-:option:`-shared`. If a compiler option requires an argument, you'll have to
-supply multiple :option:`-Xcompiler` options; for example, to pass ``-x c++``
-the :file:`Setup` file would have to contain ``-Xcompiler -x -Xcompiler c++``.
-
-Compiler flags can also be supplied through setting the :envvar:`CFLAGS`
-environment variable. If set, the contents of :envvar:`CFLAGS` will be added to
-the compiler flags specified in the :file:`Setup` file.
-
-
-.. _inst-non-ms-compilers:
-
-Using non-Microsoft compilers on Windows
-----------------------------------------
-
-.. sectionauthor:: Rene Liebscher <R.Liebscher@gmx.de>
-
-
-
-Borland/CodeGear C++
-^^^^^^^^^^^^^^^^^^^^
-
-This subsection describes the necessary steps to use Distutils with the Borland
-C++ compiler version 5.5. First you have to know that Borland's object file
-format (OMF) is different from the format used by the Python version you can
-download from the Python or ActiveState Web site. (Python is built with
-Microsoft Visual C++, which uses COFF as the object file format.) For this
-reason you have to convert Python's library :file:`python25.lib` into the
-Borland format. You can do this as follows:
-
-.. Should we mention that users have to create cfg-files for the compiler?
-.. see also http://community.borland.com/article/0,1410,21205,00.html
-
-::
-
- coff2omf python25.lib python25_bcpp.lib
-
-The :file:`coff2omf` program comes with the Borland compiler. The file
-:file:`python25.lib` is in the :file:`Libs` directory of your Python
-installation. If your extension uses other libraries (zlib, ...) you have to
-convert them too.
-
-The converted files have to reside in the same directories as the normal
-libraries.
-
-How does Distutils manage to use these libraries with their changed names? If
-the extension needs a library (eg. :file:`foo`) Distutils checks first if it
-finds a library with suffix :file:`_bcpp` (eg. :file:`foo_bcpp.lib`) and then
-uses this library. In the case it doesn't find such a special library it uses
-the default name (:file:`foo.lib`.) [#]_
-
-To let Distutils compile your extension with Borland C++ you now have to type::
-
- python setup.py build --compiler=bcpp
-
-If you want to use the Borland C++ compiler as the default, you could specify
-this in your personal or system-wide configuration file for Distutils (see
-section :ref:`inst-config-files`.)
-
-
-.. seealso::
-
- `C++Builder Compiler <http://www.codegear.com/downloads/free/cppbuilder>`_
- Information about the free C++ compiler from Borland, including links to the
- download pages.
-
- `Creating Python Extensions Using Borland's Free Compiler <http://www.cyberus.ca/~g_will/pyExtenDL.shtml>`_
- Document describing how to use Borland's free command-line C++ compiler to build
- Python.
-
-
-GNU C / Cygwin / MinGW
-^^^^^^^^^^^^^^^^^^^^^^
-
-This section describes the necessary steps to use Distutils with the GNU C/C++
-compilers in their Cygwin and MinGW distributions. [#]_ For a Python interpreter
-that was built with Cygwin, everything should work without any of these
-following steps.
-
-Not all extensions can be built with MinGW or Cygwin, but many can. Extensions
-most likely to not work are those that use C++ or depend on Microsoft Visual C
-extensions.
-
-To let Distutils compile your extension with Cygwin you have to type::
-
- python setup.py build --compiler=cygwin
-
-and for Cygwin in no-cygwin mode [#]_ or for MinGW type::
-
- python setup.py build --compiler=mingw32
-
-If you want to use any of these options/compilers as default, you should
-consider writing it in your personal or system-wide configuration file for
-Distutils (see section :ref:`inst-config-files`.)
-
-Older Versions of Python and MinGW
-""""""""""""""""""""""""""""""""""
-The following instructions only apply if you're using a version of Python
-inferior to 2.4.1 with a MinGW inferior to 3.0.0 (with
-binutils-2.13.90-20030111-1).
-
-These compilers require some special libraries. This task is more complex than
-for Borland's C++, because there is no program to convert the library. First
-you have to create a list of symbols which the Python DLL exports. (You can find
-a good program for this task at
-http://www.emmestech.com/software/pexports-0.43/download_pexports.html).
-
-.. I don't understand what the next line means. --amk
-.. (inclusive the references on data structures.)
-
-::
-
- pexports python25.dll >python25.def
-
-The location of an installed :file:`python25.dll` will depend on the
-installation options and the version and language of Windows. In a "just for
-me" installation, it will appear in the root of the installation directory. In
-a shared installation, it will be located in the system directory.
-
-Then you can create from these information an import library for gcc. ::
-
- /cygwin/bin/dlltool --dllname python25.dll --def python25.def --output-lib libpython25.a
-
-The resulting library has to be placed in the same directory as
-:file:`python25.lib`. (Should be the :file:`libs` directory under your Python
-installation directory.)
-
-If your extension uses other libraries (zlib,...) you might have to convert
-them too. The converted files have to reside in the same directories as the
-normal libraries do.
+ install
+ pysetup
+ pysetup-config
+ pysetup-servers
.. seealso::
- `Building Python modules on MS Windows platform with MinGW <http://www.zope.org/Members/als/tips/win32_mingw_modules>`_
- Information about building the required libraries for the MinGW environment.
-
-
-.. rubric:: Footnotes
-
-.. [#] This also means you could replace all existing COFF-libraries with OMF-libraries
- of the same name.
-
-.. [#] Check http://sources.redhat.com/cygwin/ and http://www.mingw.org/ for more
- information
+ :ref:`packaging-index`
+ The manual for developers of Python projects who want to package and
+ distribute them. This describes how to use :mod:`packaging` to make
+ projects easily found and added to an existing Python installation.
-.. [#] Then you have no POSIX emulation available, but you also don't need
- :file:`cygwin1.dll`.
+ :mod:`packaging`
+ A library reference for developers of packaging tools wanting to use
+ standalone building blocks like :mod:`~packaging.version` or
+ :mod:`~packaging.metadata`, or extend Packaging itself.
diff --git a/Doc/install/install.rst b/Doc/install/install.rst
new file mode 100644
index 0000000..33f3e9c
--- /dev/null
+++ b/Doc/install/install.rst
@@ -0,0 +1,1117 @@
+.. highlightlang:: none
+
+====================================
+Installing Python projects: overwiew
+====================================
+
+.. _packaging-install-intro:
+
+Introduction
+============
+
+Although Python's extensive standard library covers many programming needs,
+there often comes a time when you need to add new functionality to your Python
+installation in the form of third-party modules. This might be necessary to
+support your own programming, or to support an application that you want to use
+and that happens to be written in Python.
+
+In the past, there was little support for adding third-party modules to an
+existing Python installation. With the introduction of the Python Distribution
+Utilities (Distutils for short) in Python 2.0, this changed. However, not all
+problems were solved; end-users had to rely on ``easy_install`` or
+``pip`` to download third-party modules from PyPI, uninstall distributions or do
+other maintenance operations. Packaging is a more complete replacement for
+Distutils, in the standard library, with a backport named Distutils2 available
+for older Python versions.
+
+This document is aimed primarily at people who need to install third-party
+Python modules: end-users and system administrators who just need to get some
+Python application running, and existing Python programmers who want to add
+new goodies to their toolbox. You don't need to know Python to read this
+document; there will be some brief forays into using Python's interactive mode
+to explore your installation, but that's it. If you're looking for information
+on how to distribute your own Python modules so that others may use them, see
+the :ref:`packaging-index` manual.
+
+
+.. _packaging-trivial-install:
+
+Best case: trivial installation
+-------------------------------
+
+In the best case, someone will have prepared a special version of the module
+distribution you want to install that is targeted specifically at your platform
+and can be installed just like any other software on your platform. For example,
+the module's developer might make an executable installer available for Windows
+users, an RPM package for users of RPM-based Linux systems (Red Hat, SuSE,
+Mandrake, and many others), a Debian package for users of Debian and derivative
+systems, and so forth.
+
+In that case, you would use the standard system tools to download and install
+the specific installer for your platform and its dependencies.
+
+Of course, things will not always be that easy. You might be interested in a
+module whose distribution doesn't have an easy-to-use installer for your
+platform. In that case, you'll have to start with the source distribution
+released by the module's author/maintainer. Installing from a source
+distribution is not too hard, as long as the modules are packaged in the
+standard way. The bulk of this document addresses the building and installing
+of modules from standard source distributions.
+
+
+.. _packaging-distutils:
+
+The Python standard: Distutils
+------------------------------
+
+If you download a source distribution of a module, it will be obvious whether
+it was packaged and distributed using Distutils. First, the distribution's name
+and version number will be featured prominently in the name of the downloaded
+archive, e.g. :file:`foo-1.0.tar.gz` or :file:`widget-0.9.7.zip`. Next, the
+archive will unpack into a similarly-named directory: :file:`foo-1.0` or
+:file:`widget-0.9.7`. Additionally, the distribution may contain a
+:file:`setup.cfg` file and a file named :file:`README.txt` ---or possibly just
+:file:`README`--- explaining that building and installing the module
+distribution is a simple matter of issuing the following command at your shell's
+prompt::
+
+ python setup.py install
+
+Third-party projects have extended Distutils to work around its limitations or
+add functionality. After some years of near-inactivity in Distutils, a new
+maintainer has started to standardize good ideas in PEPs and implement them in a
+new, improved version of Distutils, called Distutils2 or Packaging.
+
+
+.. _packaging-new-standard:
+
+The new standard: Packaging
+---------------------------
+
+The rules described in the first paragraph above apply to Packaging-based
+projects too: a source distribution will have a name like
+:file:`widget-0.9.7.zip`. One of the main differences with Distutils is that
+distributions no longer have a :file:`setup.py` script; it used to cause a
+number of issues. Now there is a unique script installed with Python itself::
+
+ pysetup install widget-0.9.7.zip
+
+Running this command is enough to build and install projects (Python modules or
+packages, scripts or whole applications), without even having to unpack the
+archive. It is also compatible with Distutils-based distributions.
+
+Unless you have to perform non-standard installations or customize the build
+process, you can stop reading this manual ---the above command is everything you
+need to get out of it.
+
+With :program:`pysetup`, you won't even have to manually download a distribution
+before installing it; see :ref:`packaging-pysetup`.
+
+
+.. _packaging-standard-install:
+
+Standard build and install
+==========================
+
+As described in section :ref:`packaging-new-standard`, building and installing
+a module distribution using Packaging usually comes down to one simple
+command::
+
+ pysetup run install_dist
+
+This is a command that should be run in a terminal. On Windows, it is called a
+command prompt and found in :menuselection:`Start --> Accessories`; Powershell
+is a popular alternative.
+
+
+.. _packaging-platform-variations:
+
+Platform variations
+-------------------
+
+The setup command is meant to be run from the root directory of the source
+distribution, i.e. the top-level subdirectory that the module source
+distribution unpacks into. For example, if you've just downloaded a module
+source distribution :file:`foo-1.0.tar.gz` onto a Unix system, the normal
+steps to follow are these::
+
+ gunzip -c foo-1.0.tar.gz | tar xf - # unpacks into directory foo-1.0
+ cd foo-1.0
+ pysetup run install_dist
+
+On Windows, you'd probably download :file:`foo-1.0.zip`. If you downloaded the
+archive file to :file:`C:\\Temp`, then it would unpack into
+:file:`C:\\Temp\\foo-1.0`. To actually unpack the archive, you can use either
+an archive manipulator with a graphical user interface (such as WinZip or 7-Zip)
+or a command-line tool (such as :program:`unzip`, :program:`pkunzip` or, again,
+:program:`7z`). Then, open a command prompt window and run::
+
+ cd c:\Temp\foo-1.0
+ pysetup run install_dist
+
+
+.. _packaging-splitting-up:
+
+Splitting the job up
+--------------------
+
+Running ``pysetup run install_dist`` builds and installs all modules in one go. If you
+prefer to work incrementally ---especially useful if you want to customize the
+build process, or if things are going wrong--- you can use the setup script to
+do one thing at a time. This is a valuable tool when different users will perform
+separately the build and install steps. For example, you might want to build a
+module distribution and hand it off to a system administrator for installation
+(or do it yourself, but with super-user or admin privileges).
+
+For example, to build everything in one step and then install everything
+in a second step, you aptly invoke two distinct Packaging commands::
+
+ pysetup run build
+ pysetup run install_dist
+
+If you do this, you will notice that invoking the :command:`install_dist` command
+first runs the :command:`build` command, which ---in this case--- quickly
+notices it can spare itself the work, since everything in the :file:`build`
+directory is up-to-date.
+
+You may often ignore this ability to divide the process in steps if all you do
+is installing modules downloaded from the Internet, but it's very handy for
+more advanced tasks. If you find yourself in the need for distributing your own
+Python modules and extensions, though, you'll most likely run many individual
+Packaging commands.
+
+
+.. _packaging-how-build-works:
+
+How building works
+------------------
+
+As implied above, the :command:`build` command is responsible for collecting
+and placing the files to be installed into a *build directory*. By default,
+this is :file:`build`, under the distribution root. If you're excessively
+concerned with speed, or want to keep the source tree pristine, you can specify
+a different build directory with the :option:`--build-base` option. For example::
+
+ pysetup run build --build-base /tmp/pybuild/foo-1.0
+
+(Or you could do this permanently with a directive in your system or personal
+Packaging configuration file; see section :ref:`packaging-config-files`.)
+In the usual case, however, all this is unnecessary.
+
+The build tree's default layout looks like so::
+
+ --- build/ --- lib/
+ or
+ --- build/ --- lib.<plat>/
+ temp.<plat>/
+
+where ``<plat>`` expands to a brief description of the current OS/hardware
+platform and Python version. The first form, with just a :file:`lib` directory,
+is used for pure module distributions (module distributions that
+include only pure Python modules). If a module distribution contains any
+extensions (modules written in C/C++), then the second form, with two ``<plat>``
+directories, is used. In that case, the :file:`temp.{plat}` directory holds
+temporary files generated during the compile/link process which are not intended
+to be installed. In either case, the :file:`lib` (or :file:`lib.{plat}`) directory
+contains all Python modules (pure Python and extensions) to be installed.
+
+In the future, more directories will be added to handle Python scripts,
+documentation, binary executables, and whatever else is required to install
+Python modules and applications.
+
+
+.. _packaging-how-install-works:
+
+How installation works
+----------------------
+
+After the :command:`build` command is run (whether explicitly or by the
+:command:`install_dist` command on your behalf), the work of the :command:`install_dist`
+command is relatively simple: all it has to do is copy the contents of
+:file:`build/lib` (or :file:`build/lib.{plat}`) to the installation directory
+of your choice.
+
+If you don't choose an installation directory ---i.e., if you just run
+``pysetup run install_dist``\ --- then the :command:`install_dist` command
+installs to the standard location for third-party Python modules. This location
+varies by platform and depending on how you built/installed Python itself. On
+Unix (and Mac OS X, which is also Unix-based), it also depends on whether the
+module distribution being installed is pure Python or contains extensions
+("non-pure"):
+
++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+
+| Platform | Standard installation location | Default value | Notes |
++=================+=====================================================+==================================================+=======+
+| Unix (pure) | :file:`{prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) |
++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+
+| Unix (non-pure) | :file:`{exec-prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) |
++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+
+| Windows | :file:`{prefix}\\Lib\\site-packages` | :file:`C:\\Python{XY}\\Lib\\site-packages` | \(2) |
++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+
+
+Notes:
+
+(1)
+ Most Linux distributions include Python as a standard part of the system, so
+ :file:`{prefix}` and :file:`{exec-prefix}` are usually both :file:`/usr` on
+ Linux. If you build Python yourself on Linux (or any Unix-like system), the
+ default :file:`{prefix}` and :file:`{exec-prefix}` are :file:`/usr/local`.
+
+(2)
+ The default installation directory on Windows was :file:`C:\\Program
+ Files\\Python` under Python 1.6a1, 1.5.2, and earlier.
+
+:file:`{prefix}` and :file:`{exec-prefix}` stand for the directories that Python
+is installed to, and where it finds its libraries at run-time. They are always
+the same under Windows, and very often the same under Unix and Mac OS X. You
+can find out what your Python installation uses for :file:`{prefix}` and
+:file:`{exec-prefix}` by running Python in interactive mode and typing a few
+simple commands.
+
+.. TODO link to Doc/using instead of duplicating
+
+To start the interactive Python interpreter, you need to follow a slightly
+different recipe for each platform. Under Unix, just type :command:`python` at
+the shell prompt. Under Windows (assuming the Python executable is on your
+:envvar:`PATH`, which is the usual case), you can choose :menuselection:`Start --> Run`,
+type ``python`` and press ``enter``. Alternatively, you can simply execute
+:command:`python` at a command prompt (:menuselection:`Start --> Accessories`)
+or in Powershell.
+
+Once the interpreter is started, you type Python code at the prompt. For
+example, on my Linux system, I type the three Python statements shown below,
+and get the output as shown, to find out my :file:`{prefix}` and :file:`{exec-prefix}`::
+
+ Python 3.3 (r32:88445, Apr 2 2011, 10:43:54)
+ Type "help", "copyright", "credits" or "license" for more information.
+ >>> import sys
+ >>> sys.prefix
+ '/usr'
+ >>> sys.exec_prefix
+ '/usr'
+
+A few other placeholders are used in this document: :file:`{X.Y}` stands for the
+version of Python, for example ``3.2``; :file:`{abiflags}` will be replaced by
+the value of :data:`sys.abiflags` or the empty string for platforms which don't
+define ABI flags; :file:`{distname}` will be replaced by the name of the module
+distribution being installed. Dots and capitalization are important in the
+paths; for example, a value that uses ``python3.2`` on UNIX will typically use
+``Python32`` on Windows.
+
+If you don't want to install modules to the standard location, or if you don't
+have permission to write there, then you need to read about alternate
+installations in section :ref:`packaging-alt-install`. If you want to customize your
+installation directories more heavily, see section :ref:`packaging-custom-install`.
+
+
+.. _packaging-alt-install:
+
+Alternate installation
+======================
+
+Often, it is necessary or desirable to install modules to a location other than
+the standard location for third-party Python modules. For example, on a Unix
+system you might not have permission to write to the standard third-party module
+directory. Or you might wish to try out a module before making it a standard
+part of your local Python installation. This is especially true when upgrading
+a distribution already present: you want to make sure your existing base of
+scripts still works with the new version before actually upgrading.
+
+The Packaging :command:`install_dist` command is designed to make installing module
+distributions to an alternate location simple and painless. The basic idea is
+that you supply a base directory for the installation, and the
+:command:`install_dist` command picks a set of directories (called an *installation
+scheme*) under this base directory in which to install files. The details
+differ across platforms, so read whichever of the following sections applies to
+you.
+
+Note that the various alternate installation schemes are mutually exclusive: you
+can pass ``--user``, or ``--home``, or ``--prefix`` and ``--exec-prefix``, or
+``--install-base`` and ``--install-platbase``, but you can't mix from these
+groups.
+
+
+.. _packaging-alt-install-user:
+
+Alternate installation: the user scheme
+---------------------------------------
+
+This scheme is designed to be the most convenient solution for users that don't
+have write permission to the global site-packages directory or don't want to
+install into it. It is enabled with a simple option::
+
+ pysetup run install_dist --user
+
+Files will be installed into subdirectories of :data:`site.USER_BASE` (written
+as :file:`{userbase}` hereafter). This scheme installs pure Python modules and
+extension modules in the same location (also known as :data:`site.USER_SITE`).
+Here are the values for UNIX, including non-framework builds on Mac OS X:
+
+=============== ===========================================================
+Type of file Installation directory
+=============== ===========================================================
+modules :file:`{userbase}/lib/python{X.Y}/site-packages`
+scripts :file:`{userbase}/bin`
+data :file:`{userbase}`
+C headers :file:`{userbase}/include/python{X.Y}`
+=============== ===========================================================
+
+Framework builds on Mac OS X use these paths:
+
+=============== ===========================================================
+Type of file Installation directory
+=============== ===========================================================
+modules :file:`{userbase}/lib/python/site-packages`
+scripts :file:`{userbase}/bin`
+data :file:`{userbase}`
+C headers :file:`{userbase}/include/python`
+=============== ===========================================================
+
+And here are the values used on Windows:
+
+=============== ===========================================================
+Type of file Installation directory
+=============== ===========================================================
+modules :file:`{userbase}\\Python{XY}\\site-packages`
+scripts :file:`{userbase}\\Scripts`
+data :file:`{userbase}`
+C headers :file:`{userbase}\\Python{XY}\\Include`
+=============== ===========================================================
+
+The advantage of using this scheme compared to the other ones described below is
+that the user site-packages directory is under normal conditions always included
+in :data:`sys.path` (see :mod:`site` for more information), which means that
+there is no additional step to perform after running ``pysetup`` to finalize the
+installation.
+
+The :command:`build_ext` command also has a ``--user`` option to add
+:file:`{userbase}/include` to the compiler search path for header files and
+:file:`{userbase}/lib` to the compiler search path for libraries as well as to
+the runtime search path for shared C libraries (rpath).
+
+
+.. _packaging-alt-install-home:
+
+Alternate installation: the home scheme
+---------------------------------------
+
+The idea behind the "home scheme" is that you build and maintain a personal
+stash of Python modules. This scheme's name is derived from the concept of a
+"home" directory on Unix, since it's not unusual for a Unix user to make their
+home directory have a layout similar to :file:`/usr/` or :file:`/usr/local/`.
+In spite of its name's origin, this scheme can be used by anyone, regardless
+of the operating system.
+
+Installing a new module distribution in this way is as simple as ::
+
+ pysetup run install_dist --home <dir>
+
+where you can supply any directory you like for the :option:`--home` option. On
+Unix, lazy typists can just type a tilde (``~``); the :command:`install_dist` command
+will expand this to your home directory::
+
+ pysetup run install_dist --home ~
+
+To make Python find the distributions installed with this scheme, you may have
+to :ref:`modify Python's search path <inst-search-path>` or edit
+:mod:`sitecustomize` (see :mod:`site`) to call :func:`site.addsitedir` or edit
+:data:`sys.path`.
+
+The :option:`--home` option defines the base directory for the installation.
+Under it, files are installed to the following directories:
+
+=============== ===========================================================
+Type of file Installation directory
+=============== ===========================================================
+modules :file:`{home}/lib/python`
+scripts :file:`{home}/bin`
+data :file:`{home}`
+C headers :file:`{home}/include/python`
+=============== ===========================================================
+
+(Mentally replace slashes with backslashes if you're on Windows.)
+
+
+.. _packaging-alt-install-prefix-unix:
+
+Alternate installation: Unix (the prefix scheme)
+------------------------------------------------
+
+The "prefix scheme" is useful when you wish to use one Python installation to
+run the build command, but install modules into the third-party module directory
+of a different Python installation (or something that looks like a different
+Python installation). If this sounds a trifle unusual, it is ---that's why the
+user and home schemes come before. However, there are at least two known cases
+where the prefix scheme will be useful.
+
+First, consider that many Linux distributions put Python in :file:`/usr`, rather
+than the more traditional :file:`/usr/local`. This is entirely appropriate,
+since in those cases Python is part of "the system" rather than a local add-on.
+However, if you are installing Python modules from source, you probably want
+them to go in :file:`/usr/local/lib/python2.{X}` rather than
+:file:`/usr/lib/python2.{X}`. This can be done with ::
+
+ pysetup run install_dist --prefix /usr/local
+
+Another possibility is a network filesystem where the name used to write to a
+remote directory is different from the name used to read it: for example, the
+Python interpreter accessed as :file:`/usr/local/bin/python` might search for
+modules in :file:`/usr/local/lib/python2.{X}`, but those modules would have to
+be installed to, say, :file:`/mnt/{@server}/export/lib/python2.{X}`. This could
+be done with ::
+
+ pysetup run install_dist --prefix=/mnt/@server/export
+
+In either case, the :option:`--prefix` option defines the installation base, and
+the :option:`--exec-prefix` option defines the platform-specific installation
+base, which is used for platform-specific files. (Currently, this just means
+non-pure module distributions, but could be expanded to C libraries, binary
+executables, etc.) If :option:`--exec-prefix` is not supplied, it defaults to
+:option:`--prefix`. Files are installed as follows:
+
+================= ==========================================================
+Type of file Installation directory
+================= ==========================================================
+Python modules :file:`{prefix}/lib/python{X.Y}/site-packages`
+extension modules :file:`{exec-prefix}/lib/python{X.Y}/site-packages`
+scripts :file:`{prefix}/bin`
+data :file:`{prefix}`
+C headers :file:`{prefix}/include/python{X.Y}{abiflags}`
+================= ==========================================================
+
+.. XXX misses an entry for platinclude
+
+There is no requirement that :option:`--prefix` or :option:`--exec-prefix`
+actually point to an alternate Python installation; if the directories listed
+above do not already exist, they are created at installation time.
+
+Incidentally, the real reason the prefix scheme is important is simply that a
+standard Unix installation uses the prefix scheme, but with :option:`--prefix`
+and :option:`--exec-prefix` supplied by Python itself as ``sys.prefix`` and
+``sys.exec_prefix``. Thus, you might think you'll never use the prefix scheme,
+but every time you run ``pysetup run install_dist`` without any other
+options, you're using it.
+
+Note that installing extensions to an alternate Python installation doesn't have
+anything to do with how those extensions are built: in particular, extensions
+will be compiled using the Python header files (:file:`Python.h` and friends)
+installed with the Python interpreter used to run the build command. It is
+therefore your responsibility to ensure compatibility between the interpreter
+intended to run extensions installed in this way and the interpreter used to
+build these same extensions. To avoid problems, it is best to make sure that
+the two interpreters are the same version of Python (possibly different builds,
+or possibly copies of the same build). (Of course, if your :option:`--prefix`
+and :option:`--exec-prefix` don't even point to an alternate Python installation,
+this is immaterial.)
+
+
+.. _packaging-alt-install-prefix-windows:
+
+Alternate installation: Windows (the prefix scheme)
+---------------------------------------------------
+
+Windows has a different and vaguer notion of home directories than Unix, and
+since its standard Python installation is simpler, the :option:`--prefix` option
+has traditionally been used to install additional packages to arbitrary
+locations. ::
+
+ pysetup run install_dist --prefix "\Temp\Python"
+
+to install modules to the :file:`\\Temp\\Python` directory on the current drive.
+
+The installation base is defined by the :option:`--prefix` option; the
+:option:`--exec-prefix` option is not supported under Windows, which means that
+pure Python modules and extension modules are installed into the same location.
+Files are installed as follows:
+
+=============== ==========================================================
+Type of file Installation directory
+=============== ==========================================================
+modules :file:`{prefix}\\Lib\\site-packages`
+scripts :file:`{prefix}\\Scripts`
+data :file:`{prefix}`
+C headers :file:`{prefix}\\Include`
+=============== ==========================================================
+
+
+.. _packaging-custom-install:
+
+Custom installation
+===================
+
+Sometimes, the alternate installation schemes described in section
+:ref:`packaging-alt-install` just don't do what you want. You might want to tweak
+just one or two directories while keeping everything under the same base
+directory, or you might want to completely redefine the installation scheme.
+In either case, you're creating a *custom installation scheme*.
+
+To create a custom installation scheme, you start with one of the alternate
+schemes and override some of the installation directories used for the various
+types of files, using these options:
+
+====================== =======================
+Type of file Override option
+====================== =======================
+Python modules ``--install-purelib``
+extension modules ``--install-platlib``
+all modules ``--install-lib``
+scripts ``--install-scripts``
+data ``--install-data``
+C headers ``--install-headers``
+====================== =======================
+
+These override options can be relative, absolute,
+or explicitly defined in terms of one of the installation base directories.
+(There are two installation base directories, and they are normally the same
+---they only differ when you use the Unix "prefix scheme" and supply different
+``--prefix`` and ``--exec-prefix`` options; using ``--install-lib`` will
+override values computed or given for ``--install-purelib`` and
+``--install-platlib``, and is recommended for schemes that don't make a
+difference between Python and extension modules.)
+
+For example, say you're installing a module distribution to your home directory
+under Unix, but you want scripts to go in :file:`~/scripts` rather than
+:file:`~/bin`. As you might expect, you can override this directory with the
+:option:`--install-scripts` option and, in this case, it makes most sense to supply
+a relative path, which will be interpreted relative to the installation base
+directory (in our example, your home directory)::
+
+ pysetup run install_dist --home ~ --install-scripts scripts
+
+Another Unix example: suppose your Python installation was built and installed
+with a prefix of :file:`/usr/local/python`. Thus, in a standard installation,
+scripts will wind up in :file:`/usr/local/python/bin`. If you want them in
+:file:`/usr/local/bin` instead, you would supply this absolute directory for
+the :option:`--install-scripts` option::
+
+ pysetup run install_dist --install-scripts /usr/local/bin
+
+This command performs an installation using the "prefix scheme", where the
+prefix is whatever your Python interpreter was installed with ---in this case,
+:file:`/usr/local/python`.
+
+If you maintain Python on Windows, you might want third-party modules to live in
+a subdirectory of :file:`{prefix}`, rather than right in :file:`{prefix}`
+itself. This is almost as easy as customizing the script installation directory
+---you just have to remember that there are two types of modules to worry about,
+Python and extension modules, which can conveniently be both controlled by one
+option::
+
+ pysetup run install_dist --install-lib Site
+
+.. XXX Nothing is installed right under prefix in windows, is it??
+
+The specified installation directory is relative to :file:`{prefix}`. Of
+course, you also have to ensure that this directory is in Python's module
+search path, such as by putting a :file:`.pth` file in a site directory (see
+:mod:`site`). See section :ref:`packaging-search-path` to find out how to modify
+Python's search path.
+
+If you want to define an entire installation scheme, you just have to supply all
+of the installation directory options. Using relative paths is recommended here.
+For example, if you want to maintain all Python module-related files under
+:file:`python` in your home directory, and you want a separate directory for
+each platform that you use your home directory from, you might define the
+following installation scheme::
+
+ pysetup run install_dist --home ~ \
+ --install-purelib python/lib \
+ --install-platlib python/'lib.$PLAT' \
+ --install-scripts python/scripts \
+ --install-data python/data
+
+or, equivalently, ::
+
+ pysetup run install_dist --home ~/python \
+ --install-purelib lib \
+ --install-platlib 'lib.$PLAT' \
+ --install-scripts scripts \
+ --install-data data
+
+``$PLAT`` doesn't need to be defined as an environment variable ---it will also
+be expanded by Packaging as it parses your command line options, just as it
+does when parsing your configuration file(s). (More on that later.)
+
+Obviously, specifying the entire installation scheme every time you install a
+new module distribution would be very tedious. To spare you all that work, you
+can store it in a Packaging configuration file instead (see section
+:ref:`packaging-config-files`), like so::
+
+ [install_dist]
+ install-base = $HOME
+ install-purelib = python/lib
+ install-platlib = python/lib.$PLAT
+ install-scripts = python/scripts
+ install-data = python/data
+
+or, equivalently, ::
+
+ [install_dist]
+ install-base = $HOME/python
+ install-purelib = lib
+ install-platlib = lib.$PLAT
+ install-scripts = scripts
+ install-data = data
+
+Note that these two are *not* equivalent if you override their installation
+base directory when running the setup script. For example, ::
+
+ pysetup run install_dist --install-base /tmp
+
+would install pure modules to :file:`/tmp/python/lib` in the first case, and
+to :file:`/tmp/lib` in the second case. (For the second case, you'd probably
+want to supply an installation base of :file:`/tmp/python`.)
+
+You may have noticed the use of ``$HOME`` and ``$PLAT`` in the sample
+configuration file. These are Packaging configuration variables, which
+bear a strong resemblance to environment variables. In fact, you can use
+environment variables in configuration files on platforms that have such a notion, but
+Packaging additionally defines a few extra variables that may not be in your
+environment, such as ``$PLAT``. Of course, on systems that don't have
+environment variables, such as Mac OS 9, the configuration variables supplied by
+the Packaging are the only ones you can use. See section :ref:`packaging-config-files`
+for details.
+
+.. XXX which vars win out eventually in case of clash env or Packaging?
+
+.. XXX need some Windows examples---when would custom installation schemes be
+ needed on those platforms?
+
+
+.. XXX Move this section to Doc/using
+
+.. _packaging-search-path:
+
+Modifying Python's search path
+------------------------------
+
+When the Python interpreter executes an :keyword:`import` statement, it searches
+for both Python code and extension modules along a search path. A default value
+for this path is configured into the Python binary when the interpreter is built.
+You can obtain the search path by importing the :mod:`sys` module and printing
+the value of ``sys.path``. ::
+
+ $ python
+ Python 2.2 (#11, Oct 3 2002, 13:31:27)
+ [GCC 2.96 20000731 (Red Hat Linux 7.3 2.96-112)] on linux2
+ Type "help", "copyright", "credits" or "license" for more information.
+ >>> import sys
+ >>> sys.path
+ ['', '/usr/local/lib/python2.3', '/usr/local/lib/python2.3/plat-linux2',
+ '/usr/local/lib/python2.3/lib-tk', '/usr/local/lib/python2.3/lib-dynload',
+ '/usr/local/lib/python2.3/site-packages']
+ >>>
+
+The null string in ``sys.path`` represents the current working directory.
+
+The expected convention for locally installed packages is to put them in the
+:file:`{...}/site-packages/` directory, but you may want to choose a different
+location for some reason. For example, if your site kept by convention all web
+server-related software under :file:`/www`. Add-on Python modules might then
+belong in :file:`/www/python`, and in order to import them, this directory would
+have to be added to ``sys.path``. There are several ways to solve this problem.
+
+The most convenient way is to add a path configuration file to a directory
+that's already on Python's path, usually to the :file:`.../site-packages/`
+directory. Path configuration files have an extension of :file:`.pth`, and each
+line must contain a single path that will be appended to ``sys.path``. (Because
+the new paths are appended to ``sys.path``, modules in the added directories
+will not override standard modules. This means you can't use this mechanism for
+installing fixed versions of standard modules.)
+
+Paths can be absolute or relative, in which case they're relative to the
+directory containing the :file:`.pth` file. See the documentation of
+the :mod:`site` module for more information.
+
+A slightly less convenient way is to edit the :file:`site.py` file in Python's
+standard library, and modify ``sys.path``. :file:`site.py` is automatically
+imported when the Python interpreter is executed, unless the :option:`-S` switch
+is supplied to suppress this behaviour. So you could simply edit
+:file:`site.py` and add two lines to it::
+
+ import sys
+ sys.path.append('/www/python/')
+
+However, if you reinstall the same major version of Python (perhaps when
+upgrading from 3.3 to 3.3.1, for example) :file:`site.py` will be overwritten by
+the stock version. You'd have to remember that it was modified and save a copy
+before doing the installation.
+
+Alternatively, there are two environment variables that can modify ``sys.path``.
+:envvar:`PYTHONHOME` sets an alternate value for the prefix of the Python
+installation. For example, if :envvar:`PYTHONHOME` is set to ``/www/python``,
+the search path will be set to ``['', '/www/python/lib/pythonX.Y/',
+'/www/python/lib/pythonX.Y/plat-linux2', ...]``.
+
+The :envvar:`PYTHONPATH` variable can be set to a list of paths that will be
+added to the beginning of ``sys.path``. For example, if :envvar:`PYTHONPATH` is
+set to ``/www/python:/opt/py``, the search path will begin with
+``['/www/python', '/opt/py']``. (Note that directories must exist in order to
+be added to ``sys.path``; the :mod:`site` module removes non-existent paths.)
+
+Finally, ``sys.path`` is just a regular Python list, so any Python application
+can modify it by adding or removing entries.
+
+
+.. _packaging-config-files:
+
+Configuration files for Packaging
+=================================
+
+As mentioned above, you can use configuration files to store personal or site
+preferences for any option supported by any Packaging command. Depending on your
+platform, you can use one of two or three possible configuration files. These
+files will be read before parsing the command-line, so they take precedence over
+default values. In turn, the command-line will override configuration files.
+Lastly, if there are multiple configuration files, values from files read
+earlier will be overridden by values from files read later.
+
+.. XXX "one of two or three possible..." seems wrong info. Below always 3 files
+ are indicated in the tables.
+
+
+.. _packaging-config-filenames:
+
+Location and names of configuration files
+-----------------------------------------
+
+The name and location of the configuration files vary slightly across
+platforms. On Unix and Mac OS X, these are the three configuration files listed
+in the order they are processed:
+
++--------------+----------------------------------------------------------+-------+
+| Type of file | Location and filename | Notes |
++==============+==========================================================+=======+
+| system | :file:`{prefix}/lib/python{ver}/packaging/packaging.cfg` | \(1) |
++--------------+----------------------------------------------------------+-------+
+| personal | :file:`$HOME/.pydistutils.cfg` | \(2) |
++--------------+----------------------------------------------------------+-------+
+| local | :file:`setup.cfg` | \(3) |
++--------------+----------------------------------------------------------+-------+
+
+Similarly, the configuration files on Windows ---also listed in the order they
+are processed--- are these:
+
++--------------+-------------------------------------------------+-------+
+| Type of file | Location and filename | Notes |
++==============+=================================================+=======+
+| system | :file:`{prefix}\\Lib\\packaging\\packaging.cfg` | \(4) |
++--------------+-------------------------------------------------+-------+
+| personal | :file:`%HOME%\\pydistutils.cfg` | \(5) |
++--------------+-------------------------------------------------+-------+
+| local | :file:`setup.cfg` | \(3) |
++--------------+-------------------------------------------------+-------+
+
+On all platforms, the *personal* file can be temporarily disabled by
+means of the `--no-user-cfg` option.
+
+Notes:
+
+(1)
+ Strictly speaking, the system-wide configuration file lives in the directory
+ where Packaging is installed.
+
+(2)
+ On Unix, if the :envvar:`HOME` environment variable is not defined, the
+ user's home directory will be determined with the :func:`getpwuid` function
+ from the standard :mod:`pwd` module. Packaging uses the
+ :func:`os.path.expanduser` function to do this.
+
+(3)
+ I.e., in the current directory (usually the location of the setup script).
+
+(4)
+ (See also note (1).) Python's default installation prefix is
+ :file:`C:\\Python`, so the system configuration file is normally
+ :file:`C:\\Python\\Lib\\packaging\\packaging.cfg`.
+
+(5)
+ On Windows, if the :envvar:`HOME` environment variable is not defined,
+ :envvar:`USERPROFILE` then :envvar:`HOMEDRIVE` and :envvar:`HOMEPATH` will
+ be tried. Packaging uses the :func:`os.path.expanduser` function to do this.
+
+
+.. _packaging-config-syntax:
+
+Syntax of configuration files
+-----------------------------
+
+All Packaging configuration files share the same syntax. Options defined in
+them are grouped into sections, and each Packaging command gets its own section.
+Additionally, there's a ``global`` section for options that affect every command.
+Sections consist of one or more lines containing a single option specified as
+``option = value``.
+
+For example, here's a complete configuration file that forces all commands to
+run quietly by default::
+
+ [global]
+ verbose = 0
+
+If this was the system configuration file, it would affect all processing
+of any Python module distribution by any user on the current system. If it was
+installed as your personal configuration file (on systems that support them),
+it would affect only module distributions processed by you. Lastly, if it was
+used as the :file:`setup.cfg` for a particular module distribution, it would
+affect that distribution only.
+
+.. XXX "(on systems that support them)" seems wrong info
+
+If you wanted to, you could override the default "build base" directory and
+make the :command:`build\*` commands always forcibly rebuild all files with
+the following::
+
+ [build]
+ build-base = blib
+ force = 1
+
+which corresponds to the command-line arguments::
+
+ pysetup run build --build-base blib --force
+
+except that including the :command:`build` command on the command-line means
+that command will be run. Including a particular command in configuration files
+has no such implication; it only means that if the command is run, the options
+for it in the configuration file will apply. (This is also true if you run
+other commands that derive values from it.)
+
+You can find out the complete list of options for any command using the
+:option:`--help` option, e.g.::
+
+ pysetup run build --help
+
+and you can find out the complete list of global options by using
+:option:`--help` without a command::
+
+ pysetup run --help
+
+See also the "Reference" section of the "Distributing Python Modules" manual.
+
+.. XXX no links to the relevant section exist.
+
+
+.. _packaging-building-ext:
+
+Building extensions: tips and tricks
+====================================
+
+Whenever possible, Packaging tries to use the configuration information made
+available by the Python interpreter used to run `pysetup`.
+For example, the same compiler and linker flags used to compile Python will also
+be used for compiling extensions. Usually this will work well, but in
+complicated situations this might be inappropriate. This section discusses how
+to override the usual Packaging behaviour.
+
+
+.. _packaging-tweak-flags:
+
+Tweaking compiler/linker flags
+------------------------------
+
+Compiling a Python extension written in C or C++ will sometimes require
+specifying custom flags for the compiler and linker in order to use a particular
+library or produce a special kind of object code. This is especially true if the
+extension hasn't been tested on your platform, or if you're trying to
+cross-compile Python.
+
+.. TODO update to new setup.cfg
+
+In the most general case, the extension author might have foreseen that
+compiling the extensions would be complicated, and provided a :file:`Setup` file
+for you to edit. This will likely only be done if the module distribution
+contains many separate extension modules, or if they often require elaborate
+sets of compiler flags in order to work.
+
+A :file:`Setup` file, if present, is parsed in order to get a list of extensions
+to build. Each line in a :file:`Setup` describes a single module. Lines have
+the following structure::
+
+ module ... [sourcefile ...] [cpparg ...] [library ...]
+
+
+Let's examine each of the fields in turn.
+
+* *module* is the name of the extension module to be built, and should be a
+ valid Python identifier. You can't just change this in order to rename a module
+ (edits to the source code would also be needed), so this should be left alone.
+
+* *sourcefile* is anything that's likely to be a source code file, at least
+ judging by the filename. Filenames ending in :file:`.c` are assumed to be
+ written in C, filenames ending in :file:`.C`, :file:`.cc`, and :file:`.c++` are
+ assumed to be C++, and filenames ending in :file:`.m` or :file:`.mm` are assumed
+ to be in Objective C.
+
+* *cpparg* is an argument for the C preprocessor, and is anything starting with
+ :option:`-I`, :option:`-D`, :option:`-U` or :option:`-C`.
+
+* *library* is anything ending in :file:`.a` or beginning with :option:`-l` or
+ :option:`-L`.
+
+If a particular platform requires a special library on your platform, you can
+add it by editing the :file:`Setup` file and running ``pysetup run build``.
+For example, if the module defined by the line ::
+
+ foo foomodule.c
+
+must be linked with the math library :file:`libm.a` on your platform, simply add
+:option:`-lm` to the line::
+
+ foo foomodule.c -lm
+
+Arbitrary switches intended for the compiler or the linker can be supplied with
+the :option:`-Xcompiler` *arg* and :option:`-Xlinker` *arg* options::
+
+ foo foomodule.c -Xcompiler -o32 -Xlinker -shared -lm
+
+The next option after :option:`-Xcompiler` and :option:`-Xlinker` will be
+appended to the proper command line, so in the above example the compiler will
+be passed the :option:`-o32` option, and the linker will be passed
+:option:`-shared`. If a compiler option requires an argument, you'll have to
+supply multiple :option:`-Xcompiler` options; for example, to pass ``-x c++``
+the :file:`Setup` file would have to contain ``-Xcompiler -x -Xcompiler c++``.
+
+Compiler flags can also be supplied through setting the :envvar:`CFLAGS`
+environment variable. If set, the contents of :envvar:`CFLAGS` will be added to
+the compiler flags specified in the :file:`Setup` file.
+
+
+.. _packaging-non-ms-compilers:
+
+Using non-Microsoft compilers on Windows
+----------------------------------------
+
+.. sectionauthor:: Rene Liebscher <R.Liebscher@gmx.de>
+
+
+
+Borland/CodeGear C++
+^^^^^^^^^^^^^^^^^^^^
+
+This subsection describes the necessary steps to use Packaging with the Borland
+C++ compiler version 5.5. First you have to know that Borland's object file
+format (OMF) is different from the format used by the Python version you can
+download from the Python or ActiveState Web site. (Python is built with
+Microsoft Visual C++, which uses COFF as the object file format.) For this
+reason, you have to convert Python's library :file:`python25.lib` into the
+Borland format. You can do this as follows:
+
+.. Should we mention that users have to create cfg-files for the compiler?
+.. see also http://community.borland.com/article/0,1410,21205,00.html
+
+::
+
+ coff2omf python25.lib python25_bcpp.lib
+
+The :file:`coff2omf` program comes with the Borland compiler. The file
+:file:`python25.lib` is in the :file:`Libs` directory of your Python
+installation. If your extension uses other libraries (zlib, ...) you have to
+convert them too.
+
+The converted files have to reside in the same directories as the normal
+libraries.
+
+How does Packaging manage to use these libraries with their changed names? If
+the extension needs a library (eg. :file:`foo`) Packaging checks first if it
+finds a library with suffix :file:`_bcpp` (eg. :file:`foo_bcpp.lib`) and then
+uses this library. In the case it doesn't find such a special library it uses
+the default name (:file:`foo.lib`.) [#]_
+
+To let Packaging compile your extension with Borland, C++ you now have to
+type::
+
+ pysetup run build --compiler bcpp
+
+If you want to use the Borland C++ compiler as the default, you could specify
+this in your personal or system-wide configuration file for Packaging (see
+section :ref:`packaging-config-files`.)
+
+
+.. seealso::
+
+ `C++Builder Compiler <http://www.codegear.com/downloads/free/cppbuilder>`_
+ Information about the free C++ compiler from Borland, including links to the
+ download pages.
+
+ `Creating Python Extensions Using Borland's Free Compiler <http://www.cyberus.ca/~g_will/pyExtenDL.shtml>`_
+ Document describing how to use Borland's free command-line C++ compiler to build
+ Python.
+
+
+GNU C / Cygwin / MinGW
+^^^^^^^^^^^^^^^^^^^^^^
+
+This section describes the necessary steps to use Packaging with the GNU C/C++
+compilers in their Cygwin and MinGW distributions. [#]_ For a Python interpreter
+that was built with Cygwin, everything should work without any of these
+following steps.
+
+Not all extensions can be built with MinGW or Cygwin, but many can. Extensions
+most likely to not work are those that use C++ or depend on Microsoft Visual C
+extensions.
+
+To let Packaging compile your extension with Cygwin, you have to type::
+
+ pysetup run build --compiler=cygwin
+
+and for Cygwin in no-cygwin mode [#]_ or for MinGW, type::
+
+ pysetup run build --compiler=mingw32
+
+If you want to use any of these options/compilers as default, you should
+consider writing it in your personal or system-wide configuration file for
+Packaging (see section :ref:`packaging-config-files`.)
+
+Older Versions of Python and MinGW
+""""""""""""""""""""""""""""""""""
+The following instructions only apply if you're using a version of Python
+inferior to 2.4.1 with a MinGW inferior to 3.0.0 (with
+:file:`binutils-2.13.90-20030111-1`).
+
+These compilers require some special libraries. This task is more complex than
+for Borland's C++, because there is no program to convert the library. First
+you have to create a list of symbols which the Python DLL exports. (You can find
+a good program for this task at
+http://www.emmestech.com/software/pexports-0.43/download_pexports.html).
+
+.. I don't understand what the next line means. --amk
+ (inclusive the references on data structures.)
+
+::
+
+ pexports python25.dll > python25.def
+
+The location of an installed :file:`python25.dll` will depend on the
+installation options and the version and language of Windows. In a "just for
+me" installation, it will appear in the root of the installation directory. In
+a shared installation, it will be located in the system directory.
+
+Then you can create from these information an import library for gcc. ::
+
+ /cygwin/bin/dlltool --dllname python25.dll --def python25.def --output-lib libpython25.a
+
+The resulting library has to be placed in the same directory as
+:file:`python25.lib`. (Should be the :file:`libs` directory under your Python
+installation directory.)
+
+If your extension uses other libraries (zlib,...) you might have to convert
+them too. The converted files have to reside in the same directories as the
+normal libraries do.
+
+
+.. seealso::
+
+ `Building Python modules on MS Windows platform with MinGW <http://www.zope.org/Members/als/tips/win32_mingw_modules>`_
+ Information about building the required libraries for the MinGW
+ environment.
+
+
+.. rubric:: Footnotes
+
+.. [#] This also means you could replace all existing COFF-libraries with
+ OMF-libraries of the same name.
+
+.. [#] Check http://sources.redhat.com/cygwin/ and http://www.mingw.org/ for
+ more information.
+
+.. [#] Then you have no POSIX emulation available, but you also don't need
+ :file:`cygwin1.dll`.
diff --git a/Doc/install/pysetup-config.rst b/Doc/install/pysetup-config.rst
new file mode 100644
index 0000000..a473bfe
--- /dev/null
+++ b/Doc/install/pysetup-config.rst
@@ -0,0 +1,44 @@
+.. _packaging-pysetup-config:
+
+=====================
+Pysetup Configuration
+=====================
+
+Pysetup supports two configuration files: :file:`.pypirc` and :file:`packaging.cfg`.
+
+.. FIXME integrate with configfile instead of duplicating
+
+Configuring indexes
+-------------------
+
+You can configure additional indexes in :file:`.pypirc` to be used for index-related
+operations. By default, all configured index-servers and package-servers will be used
+in an additive fashion. To limit operations to specific indexes, use the :option:`--index`
+and :option:`--package-server options`::
+
+ $ pysetup install --index pypi --package-server django some.project
+
+Adding indexes to :file:`.pypirc`::
+
+ [packaging]
+ index-servers =
+ pypi
+ other
+
+ package-servers =
+ django
+
+ [pypi]
+ repository: <repository-url>
+ username: <username>
+ password: <password>
+
+ [other]
+ repository: <repository-url>
+ username: <username>
+ password: <password>
+
+ [django]
+ repository: <repository-url>
+ username: <username>
+ password: <password>
diff --git a/Doc/install/pysetup-servers.rst b/Doc/install/pysetup-servers.rst
new file mode 100644
index 0000000..c6106de
--- /dev/null
+++ b/Doc/install/pysetup-servers.rst
@@ -0,0 +1,61 @@
+.. _packaging-pysetup-servers:
+
+===============
+Package Servers
+===============
+
+Pysetup supports installing Python packages from *Package Servers* in addition
+to PyPI indexes and mirrors.
+
+Package Servers are simple directory listings of Python distributions. Directories
+can be served via HTTP or a local file system. This is useful when you want to
+dump source distributions in a directory and not worry about the full index structure.
+
+Serving distributions from Apache
+---------------------------------
+::
+
+ $ mkdir -p /var/www/html/python/distributions
+ $ cp *.tar.gz /var/www/html/python/distributions/
+
+ <VirtualHost python.example.org:80>
+ ServerAdmin webmaster@domain.com
+ DocumentRoot "/var/www/html/python"
+ ServerName python.example.org
+ ErrorLog logs/python.example.org-error.log
+ CustomLog logs/python.example.org-access.log common
+ Options Indexes FollowSymLinks MultiViews
+ DirectoryIndex index.html index.htm
+
+ <Directory "/var/www/html/python/distributions">
+ Options Indexes FollowSymLinks MultiViews
+ Order allow,deny
+ Allow from all
+ </Directory>
+ </VirtualHost>
+
+Add the Apache based distribution server to :file:`.pypirc`::
+
+ [packaging]
+ package-servers =
+ apache
+
+ [apache]
+ repository: http://python.example.org/distributions/
+
+
+Serving distributions from a file system
+----------------------------------------
+::
+
+ $ mkdir -p /data/python/distributions
+ $ cp *.tar.gz /data/python/distributions/
+
+Add the directory to :file:`.pypirc`::
+
+ [packaging]
+ package-servers =
+ local
+
+ [local]
+ repository: file:///data/python/distributions/
diff --git a/Doc/install/pysetup.rst b/Doc/install/pysetup.rst
new file mode 100644
index 0000000..f6f1f83
--- /dev/null
+++ b/Doc/install/pysetup.rst
@@ -0,0 +1,164 @@
+.. _packaging-pysetup:
+
+================
+Pysetup Tutorial
+================
+
+Getting started
+---------------
+
+Pysetup is a simple script that supports the following features:
+
+- install, remove, list, and verify Python packages;
+- search for available packages on PyPI or any *Simple Index*;
+- verify installed packages (md5sum, installed files, version).
+
+
+Finding out what's installed
+----------------------------
+
+Pysetup makes it easy to find out what Python packages are installed::
+
+ $ pysetup list virtualenv
+ 'virtualenv' 1.6 at '/opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info'
+
+ $ pysetup list
+ 'pyverify' 0.8.1 at '/opt/python3.3/lib/python3.3/site-packages/pyverify-0.8.1.dist-info'
+ 'virtualenv' 1.6 at '/opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info'
+ ...
+
+
+Installing a distribution
+-------------------------
+
+Pysetup can install a Python project from the following sources:
+
+- PyPI and Simple Indexes;
+- source directories containing a valid :file:`setup.py` or :file:`setup.cfg`;
+- distribution source archives (:file:`project-1.0.tar.gz`, :file:`project-1.0.zip`);
+- HTTP (http://host/packages/project-1.0.tar.gz).
+
+
+Installing from PyPI and Simple Indexes::
+
+ $ pysetup install project
+ $ pysetup install project==1.0
+
+Installing from a distribution source archive::
+
+ $ pysetup install project-1.0.tar.gz
+
+Installing from a source directory containing a valid :file:`setup.py` or
+:file:`setup.cfg`::
+
+ $ cd path/to/source/directory
+ $ pysetup install
+
+ $ pysetup install path/to/source/directory
+
+Installing from HTTP::
+
+ $ pysetup install http://host/packages/project-1.0.tar.gz
+
+
+Retrieving metadata
+-------------------
+
+You can gather metadata from two sources, a project's source directory or an
+installed distribution. The `pysetup metadata` command can retrieve one or
+more metadata fields using the `-f` option and a metadata field as the
+argument. ::
+
+ $ pysetup metadata virtualenv -f version -f name
+ Version:
+ 1.6
+ Name:
+ virtualenv
+
+ $ pysetup metadata virtualenv
+ Metadata-Version:
+ 1.0
+ Name:
+ virtualenv
+ Version:
+ 1.6
+ Platform:
+ UNKNOWN
+ Summary:
+ Virtual Python Environment builder
+ ...
+
+.. seealso::
+
+ There are three metadata versions, 1.0, 1.1, and 1.2. The following PEPs
+ describe specifics of the field names, and their semantics and usage. 1.0
+ :PEP:`241`, 1.1 :PEP:`314`, and 1.2 :PEP:`345`
+
+
+Removing a distribution
+-----------------------
+
+You can remove one or more installed distributions using the `pysetup remove`
+command::
+
+ $ pysetup remove virtualenv
+ removing 'virtualenv':
+ /opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info/dependency_links.txt
+ /opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info/entry_points.txt
+ /opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info/not-zip-safe
+ /opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info/PKG-INFO
+ /opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info/SOURCES.txt
+ /opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info/top_level.txt
+ Proceed (y/n)? y
+ success: removed 6 files and 1 dirs
+
+The optional '-y' argument auto confirms, skipping the conformation prompt::
+
+ $ pysetup remove virtualenv -y
+
+
+Getting help
+------------
+
+All pysetup actions take the `-h` and `--help` options which prints the commands
+help string to stdout. ::
+
+ $ pysetup remove -h
+ Usage: pysetup remove dist [-y]
+ or: pysetup remove --help
+
+ Uninstall a Python package.
+
+ positional arguments:
+ dist installed distribution name
+
+ optional arguments:
+ -y auto confirm package removal
+
+Getting a list of all pysetup actions and global options::
+
+ $ pysetup --help
+ Usage: pysetup [options] action [action_options]
+
+ Actions:
+ run: Run one or several commands
+ metadata: Display the metadata of a project
+ install: Install a project
+ remove: Remove a project
+ search: Search for a project in the indexes
+ list: List installed projects
+ graph: Display a graph
+ create: Create a project
+ generate-setup: Generate a backward-comptatible setup.py
+
+ To get more help on an action, use:
+
+ pysetup action --help
+
+ Global options:
+ --verbose (-v) run verbosely (default)
+ --quiet (-q) run quietly (turns verbosity off)
+ --dry-run (-n) don't actually do anything
+ --help (-h) show detailed help message
+ --no-user-cfg ignore pydistutils.cfg in your home directory
+ --version Display the version
diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst
index 369e9cd..e7e7504 100644
--- a/Doc/library/_thread.rst
+++ b/Doc/library/_thread.rst
@@ -35,6 +35,9 @@ It defines the following constants and functions:
Raised on thread-specific errors.
+ .. versionchanged:: 3.3
+ This is now a synonym of the built-in :exc:`RuntimeError`.
+
.. data:: LockType
diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst
index 1048b24..5afc847 100644
--- a/Doc/library/abc.rst
+++ b/Doc/library/abc.rst
@@ -18,7 +18,7 @@ regarding a type hierarchy for numbers based on ABCs.)
The :mod:`collections` module has some concrete classes that derive from
ABCs; these can, of course, be further derived. In addition the
-:mod:`collections` module has some ABCs that can be used to test whether
+:mod:`collections.abc` submodule has some ABCs that can be used to test whether
a class or instance provides a particular interface, for example, is it
hashable or a mapping.
@@ -55,6 +55,9 @@ This module provides the following class:
assert issubclass(tuple, MyABC)
assert isinstance((), MyABC)
+ .. versionchanged:: 3.3
+ Returns the registered subclass, to allow usage as a class decorator.
+
You can also override this method in an abstract base class:
.. method:: __subclasshook__(subclass)
@@ -191,7 +194,7 @@ It also provides the following decorators:
.. versionadded:: 3.2
-.. function:: abstractproperty(fget=None, fset=None, fdel=None, doc=None)
+.. decorator:: abstractproperty(fget=None, fset=None, fdel=None, doc=None)
A subclass of the built-in :func:`property`, indicating an abstract property.
diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst
index 9c7ef52..7a92052 100644
--- a/Doc/library/argparse.rst
+++ b/Doc/library/argparse.rst
@@ -362,16 +362,16 @@ formatter_class
^^^^^^^^^^^^^^^
:class:`ArgumentParser` objects allow the help formatting to be customized by
-specifying an alternate formatting class. Currently, there are three such
+specifying an alternate formatting class. Currently, there are four such
classes:
.. class:: RawDescriptionHelpFormatter
RawTextHelpFormatter
ArgumentDefaultsHelpFormatter
+ MetavarTypeHelpFormatter
-The first two allow more control over how textual descriptions are displayed,
-while the last automatically adds information about argument default values.
-
+:class:`RawDescriptionHelpFormatter` and :class:`RawTextHelpFormatter` give
+more control over how textual descriptions are displayed.
By default, :class:`ArgumentParser` objects line-wrap the description_ and
epilog_ texts in command-line help messages::
@@ -424,8 +424,8 @@ should not be line-wrapped::
:class:`RawTextHelpFormatter` maintains whitespace for all sorts of help text,
including argument descriptions.
-The other formatter class available, :class:`ArgumentDefaultsHelpFormatter`,
-will add information about the default value of each of the arguments::
+:class:`ArgumentDefaultsHelpFormatter` automatically adds information about
+default values to each of the argument help messages::
>>> parser = argparse.ArgumentParser(
... prog='PROG',
@@ -442,6 +442,25 @@ will add information about the default value of each of the arguments::
-h, --help show this help message and exit
--foo FOO FOO! (default: 42)
+:class:`MetavarTypeHelpFormatter` uses the name of the type_ argument for each
+argument as the display name for its values (rather than using the dest_
+as the regular formatter does)::
+
+ >>> parser = argparse.ArgumentParser(
+ ... prog='PROG',
+ ... formatter_class=argparse.MetavarTypeHelpFormatter)
+ >>> parser.add_argument('--foo', type=int)
+ >>> parser.add_argument('bar', type=float)
+ >>> parser.print_help()
+ usage: PROG [-h] [--foo int] float
+
+ positional arguments:
+ float
+
+ optional arguments:
+ -h, --help show this help message and exit
+ --foo int
+
conflict_handler
^^^^^^^^^^^^^^^^
diff --git a/Doc/library/array.rst b/Doc/library/array.rst
index d563cce..3e275a2 100644
--- a/Doc/library/array.rst
+++ b/Doc/library/array.rst
@@ -14,36 +14,44 @@ them is constrained. The type is specified at object creation time by using a
:dfn:`type code`, which is a single character. The following type codes are
defined:
-+-----------+----------------+-------------------+-----------------------+
-| Type code | C Type | Python Type | Minimum size in bytes |
-+===========+================+===================+=======================+
-| ``'b'`` | signed char | int | 1 |
-+-----------+----------------+-------------------+-----------------------+
-| ``'B'`` | unsigned char | int | 1 |
-+-----------+----------------+-------------------+-----------------------+
-| ``'u'`` | Py_UNICODE | Unicode character | 2 (see note) |
-+-----------+----------------+-------------------+-----------------------+
-| ``'h'`` | signed short | int | 2 |
-+-----------+----------------+-------------------+-----------------------+
-| ``'H'`` | unsigned short | int | 2 |
-+-----------+----------------+-------------------+-----------------------+
-| ``'i'`` | signed int | int | 2 |
-+-----------+----------------+-------------------+-----------------------+
-| ``'I'`` | unsigned int | int | 2 |
-+-----------+----------------+-------------------+-----------------------+
-| ``'l'`` | signed long | int | 4 |
-+-----------+----------------+-------------------+-----------------------+
-| ``'L'`` | unsigned long | int | 4 |
-+-----------+----------------+-------------------+-----------------------+
-| ``'f'`` | float | float | 4 |
-+-----------+----------------+-------------------+-----------------------+
-| ``'d'`` | double | float | 8 |
-+-----------+----------------+-------------------+-----------------------+
-
-.. note::
-
- The ``'u'`` typecode corresponds to Python's unicode character. On narrow
- Unicode builds this is 2-bytes, on wide builds this is 4-bytes.
++-----------+--------------------+-------------------+-----------------------+-------+
+| Type code | C Type | Python Type | Minimum size in bytes | Notes |
++===========+====================+===================+=======================+=======+
+| ``'b'`` | signed char | int | 1 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'B'`` | unsigned char | int | 1 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'u'`` | Py_UCS4 | Unicode character | 4 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'h'`` | signed short | int | 2 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'H'`` | unsigned short | int | 2 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'i'`` | signed int | int | 2 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'I'`` | unsigned int | int | 2 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'l'`` | signed long | int | 4 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'L'`` | unsigned long | int | 4 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'q'`` | signed long long | int | 8 | \(1) |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'Q'`` | unsigned long long | int | 8 | \(1) |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'f'`` | float | float | 4 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+| ``'d'`` | double | float | 8 | |
++-----------+--------------------+-------------------+-----------------------+-------+
+
+Notes:
+
+(1)
+ The ``'q'`` and ``'Q'`` type codes are available only if
+ the platform C compiler used to build Python supports C :c:type:`long long`,
+ or, on Windows, :c:type:`__int64`.
+
+ .. versionadded:: 3.3
The actual representation of values is determined by the machine architecture
(strictly speaking, by the C implementation). The actual size can be accessed
diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst
index e2c0b6d..16de3ca 100644
--- a/Doc/library/ast.rst
+++ b/Doc/library/ast.rst
@@ -96,9 +96,6 @@ Node classes
Abstract Grammar
----------------
-The module defines a string constant ``__version__`` which is the decimal
-Subversion revision number of the file shown below.
-
The abstract grammar is currently defined as follows:
.. literalinclude:: ../../Parser/Python.asdl
diff --git a/Doc/library/asyncore.rst b/Doc/library/asyncore.rst
index 619b7bb..5411c30 100644
--- a/Doc/library/asyncore.rst
+++ b/Doc/library/asyncore.rst
@@ -184,12 +184,14 @@ any that have been added to the map during asynchronous service) is closed.
Most of these are nearly identical to their socket partners.
- .. method:: create_socket(family, type)
+ .. method:: create_socket(family=socket.AF_INET, type=socket.SOCK_STREAM)
This is identical to the creation of a normal socket, and will use the
same options for creation. Refer to the :mod:`socket` documentation for
information on creating sockets.
+ .. versionchanged:: 3.3 family and type arguments can be omitted.
+
.. method:: connect(address)
@@ -280,7 +282,7 @@ implement its socket handling::
def __init__(self, host, path):
asyncore.dispatcher.__init__(self)
- self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+ self.create_socket()
self.connect( (host, 80) )
self.buffer = bytes('GET %s HTTP/1.0\r\nHost: %s\r\n\r\n' %
(path, host), 'ascii')
@@ -327,7 +329,7 @@ connections and dispatches the incoming connections to a handler::
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
- self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+ self.create_socket()
self.set_reuse_addr()
self.bind((host, port))
self.listen(5)
diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst
index f2dccc2..54131f5 100644
--- a/Doc/library/atexit.rst
+++ b/Doc/library/atexit.rst
@@ -67,8 +67,9 @@ automatically when the program terminates without relying on the application
making an explicit call into this module at termination. ::
try:
- _count = int(open("/tmp/counter").read())
- except IOError:
+ with open("/tmp/counter") as infile:
+ _count = int(infile.read())
+ except FileNotFoundError:
_count = 0
def incrcounter(n):
@@ -76,7 +77,8 @@ making an explicit call into this module at termination. ::
_count = _count + n
def savecounter():
- open("/tmp/counter", "w").write("%d" % _count)
+ with open("/tmp/counter", "w") as outfile:
+ outfile.write("%d" % _count)
import atexit
atexit.register(savecounter)
diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst
index d9a2bad..87f2cf3 100644
--- a/Doc/library/bz2.rst
+++ b/Doc/library/bz2.rst
@@ -1,189 +1,172 @@
-:mod:`bz2` --- Compression compatible with :program:`bzip2`
-===========================================================
+:mod:`bz2` --- Support for :program:`bzip2` compression
+=======================================================
.. module:: bz2
- :synopsis: Interface to compression and decompression routines
- compatible with bzip2.
+ :synopsis: Interfaces for bzip2 compression and decompression.
.. moduleauthor:: Gustavo Niemeyer <niemeyer@conectiva.com>
+.. moduleauthor:: Nadeem Vawda <nadeem.vawda@gmail.com>
.. sectionauthor:: Gustavo Niemeyer <niemeyer@conectiva.com>
+.. sectionauthor:: Nadeem Vawda <nadeem.vawda@gmail.com>
-This module provides a comprehensive interface for the bz2 compression library.
-It implements a complete file interface, one-shot (de)compression functions, and
-types for sequential (de)compression.
+This module provides a comprehensive interface for compressing and
+decompressing data using the bzip2 compression algorithm.
-For other archive formats, see the :mod:`gzip`, :mod:`zipfile`, and
+For related file formats, see the :mod:`gzip`, :mod:`zipfile`, and
:mod:`tarfile` modules.
-Here is a summary of the features offered by the bz2 module:
+The :mod:`bz2` module contains:
-* :class:`BZ2File` class implements a complete file interface, including
- :meth:`~BZ2File.readline`, :meth:`~BZ2File.readlines`,
- :meth:`~BZ2File.writelines`, :meth:`~BZ2File.seek`, etc;
+* The :class:`BZ2File` class for reading and writing compressed files.
+* The :class:`BZ2Compressor` and :class:`BZ2Decompressor` classes for
+ incremental (de)compression.
+* The :func:`compress` and :func:`decompress` functions for one-shot
+ (de)compression.
-* :class:`BZ2File` class implements emulated :meth:`~BZ2File.seek` support;
-
-* :class:`BZ2File` class implements universal newline support;
-
-* :class:`BZ2File` class offers an optimized line iteration using a readahead
- algorithm;
-
-* Sequential (de)compression supported by :class:`BZ2Compressor` and
- :class:`BZ2Decompressor` classes;
-
-* One-shot (de)compression supported by :func:`compress` and :func:`decompress`
- functions;
-
-* Thread safety uses individual locking mechanism.
+All of the classes in this module may safely be accessed from multiple threads.
(De)compression of files
------------------------
-Handling of compressed files is offered by the :class:`BZ2File` class.
-
-
-.. class:: BZ2File(filename, mode='r', buffering=0, compresslevel=9)
-
- Open a bz2 file. Mode can be either ``'r'`` or ``'w'``, for reading (default)
- or writing. When opened for writing, the file will be created if it doesn't
- exist, and truncated otherwise. If *buffering* is given, ``0`` means
- unbuffered, and larger numbers specify the buffer size; the default is
- ``0``. If *compresslevel* is given, it must be a number between ``1`` and
- ``9``; the default is ``9``. Add a ``'U'`` to mode to open the file for input
- with universal newline support. Any line ending in the input file will be
- seen as a ``'\n'`` in Python. Also, a file so opened gains the attribute
- :attr:`newlines`; the value for this attribute is one of ``None`` (no newline
- read yet), ``'\r'``, ``'\n'``, ``'\r\n'`` or a tuple containing all the
- newline types seen. Universal newlines are available only when
- reading. Instances support iteration in the same way as normal :class:`file`
- instances.
+.. class:: BZ2File(filename=None, mode='r', buffering=None, compresslevel=9, fileobj=None)
- :class:`BZ2File` supports the :keyword:`with` statement.
+ Open a bzip2-compressed file.
- .. versionchanged:: 3.1
- Support for the :keyword:`with` statement was added.
+ The :class:`BZ2File` can wrap an existing :term:`file object` (given by
+ *fileobj*), or operate directly on a named file (named by *filename*).
+ Exactly one of these two parameters should be provided.
+ The *mode* argument can be either ``'r'`` for reading (default), ``'w'`` for
+ overwriting, or ``'a'`` for appending. If *fileobj* is provided, a mode of
+ ``'w'`` does not truncate the file, and is instead equivalent to ``'a'``.
- .. method:: close()
+ The *buffering* argument is ignored. Its use is deprecated.
- Close the file. Sets data attribute :attr:`closed` to true. A closed file
- cannot be used for further I/O operations. :meth:`close` may be called
- more than once without error.
+ If *mode* is ``'w'`` or ``'a'``, *compresslevel* can be a number between
+ ``1`` and ``9`` specifying the level of compression: ``1`` produces the
+ least compression, and ``9`` (default) produces the most compression.
+ If *mode* is ``'r'``, the input file may be the concatenation of multiple
+ compressed streams.
- .. method:: read([size])
+ :class:`BZ2File` provides all of the members specified by the
+ :class:`io.BufferedIOBase`, except for :meth:`detach` and :meth:`truncate`.
+ Iteration and the :keyword:`with` statement are supported.
- Read at most *size* uncompressed bytes, returned as a byte string. If the
- *size* argument is negative or omitted, read until EOF is reached.
+ :class:`BZ2File` also provides the following method:
+ .. method:: peek([n])
- .. method:: readline([size])
+ Return buffered data without advancing the file position. At least one
+ byte of data will be returned (unless at EOF). The exact number of bytes
+ returned is unspecified.
- Return the next line from the file, as a byte string, retaining newline.
- A non-negative *size* argument limits the maximum number of bytes to
- return (an incomplete line may be returned then). Return an empty byte
- string at EOF.
+ .. versionadded:: 3.3
+ .. versionchanged:: 3.1
+ Support for the :keyword:`with` statement was added.
- .. method:: readlines([size])
-
- Return a list of lines read. The optional *size* argument, if given, is an
- approximate bound on the total number of bytes in the lines returned.
+ .. versionchanged:: 3.3
+ The :meth:`fileno`, :meth:`readable`, :meth:`seekable`, :meth:`writable`,
+ :meth:`read1` and :meth:`readinto` methods were added.
+ .. versionchanged:: 3.3
+ The *fileobj* argument to the constructor was added.
- .. method:: seek(offset[, whence])
+ .. versionchanged:: 3.3
+ The ``'a'`` (append) mode was added, along with support for reading
+ multi-stream files.
- Move to new file position. Argument *offset* is a byte count. Optional
- argument *whence* defaults to ``os.SEEK_SET`` or ``0`` (offset from start
- of file; offset should be ``>= 0``); other values are ``os.SEEK_CUR`` or
- ``1`` (move relative to current position; offset can be positive or
- negative), and ``os.SEEK_END`` or ``2`` (move relative to end of file;
- offset is usually negative, although many platforms allow seeking beyond
- the end of a file).
- Note that seeking of bz2 files is emulated, and depending on the
- parameters the operation may be extremely slow.
+Incremental (de)compression
+---------------------------
+.. class:: BZ2Compressor(compresslevel=9)
- .. method:: tell()
+ Create a new compressor object. This object may be used to compress data
+ incrementally. For one-shot compression, use the :func:`compress` function
+ instead.
- Return the current file position, an integer.
+ *compresslevel*, if given, must be a number between ``1`` and ``9``. The
+ default is ``9``.
+ .. method:: compress(data)
- .. method:: write(data)
+ Provide data to the compressor object. Returns a chunk of compressed data
+ if possible, or an empty byte string otherwise.
- Write the byte string *data* to file. Note that due to buffering,
- :meth:`close` may be needed before the file on disk reflects the data
- written.
+ When you have finished providing data to the compressor, call the
+ :meth:`flush` method to finish the compression process.
- .. method:: writelines(sequence_of_byte_strings)
+ .. method:: flush()
- Write the sequence of byte strings to the file. Note that newlines are not
- added. The sequence can be any iterable object producing byte strings.
- This is equivalent to calling write() for each byte string.
+ Finish the compression process. Returns the compressed data left in
+ internal buffers.
+ The compressor object may not be used after this method has been called.
-Sequential (de)compression
---------------------------
-Sequential compression and decompression is done using the classes
-:class:`BZ2Compressor` and :class:`BZ2Decompressor`.
+.. class:: BZ2Decompressor()
+ Create a new decompressor object. This object may be used to decompress data
+ incrementally. For one-shot compression, use the :func:`decompress` function
+ instead.
-.. class:: BZ2Compressor(compresslevel=9)
+ .. note::
+ This class does not transparently handle inputs containing multiple
+ compressed streams, unlike :func:`decompress` and :class:`BZ2File`. If
+ you need to decompress a multi-stream input with :class:`BZ2Decompressor`,
+ you must use a new decompressor for each stream.
- Create a new compressor object. This object may be used to compress data
- sequentially. If you want to compress data in one shot, use the
- :func:`compress` function instead. The *compresslevel* parameter, if given,
- must be a number between ``1`` and ``9``; the default is ``9``.
+ .. method:: decompress(data)
- .. method:: compress(data)
+ Provide data to the decompressor object. Returns a chunk of decompressed
+ data if possible, or an empty byte string otherwise.
- Provide more data to the compressor object. It will return chunks of
- compressed data whenever possible. When you've finished providing data to
- compress, call the :meth:`flush` method to finish the compression process,
- and return what is left in internal buffers.
+ Attempting to decompress data after the end of the current stream is
+ reached raises an :exc:`EOFError`. If any data is found after the end of
+ the stream, it is ignored and saved in the :attr:`unused_data` attribute.
- .. method:: flush()
+ .. attribute:: eof
- Finish the compression process and return what is left in internal
- buffers. You must not use the compressor object after calling this method.
+ True if the end-of-stream marker has been reached.
+ .. versionadded:: 3.3
-.. class:: BZ2Decompressor()
- Create a new decompressor object. This object may be used to decompress data
- sequentially. If you want to decompress data in one shot, use the
- :func:`decompress` function instead.
+ .. attribute:: unused_data
- .. method:: decompress(data)
+ Data found after the end of the compressed stream.
- Provide more data to the decompressor object. It will return chunks of
- decompressed data whenever possible. If you try to decompress data after
- the end of stream is found, :exc:`EOFError` will be raised. If any data
- was found after the end of stream, it'll be ignored and saved in
- :attr:`unused_data` attribute.
+ If this attribute is accessed before the end of the stream has been
+ reached, its value will be ``b''``.
One-shot (de)compression
------------------------
-One-shot compression and decompression is provided through the :func:`compress`
-and :func:`decompress` functions.
+.. function:: compress(data, compresslevel=9)
+ Compress *data*.
-.. function:: compress(data, compresslevel=9)
+ *compresslevel*, if given, must be a number between ``1`` and ``9``. The
+ default is ``9``.
- Compress *data* in one shot. If you want to compress data sequentially, use
- an instance of :class:`BZ2Compressor` instead. The *compresslevel* parameter,
- if given, must be a number between ``1`` and ``9``; the default is ``9``.
+ For incremental compression, use a :class:`BZ2Compressor` instead.
.. function:: decompress(data)
- Decompress *data* in one shot. If you want to decompress data sequentially,
- use an instance of :class:`BZ2Decompressor` instead.
+ Decompress *data*.
+
+ If *data* is the concatenation of multiple compressed streams, decompress
+ all of the streams.
+
+ For incremental decompression, use a :class:`BZ2Decompressor` instead.
+
+ .. versionchanged:: 3.3
+ Support for multi-stream inputs was added.
diff --git a/Doc/library/chunk.rst b/Doc/library/chunk.rst
index d3558a4..c1ba497 100644
--- a/Doc/library/chunk.rst
+++ b/Doc/library/chunk.rst
@@ -84,8 +84,9 @@ instance will fail with a :exc:`EOFError` exception.
Close and skip to the end of the chunk. This does not close the
underlying file.
- The remaining methods will raise :exc:`IOError` if called after the
- :meth:`close` method has been called.
+ The remaining methods will raise :exc:`OSError` if called after the
+ :meth:`close` method has been called. Before Python 3.3, they used to
+ raise :exc:`IOError`, now an alias of :exc:`OSError`.
.. method:: isatty()
diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst
index fd7f453..0c43bb8 100644
--- a/Doc/library/cmd.rst
+++ b/Doc/library/cmd.rst
@@ -285,8 +285,8 @@ immediate playback::
def do_playback(self, arg):
'Playback commands from a file: PLAYBACK rose.cmd'
self.close()
- cmds = open(arg).read().splitlines()
- self.cmdqueue.extend(cmds)
+ with open(arg) as f:
+ self.cmdqueue.extend(f.read().splitlines())
def precmd(self, line):
line = line.lower()
if self.file and 'playback' not in line:
diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst
index 7747794..4523c7f 100644
--- a/Doc/library/codecs.rst
+++ b/Doc/library/codecs.rst
@@ -458,7 +458,8 @@ define in order to be compatible with the Python codec registry.
.. method:: reset()
- Reset the encoder to the initial state.
+ Reset the encoder to the initial state. The output is discarded: call
+ ``.encode('', final=True)`` to reset the encoder and to get the output.
.. method:: IncrementalEncoder.getstate()
@@ -786,11 +787,9 @@ methods and attributes from the underlying stream.
Encodings and Unicode
---------------------
-Strings are stored internally as sequences of codepoints (to be precise
-as :c:type:`Py_UNICODE` arrays). Depending on the way Python is compiled (either
-via ``--without-wide-unicode`` or ``--with-wide-unicode``, with the
-former being the default) :c:type:`Py_UNICODE` is either a 16-bit or 32-bit data
-type. Once a string object is used outside of CPU and memory, CPU endianness
+Strings are stored internally as sequences of codepoints in range ``0 - 10FFFF``
+(see :pep:`393` for more details about the implementation).
+Once a string object is used outside of CPU and memory, CPU endianness
and how these arrays are stored as bytes become an issue. Transforming a
string object into a sequence of bytes is called encoding and recreating the
string object from the sequence of bytes is known as decoding. There are many
@@ -901,6 +900,15 @@ is meant to be exhaustive. Notice that spelling alternatives that only differ in
case or use a hyphen instead of an underscore are also valid aliases; therefore,
e.g. ``'utf-8'`` is a valid alias for the ``'utf_8'`` codec.
+.. impl-detail::
+
+ Some common encodings can bypass the codecs lookup machinery to
+ improve performance. These optimization opportunities are only
+ recognized by CPython for a limited set of aliases: utf-8, utf8,
+ latin-1, latin1, iso-8859-1, mbcs (Windows only), ascii, utf-16,
+ and utf-32. Using alternative spellings for these encodings may
+ result in slower execution.
+
Many of the character sets support the same languages. They vary in individual
characters (e.g. whether the EURO SIGN is supported or not), and in the
assignment of characters to code positions. For the European languages in
@@ -1003,6 +1011,11 @@ particular, the following variants typically exist:
+-----------------+--------------------------------+--------------------------------+
| cp1258 | windows-1258 | Vietnamese |
+-----------------+--------------------------------+--------------------------------+
+| cp65001 | | Windows only: Windows UTF-8 |
+| | | (``CP_UTF8``) |
+| | | |
+| | | .. versionadded:: 3.3 |
++-----------------+--------------------------------+--------------------------------+
| euc_jp | eucjp, ujis, u-jis | Japanese |
+-----------------+--------------------------------+--------------------------------+
| euc_jis_2004 | jisx0213, eucjis2004 | Japanese |
@@ -1272,12 +1285,13 @@ functions can be used directly if desired.
.. module:: encodings.mbcs
:synopsis: Windows ANSI codepage
-Encode operand according to the ANSI codepage (CP_ACP). This codec only
-supports ``'strict'`` and ``'replace'`` error handlers to encode, and
-``'strict'`` and ``'ignore'`` error handlers to decode.
+Encode operand according to the ANSI codepage (CP_ACP).
Availability: Windows only.
+.. versionchanged:: 3.3
+ Support any error handler.
+
.. versionchanged:: 3.2
Before 3.2, the *errors* argument was ignored; ``'replace'`` was always used
to encode, and ``'ignore'`` to decode.
diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst
new file mode 100644
index 0000000..9873489
--- /dev/null
+++ b/Doc/library/collections.abc.rst
@@ -0,0 +1,182 @@
+:mod:`collections.abc` --- Abstract Base Classes for Containers
+===============================================================
+
+.. module:: collections.abc
+ :synopsis: Abstract base classes for containers
+.. moduleauthor:: Raymond Hettinger <python at rcn.com>
+.. sectionauthor:: Raymond Hettinger <python at rcn.com>
+
+.. versionadded:: 3.3
+ Formerly, this module was part of the :mod:`collections` module.
+
+.. testsetup:: *
+
+ from collections import *
+ import itertools
+ __name__ = '<doctest>'
+
+**Source code:** :source:`Lib/collections/abc.py`
+
+--------------
+
+This module provides :term:`abstract base classes <abstract base class>` that
+can be used to test whether a class provides a particular interface; for
+example, whether it is hashable or whether it is a mapping.
+
+
+.. _collections-abstract-base-classes:
+
+Collections Abstract Base Classes
+---------------------------------
+
+The collections module offers the following :term:`ABCs <abstract base class>`:
+
+========================= ===================== ====================== ====================================================
+ABC Inherits from Abstract Methods Mixin Methods
+========================= ===================== ====================== ====================================================
+:class:`Container` ``__contains__``
+:class:`Hashable` ``__hash__``
+:class:`Iterable` ``__iter__``
+:class:`Iterator` :class:`Iterable` ``__next__`` ``__iter__``
+:class:`Sized` ``__len__``
+:class:`Callable` ``__call__``
+
+:class:`Sequence` :class:`Sized`, ``__getitem__`` ``__contains__``, ``__iter__``, ``__reversed__``,
+ :class:`Iterable`, ``index``, and ``count``
+ :class:`Container`
+
+:class:`MutableSequence` :class:`Sequence` ``__setitem__``, Inherited :class:`Sequence` methods and
+ ``__delitem__``, ``append``, ``reverse``, ``extend``, ``pop``,
+ ``insert`` ``remove``, ``clear``, and ``__iadd__``
+
+:class:`Set` :class:`Sized`, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``,
+ :class:`Iterable`, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``,
+ :class:`Container` ``__sub__``, ``__xor__``, and ``isdisjoint``
+
+:class:`MutableSet` :class:`Set` ``add``, Inherited :class:`Set` methods and
+ ``discard`` ``clear``, ``pop``, ``remove``, ``__ior__``,
+ ``__iand__``, ``__ixor__``, and ``__isub__``
+
+:class:`Mapping` :class:`Sized`, ``__getitem__`` ``__contains__``, ``keys``, ``items``, ``values``,
+ :class:`Iterable`, ``get``, ``__eq__``, and ``__ne__``
+ :class:`Container`
+
+:class:`MutableMapping` :class:`Mapping` ``__setitem__``, Inherited :class:`Mapping` methods and
+ ``__delitem__`` ``pop``, ``popitem``, ``clear``, ``update``,
+ and ``setdefault``
+
+
+:class:`MappingView` :class:`Sized` ``__len__``
+:class:`ItemsView` :class:`MappingView`, ``__contains__``,
+ :class:`Set` ``__iter__``
+:class:`KeysView` :class:`MappingView`, ``__contains__``,
+ :class:`Set` ``__iter__``
+:class:`ValuesView` :class:`MappingView` ``__contains__``, ``__iter__``
+========================= ===================== ====================== ====================================================
+
+
+.. class:: Container
+ Hashable
+ Sized
+ Callable
+
+ ABCs for classes that provide respectively the methods :meth:`__contains__`,
+ :meth:`__hash__`, :meth:`__len__`, and :meth:`__call__`.
+
+.. class:: Iterable
+
+ ABC for classes that provide the :meth:`__iter__` method.
+ See also the definition of :term:`iterable`.
+
+.. class:: Iterator
+
+ ABC for classes that provide the :meth:`__iter__` and :meth:`next` methods.
+ See also the definition of :term:`iterator`.
+
+.. class:: Sequence
+ MutableSequence
+
+ ABCs for read-only and mutable :term:`sequences <sequence>`.
+
+.. class:: Set
+ MutableSet
+
+ ABCs for read-only and mutable sets.
+
+.. class:: Mapping
+ MutableMapping
+
+ ABCs for read-only and mutable :term:`mappings <mapping>`.
+
+.. class:: MappingView
+ ItemsView
+ KeysView
+ ValuesView
+
+ ABCs for mapping, items, keys, and values :term:`views <view>`.
+
+
+These ABCs allow us to ask classes or instances if they provide
+particular functionality, for example::
+
+ size = None
+ if isinstance(myvar, collections.Sized):
+ size = len(myvar)
+
+Several of the ABCs are also useful as mixins that make it easier to develop
+classes supporting container APIs. For example, to write a class supporting
+the full :class:`Set` API, it only necessary to supply the three underlying
+abstract methods: :meth:`__contains__`, :meth:`__iter__`, and :meth:`__len__`.
+The ABC supplies the remaining methods such as :meth:`__and__` and
+:meth:`isdisjoint` ::
+
+ class ListBasedSet(collections.Set):
+ ''' Alternate set implementation favoring space over speed
+ and not requiring the set elements to be hashable. '''
+ def __init__(self, iterable):
+ self.elements = lst = []
+ for value in iterable:
+ if value not in lst:
+ lst.append(value)
+ def __iter__(self):
+ return iter(self.elements)
+ def __contains__(self, value):
+ return value in self.elements
+ def __len__(self):
+ return len(self.elements)
+
+ s1 = ListBasedSet('abcdef')
+ s2 = ListBasedSet('defghi')
+ overlap = s1 & s2 # The __and__() method is supported automatically
+
+Notes on using :class:`Set` and :class:`MutableSet` as a mixin:
+
+(1)
+ Since some set operations create new sets, the default mixin methods need
+ a way to create new instances from an iterable. The class constructor is
+ assumed to have a signature in the form ``ClassName(iterable)``.
+ That assumption is factored-out to an internal classmethod called
+ :meth:`_from_iterable` which calls ``cls(iterable)`` to produce a new set.
+ If the :class:`Set` mixin is being used in a class with a different
+ constructor signature, you will need to override :meth:`_from_iterable`
+ with a classmethod that can construct new instances from
+ an iterable argument.
+
+(2)
+ To override the comparisons (presumably for speed, as the
+ semantics are fixed), redefine :meth:`__le__` and
+ then the other operations will automatically follow suit.
+
+(3)
+ The :class:`Set` mixin provides a :meth:`_hash` method to compute a hash value
+ for the set; however, :meth:`__hash__` is not defined because not all sets
+ are hashable or immutable. To add set hashabilty using mixins,
+ inherit from both :meth:`Set` and :meth:`Hashable`, then define
+ ``__hash__ = Set._hash``.
+
+.. seealso::
+
+ * `OrderedSet recipe <http://code.activestate.com/recipes/576694/>`_ for an
+ example built on :class:`MutableSet`.
+
+ * For more about ABCs, see the :mod:`abc` module and :pep:`3119`.
diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst
index e512bf1..c9e3864 100644
--- a/Doc/library/collections.rst
+++ b/Doc/library/collections.rst
@@ -12,7 +12,7 @@
import itertools
__name__ = '<doctest>'
-**Source code:** :source:`Lib/collections.py` and :source:`Lib/_abcoll.py`
+**Source code:** :source:`Lib/collections/__init__.py`
--------------
@@ -23,6 +23,7 @@ Python's general purpose built-in containers, :class:`dict`, :class:`list`,
===================== ====================================================================
:func:`namedtuple` factory function for creating tuple subclasses with named fields
:class:`deque` list-like container with fast appends and pops on either end
+:class:`ChainMap` dict-like class for creating a single view of multiple mappings
:class:`Counter` dict subclass for counting hashable objects
:class:`OrderedDict` dict subclass that remembers the order entries were added
:class:`defaultdict` dict subclass that calls a factory function to supply missing values
@@ -31,12 +32,125 @@ Python's general purpose built-in containers, :class:`dict`, :class:`list`,
:class:`UserString` wrapper around string objects for easier string subclassing
===================== ====================================================================
-In addition to the concrete container classes, the collections module provides
-:ref:`abstract base classes <collections-abstract-base-classes>` that can be
-used to test whether a class provides a particular interface, for example,
-whether it is hashable or a mapping.
+.. versionchanged:: 3.3
+ Moved :ref:`collections-abstract-base-classes` to the :mod:`collections.abc` module.
+ For backwards compatibility, they continue to be visible in this module
+ as well.
+:class:`ChainMap` objects
+-------------------------
+
+A :class:`ChainMap` class is provided for quickly linking a number of mappings
+so they can be treated as a single unit. It is often much faster than creating
+a new dictionary and running multiple :meth:`~dict.update` calls.
+
+The class can be used to simulate nested scopes and is useful in templating.
+
+.. class:: ChainMap(*maps)
+
+ A :class:`ChainMap` groups multiple dicts or other mappings together to
+ create a single, updateable view. If no *maps* are specified, a single empty
+ dictionary is provided so that a new chain always has at least one mapping.
+
+ The underlying mappings are stored in a list. That list is public and can
+ accessed or updated using the *maps* attribute. There is no other state.
+
+ Lookups search the underlying mappings successively until a key is found. In
+ contrast, writes, updates, and deletions only operate on the first mapping.
+
+ A :class:`ChainMap` incorporates the underlying mappings by reference. So, if
+ one of the underlying mappings gets updated, those changes will be reflected
+ in :class:`ChainMap`.
+
+ All of the usual dictionary methods are supported. In addition, there is a
+ *maps* attribute, a method for creating new subcontexts, and a property for
+ accessing all but the first mapping:
+
+ .. attribute:: maps
+
+ A user updateable list of mappings. The list is ordered from
+ first-searched to last-searched. It is the only stored state and can
+ be modified to change which mappings are searched. The list should
+ always contain at least one mapping.
+
+ .. method:: new_child()
+
+ Returns a new :class:`ChainMap` containing a new :class:`dict` followed by
+ all of the maps in the current instance. A call to ``d.new_child()`` is
+ equivalent to: ``ChainMap({}, *d.maps)``. This method is used for
+ creating subcontexts that can be updated without altering values in any
+ of the parent mappings.
+
+ .. method:: parents()
+
+ Returns a new :class:`ChainMap` containing all of the maps in the current
+ instance except the first one. This is useful for skipping the first map
+ in the search. The use-cases are similar to those for the
+ :keyword:`nonlocal` keyword used in :term:`nested scopes <nested scope>`.
+ The use-cases also parallel those for the builtin :func:`super` function.
+ A reference to ``d.parents`` is equivalent to: ``ChainMap(*d.maps[1:])``.
+
+ .. versionadded:: 3.3
+
+ Example of simulating Python's internal lookup chain::
+
+ import builtins
+ pylookup = ChainMap(locals(), globals(), vars(builtins))
+
+ Example of letting user specified values take precedence over environment
+ variables which in turn take precedence over default values::
+
+ import os, argparse
+ defaults = {'color': 'red', 'user': guest}
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-u', '--user')
+ parser.add_argument('-c', '--color')
+ user_specified = vars(parser.parse_args())
+ combined = ChainMap(user_specified, os.environ, defaults)
+
+ Example patterns for using the :class:`ChainMap` class to simulate nested
+ contexts::
+
+ c = ChainMap() Create root context
+ d = c.new_child() Create nested child context
+ e = c.new_child() Child of c, independent from d
+ e.maps[0] Current context dictionary -- like Python's locals()
+ e.maps[-1] Root context -- like Python's globals()
+ e.parents Enclosing context chain -- like Python's nonlocals
+
+ d['x'] Get first key in the chain of contexts
+ d['x'] = 1 Set value in current context
+ del['x'] Delete from current context
+ list(d) All nested values
+ k in d Check all nested values
+ len(d) Number of nested values
+ d.items() All nested items
+ dict(d) Flatten into a regular dictionary
+
+ .. seealso::
+
+ * The `MultiContext class
+ <http://svn.enthought.com/svn/enthought/CodeTools/trunk/enthought/contexts/multi_context.py>`_
+ in the Enthought `CodeTools package
+ <https://github.com/enthought/codetools>`_ has options to support
+ writing to any mapping in the chain.
+
+ * Django's `Context class
+ <http://code.djangoproject.com/browser/django/trunk/django/template/context.py>`_
+ for templating is a read-only chain of mappings. It also features
+ pushing and popping of contexts similar to the
+ :meth:`~collections.ChainMap.new_child` method and the
+ :meth:`~collections.ChainMap.parents` property.
+
+ * The `Nested Contexts recipe
+ <http://code.activestate.com/recipes/577434/>`_ has options to control
+ whether writes and other mutations apply only to the first mapping or to
+ any mapping in the chain.
+
+ * A `greatly simplified read-only version of Chainmap
+ <http://code.activestate.com/recipes/305268/>`_.
+
:class:`Counter` objects
------------------------
@@ -149,7 +263,7 @@ Common patterns for working with :class:`Counter` objects::
c.items() # convert to a list of (elem, cnt) pairs
Counter(dict(list_of_pairs)) # convert from a list of (elem, cnt) pairs
c.most_common()[:-n:-1] # n least common elements
- c += Counter() # remove zero and negative counts
+ +c # remove zero and negative counts
Several mathematical operations are provided for combining :class:`Counter`
objects to produce multisets (counters that have counts greater than zero).
@@ -169,6 +283,18 @@ counts, but the output will exclude results with counts of zero or less.
>>> c | d # union: max(c[x], d[x])
Counter({'a': 3, 'b': 2})
+Unary addition and substraction are shortcuts for adding an empty counter
+or subtracting from an empty counter.
+
+ >>> c = Counter(a=2, b=-4)
+ >>> +c
+ Counter({'a': 2})
+ >>> -c
+ Counter({'b': 4})
+
+.. versionadded:: 3.3
+ Added support for unary plus, unary minus, and in-place multiset operations.
+
.. note::
Counters were primarily designed to work with positive integers to represent
@@ -398,7 +524,8 @@ in Unix::
def tail(filename, n=10):
'Return the last n lines of a file'
- return deque(open(filename), n)
+ with open(filename) as f:
+ return deque(f, n)
Another approach to using deques is to maintain a sequence of recently
added elements by appending to the right and popping to the left::
@@ -545,7 +672,7 @@ Setting the :attr:`default_factory` to :class:`set` makes the
... d[k].add(v)
...
>>> list(d.items())
- [('blue', set([2, 4])), ('red', set([1, 3]))]
+ [('blue', {2, 4}), ('red', {1, 3})]
:func:`namedtuple` Factory Function for Tuples with Named Fields
@@ -578,7 +705,9 @@ they add the ability to access fields by name instead of position index.
converted to ``['abc', '_1', 'ghi', '_3']``, eliminating the keyword
``def`` and the duplicate fieldname ``abc``.
- If *verbose* is true, the class definition is printed just before being built.
+ If *verbose* is true, the class definition is printed after it is
+ built. This option is outdated; instead, it is simpler to print the
+ :attr:`_source` attribute.
Named tuple instances do not have per-instance dictionaries, so they are
lightweight and require no more memory than regular tuples.
@@ -592,53 +721,6 @@ they add the ability to access fields by name instead of position index.
>>> # Basic example
>>> Point = namedtuple('Point', ['x', 'y'])
- >>> p = Point(x=10, y=11)
-
- >>> # Example using the verbose option to print the class definition
- >>> Point = namedtuple('Point', 'x y', verbose=True)
- class Point(tuple):
- 'Point(x, y)'
- <BLANKLINE>
- __slots__ = ()
- <BLANKLINE>
- _fields = ('x', 'y')
- <BLANKLINE>
- def __new__(_cls, x, y):
- 'Create a new instance of Point(x, y)'
- return _tuple.__new__(_cls, (x, y))
- <BLANKLINE>
- @classmethod
- def _make(cls, iterable, new=tuple.__new__, len=len):
- 'Make a new Point object from a sequence or iterable'
- result = new(cls, iterable)
- if len(result) != 2:
- raise TypeError('Expected 2 arguments, got %d' % len(result))
- return result
- <BLANKLINE>
- def __repr__(self):
- 'Return a nicely formatted representation string'
- return self.__class__.__name__ + '(x=%r, y=%r)' % self
- <BLANKLINE>
- def _asdict(self):
- 'Return a new OrderedDict which maps field names to their values'
- return OrderedDict(zip(self._fields, self))
- <BLANKLINE>
- __dict__ = property(_asdict)
- <BLANKLINE>
- def _replace(_self, **kwds):
- 'Return a new Point object replacing specified fields with new values'
- result = _self._make(map(kwds.pop, ('x', 'y'), _self))
- if kwds:
- raise ValueError('Got unexpected field names: %r' % list(kwds.keys()))
- return result
- <BLANKLINE>
- def __getnewargs__(self):
- 'Return self as a plain tuple. Used by copy and pickle.'
- return tuple(self)
- <BLANKLINE>
- x = _property(_itemgetter(0), doc='Alias for field number 0')
- y = _property(_itemgetter(1), doc='Alias for field number 1')
-
>>> p = Point(11, y=22) # instantiate with positional or keyword arguments
>>> p[0] + p[1] # indexable like the plain tuple (11, 22)
33
@@ -667,7 +749,7 @@ by the :mod:`csv` or :mod:`sqlite3` modules::
print(emp.name, emp.title)
In addition to the methods inherited from tuples, named tuples support
-three additional methods and one attribute. To prevent conflicts with
+three additional methods and two attributes. To prevent conflicts with
field names, the method and attribute names start with an underscore.
.. classmethod:: somenamedtuple._make(iterable)
@@ -705,6 +787,15 @@ field names, the method and attribute names start with an underscore.
>>> for partnum, record in inventory.items():
... inventory[partnum] = record._replace(price=newprices[partnum], timestamp=time.now())
+.. attribute:: somenamedtuple._source
+
+ A string with the pure Python source code used to create the named
+ tuple class. The source makes the named tuple self-documenting.
+ It can be printed, executed using :func:`exec`, or saved to a file
+ and imported.
+
+ .. versionadded:: 3.3
+
.. attribute:: somenamedtuple._fields
Tuple of strings listing the field names. Useful for introspection
@@ -753,7 +844,6 @@ a fixed-width print format:
The subclass shown above sets ``__slots__`` to an empty tuple. This helps
keep memory requirements low by preventing the creation of instance dictionaries.
-
Subclassing is not useful for adding new, stored fields. Instead, simply
create a new named tuple type from the :attr:`_fields` attribute:
@@ -765,6 +855,7 @@ customize a prototype instance:
>>> Account = namedtuple('Account', 'owner balance transaction_count')
>>> default_account = Account('<owner name>', 0.0, 0)
>>> johns_account = default_account._replace(owner='John')
+ >>> janes_account = default_account._replace(owner='Jane')
Enumerated constants can be implemented with named tuples, but it is simpler
and more efficient to use a simple class declaration:
@@ -983,161 +1074,3 @@ attribute.
be an instance of :class:`bytes`, :class:`str`, :class:`UserString` (or a
subclass) or an arbitrary sequence which can be converted into a string using
the built-in :func:`str` function.
-
-
-.. _collections-abstract-base-classes:
-
-ABCs - abstract base classes
-----------------------------
-
-The collections module offers the following :term:`ABCs <abstract base class>`:
-
-========================= ===================== ====================== ====================================================
-ABC Inherits from Abstract Methods Mixin Methods
-========================= ===================== ====================== ====================================================
-:class:`Container` ``__contains__``
-:class:`Hashable` ``__hash__``
-:class:`Iterable` ``__iter__``
-:class:`Iterator` :class:`Iterable` ``__next__`` ``__iter__``
-:class:`Sized` ``__len__``
-:class:`Callable` ``__call__``
-
-:class:`Sequence` :class:`Sized`, ``__getitem__`` ``__contains__``, ``__iter__``, ``__reversed__``,
- :class:`Iterable`, ``index``, and ``count``
- :class:`Container`
-
-:class:`MutableSequence` :class:`Sequence` ``__setitem__``, Inherited :class:`Sequence` methods and
- ``__delitem__``, ``append``, ``reverse``, ``extend``, ``pop``,
- ``insert`` ``remove``, and ``__iadd__``
-
-:class:`Set` :class:`Sized`, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``,
- :class:`Iterable`, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``,
- :class:`Container` ``__sub__``, ``__xor__``, and ``isdisjoint``
-
-:class:`MutableSet` :class:`Set` ``add``, Inherited :class:`Set` methods and
- ``discard`` ``clear``, ``pop``, ``remove``, ``__ior__``,
- ``__iand__``, ``__ixor__``, and ``__isub__``
-
-:class:`Mapping` :class:`Sized`, ``__getitem__`` ``__contains__``, ``keys``, ``items``, ``values``,
- :class:`Iterable`, ``get``, ``__eq__``, and ``__ne__``
- :class:`Container`
-
-:class:`MutableMapping` :class:`Mapping` ``__setitem__``, Inherited :class:`Mapping` methods and
- ``__delitem__`` ``pop``, ``popitem``, ``clear``, ``update``,
- and ``setdefault``
-
-
-:class:`MappingView` :class:`Sized` ``__len__``
-:class:`ItemsView` :class:`MappingView`, ``__contains__``,
- :class:`Set` ``__iter__``
-:class:`KeysView` :class:`MappingView`, ``__contains__``,
- :class:`Set` ``__iter__``
-:class:`ValuesView` :class:`MappingView` ``__contains__``, ``__iter__``
-========================= ===================== ====================== ====================================================
-
-
-.. class:: Container
- Hashable
- Sized
- Callable
-
- ABCs for classes that provide respectively the methods :meth:`__contains__`,
- :meth:`__hash__`, :meth:`__len__`, and :meth:`__call__`.
-
-.. class:: Iterable
-
- ABC for classes that provide the :meth:`__iter__` method.
- See also the definition of :term:`iterable`.
-
-.. class:: Iterator
-
- ABC for classes that provide the :meth:`__iter__` and :meth:`next` methods.
- See also the definition of :term:`iterator`.
-
-.. class:: Sequence
- MutableSequence
-
- ABCs for read-only and mutable :term:`sequences <sequence>`.
-
-.. class:: Set
- MutableSet
-
- ABCs for read-only and mutable sets.
-
-.. class:: Mapping
- MutableMapping
-
- ABCs for read-only and mutable :term:`mappings <mapping>`.
-
-.. class:: MappingView
- ItemsView
- KeysView
- ValuesView
-
- ABCs for mapping, items, keys, and values :term:`views <view>`.
-
-
-These ABCs allow us to ask classes or instances if they provide
-particular functionality, for example::
-
- size = None
- if isinstance(myvar, collections.Sized):
- size = len(myvar)
-
-Several of the ABCs are also useful as mixins that make it easier to develop
-classes supporting container APIs. For example, to write a class supporting
-the full :class:`Set` API, it only necessary to supply the three underlying
-abstract methods: :meth:`__contains__`, :meth:`__iter__`, and :meth:`__len__`.
-The ABC supplies the remaining methods such as :meth:`__and__` and
-:meth:`isdisjoint` ::
-
- class ListBasedSet(collections.Set):
- ''' Alternate set implementation favoring space over speed
- and not requiring the set elements to be hashable. '''
- def __init__(self, iterable):
- self.elements = lst = []
- for value in iterable:
- if value not in lst:
- lst.append(value)
- def __iter__(self):
- return iter(self.elements)
- def __contains__(self, value):
- return value in self.elements
- def __len__(self):
- return len(self.elements)
-
- s1 = ListBasedSet('abcdef')
- s2 = ListBasedSet('defghi')
- overlap = s1 & s2 # The __and__() method is supported automatically
-
-Notes on using :class:`Set` and :class:`MutableSet` as a mixin:
-
-(1)
- Since some set operations create new sets, the default mixin methods need
- a way to create new instances from an iterable. The class constructor is
- assumed to have a signature in the form ``ClassName(iterable)``.
- That assumption is factored-out to an internal classmethod called
- :meth:`_from_iterable` which calls ``cls(iterable)`` to produce a new set.
- If the :class:`Set` mixin is being used in a class with a different
- constructor signature, you will need to override :meth:`_from_iterable`
- with a classmethod that can construct new instances from
- an iterable argument.
-
-(2)
- To override the comparisons (presumably for speed, as the
- semantics are fixed), redefine :meth:`__le__` and
- then the other operations will automatically follow suit.
-
-(3)
- The :class:`Set` mixin provides a :meth:`_hash` method to compute a hash value
- for the set; however, :meth:`__hash__` is not defined because not all sets
- are hashable or immutable. To add set hashabilty using mixins,
- inherit from both :meth:`Set` and :meth:`Hashable`, then define
- ``__hash__ = Set._hash``.
-
-.. seealso::
-
- * `OrderedSet recipe <http://code.activestate.com/recipes/576694/>`_ for an
- example built on :class:`MutableSet`.
-
- * For more about ABCs, see the :mod:`abc` module and :pep:`3119`.
diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst
index 29ffc0d..a88f10c 100644
--- a/Doc/library/concurrent.futures.rst
+++ b/Doc/library/concurrent.futures.rst
@@ -169,6 +169,12 @@ to a :class:`ProcessPoolExecutor` will result in deadlock.
of at most *max_workers* processes. If *max_workers* is ``None`` or not
given, it will default to the number of processors on the machine.
+ .. versionchanged:: 3.3
+ When one of the worker processes terminates abruptly, a
+ :exc:`BrokenProcessPool` error is now raised. Previously, behaviour
+ was undefined but operations on the executor or its futures would often
+ freeze or deadlock.
+
.. _processpoolexecutor-example:
@@ -369,3 +375,16 @@ Module Functions
:pep:`3148` -- futures - execute computations asynchronously
The proposal which described this feature for inclusion in the Python
standard library.
+
+
+Exception classes
+-----------------
+
+.. exception:: BrokenProcessPool
+
+ Derived from :exc:`RuntimeError`, this exception class is raised when
+ one of the workers of a :class:`ProcessPoolExecutor` has terminated
+ in a non-clean fashion (for example, if it was killed from the outside).
+
+ .. versionadded:: 3.3
+
diff --git a/Doc/library/crypt.rst b/Doc/library/crypt.rst
index 0be571e..1ba2ed3 100644
--- a/Doc/library/crypt.rst
+++ b/Doc/library/crypt.rst
@@ -15,9 +15,9 @@
This module implements an interface to the :manpage:`crypt(3)` routine, which is
a one-way hash function based upon a modified DES algorithm; see the Unix man
-page for further details. Possible uses include allowing Python scripts to
-accept typed passwords from the user, or attempting to crack Unix passwords with
-a dictionary.
+page for further details. Possible uses include storing hashed passwords
+so you can check passwords without storing the actual password, or attempting
+to crack Unix passwords with a dictionary.
.. index:: single: crypt(3)
@@ -26,15 +26,74 @@ the :manpage:`crypt(3)` routine in the running system. Therefore, any
extensions available on the current implementation will also be available on
this module.
+Hashing Methods
+---------------
-.. function:: crypt(word, salt)
+.. versionadded:: 3.3
+
+The :mod:`crypt` module defines the list of hashing methods (not all methods
+are available on all platforms):
+
+.. data:: METHOD_SHA512
+
+ A Modular Crypt Format method with 16 character salt and 86 character
+ hash. This is the strongest method.
+
+.. data:: METHOD_SHA256
+
+ Another Modular Crypt Format method with 16 character salt and 43
+ character hash.
+
+.. data:: METHOD_MD5
+
+ Another Modular Crypt Format method with 8 character salt and 22
+ character hash.
+
+.. data:: METHOD_CRYPT
+
+ The traditional method with a 2 character salt and 13 characters of
+ hash. This is the weakest method.
+
+
+Module Attributes
+-----------------
+
+.. versionadded:: 3.3
+
+.. attribute:: methods
+
+ A list of available password hashing algorithms, as
+ ``crypt.METHOD_*`` objects. This list is sorted from strongest to
+ weakest, and is guaranteed to have at least ``crypt.METHOD_CRYPT``.
+
+
+Module Functions
+----------------
+
+The :mod:`crypt` module defines the following functions:
+
+.. function:: crypt(word, salt=None)
*word* will usually be a user's password as typed at a prompt or in a graphical
- interface. *salt* is usually a random two-character string which will be used
- to perturb the DES algorithm in one of 4096 ways. The characters in *salt* must
- be in the set ``[./a-zA-Z0-9]``. Returns the hashed password as a string, which
- will be composed of characters from the same alphabet as the salt (the first two
- characters represent the salt itself).
+ interface. The optional *salt* is either a string as returned from
+ :func:`mksalt`, one of the ``crypt.METHOD_*`` values (though not all
+ may be available on all platforms), or a full encrypted password
+ including salt, as returned by this function. If *salt* is not
+ provided, the strongest method will be used (as returned by
+ :func:`methods`.
+
+ Checking a password is usually done by passing the plain-text password
+ as *word* and the full results of a previous :func:`crypt` call,
+ which should be the same as the results of this call.
+
+ *salt* (either a random 2 or 16 character string, possibly prefixed with
+ ``$digit$`` to indicate the method) which will be used to perturb the
+ encryption algorithm. The characters in *salt* must be in the set
+ ``[./a-zA-Z0-9]``, with the exception of Modular Crypt Format which
+ prefixes a ``$digit$``.
+
+ Returns the hashed password as a string, which will be composed of
+ characters from the same alphabet as the salt.
.. index:: single: crypt(3)
@@ -42,18 +101,48 @@ this module.
different sizes in the *salt*, it is recommended to use the full crypted
password as salt when checking for a password.
+ .. versionchanged:: 3.3
+ Accept ``crypt.METHOD_*`` values in addition to strings for *salt*.
+
+
+.. function:: mksalt(method=None)
+
+ Return a randomly generated salt of the specified method. If no
+ *method* is given, the strongest method available as returned by
+ :func:`methods` is used.
+
+ The return value is a string either of 2 characters in length for
+ ``crypt.METHOD_CRYPT``, or 19 characters starting with ``$digit$`` and
+ 16 random characters from the set ``[./a-zA-Z0-9]``, suitable for
+ passing as the *salt* argument to :func:`crypt`.
+
+ .. versionadded:: 3.3
+
+Examples
+--------
+
A simple example illustrating typical use::
- import crypt, getpass, pwd
+ import pwd
+ import crypt
+ import getpass
def login():
- username = input('Python login:')
+ username = input('Python login: ')
cryptedpasswd = pwd.getpwnam(username)[1]
if cryptedpasswd:
if cryptedpasswd == 'x' or cryptedpasswd == '*':
- raise "Sorry, currently no support for shadow passwords"
+ raise ValueError('no support for shadow passwords')
cleartext = getpass.getpass()
return crypt.crypt(cleartext, cryptedpasswd) == cryptedpasswd
else:
- return 1
+ return True
+
+To generate a hash of a password using the strongest available method and
+check it against the original::
+
+ import crypt
+ hashed = crypt.crypt(plaintext)
+ if hashed != crypt.crypt(plaintext, hashed):
+ raise ValueError("hashed version doesn't validate against original")
diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst
index edbe726..ec0dfcc 100644
--- a/Doc/library/csv.rst
+++ b/Doc/library/csv.rst
@@ -11,15 +11,15 @@
pair: data; tabular
The so-called CSV (Comma Separated Values) format is the most common import and
-export format for spreadsheets and databases. There is no "CSV standard", so
-the format is operationally defined by the many applications which read and
-write it. The lack of a standard means that subtle differences often exist in
-the data produced and consumed by different applications. These differences can
-make it annoying to process CSV files from multiple sources. Still, while the
-delimiters and quoting characters vary, the overall format is similar enough
-that it is possible to write a single module which can efficiently manipulate
-such data, hiding the details of reading and writing the data from the
-programmer.
+export format for spreadsheets and databases. CSV format was used for many
+years prior to attempts to describe the format in a standardized way in
+:rfc:`4180`. The lack of a well-defined standard means that subtle differences
+often exist in the data produced and consumed by different applications. These
+differences can make it annoying to process CSV files from multiple sources.
+Still, while the delimiters and quoting characters vary, the overall format is
+similar enough that it is possible to write a single module which can
+efficiently manipulate such data, hiding the details of reading and writing the
+data from the programmer.
The :mod:`csv` module implements classes to read and write tabular data in CSV
format. It allows programmers to say, "write this data in the format preferred
diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst
index fce5913..8a22e94 100644
--- a/Doc/library/ctypes.rst
+++ b/Doc/library/ctypes.rst
@@ -39,9 +39,14 @@ loads libraries which export functions using the standard ``cdecl`` calling
convention, while *windll* libraries call functions using the ``stdcall``
calling convention. *oledll* also uses the ``stdcall`` calling convention, and
assumes the functions return a Windows :c:type:`HRESULT` error code. The error
-code is used to automatically raise a :class:`WindowsError` exception when the
+code is used to automatically raise a :class:`OSError` exception when the
function call fails.
+.. versionchanged:: 3.3
+ Windows errors used to raise :exc:`WindowsError`, which is now an alias
+ of :exc:`OSError`.
+
+
Here are some examples for Windows. Note that ``msvcrt`` is the MS standard C
library containing most standard C functions, and uses the cdecl calling
convention::
@@ -189,7 +194,7 @@ argument values::
>>> windll.kernel32.GetModuleHandleA(32) # doctest: +WINDOWS
Traceback (most recent call last):
File "<stdin>", line 1, in ?
- WindowsError: exception: access violation reading 0x00000020
+ OSError: exception: access violation reading 0x00000020
>>>
There are, however, enough ways to crash Python with :mod:`ctypes`, so you
@@ -491,7 +496,7 @@ useful to check for error return values and automatically raise an exception::
Traceback (most recent call last):
File "<stdin>", line 1, in ?
File "<stdin>", line 3, in ValidHandle
- WindowsError: [Errno 126] The specified module could not be found.
+ OSError: [Errno 126] The specified module could not be found.
>>>
``WinError`` is a function which will call Windows ``FormatMessage()`` api to
@@ -1345,7 +1350,10 @@ way is to instantiate one of the following classes:
assumed to return the windows specific :class:`HRESULT` code. :class:`HRESULT`
values contain information specifying whether the function call failed or
succeeded, together with additional error code. If the return value signals a
- failure, an :class:`WindowsError` is automatically raised.
+ failure, an :class:`OSError` is automatically raised.
+
+ .. versionchanged:: 3.3
+ :exc:`WindowsError` used to be raised.
.. class:: WinDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False)
@@ -1966,11 +1974,14 @@ Utility functions
.. function:: WinError(code=None, descr=None)
Windows only: this function is probably the worst-named thing in ctypes. It
- creates an instance of WindowsError. If *code* is not specified,
+ creates an instance of OSError. If *code* is not specified,
``GetLastError`` is called to determine the error code. If *descr* is not
specified, :func:`FormatError` is called to get a textual description of the
error.
+ .. versionchanged:: 3.3
+ An instance of :exc:`WindowsError` used to be created.
+
.. function:: wstring_at(address, size=-1)
diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst
index f31b9c5..d6a35c3 100644
--- a/Doc/library/curses.rst
+++ b/Doc/library/curses.rst
@@ -598,6 +598,17 @@ The module :mod:`curses` defines the following functions:
Only one *ch* can be pushed before :meth:`getch` is called.
+.. function:: unget_wch(ch)
+
+ Push *ch* so the next :meth:`get_wch` will return it.
+
+ .. note::
+
+ Only one *ch* can be pushed before :meth:`get_wch` is called.
+
+ .. versionadded:: 3.3
+
+
.. function:: ungetmouse(id, x, y, z, bstate)
Push a :const:`KEY_MOUSE` event onto the input queue, associating the given
@@ -846,6 +857,14 @@ the following methods:
until a key is pressed.
+.. method:: window.get_wch([y, x])
+
+ Get a wide character. Like :meth:`getch`, but the integer returned is the
+ Unicode code point for the key pressed, so it can be passed to :func:`chr`.
+
+ .. versionadded:: 3.3
+
+
.. method:: window.getkey([y, x])
Get a character, returning a string instead of an integer, as :meth:`getch`
diff --git a/Doc/library/datatypes.rst b/Doc/library/datatypes.rst
index 6b4a71a..8e33c1f 100644
--- a/Doc/library/datatypes.rst
+++ b/Doc/library/datatypes.rst
@@ -21,6 +21,7 @@ The following modules are documented in this chapter:
datetime.rst
calendar.rst
collections.rst
+ collections.abc.rst
heapq.rst
bisect.rst
array.rst
diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst
index 1f4cfba..a6c1997 100644
--- a/Doc/library/datetime.rst
+++ b/Doc/library/datetime.rst
@@ -721,6 +721,22 @@ Other constructors, all class methods:
It's common for this to be restricted to years in 1970 through 2038. See also
:meth:`fromtimestamp`.
+ On the POSIX compliant platforms, ``utcfromtimestamp(timestamp)``
+ is equivalent to the following expression::
+
+ datetime(1970, 1, 1) + timedelta(seconds=timestamp)
+
+ There is no method to obtain the timestamp from a :class:`datetime`
+ instance, but POSIX timestamp corresponding to a :class:`datetime`
+ instance ``dt`` can be easily calculated as follows. For a naive
+ ``dt``::
+
+ timestamp = (dt - datetime(1970, 1, 1)) / timedelta(seconds=1)
+
+ And for an aware ``dt``::
+
+ timestamp = (dt - datetime(1970, 1, 1, tzinfo=timezone.utc)) / timedelta(seconds=1)
+
.. classmethod:: datetime.fromordinal(ordinal)
@@ -1564,11 +1580,12 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)).
:class:`timezone` Objects
--------------------------
-A :class:`timezone` object represents a timezone that is defined by a
-fixed offset from UTC. Note that objects of this class cannot be used
-to represent timezone information in the locations where different
-offsets are used in different days of the year or where historical
-changes have been made to civil time.
+The :class:`timezone` class is a subclass of :class:`tzinfo`, each
+instance of which represents a timezone defined by a fixed offset from
+UTC. Note that objects of this class cannot be used to represent
+timezone information in the locations where different offsets are used
+in different days of the year or where historical changes have been
+made to civil time.
.. class:: timezone(offset[, name])
@@ -1737,8 +1754,7 @@ format codes.
| | decimal number [00,99]. | |
+-----------+--------------------------------+-------+
| ``%Y`` | Year with century as a decimal | \(5) |
-| | number [0001,9999] (strptime), | |
-| | [1000,9999] (strftime). | |
+| | number [0001,9999]. | |
+-----------+--------------------------------+-------+
| ``%z`` | UTC offset in the form +HHMM | \(6) |
| | or -HHMM (empty string if the | |
@@ -1772,10 +1788,7 @@ Notes:
calculations when the day of the week and the year are specified.
(5)
- For technical reasons, :meth:`strftime` method does not support
- dates before year 1000: ``t.strftime(format)`` will raise a
- :exc:`ValueError` when ``t.year < 1000`` even if ``format`` does
- not contain ``%Y`` directive. The :meth:`strptime` method can
+ The :meth:`strptime` method can
parse years in the full [1, 9999] range, but years < 1000 must be
zero-filled to 4-digit width.
@@ -1783,6 +1796,10 @@ Notes:
In previous versions, :meth:`strftime` method was restricted to
years >= 1900.
+ .. versionchanged:: 3.3
+ In version 3.2, :meth:`strftime` method was restricted to
+ years >= 1000.
+
(6)
For example, if :meth:`utcoffset` returns ``timedelta(hours=-3, minutes=-30)``,
``%z`` is replaced with the string ``'-0330'``.
diff --git a/Doc/library/debug.rst b/Doc/library/debug.rst
index b2ee4fa..c69fb1c 100644
--- a/Doc/library/debug.rst
+++ b/Doc/library/debug.rst
@@ -10,7 +10,8 @@ allowing you to identify bottlenecks in your programs.
.. toctree::
bdb.rst
+ faulthandler.rst
pdb.rst
profile.rst
timeit.rst
- trace.rst \ No newline at end of file
+ trace.rst
diff --git a/Doc/library/depgraph-output.png b/Doc/library/depgraph-output.png
new file mode 100644
index 0000000..960bb1b
--- /dev/null
+++ b/Doc/library/depgraph-output.png
Binary files differ
diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst
index bdc37b3..836e240 100644
--- a/Doc/library/difflib.rst
+++ b/Doc/library/difflib.rst
@@ -752,8 +752,8 @@ It is also contained in the Python source distribution, as
# we're passing these as arguments to the diff function
fromdate = time.ctime(os.stat(fromfile).st_mtime)
todate = time.ctime(os.stat(tofile).st_mtime)
- fromlines = open(fromfile, 'U').readlines()
- tolines = open(tofile, 'U').readlines()
+ with open(fromlines) as fromf, open(tofile) as tof:
+ fromlines, tolines = list(fromf), list(tof)
if options.u:
diff = difflib.unified_diff(fromlines, tolines, fromfile, tofile,
diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst
index 79cc583..cb429c8 100644
--- a/Doc/library/dis.rst
+++ b/Doc/library/dis.rst
@@ -171,11 +171,6 @@ The Python compiler currently generates the following bytecode instructions.
**General instructions**
-.. opcode:: STOP_CODE
-
- Indicates end-of-code to the compiler, not used by the interpreter.
-
-
.. opcode:: NOP
Do nothing code. Used as a placeholder by the bytecode optimizer.
diff --git a/Doc/library/distutils.rst b/Doc/library/distutils.rst
index 238b79d..53a69ae 100644
--- a/Doc/library/distutils.rst
+++ b/Doc/library/distutils.rst
@@ -12,18 +12,26 @@ additional modules into a Python installation. The new modules may be either
100%-pure Python, or may be extension modules written in C, or may be
collections of Python packages which include modules coded in both Python and C.
-This package is discussed in two separate chapters:
+.. deprecated:: 3.3
+ :mod:`packaging` replaces Distutils. See :ref:`packaging-index` and
+ :ref:`packaging-install-index`.
+User documentation and API reference are provided in another document:
+
.. seealso::
:ref:`distutils-index`
The manual for developers and packagers of Python modules. This describes
how to prepare :mod:`distutils`\ -based packages so that they may be
- easily installed into an existing Python installation.
+ easily installed into an existing Python installation. If also contains
+ instructions for end-users wanting to install a distutils-based package,
+ :ref:`install-index`.
+
+
+.. trick to silence a Sphinx warning
- :ref:`install-index`
- An "administrators" manual which includes information on installing
- modules into an existing Python installation. You do not need to be a
- Python programmer to read this manual.
+.. toctree::
+ :hidden:
+ ../distutils/index
diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst
index 85b32fe..847d7e4 100644
--- a/Doc/library/email.generator.rst
+++ b/Doc/library/email.generator.rst
@@ -32,7 +32,8 @@ Here are the public methods of the :class:`Generator` class, imported from the
:mod:`email.generator` module:
-.. class:: Generator(outfp, mangle_from_=True, maxheaderlen=78)
+.. class:: Generator(outfp, mangle_from_=True, maxheaderlen=78, *, \
+ policy=policy.default)
The constructor for the :class:`Generator` class takes a :term:`file-like object`
called *outfp* for an argument. *outfp* must support the :meth:`write` method
@@ -53,10 +54,16 @@ Here are the public methods of the :class:`Generator` class, imported from the
:class:`~email.header.Header` class. Set to zero to disable header wrapping.
The default is 78, as recommended (but not required) by :rfc:`2822`.
+ The *policy* keyword specifies a :mod:`~email.policy` object that controls a
+ number of aspects of the generator's operation. The default policy
+ maintains backward compatibility.
+
+ .. versionchanged:: 3.3 Added the *policy* keyword.
+
The other public :class:`Generator` methods are:
- .. method:: flatten(msg, unixfrom=False, linesep='\\n')
+ .. method:: flatten(msg, unixfrom=False, linesep=None)
Print the textual representation of the message object structure rooted at
*msg* to the output file specified when the :class:`Generator` instance
@@ -72,12 +79,13 @@ Here are the public methods of the :class:`Generator` class, imported from the
Note that for subparts, no envelope header is ever printed.
Optional *linesep* specifies the line separator character used to
- terminate lines in the output. It defaults to ``\n`` because that is
- the most useful value for Python application code (other library packages
- expect ``\n`` separated lines). ``linesep=\r\n`` can be used to
- generate output with RFC-compliant line separators.
+ terminate lines in the output. If specified it overrides the value
+ specified by the ``Generator``\'s ``policy``.
- Messages parsed with a Bytes parser that have a
+ Because strings cannot represent non-ASCII bytes, ``Generator`` ignores
+ the value of the :attr:`~email.policy.Policy.must_be_7bit`
+ :mod:`~email.policy` setting and operates as if it were set ``True``.
+ This means that messages parsed with a Bytes parser that have a
:mailheader:`Content-Transfer-Encoding` of 8bit will be converted to a
use a 7bit Content-Transfer-Encoding. Non-ASCII bytes in the headers
will be :rfc:`2047` encoded with a charset of `unknown-8bit`.
@@ -103,7 +111,8 @@ As a convenience, see the :class:`~email.message.Message` methods
formatted string representation of a message object. For more detail, see
:mod:`email.message`.
-.. class:: BytesGenerator(outfp, mangle_from_=True, maxheaderlen=78)
+.. class:: BytesGenerator(outfp, mangle_from_=True, maxheaderlen=78, *, \
+ policy=policy.default)
The constructor for the :class:`BytesGenerator` class takes a binary
:term:`file-like object` called *outfp* for an argument. *outfp* must
@@ -125,19 +134,31 @@ formatted string representation of a message object. For more detail, see
wrapping. The default is 78, as recommended (but not required) by
:rfc:`2822`.
+ The *policy* keyword specifies a :mod:`~email.policy` object that controls a
+ number of aspects of the generator's operation. The default policy
+ maintains backward compatibility.
+
+ .. versionchanged:: 3.3 Added the *policy* keyword.
+
The other public :class:`BytesGenerator` methods are:
- .. method:: flatten(msg, unixfrom=False, linesep='\n')
+ .. method:: flatten(msg, unixfrom=False, linesep=None)
Print the textual representation of the message object structure rooted
at *msg* to the output file specified when the :class:`BytesGenerator`
instance was created. Subparts are visited depth-first and the resulting
- text will be properly MIME encoded. If the input that created the *msg*
- contained bytes with the high bit set and those bytes have not been
- modified, they will be copied faithfully to the output, even if doing so
- is not strictly RFC compliant. (To produce strictly RFC compliant
- output, use the :class:`Generator` class.)
+ text will be properly MIME encoded. If the :mod:`~email.policy` option
+ :attr:`~email.policy.Policy.must_be_7bit` is ``False`` (the default),
+ then any bytes with the high bit set in the original parsed message that
+ have not been modified will be copied faithfully to the output. If
+ ``must_be_7bit`` is true, the bytes will be converted as needed using an
+ ASCII content-transfer-encoding. In particular, RFC-invalid non-ASCII
+ bytes in headers will be encoded using the MIME ``unknown-8bit``
+ character set, thus rendering them RFC-compliant.
+
+ .. XXX: There should be a complementary option that just does the RFC
+ compliance transformation but leaves CTE 8bit parts alone.
Messages parsed with a Bytes parser that have a
:mailheader:`Content-Transfer-Encoding` of 8bit will be reconstructed
@@ -152,10 +173,8 @@ formatted string representation of a message object. For more detail, see
Note that for subparts, no envelope header is ever printed.
Optional *linesep* specifies the line separator character used to
- terminate lines in the output. It defaults to ``\n`` because that is
- the most useful value for Python application code (other library packages
- expect ``\n`` separated lines). ``linesep=\r\n`` can be used to
- generate output with RFC-compliant line separators.
+ terminate lines in the output. If specified it overrides the value
+ specified by the ``Generator``\ 's ``policy``.
.. method:: clone(fp)
diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst
index 77a0b69..e70b429 100644
--- a/Doc/library/email.parser.rst
+++ b/Doc/library/email.parser.rst
@@ -58,12 +58,18 @@ list of defects that it can find.
Here is the API for the :class:`FeedParser`:
-.. class:: FeedParser(_factory=email.message.Message)
+.. class:: FeedParser(_factory=email.message.Message, *, policy=policy.default)
Create a :class:`FeedParser` instance. Optional *_factory* is a no-argument
callable that will be called whenever a new message object is needed. It
defaults to the :class:`email.message.Message` class.
+ The *policy* keyword specifies a :mod:`~email.policy` object that controls a
+ number of aspects of the parser's operation. The default policy maintains
+ backward compatibility.
+
+ .. versionchanged:: 3.3 Added the *policy* keyword.
+
.. method:: feed(data)
Feed the :class:`FeedParser` some more data. *data* should be a string
@@ -94,15 +100,17 @@ Parser class API
The :class:`Parser` class, imported from the :mod:`email.parser` module,
provides an API that can be used to parse a message when the complete contents
of the message are available in a string or file. The :mod:`email.parser`
-module also provides a second class, called :class:`HeaderParser` which can be
-used if you're only interested in the headers of the message.
-:class:`HeaderParser` can be much faster in these situations, since it does not
-attempt to parse the message body, instead setting the payload to the raw body
-as a string. :class:`HeaderParser` has the same API as the :class:`Parser`
-class.
+module also provides header-only parsers, called :class:`HeaderParser` and
+:class:`BytesHeaderParser`, which can be used if you're only interested in the
+headers of the message. :class:`HeaderParser` and :class:`BytesHeaderParser`
+can be much faster in these situations, since they do not attempt to parse the
+message body, instead setting the payload to the raw body as a string. They
+have the same API as the :class:`Parser` and :class:`BytesParser` classes.
+.. versionadded:: 3.3 BytesHeaderParser
-.. class:: Parser(_class=email.message.Message, strict=None)
+
+.. class:: Parser(_class=email.message.Message, *, policy=policy.default)
The constructor for the :class:`Parser` class takes an optional argument
*_class*. This must be a callable factory (such as a function or a class), and
@@ -110,13 +118,13 @@ class.
:class:`~email.message.Message` (see :mod:`email.message`). The factory will
be called without arguments.
- The optional *strict* flag is ignored.
+ The *policy* keyword specifies a :mod:`~email.policy` object that controls a
+ number of aspects of the parser's operation. The default policy maintains
+ backward compatibility.
- .. deprecated:: 2.4
- Because the :class:`Parser` class is a backward compatible API wrapper
- around the new-in-Python 2.4 :class:`FeedParser`, *all* parsing is
- effectively non-strict. You should simply stop passing a *strict* flag to
- the :class:`Parser` constructor.
+ .. versionchanged:: 3.3
+ Removed the *strict* argument that was deprecated in 2.4. Added the
+ *policy* keyword.
The other public :class:`Parser` methods are:
@@ -147,12 +155,18 @@ class.
the entire contents of the file.
-.. class:: BytesParser(_class=email.message.Message, strict=None)
+.. class:: BytesParser(_class=email.message.Message, *, policy=policy.default)
This class is exactly parallel to :class:`Parser`, but handles bytes input.
The *_class* and *strict* arguments are interpreted in the same way as for
- the :class:`Parser` constructor. *strict* is supported only to make porting
- code easier; it is deprecated.
+ the :class:`Parser` constructor.
+
+ The *policy* keyword specifies a :mod:`~email.policy` object that
+ controls a number of aspects of the parser's operation. The default
+ policy maintains backward compatibility.
+
+ .. versionchanged:: 3.3
+ Removed the *strict* argument. Added the *policy* keyword.
.. method:: parse(fp, headeronly=False)
@@ -190,34 +204,48 @@ in the top-level :mod:`email` package namespace.
.. currentmodule:: email
-.. function:: message_from_string(s, _class=email.message.Message, strict=None)
+.. function:: message_from_string(s, _class=email.message.Message, *, \
+ policy=policy.default)
Return a message object structure from a string. This is exactly equivalent to
- ``Parser().parsestr(s)``. Optional *_class* and *strict* are interpreted as
+ ``Parser().parsestr(s)``. *_class* and *policy* are interpreted as
with the :class:`Parser` class constructor.
-.. function:: message_from_bytes(s, _class=email.message.Message, strict=None)
+ .. versionchanged:: 3.3
+ Removed the *strict* argument. Added the *policy* keyword.
+
+.. function:: message_from_bytes(s, _class=email.message.Message, *, \
+ policy=policy.default)
Return a message object structure from a byte string. This is exactly
equivalent to ``BytesParser().parsebytes(s)``. Optional *_class* and
*strict* are interpreted as with the :class:`Parser` class constructor.
.. versionadded:: 3.2
+ .. versionchanged:: 3.3
+ Removed the *strict* argument. Added the *policy* keyword.
-.. function:: message_from_file(fp, _class=email.message.Message, strict=None)
+.. function:: message_from_file(fp, _class=email.message.Message, *, \
+ policy=policy.default)
Return a message object structure tree from an open :term:`file object`.
- This is exactly equivalent to ``Parser().parse(fp)``. Optional *_class*
- and *strict* are interpreted as with the :class:`Parser` class constructor.
+ This is exactly equivalent to ``Parser().parse(fp)``. *_class*
+ and *policy* are interpreted as with the :class:`Parser` class constructor.
+
+ .. versionchanged::
+ Removed the *strict* argument. Added the *policy* keyword.
-.. function:: message_from_binary_file(fp, _class=email.message.Message, strict=None)
+.. function:: message_from_binary_file(fp, _class=email.message.Message, *, \
+ policy=policy.default)
Return a message object structure tree from an open binary :term:`file
object`. This is exactly equivalent to ``BytesParser().parse(fp)``.
- Optional *_class* and *strict* are interpreted as with the :class:`Parser`
+ *_class* and *policy* are interpreted as with the :class:`Parser`
class constructor.
.. versionadded:: 3.2
+ .. versionchanged:: 3.3
+ Removed the *strict* argument. Added the *policy* keyword.
Here's an example of how you might use this at an interactive Python prompt::
diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst
new file mode 100644
index 0000000..a77e510
--- /dev/null
+++ b/Doc/library/email.policy.rst
@@ -0,0 +1,182 @@
+:mod:`email`: Policy Objects
+----------------------------
+
+.. module:: email.policy
+ :synopsis: Controlling the parsing and generating of messages
+
+.. versionadded:: 3.3
+
+
+The :mod:`email` package's prime focus is the handling of email messages as
+described by the various email and MIME RFCs. However, the general format of
+email messages (a block of header fields each consisting of a name followed by
+a colon followed by a value, the whole block followed by a blank line and an
+arbitrary 'body'), is a format that has found utility outside of the realm of
+email. Some of these uses conform fairly closely to the main RFCs, some do
+not. And even when working with email, there are times when it is desirable to
+break strict compliance with the RFCs.
+
+Policy objects give the email package the flexibility to handle all these
+disparate use cases.
+
+A :class:`Policy` object encapsulates a set of attributes and methods that
+control the behavior of various components of the email package during use.
+:class:`Policy` instances can be passed to various classes and methods in the
+email package to alter the default behavior. The settable values and their
+defaults are described below. The :mod:`policy` module also provides some
+pre-created :class:`Policy` instances. In addition to a :const:`default`
+instance, there are instances tailored for certain applications. For example
+there is an :const:`SMTP` :class:`Policy` with defaults appropriate for
+generating output to be sent to an SMTP server. These are listed `below
+<Policy Instances>`.
+
+In general an application will only need to deal with setting the policy at the
+input and output boundaries. Once parsed, a message is represented by a
+:class:`~email.message.Message` object, which is designed to be independent of
+the format that the message has "on the wire" when it is received, transmitted,
+or displayed. Thus, a :class:`Policy` can be specified when parsing a message
+to create a :class:`~email.message.Message`, and again when turning the
+:class:`~email.message.Message` into some other representation. While often a
+program will use the same :class:`Policy` for both input and output, the two
+can be different.
+
+As an example, the following code could be used to read an email message from a
+file on disk and pass it to the system ``sendmail`` program on a Unix system::
+
+ >>> from email import msg_from_binary_file
+ >>> from email.generator import BytesGenerator
+ >>> import email.policy
+ >>> from subprocess import Popen, PIPE
+ >>> with open('mymsg.txt', 'b') as f:
+ ... Msg = msg_from_binary_file(f, policy=email.policy.mbox)
+ >>> p = Popen(['sendmail', msg['To'][0].address], stdin=PIPE)
+ >>> g = BytesGenerator(p.stdin, policy=email.policy.SMTP)
+ >>> g.flatten(msg)
+ >>> p.stdin.close()
+ >>> rc = p.wait()
+
+Some email package methods accept a *policy* keyword argument, allowing the
+policy to be overridden for that method. For example, the following code uses
+the :meth:`email.message.Message.as_string` method of the *msg* object from the
+previous example and re-write it to a file using the native line separators for
+the platform on which it is running::
+
+ >>> import os
+ >>> mypolicy = email.policy.Policy(linesep=os.linesep)
+ >>> with open('converted.txt', 'wb') as f:
+ ... f.write(msg.as_string(policy=mypolicy))
+
+Policy instances are immutable, but they can be cloned, accepting the same
+keyword arguments as the class constructor and returning a new :class:`Policy`
+instance that is a copy of the original but with the specified attributes
+values changed. For example, the following creates an SMTP policy that will
+raise any defects detected as errors::
+
+ >>> strict_SMTP = email.policy.SMTP.clone(raise_on_defect=True)
+
+Policy objects can also be combined using the addition operator, producing a
+policy object whose settings are a combination of the non-default values of the
+summed objects::
+
+ >>> strict_SMTP = email.policy.SMTP + email.policy.strict
+
+This operation is not commutative; that is, the order in which the objects are
+added matters. To illustrate::
+
+ >>> Policy = email.policy.Policy
+ >>> apolicy = Policy(max_line_length=100) + Policy(max_line_length=80)
+ >>> apolicy.max_line_length
+ 80
+ >>> apolicy = Policy(max_line_length=80) + Policy(max_line_length=100)
+ >>> apolicy.max_line_length
+ 100
+
+
+.. class:: Policy(**kw)
+
+ The valid constructor keyword arguments are any of the attributes listed
+ below.
+
+ .. attribute:: max_line_length
+
+ The maximum length of any line in the serialized output, not counting the
+ end of line character(s). Default is 78, per :rfc:`5322`. A value of
+ ``0`` or :const:`None` indicates that no line wrapping should be
+ done at all.
+
+ .. attribute:: linesep
+
+ The string to be used to terminate lines in serialized output. The
+ default is ``\n`` because that's the internal end-of-line discipline used
+ by Python, though ``\r\n`` is required by the RFCs. See `Policy
+ Instances`_ for policies that use an RFC conformant linesep. Setting it
+ to :attr:`os.linesep` may also be useful.
+
+ .. attribute:: must_be_7bit
+
+ If ``True``, data output by a bytes generator is limited to ASCII
+ characters. If :const:`False` (the default), then bytes with the high
+ bit set are preserved and/or allowed in certain contexts (for example,
+ where possible a content transfer encoding of ``8bit`` will be used).
+ String generators act as if ``must_be_7bit`` is ``True`` regardless of
+ the policy in effect, since a string cannot represent non-ASCII bytes.
+
+ .. attribute:: raise_on_defect
+
+ If :const:`True`, any defects encountered will be raised as errors. If
+ :const:`False` (the default), defects will be passed to the
+ :meth:`register_defect` method.
+
+ :mod:`Policy` object also have the following methods:
+
+ .. method:: handle_defect(obj, defect)
+
+ *obj* is the object on which to register the defect. *defect* should be
+ an instance of a subclass of :class:`~email.errors.Defect`.
+ If :attr:`raise_on_defect`
+ is ``True`` the defect is raised as an exception. Otherwise *obj* and
+ *defect* are passed to :meth:`register_defect`. This method is intended
+ to be called by parsers when they encounter defects, and will not be
+ called by code that uses the email library unless that code is
+ implementing an alternate parser.
+
+ .. method:: register_defect(obj, defect)
+
+ *obj* is the object on which to register the defect. *defect* should be
+ a subclass of :class:`~email.errors.Defect`. This method is part of the
+ public API so that custom ``Policy`` subclasses can implement alternate
+ handling of defects. The default implementation calls the ``append``
+ method of the ``defects`` attribute of *obj*.
+
+ .. method:: clone(obj, *kw)
+
+ Return a new :class:`Policy` instance whose attributes have the same
+ values as the current instance, except where those attributes are
+ given new values by the keyword arguments.
+
+
+Policy Instances
+^^^^^^^^^^^^^^^^
+
+The following instances of :class:`Policy` provide defaults suitable for
+specific common application domains.
+
+.. data:: default
+
+ An instance of :class:`Policy` with all defaults unchanged.
+
+.. data:: SMTP
+
+ Output serialized from a message will conform to the email and SMTP
+ RFCs. The only changed attribute is :attr:`linesep`, which is set to
+ ``\r\n``.
+
+.. data:: HTTP
+
+ Suitable for use when serializing headers for use in HTTP traffic.
+ :attr:`linesep` is set to ``\r\n``, and :attr:`max_line_length` is set to
+ :const:`None` (unlimited).
+
+.. data:: strict
+
+ :attr:`raise_on_defect` is set to :const:`True`.
diff --git a/Doc/library/email.rst b/Doc/library/email.rst
index 4530b95..fc206f4 100644
--- a/Doc/library/email.rst
+++ b/Doc/library/email.rst
@@ -51,6 +51,7 @@ Contents of the :mod:`email` package documentation:
email.message.rst
email.parser.rst
email.generator.rst
+ email.policy.rst
email.mime.rst
email.header.rst
email.charset.rst
diff --git a/Doc/library/email.util.rst b/Doc/library/email.util.rst
index f7b777a..2f9ef89 100644
--- a/Doc/library/email.util.rst
+++ b/Doc/library/email.util.rst
@@ -29,13 +29,20 @@ There are several useful utilities provided in the :mod:`email.utils` module:
fails, in which case a 2-tuple of ``('', '')`` is returned.
-.. function:: formataddr(pair)
+.. function:: formataddr(pair, charset='utf-8')
The inverse of :meth:`parseaddr`, this takes a 2-tuple of the form ``(realname,
email_address)`` and returns the string value suitable for a :mailheader:`To` or
:mailheader:`Cc` header. If the first element of *pair* is false, then the
second element is returned unmodified.
+ Optional *charset* is the character set that will be used in the :rfc:`2047`
+ encoding of the ``realname`` if the ``realname`` contains non-ASCII
+ characters. Can be an instance of :class:`str` or a
+ :class:`~email.charset.Charset`. Defaults to ``utf-8``.
+
+ .. versionchanged: 3.3 added the *charset* option
+
.. function:: getaddresses(fieldvalues)
@@ -74,6 +81,20 @@ There are several useful utilities provided in the :mod:`email.utils` module:
indexes 6, 7, and 8 of the result tuple are not usable.
+.. function:: parsedate_to_datetime(date)
+
+ The inverse of :func:`format_datetime`. Performs the same function as
+ :func:`parsedate`, but on success returns a :mod:`~datetime.datetime`. If
+ the input date has a timezone of ``-0000``, the ``datetime`` will be a naive
+ ``datetime``, and if the date is conforming to the RFCs it will represent a
+ time in UTC but with no indication of the actual source timezone of the
+ message the date comes from. If the input date has any other valid timezone
+ offset, the ``datetime`` will be an aware ``datetime`` with the
+ corresponding a :class:`~datetime.timezone` :class:`~datetime.tzinfo`.
+
+ .. versionadded:: 3.3
+
+
.. function:: mktime_tz(tuple)
Turn a 10-tuple as returned by :func:`parsedate_tz` into a UTC timestamp. It
@@ -105,6 +126,20 @@ There are several useful utilities provided in the :mod:`email.utils` module:
``False``. The default is ``False``.
+.. function:: format_datetime(dt, usegmt=False)
+
+ Like ``formatdate``, but the input is a :mod:`datetime` instance. If it is
+ a naive datetime, it is assumed to be "UTC with no information about the
+ source timezone", and the conventional ``-0000`` is used for the timezone.
+ If it is an aware ``datetime``, then the numeric timezone offset is used.
+ If it is an aware timezone with offset zero, then *usegmt* may be set to
+ ``True``, in which case the string ``GMT`` is used instead of the numeric
+ timezone offset. This provides a way to generate standards conformant HTTP
+ date headers.
+
+ .. versionadded:: 3.3
+
+
.. function:: make_msgid(idstring=None, domain=None)
Returns a string suitable for an :rfc:`2822`\ -compliant
diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst
index 528febd..ca8e4d8 100644
--- a/Doc/library/exceptions.rst
+++ b/Doc/library/exceptions.rst
@@ -34,6 +34,10 @@ programmers are encouraged to at least derive new exceptions from the
defining exceptions is available in the Python Tutorial under
:ref:`tut-userexceptions`.
+
+Base classes
+------------
+
The following exceptions are used mostly as base classes for other exceptions.
.. exception:: BaseException
@@ -90,27 +94,8 @@ The following exceptions are used mostly as base classes for other exceptions.
can be raised directly by :func:`codecs.lookup`.
-.. exception:: EnvironmentError
-
- The base class for exceptions that can occur outside the Python system:
- :exc:`IOError`, :exc:`OSError`. When exceptions of this type are created with a
- 2-tuple, the first item is available on the instance's :attr:`errno` attribute
- (it is assumed to be an error number), and the second item is available on the
- :attr:`strerror` attribute (it is usually the associated error message). The
- tuple itself is also available on the :attr:`args` attribute.
-
- When an :exc:`EnvironmentError` exception is instantiated with a 3-tuple, the
- first two items are available as above, while the third item is available on the
- :attr:`filename` attribute. However, for backwards compatibility, the
- :attr:`args` attribute contains only a 2-tuple of the first two constructor
- arguments.
-
- The :attr:`filename` attribute is ``None`` when this exception is created with
- other than 3 arguments. The :attr:`errno` and :attr:`strerror` attributes are
- also ``None`` when the instance was created with other than 2 or 3 arguments.
- In this last case, :attr:`args` contains the verbatim constructor arguments as a
- tuple.
-
+Concrete exceptions
+-------------------
The following exceptions are the exceptions that are usually raised.
@@ -151,16 +136,6 @@ The following exceptions are the exceptions that are usually raised.
it is technically not an error.
-.. exception:: IOError
-
- Raised when an I/O operation (such as the built-in :func:`print` or
- :func:`open` functions or a method of a :term:`file object`) fails for an
- I/O-related reason, e.g., "file not found" or "disk full".
-
- This class is derived from :exc:`EnvironmentError`. See the discussion above
- for more information on exception instance attributes.
-
-
.. exception:: ImportError
Raised when an :keyword:`import` statement fails to find the module definition
@@ -221,17 +196,30 @@ The following exceptions are the exceptions that are usually raised.
.. index:: module: errno
- This exception is derived from :exc:`EnvironmentError`. It is raised when a
- function returns a system-related error (not for illegal argument types or
- other incidental errors). The :attr:`errno` attribute is a numeric error
- code from :c:data:`errno`, and the :attr:`strerror` attribute is the
- corresponding string, as would be printed by the C function :c:func:`perror`.
- See the module :mod:`errno`, which contains names for the error codes defined
- by the underlying operating system.
+ This exception is raised when a system function returns a system-related
+ error, including I/O failures such as "file not found" or "disk full"
+ (not for illegal argument types or other incidental errors). Often a
+ subclass of :exc:`OSError` will actually be raised as described in
+ `OS exceptions`_ below. The :attr:`errno` attribute is a numeric error
+ code from the C variable :c:data:`errno`.
+
+ Under Windows, the :attr:`winerror` attribute gives you the native
+ Windows error code. The :attr:`errno` attribute is then an approximate
+ translation, in POSIX terms, of that native error code.
- For exceptions that involve a file system path (such as :func:`chdir` or
- :func:`unlink`), the exception instance will contain a third attribute,
- :attr:`filename`, which is the file name passed to the function.
+ Under all platforms, the :attr:`strerror` attribute is the corresponding
+ error message as provided by the operating system (as formatted by the C
+ functions :c:func:`perror` under POSIX, and :c:func:`FormatMessage`
+ Windows).
+
+ For exceptions that involve a file system path (such as :func:`open` or
+ :func:`os.unlink`), the exception instance will contain an additional
+ attribute, :attr:`filename`, which is the file name passed to the function.
+
+ .. versionchanged:: 3.3
+ :exc:`EnvironmentError`, :exc:`IOError`, :exc:`WindowsError`,
+ :exc:`VMSError`, :exc:`socket.error`, :exc:`select.error` and
+ :exc:`mmap.error` have been merged into :exc:`OSError`.
.. exception:: OverflowError
@@ -372,27 +360,142 @@ The following exceptions are the exceptions that are usually raised.
more precise exception such as :exc:`IndexError`.
-.. exception:: VMSError
+.. exception:: ZeroDivisionError
- Only available on VMS. Raised when a VMS-specific error occurs.
+ Raised when the second argument of a division or modulo operation is zero. The
+ associated value is a string indicating the type of the operands and the
+ operation.
+
+
+The following exceptions are kept for compatibility with previous versions;
+starting from Python 3.3, they are aliases of :exc:`OSError`.
+
+.. exception:: EnvironmentError
+
+.. exception:: IOError
+.. exception:: VMSError
+
+ Only available on VMS.
.. exception:: WindowsError
- Raised when a Windows-specific error occurs or when the error number does not
- correspond to an :c:data:`errno` value. The :attr:`winerror` and
- :attr:`strerror` values are created from the return values of the
- :c:func:`GetLastError` and :c:func:`FormatMessage` functions from the Windows
- Platform API. The :attr:`errno` value maps the :attr:`winerror` value to
- corresponding ``errno.h`` values. This is a subclass of :exc:`OSError`.
+ Only available on Windows.
-.. exception:: ZeroDivisionError
+OS exceptions
+^^^^^^^^^^^^^
+
+The following exceptions are subclasses of :exc:`OSError`, they get raised
+depending on the system error code.
+
+.. exception:: BlockingIOError
+
+ Raised when an operation would block on an object (e.g. socket) set
+ for non-blocking operation.
+ Corresponds to :c:data:`errno` ``EAGAIN``, ``EALREADY``,
+ ``EWOULDBLOCK`` and ``EINPROGRESS``.
+
+ In addition to those of :exc:`OSError`, :exc:`BlockingIOError` can have
+ one more attribute:
+
+ .. attribute:: characters_written
+
+ An integer containing the number of characters written to the stream
+ before it blocked. This attribute is available when using the
+ buffered I/O classes from the :mod:`io` module.
+
+.. exception:: ChildProcessError
+
+ Raised when an operation on a child process failed.
+ Corresponds to :c:data:`errno` ``ECHILD``.
+
+.. exception:: ConnectionError
+
+ A base class for connection-related issues. Subclasses are
+ :exc:`BrokenPipeError`, :exc:`ConnectionAbortedError`,
+ :exc:`ConnectionRefusedError` and :exc:`ConnectionResetError`.
+
+ .. exception:: BrokenPipeError
+
+ A subclass of :exc:`ConnectionError`, raised when trying to write on a
+ pipe while the other end has been closed, or trying to write on a socket
+ which has been shutdown for writing.
+ Corresponds to :c:data:`errno` ``EPIPE`` and ``ESHUTDOWN``.
+
+ .. exception:: ConnectionAbortedError
+
+ A subclass of :exc:`ConnectionError`, raised when a connection attempt
+ is aborted by the peer.
+ Corresponds to :c:data:`errno` ``ECONNABORTED``.
+
+ .. exception:: ConnectionRefusedError
+
+ A subclass of :exc:`ConnectionError`, raised when a connection attempt
+ is refused by the peer.
+ Corresponds to :c:data:`errno` ``ECONNREFUSED``.
+
+ .. exception:: ConnectionResetError
+
+ A subclass of :exc:`ConnectionError`, raised when a connection is
+ reset by the peer.
+ Corresponds to :c:data:`errno` ``ECONNRESET``.
+
+.. exception:: FileExistsError
+
+ Raised when trying to create a file or directory which already exists.
+ Corresponds to :c:data:`errno` ``EEXIST``.
+
+.. exception:: FileNotFoundError
+
+ Raised when a file or directory is requested but doesn't exist.
+ Corresponds to :c:data:`errno` ``ENOENT``.
+
+.. exception:: InterruptedError
+
+ Raised when a system call is interrupted by an incoming signal.
+ Corresponds to :c:data:`errno` ``EEINTR``.
+
+.. exception:: IsADirectoryError
+
+ Raised when a file operation (such as :func:`os.remove`) is requested
+ on a directory.
+ Corresponds to :c:data:`errno` ``EISDIR``.
+
+.. exception:: NotADirectoryError
+
+ Raised when a directory operation (such as :func:`os.listdir`) is requested
+ on something which is not a directory.
+ Corresponds to :c:data:`errno` ``ENOTDIR``.
+
+.. exception:: PermissionError
+
+ Raised when trying to run an operation without the adequate access
+ rights - for example filesystem permissions.
+ Corresponds to :c:data:`errno` ``EACCES`` and ``EPERM``.
+
+.. exception:: ProcessLookupError
+
+ Raised when a given process doesn't exist.
+ Corresponds to :c:data:`errno` ``ESRCH``.
+
+.. exception:: TimeoutError
+
+ Raised when a system function timed out at the system level.
+ Corresponds to :c:data:`errno` ``ETIMEDOUT``.
+
+.. versionadded:: 3.3
+ All the above :exc:`OSError` subclasses were added.
+
+
+.. seealso::
+
+ :pep:`3151` - Reworking the OS and IO exception hierarchy
+ PEP written and implemented by Antoine Pitrou.
- Raised when the second argument of a division or modulo operation is zero. The
- associated value is a string indicating the type of the operands and the
- operation.
+Warnings
+--------
The following exceptions are used as warning categories; see the :mod:`warnings`
module for more information.
diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst
new file mode 100644
index 0000000..c9b9546
--- /dev/null
+++ b/Doc/library/faulthandler.rst
@@ -0,0 +1,136 @@
+:mod:`faulthandler` --- Dump the Python traceback
+=================================================
+
+.. module:: faulthandler
+ :synopsis: Dump the Python traceback.
+
+This module contains functions to dump Python tracebacks explicitly, on a fault,
+after a timeout, or on a user signal. Call :func:`faulthandler.enable` to
+install fault handlers for the :const:`SIGSEGV`, :const:`SIGFPE`,
+:const:`SIGABRT`, :const:`SIGBUS`, and :const:`SIGILL` signals. You can also
+enable them at startup by setting the :envvar:`PYTHONFAULTHANDLER` environment
+variable or by using :option:`-X` ``faulthandler`` command line option.
+
+The fault handler is compatible with system fault handlers like Apport or the
+Windows fault handler. The module uses an alternative stack for signal handlers
+if the :c:func:`sigaltstack` function is available. This allows it to dump the
+traceback even on a stack overflow.
+
+The fault handler is called on catastrophic cases and therefore can only use
+signal-safe functions (e.g. it cannot allocate memory on the heap). Because of
+this limitation traceback dumping is minimal compared to normal Python
+tracebacks:
+
+* Only ASCII is supported. The ``backslashreplace`` error handler is used on
+ encoding.
+* Each string is limited to 100 characters.
+* Only the filename, the function name and the line number are
+ displayed. (no source code)
+* It is limited to 100 frames and 100 threads.
+
+By default, the Python traceback is written to :data:`sys.stderr`. To see
+tracebacks, applications must be run in the terminal. A log file can
+alternatively be passed to :func:`faulthandler.enable`.
+
+The module is implemented in C, so tracebacks can be dumped on a crash or when
+Python is deadlocked.
+
+.. versionadded:: 3.3
+
+
+Dump the traceback
+------------------
+
+.. function:: dump_traceback(file=sys.stderr, all_threads=True)
+
+ Dump the tracebacks of all threads into *file*. If *all_threads* is
+ ``False``, dump only the current thread.
+
+
+Fault handler state
+-------------------
+
+.. function:: enable(file=sys.stderr, all_threads=True)
+
+ Enable the fault handler: install handlers for the :const:`SIGSEGV`,
+ :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and :const:`SIGILL`
+ signals to dump the Python traceback. If *all_threads* is ``True``,
+ produce tracebacks for every running thread. Otherwise, dump only the current
+ thread.
+
+.. function:: disable()
+
+ Disable the fault handler: uninstall the signal handlers installed by
+ :func:`enable`.
+
+.. function:: is_enabled()
+
+ Check if the fault handler is enabled.
+
+
+Dump the tracebacks after a timeout
+-----------------------------------
+
+.. function:: dump_tracebacks_later(timeout, repeat=False, file=sys.stderr, exit=False)
+
+ Dump the tracebacks of all threads, after a timeout of *timeout* seconds, or
+ every *timeout* seconds if *repeat* is ``True``. If *exit* is ``True``, call
+ :c:func:`_exit` with status=1 after dumping the tracebacks. (Note
+ :c:func:`_exit` exits the process immediately, which means it doesn't do any
+ cleanup like flushing file buffers.) If the function is called twice, the new
+ call replaces previous parameters and resets the timeout. The timer has a
+ sub-second resolution.
+
+ This function is implemented using a watchdog thread and therefore is not
+ available if Python is compiled with threads disabled.
+
+.. function:: cancel_dump_tracebacks_later()
+
+ Cancel the last call to :func:`dump_tracebacks_later`.
+
+
+Dump the traceback on a user signal
+-----------------------------------
+
+.. function:: register(signum, file=sys.stderr, all_threads=True, chain=False)
+
+ Register a user signal: install a handler for the *signum* signal to dump
+ the traceback of all threads, or of the current thread if *all_threads* is
+ ``False``, into *file*. Call the previous handler if chain is ``True``.
+
+ Not available on Windows.
+
+.. function:: unregister(signum)
+
+ Unregister a user signal: uninstall the handler of the *signum* signal
+ installed by :func:`register`. Return ``True`` if the signal was registered,
+ ``False`` otherwise.
+
+ Not available on Windows.
+
+
+File descriptor issue
+---------------------
+
+:func:`enable`, :func:`dump_tracebacks_later` and :func:`register` keep the
+file descriptor of their *file* argument. If the file is closed and its file
+descriptor is reused by a new file, or if :func:`os.dup2` is used to replace
+the file descriptor, the traceback will be written into a different file. Call
+these functions again each time that the file is replaced.
+
+
+Example
+-------
+
+Example of a segmentation fault on Linux: ::
+
+ $ python -q -X faulthandler
+ >>> import ctypes
+ >>> ctypes.string_at(0)
+ Fatal Python error: Segmentation fault
+
+ Current thread 0x00007fb899f39700:
+ File "/home/python/cpython/Lib/ctypes/__init__.py", line 486 in string_at
+ File "<stdin>", line 1 in <module>
+ Segmentation fault
+
diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst
index 6192400..9a9cdc1 100644
--- a/Doc/library/fcntl.rst
+++ b/Doc/library/fcntl.rst
@@ -19,6 +19,11 @@ argument. This can be an integer file descriptor, such as returned by
``sys.stdin.fileno()``, or a :class:`io.IOBase` object, such as ``sys.stdin``
itself, which provides a :meth:`fileno` that returns a genuine file descriptor.
+.. versionchanged:: 3.3
+ Operations in this module used to raise a :exc:`IOError` where they now
+ raise a :exc:`OSError`.
+
+
The module defines the following functions:
@@ -40,7 +45,7 @@ The module defines the following functions:
larger than 1024 bytes, this is most likely to result in a segmentation
violation or a more subtle data corruption.
- If the :c:func:`fcntl` fails, an :exc:`IOError` is raised.
+ If the :c:func:`fcntl` fails, an :exc:`OSError` is raised.
.. function:: ioctl(fd, op[, arg[, mutate_flag]])
@@ -107,7 +112,7 @@ The module defines the following functions:
When *operation* is :const:`LOCK_SH` or :const:`LOCK_EX`, it can also be
bitwise ORed with :const:`LOCK_NB` to avoid blocking on lock acquisition.
If :const:`LOCK_NB` is used and the lock cannot be acquired, an
- :exc:`IOError` will be raised and the exception will have an *errno*
+ :exc:`OSError` will be raised and the exception will have an *errno*
attribute set to :const:`EACCES` or :const:`EAGAIN` (depending on the
operating system; for portability, check for both values). On at least some
systems, :const:`LOCK_EX` can only be used if the file descriptor refers to a
diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst
index ac44311..f8ec436 100644
--- a/Doc/library/fileinput.rst
+++ b/Doc/library/fileinput.rst
@@ -28,7 +28,10 @@ as the first argument to :func:`.input`. A single file name is also allowed.
All files are opened in text mode by default, but you can override this by
specifying the *mode* parameter in the call to :func:`.input` or
:class:`FileInput`. If an I/O error occurs during opening or reading a file,
-:exc:`IOError` is raised.
+:exc:`OSError` is raised.
+
+.. versionchanged:: 3.3
+ :exc:`IOError` used to be raised; it is now an alias of :exc:`OSError`.
If ``sys.stdin`` is used more than once, the second and further use will return
no lines, except perhaps for interactive use, or if it has been explicitly reset
@@ -168,10 +171,6 @@ and the backup file remains around; by default, the extension is ``'.bak'`` and
it is deleted when the output file is closed. In-place filtering is disabled
when standard input is read.
-.. note::
-
- The current implementation does not work for MS-DOS 8+3 filesystems.
-
The two following opening hooks are provided by this module:
diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst
index 5bbef4f..3cc295a 100644
--- a/Doc/library/ftplib.rst
+++ b/Doc/library/ftplib.rst
@@ -40,7 +40,7 @@ Here's a sample session using the :mod:`ftplib` module::
The module defines the following items:
-.. class:: FTP(host='', user='', passwd='', acct=''[, timeout])
+.. class:: FTP(host='', user='', passwd='', acct='', timeout=None, source_address=None)
Return a new instance of the :class:`FTP` class. When *host* is given, the
method call ``connect(host)`` is made. When *user* is given, additionally
@@ -48,7 +48,8 @@ The module defines the following items:
*acct* default to the empty string when not given). The optional *timeout*
parameter specifies a timeout in seconds for blocking operations like the
connection attempt (if is not specified, the global default timeout setting
- will be used).
+ will be used). *source_address* is a 2-tuple ``(host, port)`` for the socket
+ to bind to as its source address before connecting.
:class:`FTP` class supports the :keyword:`with` statement. Here is a sample
on how using it:
@@ -68,8 +69,11 @@ The module defines the following items:
.. versionchanged:: 3.2
Support for the :keyword:`with` statement was added.
+ .. versionchanged:: 3.3
+ *source_address* parameter was added.
-.. class:: FTP_TLS(host='', user='', passwd='', acct='', [keyfile[, certfile[, context[, timeout]]]])
+
+.. class:: FTP_TLS(host='', user='', passwd='', acct='', keyfile=None, certfile=None, context=None, timeout=None, source_address=None)
A :class:`FTP` subclass which adds TLS support to FTP as described in
:rfc:`4217`.
@@ -80,10 +84,15 @@ The module defines the following items:
private key and certificate chain file name for the SSL connection.
*context* parameter is a :class:`ssl.SSLContext` object which allows
bundling SSL configuration options, certificates and private keys into a
- single (potentially long-lived) structure.
+ single (potentially long-lived) structure. *source_address* is a 2-tuple
+ ``(host, port)`` for the socket to bind to as its source address before
+ connecting.
.. versionadded:: 3.2
+ .. versionchanged:: 3.3
+ *source_address* parameter was added.
+
Here's a sample session using the :class:`FTP_TLS` class:
>>> from ftplib import FTP_TLS
@@ -135,8 +144,7 @@ The module defines the following items:
The set of all exceptions (as a tuple) that methods of :class:`FTP`
instances may raise as a result of problems with the FTP connection (as
opposed to programming errors made by the caller). This set includes the
- four exceptions listed above as well as :exc:`socket.error` and
- :exc:`IOError`.
+ four exceptions listed above as well as :exc:`OSError`.
.. seealso::
@@ -174,7 +182,7 @@ followed by ``lines`` for the text version or ``binary`` for the binary version.
debugging output, logging each line sent and received on the control connection.
-.. method:: FTP.connect(host='', port=0[, timeout])
+.. method:: FTP.connect(host='', port=0, timeout=None, source_address=None)
Connect to the given host and port. The default port number is ``21``, as
specified by the FTP protocol specification. It is rarely needed to specify a
@@ -182,10 +190,14 @@ followed by ``lines`` for the text version or ``binary`` for the binary version.
instance; it should not be called at all if a host was given when the instance
was created. All other methods can only be used after a connection has been
made.
-
The optional *timeout* parameter specifies a timeout in seconds for the
connection attempt. If no *timeout* is passed, the global default timeout
setting will be used.
+ *source_address* is a 2-tuple ``(host, port)`` for the socket to bind to as
+ its source address before connecting.
+
+ .. versionchanged:: 3.3
+ *source_address* parameter was added.
.. method:: FTP.getwelcome()
@@ -241,13 +253,12 @@ followed by ``lines`` for the text version or ``binary`` for the binary version.
Retrieve a file or directory listing in ASCII transfer mode. *cmd* should be
an appropriate ``RETR`` command (see :meth:`retrbinary`) or a command such as
- ``LIST``, ``NLST`` or ``MLSD`` (usually just the string ``'LIST'``).
+ ``LIST`` or ``NLST`` (usually just the string ``'LIST'``).
``LIST`` retrieves a list of files and information about those files.
- ``NLST`` retrieves a list of file names. On some servers, ``MLSD`` retrieves
- a machine readable list of files and information about those files. The
- *callback* function is called for each line with a string argument containing
- the line with the trailing CRLF stripped. The default *callback* prints the
- line to ``sys.stdout``.
+ ``NLST`` retrieves a list of file names.
+ The *callback* function is called for each line with a string argument
+ containing the line with the trailing CRLF stripped. The default *callback*
+ prints the line to ``sys.stdout``.
.. method:: FTP.set_pasv(boolean)
@@ -307,6 +318,20 @@ followed by ``lines`` for the text version or ``binary`` for the binary version.
in :meth:`transfercmd`.
+.. method:: FTP.mlsd(path="", facts=[])
+
+ List a directory in a standardized format by using MLSD command
+ (:rfc:`3659`). If *path* is omitted the current directory is assumed.
+ *facts* is a list of strings representing the type of information desired
+ (e.g. ``["type", "size", "perm"]``). Return a generator object yielding a
+ tuple of two elements for every file found in path. First element is the
+ file name, the second one is a dictionary containing facts about the file
+ name. Content of this dictionary might be limited by the *facts* argument
+ but server is not guaranteed to return all requested facts.
+
+ .. versionadded:: 3.3
+
+
.. method:: FTP.nlst(argument[, ...])
Return a list of file names as returned by the ``NLST`` command. The
@@ -314,6 +339,8 @@ followed by ``lines`` for the text version or ``binary`` for the binary version.
directory). Multiple arguments can be used to pass non-standard options to
the ``NLST`` command.
+ .. deprecated:: 3.3 use :meth:`mlsd` instead.
+
.. method:: FTP.dir(argument[, ...])
@@ -324,6 +351,8 @@ followed by ``lines`` for the text version or ``binary`` for the binary version.
as a *callback* function as for :meth:`retrlines`; the default prints to
``sys.stdout``. This method returns ``None``.
+ .. deprecated:: 3.3 use :meth:`mlsd` instead.
+
.. method:: FTP.rename(fromname, toname)
@@ -396,6 +425,14 @@ FTP_TLS Objects
Set up secure control connection by using TLS or SSL, depending on what specified in :meth:`ssl_version` attribute.
+.. method:: FTP_TLS.ccc()
+
+ Revert control channel back to plaintext. This can be useful to take
+ advantage of firewalls that know how to handle NAT with non-secure FTP
+ without opening fixed ports.
+
+ .. versionadded:: 3.3
+
.. method:: FTP_TLS.prot_p()
Set up secure data connection.
diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index e5d4a99..f9af3d8 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -152,10 +152,6 @@ are always available. They are listed here in alphabetical order.
1,114,111 (0x10FFFF in base 16). :exc:`ValueError` will be raised if *i* is
outside that range.
- Note that on narrow Unicode builds, the result is a string of
- length two for *i* greater than 65,535 (0xFFFF in hexadecimal).
-
-
.. function:: classmethod(function)
@@ -780,10 +776,10 @@ are always available. They are listed here in alphabetical order.
:meth:`__index__` method that returns an integer.
-.. function:: open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True)
+.. function:: open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None)
Open *file* and return a corresponding stream. If the file cannot be opened,
- an :exc:`IOError` is raised.
+ an :exc:`OSError` is raised.
*file* is either a string or bytes object giving the pathname (absolute or
relative to the current working directory) of the file to be opened or
@@ -887,6 +883,15 @@ are always available. They are listed here in alphabetical order.
closed. If a filename is given *closefd* has no effect and must be ``True``
(the default).
+ A custom opener can be used by passing a callable as *opener*. The underlying
+ file descriptor for the file object is then obtained by calling *opener* with
+ (*file*, *flags*). *opener* must return an open file descriptor (passing
+ :mod:`os.open` as *opener* results in functionality similar to passing
+ ``None``).
+
+ .. versionchanged:: 3.3
+ The *opener* parameter was added.
+
The type of file object returned by the :func:`open` function depends on the
mode. When :func:`open` is used to open a file in a text mode (``'w'``,
``'r'``, ``'wt'``, ``'rt'``, etc.), it returns a subclass of
@@ -912,6 +917,9 @@ are always available. They are listed here in alphabetical order.
(where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`,
and :mod:`shutil`.
+ .. versionchanged:: 3.3
+ :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`.
+
.. XXX works for bytes too, but should it?
.. function:: ord(c)
@@ -921,9 +929,6 @@ are always available. They are listed here in alphabetical order.
point of that character. For example, ``ord('a')`` returns the integer ``97``
and ``ord('\u2020')`` returns ``8224``. This is the inverse of :func:`chr`.
- On wide Unicode builds, if the argument length is not one, a
- :exc:`TypeError` will be raised. On narrow Unicode builds, strings
- of length two are accepted when they form a UTF-16 surrogate pair.
.. function:: pow(x, y[, z])
@@ -1074,6 +1079,13 @@ are always available. They are listed here in alphabetical order.
>>> r[-1]
18
+ Testing range objects for equality with ``==`` and ``!=`` compares
+ them as sequences. That is, two range objects are considered equal if
+ they represent the same sequence of values. (Note that two range
+ objects that compare equal might have different :attr:`start`,
+ :attr:`stop` and :attr:`step` attributes, for example ``range(0) ==
+ range(2, 1, 3)`` or ``range(0, 3, 2) == range(0, 4, 2)``.)
+
Ranges containing absolute values larger than :data:`sys.maxsize` are permitted
but some features (such as :func:`len`) will raise :exc:`OverflowError`.
@@ -1083,6 +1095,11 @@ are always available. They are listed here in alphabetical order.
Test integers for membership in constant time instead of iterating
through all items.
+ .. versionchanged:: 3.3
+ Define '==' and '!=' to compare range objects based on the
+ sequence of values they define (instead of comparing based on
+ object identity).
+
.. function:: repr(object)
diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst
index 2316e80..4eaf54e 100644
--- a/Doc/library/functools.rst
+++ b/Doc/library/functools.rst
@@ -40,7 +40,7 @@ The :mod:`functools` module defines the following functions:
.. versionadded:: 3.2
-.. decorator:: lru_cache(maxsize=100)
+.. decorator:: lru_cache(maxsize=100, typed=False)
Decorator to wrap a function with a memoizing callable that saves up to the
*maxsize* most recent calls. It can save time when an expensive or I/O bound
@@ -52,6 +52,10 @@ The :mod:`functools` module defines the following functions:
If *maxsize* is set to None, the LRU feature is disabled and the cache
can grow without bound.
+ If *typed* is set to True, function arguments of different types will be
+ cached separately. For example, ``f(3)`` and ``f(3.0)`` will be treated
+ as distinct calls with distinct results.
+
To help measure the effectiveness of the cache and tune the *maxsize*
parameter, the wrapped function is instrumented with a :func:`cache_info`
function that returns a :term:`named tuple` showing *hits*, *misses*,
@@ -67,8 +71,8 @@ The :mod:`functools` module defines the following functions:
An `LRU (least recently used) cache
<http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used>`_ works
- best when more recent calls are the best predictors of upcoming calls (for
- example, the most popular articles on a news server tend to change daily).
+ best when the most recent calls are the best predictors of upcoming calls (for
+ example, the most popular articles on a news server tend to change each day).
The cache's size limit assures that the cache does not grow without bound on
long-running processes such as web servers.
@@ -111,6 +115,9 @@ The :mod:`functools` module defines the following functions:
.. versionadded:: 3.2
+ .. versionchanged:: 3.3
+ Added the *typed* option.
+
.. decorator:: total_ordering
Given a class defining one or more rich comparison ordering methods, this
diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst
index 0fa022c..825311b 100644
--- a/Doc/library/gettext.rst
+++ b/Doc/library/gettext.rst
@@ -185,10 +185,13 @@ class can also install themselves in the built-in namespace as the function
translation object from the cache; the actual instance data is still shared with
the cache.
- If no :file:`.mo` file is found, this function raises :exc:`IOError` if
+ If no :file:`.mo` file is found, this function raises :exc:`OSError` if
*fallback* is false (which is the default), and returns a
:class:`NullTranslations` instance if *fallback* is true.
+ .. versionchanged:: 3.3
+ :exc:`IOError` used to be raised instead of :exc:`OSError`.
+
.. function:: install(domain, localedir=None, codeset=None, names=None)
@@ -342,7 +345,7 @@ The entire set of key/value pairs are placed into a dictionary and set as the
If the :file:`.mo` file's magic number is invalid, or if other problems occur
while reading the file, instantiating a :class:`GNUTranslations` class can raise
-:exc:`IOError`.
+:exc:`OSError`.
The following methods are overridden from the base class implementation:
diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst
index 48a8694..a5e08c9 100644
--- a/Doc/library/gzip.rst
+++ b/Doc/library/gzip.rst
@@ -72,7 +72,7 @@ The module defines the following items:
:class:`GzipFile` supports the :class:`io.BufferedIOBase` interface,
including iteration and the :keyword:`with` statement. Only the
- :meth:`read1` and :meth:`truncate` methods aren't implemented.
+ :meth:`truncate` method isn't implemented.
:class:`GzipFile` also provides the following method:
@@ -94,6 +94,9 @@ The module defines the following items:
.. versionchanged:: 3.2
Support for unseekable files was added.
+ .. versionchanged:: 3.3
+ The :meth:`io.BufferedIOBase.read1` method is now implemented.
+
.. function:: open(filename, mode='rb', compresslevel=9)
diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst
index 9771496..1fe775f 100644
--- a/Doc/library/http.cookiejar.rst
+++ b/Doc/library/http.cookiejar.rst
@@ -40,7 +40,11 @@ The module defines the following exception:
.. exception:: LoadError
Instances of :class:`FileCookieJar` raise this exception on failure to load
- cookies from a file. :exc:`LoadError` is a subclass of :exc:`IOError`.
+ cookies from a file. :exc:`LoadError` is a subclass of :exc:`OSError`.
+
+ .. versionchanged:: 3.3
+ LoadError was made a subclass of :exc:`OSError` instead of
+ :exc:`IOError`.
The following classes are provided:
@@ -257,9 +261,12 @@ contained :class:`Cookie` objects.
Arguments are as for :meth:`save`.
The named file must be in the format understood by the class, or
- :exc:`LoadError` will be raised. Also, :exc:`IOError` may be raised, for
+ :exc:`LoadError` will be raised. Also, :exc:`OSError` may be raised, for
example if the file does not exist.
+ .. versionchanged:: 3.3
+ :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`.
+
.. method:: FileCookieJar.revert(filename=None, ignore_discard=False, ignore_expires=False)
diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst
index e3a3a10..d9aaa72 100644
--- a/Doc/library/http.server.rst
+++ b/Doc/library/http.server.rst
@@ -179,19 +179,29 @@ of which this module provides three different variants:
.. method:: send_response(code, message=None)
- Sends a response header and logs the accepted request. The HTTP response
- line is sent, followed by *Server* and *Date* headers. The values for
- these two headers are picked up from the :meth:`version_string` and
- :meth:`date_time_string` methods, respectively.
+ Adds a response header to the headers buffer and logs the accepted
+ request. The HTTP response line is written to the internal buffer,
+ followed by *Server* and *Date* headers. The values for these two headers
+ are picked up from the :meth:`version_string` and
+ :meth:`date_time_string` methods, respectively. If the server does not
+ intend to send any other headers using the :meth:`send_header` method,
+ then :meth:`send_response` should be followed by a :meth:`end_headers`
+ call.
+
+ .. versionchanged:: 3.3
+ Headers are stored to an internal buffer and :meth:`end_headers`
+ needs to be called explicitly.
+
.. method:: send_header(keyword, value)
- Stores the HTTP header to an internal buffer which will be written to the
- output stream when :meth:`end_headers` method is invoked.
- *keyword* should specify the header keyword, with *value*
- specifying its value.
+ Adds the HTTP header to an internal buffer which will be written to the
+ output stream when either :meth:`end_headers` or :meth:`flush_headers` is
+ invoked. *keyword* should specify the header keyword, with *value*
+ specifying its value. Note that, after the send_header calls are done,
+ :meth:`end_headers` MUST BE called in order to complete the operation.
- .. versionchanged:: 3.2 Storing the headers in an internal buffer
+ .. versionchanged:: 3.2 Headers are stored in an internal buffer.
.. method:: send_response_only(code, message=None)
@@ -205,10 +215,19 @@ of which this module provides three different variants:
.. method:: end_headers()
- Write the buffered HTTP headers to the output stream and send a blank
- line, indicating the end of the HTTP headers in the response.
+ Adds a blank line
+ (indicating the end of the HTTP headers in the response)
+ to the headers buffer and calls :meth:`flush_headers()`.
+
+ .. versionchanged:: 3.2
+ The buffered headers are written to the output stream.
+
+ .. method:: flush_headers()
+
+ Finally send the headers to the output stream and flush the internal
+ headers buffer.
- .. versionchanged:: 3.2 Writing the buffered headers to the output stream.
+ .. versionadded:: 3.3
.. method:: log_request(code='-', size='-')
@@ -299,7 +318,7 @@ of which this module provides three different variants:
response if the :func:`listdir` fails.
If the request was mapped to a file, it is opened and the contents are
- returned. Any :exc:`IOError` exception in opening the requested file is
+ returned. Any :exc:`OSError` exception in opening the requested file is
mapped to a ``404``, ``'File not found'`` error. Otherwise, the content
type is guessed by calling the :meth:`guess_type` method, which in turn
uses the *extensions_map* variable.
diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst
index 1d92fe5..592e4b0 100644
--- a/Doc/library/imaplib.rst
+++ b/Doc/library/imaplib.rst
@@ -64,14 +64,21 @@ Three exceptions are defined as attributes of the :class:`IMAP4` class:
There's also a subclass for secure connections:
-.. class:: IMAP4_SSL(host='', port=IMAP4_SSL_PORT, keyfile=None, certfile=None)
+.. class:: IMAP4_SSL(host='', port=IMAP4_SSL_PORT, keyfile=None, certfile=None, ssl_context=None)
This is a subclass derived from :class:`IMAP4` that connects over an SSL
encrypted socket (to use this class you need a socket module that was compiled
with SSL support). If *host* is not specified, ``''`` (the local host) is used.
If *port* is omitted, the standard IMAP4-over-SSL port (993) is used. *keyfile*
and *certfile* are also optional - they can contain a PEM formatted private key
- and certificate chain file for the SSL connection.
+ and certificate chain file for the SSL connection. *ssl_context* parameter is a
+ :class:`ssl.SSLContext` object which allows bundling SSL configuration
+ options, certificates and private keys into a single (potentially long-lived)
+ structure. Note that the *keyfile*/*certfile* parameters are mutually exclusive with *ssl_context*,
+ a :class:`ValueError` is thrown if *keyfile*/*certfile* is provided along with *ssl_context*.
+
+ .. versionchanged:: 3.3
+ *ssl_context* parameter added.
The second subclass allows for connections created by a child process:
diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst
index d127ce8..ac6ae99 100644
--- a/Doc/library/inspect.rst
+++ b/Doc/library/inspect.rst
@@ -355,17 +355,25 @@ Retrieving source code
argument may be a module, class, method, function, traceback, frame, or code
object. The source code is returned as a list of the lines corresponding to the
object and the line number indicates where in the original source file the first
- line of code was found. An :exc:`IOError` is raised if the source code cannot
+ line of code was found. An :exc:`OSError` is raised if the source code cannot
be retrieved.
+ .. versionchanged:: 3.3
+ :exc:`OSError` is raised instead of :exc:`IOError`, now an alias of the
+ former.
+
.. function:: getsource(object)
Return the text of the source code for an object. The argument may be a module,
class, method, function, traceback, frame, or code object. The source code is
- returned as a single string. An :exc:`IOError` is raised if the source code
+ returned as a single string. An :exc:`OSError` is raised if the source code
cannot be retrieved.
+ .. versionchanged:: 3.3
+ :exc:`OSError` is raised instead of :exc:`IOError`, now an alias of the
+ former.
+
.. function:: cleandoc(doc)
diff --git a/Doc/library/io.rst b/Doc/library/io.rst
index 4da6195..1da7e4c 100644
--- a/Doc/library/io.rst
+++ b/Doc/library/io.rst
@@ -33,6 +33,10 @@ giving a :class:`str` object to the ``write()`` method of a binary stream
will raise a ``TypeError``. So will giving a :class:`bytes` object to the
``write()`` method of a text stream.
+.. versionchanged:: 3.3
+ Operations defined in this module used to raise :exc:`IOError`, which is
+ now an alias of :exc:`OSError`.
+
Text I/O
^^^^^^^^
@@ -109,21 +113,13 @@ High-level Module Interface
.. exception:: BlockingIOError
- Error raised when blocking would occur on a non-blocking stream. It inherits
- :exc:`IOError`.
-
- In addition to those of :exc:`IOError`, :exc:`BlockingIOError` has one
- attribute:
-
- .. attribute:: characters_written
-
- An integer containing the number of characters written to the stream
- before it blocked.
+ This is a compatibility alias for the builtin :exc:`BlockingIOError`
+ exception.
.. exception:: UnsupportedOperation
- An exception inheriting :exc:`IOError` and :exc:`ValueError` that is raised
+ An exception inheriting :exc:`OSError` and :exc:`ValueError` that is raised
when an unsupported operation is called on a stream.
@@ -202,8 +198,8 @@ I/O Base Classes
Even though :class:`IOBase` does not declare :meth:`read`, :meth:`readinto`,
or :meth:`write` because their signatures will vary, implementations and
clients should consider those methods part of the interface. Also,
- implementations may raise a :exc:`IOError` when operations they do not
- support are called.
+ implementations may raise a :exc:`ValueError` (or :exc:`UnsupportedOperation`)
+ when operations they do not support are called.
The basic type used for binary data read from or written to a file is
:class:`bytes`. :class:`bytearray`\s are accepted too, and in some cases
@@ -211,7 +207,7 @@ I/O Base Classes
:class:`str` data.
Note that calling any method (even inquiries) on a closed stream is
- undefined. Implementations may raise :exc:`IOError` in this case.
+ undefined. Implementations may raise :exc:`ValueError` in this case.
IOBase (and its subclasses) support the iterator protocol, meaning that an
:class:`IOBase` object can be iterated over yielding the lines in a stream.
@@ -244,7 +240,7 @@ I/O Base Classes
.. method:: fileno()
Return the underlying file descriptor (an integer) of the stream if it
- exists. An :exc:`IOError` is raised if the IO object does not use a file
+ exists. An :exc:`OSError` is raised if the IO object does not use a file
descriptor.
.. method:: flush()
@@ -260,7 +256,7 @@ I/O Base Classes
.. method:: readable()
Return ``True`` if the stream can be read from. If False, :meth:`read`
- will raise :exc:`IOError`.
+ will raise :exc:`OSError`.
.. method:: readline(limit=-1)
@@ -298,7 +294,7 @@ I/O Base Classes
.. method:: seekable()
Return ``True`` if the stream supports random access. If ``False``,
- :meth:`seek`, :meth:`tell` and :meth:`truncate` will raise :exc:`IOError`.
+ :meth:`seek`, :meth:`tell` and :meth:`truncate` will raise :exc:`OSError`.
.. method:: tell()
@@ -316,7 +312,7 @@ I/O Base Classes
.. method:: writable()
Return ``True`` if the stream supports writing. If ``False``,
- :meth:`write` and :meth:`truncate` will raise :exc:`IOError`.
+ :meth:`write` and :meth:`truncate` will raise :exc:`OSError`.
.. method:: writelines(lines)
@@ -450,7 +446,7 @@ I/O Base Classes
Write the given bytes or bytearray object, *b* and return the number
of bytes written (never less than ``len(b)``, since if the write fails
- an :exc:`IOError` will be raised). Depending on the actual
+ an :exc:`OSError` will be raised). Depending on the actual
implementation, these bytes may be readily written to the underlying
stream, or held in a buffer for performance and latency reasons.
@@ -462,7 +458,7 @@ I/O Base Classes
Raw File I/O
^^^^^^^^^^^^
-.. class:: FileIO(name, mode='r', closefd=True)
+.. class:: FileIO(name, mode='r', closefd=True, opener=None)
:class:`FileIO` represents an OS-level file containing bytes data.
It implements the :class:`RawIOBase` interface (and therefore the
@@ -483,6 +479,15 @@ Raw File I/O
The :meth:`read` (when called with a positive argument), :meth:`readinto`
and :meth:`write` methods on this class will only make one system call.
+ A custom opener can be used by passing a callable as *opener*. The underlying
+ file descriptor for the file object is then obtained by calling *opener* with
+ (*name*, *flags*). *opener* must return an open file descriptor (passing
+ :mod:`os.open` as *opener* results in functionality similar to passing
+ ``None``).
+
+ .. versionchanged:: 3.3
+ The *opener* parameter was added.
+
In addition to the attributes and methods from :class:`IOBase` and
:class:`RawIOBase`, :class:`FileIO` provides the following data
attributes and methods:
@@ -710,7 +715,8 @@ Text I/O
written.
-.. class:: TextIOWrapper(buffer, encoding=None, errors=None, newline=None, line_buffering=False)
+.. class:: TextIOWrapper(buffer, encoding=None, errors=None, newline=None, \
+ line_buffering=False, write_through=False)
A buffered text stream over a :class:`BufferedIOBase` binary stream.
It inherits :class:`TextIOBase`.
@@ -741,6 +747,13 @@ Text I/O
If *line_buffering* is ``True``, :meth:`flush` is implied when a call to
write contains a newline character.
+ If *write_through* is ``True``, calls to :meth:`write` are guaranteed
+ not to be buffered: any data written on the :class:`TextIOWrapper`
+ object is immediately handled to its underlying binary *buffer*.
+
+ .. versionchanged:: 3.3
+ The *write_through* argument has been added.
+
:class:`TextIOWrapper` provides one attribute in addition to those of
:class:`TextIOBase` and its parents:
diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst
index 28625e8..7b5922a 100644
--- a/Doc/library/itertools.rst
+++ b/Doc/library/itertools.rst
@@ -46,7 +46,7 @@ Iterator Arguments Results
==================== ============================ ================================================= =============================================================
Iterator Arguments Results Example
==================== ============================ ================================================= =============================================================
-:func:`accumulate` p p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) --> 1 3 6 10 15``
+:func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) --> 1 3 6 10 15``
:func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') --> A B C D E F``
:func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F``
:func:`dropwhile` pred, seq seq[n], seq[n+1], starting when pred fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1``
@@ -84,23 +84,61 @@ The following module functions all construct and return iterators. Some provide
streams of infinite length, so they should only be accessed by functions or
loops that truncate the stream.
-.. function:: accumulate(iterable)
+.. function:: accumulate(iterable[, func])
Make an iterator that returns accumulated sums. Elements may be any addable
- type including :class:`Decimal` or :class:`Fraction`. Equivalent to::
+ type including :class:`Decimal` or :class:`Fraction`. If the optional
+ *func* argument is supplied, it should be a function of two arguments
+ and it will be used instead of addition.
- def accumulate(iterable):
+ Equivalent to::
+
+ def accumulate(iterable, func=operator.add):
'Return running totals'
# accumulate([1,2,3,4,5]) --> 1 3 6 10 15
+ # accumulate([1,2,3,4,5], operator.mul) --> 1 2 6 24 120
it = iter(iterable)
total = next(it)
yield total
for element in it:
- total = total + element
+ total = func(total, element)
yield total
+ There are a number of uses for the *func* argument. It can be set to
+ :func:`min` for a running minimum, :func:`max` for a running maximum, or
+ :func:`operator.mul` for a running product. Amortization tables can be
+ built by accumulating interest and applying payments. First-order
+ `recurrence relations <http://en.wikipedia.org/wiki/Recurrence_relation>`_
+ can be modeled by supplying the initial value in the iterable and using only
+ the accumulated total in *func* argument::
+
+ >>> data = [3, 4, 6, 2, 1, 9, 0, 7, 5, 8]
+ >>> list(accumulate(data, operator.mul)) # running product
+ [3, 12, 72, 144, 144, 1296, 0, 0, 0, 0]
+ >>> list(accumulate(data, max)) # running maximum
+ [3, 4, 6, 6, 6, 9, 9, 9, 9, 9]
+
+ # Amortize a 5% loan of 1000 with 4 annual payments of 90
+ >>> cashflows = [1000, -90, -90, -90, -90]
+ >>> list(accumulate(cashflows, lambda bal, pmt: bal*1.05 + pmt))
+ [1000, 960.0, 918.0, 873.9000000000001, 827.5950000000001]
+
+ # Chaotic recurrence relation http://en.wikipedia.org/wiki/Logistic_map
+ >>> logistic_map = lambda x, _: r * x * (1 - x)
+ >>> r = 3.8
+ >>> x0 = 0.4
+ >>> inputs = repeat(x0, 36) # only the initial value is used
+ >>> [format(x, '.2f') for x in accumulate(inputs, logistic_map)]
+ ['0.40', '0.91', '0.30', '0.81', '0.60', '0.92', '0.29', '0.79', '0.63',
+ '0.88' ,'0.39', '0.90', '0.33', '0.84', '0.52', '0.95', '0.18', '0.57',
+ '0.93', '0.25', '0.71', '0.79', '0.63', '0.88', '0.39', '0.91', '0.32',
+ '0.83', '0.54', '0.95', '0.20', '0.60', '0.91', '0.30', '0.80', '0.60']
+
.. versionadded:: 3.2
+ .. versionchanged:: 3.3
+ Added the optional *func* parameter.
+
.. function:: chain(*iterables)
Make an iterator that returns elements from the first iterable until it is
diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst
index c4dd438..3911d4d 100644
--- a/Doc/library/logging.handlers.rst
+++ b/Doc/library/logging.handlers.rst
@@ -452,6 +452,15 @@ supports sending logging messages to a remote or local Unix syslog.
behaviour) but can be set to ``False`` on a ``SysLogHandler`` instance
in order for that instance to *not* append the NUL terminator.
+ .. versionchanged:: 3.3
+ (See: :issue:`12419`.) In earlier versions, there was no facility for
+ an "ident" or "tag" prefix to identify the source of the message. This
+ can now be specified using a class-level attribute, defaulting to
+ ``""`` to preserve existing behaviour, but which can be overridden on
+ a ``SysLogHandler`` instance in order for that instance to prepend
+ the ident to every message handled. Note that the provided ident must
+ be text, not bytes, and is prepended to the message exactly as is.
+
.. method:: encodePriority(facility, priority)
Encodes the facility and priority into an integer. You can pass in strings
@@ -859,6 +868,15 @@ possible, while any potentially slow operations (such as sending an email via
Note that if you don't call this before your application exits, there
may be some records still left on the queue, which won't be processed.
+ .. method:: enqueue_sentinel()
+
+ Writes a sentinel to the queue to tell the listener to quit. This
+ implementation uses ``put_nowait()``. You may want to override this
+ method if you want to use timeouts or work with custom queue
+ implementations.
+
+ .. versionadded:: 3.3
+
.. seealso::
diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst
index 5d385c5..d318948 100644
--- a/Doc/library/logging.rst
+++ b/Doc/library/logging.rst
@@ -189,6 +189,9 @@ instantiated directly, but always through the module-level function
Logs a message with level :const:`WARNING` on this logger. The arguments are
interpreted as for :meth:`debug`.
+ .. note:: There is an obsolete method ``warn`` which is functionally
+ identical to ``warning``. As ``warn`` is deprecated, please do not use
+ it - use ``warning`` instead.
.. method:: Logger.error(msg, *args, **kwargs)
@@ -461,6 +464,19 @@ The useful mapping keys in a :class:`LogRecord` are given in the section on
want all logging times to be shown in GMT, set the ``converter``
attribute in the ``Formatter`` class.
+ .. versionchanged:: 3.3
+ Previously, the default ISO 8601 format was hard-coded as in this
+ example: ``2010-09-06 22:38:15,292`` where the part before the comma is
+ handled by a strptime format string (``'%Y-%m-%d %H:%M:%S'``), and the
+ part after the comma is a millisecond value. Because strptime does not
+ have a format placeholder for milliseconds, the millisecond value is
+ appended using another format string, ``'%s,%03d'`` – and both of these
+ format strings have been hardcoded into this method. With the change,
+ these strings are defined as class-level attributes which can be
+ overridden at the instance level when desired. The names of the
+ attributes are ``default_time_format`` (for the strptime format string)
+ and ``default_msec_format`` (for appending the millisecond value).
+
.. method:: formatException(exc_info)
Formats the specified exception information (a standard exception tuple as
@@ -867,8 +883,12 @@ functions.
.. function:: warning(msg, *args, **kwargs)
- Logs a message with level :const:`WARNING` on the root logger. The arguments are
- interpreted as for :func:`debug`.
+ Logs a message with level :const:`WARNING` on the root logger. The arguments
+ are interpreted as for :func:`debug`.
+
+ .. note:: There is an obsolete function ``warn`` which is functionally
+ identical to ``warning``. As ``warn`` is deprecated, please do not use
+ it - use ``warning`` instead.
.. function:: error(msg, *args, **kwargs)
@@ -993,12 +1013,27 @@ functions.
| ``stream`` | Use the specified stream to initialize the |
| | StreamHandler. Note that this argument is |
| | incompatible with 'filename' - if both are |
- | | present, 'stream' is ignored. |
+ | | present, a ``ValueError`` is raised. |
+ +--------------+---------------------------------------------+
+ | ``handlers`` | If specified, this should be an iterable of |
+ | | already created handlers to add to the root |
+ | | logger. Any handlers which don't already |
+ | | have a formatter set will be assigned the |
+ | | default formatter created in this function. |
+ | | Note that this argument is incompatible |
+ | | with 'filename' or 'stream' - if both are |
+ | | present, a ``ValueError`` is raised. |
+--------------+---------------------------------------------+
.. versionchanged:: 3.2
The ``style`` argument was added.
+ .. versionchanged:: 3.3
+ The ``handlers`` argument was added. Additional checks were added to
+ catch situations where incompatible arguments are specified (e.g.
+ ``handlers`` together with ``stream`` or ``filename``, or ``stream``
+ together with ``filename``).
+
.. function:: shutdown()
diff --git a/Doc/library/math.rst b/Doc/library/math.rst
index 98c5b33..62c0f34 100644
--- a/Doc/library/math.rst
+++ b/Doc/library/math.rst
@@ -184,6 +184,19 @@ Power and logarithmic functions
result is calculated in a way which is accurate for *x* near zero.
+.. function:: log2(x)
+
+ Return the base-2 logarithm of *x*. This is usually more accurate than
+ ``log(x, 2)``.
+
+ .. versionadded:: 3.3
+
+ .. seealso::
+
+ :meth:`int.bit_length` returns the number of bits necessary to represent
+ an integer in binary, excluding the sign and leading zeros.
+
+
.. function:: log10(x)
Return the base-10 logarithm of *x*. This is usually more accurate
diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst
index 5f0f004..1a19a7e 100644
--- a/Doc/library/mmap.rst
+++ b/Doc/library/mmap.rst
@@ -196,12 +196,16 @@ To map anonymous memory, -1 should be passed as the fileno along with the length
move will raise a :exc:`TypeError` exception.
- .. method:: read(num)
+ .. method:: read([n])
- Return a :class:`bytes` containing up to *num* bytes starting from the
- current file position; the file position is updated to point after the
- bytes that were returned.
+ Return a :class:`bytes` containing up to *n* bytes starting from the
+ current file position. If the argument is omitted, *None* or negative,
+ return all bytes from the current file position to the end of the
+ mapping. The file position is updated to point after the bytes that were
+ returned.
+ .. versionchanged:: 3.3
+ Argument can be omitted or *None*.
.. method:: read_byte()
diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst
index 889a0c5..9d23720 100644
--- a/Doc/library/msvcrt.rst
+++ b/Doc/library/msvcrt.rst
@@ -20,6 +20,11 @@ api. The normal API deals only with ASCII characters and is of limited use
for internationalized applications. The wide char API should be used where
ever possible
+.. versionchanged:: 3.3
+ Operations in this module now raise :exc:`OSError` where :exc:`IOError`
+ was raised.
+
+
.. _msvcrt-files:
File Operations
@@ -29,7 +34,7 @@ File Operations
.. function:: locking(fd, mode, nbytes)
Lock part of a file based on file descriptor *fd* from the C runtime. Raises
- :exc:`IOError` on failure. The locked region of the file extends from the
+ :exc:`OSError` on failure. The locked region of the file extends from the
current file position for *nbytes* bytes, and may continue beyond the end of the
file. *mode* must be one of the :const:`LK_\*` constants listed below. Multiple
regions in a file may be locked at the same time, but may not overlap. Adjacent
@@ -41,13 +46,13 @@ File Operations
Locks the specified bytes. If the bytes cannot be locked, the program
immediately tries again after 1 second. If, after 10 attempts, the bytes cannot
- be locked, :exc:`IOError` is raised.
+ be locked, :exc:`OSError` is raised.
.. data:: LK_NBLCK
LK_NBRLCK
- Locks the specified bytes. If the bytes cannot be locked, :exc:`IOError` is
+ Locks the specified bytes. If the bytes cannot be locked, :exc:`OSError` is
raised.
@@ -73,7 +78,7 @@ File Operations
.. function:: get_osfhandle(fd)
- Return the file handle for the file descriptor *fd*. Raises :exc:`IOError` if
+ Return the file handle for the file descriptor *fd*. Raises :exc:`OSError` if
*fd* is not recognized.
@@ -144,4 +149,4 @@ Other Functions
.. function:: heapmin()
Force the :c:func:`malloc` heap to clean itself up and return unused blocks to
- the operating system. On failure, this raises :exc:`IOError`.
+ the operating system. On failure, this raises :exc:`OSError`.
diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst
index d8a554d..851b3cf 100644
--- a/Doc/library/multiprocessing.rst
+++ b/Doc/library/multiprocessing.rst
@@ -297,7 +297,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the
:class:`Process` and exceptions
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-.. class:: Process([group[, target[, name[, args[, kwargs]]]]])
+.. class:: Process([group[, target[, name[, args[, kwargs]]]]], *, daemon=None)
Process objects represent activity that is run in a separate process. The
:class:`Process` class has equivalents of all the methods of
@@ -312,13 +312,19 @@ The :mod:`multiprocessing` package mostly replicates the API of the
:sub:`1`,N\ :sub:`2`,...,N\ :sub:`k` is a sequence of integers whose length
is determined by the *generation* of the process. *args* is the argument
tuple for the target invocation. *kwargs* is a dictionary of keyword
- arguments for the target invocation. By default, no arguments are passed to
- *target*.
+ arguments for the target invocation. If provided, the keyword-only *daemon* argument
+ sets the process :attr:`daemon` flag to ``True`` or ``False``. If ``None``
+ (the default), this flag will be inherited from the creating process.
+
+ By default, no arguments are passed to *target*.
If a subclass overrides the constructor, it must make sure it invokes the
base class constructor (:meth:`Process.__init__`) before doing anything else
to the process.
+ .. versionchanged:: 3.3
+ Added the *daemon* argument.
+
.. method:: run()
Method representing the process's activity.
@@ -337,10 +343,9 @@ The :mod:`multiprocessing` package mostly replicates the API of the
.. method:: join([timeout])
- Block the calling thread until the process whose :meth:`join` method is
- called terminates or until the optional timeout occurs.
-
- If *timeout* is ``None`` then there is no timeout.
+ If the optional argument *timeout* is ``None`` (the default), the method
+ blocks until the process whose :meth:`join` method is called terminates.
+ If *timeout* is a positive number, it blocks at most *timeout* seconds.
A process can be joined many times.
@@ -405,6 +410,20 @@ The :mod:`multiprocessing` package mostly replicates the API of the
See :ref:`multiprocessing-auth-keys`.
+ .. attribute:: sentinel
+
+ A numeric handle of a system object which will become "ready" when
+ the process ends.
+
+ On Windows, this is an OS handle usable with the ``WaitForSingleObject``
+ and ``WaitForMultipleObjects`` family of API calls. On Unix, this is
+ a file descriptor usable with primitives from the :mod:`select` module.
+
+ You can use this value if you want to wait on several events at once.
+ Otherwise calling :meth:`join()` is simpler.
+
+ .. versionadded:: 3.3
+
.. method:: terminate()
Terminate the process. On Unix this is done using the ``SIGTERM`` signal;
@@ -765,9 +784,14 @@ Connection objects usually created using :func:`Pipe` -- see also
to receive and the other end has closed.
If *maxlength* is specified and the message is longer than *maxlength*
- then :exc:`IOError` is raised and the connection will no longer be
+ then :exc:`OSError` is raised and the connection will no longer be
readable.
+ .. versionchanged:: 3.3
+ This function used to raise a :exc:`IOError`, which is now an
+ alias of :exc:`OSError`.
+
+
.. method:: recv_bytes_into(buffer[, offset])
Read into *buffer* a complete message of byte data sent from the other end
diff --git a/Doc/library/nntplib.rst b/Doc/library/nntplib.rst
index ef507e1..01294e8 100644
--- a/Doc/library/nntplib.rst
+++ b/Doc/library/nntplib.rst
@@ -70,10 +70,23 @@ The module itself defines the following classes:
connecting to an NNTP server on the local machine and intend to call
reader-specific commands, such as ``group``. If you get unexpected
:exc:`NNTPPermanentError`\ s, you might need to set *readermode*.
+ :class:`NNTP` class supports the :keyword:`with` statement to
+ unconditionally consume :exc:`socket.error` exceptions and to close the NNTP
+ connection when done. Here is a sample on how using it:
+
+ >>> from nntplib import NNTP
+ >>> with NNTP('news.gmane.org') as n:
+ ... n.group('gmane.comp.python.committers')
+ ...
+ ('211 1755 1 1755 gmane.comp.python.committers', 1755, 1, 1755, 'gmane.comp.python.committers')
+ >>>
+
.. versionchanged:: 3.2
*usenetrc* is now False by default.
+ .. versionchanged:: 3.3
+ Support for the :keyword:`with` statement was added.
.. class:: NNTP_SSL(host, port=563, user=None, password=None, ssl_context=None, readermode=None, usenetrc=False, [timeout])
diff --git a/Doc/library/os.rst b/Doc/library/os.rst
index 767c40f..fb794d3 100644
--- a/Doc/library/os.rst
+++ b/Doc/library/os.rst
@@ -221,6 +221,17 @@ process and user.
Availability: Unix.
+.. function:: getgrouplist(user, group)
+
+ Return list of group ids that *user* belongs to. If *group* is not in the
+ list, it is included; typically, *group* is specified as the group ID
+ field from the password record for *user*.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
.. function:: getgroups()
Return list of supplemental group ids associated with the current process.
@@ -288,6 +299,22 @@ process and user.
.. versionchanged:: 3.2
Added support for Windows.
+.. function:: getpriority(which, who)
+
+ .. index:: single: process; scheduling priority
+
+ Get program scheduling priority. The value *which* is one of
+ :const:`PRIO_PROCESS`, :const:`PRIO_PGRP`, or :const:`PRIO_USER`, and *who*
+ is interpreted relative to *which* (a process identifier for
+ :const:`PRIO_PROCESS`, process group identifier for :const:`PRIO_PGRP`, and a
+ user ID for :const:`PRIO_USER`). A zero value for *who* denotes
+ (respectively) the calling process, the process group of the calling process,
+ or the real user ID of the calling process.
+
+ Availability: Unix
+
+ .. versionadded:: 3.3
+
.. function:: getresuid()
Return a tuple (ruid, euid, suid) denoting the current process's
@@ -338,6 +365,15 @@ process and user.
.. versionadded:: 3.2
+.. data:: PRIO_PROCESS
+ PRIO_PGRP
+ PRIO_USER
+
+ Parameters for :func:`getpriority` and :func:`setpriority` functions.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
.. function:: putenv(key, value)
@@ -407,6 +443,25 @@ process and user.
Availability: Unix.
+.. function:: setpriority(which, who, priority)
+
+ .. index:: single: process; scheduling priority
+
+ Set program scheduling priority. The value *which* is one of
+ :const:`PRIO_PROCESS`, :const:`PRIO_PGRP`, or :const:`PRIO_USER`, and *who*
+ is interpreted relative to *which* (a process identifier for
+ :const:`PRIO_PROCESS`, process group identifier for :const:`PRIO_PGRP`, and a
+ user ID for :const:`PRIO_USER`). A zero value for *who* denotes
+ (respectively) the calling process, the process group of the calling process,
+ or the real user ID of the calling process.
+ *priority* is a value in the range -20 to 19. The default priority is 0;
+ lower priorities cause more favorable scheduling.
+
+ Availability: Unix
+
+ .. versionadded:: 3.3
+
+
.. function:: setregid(rgid, egid)
Set the current process's real and effective group ids.
@@ -564,6 +619,21 @@ associated with a :term:`file object` when required. Note that using the file
descriptor directly will bypass the file object methods, ignoring aspects such
as internal buffering of data.
+.. data:: AT_SYMLINK_NOFOLLOW
+ AT_EACCESS
+ AT_FDCWD
+ AT_REMOVEDIR
+ AT_SYMLINK_FOLLOW
+ UTIME_NOW
+ UTIME_OMIT
+
+ These parameters are used as flags to the \*at family of functions.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
.. function:: close(fd)
Close file descriptor *fd*.
@@ -612,6 +682,19 @@ as internal buffering of data.
Availability: Unix, Windows.
+.. function:: faccessat(dirfd, path, mode, flags=0)
+
+ Like :func:`access` but if *path* is relative, it is taken as relative to *dirfd*.
+ *flags* is optional and can be constructed by ORing together zero or more
+ of these values: :data:`AT_SYMLINK_NOFOLLOW`, :data:`AT_EACCESS`.
+ If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path*
+ is interpreted relative to the current working directory.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
.. function:: fchmod(fd, mode)
Change the mode of the file given by *fd* to the numeric *mode*. See the docs
@@ -620,6 +703,18 @@ as internal buffering of data.
Availability: Unix.
+.. function:: fchmodat(dirfd, path, mode, flags=0)
+
+ Like :func:`chmod` but if *path* is relative, it is taken as relative to *dirfd*.
+ *flags* is optional and may be 0 or :data:`AT_SYMLINK_NOFOLLOW`.
+ If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path*
+ is interpreted relative to the current working directory.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
.. function:: fchown(fd, uid, gid)
Change the owner and group id of the file given by *fd* to the numeric *uid*
@@ -628,6 +723,18 @@ as internal buffering of data.
Availability: Unix.
+.. function:: fchownat(dirfd, path, uid, gid, flags=0)
+
+ Like :func:`chown` but if *path* is relative, it is taken as relative to *dirfd*.
+ *flags* is optional and may be 0 or :data:`AT_SYMLINK_NOFOLLOW`.
+ If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path*
+ is interpreted relative to the current working directory.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
.. function:: fdatasync(fd)
Force write of file with filedescriptor *fd* to disk. Does not force update of
@@ -639,6 +746,47 @@ as internal buffering of data.
This function is not available on MacOS.
+.. function:: fgetxattr(fd, attr)
+
+ This works exactly like :func:`getxattr` but operates on a file descriptor,
+ *fd*, instead of a path.
+
+ Availability: Linux
+
+ .. versionadded:: 3.3
+
+
+.. function:: flistxattr(fd)
+
+ This is exactly like :func:`listxattr` but operates on a file descriptor,
+ *fd*, instead of a path.
+
+ Availability: Linux
+
+ .. versionadded:: 3.3
+
+
+.. function:: fdlistdir(fd)
+
+ Like :func:`listdir`, but uses a file descriptor instead and always returns
+ strings. After execution of this function, *fd* will be closed.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. function:: fexecve(fd, args, env)
+
+ Execute the program specified by a file descriptor *fd* with arguments given
+ by *args* and environment given by *env*, replacing the current process.
+ *args* and *env* are given as in :func:`execve`.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
.. function:: fpathconf(fd, name)
Return system configuration information relevant to an open file. *name*
@@ -663,6 +811,17 @@ as internal buffering of data.
Availability: Unix, Windows.
+.. function:: fstatat(dirfd, path, flags=0)
+
+ Like :func:`stat` but if *path* is relative, it is taken as relative to *dirfd*.
+ *flags* is optional and may be 0 or :data:`AT_SYMLINK_NOFOLLOW`.
+ If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path*
+ is interpreted relative to the current working directory.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
.. function:: fstatvfs(fd)
@@ -692,6 +851,78 @@ as internal buffering of data.
Availability: Unix.
+.. function:: fremovexattr(fd, attr)
+
+ This works exactly like :func:`removexattr` but operates on a file
+ descriptor, *fd*, instead of a path.
+
+ Availability: Linux
+
+ .. versionadded:: 3.3
+
+
+.. function:: fsetxattr(fd, attr, value, flags=0)
+
+ This works exactly like :func:`setxattr` but on a file descriptor, *fd*,
+ instead of a path.
+
+
+ Availability: Linux
+
+ .. versionadded:: 3.3
+
+
+.. function:: futimesat(dirfd, path, (atime, mtime))
+ futimesat(dirfd, path, None)
+
+ Like :func:`utime` but if *path* is relative, it is taken as relative to *dirfd*.
+ If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path*
+ is interpreted relative to the current working directory.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. function:: futimens(fd, (atime_sec, atime_nsec), (mtime_sec, mtime_nsec))
+ futimens(fd, None, None)
+
+ Updates the timestamps of a file specified by the file descriptor *fd*, with
+ nanosecond precision.
+ The second form sets *atime* and *mtime* to the current time.
+ If *atime_nsec* or *mtime_nsec* is specified as :data:`UTIME_NOW`, the corresponding
+ timestamp is updated to the current time.
+ If *atime_nsec* or *mtime_nsec* is specified as :data:`UTIME_OMIT`, the corresponding
+ timestamp is not updated.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. data:: UTIME_NOW
+ UTIME_OMIT
+
+ Flags used with :func:`futimens` to specify that the timestamp must be
+ updated either to the current time or not updated at all.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. function:: futimes(fd, (atime, mtime))
+ futimes(fd, None)
+
+ Set the access and modified time of the file specified by the file
+ descriptor *fd* to the given values. If the second form is used, set the
+ access and modified times to the current time.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
.. function:: isatty(fd)
Return ``True`` if the file descriptor *fd* is open and connected to a
@@ -700,6 +931,44 @@ as internal buffering of data.
Availability: Unix.
+.. function:: linkat(srcfd, srcpath, dstfd, dstpath, flags=0)
+
+ Like :func:`link` but if *srcpath* is relative, it is taken as relative to *srcfd*
+ and if *dstpath* is relative, it is taken as relative to *dstfd*.
+ *flags* is optional and may be 0 or :data:`AT_SYMLINK_FOLLOW`.
+ If *srcpath* is relative and *srcfd* is the special value :data:`AT_FDCWD`, then
+ *srcpath* is interpreted relative to the current working directory. This
+ also applies for *dstpath*.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. function:: lockf(fd, cmd, len)
+
+ Apply, test or remove a POSIX lock on an open file descriptor.
+ *fd* is an open file descriptor.
+ *cmd* specifies the command to use - one of :data:`F_LOCK`, :data:`F_TLOCK`,
+ :data:`F_ULOCK` or :data:`F_TEST`.
+ *len* specifies the section of the file to lock.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. data:: F_LOCK
+ F_TLOCK
+ F_ULOCK
+ F_TEST
+
+ Flags that specify what action :func:`lockf` will take.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
.. function:: lseek(fd, pos, how)
Set the current position of file descriptor *fd* to position *pos*, modified
@@ -719,6 +988,39 @@ as internal buffering of data.
respectively. Availability: Windows, Unix.
+.. function:: mkdirat(dirfd, path, mode=0o777)
+
+ Like :func:`mkdir` but if *path* is relative, it is taken as relative to *dirfd*.
+ If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path*
+ is interpreted relative to the current working directory.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. function:: mkfifoat(dirfd, path, mode=0o666)
+
+ Like :func:`mkfifo` but if *path* is relative, it is taken as relative to *dirfd*.
+ If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path*
+ is interpreted relative to the current working directory.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. function:: mknodat(dirfd, path, mode=0o600, device=0)
+
+ Like :func:`mknod` but if *path* is relative, it is taken as relative to *dirfd*.
+ If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path*
+ is interpreted relative to the current working directory.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
.. function:: open(file, flags[, mode])
Open the file *file* and set various flags according to *flags* and possibly
@@ -741,6 +1043,17 @@ as internal buffering of data.
wrap a file descriptor in a file object, use :func:`fdopen`.
+.. function:: openat(dirfd, path, flags, mode=0o777)
+
+ Like :func:`open` but if *path* is relative, it is taken as relative to *dirfd*.
+ If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path*
+ is interpreted relative to the current working directory.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
.. function:: openpty()
.. index:: module: pty
@@ -760,6 +1073,79 @@ as internal buffering of data.
Availability: Unix, Windows.
+.. function:: pipe2(flags)
+
+ Create a pipe with *flags* set atomically.
+ *flags* can be constructed by ORing together one or more of these values:
+ :data:`O_NONBLOCK`, :data:`O_CLOEXEC`.
+ Return a pair of file descriptors ``(r, w)`` usable for reading and writing,
+ respectively.
+
+ Availability: some flavors of Unix.
+
+ .. versionadded:: 3.3
+
+
+.. function:: posix_fallocate(fd, offset, len)
+
+ Ensures that enough disk space is allocated for the file specified by *fd*
+ starting from *offset* and continuing for *len* bytes.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. function:: posix_fadvise(fd, offset, len, advice)
+
+ Announces an intention to access data in a specific pattern thus allowing
+ the kernel to make optimizations.
+ The advice applies to the region of the file specified by *fd* starting at
+ *offset* and continuing for *len* bytes.
+ *advice* is one of :data:`POSIX_FADV_NORMAL`, :data:`POSIX_FADV_SEQUENTIAL`,
+ :data:`POSIX_FADV_RANDOM`, :data:`POSIX_FADV_NOREUSE`,
+ :data:`POSIX_FADV_WILLNEED` or :data:`POSIX_FADV_DONTNEED`.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. data:: POSIX_FADV_NORMAL
+ POSIX_FADV_SEQUENTIAL
+ POSIX_FADV_RANDOM
+ POSIX_FADV_NOREUSE
+ POSIX_FADV_WILLNEED
+ POSIX_FADV_DONTNEED
+
+ Flags that can be used in *advice* in :func:`posix_fadvise` that specify
+ the access pattern that is likely to be used.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. function:: pread(fd, buffersize, offset)
+
+ Read from a file descriptor, *fd*, at a position of *offset*. It will read up
+ to *buffersize* number of bytes. The file offset remains unchanged.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. function:: pwrite(fd, string, offset)
+
+ Write *string* to a file descriptor, *fd*, from *offset*, leaving the file
+ offset unchanged.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
.. function:: read(fd, n)
Read at most *n* bytes from file descriptor *fd*. Return a bytestring containing the
@@ -777,6 +1163,93 @@ as internal buffering of data.
:meth:`~file.readline` methods.
+.. function:: sendfile(out, in, offset, nbytes)
+ sendfile(out, in, offset, nbytes, headers=None, trailers=None, flags=0)
+
+ Copy *nbytes* bytes from file descriptor *in* to file descriptor *out*
+ starting at *offset*.
+ Return the number of bytes sent. When EOF is reached return 0.
+
+ The first function notation is supported by all platforms that define
+ :func:`sendfile`.
+
+ On Linux, if *offset* is given as ``None``, the bytes are read from the
+ current position of *in* and the position of *in* is updated.
+
+ The second case may be used on Mac OS X and FreeBSD where *headers* and
+ *trailers* are arbitrary sequences of buffers that are written before and
+ after the data from *in* is written. It returns the same as the first case.
+
+ On Mac OS X and FreeBSD, a value of 0 for *nbytes* specifies to send until
+ the end of *in* is reached.
+
+ On Solaris, *out* may be the file descriptor of a regular file or the file
+ descriptor of a socket. On all other platforms, *out* must be the file
+ descriptor of an open socket.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. data:: SF_NODISKIO
+ SF_MNOWAIT
+ SF_SYNC
+
+ Parameters to the :func:`sendfile` function, if the implementation supports
+ them.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. function:: readlinkat(dirfd, path)
+
+ Like :func:`readlink` but if *path* is relative, it is taken as relative to *dirfd*.
+ If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path*
+ is interpreted relative to the current working directory.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. function:: renameat(olddirfd, oldpath, newdirfd, newpath)
+
+ Like :func:`rename` but if *oldpath* is relative, it is taken as relative to
+ *olddirfd* and if *newpath* is relative, it is taken as relative to *newdirfd*.
+ If *oldpath* is relative and *olddirfd* is the special value :data:`AT_FDCWD`, then
+ *oldpath* is interpreted relative to the current working directory. This
+ also applies for *newpath*.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. function:: symlinkat(src, dstfd, dst)
+
+ Like :func:`symlink` but if *dst* is relative, it is taken as relative to *dstfd*.
+ If *dst* is relative and *dstfd* is the special value :data:`AT_FDCWD`, then *dst*
+ is interpreted relative to the current working directory.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. function:: readv(fd, buffers)
+
+ Read from a file descriptor into a number of writable buffers. *buffers* is
+ an arbitrary sequence of writable buffers. Returns the total number of bytes
+ read.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
.. function:: tcgetpgrp(fd)
Return the process group associated with the terminal given by *fd* (an open
@@ -802,6 +1275,38 @@ as internal buffering of data.
Availability: Unix.
+.. function:: unlinkat(dirfd, path, flags=0)
+
+ Like :func:`unlink` but if *path* is relative, it is taken as relative to *dirfd*.
+ *flags* is optional and may be 0 or :data:`AT_REMOVEDIR`. If :data:`AT_REMOVEDIR` is
+ specified, :func:`unlinkat` behaves like :func:`rmdir`.
+ If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path*
+ is interpreted relative to the current working directory.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. function:: utimensat(dirfd, path, (atime_sec, atime_nsec), (mtime_sec, mtime_nsec), flags)
+ utimensat(dirfd, path, None, None, flags)
+
+ Updates the timestamps of a file with nanosecond precision.
+ The second form sets *atime* and *mtime* to the current time.
+ If *atime_nsec* or *mtime_nsec* is specified as :data:`UTIME_NOW`, the corresponding
+ timestamp is updated to the current time.
+ If *atime_nsec* or *mtime_nsec* is specified as :data:`UTIME_OMIT`, the corresponding
+ timestamp is not updated.
+ If *path* is relative, it is taken as relative to *dirfd*.
+ *flags* is optional and may be 0 or :data:`AT_SYMLINK_NOFOLLOW`.
+ If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path*
+ is interpreted relative to the current working directory.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
.. function:: write(fd, str)
Write the bytestring in *str* to file descriptor *fd*. Return the number of
@@ -818,6 +1323,17 @@ as internal buffering of data.
:meth:`~file.write` method.
+.. function:: writev(fd, buffers)
+
+ Write the contents of *buffers* to file descriptor *fd*, where *buffers*
+ is an arbitrary sequence of buffers.
+ Returns the total number of bytes written.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
.. _open-constants:
``open()`` flag constants
@@ -849,9 +1365,12 @@ or `the MSDN <http://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Window
O_NOCTTY
O_SHLOCK
O_EXLOCK
+ O_CLOEXEC
These constants are only available on Unix.
+ .. versionchanged:: 3.3
+ Add :data:`O_CLOEXEC` constant.
.. data:: O_BINARY
O_NOINHERIT
@@ -874,6 +1393,19 @@ or `the MSDN <http://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Window
the C library.
+.. data:: RTLD_LAZY
+ RTLD_NOW
+ RTLD_GLOBAL
+ RTLD_LOCAL
+ RTLD_NODELETE
+ RTLD_NOLOAD
+ RTLD_DEEPBIND
+
+ See the Unix manual page :manpage:`dlopen(3)`.
+
+ .. versionadded:: 3.3
+
+
.. _os-file-dir:
Files and Directories
@@ -909,11 +1441,8 @@ Files and Directories
try:
fp = open("myfile")
- except IOError as e:
- if e.errno == errno.EACCES:
- return "some default data"
- # Not a permission error.
- raise
+ except PermissionError:
+ return "some default data"
else:
with fp:
return fp.read()
@@ -1049,9 +1578,23 @@ Files and Directories
Change the owner and group id of *path* to the numeric *uid* and *gid*. To leave
one of the ids unchanged, set it to -1.
+ See :func:`shutil.chown` for a higher-level function that accepts names in
+ addition to numeric ids.
+
Availability: Unix.
+.. function:: getxattr(path, attr)
+
+ Return the value of the extended filesystem attribute *attr* for
+ *path*. *attr* can be bytes or str. If it is str, it is encoded with the
+ filesystem encoding.
+
+ Availability: Linux
+
+ .. versionadded:: 3.3
+
+
.. function:: lchflags(path, flags)
Set the flags of *path* to the numeric *flags*, like :func:`chflags`, but do not
@@ -1077,6 +1620,15 @@ Files and Directories
Availability: Unix.
+.. function:: lgetxattr(path, attr)
+
+ This works exactly like :func:`getxattr` but doesn't follow symlinks.
+
+ Availability: Linux
+
+ .. versionadded:: 3.3
+
+
.. function:: link(source, link_name)
Create a hard link pointing to *source* named *link_name*.
@@ -1101,6 +1653,44 @@ Files and Directories
.. versionchanged:: 3.2
The *path* parameter became optional.
+
+.. function:: listxattr(path)
+
+ Return a list of the extended filesystem attributes on *path*. Attributes are
+ returned as string decoded with the filesystem encoding.
+
+ Availability: Linux
+
+ .. versionadded:: 3.3
+
+
+.. function:: llistxattr(path)
+
+ This works exactly like :func:`listxattr` but doesn't follow symlinks.
+
+ Availability: Linux
+
+ .. versionadded:: 3.3
+
+
+.. function:: lremovexattr(path, attr)
+
+ This works exactly like :func:`removexattr` but doesn't follow symlinks.
+
+ Availability: Linux
+
+ .. versionadded:: 3.3
+
+
+.. function:: lsetxattr(path, attr, value, flags=0)
+
+ This works exactly like :func:`setxattr` but doesn't follow symlinks.
+
+ Availability: Linux
+
+ .. versionadded:: 3.3
+
+
.. function:: lstat(path)
Perform the equivalent of an :c:func:`lstat` system call on the given path.
@@ -1112,6 +1702,17 @@ Files and Directories
Added support for Windows 6.0 (Vista) symbolic links.
+.. function:: lutimes(path, (atime, mtime))
+ lutimes(path, None)
+
+ Like :func:`utime`, but if *path* is a symbolic link, it is not
+ dereferenced.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
.. function:: mkfifo(path[, mode])
Create a FIFO (a named pipe) named *path* with numeric mode *mode*. The
@@ -1263,6 +1864,17 @@ Files and Directories
successfully removed.
+.. function:: removexattr(path, attr)
+
+ Removes the extended filesystem attribute *attr* from *path*. *attr* should
+ be bytes or str. If it is a string, it is encoded with the filesystem
+ encoding.
+
+ Availability: Linux
+
+ .. versionadded:: 3.3
+
+
.. function:: rename(src, dst)
Rename the file or directory *src* to *dst*. If *dst* is a directory,
@@ -1299,6 +1911,44 @@ Files and Directories
Availability: Unix, Windows.
+.. data:: XATTR_SIZE_MAX
+
+ The maximum size the value of an extended attribute can be. Currently, this
+ is 64 kilobytes on Linux.
+
+
+.. data:: XATTR_CREATE
+
+ This is a possible value for the flags argument in :func:`setxattr`. It
+ indicates the operation must create an attribute.
+
+
+.. data:: XATTR_REPLACE
+
+ This is a possible value for the flags argument in :func:`setxattr`. It
+ indicates the operation must replace an existing attribute.
+
+
+.. function:: setxattr(path, attr, value, flags=0)
+
+ Set the extended filesystem attribute *attr* on *path* to *value*. *attr*
+ must be a bytes or str with no embedded NULs. If it is str, it is encoded
+ with the filesystem encoding. *flags* may be :data:`XATTR_REPLACE` or
+ :data:`XATTR_CREATE`. If :data:`XATTR_REPLACE` is given and the attribute
+ does not exist, ``EEXISTS`` will be raised. If :data:`XATTR_CREATE` is given
+ and the attribute already exists, the attribute will not be created and
+ ``ENODATA`` will be raised.
+
+ Availability: Linux
+
+ .. note::
+
+ A bug in Linux kernel versions less than 2.6.39 caused the flags argument
+ to be ignored on some filesystems.
+
+ .. versionadded:: 3.3
+
+
.. function:: stat(path)
Perform the equivalent of a :c:func:`stat` system call on the given path.
@@ -1456,6 +2106,25 @@ Files and Directories
Added support for Windows 6.0 (Vista) symbolic links.
+.. function:: sync()
+
+ Force write of everything to disk.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. function:: truncate(path, length)
+
+ Truncate the file corresponding to *path*, so that it is at most
+ *length* bytes in size.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
.. function:: unlink(path)
Remove (delete) the file *path*. This is the same function as
@@ -1828,6 +2497,8 @@ written in Python, such as a mail server's external command delivery program.
will be set to *sig*. The Windows version of :func:`kill` additionally takes
process handles to be killed.
+ See also :func:`signal.pthread_kill`.
+
.. versionadded:: 3.2
Windows support.
@@ -2038,6 +2709,58 @@ written in Python, such as a mail server's external command delivery program.
Availability: Unix.
+.. function:: waitid(idtype, id, options)
+
+ Wait for the completion of one or more child processes.
+ *idtype* can be :data:`P_PID`, :data:`P_PGID` or :data:`P_ALL`.
+ *id* specifies the pid to wait on.
+ *options* is constructed from the ORing of one or more of :data:`WEXITED`,
+ :data:`WSTOPPED` or :data:`WCONTINUED` and additionally may be ORed with
+ :data:`WNOHANG` or :data:`WNOWAIT`. The return value is an object
+ representing the data contained in the :c:type:`siginfo_t` structure, namely:
+ :attr:`si_pid`, :attr:`si_uid`, :attr:`si_signo`, :attr:`si_status`,
+ :attr:`si_code` or ``None`` if :data:`WNOHANG` is specified and there are no
+ children in a waitable state.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+.. data:: P_PID
+ P_PGID
+ P_ALL
+
+ These are the possible values for *idtype* in :func:`waitid`. They affect
+ how *id* is interpreted.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+.. data:: WEXITED
+ WSTOPPED
+ WNOWAIT
+
+ Flags that can be used in *options* in :func:`waitid` that specify what
+ child signal to wait for.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. data:: CLD_EXITED
+ CLD_DUMPED
+ CLD_TRAPPED
+ CLD_CONTINUED
+
+ These are the possible values for :attr:`si_code` in the result returned by
+ :func:`waitid`.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
.. function:: waitpid(pid, options)
@@ -2179,6 +2902,155 @@ used to determine the disposition of a process.
Availability: Unix.
+Interface to the scheduler
+--------------------------
+
+These functions control how a process is allocated CPU time by the operating
+system. They are only available on some Unix platforms. For more detailed
+information, consult your Unix manpages.
+
+.. versionadded:: 3.3
+
+The following scheduling policies are exposed if they are a supported by the
+operating system.
+
+.. data:: SCHED_OTHER
+
+ The default scheduling policy.
+
+.. data:: SCHED_BATCH
+
+ Scheduling policy for CPU-intensive processes that tries to preserve
+ interactivity on the rest of the computer.
+
+.. data:: SCHED_IDLE
+
+ Scheduling policy for extremely low priority background tasks.
+
+.. data:: SCHED_SPORADIC
+
+ Scheduling policy for sporadic server programs.
+
+.. data:: SCHED_FIFO
+
+ A First In First Out scheduling policy.
+
+.. data:: SCHED_RR
+
+ A round-robin scheduling policy.
+
+.. data:: SCHED_RESET_ON_FORK
+
+ This flag can OR'ed with any other scheduling policy. When a process with
+ this flag set forks, its child's scheduling policy and priority are reset to
+ the default.
+
+
+.. class:: sched_param(sched_priority)
+
+ This class represents tunable scheduling parameters used in
+ :func:`sched_setparam`, :func:`sched_setscheduler`, and
+ :func:`sched_getparam`. It is immutable.
+
+ At the moment, there is only one possible parameter:
+
+ .. attribute:: sched_priority
+
+ The scheduling priority for a scheduling policy.
+
+
+.. function:: sched_get_priority_min(policy)
+
+ Get the minimum priority value for *policy*. *policy* is one of the
+ scheduling policy constants above.
+
+
+.. function:: sched_get_priority_max(policy)
+
+ Get the maximum priority value for *policy*. *policy* is one of the
+ scheduling policy constants above.
+
+
+.. function:: sched_setscheduler(pid, policy, param)
+
+ Set the scheduling policy for the process with PID *pid*. A *pid* of 0 means
+ the calling process. *policy* is one of the scheduling policy constants
+ above. *param* is a :class:`sched_param` instance.
+
+
+.. function:: sched_getscheduler(pid)
+
+ Return the scheduling policy for the process with PID *pid*. A *pid* of 0
+ means the calling process. The result is one of the scheduling policy
+ constants above.
+
+
+.. function:: sched_setparam(pid, param)
+
+ Set a scheduling parameters for the process with PID *pid*. A *pid* of 0 means
+ the calling process. *param* is a :class:`sched_param` instance.
+
+
+.. function:: sched_getparam(pid)
+
+ Return the scheduling parameters as a :class:`sched_param` instance for the
+ process with PID *pid*. A *pid* of 0 means the calling process.
+
+
+.. function:: sched_rr_get_interval(pid)
+
+ Return the round-robin quantum in seconds for the process with PID *pid*. A
+ *pid* of 0 means the calling process.
+
+
+.. function:: sched_yield()
+
+ Voluntarily relinquish the CPU.
+
+
+.. class:: cpu_set(ncpus)
+
+ :class:`cpu_set` represents a set of CPUs on which a process is eligible to
+ run. *ncpus* is the number of CPUs the set should describe. Methods on
+ :class:`cpu_set` allow CPUs to be add or removed.
+
+ :class:`cpu_set` supports the AND, OR, and XOR bitwise operations. For
+ example, given two cpu_sets, ``one`` and ``two``, ``one | two`` returns a
+ :class:`cpu_set` containing the cpus enabled both in ``one`` and ``two``.
+
+ .. method:: set(i)
+
+ Enable CPU *i*.
+
+ .. method:: clear(i)
+
+ Remove CPU *i*.
+
+ .. method:: isset(i)
+
+ Return ``True`` if CPU *i* is enabled in the set.
+
+ .. method:: count()
+
+ Return the number of enabled CPUs in the set.
+
+ .. method:: zero()
+
+ Clear the set completely.
+
+
+.. function:: sched_setaffinity(pid, mask)
+
+ Restrict the process with PID *pid* to a set of CPUs. *mask* is a
+ :class:`cpu_set` instance.
+
+
+.. function:: sched_getaffinity(pid, size)
+
+ Return the :class:`cpu_set` the process with PID *pid* is restricted to. The
+ result will contain *size* CPUs.
+
+
.. _os-path:
Miscellaneous System Information
diff --git a/Doc/library/ossaudiodev.rst b/Doc/library/ossaudiodev.rst
index 0a08428..51c5857 100644
--- a/Doc/library/ossaudiodev.rst
+++ b/Doc/library/ossaudiodev.rst
@@ -38,6 +38,10 @@ the standard audio interface for Linux and recent versions of FreeBSD.
This probably all warrants a footnote or two, but I don't understand
things well enough right now to write it! --GPW
+.. versionchanged:: 3.3
+ Operations in this module now raise :exc:`OSError` where :exc:`IOError`
+ was raised.
+
.. seealso::
@@ -56,7 +60,7 @@ the standard audio interface for Linux and recent versions of FreeBSD.
what went wrong.
(If :mod:`ossaudiodev` receives an error from a system call such as
- :c:func:`open`, :c:func:`write`, or :c:func:`ioctl`, it raises :exc:`IOError`.
+ :c:func:`open`, :c:func:`write`, or :c:func:`ioctl`, it raises :exc:`OSError`.
Errors detected directly by :mod:`ossaudiodev` result in :exc:`OSSAudioError`.)
(For backwards compatibility, the exception class is also available as
@@ -168,7 +172,7 @@ The following methods each map to exactly one :func:`ioctl` system call. The
correspondence is obvious: for example, :meth:`setfmt` corresponds to the
``SNDCTL_DSP_SETFMT`` ioctl, and :meth:`sync` to ``SNDCTL_DSP_SYNC`` (this can
be useful when consulting the OSS documentation). If the underlying
-:func:`ioctl` fails, they all raise :exc:`IOError`.
+:func:`ioctl` fails, they all raise :exc:`OSError`.
.. method:: oss_audio_device.nonblock()
@@ -344,7 +348,7 @@ The mixer object provides two file-like methods:
.. method:: oss_mixer_device.close()
This method closes the open mixer device file. Any further attempts to use the
- mixer after this file is closed will raise an :exc:`IOError`.
+ mixer after this file is closed will raise an :exc:`OSError`.
.. method:: oss_mixer_device.fileno()
@@ -403,7 +407,7 @@ The remaining methods are specific to audio mixing:
returned, but both volumes are the same.
Raises :exc:`OSSAudioError` if an invalid control was is specified, or
- :exc:`IOError` if an unsupported control is specified.
+ :exc:`OSError` if an unsupported control is specified.
.. method:: oss_mixer_device.set(control, (left, right))
@@ -427,7 +431,7 @@ The remaining methods are specific to audio mixing:
.. method:: oss_mixer_device.set_recsrc(bitmask)
Call this function to specify a recording source. Returns a bitmask indicating
- the new recording source (or sources) if successful; raises :exc:`IOError` if an
+ the new recording source (or sources) if successful; raises :exc:`OSError` if an
invalid source was specified. To set the current recording source to the
microphone input::
diff --git a/Doc/library/packaging-misc.rst b/Doc/library/packaging-misc.rst
new file mode 100644
index 0000000..5e56247
--- /dev/null
+++ b/Doc/library/packaging-misc.rst
@@ -0,0 +1,27 @@
+.. temporary file for modules that don't need a dedicated file yet
+
+:mod:`packaging.errors` --- Packaging exceptions
+================================================
+
+.. module:: packaging.errors
+ :synopsis: Packaging exceptions.
+
+
+Provides exceptions used by the Packaging modules. Note that Packaging modules
+may raise standard exceptions; in particular, SystemExit is usually raised for
+errors that are obviously the end-user's fault (e.g. bad command-line arguments).
+
+This module is safe to use in ``from ... import *`` mode; it only exports
+symbols whose names start with ``Packaging`` and end with ``Error``.
+
+
+:mod:`packaging.manifest` --- The Manifest class
+================================================
+
+.. module:: packaging.manifest
+ :synopsis: The Manifest class, used for poking about the file system and
+ building lists of files.
+
+
+This module provides the :class:`Manifest` class, used for poking about the
+filesystem and building lists of files.
diff --git a/Doc/library/packaging.command.rst b/Doc/library/packaging.command.rst
new file mode 100644
index 0000000..6a85351
--- /dev/null
+++ b/Doc/library/packaging.command.rst
@@ -0,0 +1,111 @@
+:mod:`packaging.command` --- Standard Packaging commands
+========================================================
+
+.. module:: packaging.command
+ :synopsis: Standard packaging commands.
+
+
+This subpackage contains one module for each standard Packaging command, such as
+:command:`build` or :command:`upload`. Each command is implemented as a
+separate module, with the command name as the name of the module and of the
+class defined therein.
+
+
+
+:mod:`packaging.command.cmd` --- Abstract base class for Packaging commands
+===========================================================================
+
+.. module:: packaging.command.cmd
+ :synopsis: Abstract base class for commands.
+
+
+This module supplies the abstract base class :class:`Command`. This class is
+subclassed by the modules in the packaging.command subpackage.
+
+
+.. class:: Command(dist)
+
+ Abstract base class for defining command classes, the "worker bees" of the
+ Packaging. A useful analogy for command classes is to think of them as
+ subroutines with local variables called *options*. The options are declared
+ in :meth:`initialize_options` and defined (given their final values) in
+ :meth:`finalize_options`, both of which must be defined by every command
+ class. The distinction between the two is necessary because option values
+ might come from the outside world (command line, config file, ...), and any
+ options dependent on other options must be computed after these outside
+ influences have been processed --- hence :meth:`finalize_options`. The body
+ of the subroutine, where it does all its work based on the values of its
+ options, is the :meth:`run` method, which must also be implemented by every
+ command class.
+
+ The class constructor takes a single argument *dist*, a
+ :class:`~packaging.dist.Distribution` instance.
+
+
+Creating a new Packaging command
+--------------------------------
+
+This section outlines the steps to create a new Packaging command.
+
+.. XXX the following paragraph is focused on the stdlib; expand it to document
+ how to write and register a command in third-party projects
+
+A new command lives in a module in the :mod:`packaging.command` package. There
+is a sample template in that directory called :file:`command_template`. Copy
+this file to a new module with the same name as the new command you're
+implementing. This module should implement a class with the same name as the
+module (and the command). So, for instance, to create the command
+``peel_banana`` (so that users can run ``setup.py peel_banana``), you'd copy
+:file:`command_template` to :file:`packaging/command/peel_banana.py`, then edit
+it so that it's implementing the class :class:`peel_banana`, a subclass of
+:class:`Command`. It must define the following methods:
+
+.. method:: Command.initialize_options()
+
+ Set default values for all the options that this command supports. Note that
+ these defaults may be overridden by other commands, by the setup script, by
+ config files, or by the command line. Thus, this is not the place to code
+ dependencies between options; generally, :meth:`initialize_options`
+ implementations are just a bunch of ``self.foo = None`` assignments.
+
+
+.. method:: Command.finalize_options()
+
+ Set final values for all the options that this command supports. This is
+ always called as late as possible, i.e. after any option assignments from the
+ command line or from other commands have been done. Thus, this is the place
+ to code option dependencies: if *foo* depends on *bar*, then it is safe to
+ set *foo* from *bar* as long as *foo* still has the same value it was
+ assigned in :meth:`initialize_options`.
+
+
+.. method:: Command.run()
+
+ A command's raison d'etre: carry out the action it exists to perform,
+ controlled by the options initialized in :meth:`initialize_options`,
+ customized by other commands, the setup script, the command line, and config
+ files, and finalized in :meth:`finalize_options`. All terminal output and
+ filesystem interaction should be done by :meth:`run`.
+
+
+Command classes may define this attribute:
+
+
+.. attribute:: Command.sub_commands
+
+ *sub_commands* formalizes the notion of a "family" of commands,
+ e.g. ``install_dist`` as the parent with sub-commands ``install_lib``,
+ ``install_headers``, etc. The parent of a family of commands defines
+ *sub_commands* as a class attribute; it's a list of 2-tuples ``(command_name,
+ predicate)``, with *command_name* a string and *predicate* a function, a
+ string or ``None``. *predicate* is a method of the parent command that
+ determines whether the corresponding command is applicable in the current
+ situation. (E.g. ``install_headers`` is only applicable if we have any C
+ header files to install.) If *predicate* is ``None``, that command is always
+ applicable.
+
+ *sub_commands* is usually defined at the *end* of a class, because
+ predicates can be methods of the class, so they must already have been
+ defined. The canonical example is the :command:`install_dist` command.
+
+.. XXX document how to add a custom command to another one's subcommands
diff --git a/Doc/library/packaging.compiler.rst b/Doc/library/packaging.compiler.rst
new file mode 100644
index 0000000..10780d1
--- /dev/null
+++ b/Doc/library/packaging.compiler.rst
@@ -0,0 +1,681 @@
+:mod:`packaging.compiler` --- Compiler classes
+==============================================
+
+.. module:: packaging.compiler
+ :synopsis: Compiler classes to build C/C++ extensions or libraries.
+
+
+This subpackage contains an abstract base class representing a compiler and
+concrete implementations for common compilers. The compiler classes should not
+be instantiated directly, but created using the :func:`new_compiler` factory
+function. Compiler types provided by Packaging are listed in
+:ref:`packaging-standard-compilers`.
+
+
+Public functions
+----------------
+
+.. function:: new_compiler(plat=None, compiler=None, verbose=0, dry_run=0, force=0)
+
+ Factory function to generate an instance of some
+ :class:`~.ccompiler.CCompiler` subclass for the requested platform or
+ compiler type.
+
+ If no argument is given for *plat* and *compiler*, the default compiler type
+ for the platform (:attr:`os.name`) will be used: ``'unix'`` for Unix and
+ Mac OS X, ``'msvc'`` for Windows.
+
+ If *plat* is given, it must be one of ``'posix'``, ``'darwin'`` or ``'nt'``.
+ An invalid value will not raise an exception but use the default compiler
+ type for the current platform.
+
+ .. XXX errors should never pass silently; this behavior is particularly
+ harmful when a compiler type is given as first argument
+
+ If *compiler* is given, *plat* will be ignored, allowing you to get for
+ example a ``'unix'`` compiler object under Windows or an ``'msvc'`` compiler
+ under Unix. However, not all compiler types can be instantiated on every
+ platform.
+
+
+.. function:: customize_compiler(compiler)
+
+ Do any platform-specific customization of a CCompiler instance. Mainly
+ needed on Unix to plug in the information that varies across Unices and is
+ stored in CPython's Makefile.
+
+
+.. function:: gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries)
+
+ Generate linker options for searching library directories and linking with
+ specific libraries. *libraries* and *library_dirs* are, respectively, lists
+ of library names (not filenames!) and search directories. Returns a list of
+ command-line options suitable for use with some compiler (depending on the
+ two format strings passed in).
+
+
+.. function:: gen_preprocess_options(macros, include_dirs)
+
+ Generate C preprocessor options (:option:`-D`, :option:`-U`, :option:`-I`) as
+ used by at least two types of compilers: the typical Unix compiler and Visual
+ C++. *macros* is the usual thing, a list of 1- or 2-tuples, where ``(name,)``
+ means undefine (:option:`-U`) macro *name*, and ``(name, value)`` means
+ define (:option:`-D`) macro *name* to *value*. *include_dirs* is just a list
+ of directory names to be added to the header file search path (:option:`-I`).
+ Returns a list of command-line options suitable for either Unix compilers or
+ Visual C++.
+
+
+.. function:: get_default_compiler(osname, platform)
+
+ Determine the default compiler to use for the given platform.
+
+ *osname* should be one of the standard Python OS names (i.e. the ones
+ returned by ``os.name``) and *platform* the common value returned by
+ ``sys.platform`` for the platform in question.
+
+ The default values are ``os.name`` and ``sys.platform``.
+
+
+.. function:: set_compiler(location)
+
+ Add or change a compiler
+
+
+.. function:: show_compilers()
+
+ Print list of available compilers (used by the :option:`--help-compiler`
+ options to :command:`build`, :command:`build_ext`, :command:`build_clib`).
+
+
+.. _packaging-standard-compilers:
+
+Standard compilers
+------------------
+
+Concrete subclasses of :class:`~.ccompiler.CCompiler` are provided in submodules
+of the :mod:`packaging.compiler` package. You do not need to import them, using
+:func:`new_compiler` is the public API to use. This table documents the
+standard compilers; be aware that they can be replaced by other classes on your
+platform.
+
+=============== ======================================================== =======
+name description notes
+=============== ======================================================== =======
+``'unix'`` typical Unix-style command-line C compiler [#]_
+``'msvc'`` Microsoft compiler [#]_
+``'bcpp'`` Borland C++ compiler
+``'cygwin'`` Cygwin compiler (Windows port of GCC)
+``'mingw32'`` Mingw32 port of GCC (same as Cygwin in no-Cygwin mode)
+=============== ======================================================== =======
+
+
+.. [#] The Unix compiler class assumes this behavior:
+
+ * macros defined with :option:`-Dname[=value]`
+
+ * macros undefined with :option:`-Uname`
+
+ * include search directories specified with :option:`-Idir`
+
+ * libraries specified with :option:`-llib`
+
+ * library search directories specified with :option:`-Ldir`
+
+ * compile handled by :program:`cc` (or similar) executable with
+ :option:`-c` option: compiles :file:`.c` to :file:`.o`
+
+ * link static library handled by :program:`ar` command (possibly with
+ :program:`ranlib`)
+
+ * link shared library handled by :program:`cc` :option:`-shared`
+
+
+.. [#] On Windows, extension modules typically need to be compiled with the same
+ compiler that was used to compile CPython (for example Microsoft Visual
+ Studio .NET 2003 for CPython 2.4 and 2.5). The AMD64 and Itanium
+ binaries are created using the Platform SDK.
+
+ Under the hood, there are actually two different subclasses of
+ :class:`~.ccompiler.CCompiler` defined: one is compatible with MSVC 2005
+ and 2008, the other works with older versions. This should not be a
+ concern for regular use of the functions in this module.
+
+ Packaging will normally choose the right compiler, linker etc. on its
+ own. To override this choice, the environment variables
+ *DISTUTILS_USE_SDK* and *MSSdk* must be both set. *MSSdk* indicates that
+ the current environment has been setup by the SDK's ``SetEnv.Cmd``
+ script, or that the environment variables had been registered when the
+ SDK was installed; *DISTUTILS_USE_SDK* indicates that the user has made
+ an explicit choice to override the compiler selection done by Packaging.
+
+ .. TODO document the envvars in Doc/using and the man page
+
+
+:mod:`packaging.compiler.ccompiler` --- CCompiler base class
+============================================================
+
+.. module:: packaging.compiler.ccompiler
+ :synopsis: Abstract CCompiler class.
+
+
+This module provides the abstract base class for the :class:`CCompiler`
+classes. A :class:`CCompiler` instance can be used for all the compile and
+link steps needed to build a single project. Methods are provided to set
+options for the compiler --- macro definitions, include directories, link path,
+libraries and the like.
+
+.. class:: CCompiler([verbose=0, dry_run=0, force=0])
+
+ The abstract base class :class:`CCompiler` defines the interface that must be
+ implemented by real compiler classes. The class also has some utility
+ methods used by several compiler classes.
+
+ The basic idea behind a compiler abstraction class is that each instance can
+ be used for all the compile/link steps in building a single project. Thus,
+ attributes common to all of those compile and link steps --- include
+ directories, macros to define, libraries to link against, etc. --- are
+ attributes of the compiler instance. To allow for variability in how
+ individual files are treated, most of those attributes may be varied on a
+ per-compilation or per-link basis.
+
+ The constructor for each subclass creates an instance of the Compiler object.
+ Flags are *verbose* (show verbose output), *dry_run* (don't actually execute
+ the steps) and *force* (rebuild everything, regardless of dependencies). All
+ of these flags default to ``0`` (off). Note that you probably don't want to
+ instantiate :class:`CCompiler` or one of its subclasses directly - use the
+ :func:`packaging.CCompiler.new_compiler` factory function instead.
+
+ The following methods allow you to manually alter compiler options for the
+ instance of the Compiler class.
+
+
+ .. method:: CCompiler.add_include_dir(dir)
+
+ Add *dir* to the list of directories that will be searched for header
+ files. The compiler is instructed to search directories in the order in
+ which they are supplied by successive calls to :meth:`add_include_dir`.
+
+
+ .. method:: CCompiler.set_include_dirs(dirs)
+
+ Set the list of directories that will be searched to *dirs* (a list of
+ strings). Overrides any preceding calls to :meth:`add_include_dir`;
+ subsequent calls to :meth:`add_include_dir` add to the list passed to
+ :meth:`set_include_dirs`. This does not affect any list of standard
+ include directories that the compiler may search by default.
+
+
+ .. method:: CCompiler.add_library(libname)
+
+ Add *libname* to the list of libraries that will be included in all links
+ driven by this compiler object. Note that *libname* should *not* be the
+ name of a file containing a library, but the name of the library itself:
+ the actual filename will be inferred by the linker, the compiler, or the
+ compiler class (depending on the platform).
+
+ The linker will be instructed to link against libraries in the order they
+ were supplied to :meth:`add_library` and/or :meth:`set_libraries`. It is
+ perfectly valid to duplicate library names; the linker will be instructed
+ to link against libraries as many times as they are mentioned.
+
+
+ .. method:: CCompiler.set_libraries(libnames)
+
+ Set the list of libraries to be included in all links driven by this
+ compiler object to *libnames* (a list of strings). This does not affect
+ any standard system libraries that the linker may include by default.
+
+
+ .. method:: CCompiler.add_library_dir(dir)
+
+ Add *dir* to the list of directories that will be searched for libraries
+ specified to :meth:`add_library` and :meth:`set_libraries`. The linker
+ will be instructed to search for libraries in the order they are supplied
+ to :meth:`add_library_dir` and/or :meth:`set_library_dirs`.
+
+
+ .. method:: CCompiler.set_library_dirs(dirs)
+
+ Set the list of library search directories to *dirs* (a list of strings).
+ This does not affect any standard library search path that the linker may
+ search by default.
+
+
+ .. method:: CCompiler.add_runtime_library_dir(dir)
+
+ Add *dir* to the list of directories that will be searched for shared
+ libraries at runtime.
+
+
+ .. method:: CCompiler.set_runtime_library_dirs(dirs)
+
+ Set the list of directories to search for shared libraries at runtime to
+ *dirs* (a list of strings). This does not affect any standard search path
+ that the runtime linker may search by default.
+
+
+ .. method:: CCompiler.define_macro(name[, value=None])
+
+ Define a preprocessor macro for all compilations driven by this compiler
+ object. The optional parameter *value* should be a string; if it is not
+ supplied, then the macro will be defined without an explicit value and the
+ exact outcome depends on the compiler used (XXX true? does ANSI say
+ anything about this?)
+
+
+ .. method:: CCompiler.undefine_macro(name)
+
+ Undefine a preprocessor macro for all compilations driven by this compiler
+ object. If the same macro is defined by :meth:`define_macro` and
+ undefined by :meth:`undefine_macro` the last call takes precedence
+ (including multiple redefinitions or undefinitions). If the macro is
+ redefined/undefined on a per-compilation basis (i.e. in the call to
+ :meth:`compile`), then that takes precedence.
+
+
+ .. method:: CCompiler.add_link_object(object)
+
+ Add *object* to the list of object files (or analogues, such as explicitly
+ named library files or the output of "resource compilers") to be included
+ in every link driven by this compiler object.
+
+
+ .. method:: CCompiler.set_link_objects(objects)
+
+ Set the list of object files (or analogues) to be included in every link
+ to *objects*. This does not affect any standard object files that the
+ linker may include by default (such as system libraries).
+
+ The following methods implement methods for autodetection of compiler
+ options, providing some functionality similar to GNU :program:`autoconf`.
+
+
+ .. method:: CCompiler.detect_language(sources)
+
+ Detect the language of a given file, or list of files. Uses the instance
+ attributes :attr:`language_map` (a dictionary), and :attr:`language_order`
+ (a list) to do the job.
+
+
+ .. method:: CCompiler.find_library_file(dirs, lib[, debug=0])
+
+ Search the specified list of directories for a static or shared library file
+ *lib* and return the full path to that file. If *debug* is true, look for a
+ debugging version (if that makes sense on the current platform). Return
+ ``None`` if *lib* wasn't found in any of the specified directories.
+
+
+ .. method:: CCompiler.has_function(funcname [, includes=None, include_dirs=None, libraries=None, library_dirs=None])
+
+ Return a boolean indicating whether *funcname* is supported on the current
+ platform. The optional arguments can be used to augment the compilation
+ environment by providing additional include files and paths and libraries and
+ paths.
+
+
+ .. method:: CCompiler.library_dir_option(dir)
+
+ Return the compiler option to add *dir* to the list of directories searched for
+ libraries.
+
+
+ .. method:: CCompiler.library_option(lib)
+
+ Return the compiler option to add *dir* to the list of libraries linked into the
+ shared library or executable.
+
+
+ .. method:: CCompiler.runtime_library_dir_option(dir)
+
+ Return the compiler option to add *dir* to the list of directories searched for
+ runtime libraries.
+
+
+ .. method:: CCompiler.set_executables(**args)
+
+ Define the executables (and options for them) that will be run to perform the
+ various stages of compilation. The exact set of executables that may be
+ specified here depends on the compiler class (via the 'executables' class
+ attribute), but most will have:
+
+ +--------------+------------------------------------------+
+ | attribute | description |
+ +==============+==========================================+
+ | *compiler* | the C/C++ compiler |
+ +--------------+------------------------------------------+
+ | *linker_so* | linker used to create shared objects and |
+ | | libraries |
+ +--------------+------------------------------------------+
+ | *linker_exe* | linker used to create binary executables |
+ +--------------+------------------------------------------+
+ | *archiver* | static library creator |
+ +--------------+------------------------------------------+
+
+ On platforms with a command line (Unix, DOS/Windows), each of these is a string
+ that will be split into executable name and (optional) list of arguments.
+ (Splitting the string is done similarly to how Unix shells operate: words are
+ delimited by spaces, but quotes and backslashes can override this. See
+ :func:`packaging.util.split_quoted`.)
+
+ The following methods invoke stages in the build process.
+
+
+ .. method:: CCompiler.compile(sources[, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None])
+
+ Compile one or more source files. Generates object files (e.g. transforms a
+ :file:`.c` file to a :file:`.o` file.)
+
+ *sources* must be a list of filenames, most likely C/C++ files, but in reality
+ anything that can be handled by a particular compiler and compiler class (e.g.
+ an ``'msvc'`` compiler can handle resource files in *sources*). Return a list of
+ object filenames, one per source filename in *sources*. Depending on the
+ implementation, not all source files will necessarily be compiled, but all
+ corresponding object filenames will be returned.
+
+ If *output_dir* is given, object files will be put under it, while retaining
+ their original path component. That is, :file:`foo/bar.c` normally compiles to
+ :file:`foo/bar.o` (for a Unix implementation); if *output_dir* is *build*, then
+ it would compile to :file:`build/foo/bar.o`.
+
+ *macros*, if given, must be a list of macro definitions. A macro definition is
+ either a ``(name, value)`` 2-tuple or a ``(name,)`` 1-tuple. The former defines
+ a macro; if the value is ``None``, the macro is defined without an explicit
+ value. The 1-tuple case undefines a macro. Later
+ definitions/redefinitions/undefinitions take precedence.
+
+ *include_dirs*, if given, must be a list of strings, the directories to add to
+ the default include file search path for this compilation only.
+
+ *debug* is a boolean; if true, the compiler will be instructed to output debug
+ symbols in (or alongside) the object file(s).
+
+ *extra_preargs* and *extra_postargs* are implementation-dependent. On platforms
+ that have the notion of a command line (e.g. Unix, DOS/Windows), they are most
+ likely lists of strings: extra command-line arguments to prepend/append to the
+ compiler command line. On other platforms, consult the implementation class
+ documentation. In any event, they are intended as an escape hatch for those
+ occasions when the abstract compiler framework doesn't cut the mustard.
+
+ *depends*, if given, is a list of filenames that all targets depend on. If a
+ source file is older than any file in depends, then the source file will be
+ recompiled. This supports dependency tracking, but only at a coarse
+ granularity.
+
+ Raises :exc:`CompileError` on failure.
+
+
+ .. method:: CCompiler.create_static_lib(objects, output_libname[, output_dir=None, debug=0, target_lang=None])
+
+ Link a bunch of stuff together to create a static library file. The "bunch of
+ stuff" consists of the list of object files supplied as *objects*, the extra
+ object files supplied to :meth:`add_link_object` and/or
+ :meth:`set_link_objects`, the libraries supplied to :meth:`add_library` and/or
+ :meth:`set_libraries`, and the libraries supplied as *libraries* (if any).
+
+ *output_libname* should be a library name, not a filename; the filename will be
+ inferred from the library name. *output_dir* is the directory where the library
+ file will be put. XXX defaults to what?
+
+ *debug* is a boolean; if true, debugging information will be included in the
+ library (note that on most platforms, it is the compile step where this matters:
+ the *debug* flag is included here just for consistency).
+
+ *target_lang* is the target language for which the given objects are being
+ compiled. This allows specific linkage time treatment of certain languages.
+
+ Raises :exc:`LibError` on failure.
+
+
+ .. method:: CCompiler.link(target_desc, objects, output_filename[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, target_lang=None])
+
+ Link a bunch of stuff together to create an executable or shared library file.
+
+ The "bunch of stuff" consists of the list of object files supplied as *objects*.
+ *output_filename* should be a filename. If *output_dir* is supplied,
+ *output_filename* is relative to it (i.e. *output_filename* can provide
+ directory components if needed).
+
+ *libraries* is a list of libraries to link against. These are library names,
+ not filenames, since they're translated into filenames in a platform-specific
+ way (e.g. *foo* becomes :file:`libfoo.a` on Unix and :file:`foo.lib` on
+ DOS/Windows). However, they can include a directory component, which means the
+ linker will look in that specific directory rather than searching all the normal
+ locations.
+
+ *library_dirs*, if supplied, should be a list of directories to search for
+ libraries that were specified as bare library names (i.e. no directory
+ component). These are on top of the system default and those supplied to
+ :meth:`add_library_dir` and/or :meth:`set_library_dirs`. *runtime_library_dirs*
+ is a list of directories that will be embedded into the shared library and used
+ to search for other shared libraries that \*it\* depends on at run-time. (This
+ may only be relevant on Unix.)
+
+ *export_symbols* is a list of symbols that the shared library will export.
+ (This appears to be relevant only on Windows.)
+
+ *debug* is as for :meth:`compile` and :meth:`create_static_lib`, with the
+ slight distinction that it actually matters on most platforms (as opposed to
+ :meth:`create_static_lib`, which includes a *debug* flag mostly for form's
+ sake).
+
+ *extra_preargs* and *extra_postargs* are as for :meth:`compile` (except of
+ course that they supply command-line arguments for the particular linker being
+ used).
+
+ *target_lang* is the target language for which the given objects are being
+ compiled. This allows specific linkage time treatment of certain languages.
+
+ Raises :exc:`LinkError` on failure.
+
+
+ .. method:: CCompiler.link_executable(objects, output_progname[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, target_lang=None])
+
+ Link an executable. *output_progname* is the name of the file executable, while
+ *objects* are a list of object filenames to link in. Other arguments are as for
+ the :meth:`link` method.
+
+
+ .. method:: CCompiler.link_shared_lib(objects, output_libname[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, target_lang=None])
+
+ Link a shared library. *output_libname* is the name of the output library,
+ while *objects* is a list of object filenames to link in. Other arguments are
+ as for the :meth:`link` method.
+
+
+ .. method:: CCompiler.link_shared_object(objects, output_filename[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, target_lang=None])
+
+ Link a shared object. *output_filename* is the name of the shared object that
+ will be created, while *objects* is a list of object filenames to link in.
+ Other arguments are as for the :meth:`link` method.
+
+
+ .. method:: CCompiler.preprocess(source[, output_file=None, macros=None, include_dirs=None, extra_preargs=None, extra_postargs=None])
+
+ Preprocess a single C/C++ source file, named in *source*. Output will be written
+ to file named *output_file*, or *stdout* if *output_file* not supplied.
+ *macros* is a list of macro definitions as for :meth:`compile`, which will
+ augment the macros set with :meth:`define_macro` and :meth:`undefine_macro`.
+ *include_dirs* is a list of directory names that will be added to the default
+ list, in the same way as :meth:`add_include_dir`.
+
+ Raises :exc:`PreprocessError` on failure.
+
+ The following utility methods are defined by the :class:`CCompiler` class, for
+ use by the various concrete subclasses.
+
+
+ .. method:: CCompiler.executable_filename(basename[, strip_dir=0, output_dir=''])
+
+ Returns the filename of the executable for the given *basename*. Typically for
+ non-Windows platforms this is the same as the basename, while Windows will get
+ a :file:`.exe` added.
+
+
+ .. method:: CCompiler.library_filename(libname[, lib_type='static', strip_dir=0, output_dir=''])
+
+ Returns the filename for the given library name on the current platform. On Unix
+ a library with *lib_type* of ``'static'`` will typically be of the form
+ :file:`liblibname.a`, while a *lib_type* of ``'dynamic'`` will be of the form
+ :file:`liblibname.so`.
+
+
+ .. method:: CCompiler.object_filenames(source_filenames[, strip_dir=0, output_dir=''])
+
+ Returns the name of the object files for the given source files.
+ *source_filenames* should be a list of filenames.
+
+
+ .. method:: CCompiler.shared_object_filename(basename[, strip_dir=0, output_dir=''])
+
+ Returns the name of a shared object file for the given file name *basename*.
+
+
+ .. method:: CCompiler.execute(func, args[, msg=None, level=1])
+
+ Invokes :func:`packaging.util.execute` This method invokes a Python function
+ *func* with the given arguments *args*, after logging and taking into account
+ the *dry_run* flag. XXX see also.
+
+
+ .. method:: CCompiler.spawn(cmd)
+
+ Invokes :func:`packaging.util.spawn`. This invokes an external process to run
+ the given command. XXX see also.
+
+
+ .. method:: CCompiler.mkpath(name[, mode=511])
+
+ Invokes :func:`packaging.dir_util.mkpath`. This creates a directory and any
+ missing ancestor directories. XXX see also.
+
+
+ .. method:: CCompiler.move_file(src, dst)
+
+ Invokes :meth:`packaging.file_util.move_file`. Renames *src* to *dst*. XXX see
+ also.
+
+
+:mod:`packaging.compiler.extension` --- The Extension class
+===========================================================
+
+.. module:: packaging.compiler.extension
+ :synopsis: Class used to represent C/C++ extension modules.
+
+
+This module provides the :class:`Extension` class, used to represent C/C++
+extension modules.
+
+.. class:: Extension
+
+ The Extension class describes a single C or C++ extension module. It accepts
+ the following keyword arguments in its constructor:
+
+ +------------------------+--------------------------------+---------------------------+
+ | argument name | value | type |
+ +========================+================================+===========================+
+ | *name* | the full name of the | string |
+ | | extension, including any | |
+ | | packages --- i.e. *not* a | |
+ | | filename or pathname, but | |
+ | | Python dotted name | |
+ +------------------------+--------------------------------+---------------------------+
+ | *sources* | list of source filenames, | list of strings |
+ | | relative to the distribution | |
+ | | root (where the setup script | |
+ | | lives), in Unix form (slash- | |
+ | | separated) for portability. | |
+ | | Source files may be C, C++, | |
+ | | SWIG (.i), platform-specific | |
+ | | resource files, or whatever | |
+ | | else is recognized by the | |
+ | | :command:`build_ext` command | |
+ | | as source for a Python | |
+ | | extension. | |
+ +------------------------+--------------------------------+---------------------------+
+ | *include_dirs* | list of directories to search | list of strings |
+ | | for C/C++ header files (in | |
+ | | Unix form for portability) | |
+ +------------------------+--------------------------------+---------------------------+
+ | *define_macros* | list of macros to define; each | list of tuples |
+ | | macro is defined using a | |
+ | | 2-tuple ``(name, value)``, | |
+ | | where *value* is | |
+ | | either the string to define it | |
+ | | to or ``None`` to define it | |
+ | | without a particular value | |
+ | | (equivalent of ``#define FOO`` | |
+ | | in source or :option:`-DFOO` | |
+ | | on Unix C compiler command | |
+ | | line) | |
+ +------------------------+--------------------------------+---------------------------+
+ | *undef_macros* | list of macros to undefine | list of strings |
+ | | explicitly | |
+ +------------------------+--------------------------------+---------------------------+
+ | *library_dirs* | list of directories to search | list of strings |
+ | | for C/C++ libraries at link | |
+ | | time | |
+ +------------------------+--------------------------------+---------------------------+
+ | *libraries* | list of library names (not | list of strings |
+ | | filenames or paths) to link | |
+ | | against | |
+ +------------------------+--------------------------------+---------------------------+
+ | *runtime_library_dirs* | list of directories to search | list of strings |
+ | | for C/C++ libraries at run | |
+ | | time (for shared extensions, | |
+ | | this is when the extension is | |
+ | | loaded) | |
+ +------------------------+--------------------------------+---------------------------+
+ | *extra_objects* | list of extra files to link | list of strings |
+ | | with (e.g. object files not | |
+ | | implied by 'sources', static | |
+ | | library that must be | |
+ | | explicitly specified, binary | |
+ | | resource files, etc.) | |
+ +------------------------+--------------------------------+---------------------------+
+ | *extra_compile_args* | any extra platform- and | list of strings |
+ | | compiler-specific information | |
+ | | to use when compiling the | |
+ | | source files in 'sources'. For | |
+ | | platforms and compilers where | |
+ | | a command line makes sense, | |
+ | | this is typically a list of | |
+ | | command-line arguments, but | |
+ | | for other platforms it could | |
+ | | be anything. | |
+ +------------------------+--------------------------------+---------------------------+
+ | *extra_link_args* | any extra platform- and | list of strings |
+ | | compiler-specific information | |
+ | | to use when linking object | |
+ | | files together to create the | |
+ | | extension (or to create a new | |
+ | | static Python interpreter). | |
+ | | Similar interpretation as for | |
+ | | 'extra_compile_args'. | |
+ +------------------------+--------------------------------+---------------------------+
+ | *export_symbols* | list of symbols to be exported | list of strings |
+ | | from a shared extension. Not | |
+ | | used on all platforms, and not | |
+ | | generally necessary for Python | |
+ | | extensions, which typically | |
+ | | export exactly one symbol: | |
+ | | ``init`` + extension_name. | |
+ +------------------------+--------------------------------+---------------------------+
+ | *depends* | list of files that the | list of strings |
+ | | extension depends on | |
+ +------------------------+--------------------------------+---------------------------+
+ | *language* | extension language (i.e. | string |
+ | | ``'c'``, ``'c++'``, | |
+ | | ``'objc'``). Will be detected | |
+ | | from the source extensions if | |
+ | | not provided. | |
+ +------------------------+--------------------------------+---------------------------+
+ | *optional* | specifies that a build failure | boolean |
+ | | in the extension should not | |
+ | | abort the build process, but | |
+ | | simply skip the extension. | |
+ +------------------------+--------------------------------+---------------------------+
+
+To distribute extension modules that live in a package (e.g. ``package.ext``),
+you need to create a :file:`{package}/__init__.py` file to let Python recognize
+and import your module.
diff --git a/Doc/library/packaging.database.rst b/Doc/library/packaging.database.rst
new file mode 100644
index 0000000..aaa2cb9
--- /dev/null
+++ b/Doc/library/packaging.database.rst
@@ -0,0 +1,324 @@
+:mod:`packaging.database` --- Database of installed distributions
+=================================================================
+
+.. module:: packaging.database
+ :synopsis: Functions to query and manipulate installed distributions.
+
+
+This module provides an implementation of :PEP:`376`. It was originally
+intended to land in :mod:`pkgutil`, but with the inclusion of Packaging in the
+standard library, it was thought best to include it in a submodule of
+:mod:`packaging`, leaving :mod:`pkgutil` to deal with imports.
+
+Installed Python distributions are represented by instances of
+:class:`Distribution`, or :class:`EggInfoDistribution` for legacy egg formats.
+Most functions also provide an extra argument ``use_egg_info`` to take legacy
+distributions into account.
+
+
+Classes representing installed distributions
+--------------------------------------------
+
+.. class:: Distribution(path)
+
+ Class representing an installed distribution. It is different from
+ :class:`packaging.dist.Distribution` which holds the list of files, the
+ metadata and options during the run of a Packaging command.
+
+ Instantiate with the *path* to a ``.dist-info`` directory. Instances can be
+ compared and sorted. Other available methods are:
+
+ .. XXX describe how comparison works
+
+ .. method:: get_distinfo_file(path, binary=False)
+
+ Return a read-only file object for a file located at
+ :file:`{project}-{version}.dist-info/{path}`. *path* should be a
+ ``'/'``-separated path relative to the ``.dist-info`` directory or an
+ absolute path; if it is an absolute path and doesn't start with the path
+ to the :file:`.dist-info` directory, a :class:`PackagingError` is raised.
+
+ If *binary* is ``True``, the file is opened in binary mode.
+
+ .. method:: get_resource_path(relative_path)
+
+ .. TODO
+
+ .. method:: list_distinfo_files(local=False)
+
+ Return an iterator over all files located in the :file:`.dist-info`
+ directory. If *local* is ``True``, each returned path is transformed into
+ a local absolute path, otherwise the raw value found in the :file:`RECORD`
+ file is returned.
+
+ .. method:: list_installed_files(local=False)
+
+ Iterate over the files installed with the distribution and registered in
+ the :file:`RECORD` file and yield a tuple ``(path, md5, size)`` for each
+ line. If *local* is ``True``, the returned path is transformed into a
+ local absolute path, otherwise the raw value is returned.
+
+ A local absolute path is an absolute path in which occurrences of ``'/'``
+ have been replaced by :data:`os.sep`.
+
+ .. method:: uses(path)
+
+ Check whether *path* was installed by this distribution (i.e. if the path
+ is present in the :file:`RECORD` file). *path* can be a local absolute
+ path or a relative ``'/'``-separated path. Returns a boolean.
+
+ Available attributes:
+
+ .. attribute:: metadata
+
+ Instance of :class:`packaging.metadata.Metadata` filled with the contents
+ of the :file:`{project}-{version}.dist-info/METADATA` file.
+
+ .. attribute:: name
+
+ Shortcut for ``metadata['Name']``.
+
+ .. attribute:: version
+
+ Shortcut for ``metadata['Version']``.
+
+ .. attribute:: requested
+
+ Boolean indicating whether this distribution was requested by the user of
+ automatically installed as a dependency.
+
+
+.. class:: EggInfoDistribution(path)
+
+ Class representing a legacy distribution. It is compatible with distutils'
+ and setuptools' :file:`.egg-info` and :file:`.egg` files and directories.
+
+ .. FIXME should be named EggDistribution
+
+ Instantiate with the *path* to an egg file or directory. Instances can be
+ compared and sorted. Other available methods are:
+
+ .. method:: list_installed_files(local=False)
+
+ .. method:: uses(path)
+
+ Available attributes:
+
+ .. attribute:: metadata
+
+ Instance of :class:`packaging.metadata.Metadata` filled with the contents
+ of the :file:`{project-version}.egg-info/PKG-INFO` or
+ :file:`{project-version}.egg` file.
+
+ .. attribute:: name
+
+ Shortcut for ``metadata['Name']``.
+
+ .. attribute:: version
+
+ Shortcut for ``metadata['Version']``.
+
+
+Functions to work with the database
+-----------------------------------
+
+.. function:: get_distribution(name, use_egg_info=False, paths=None)
+
+ Return an instance of :class:`Distribution` or :class:`EggInfoDistribution`
+ for the first installed distribution matching *name*. Egg distributions are
+ considered only if *use_egg_info* is true; if both a dist-info and an egg
+ file are found, the dist-info prevails. The directories to be searched are
+ given in *paths*, which defaults to :data:`sys.path`. Return ``None`` if no
+ matching distribution is found.
+
+ .. FIXME param should be named use_egg
+
+
+.. function:: get_distributions(use_egg_info=False, paths=None)
+
+ Return an iterator of :class:`Distribution` instances for all installed
+ distributions found in *paths* (defaults to :data:`sys.path`). If
+ *use_egg_info* is true, also return instances of :class:`EggInfoDistribution`
+ for legacy distributions found.
+
+
+.. function:: get_file_users(path)
+
+ Return an iterator over all distributions using *path*, a local absolute path
+ or a relative ``'/'``-separated path.
+
+ .. XXX does this work with prefixes or full file path only?
+
+
+.. function:: obsoletes_distribution(name, version=None, use_egg_info=False)
+
+ Return an iterator over all distributions that declare they obsolete *name*.
+ *version* is an optional argument to match only specific releases (see
+ :mod:`packaging.version`). If *use_egg_info* is true, legacy egg
+ distributions will be considered as well.
+
+
+.. function:: provides_distribution(name, version=None, use_egg_info=False)
+
+ Return an iterator over all distributions that declare they provide *name*.
+ *version* is an optional argument to match only specific releases (see
+ :mod:`packaging.version`). If *use_egg_info* is true, legacy egg
+ distributions will be considered as well.
+
+
+Utility functions
+-----------------
+
+.. function:: distinfo_dirname(name, version)
+
+ Escape *name* and *version* into a filename-safe form and return the
+ directory name built from them, for example
+ :file:`{safename}-{safeversion}.dist-info.` In *name*, runs of
+ non-alphanumeric characters are replaced with one ``'_'``; in *version*,
+ spaces become dots, and runs of other non-alphanumeric characters (except
+ dots) a replaced by one ``'-'``.
+
+ .. XXX wth spaces in version numbers?
+
+For performance purposes, the list of distributions is being internally
+cached. Caching is enabled by default, but you can control it with these
+functions:
+
+.. function:: clear_cache()
+
+ Clear the cache.
+
+.. function:: disable_cache()
+
+ Disable the cache, without clearing it.
+
+.. function:: enable_cache()
+
+ Enable the internal cache, without clearing it.
+
+
+Examples
+--------
+
+Print all information about a distribution
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Given a path to a ``.dist-info`` distribution, we shall print out all
+information that can be obtained using functions provided in this module::
+
+ import sys
+ import packaging.database
+
+ path = input()
+ # first create the Distribution instance
+ try:
+ dist = packaging.database.Distribution(path)
+ except FileNotFoundError:
+ sys.exit('No such distribution')
+
+ print('Information about %r' % dist.name)
+ print()
+
+ print('Files')
+ print('=====')
+ for path, md5, size in dist.list_installed_files():
+ print('* Path: %s' % path)
+ print(' Hash %s, Size: %s bytes' % (md5, size))
+ print()
+
+ print('Metadata')
+ print('========')
+ for key, value in dist.metadata.items():
+ print('%20s: %s' % (key, value))
+ print()
+
+ print('Extra')
+ print('=====')
+ if dist.requested:
+ print('* It was installed by user request')
+ else:
+ print('* It was installed as a dependency')
+
+If we save the script above as ``print_info.py``, we can use it to extract
+information from a :file:`.dist-info` directory. By typing in the console:
+
+.. code-block:: sh
+
+ $ echo /tmp/choxie/choxie-2.0.0.9.dist-info | python3 print_info.py
+
+we get the following output:
+
+.. code-block:: none
+
+ Information about 'choxie'
+
+ Files
+ =====
+ * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9/truffles.py
+ Hash 5e052db6a478d06bad9ae033e6bc08af, Size: 111 bytes
+ * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9/choxie/chocolate.py
+ Hash ac56bf496d8d1d26f866235b95f31030, Size: 214 bytes
+ * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9/choxie/__init__.py
+ Hash 416aab08dfa846f473129e89a7625bbc, Size: 25 bytes
+ * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9.dist-info/INSTALLER
+ Hash d41d8cd98f00b204e9800998ecf8427e, Size: 0 bytes
+ * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9.dist-info/METADATA
+ Hash 696a209967fef3c8b8f5a7bb10386385, Size: 225 bytes
+ * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9.dist-info/REQUESTED
+ Hash d41d8cd98f00b204e9800998ecf8427e, Size: 0 bytes
+ * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9.dist-info/RECORD
+ Hash None, Size: None bytes
+
+ Metadata
+ ========
+ Metadata-Version: 1.2
+ Name: choxie
+ Version: 2.0.0.9
+ Platform: []
+ Supported-Platform: UNKNOWN
+ Summary: Chocolate with a kick!
+ Description: UNKNOWN
+ Keywords: []
+ Home-page: UNKNOWN
+ Author: UNKNOWN
+ Author-email: UNKNOWN
+ Maintainer: UNKNOWN
+ Maintainer-email: UNKNOWN
+ License: UNKNOWN
+ Classifier: []
+ Download-URL: UNKNOWN
+ Obsoletes-Dist: ['truffles (<=0.8,>=0.5)', 'truffles (<=0.9,>=0.6)']
+ Project-URL: []
+ Provides-Dist: ['truffles (1.0)']
+ Requires-Dist: ['towel-stuff (0.1)']
+ Requires-Python: UNKNOWN
+ Requires-External: []
+
+ Extra
+ =====
+ * It was installed as a dependency
+
+
+Find out obsoleted distributions
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Now, we take tackle a different problem, we are interested in finding out
+which distributions have been obsoleted. This can be easily done as follows::
+
+ import packaging.database
+
+ # iterate over all distributions in the system
+ for dist in packaging.database.get_distributions():
+ name, version = dist.name, dist.version
+ # find out which distributions obsolete this name/version combination
+ replacements = packaging.database.obsoletes_distribution(name, version)
+ if replacements:
+ print('%r %s is obsoleted by' % (name, version),
+ ', '.join(repr(r.name) for r in replacements))
+
+This is how the output might look like:
+
+.. code-block:: none
+
+ 'strawberry' 0.6 is obsoleted by 'choxie'
+ 'grammar' 1.0a4 is obsoleted by 'towel-stuff'
diff --git a/Doc/library/packaging.depgraph.rst b/Doc/library/packaging.depgraph.rst
new file mode 100644
index 0000000..c384788
--- /dev/null
+++ b/Doc/library/packaging.depgraph.rst
@@ -0,0 +1,199 @@
+:mod:`packaging.depgraph` --- Dependency graph builder
+======================================================
+
+.. module:: packaging.depgraph
+ :synopsis: Graph builder for dependencies between releases.
+
+
+This module provides the means to analyse the dependencies between various
+distributions and to create a graph representing these dependency relationships.
+In this document, "distribution" refers to an instance of
+:class:`packaging.database.Distribution` or
+:class:`packaging.database.EggInfoDistribution`.
+
+.. XXX terminology problem with dist vs. release: dists are installed, but deps
+ use releases
+
+.. XXX explain how to use it with dists not installed: Distribution can only be
+ instantiated with a path, but this module is useful for remote dist too
+
+.. XXX functions should accept and return iterators, not lists
+
+
+The :class:`DependencyGraph` class
+----------------------------------
+
+.. class:: DependencyGraph
+
+ Represent a dependency graph between releases. The nodes are distribution
+ instances; the edge model dependencies. An edge from ``a`` to ``b`` means
+ that ``a`` depends on ``b``.
+
+ .. method:: add_distribution(distribution)
+
+ Add *distribution* to the graph.
+
+ .. method:: add_edge(x, y, label=None)
+
+ Add an edge from distribution *x* to distribution *y* with the given
+ *label* (string).
+
+ .. method:: add_missing(distribution, requirement)
+
+ Add a missing *requirement* (string) for the given *distribution*.
+
+ .. method:: repr_node(dist, level=1)
+
+ Print a subgraph starting from *dist*. *level* gives the depth of the
+ subgraph.
+
+ Direct access to the graph nodes and edges is provided through these
+ attributes:
+
+ .. attribute:: adjacency_list
+
+ Dictionary mapping distributions to a list of ``(other, label)`` tuples
+ where ``other`` is a distribution and the edge is labeled with ``label``
+ (i.e. the version specifier, if such was provided).
+
+ .. attribute:: reverse_list
+
+ Dictionary mapping distributions to a list of predecessors. This allows
+ efficient traversal.
+
+ .. attribute:: missing
+
+ Dictionary mapping distributions to a list of requirements that were not
+ provided by any distribution.
+
+
+Auxiliary functions
+-------------------
+
+.. function:: dependent_dists(dists, dist)
+
+ Recursively generate a list of distributions from *dists* that are dependent
+ on *dist*.
+
+ .. XXX what does member mean here: "dist is a member of *dists* for which we
+ are interested"
+
+.. function:: generate_graph(dists)
+
+ Generate a :class:`DependencyGraph` from the given list of distributions.
+
+ .. XXX make this alternate constructor a DepGraph classmethod or rename;
+ 'generate' can suggest it creates a file or an image, use 'make'
+
+.. function:: graph_to_dot(graph, f, skip_disconnected=True)
+
+ Write a DOT output for the graph to the file-like object *f*.
+
+ If *skip_disconnected* is true, all distributions that are not dependent on
+ any other distribution are skipped.
+
+ .. XXX why is this not a DepGraph method?
+
+
+Example Usage
+-------------
+
+Depict all dependenciess in the system
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+First, we shall generate a graph of all the distributions on the system
+and then create an image out of it using the tools provided by
+`Graphviz <http://www.graphviz.org/>`_::
+
+ from packaging.database import get_distributions
+ from packaging.depgraph import generate_graph
+
+ dists = list(get_distributions())
+ graph = generate_graph(dists)
+
+It would be interesting to print out the missing requirements. This can be done
+as follows::
+
+ for dist, reqs in graph.missing.items():
+ if reqs:
+ reqs = ' ,'.join(repr(req) for req in reqs)
+ print('Missing dependencies for %r: %s' % (dist.name, reqs))
+
+Example output is:
+
+.. code-block:: none
+
+ Missing dependencies for 'TurboCheetah': 'Cheetah'
+ Missing dependencies for 'TurboGears': 'ConfigObj', 'DecoratorTools', 'RuleDispatch'
+ Missing dependencies for 'jockey': 'PyKDE4.kdecore', 'PyKDE4.kdeui', 'PyQt4.QtCore', 'PyQt4.QtGui'
+ Missing dependencies for 'TurboKid': 'kid'
+ Missing dependencies for 'TurboJson: 'DecoratorTools', 'RuleDispatch'
+
+Now, we proceed with generating a graphical representation of the graph. First
+we write it to a file, and then we generate a PNG image using the
+:program:`dot` command-line tool::
+
+ from packaging.depgraph import graph_to_dot
+ with open('output.dot', 'w') as f:
+ # only show the interesting distributions, skipping the disconnected ones
+ graph_to_dot(graph, f, skip_disconnected=True)
+
+We can create the final picture using:
+
+.. code-block:: sh
+
+ $ dot -Tpng output.dot > output.png
+
+An example result is:
+
+.. figure:: depgraph-output.png
+ :alt: Example PNG output from packaging.depgraph and dot
+
+If you want to include egg distributions as well, then the code requires only
+one change, namely the line::
+
+ dists = list(packaging.database.get_distributions())
+
+has to be replaced with::
+
+ dists = list(packaging.database.get_distributions(use_egg_info=True))
+
+On many platforms, a richer graph is obtained because at the moment most
+distributions are provided in the egg rather than the new standard
+``.dist-info`` format.
+
+.. XXX missing image
+
+ An example of a more involved graph for illustrative reasons can be seen
+ here:
+
+ .. image:: depgraph_big.png
+
+
+List all dependent distributions
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+We will list all distributions that are dependent on some given distibution.
+This time, egg distributions will be considered as well::
+
+ import sys
+ from packaging.database import get_distribution, get_distributions
+ from packaging.depgraph import dependent_dists
+
+ dists = list(get_distributions(use_egg_info=True))
+ dist = get_distribution('bacon', use_egg_info=True)
+ if dist is None:
+ sys.exit('No such distribution in the system')
+
+ deps = dependent_dists(dists, dist)
+ deps = ', '.join(repr(x.name) for x in deps)
+ print('Distributions depending on %r: %s' % (dist.name, deps))
+
+And this is example output:
+
+.. with the dependency relationships as in the previous section
+ (depgraph_big)
+
+.. code-block:: none
+
+ Distributions depending on 'bacon': 'towel-stuff', 'choxie', 'grammar'
diff --git a/Doc/library/packaging.dist.rst b/Doc/library/packaging.dist.rst
new file mode 100644
index 0000000..25cb62b
--- /dev/null
+++ b/Doc/library/packaging.dist.rst
@@ -0,0 +1,108 @@
+:mod:`packaging.dist` --- The Distribution class
+================================================
+
+.. module:: packaging.dist
+ :synopsis: Core Distribution class.
+
+
+This module provides the :class:`Distribution` class, which represents the
+module distribution being built/packaged/distributed/installed.
+
+.. class:: Distribution(arguments)
+
+ A :class:`Distribution` describes how to build, package, distribute and
+ install a Python project.
+
+ The arguments accepted by the constructor are laid out in the following
+ table. Some of them will end up in a metadata object, the rest will become
+ data attributes of the :class:`Distribution` instance.
+
+ .. TODO improve constructor to take a Metadata object + named params?
+ (i.e. Distribution(metadata, cmdclass, py_modules, etc)
+ .. TODO also remove obsolete(?) script_name, etc. parameters? see what
+ py2exe and other tools need
+
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | argument name | value | type |
+ +====================+================================+=============================================================+
+ | *name* | The name of the project | a string |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *version* | The version number of the | a string |
+ | | release; see | |
+ | | :mod:`packaging.version` | |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *summary* | A single line describing the | a string |
+ | | project | |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *description* | Longer description of the | a string |
+ | | project | |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *author* | The name of the project author | a string |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *author_email* | The email address of the | a string |
+ | | project author | |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *maintainer* | The name of the current | a string |
+ | | maintainer, if different from | |
+ | | the author | |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *maintainer_email* | The email address of the | a string |
+ | | current maintainer, if | |
+ | | different from the author | |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *home_page* | A URL for the proejct | a string |
+ | | (homepage) | |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *download_url* | A URL to download the project | a string |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *packages* | A list of Python packages that | a list of strings |
+ | | packaging will manipulate | |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *py_modules* | A list of Python modules that | a list of strings |
+ | | packaging will manipulate | |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *scripts* | A list of standalone scripts | a list of strings |
+ | | to be built and installed | |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *ext_modules* | A list of Python extensions to | a list of instances of |
+ | | be built | :class:`packaging.compiler.extension.Extension` |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *classifiers* | A list of categories for the | a list of strings; valid classifiers are listed on `PyPi |
+ | | distribution | <http://pypi.python.org/pypi?:action=list_classifiers>`_. |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *distclass* | the :class:`Distribution` | a subclass of |
+ | | class to use | :class:`packaging.dist.Distribution` |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *script_name* | The name of the setup.py | a string |
+ | | script - defaults to | |
+ | | ``sys.argv[0]`` | |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *script_args* | Arguments to supply to the | a list of strings |
+ | | setup script | |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *options* | default options for the setup | a string |
+ | | script | |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *license* | The license for the | a string |
+ | | distribution; should be used | |
+ | | when there is no suitable | |
+ | | License classifier, or to | |
+ | | refine a classifier | |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *keywords* | Descriptive keywords; used by | a list of strings or a comma-separated string |
+ | | catalogs such as PyPI | |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *platforms* | Platforms compatible with this | a list of strings or a comma-separated string |
+ | | distribution; should be used | |
+ | | when there is no suitable | |
+ | | Platform classifier | |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *cmdclass* | A mapping of command names to | a dictionary |
+ | | :class:`Command` subclasses | |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *data_files* | A list of data files to | a list |
+ | | install | |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
+ | *package_dir* | A mapping of Python packages | a dictionary |
+ | | to directory names | |
+ +--------------------+--------------------------------+-------------------------------------------------------------+
diff --git a/Doc/library/packaging.fancy_getopt.rst b/Doc/library/packaging.fancy_getopt.rst
new file mode 100644
index 0000000..2c69341
--- /dev/null
+++ b/Doc/library/packaging.fancy_getopt.rst
@@ -0,0 +1,75 @@
+:mod:`packaging.fancy_getopt` --- Wrapper around the getopt module
+==================================================================
+
+.. module:: packaging.fancy_getopt
+ :synopsis: Additional getopt functionality.
+
+
+.. warning::
+ This module is deprecated and will be replaced with :mod:`optparse`.
+
+This module provides a wrapper around the standard :mod:`getopt` module that
+provides the following additional features:
+
+* short and long options are tied together
+
+* options have help strings, so :func:`fancy_getopt` could potentially create a
+ complete usage summary
+
+* options set attributes of a passed-in object
+
+* boolean options can have "negative aliases" --- e.g. if :option:`--quiet` is
+ the "negative alias" of :option:`--verbose`, then :option:`--quiet` on the
+ command line sets *verbose* to false.
+
+.. function:: fancy_getopt(options, negative_opt, object, args)
+
+ Wrapper function. *options* is a list of ``(long_option, short_option,
+ help_string)`` 3-tuples as described in the constructor for
+ :class:`FancyGetopt`. *negative_opt* should be a dictionary mapping option names
+ to option names, both the key and value should be in the *options* list.
+ *object* is an object which will be used to store values (see the :meth:`getopt`
+ method of the :class:`FancyGetopt` class). *args* is the argument list. Will use
+ ``sys.argv[1:]`` if you pass ``None`` as *args*.
+
+
+.. class:: FancyGetopt([option_table=None])
+
+ The option_table is a list of 3-tuples: ``(long_option, short_option,
+ help_string)``
+
+ If an option takes an argument, its *long_option* should have ``'='`` appended;
+ *short_option* should just be a single character, no ``':'`` in any case.
+ *short_option* should be ``None`` if a *long_option* doesn't have a
+ corresponding *short_option*. All option tuples must have long options.
+
+The :class:`FancyGetopt` class provides the following methods:
+
+
+.. method:: FancyGetopt.getopt([args=None, object=None])
+
+ Parse command-line options in args. Store as attributes on *object*.
+
+ If *args* is ``None`` or not supplied, uses ``sys.argv[1:]``. If *object* is
+ ``None`` or not supplied, creates a new :class:`OptionDummy` instance, stores
+ option values there, and returns a tuple ``(args, object)``. If *object* is
+ supplied, it is modified in place and :func:`getopt` just returns *args*; in
+ both cases, the returned *args* is a modified copy of the passed-in *args* list,
+ which is left untouched.
+
+ .. TODO and args returned are?
+
+
+.. method:: FancyGetopt.get_option_order()
+
+ Returns the list of ``(option, value)`` tuples processed by the previous run of
+ :meth:`getopt` Raises :exc:`RuntimeError` if :meth:`getopt` hasn't been called
+ yet.
+
+
+.. method:: FancyGetopt.generate_help([header=None])
+
+ Generate help text (a list of strings, one per suggested line of output) from
+ the option table for this :class:`FancyGetopt` object.
+
+ If supplied, prints the supplied *header* at the top of the help.
diff --git a/Doc/library/packaging.install.rst b/Doc/library/packaging.install.rst
new file mode 100644
index 0000000..3e00750
--- /dev/null
+++ b/Doc/library/packaging.install.rst
@@ -0,0 +1,112 @@
+:mod:`packaging.install` --- Installation tools
+===============================================
+
+.. module:: packaging.install
+ :synopsis: Download and installation building blocks
+
+
+Packaging provides a set of tools to deal with downloads and installation of
+distributions. Their role is to download the distribution from indexes, resolve
+the dependencies, and provide a safe way to install distributions. An operation
+that fails will cleanly roll back, not leave half-installed distributions on the
+system. Here's the basic process followed:
+
+#. Move all distributions that will be removed to a temporary location.
+
+#. Install all the distributions that will be installed in a temporary location.
+
+#. If the installation fails, move the saved distributions back to their
+ location and delete the installed distributions.
+
+#. Otherwise, move the installed distributions to the right location and delete
+ the temporary locations.
+
+This is a higher-level module built on :mod:`packaging.database` and
+:mod:`packaging.pypi`.
+
+
+Public functions
+----------------
+
+.. function:: get_infos(requirements, index=None, installed=None, \
+ prefer_final=True)
+
+ Return information about what's going to be installed and upgraded.
+ *requirements* is a string containing the requirements for this
+ project, for example ``'FooBar 1.1'`` or ``'BarBaz (<1.2)'``.
+
+ .. XXX are requirements comma-separated?
+
+ If you want to use another index than the main PyPI, give its URI as *index*
+ argument.
+
+ *installed* is a list of already installed distributions used to find
+ satisfied dependencies, obsoleted distributions and eventual conflicts.
+
+ By default, alpha, beta and candidate versions are not picked up. Set
+ *prefer_final* to false to accept them too.
+
+ The results are returned in a dictionary containing all the information
+ needed to perform installation of the requirements with the
+ :func:`install_from_infos` function:
+
+ >>> get_install_info("FooBar (<=1.2)")
+ {'install': [<FooBar 1.1>], 'remove': [], 'conflict': []}
+
+ .. TODO should return tuple or named tuple, not dict
+ .. TODO use "predicate" or "requirement" consistently in version and here
+ .. FIXME "info" cannot be plural in English, s/infos/info/
+
+
+.. function:: install(project)
+
+
+.. function:: install_dists(dists, path, paths=None)
+
+ Safely install all distributions provided in *dists* into *path*. *paths* is
+ a list of paths where already-installed distributions will be looked for to
+ find satisfied dependencies and conflicts (default: :data:`sys.path`).
+ Returns a list of installed dists.
+
+ .. FIXME dists are instances of what?
+
+
+.. function:: install_from_infos(install_path=None, install=[], remove=[], \
+ conflicts=[], paths=None)
+
+ Safely install and remove given distributions. This function is designed to
+ work with the return value of :func:`get_infos`: *install*, *remove* and
+ *conflicts* should be list of distributions returned by :func:`get_infos`.
+ If *install* is not empty, *install_path* must be given to specify the path
+ where the distributions should be installed. *paths* is a list of paths
+ where already-installed distributions will be looked for (default:
+ :data:`sys.path`).
+
+ This function is a very basic installer; if *conflicts* is not empty, the
+ system will be in a conflicting state after the function completes. It is a
+ building block for more sophisticated installers with conflict resolution
+ systems.
+
+ .. TODO document typical value for install_path
+ .. TODO document integration with default schemes, esp. user site-packages
+
+
+.. function:: install_local_project(path)
+
+ Install a distribution from a source directory, which must contain either a
+ Packaging-compliant :file:`setup.cfg` file or a legacy Distutils
+ :file:`setup.py` script (in which case Distutils will be used under the hood
+ to perform the installation).
+
+
+.. function:: remove(project_name, paths=None, auto_confirm=True)
+
+ Remove one distribution from the system.
+
+ .. FIXME this is the only function using "project" instead of dist/release
+
+..
+ Example usage
+ --------------
+
+ Get the scheme of what's gonna be installed if we install "foobar":
diff --git a/Doc/library/packaging.metadata.rst b/Doc/library/packaging.metadata.rst
new file mode 100644
index 0000000..332d69d
--- /dev/null
+++ b/Doc/library/packaging.metadata.rst
@@ -0,0 +1,122 @@
+:mod:`packaging.metadata` --- Metadata handling
+===============================================
+
+.. module:: packaging.metadata
+ :synopsis: Class holding the metadata of a release.
+
+
+.. TODO use sphinx-autogen to generate basic doc from the docstrings
+
+.. class:: Metadata
+
+ This class can read and write metadata files complying with any of the
+ defined versions: 1.0 (:PEP:`241`), 1.1 (:PEP:`314`) and 1.2 (:PEP:`345`). It
+ implements methods to parse Metadata files and write them, and a mapping
+ interface to its contents.
+
+ The :PEP:`345` implementation supports the micro-language for the environment
+ markers, and displays warnings when versions that are supposed to be
+ :PEP:`386`-compliant are violating the specification.
+
+
+Reading metadata
+----------------
+
+The :class:`Metadata` class can be instantiated
+with the path of the metadata file, and provides a dict-like interface to the
+values::
+
+ >>> from packaging.metadata import Metadata
+ >>> metadata = Metadata('PKG-INFO')
+ >>> metadata.keys()[:5]
+ ('Metadata-Version', 'Name', 'Version', 'Platform', 'Supported-Platform')
+ >>> metadata['Name']
+ 'CLVault'
+ >>> metadata['Version']
+ '0.5'
+ >>> metadata['Requires-Dist']
+ ["pywin32; sys.platform == 'win32'", "Sphinx"]
+
+
+The fields that support environment markers can be automatically ignored if
+the object is instantiated using the ``platform_dependent`` option.
+:class:`Metadata` will interpret in this case
+the markers and will automatically remove the fields that are not compliant
+with the running environment. Here's an example under Mac OS X. The win32
+dependency we saw earlier is ignored::
+
+ >>> from packaging.metadata import Metadata
+ >>> metadata = Metadata('PKG-INFO', platform_dependent=True)
+ >>> metadata['Requires-Dist']
+ ['Sphinx']
+
+
+If you want to provide your own execution context, let's say to test the
+metadata under a particular environment that is not the current environment,
+you can provide your own values in the ``execution_context`` option, which
+is the dict that may contain one or more keys of the context the micro-language
+expects.
+
+Here's an example, simulating a win32 environment::
+
+ >>> from packaging.metadata import Metadata
+ >>> context = {'sys.platform': 'win32'}
+ >>> metadata = Metadata('PKG-INFO', platform_dependent=True,
+ ... execution_context=context)
+ ...
+ >>> metadata['Requires-Dist'] = ["pywin32; sys.platform == 'win32'",
+ ... "Sphinx"]
+ ...
+ >>> metadata['Requires-Dist']
+ ['pywin32', 'Sphinx']
+
+
+Writing metadata
+----------------
+
+Writing metadata can be done using the ``write`` method::
+
+ >>> metadata.write('/to/my/PKG-INFO')
+
+The class will pick the best version for the metadata, depending on the values
+provided. If all the values provided exist in all versions, the class will
+use :attr:`PKG_INFO_PREFERRED_VERSION`. It is set by default to 1.0, the most
+widespread version.
+
+
+Conflict checking and best version
+----------------------------------
+
+Some fields in :PEP:`345` have to comply with the version number specification
+defined in :PEP:`386`. When they don't comply, a warning is emitted::
+
+ >>> from packaging.metadata import Metadata
+ >>> metadata = Metadata()
+ >>> metadata['Requires-Dist'] = ['Funky (Groovie)']
+ "Funky (Groovie)" is not a valid predicate
+ >>> metadata['Requires-Dist'] = ['Funky (1.2)']
+
+See also :mod:`packaging.version`.
+
+
+.. TODO talk about check()
+
+
+:mod:`packaging.markers` --- Environment markers
+================================================
+
+.. module:: packaging.markers
+ :synopsis: Micro-language for environment markers
+
+
+This is an implementation of environment markers `as defined in PEP 345
+<http://www.python.org/dev/peps/pep-0345/#environment-markers>`_. It is used
+for some metadata fields.
+
+.. function:: interpret(marker, execution_context=None)
+
+ Interpret a marker and return a boolean result depending on the environment.
+ Example:
+
+ >>> interpret("python_version > '1.0'")
+ True
diff --git a/Doc/library/packaging.pypi.dist.rst b/Doc/library/packaging.pypi.dist.rst
new file mode 100644
index 0000000..aaaaab7
--- /dev/null
+++ b/Doc/library/packaging.pypi.dist.rst
@@ -0,0 +1,114 @@
+:mod:`packaging.pypi.dist` --- Classes representing query results
+=================================================================
+
+.. module:: packaging.pypi.dist
+ :synopsis: Classes representing the results of queries to indexes.
+
+
+Information coming from the indexes is held in instances of the classes defined
+in this module.
+
+Keep in mind that each project (eg. FooBar) can have several releases
+(eg. 1.1, 1.2, 1.3), and each of these releases can be provided in multiple
+distributions (eg. a source distribution, a binary one, etc).
+
+
+ReleaseInfo
+-----------
+
+Each release has a project name, version, metadata, and related distributions.
+
+This information is stored in :class:`ReleaseInfo`
+objects.
+
+.. class:: ReleaseInfo
+
+
+DistInfo
+---------
+
+:class:`DistInfo` is a simple class that contains
+information related to distributions; mainly the URLs where distributions
+can be found.
+
+.. class:: DistInfo
+
+
+ReleasesList
+------------
+
+The :mod:`~packaging.pypi.dist` module provides a class which works
+with lists of :class:`ReleaseInfo` classes;
+used to filter and order results.
+
+.. class:: ReleasesList
+
+
+Example usage
+-------------
+
+Build a list of releases and order them
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Assuming we have a list of releases::
+
+ >>> from packaging.pypi.dist import ReleasesList, ReleaseInfo
+ >>> fb10 = ReleaseInfo("FooBar", "1.0")
+ >>> fb11 = ReleaseInfo("FooBar", "1.1")
+ >>> fb11a = ReleaseInfo("FooBar", "1.1a1")
+ >>> ReleasesList("FooBar", [fb11, fb11a, fb10])
+ >>> releases.sort_releases()
+ >>> releases.get_versions()
+ ['1.1', '1.1a1', '1.0']
+ >>> releases.add_release("1.2a1")
+ >>> releases.get_versions()
+ ['1.1', '1.1a1', '1.0', '1.2a1']
+ >>> releases.sort_releases()
+ ['1.2a1', '1.1', '1.1a1', '1.0']
+ >>> releases.sort_releases(prefer_final=True)
+ >>> releases.get_versions()
+ ['1.1', '1.0', '1.2a1', '1.1a1']
+
+
+Add distribution related information to releases
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+It's easy to add distribution information to releases::
+
+ >>> from packaging.pypi.dist import ReleasesList, ReleaseInfo
+ >>> r = ReleaseInfo("FooBar", "1.0")
+ >>> r.add_distribution("sdist", url="http://example.org/foobar-1.0.tar.gz")
+ >>> r.dists
+ {'sdist': FooBar 1.0 sdist}
+ >>> r['sdist'].url
+ {'url': 'http://example.org/foobar-1.0.tar.gz', 'hashname': None, 'hashval':
+ None, 'is_external': True}
+
+
+Getting attributes from the dist objects
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To abstract querying information returned from the indexes, attributes and
+release information can be retrieved directly from dist objects.
+
+For instance, if you have a release instance that does not contain the metadata
+attribute, it can be fetched by using the "fetch_metadata" method::
+
+ >>> r = Release("FooBar", "1.1")
+ >>> print r.metadata
+ None # metadata field is actually set to "None"
+ >>> r.fetch_metadata()
+ <Metadata for FooBar 1.1>
+
+.. XXX add proper roles to these constructs
+
+
+It's possible to retrieve a project's releases (`fetch_releases`),
+metadata (`fetch_metadata`) and distributions (`fetch_distributions`) using
+a similar work flow.
+
+.. XXX what is possible?
+
+Internally, this is possible because while retrieving information about
+projects, releases or distributions, a reference to the client used is
+stored which can be accessed using the objects `_index` attribute.
diff --git a/Doc/library/packaging.pypi.rst b/Doc/library/packaging.pypi.rst
new file mode 100644
index 0000000..14602ce
--- /dev/null
+++ b/Doc/library/packaging.pypi.rst
@@ -0,0 +1,74 @@
+:mod:`packaging.pypi` --- Interface to projects indexes
+=======================================================
+
+.. module:: packaging.pypi
+ :synopsis: Low-level and high-level APIs to query projects indexes.
+
+
+Packaging queries PyPI to get information about projects or download them. The
+low-level facilities used internally are also part of the public API designed to
+be used by other tools.
+
+The :mod:`packaging.pypi` package provides those facilities, which can be
+used to access information about Python projects registered at indexes, the
+main one being PyPI, located ad http://pypi.python.org/.
+
+There is two ways to retrieve data from these indexes: a screen-scraping
+interface called the "simple API", and XML-RPC. The first one uses HTML pages
+located under http://pypi.python.org/simple/, the second one makes XML-RPC
+requests to http://pypi.python.org/pypi/. All functions and classes also work
+with other indexes such as mirrors, which typically implement only the simple
+interface.
+
+Packaging provides a class that wraps both APIs to provide full query and
+download functionality: :class:`packaging.pypi.client.ClientWrapper`. If you
+want more control, you can use the underlying classes
+:class:`packaging.pypi.simple.Crawler` and :class:`packaging.pypi.xmlrpc.Client`
+to connect to one specific interface.
+
+
+:mod:`packaging.pypi.client` --- High-level query API
+=====================================================
+
+.. module:: packaging.pypi.client
+ :synopsis: Wrapper around :mod;`packaging.pypi.xmlrpc` and
+ :mod:`packaging.pypi.simple` to query indexes.
+
+
+This module provides a high-level API to query indexes and search
+for releases and distributions. The aim of this module is to choose the best
+way to query the API automatically, either using XML-RPC or the simple index,
+with a preference toward the latter.
+
+.. class:: ClientWrapper
+
+ Instances of this class will use the simple interface or XML-RPC requests to
+ query indexes and return :class:`packaging.pypi.dist.ReleaseInfo` and
+ :class:`packaging.pypi.dist.ReleasesList` objects.
+
+ .. method:: find_projects
+
+ .. method:: get_release
+
+ .. method:: get_releases
+
+
+:mod:`packaging.pypi.base` --- Base class for index crawlers
+============================================================
+
+.. module:: packaging.pypi.base
+ :synopsis: Base class used to implement crawlers.
+
+
+.. class:: BaseClient(prefer_final, prefer_source)
+
+ Base class containing common methods for the index crawlers or clients. One
+ method is currently defined:
+
+ .. method:: download_distribution(requirements, temp_path=None, \
+ prefer_source=None, prefer_final=None)
+
+ Download a distribution from the last release according to the
+ requirements. If *temp_path* is provided, download to this path,
+ otherwise, create a temporary directory for the download. If a release is
+ found, the full path to the downloaded file is returned.
diff --git a/Doc/library/packaging.pypi.simple.rst b/Doc/library/packaging.pypi.simple.rst
new file mode 100644
index 0000000..9153738
--- /dev/null
+++ b/Doc/library/packaging.pypi.simple.rst
@@ -0,0 +1,218 @@
+:mod:`packaging.pypi.simple` --- Crawler using the PyPI "simple" interface
+==========================================================================
+
+.. module:: packaging.pypi.simple
+ :synopsis: Crawler using the screen-scraping "simple" interface to fetch info
+ and distributions.
+
+
+The class provided by :mod:`packaging.pypi.simple` can access project indexes
+and provide useful information about distributions. PyPI, other indexes and
+local indexes are supported.
+
+You should use this module to search distributions by name and versions, process
+index external pages and download distributions. It is not suited for things
+that will end up in too long index processing (like "finding all distributions
+with a specific version, no matter the name"); use :mod:`packaging.pypi.xmlrpc`
+for that.
+
+
+API
+---
+
+.. class:: Crawler(index_url=DEFAULT_SIMPLE_INDEX_URL, \
+ prefer_final=False, prefer_source=True, \
+ hosts=('*',), follow_externals=False, \
+ mirrors_url=None, mirrors=None, timeout=15, \
+ mirrors_max_tries=0)
+
+ *index_url* is the address of the index to use for requests.
+
+ The first two parameters control the query results. *prefer_final*
+ indicates whether a final version (not alpha, beta or candidate) is to be
+ prefered over a newer but non-final version (for example, whether to pick
+ up 1.0 over 2.0a3). It is used only for queries that don't give a version
+ argument. Likewise, *prefer_source* tells whether to prefer a source
+ distribution over a binary one, if no distribution argument was prodived.
+
+ Other parameters are related to external links (that is links that go
+ outside the simple index): *hosts* is a list of hosts allowed to be
+ processed if *follow_externals* is true (default behavior is to follow all
+ hosts), *follow_externals* enables or disables following external links
+ (default is false, meaning disabled).
+
+ The remaining parameters are related to the mirroring infrastructure
+ defined in :PEP:`381`. *mirrors_url* gives a URL to look on for DNS
+ records giving mirror adresses; *mirrors* is a list of mirror URLs (see
+ the PEP). If both *mirrors* and *mirrors_url* are given, *mirrors_url*
+ will only be used if *mirrors* is set to ``None``. *timeout* is the time
+ (in seconds) to wait before considering a URL has timed out;
+ *mirrors_max_tries"* is the number of times to try requesting informations
+ on mirrors before switching.
+
+ The following methods are defined:
+
+ .. method:: get_distributions(project_name, version)
+
+ Return the distributions found in the index for the given release.
+
+ .. method:: get_metadata(project_name, version)
+
+ Return the metadata found on the index for this project name and
+ version. Currently downloads and unpacks a distribution to read the
+ PKG-INFO file.
+
+ .. method:: get_release(requirements, prefer_final=None)
+
+ Return one release that fulfills the given requirements.
+
+ .. method:: get_releases(requirements, prefer_final=None, force_update=False)
+
+ Search for releases and return a
+ :class:`~packaging.pypi.dist.ReleasesList` object containing the
+ results.
+
+ .. method:: search_projects(name=None)
+
+ Search the index for projects containing the given name and return a
+ list of matching names.
+
+ See also the base class :class:`packaging.pypi.base.BaseClient` for inherited
+ methods.
+
+
+.. data:: DEFAULT_SIMPLE_INDEX_URL
+
+ The address used by default by the crawler class. It is currently
+ ``'http://a.pypi.python.org/simple/'``, the main PyPI installation.
+
+
+
+
+Usage Exemples
+---------------
+
+To help you understand how using the `Crawler` class, here are some basic
+usages.
+
+Request the simple index to get a specific distribution
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Supposing you want to scan an index to get a list of distributions for
+the "foobar" project. You can use the "get_releases" method for that.
+The get_releases method will browse the project page, and return
+:class:`ReleaseInfo` objects for each found link that rely on downloads. ::
+
+ >>> from packaging.pypi.simple import Crawler
+ >>> crawler = Crawler()
+ >>> crawler.get_releases("FooBar")
+ [<ReleaseInfo "Foobar 1.1">, <ReleaseInfo "Foobar 1.2">]
+
+
+Note that you also can request the client about specific versions, using version
+specifiers (described in `PEP 345
+<http://www.python.org/dev/peps/pep-0345/#version-specifiers>`_)::
+
+ >>> client.get_releases("FooBar < 1.2")
+ [<ReleaseInfo "FooBar 1.1">, ]
+
+
+`get_releases` returns a list of :class:`ReleaseInfo`, but you also can get the
+best distribution that fullfil your requirements, using "get_release"::
+
+ >>> client.get_release("FooBar < 1.2")
+ <ReleaseInfo "FooBar 1.1">
+
+
+Download distributions
+^^^^^^^^^^^^^^^^^^^^^^
+
+As it can get the urls of distributions provided by PyPI, the `Crawler`
+client also can download the distributions and put it for you in a temporary
+destination::
+
+ >>> client.download("foobar")
+ /tmp/temp_dir/foobar-1.2.tar.gz
+
+
+You also can specify the directory you want to download to::
+
+ >>> client.download("foobar", "/path/to/my/dir")
+ /path/to/my/dir/foobar-1.2.tar.gz
+
+
+While downloading, the md5 of the archive will be checked, if not matches, it
+will try another time, then if fails again, raise `MD5HashDoesNotMatchError`.
+
+Internally, that's not the Crawler which download the distributions, but the
+`DistributionInfo` class. Please refer to this documentation for more details.
+
+
+Following PyPI external links
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The default behavior for packaging is to *not* follow the links provided
+by HTML pages in the "simple index", to find distributions related
+downloads.
+
+It's possible to tell the PyPIClient to follow external links by setting the
+`follow_externals` attribute, on instantiation or after::
+
+ >>> client = Crawler(follow_externals=True)
+
+or ::
+
+ >>> client = Crawler()
+ >>> client.follow_externals = True
+
+
+Working with external indexes, and mirrors
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The default `Crawler` behavior is to rely on the Python Package index stored
+on PyPI (http://pypi.python.org/simple).
+
+As you can need to work with a local index, or private indexes, you can specify
+it using the index_url parameter::
+
+ >>> client = Crawler(index_url="file://filesystem/path/")
+
+or ::
+
+ >>> client = Crawler(index_url="http://some.specific.url/")
+
+
+You also can specify mirrors to fallback on in case the first index_url you
+provided doesnt respond, or not correctly. The default behavior for
+`Crawler` is to use the list provided by Python.org DNS records, as
+described in the :PEP:`381` about mirroring infrastructure.
+
+If you don't want to rely on these, you could specify the list of mirrors you
+want to try by specifying the `mirrors` attribute. It's a simple iterable::
+
+ >>> mirrors = ["http://first.mirror","http://second.mirror"]
+ >>> client = Crawler(mirrors=mirrors)
+
+
+Searching in the simple index
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+It's possible to search for projects with specific names in the package index.
+Assuming you want to find all projects containing the "distutils" keyword::
+
+ >>> c.search_projects("distutils")
+ [<Project "collective.recipe.distutils">, <Project "Distutils">, <Project
+ "Packaging">, <Project "distutilscross">, <Project "lpdistutils">, <Project
+ "taras.recipe.distutils">, <Project "zerokspot.recipe.distutils">]
+
+
+You can also search the projects starting with a specific text, or ending with
+that text, using a wildcard::
+
+ >>> c.search_projects("distutils*")
+ [<Project "Distutils">, <Project "Packaging">, <Project "distutilscross">]
+
+ >>> c.search_projects("*distutils")
+ [<Project "collective.recipe.distutils">, <Project "Distutils">, <Project
+ "lpdistutils">, <Project "taras.recipe.distutils">, <Project
+ "zerokspot.recipe.distutils">]
diff --git a/Doc/library/packaging.pypi.xmlrpc.rst b/Doc/library/packaging.pypi.xmlrpc.rst
new file mode 100644
index 0000000..0253d68
--- /dev/null
+++ b/Doc/library/packaging.pypi.xmlrpc.rst
@@ -0,0 +1,143 @@
+:mod:`packaging.pypi.xmlrpc` --- Crawler using the PyPI XML-RPC interface
+=========================================================================
+
+.. module:: packaging.pypi.xmlrpc
+ :synopsis: Client using XML-RPC requests to fetch info and distributions.
+
+
+Indexes can be queried using XML-RPC calls, and Packaging provides a simple
+way to interface with XML-RPC.
+
+You should **use** XML-RPC when:
+
+* Searching the index for projects **on other fields than project
+ names**. For instance, you can search for projects based on the
+ author_email field.
+* Searching all the versions that have existed for a project.
+* you want to retrieve METADATAs information from releases or
+ distributions.
+
+
+You should **avoid using** XML-RPC method calls when:
+
+* Retrieving the last version of a project
+* Getting the projects with a specific name and version.
+* The simple index can match your needs
+
+
+When dealing with indexes, keep in mind that the index queries will always
+return you :class:`packaging.pypi.dist.ReleaseInfo` and
+:class:`packaging.pypi.dist.ReleasesList` objects.
+
+Some methods here share common APIs with the one you can find on
+:class:`packaging.pypi.simple`, internally, :class:`packaging.pypi.client`
+is inherited by :class:`Client`
+
+
+API
+---
+
+.. class:: Client
+
+
+Usage examples
+--------------
+
+Use case described here are use case that are not common to the other clients.
+If you want to see all the methods, please refer to API or to usage examples
+described in :class:`packaging.pypi.client.Client`
+
+
+Finding releases
+^^^^^^^^^^^^^^^^
+
+It's a common use case to search for "things" within the index. We can
+basically search for projects by their name, which is the most used way for
+users (eg. "give me the last version of the FooBar project").
+
+This can be accomplished using the following syntax::
+
+ >>> client = xmlrpc.Client()
+ >>> client.get_release("Foobar (<= 1.3))
+ <FooBar 1.2.1>
+ >>> client.get_releases("FooBar (<= 1.3)")
+ [FooBar 1.1, FooBar 1.1.1, FooBar 1.2, FooBar 1.2.1]
+
+
+And we also can find for specific fields::
+
+ >>> client.search_projects(field=value)
+
+
+You could specify the operator to use, default is "or"::
+
+ >>> client.search_projects(field=value, operator="and")
+
+
+The specific fields you can search are:
+
+* name
+* version
+* author
+* author_email
+* maintainer
+* maintainer_email
+* home_page
+* license
+* summary
+* description
+* keywords
+* platform
+* download_url
+
+
+Getting metadata information
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+XML-RPC is a prefered way to retrieve metadata information from indexes.
+It's really simple to do so::
+
+ >>> client = xmlrpc.Client()
+ >>> client.get_metadata("FooBar", "1.1")
+ <ReleaseInfo FooBar 1.1>
+
+
+Assuming we already have a :class:`packaging.pypi.ReleaseInfo` object defined,
+it's possible to pass it to the xmlrpc client to retrieve and complete its
+metadata::
+
+ >>> foobar11 = ReleaseInfo("FooBar", "1.1")
+ >>> client = xmlrpc.Client()
+ >>> returned_release = client.get_metadata(release=foobar11)
+ >>> returned_release
+ <ReleaseInfo FooBar 1.1>
+
+
+Get all the releases of a project
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To retrieve all the releases for a project, you can build them using
+`get_releases`::
+
+ >>> client = xmlrpc.Client()
+ >>> client.get_releases("FooBar")
+ [<ReleaseInfo FooBar 0.9>, <ReleaseInfo FooBar 1.0>, <ReleaseInfo 1.1>]
+
+
+Get information about distributions
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Indexes have information about projects, releases **and** distributions.
+If you're not familiar with those, please refer to the documentation of
+:mod:`packaging.pypi.dist`.
+
+It's possible to retrieve information about distributions, e.g "what are the
+existing distributions for this release ? How to retrieve them ?"::
+
+ >>> client = xmlrpc.Client()
+ >>> release = client.get_distributions("FooBar", "1.1")
+ >>> release.dists
+ {'sdist': <FooBar 1.1 sdist>, 'bdist': <FooBar 1.1 bdist>}
+
+As you see, this does not return a list of distributions, but a release,
+because a release can be used like a list of distributions.
diff --git a/Doc/library/packaging.rst b/Doc/library/packaging.rst
new file mode 100644
index 0000000..c6bff47
--- /dev/null
+++ b/Doc/library/packaging.rst
@@ -0,0 +1,75 @@
+:mod:`packaging` --- Packaging support
+======================================
+
+.. module:: packaging
+ :synopsis: Packaging system and building blocks for other packaging systems.
+.. sectionauthor:: Fred L. Drake, Jr. <fdrake@acm.org>, distutils and packaging
+ contributors
+
+
+The :mod:`packaging` package provides support for building, packaging,
+distributing and installing additional projects into a Python installation.
+Projects may include Python modules, extension modules, packages and scripts.
+:mod:`packaging` also provides building blocks for other packaging systems
+that are not tied to the command system.
+
+This manual is the reference documentation for those standalone building
+blocks and for extending Packaging. If you're looking for the user-centric
+guides to install a project or package your own code, head to `See also`__.
+
+
+Building blocks
+---------------
+
+.. toctree::
+ :maxdepth: 2
+
+ packaging-misc
+ packaging.version
+ packaging.metadata
+ packaging.database
+ packaging.depgraph
+ packaging.pypi
+ packaging.pypi.dist
+ packaging.pypi.simple
+ packaging.pypi.xmlrpc
+ packaging.install
+
+
+The command machinery
+---------------------
+
+.. toctree::
+ :maxdepth: 2
+
+ packaging.dist
+ packaging.command
+ packaging.compiler
+ packaging.fancy_getopt
+
+
+Other utilities
+----------------
+
+.. toctree::
+ :maxdepth: 2
+
+ packaging.util
+ packaging.tests.pypi_server
+
+.. XXX missing: compat config create (dir_util) run pypi.{base,mirrors}
+
+
+.. __:
+
+.. seealso::
+
+ :ref:`packaging-index`
+ The manual for developers of Python projects who want to package and
+ distribute them. This describes how to use :mod:`packaging` to make
+ projects easily found and added to an existing Python installation.
+
+ :ref:`packaging-install-index`
+ A user-centered manual which includes information on adding projects
+ into an existing Python installation. You do not need to be a Python
+ programmer to read this manual.
diff --git a/Doc/library/packaging.tests.pypi_server.rst b/Doc/library/packaging.tests.pypi_server.rst
new file mode 100644
index 0000000..f3b7720
--- /dev/null
+++ b/Doc/library/packaging.tests.pypi_server.rst
@@ -0,0 +1,105 @@
+:mod:`packaging.tests.pypi_server` --- PyPI mock server
+=======================================================
+
+.. module:: packaging.tests.pypi_server
+ :synopsis: Mock server used to test PyPI-related modules and commands.
+
+
+When you are testing code that works with Packaging, you might find these tools
+useful.
+
+
+The mock server
+---------------
+
+.. class:: PyPIServer
+
+ PyPIServer is a class that implements an HTTP server running in a separate
+ thread. All it does is record the requests for further inspection. The recorded
+ data is available under ``requests`` attribute. The default
+ HTTP response can be overridden with the ``default_response_status``,
+ ``default_response_headers`` and ``default_response_data`` attributes.
+
+ By default, when accessing the server with urls beginning with `/simple/`,
+ the server also record your requests, but will look for files under
+ the `/tests/pypiserver/simple/` path.
+
+ You can tell the sever to serve static files for other paths. This could be
+ accomplished by using the `static_uri_paths` parameter, as below::
+
+ server = PyPIServer(static_uri_paths=["first_path", "second_path"])
+
+
+ You need to create the content that will be served under the
+ `/tests/pypiserver/default` path. If you want to serve content from another
+ place, you also can specify another filesystem path (which needs to be under
+ `tests/pypiserver/`. This will replace the default behavior of the server, and
+ it will not serve content from the `default` dir ::
+
+ server = PyPIServer(static_filesystem_paths=["path/to/your/dir"])
+
+
+ If you just need to add some paths to the existing ones, you can do as shown,
+ keeping in mind that the server will always try to load paths in reverse order
+ (e.g here, try "another/super/path" then the default one) ::
+
+ server = PyPIServer(test_static_path="another/super/path")
+ server = PyPIServer("another/super/path")
+ # or
+ server.static_filesystem_paths.append("another/super/path")
+
+
+ As a result of what, in your tests, while you need to use the PyPIServer, in
+ order to isolates the test cases, the best practice is to place the common files
+ in the `default` folder, and to create a directory for each specific test case::
+
+ server = PyPIServer(static_filesystem_paths = ["default", "test_pypi_server"],
+ static_uri_paths=["simple", "external"])
+
+
+Base class and decorator for tests
+----------------------------------
+
+.. class:: PyPIServerTestCase
+
+ ``PyPIServerTestCase`` is a test case class with setUp and tearDown methods that
+ take care of a single PyPIServer instance attached as a ``pypi`` attribute on
+ the test class. Use it as one of the base classes in your test case::
+
+
+ class UploadTestCase(PyPIServerTestCase):
+
+ def test_something(self):
+ cmd = self.prepare_command()
+ cmd.ensure_finalized()
+ cmd.repository = self.pypi.full_address
+ cmd.run()
+
+ environ, request_data = self.pypi.requests[-1]
+ self.assertEqual(request_data, EXPECTED_REQUEST_DATA)
+
+
+.. decorator:: use_pypi_server
+
+ You also can use a decorator for your tests, if you do not need the same server
+ instance along all you test case. So, you can specify, for each test method,
+ some initialisation parameters for the server.
+
+ For this, you need to add a `server` parameter to your method, like this::
+
+ class SampleTestCase(TestCase):
+
+ @use_pypi_server()
+ def test_something(self, server):
+ ...
+
+
+ The decorator will instantiate the server for you, and run and stop it just
+ before and after your method call. You also can pass the server initializer,
+ just like this::
+
+ class SampleTestCase(TestCase):
+
+ @use_pypi_server("test_case_name")
+ def test_something(self, server):
+ ...
diff --git a/Doc/library/packaging.util.rst b/Doc/library/packaging.util.rst
new file mode 100644
index 0000000..2b3103c
--- /dev/null
+++ b/Doc/library/packaging.util.rst
@@ -0,0 +1,154 @@
+:mod:`packaging.util` --- Miscellaneous utility functions
+=========================================================
+
+.. module:: packaging.util
+ :synopsis: Miscellaneous utility functions.
+
+
+This module contains various helpers for the other modules.
+
+.. XXX a number of functions are missing, but the module may be split first
+ (it's ginormous right now, some things could go to compat for example)
+
+.. function:: get_platform()
+
+ Return a string that identifies the current platform. This is used mainly to
+ distinguish platform-specific build directories and platform-specific built
+ distributions. Typically includes the OS name and version and the
+ architecture (as supplied by 'os.uname()'), although the exact information
+ included depends on the OS; e.g. for IRIX the architecture isn't particularly
+ important (IRIX only runs on SGI hardware), but for Linux the kernel version
+ isn't particularly important.
+
+ Examples of returned values:
+
+ * ``linux-i586``
+ * ``linux-alpha``
+ * ``solaris-2.6-sun4u``
+ * ``irix-5.3``
+ * ``irix64-6.2``
+
+ For non-POSIX platforms, currently just returns ``sys.platform``.
+
+ For Mac OS X systems the OS version reflects the minimal version on which
+ binaries will run (that is, the value of ``MACOSX_DEPLOYMENT_TARGET``
+ during the build of Python), not the OS version of the current system.
+
+ For universal binary builds on Mac OS X the architecture value reflects
+ the univeral binary status instead of the architecture of the current
+ processor. For 32-bit universal binaries the architecture is ``fat``,
+ for 64-bit universal binaries the architecture is ``fat64``, and
+ for 4-way universal binaries the architecture is ``universal``. Starting
+ from Python 2.7 and Python 3.2 the architecture ``fat3`` is used for
+ a 3-way universal build (ppc, i386, x86_64) and ``intel`` is used for
+ a univeral build with the i386 and x86_64 architectures
+
+ Examples of returned values on Mac OS X:
+
+ * ``macosx-10.3-ppc``
+
+ * ``macosx-10.3-fat``
+
+ * ``macosx-10.5-universal``
+
+ * ``macosx-10.6-intel``
+
+ .. XXX reinvention of platform module?
+
+
+.. function:: convert_path(pathname)
+
+ Return 'pathname' as a name that will work on the native filesystem, i.e.
+ split it on '/' and put it back together again using the current directory
+ separator. Needed because filenames in the setup script are always supplied
+ in Unix style, and have to be converted to the local convention before we
+ can actually use them in the filesystem. Raises :exc:`ValueError` on
+ non-Unix-ish systems if *pathname* either starts or ends with a slash.
+
+
+.. function:: change_root(new_root, pathname)
+
+ Return *pathname* with *new_root* prepended. If *pathname* is relative, this
+ is equivalent to ``os.path.join(new_root,pathname)`` Otherwise, it requires
+ making *pathname* relative and then joining the two, which is tricky on
+ DOS/Windows.
+
+
+.. function:: check_environ()
+
+ Ensure that 'os.environ' has all the environment variables we guarantee that
+ users can use in config files, command-line options, etc. Currently this
+ includes:
+
+ * :envvar:`HOME` - user's home directory (Unix only)
+ * :envvar:`PLAT` - description of the current platform, including hardware
+ and OS (see :func:`get_platform`)
+
+
+.. function:: find_executable(executable, path=None)
+
+ Search the path for a given executable name.
+
+
+.. function:: execute(func, args[, msg=None, verbose=0, dry_run=0])
+
+ Perform some action that affects the outside world (for instance, writing to
+ the filesystem). Such actions are special because they are disabled by the
+ *dry_run* flag. This method takes care of all that bureaucracy for you;
+ all you have to do is supply the function to call and an argument tuple for
+ it (to embody the "external action" being performed), and an optional message
+ to print.
+
+
+.. function:: newer(source, target)
+
+ Return true if *source* exists and is more recently modified than *target*,
+ or if *source* exists and *target* doesn't. Return false if both exist and
+ *target* is the same age or newer than *source*. Raise
+ :exc:`PackagingFileError` if *source* does not exist.
+
+
+.. function:: strtobool(val)
+
+ Convert a string representation of truth to true (1) or false (0).
+
+ True values are ``y``, ``yes``, ``t``, ``true``, ``on`` and ``1``; false
+ values are ``n``, ``no``, ``f``, ``false``, ``off`` and ``0``. Raises
+ :exc:`ValueError` if *val* is anything else.
+
+
+.. function:: byte_compile(py_files[, optimize=0, force=0, prefix=None, base_dir=None, verbose=1, dry_run=0, direct=None])
+
+ Byte-compile a collection of Python source files to either :file:`.pyc` or
+ :file:`.pyo` files in a :file:`__pycache__` subdirectory (see :pep:`3147`),
+ or to the same directory when using the distutils2 backport on Python
+ versions older than 3.2.
+
+ *py_files* is a list of files to compile; any files that don't end in
+ :file:`.py` are silently skipped. *optimize* must be one of the following:
+
+ * ``0`` - don't optimize (generate :file:`.pyc`)
+ * ``1`` - normal optimization (like ``python -O``)
+ * ``2`` - extra optimization (like ``python -OO``)
+
+ If *force* is true, all files are recompiled regardless of timestamps.
+
+ The source filename encoded in each :term:`bytecode` file defaults to the filenames
+ listed in *py_files*; you can modify these with *prefix* and *basedir*.
+ *prefix* is a string that will be stripped off of each source filename, and
+ *base_dir* is a directory name that will be prepended (after *prefix* is
+ stripped). You can supply either or both (or neither) of *prefix* and
+ *base_dir*, as you wish.
+
+ If *dry_run* is true, doesn't actually do anything that would affect the
+ filesystem.
+
+ Byte-compilation is either done directly in this interpreter process with the
+ standard :mod:`py_compile` module, or indirectly by writing a temporary
+ script and executing it. Normally, you should let :func:`byte_compile`
+ figure out to use direct compilation or not (see the source for details).
+ The *direct* flag is used by the script generated in indirect mode; unless
+ you know what you're doing, leave it set to ``None``.
+
+ This function is independent from the running Python's :option:`-O` or
+ :option:`-B` options; it is fully controlled by the parameters passed in.
diff --git a/Doc/library/packaging.version.rst b/Doc/library/packaging.version.rst
new file mode 100644
index 0000000..f36cdab
--- /dev/null
+++ b/Doc/library/packaging.version.rst
@@ -0,0 +1,104 @@
+:mod:`packaging.version` --- Version number classes
+===================================================
+
+.. module:: packaging.version
+ :synopsis: Classes that represent project version numbers.
+
+
+This module contains classes and functions useful to deal with version numbers.
+It's an implementation of version specifiers `as defined in PEP 345
+<http://www.python.org/dev/peps/pep-0345/#version-specifiers>`_.
+
+
+Version numbers
+---------------
+
+.. class:: NormalizedVersion(self, s, error_on_huge_major_num=True)
+
+ A specific version of a distribution, as described in PEP 345. *s* is a
+ string object containing the version number (for example ``'1.2b1'``),
+ *error_on_huge_major_num* a boolean specifying whether to consider an
+ apparent use of a year or full date as the major version number an error.
+
+ The rationale for the second argument is that there were projects using years
+ or full dates as version numbers, which could cause problems with some
+ packaging systems sorting.
+
+ Instances of this class can be compared and sorted::
+
+ >>> NormalizedVersion('1.2b1') < NormalizedVersion('1.2')
+ True
+
+ :class:`NormalizedVersion` is used internally by :class:`VersionPredicate` to
+ do its work.
+
+
+.. class:: IrrationalVersionError
+
+ Exception raised when an invalid string is given to
+ :class:`NormalizedVersion`.
+
+ >>> NormalizedVersion("irrational_version_number")
+ ...
+ IrrationalVersionError: irrational_version_number
+
+
+.. function:: suggest_normalized_version(s)
+
+ Before standardization in PEP 386, various schemes were in use. Packaging
+ provides a function to try to convert any string to a valid, normalized
+ version::
+
+ >>> suggest_normalized_version('2.1-rc1')
+ 2.1c1
+
+
+ If :func:`suggest_normalized_version` can't make sense of the given string,
+ it will return ``None``::
+
+ >>> print(suggest_normalized_version('not a version'))
+ None
+
+
+Version predicates
+------------------
+
+.. class:: VersionPredicate(predicate)
+
+ This class deals with the parsing of field values like
+ ``ProjectName (>=version)``.
+
+ .. method:: match(version)
+
+ Test if a version number matches the predicate:
+
+ >>> version = VersionPredicate("ProjectName (<1.2, >1.0)")
+ >>> version.match("1.2.1")
+ False
+ >>> version.match("1.1.1")
+ True
+
+
+Validation helpers
+------------------
+
+If you want to use :term:`LBYL`-style checks instead of instantiating the
+classes and catching :class:`IrrationalVersionError` and :class:`ValueError`,
+you can use these functions:
+
+.. function:: is_valid_version(predicate)
+
+ Check whether the given string is a valid version number. Example of valid
+ strings: ``'1.2'``, ``'4.2.0.dev4'``, ``'2.5.4.post2'``.
+
+
+.. function:: is_valid_versions(predicate)
+
+ Check whether the given string is a valid value for specifying multiple
+ versions, such as in the Requires-Python field. Example: ``'2.7, >=3.2'``.
+
+
+.. function:: is_valid_predicate(predicate)
+
+ Check whether the given string is a valid version predicate. Examples:
+ ``'some.project == 4.5, <= 4.7'``, ``'speciallib (> 1.0, != 1.4.2, < 2.0)'``.
diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst
index 929936e..ce079cf 100644
--- a/Doc/library/platform.rst
+++ b/Doc/library/platform.rst
@@ -214,6 +214,10 @@ Win95/98 specific
preferring :func:`win32pipe.popen`. On Windows NT, :func:`win32pipe.popen`
should work; on Windows 9x it hangs due to bugs in the MS C library.
+ .. deprecated:: 3.3
+ This function is obsolete. Use the :mod:`subprocess` module. Check
+ especially the :ref:`subprocess-replacements` section.
+
Mac OS Platform
---------------
diff --git a/Doc/library/python.rst b/Doc/library/python.rst
index b67fbfc..07eadb4 100644
--- a/Doc/library/python.rst
+++ b/Doc/library/python.rst
@@ -25,4 +25,5 @@ overview:
inspect.rst
site.rst
fpectl.rst
+ packaging.rst
distutils.rst
diff --git a/Doc/library/random.rst b/Doc/library/random.rst
index 31cb945..2b10e6e 100644
--- a/Doc/library/random.rst
+++ b/Doc/library/random.rst
@@ -43,6 +43,12 @@ The :mod:`random` module also provides the :class:`SystemRandom` class which
uses the system function :func:`os.urandom` to generate random numbers
from sources provided by the operating system.
+.. warning::
+
+ The generators of the :mod:`random` module should not be used for security
+ purposes. Use :func:`ssl.RAND_bytes` if you require a cryptographically
+ secure pseudorandom number generator.
+
Bookkeeping functions:
diff --git a/Doc/library/re.rst b/Doc/library/re.rst
index 3dec04c..331ffa7 100644
--- a/Doc/library/re.rst
+++ b/Doc/library/re.rst
@@ -694,9 +694,12 @@ form.
.. function:: escape(string)
- Return *string* with all non-alphanumerics backslashed; this is useful if you
- want to match an arbitrary literal string that may have regular expression
- metacharacters in it.
+ Escape all the characters in pattern except ASCII letters, numbers and ``'_'``.
+ This is useful if you want to match an arbitrary literal string that may
+ have regular expression metacharacters in it.
+
+ .. versionchanged:: 3.3
+ The ``'_'`` character is no longer escaped.
.. function:: purge()
diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst
index ab55197..1134619 100644
--- a/Doc/library/readline.rst
+++ b/Doc/library/readline.rst
@@ -199,7 +199,7 @@ normally be executed automatically during interactive sessions from the user's
histfile = os.path.join(os.path.expanduser("~"), ".pyhist")
try:
readline.read_history_file(histfile)
- except IOError:
+ except FileNotFoundError:
pass
import atexit
atexit.register(readline.write_history_file, histfile)
@@ -224,7 +224,7 @@ support history save/restore. ::
if hasattr(readline, "read_history_file"):
try:
readline.read_history_file(histfile)
- except IOError:
+ except FileNotFoundError:
pass
atexit.register(self.save_history, histfile)
diff --git a/Doc/library/select.rst b/Doc/library/select.rst
index f1fd126..a4bc6fc 100644
--- a/Doc/library/select.rst
+++ b/Doc/library/select.rst
@@ -18,9 +18,10 @@ The module defines the following:
.. exception:: error
- The exception raised when an error occurs. The accompanying value is a pair
- containing the numeric error code from :c:data:`errno` and the corresponding
- string, as would be printed by the C function :c:func:`perror`.
+ A deprecated alias of :exc:`OSError`.
+
+ .. versionchanged:: 3.3
+ Following :pep:`3151`, this class was made an alias of :exc:`OSError`.
.. function:: epoll(sizehint=-1)
@@ -165,11 +166,6 @@ Edge and Level Trigger Polling (epoll) Objects
Register a fd descriptor with the epoll object.
- .. note::
-
- Registering a file descriptor that's already registered raises an
- IOError -- contrary to :ref:`poll-objects`'s register.
-
.. method:: epoll.modify(fd, eventmask)
diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst
index 0113fb7..908f996 100644
--- a/Doc/library/shlex.rst
+++ b/Doc/library/shlex.rst
@@ -34,6 +34,39 @@ The :mod:`shlex` module defines the following functions:
passing ``None`` for *s* will read the string to split from standard
input.
+
+.. function:: quote(s)
+
+ Return a shell-escaped version of the string *s*. The returned value is a
+ string that can safely be used as one token in a shell command line, for
+ cases where you cannot use a list.
+
+ This idiom would be unsafe::
+
+ >>> filename = 'somefile; rm -rf ~'
+ >>> command = 'ls -l {}'.format(filename)
+ >>> print(command) # executed by a shell: boom!
+ ls -l somefile; rm -rf ~
+
+ :func:`quote` lets you plug the security hole::
+
+ >>> command = 'ls -l {}'.format(quote(filename))
+ >>> print(command)
+ ls -l 'somefile; rm -rf ~'
+ >>> remote_command = 'ssh home {}'.format(quote(command))
+ >>> print(remote_command)
+ ssh home 'ls -l '"'"'somefile; rm -rf ~'"'"''
+
+ The quoting is compatible with UNIX shells and with :func:`split`:
+
+ >>> remote_command = split(remote_command)
+ >>> remote_command
+ ['ssh', 'home', "ls -l 'somefile; rm -rf ~'"]
+ >>> command = split(remote_command[-1])
+ >>> command
+ ['ls', '-l', 'somefile; rm -rf ~']
+
+
The :mod:`shlex` module defines the following class:
@@ -282,5 +315,4 @@ parsing rules.
* EOF is signaled with a :const:`None` value;
-* Quoted empty strings (``''``) are allowed;
-
+* Quoted empty strings (``''``) are allowed.
diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst
index a785682..6cb03b8 100644
--- a/Doc/library/shutil.rst
+++ b/Doc/library/shutil.rst
@@ -51,11 +51,14 @@ Directory and files operations
*dst* must be the complete target file name; look at :func:`copy` for a copy that
accepts a target directory path. If *src* and *dst* are the same files,
:exc:`Error` is raised.
- The destination location must be writable; otherwise, an :exc:`IOError` exception
+ The destination location must be writable; otherwise, an :exc:`OSError` exception
will be raised. If *dst* already exists, it will be replaced. Special files
such as character or block devices and pipes cannot be copied with this
function. *src* and *dst* are path names given as strings.
+ .. versionchanged:: 3.3
+ :exc:`IOError` used to be raised instead of :exc:`OSError`.
+
.. function:: copymode(src, dst)
@@ -173,6 +176,29 @@ Directory and files operations
used. Otherwise, *src* is copied (using :func:`copy2`) to *dst* and then
removed.
+.. function:: disk_usage(path)
+
+ Return disk usage statistics about the given path as a :term:`named tuple`
+ with the attributes *total*, *used* and *free*, which are the amount of
+ total, used and free space, in bytes.
+
+ .. versionadded:: 3.3
+
+ Availability: Unix, Windows.
+
+.. function:: chown(path, user=None, group=None)
+
+ Change owner *user* and/or *group* of the given *path*.
+
+ *user* can be a system user name or a uid; the same applies to *group*. At
+ least one argument is required.
+
+ See also :func:`os.chown`, the underlying function.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
.. exception:: Error
diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst
index 698b1e7..4fc3fd6 100644
--- a/Doc/library/signal.rst
+++ b/Doc/library/signal.rst
@@ -13,9 +13,6 @@ rules for working with signals and their handlers:
underlying implementation), with the exception of the handler for
:const:`SIGCHLD`, which follows the underlying implementation.
-* There is no way to "block" signals temporarily from critical sections (since
- this is not supported by all Unix flavors).
-
* Although Python signal handlers are called asynchronously as far as the Python
user is concerned, they can only occur between the "atomic" instructions of the
Python interpreter. This means that signals arriving during long calculations
@@ -119,6 +116,28 @@ The variables defined in the :mod:`signal` module are:
in user and kernel space. SIGPROF is delivered upon expiration.
+.. data:: SIG_BLOCK
+
+ A possible value for the *how* parameter to :func:`pthread_sigmask`
+ indicating that signals are to be blocked.
+
+ .. versionadded:: 3.3
+
+.. data:: SIG_UNBLOCK
+
+ A possible value for the *how* parameter to :func:`pthread_sigmask`
+ indicating that signals are to be unblocked.
+
+ .. versionadded:: 3.3
+
+.. data:: SIG_SETMASK
+
+ A possible value for the *how* parameter to :func:`pthread_sigmask`
+ indicating that the signal mask is to be replaced.
+
+ .. versionadded:: 3.3
+
+
The :mod:`signal` module defines one exception:
.. exception:: ItimerError
@@ -126,7 +145,11 @@ The :mod:`signal` module defines one exception:
Raised to signal an error from the underlying :func:`setitimer` or
:func:`getitimer` implementation. Expect this error if an invalid
interval timer or a negative time is passed to :func:`setitimer`.
- This error is a subtype of :exc:`IOError`.
+ This error is a subtype of :exc:`OSError`.
+
+ .. versionadded:: 3.3
+ This error used to be a subtype of :exc:`IOError`, which is now an
+ alias of :exc:`OSError`.
The :mod:`signal` module defines the following functions:
@@ -160,6 +183,60 @@ The :mod:`signal` module defines the following functions:
will then be called. Returns nothing. Not on Windows. (See the Unix man page
:manpage:`signal(2)`.)
+ See also :func:`sigwait`, :func:`sigwaitinfo`, :func:`sigtimedwait` and
+ :func:`sigpending`.
+
+
+.. function:: pthread_kill(thread_id, signum)
+
+ Send the signal *signum* to the thread *thread_id*, another thread in the same
+ process as the caller. The signal is asynchronously directed to thread.
+
+ Use :func:`threading.get_ident()` or the :attr:`~threading.Thread.ident`
+ attribute of :attr:`threading.Thread` to get a 'thread identifier' for
+ *thread_id*.
+
+ If *signum* is 0, then no signal is sent, but error checking is still
+ performed; this can be used to check if a thread is still running.
+
+ Availability: Unix (see the man page :manpage:`pthread_kill(3)` for further
+ information).
+
+ See also :func:`os.kill`.
+
+ .. versionadded:: 3.3
+
+
+.. function:: pthread_sigmask(how, mask)
+
+ Fetch and/or change the signal mask of the calling thread. The signal mask
+ is the set of signals whose delivery is currently blocked for the caller.
+ Return the old signal mask as a set of signals.
+
+ The behavior of the call is dependent on the value of *how*, as follows.
+
+ * :data:`SIG_BLOCK`: The set of blocked signals is the union of the current
+ set and the *mask* argument.
+ * :data:`SIG_UNBLOCK`: The signals in *mask* are removed from the current
+ set of blocked signals. It is permissible to attempt to unblock a
+ signal which is not blocked.
+ * :data:`SIG_SETMASK`: The set of blocked signals is set to the *mask*
+ argument.
+
+ *mask* is a set of signal numbers (e.g. {:const:`signal.SIGINT`,
+ :const:`signal.SIGTERM`}). Use ``range(1, signal.NSIG)`` for a full mask
+ including all signals.
+
+ For example, ``signal.pthread_sigmask(signal.SIG_BLOCK, [])`` reads the
+ signal mask of the calling thread.
+
+ Availability: Unix. See the man page :manpage:`sigprocmask(3)` and
+ :manpage:`pthread_sigmask(3)` for further information.
+
+ See also :func:`pause`, :func:`sigpending` and :func:`sigwait`.
+
+ .. versionadded:: 3.3
+
.. function:: setitimer(which, seconds[, interval])
@@ -189,13 +266,17 @@ The :mod:`signal` module defines the following functions:
.. function:: set_wakeup_fd(fd)
- Set the wakeup fd to *fd*. When a signal is received, a ``'\0'`` byte is
- written to the fd. This can be used by a library to wakeup a poll or select
- call, allowing the signal to be fully processed.
+ Set the wakeup file descriptor to *fd*. When a signal is received, the
+ signal number is written as a single byte into the fd. This can be used by
+ a library to wakeup a poll or select call, allowing the signal to be fully
+ processed.
The old wakeup fd is returned. *fd* must be non-blocking. It is up to the
library to remove any bytes before calling poll or select again.
+ Use for example ``struct.unpack('%uB' % len(data), data)`` to decode the
+ signal numbers list.
+
When threads are enabled, this function can only be called from the main thread;
attempting to call it from other threads will cause a :exc:`ValueError`
exception to be raised.
@@ -235,6 +316,74 @@ The :mod:`signal` module defines the following functions:
:const:`SIGTERM`. A :exc:`ValueError` will be raised in any other case.
+.. function:: sigpending()
+
+ Examine the set of signals that are pending for delivery to the calling
+ thread (i.e., the signals which have been raised while blocked). Return the
+ set of the pending signals.
+
+ Availability: Unix (see the man page :manpage:`sigpending(2)` for further
+ information).
+
+ See also :func:`pause`, :func:`pthread_sigmask` and :func:`sigwait`.
+
+ .. versionadded:: 3.3
+
+
+.. function:: sigwait(sigset)
+
+ Suspend execution of the calling thread until the delivery of one of the
+ signals specified in the signal set *sigset*. The function accepts the signal
+ (removes it from the pending list of signals), and returns the signal number.
+
+ Availability: Unix (see the man page :manpage:`sigwait(3)` for further
+ information).
+
+ See also :func:`pause`, :func:`pthread_sigmask`, :func:`sigpending`,
+ :func:`sigwaitinfo` and :func:`sigtimedwait`.
+
+ .. versionadded:: 3.3
+
+
+.. function:: sigwaitinfo(sigset)
+
+ Suspend execution of the calling thread until the delivery of one of the
+ signals specified in the signal set *sigset*. The function accepts the
+ signal and removes it from the pending list of signals. If one of the
+ signals in *sigset* is already pending for the calling thread, the function
+ will return immediately with information about that signal. The signal
+ handler is not called for the delivered signal. The function raises an
+ :exc:`InterruptedError` if it is interrupted by a signal that is not in
+ *sigset*.
+
+ The return value is an object representing the data contained in the
+ :c:type:`siginfo_t` structure, namely: :attr:`si_signo`, :attr:`si_code`,
+ :attr:`si_errno`, :attr:`si_pid`, :attr:`si_uid`, :attr:`si_status`,
+ :attr:`si_band`.
+
+ Availability: Unix (see the man page :manpage:`sigwaitinfo(2)` for further
+ information).
+
+ See also :func:`pause`, :func:`sigwait` and :func:`sigtimedwait`.
+
+ .. versionadded:: 3.3
+
+
+.. function:: sigtimedwait(sigset, (timeout_sec, timeout_nsec))
+
+ Like :func:`sigtimedwait`, but takes a tuple of ``(seconds, nanoseconds)``
+ as an additional argument specifying a timeout. If both *timeout_sec* and
+ *timeout_nsec* are specified as :const:`0`, a poll is performed. Returns
+ :const:`None` if a timeout occurs.
+
+ Availability: Unix (see the man page :manpage:`sigtimedwait(2)` for further
+ information).
+
+ See also :func:`pause`, :func:`sigwait` and :func:`sigwaitinfo`.
+
+ .. versionadded:: 3.3
+
+
.. _signal-example:
Example
@@ -251,7 +400,7 @@ be sent, and the handler raises an exception. ::
def handler(signum, frame):
print('Signal handler called with signal', signum)
- raise IOError("Couldn't open device!")
+ raise OSError("Couldn't open device!")
# Set the signal handler and a 5-second alarm
signal.signal(signal.SIGALRM, handler)
diff --git a/Doc/library/site.rst b/Doc/library/site.rst
index db96add..b987897 100644
--- a/Doc/library/site.rst
+++ b/Doc/library/site.rst
@@ -16,7 +16,14 @@ import can be suppressed using the interpreter's :option:`-S` option.
.. index:: triple: module; search; path
Importing this module will append site-specific paths to the module search path
-and add a few builtins.
+and add a few builtins, unless :option:`-S` was used. In that case, this module
+can be safely imported with no automatic modifications to the module search path
+or additions to the builtins. To explicitly trigger the usual site-specific
+additions, call the :func:`site.main` function.
+
+.. versionchanged:: 3.3
+ Importing the module used to trigger paths manipulation even when using
+ :option:`-S`.
.. index::
pair: site-python; directory
@@ -127,10 +134,21 @@ empty, and the path manipulations are skipped; however the import of
:func:`getuserbase` hasn't been called yet. Default value is
:file:`~/.local` for UNIX and Mac OS X non-framework builds,
:file:`~/Library/Python/{X.Y}` for Mac framework builds, and
- :file:`{%APPDATA%}\\Python` for Windows. This value is used by Distutils to
+ :file:`{%APPDATA%}\\Python` for Windows. This value is used by Packaging to
compute the installation directories for scripts, data files, Python modules,
- etc. for the :ref:`user installation scheme <inst-alt-install-user>`. See
- also :envvar:`PYTHONUSERBASE`.
+ etc. for the :ref:`user installation scheme <packaging-alt-install-user>`.
+ See also :envvar:`PYTHONUSERBASE`.
+
+
+.. function:: main()
+
+ Adds all the standard site-specific directories to the module search
+ path. This function is called automatically when this module is imported,
+ unless the :program:`python` interpreter was started with the :option:`-S`
+ flag.
+
+ .. versionchanged:: 3.3
+ This function used to be called unconditionnally.
.. function:: addsitedir(sitedir, known_paths=None)
diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst
index 5978a8f..7dd038d 100644
--- a/Doc/library/smtplib.rst
+++ b/Doc/library/smtplib.rst
@@ -20,7 +20,7 @@ details of SMTP and ESMTP operation, consult :rfc:`821` (Simple Mail Transfer
Protocol) and :rfc:`1869` (SMTP Service Extensions).
-.. class:: SMTP(host='', port=0, local_hostname=None[, timeout])
+.. class:: SMTP(host='', port=0, local_hostname=None[, timeout], source_address=None)
A :class:`SMTP` instance encapsulates an SMTP connection. It has methods
that support a full repertoire of SMTP and ESMTP operations. If the optional
@@ -29,13 +29,34 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions).
raised if the specified host doesn't respond correctly. The optional
*timeout* parameter specifies a timeout in seconds for blocking operations
like the connection attempt (if not specified, the global default timeout
- setting will be used).
+ setting will be used). The optional source_address parameter allows to bind to some
+ specific source address in a machine with multiple network interfaces,
+ and/or to some specific source TCP port. It takes a 2-tuple (host, port),
+ for the socket to bind to as its source address before connecting. If
+ omitted (or if host or port are ``''`` and/or 0 respectively) the OS default
+ behavior will be used.
For normal use, you should only require the initialization/connect,
:meth:`sendmail`, and :meth:`quit` methods. An example is included below.
+ The :class:`SMTP` class supports the :keyword:`with` statement. When used
+ like this, the SMTP ``QUIT`` command is issued automatically when the
+ :keyword:`with` statement exits. E.g.::
-.. class:: SMTP_SSL(host='', port=0, local_hostname=None, keyfile=None, certfile=None[, timeout])
+ >>> from smtplib import SMTP
+ >>> with SMTP("domain.org") as smtp:
+ ... smtp.noop()
+ ...
+ (250, b'Ok')
+ >>>
+
+ .. versionchanged:: 3.3
+ Support for the :keyword:`with` statement was added.
+
+ .. versionchanged:: 3.3
+ source_address argument was added.
+
+.. class:: SMTP_SSL(host='', port=0, local_hostname=None, keyfile=None, certfile=None[, timeout], context=None, source_address=None)
A :class:`SMTP_SSL` instance behaves exactly the same as instances of
:class:`SMTP`. :class:`SMTP_SSL` should be used for situations where SSL is
@@ -43,18 +64,33 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions).
not appropriate. If *host* is not specified, the local host is used. If
*port* is zero, the standard SMTP-over-SSL port (465) is used. *keyfile*
and *certfile* are also optional, and can contain a PEM formatted private key
- and certificate chain file for the SSL connection. The optional *timeout*
+ and certificate chain file for the SSL connection. *context* also optional, can contain
+ a SSLContext, and is an alternative to keyfile and certfile; If it is specified both
+ keyfile and certfile must be None. The optional *timeout*
parameter specifies a timeout in seconds for blocking operations like the
connection attempt (if not specified, the global default timeout setting
- will be used).
+ will be used). The optional source_address parameter allows to bind to some
+ specific source address in a machine with multiple network interfaces,
+ and/or to some specific source tcp port. It takes a 2-tuple (host, port),
+ for the socket to bind to as its source address before connecting. If
+ omitted (or if host or port are ``''`` and/or 0 respectively) the OS default
+ behavior will be used.
+
+ .. versionchanged:: 3.3
+ *context* was added.
+ .. versionchanged:: 3.3
+ source_address argument was added.
-.. class:: LMTP(host='', port=LMTP_PORT, local_hostname=None)
+
+.. class:: LMTP(host='', port=LMTP_PORT, local_hostname=None, source_address=None)
The LMTP protocol, which is very similar to ESMTP, is heavily based on the
- standard SMTP client. It's common to use Unix sockets for LMTP, so our :meth:`connect`
- method must support that as well as a regular host:port server. To specify a
- Unix socket, you must use an absolute path for *host*, starting with a '/'.
+ standard SMTP client. It's common to use Unix sockets for LMTP, so our
+ :meth:`connect` method must support that as well as a regular host:port
+ server. The optional arguments local_hostname and source_address have the
+ same meaning as that of SMTP client. To specify a Unix socket, you must use
+ an absolute path for *host*, starting with a '/'.
Authentication is supported, using the regular SMTP mechanism. When using a Unix
socket, LMTP generally don't support or require any authentication, but your
@@ -242,7 +278,7 @@ An :class:`SMTP` instance has the following methods:
No suitable authentication method was found.
-.. method:: SMTP.starttls(keyfile=None, certfile=None)
+.. method:: SMTP.starttls(keyfile=None, certfile=None, context=None)
Put the SMTP connection in TLS (Transport Layer Security) mode. All SMTP
commands that follow will be encrypted. You should then call :meth:`ehlo`
@@ -251,6 +287,9 @@ An :class:`SMTP` instance has the following methods:
If *keyfile* and *certfile* are provided, these are passed to the :mod:`socket`
module's :func:`ssl` function.
+ Optional *context* parameter is a :class:`ssl.SSLContext` object; This is an alternative to
+ using a keyfile and a certfile and if specified both *keyfile* and *certfile* should be None.
+
If there has been no previous ``EHLO`` or ``HELO`` command this session,
this method tries ESMTP ``EHLO`` first.
@@ -263,6 +302,9 @@ An :class:`SMTP` instance has the following methods:
:exc:`RuntimeError`
SSL/TLS support is not available to your Python interpreter.
+ .. versionchanged:: 3.3
+ *context* was added.
+
.. method:: SMTP.sendmail(from_addr, to_addrs, msg, mail_options=[], rcpt_options=[])
diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst
index 64b4183..3d1a6a4 100644
--- a/Doc/library/socket.rst
+++ b/Doc/library/socket.rst
@@ -80,6 +80,11 @@ Socket addresses are represented as follows:
If *addr_type* is TIPC_ADDR_ID, then *v1* is the node, *v2* is the
reference, and *v3* should be set to 0.
+- A tuple ``(interface, )`` is used for the :const:`AF_CAN` address family,
+ where *interface* is a string representing a network interface name like
+ ``'can0'``. The network interface name ``''`` can be used to receive packets
+ from all network interfaces of this family.
+
- Certain other address families (:const:`AF_BLUETOOTH`, :const:`AF_PACKET`)
support specific representations.
@@ -99,8 +104,9 @@ resolution and/or the host configuration. For deterministic behavior use a
numeric address in *host* portion.
All errors raise exceptions. The normal exceptions for invalid argument types
-and out-of-memory conditions can be raised; errors related to socket or address
-semantics raise :exc:`socket.error` or one of its subclasses.
+and out-of-memory conditions can be raised; starting from Python 3.3, errors
+related to socket or address semantics raise :exc:`OSError` or one of its
+subclasses (they used to raise :exc:`socket.error`).
Non-blocking mode is supported through :meth:`~socket.setblocking`. A
generalization of this based on timeouts is supported through
@@ -115,20 +121,15 @@ The module :mod:`socket` exports the following constants and functions:
.. exception:: error
- .. index:: module: errno
-
- A subclass of :exc:`IOError`, this exception is raised for socket-related
- errors. It is recommended that you inspect its ``errno`` attribute to
- discriminate between different kinds of errors.
+ A deprecated alias of :exc:`OSError`.
- .. seealso::
- The :mod:`errno` module contains symbolic names for the error codes
- defined by the underlying operating system.
+ .. versionchanged:: 3.3
+ Following :pep:`3151`, this class was made an alias of :exc:`OSError`.
.. exception:: herror
- A subclass of :exc:`socket.error`, this exception is raised for
+ A subclass of :exc:`OSError`, this exception is raised for
address-related errors, i.e. for functions that use *h_errno* in the POSIX
C API, including :func:`gethostbyname_ex` and :func:`gethostbyaddr`.
The accompanying value is a pair ``(h_errno, string)`` representing an
@@ -136,10 +137,12 @@ The module :mod:`socket` exports the following constants and functions:
*string* represents the description of *h_errno*, as returned by the
:c:func:`hstrerror` C function.
+ .. versionchanged:: 3.3
+ This class was made a subclass of :exc:`OSError`.
.. exception:: gaierror
- A subclass of :exc:`socket.error`, this exception is raised for
+ A subclass of :exc:`OSError`, this exception is raised for
address-related errors by :func:`getaddrinfo` and :func:`getnameinfo`.
The accompanying value is a pair ``(error, string)`` representing an error
returned by a library call. *string* represents the description of
@@ -147,15 +150,19 @@ The module :mod:`socket` exports the following constants and functions:
numeric *error* value will match one of the :const:`EAI_\*` constants
defined in this module.
+ .. versionchanged:: 3.3
+ This class was made a subclass of :exc:`OSError`.
.. exception:: timeout
- A subclass of :exc:`socket.error`, this exception is raised when a timeout
+ A subclass of :exc:`OSError`, this exception is raised when a timeout
occurs on a socket which has had timeouts enabled via a prior call to
:meth:`~socket.settimeout` (or implicitly through
:func:`~socket.setdefaulttimeout`). The accompanying value is a string
whose value is currently always "timed out".
+ .. versionchanged:: 3.3
+ This class was made a subclass of :exc:`OSError`.
.. data:: AF_UNIX
AF_INET
@@ -198,6 +205,7 @@ The module :mod:`socket` exports the following constants and functions:
SOMAXCONN
MSG_*
SOL_*
+ SCM_*
IPPROTO_*
IPPORT_*
INADDR_*
@@ -215,6 +223,19 @@ The module :mod:`socket` exports the following constants and functions:
in the Unix header files are defined; for a few symbols, default values are
provided.
+.. data:: AF_CAN
+ PF_CAN
+ SOL_CAN_*
+ CAN_*
+
+ Many constants of these forms, documented in the Linux documentation, are
+ also defined in the socket module.
+
+ Availability: Linux >= 2.6.25.
+
+ .. versionadded:: 3.3
+
+
.. data:: SIO_*
RCVALL_*
@@ -386,10 +407,14 @@ The module :mod:`socket` exports the following constants and functions:
Create a new socket using the given address family, socket type and protocol
number. The address family should be :const:`AF_INET` (the default),
- :const:`AF_INET6` or :const:`AF_UNIX`. The socket type should be
- :const:`SOCK_STREAM` (the default), :const:`SOCK_DGRAM` or perhaps one of the
- other ``SOCK_`` constants. The protocol number is usually zero and may be
- omitted in that case.
+ :const:`AF_INET6`, :const:`AF_UNIX` or :const:`AF_CAN`. The socket type
+ should be :const:`SOCK_STREAM` (the default), :const:`SOCK_DGRAM`,
+ :const:`SOCK_RAW` or perhaps one of the other ``SOCK_`` constants. The
+ protocol number is usually zero and may be omitted in that case or
+ :const:`CAN_RAW` in case the address family is :const:`AF_CAN`.
+
+ .. versionchanged:: 3.3
+ The AF_CAN family was added.
.. function:: socketpair([family[, type[, proto]]])
@@ -457,7 +482,7 @@ The module :mod:`socket` exports the following constants and functions:
Unix manual page :manpage:`inet(3)` for details.
If the IPv4 address string passed to this function is invalid,
- :exc:`socket.error` will be raised. Note that exactly what is valid depends on
+ :exc:`OSError` will be raised. Note that exactly what is valid depends on
the underlying C implementation of :c:func:`inet_aton`.
:func:`inet_aton` does not support IPv6, and :func:`inet_pton` should be used
@@ -474,7 +499,7 @@ The module :mod:`socket` exports the following constants and functions:
argument.
If the byte sequence passed to this function is not exactly 4 bytes in
- length, :exc:`socket.error` will be raised. :func:`inet_ntoa` does not
+ length, :exc:`OSError` will be raised. :func:`inet_ntoa` does not
support IPv6, and :func:`inet_ntop` should be used instead for IPv4/v6 dual
stack support.
@@ -488,7 +513,7 @@ The module :mod:`socket` exports the following constants and functions:
Supported values for *address_family* are currently :const:`AF_INET` and
:const:`AF_INET6`. If the IP address string *ip_string* is invalid,
- :exc:`socket.error` will be raised. Note that exactly what is valid depends on
+ :exc:`OSError` will be raised. Note that exactly what is valid depends on
both the value of *address_family* and the underlying implementation of
:c:func:`inet_pton`.
@@ -506,11 +531,54 @@ The module :mod:`socket` exports the following constants and functions:
Supported values for *address_family* are currently :const:`AF_INET` and
:const:`AF_INET6`. If the string *packed_ip* is not the correct length for the
specified address family, :exc:`ValueError` will be raised. A
- :exc:`socket.error` is raised for errors from the call to :func:`inet_ntop`.
+ :exc:`OSError` is raised for errors from the call to :func:`inet_ntop`.
Availability: Unix (maybe not all platforms).
+..
+ XXX: Are sendmsg(), recvmsg() and CMSG_*() available on any
+ non-Unix platforms? The old (obsolete?) 4.2BSD form of the
+ interface, in which struct msghdr has no msg_control or
+ msg_controllen members, is not currently supported.
+
+.. function:: CMSG_LEN(length)
+
+ Return the total length, without trailing padding, of an ancillary
+ data item with associated data of the given *length*. This value
+ can often be used as the buffer size for :meth:`~socket.recvmsg` to
+ receive a single item of ancillary data, but :rfc:`3542` requires
+ portable applications to use :func:`CMSG_SPACE` and thus include
+ space for padding, even when the item will be the last in the
+ buffer. Raises :exc:`OverflowError` if *length* is outside the
+ permissible range of values.
+
+ Availability: most Unix platforms, possibly others.
+
+ .. versionadded:: 3.3
+
+
+.. function:: CMSG_SPACE(length)
+
+ Return the buffer size needed for :meth:`~socket.recvmsg` to
+ receive an ancillary data item with associated data of the given
+ *length*, along with any trailing padding. The buffer space needed
+ to receive multiple items is the sum of the :func:`CMSG_SPACE`
+ values for their associated data lengths. Raises
+ :exc:`OverflowError` if *length* is outside the permissible range
+ of values.
+
+ Note that some systems might support ancillary data without
+ providing this function. Also note that setting the buffer size
+ using the results of this function may not precisely limit the
+ amount of ancillary data that can be received, since additional
+ data may be able to fit into the padding area.
+
+ Availability: most Unix platforms, possibly others.
+
+ .. versionadded:: 3.3
+
+
.. function:: getdefaulttimeout()
Return the default timeout in seconds (float) for new socket objects. A value
@@ -526,6 +594,49 @@ The module :mod:`socket` exports the following constants and functions:
meanings.
+.. function:: sethostname(name)
+
+ Set the machine's hostname to *name*. This will raise a
+ :exc:`OSError` if you don't have enough rights.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. function:: if_nameindex()
+
+ Return a list of network interface information
+ (index int, name string) tuples.
+ :exc:`OSError` if the system call fails.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. function:: if_nametoindex(if_name)
+
+ Return a network interface index number corresponding to an
+ interface name.
+ :exc:`OSError` if no interface with the given name exists.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
+.. function:: if_indextoname(if_index)
+
+ Return a network interface name corresponding to a
+ interface index number.
+ :exc:`OSError` if no interface with the given index exists.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.3
+
+
.. data:: SocketType
This is a Python type object that represents the socket object type. It is the
@@ -699,6 +810,109 @@ correspond to Unix system calls applicable to sockets.
to zero. (The format of *address* depends on the address family --- see above.)
+.. method:: socket.recvmsg(bufsize[, ancbufsize[, flags]])
+
+ Receive normal data (up to *bufsize* bytes) and ancillary data from
+ the socket. The *ancbufsize* argument sets the size in bytes of
+ the internal buffer used to receive the ancillary data; it defaults
+ to 0, meaning that no ancillary data will be received. Appropriate
+ buffer sizes for ancillary data can be calculated using
+ :func:`CMSG_SPACE` or :func:`CMSG_LEN`, and items which do not fit
+ into the buffer might be truncated or discarded. The *flags*
+ argument defaults to 0 and has the same meaning as for
+ :meth:`recv`.
+
+ The return value is a 4-tuple: ``(data, ancdata, msg_flags,
+ address)``. The *data* item is a :class:`bytes` object holding the
+ non-ancillary data received. The *ancdata* item is a list of zero
+ or more tuples ``(cmsg_level, cmsg_type, cmsg_data)`` representing
+ the ancillary data (control messages) received: *cmsg_level* and
+ *cmsg_type* are integers specifying the protocol level and
+ protocol-specific type respectively, and *cmsg_data* is a
+ :class:`bytes` object holding the associated data. The *msg_flags*
+ item is the bitwise OR of various flags indicating conditions on
+ the received message; see your system documentation for details.
+ If the receiving socket is unconnected, *address* is the address of
+ the sending socket, if available; otherwise, its value is
+ unspecified.
+
+ On some systems, :meth:`sendmsg` and :meth:`recvmsg` can be used to
+ pass file descriptors between processes over an :const:`AF_UNIX`
+ socket. When this facility is used (it is often restricted to
+ :const:`SOCK_STREAM` sockets), :meth:`recvmsg` will return, in its
+ ancillary data, items of the form ``(socket.SOL_SOCKET,
+ socket.SCM_RIGHTS, fds)``, where *fds* is a :class:`bytes` object
+ representing the new file descriptors as a binary array of the
+ native C :c:type:`int` type. If :meth:`recvmsg` raises an
+ exception after the system call returns, it will first attempt to
+ close any file descriptors received via this mechanism.
+
+ Some systems do not indicate the truncated length of ancillary data
+ items which have been only partially received. If an item appears
+ to extend beyond the end of the buffer, :meth:`recvmsg` will issue
+ a :exc:`RuntimeWarning`, and will return the part of it which is
+ inside the buffer provided it has not been truncated before the
+ start of its associated data.
+
+ On systems which support the :const:`SCM_RIGHTS` mechanism, the
+ following function will receive up to *maxfds* file descriptors,
+ returning the message data and a list containing the descriptors
+ (while ignoring unexpected conditions such as unrelated control
+ messages being received). See also :meth:`sendmsg`. ::
+
+ import socket, array
+
+ def recv_fds(sock, msglen, maxfds):
+ fds = array.array("i") # Array of ints
+ msg, ancdata, flags, addr = sock.recvmsg(msglen, socket.CMSG_LEN(maxfds * fds.itemsize))
+ for cmsg_level, cmsg_type, cmsg_data in ancdata:
+ if (cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS):
+ # Append data, ignoring any truncated integers at the end.
+ fds.fromstring(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)])
+ return msg, list(fds)
+
+ Availability: most Unix platforms, possibly others.
+
+ .. versionadded:: 3.3
+
+
+.. method:: socket.recvmsg_into(buffers[, ancbufsize[, flags]])
+
+ Receive normal data and ancillary data from the socket, behaving as
+ :meth:`recvmsg` would, but scatter the non-ancillary data into a
+ series of buffers instead of returning a new bytes object. The
+ *buffers* argument must be an iterable of objects that export
+ writable buffers (e.g. :class:`bytearray` objects); these will be
+ filled with successive chunks of the non-ancillary data until it
+ has all been written or there are no more buffers. The operating
+ system may set a limit (:func:`~os.sysconf` value ``SC_IOV_MAX``)
+ on the number of buffers that can be used. The *ancbufsize* and
+ *flags* arguments have the same meaning as for :meth:`recvmsg`.
+
+ The return value is a 4-tuple: ``(nbytes, ancdata, msg_flags,
+ address)``, where *nbytes* is the total number of bytes of
+ non-ancillary data written into the buffers, and *ancdata*,
+ *msg_flags* and *address* are the same as for :meth:`recvmsg`.
+
+ Example::
+
+ >>> import socket
+ >>> s1, s2 = socket.socketpair()
+ >>> b1 = bytearray(b'----')
+ >>> b2 = bytearray(b'0123456789')
+ >>> b3 = bytearray(b'--------------')
+ >>> s1.send(b'Mary had a little lamb')
+ 22
+ >>> s2.recvmsg_into([b1, memoryview(b2)[2:9], b3])
+ (22, [], 0, None)
+ >>> [b1, b2, b3]
+ [bytearray(b'Mary'), bytearray(b'01 had a 9'), bytearray(b'little lamb---')]
+
+ Availability: most Unix platforms, possibly others.
+
+ .. versionadded:: 3.3
+
+
.. method:: socket.recvfrom_into(buffer[, nbytes[, flags]])
Receive data from the socket, writing it into *buffer* instead of creating a
@@ -746,6 +960,41 @@ correspond to Unix system calls applicable to sockets.
above.)
+.. method:: socket.sendmsg(buffers[, ancdata[, flags[, address]]])
+
+ Send normal and ancillary data to the socket, gathering the
+ non-ancillary data from a series of buffers and concatenating it
+ into a single message. The *buffers* argument specifies the
+ non-ancillary data as an iterable of buffer-compatible objects
+ (e.g. :class:`bytes` objects); the operating system may set a limit
+ (:func:`~os.sysconf` value ``SC_IOV_MAX``) on the number of buffers
+ that can be used. The *ancdata* argument specifies the ancillary
+ data (control messages) as an iterable of zero or more tuples
+ ``(cmsg_level, cmsg_type, cmsg_data)``, where *cmsg_level* and
+ *cmsg_type* are integers specifying the protocol level and
+ protocol-specific type respectively, and *cmsg_data* is a
+ buffer-compatible object holding the associated data. Note that
+ some systems (in particular, systems without :func:`CMSG_SPACE`)
+ might support sending only one control message per call. The
+ *flags* argument defaults to 0 and has the same meaning as for
+ :meth:`send`. If *address* is supplied and not ``None``, it sets a
+ destination address for the message. The return value is the
+ number of bytes of non-ancillary data sent.
+
+ The following function sends the list of file descriptors *fds*
+ over an :const:`AF_UNIX` socket, on systems which support the
+ :const:`SCM_RIGHTS` mechanism. See also :meth:`recvmsg`. ::
+
+ import socket, array
+
+ def send_fds(sock, msg, fds):
+ return sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array("i", fds))])
+
+ Availability: most Unix platforms, possibly others.
+
+ .. versionadded:: 3.3
+
+
.. method:: socket.setblocking(flag)
Set blocking or non-blocking mode of the socket: if *flag* is false, the
@@ -934,13 +1183,13 @@ sends traffic to the first one connected successfully. ::
af, socktype, proto, canonname, sa = res
try:
s = socket.socket(af, socktype, proto)
- except socket.error as msg:
+ except OSError as msg:
s = None
continue
try:
s.bind(sa)
s.listen(1)
- except socket.error as msg:
+ except OSError as msg:
s.close()
s = None
continue
@@ -969,12 +1218,12 @@ sends traffic to the first one connected successfully. ::
af, socktype, proto, canonname, sa = res
try:
s = socket.socket(af, socktype, proto)
- except socket.error as msg:
+ except OSError as msg:
s = None
continue
try:
s.connect(sa)
- except socket.error as msg:
+ except OSError as msg:
s.close()
s = None
continue
@@ -988,7 +1237,7 @@ sends traffic to the first one connected successfully. ::
print('Received', repr(data))
-The last example shows how to write a very simple network sniffer with raw
+The next example shows how to write a very simple network sniffer with raw
sockets on Windows. The example requires administrator privileges to modify
the interface::
@@ -1013,11 +1262,51 @@ the interface::
# disabled promiscuous mode
s.ioctl(socket.SIO_RCVALL, socket.RCVALL_OFF)
+The last example shows how to use the socket interface to communicate to a CAN
+network. This example might require special priviledge::
+
+ import socket
+ import struct
+
+
+ # CAN frame packing/unpacking (see `struct can_frame` in <linux/can.h>)
+
+ can_frame_fmt = "=IB3x8s"
+ can_frame_size = struct.calcsize(can_frame_fmt)
+
+ def build_can_frame(can_id, data):
+ can_dlc = len(data)
+ data = data.ljust(8, b'\x00')
+ return struct.pack(can_frame_fmt, can_id, can_dlc, data)
+
+ def dissect_can_frame(frame):
+ can_id, can_dlc, data = struct.unpack(can_frame_fmt, frame)
+ return (can_id, can_dlc, data[:can_dlc])
+
+
+ # create a raw socket and bind it to the `vcan0` interface
+ s = socket.socket(socket.AF_CAN, socket.SOCK_RAW, socket.CAN_RAW)
+ s.bind(('vcan0',))
+
+ while True:
+ cf, addr = s.recvfrom(can_frame_size)
+
+ print('Received: can_id=%x, can_dlc=%x, data=%s' % dissect_can_frame(cf))
+
+ try:
+ s.send(cf)
+ except OSError:
+ print('Error sending CAN frame')
+
+ try:
+ s.send(build_can_frame(0x01, b'\x01\x02\x03'))
+ except OSError:
+ print('Error sending CAN frame')
Running an example several times with too small delay between executions, could
lead to this error::
- socket.error: [Errno 98] Address already in use
+ OSError: [Errno 98] Address already in use
This is because the previous execution has left the socket in a ``TIME_WAIT``
state, and can't be immediately reused.
diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst
index 177c0b1..8623380 100644
--- a/Doc/library/socketserver.rst
+++ b/Doc/library/socketserver.rst
@@ -153,8 +153,21 @@ Server Objects
.. method:: BaseServer.serve_forever(poll_interval=0.5)
Handle requests until an explicit :meth:`shutdown` request. Polls for
- shutdown every *poll_interval* seconds.
+ shutdown every *poll_interval* seconds. It also calls
+ :meth:`service_actions` which may be used by a subclass or Mixin to provide
+ various cleanup actions. For e.g. ForkingMixin class uses
+ :meth:`service_actions` to cleanup the zombie child processes.
+ .. versionchanged:: 3.3
+ Added service_actions call to the serve_forever method.
+
+
+.. method:: BaseServer.service_actions()
+
+ This is called by the serve_forever loop. This method is can be overridden
+ by Mixin's to add cleanup or service specific actions.
+
+ .. versionadded:: 3.3
.. method:: BaseServer.shutdown()
diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst
index 786bb04..f3acdc5 100644
--- a/Doc/library/sqlite3.rst
+++ b/Doc/library/sqlite3.rst
@@ -369,6 +369,22 @@ Connection Objects
method with :const:`None` for *handler*.
+.. method:: Connection.set_trace_callback(trace_callback)
+
+ Registers *trace_callback* to be called for each SQL statement that is
+ actually executed by the SQLite backend.
+
+ The only argument passed to the callback is the statement (as string) that
+ is being executed. The return value of the callback is ignored. Note that
+ the backend does not only run statements passed to the :meth:`Cursor.execute`
+ methods. Other sources include the transaction management of the Python
+ module and the execution of triggers defined in the current database.
+
+ Passing :const:`None` as *trace_callback* will disable the trace callback.
+
+ .. versionadded:: 3.3
+
+
.. method:: Connection.enable_load_extension(enabled)
This routine allows/disallows the SQLite engine to load SQLite extensions
diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst
index 497c5ba..6651a69 100644
--- a/Doc/library/ssl.rst
+++ b/Doc/library/ssl.rst
@@ -53,9 +53,53 @@ Functions, Constants, and Exceptions
(currently provided by the OpenSSL library). This signifies some
problem in the higher-level encryption and authentication layer that's
superimposed on the underlying network connection. This error
- is a subtype of :exc:`socket.error`, which in turn is a subtype of
- :exc:`IOError`. The error code and message of :exc:`SSLError` instances
- are provided by the OpenSSL library.
+ is a subtype of :exc:`OSError`. The error code and message of
+ :exc:`SSLError` instances are provided by the OpenSSL library.
+
+ .. versionchanged:: 3.3
+ :exc:`SSLError` used to be a subtype of :exc:`socket.error`.
+
+.. exception:: SSLZeroReturnError
+
+ A subclass of :exc:`SSLError` raised when trying to read or write and
+ the SSL connection has been closed cleanly. Note that this doesn't
+ mean that the underlying transport (read TCP) has been closed.
+
+ .. versionadded:: 3.3
+
+.. exception:: SSLWantReadError
+
+ A subclass of :exc:`SSLError` raised by a :ref:`non-blocking SSL socket
+ <ssl-nonblocking>` when trying to read or write data, but more data needs
+ to be received on the underlying TCP transport before the request can be
+ fulfilled.
+
+ .. versionadded:: 3.3
+
+.. exception:: SSLWantWriteError
+
+ A subclass of :exc:`SSLError` raised by a :ref:`non-blocking SSL socket
+ <ssl-nonblocking>` when trying to read or write data, but more data needs
+ to be sent on the underlying TCP transport before the request can be
+ fulfilled.
+
+ .. versionadded:: 3.3
+
+.. exception:: SSLSyscallError
+
+ A subclass of :exc:`SSLError` raised when a system error was encountered
+ while trying to fulfill an operation on a SSL socket. Unfortunately,
+ there is no easy way to inspect the original errno number.
+
+ .. versionadded:: 3.3
+
+.. exception:: SSLEOFError
+
+ A subclass of :exc:`SSLError` raised when the SSL connection has been
+ terminated abruptly. Generally, you shouldn't try to reuse the underlying
+ transport when this error is encountered.
+
+ .. versionadded:: 3.3
.. exception:: CertificateError
@@ -162,6 +206,35 @@ instead.
Random generation
^^^^^^^^^^^^^^^^^
+.. function:: RAND_bytes(num)
+
+ Returns *num* cryptographically strong pseudo-random bytes. Raises an
+ :class:`SSLError` if the PRNG has not been seeded with enough data or if the
+ operation is not supported by the current RAND method. :func:`RAND_status`
+ can be used to check the status of the PRNG and :func:`RAND_add` can be used
+ to seed the PRNG.
+
+ Read the Wikipedia article, `Cryptographically secure pseudorandom number
+ generator (CSPRNG)
+ <http://en.wikipedia.org/wiki/Cryptographically_secure_pseudorandom_number_generator>`_,
+ to get the requirements of a cryptographically generator.
+
+ .. versionadded:: 3.3
+
+.. function:: RAND_pseudo_bytes(num)
+
+ Returns (bytes, is_cryptographic): bytes are *num* pseudo-random bytes,
+ is_cryptographic is True if the bytes generated are cryptographically
+ strong. Raises an :class:`SSLError` if the operation is not supported by the
+ current RAND method.
+
+ Generated pseudo-random byte sequences will be unique if they are of
+ sufficient length, but are not necessarily unpredictable. They can be used
+ for non-cryptographic purposes and for certain purposes in cryptographic
+ protocols, but usually not for key generation etc.
+
+ .. versionadded:: 3.3
+
.. function:: RAND_status()
Returns True if the SSL pseudo-random number generator has been seeded with
@@ -171,7 +244,7 @@ Random generation
.. function:: RAND_egd(path)
- If you are running an entropy-gathering daemon (EGD) somewhere, and ``path``
+ If you are running an entropy-gathering daemon (EGD) somewhere, and *path*
is the pathname of a socket connection open to it, this will read 256 bytes
of randomness from the socket, and add it to the SSL pseudo-random number
generator to increase the security of generated secret keys. This is
@@ -182,8 +255,8 @@ Random generation
.. function:: RAND_add(bytes, entropy)
- Mixes the given ``bytes`` into the SSL pseudo-random number generator. The
- parameter ``entropy`` (a float) is a lower bound on the entropy contained in
+ Mixes the given *bytes* into the SSL pseudo-random number generator. The
+ parameter *entropy* (a float) is a lower bound on the entropy contained in
string (so you can always use :const:`0.0`). See :rfc:`1750` for more
information on sources of entropy.
@@ -239,6 +312,9 @@ Certificate handling
will attempt to validate the server certificate against that set of root
certificates, and will fail if the validation attempt fails.
+ .. versionchanged:: 3.3
+ This function is now IPv6-compatible.
+
.. function:: DER_cert_to_PEM_cert(DER_cert_bytes)
Given a certificate as a DER-encoded blob of bytes, returns a PEM-encoded
@@ -354,6 +430,13 @@ Constants
.. versionadded:: 3.2
+.. data:: CHANNEL_BINDING_TYPES
+
+ List of supported TLS channel binding types. Strings in this list
+ can be used as arguments to :meth:`SSLSocket.get_channel_binding`.
+
+ .. versionadded:: 3.3
+
.. data:: OPENSSL_VERSION
The version string of the OpenSSL library loaded by the interpreter::
@@ -463,6 +546,18 @@ SSL sockets also have the following additional methods and attributes:
version of the SSL protocol that defines its use, and the number of secret
bits being used. If no connection has been established, returns ``None``.
+.. method:: SSLSocket.get_channel_binding(cb_type="tls-unique")
+
+ Get channel binding data for current connection, as a bytes object. Returns
+ ``None`` if not connected or the handshake has not been completed.
+
+ The *cb_type* parameter allow selection of the desired channel binding
+ type. Valid channel binding types are listed in the
+ :data:`CHANNEL_BINDING_TYPES` list. Currently only the 'tls-unique' channel
+ binding, defined by :rfc:`5929`, is supported. :exc:`ValueError` will be
+ raised if an unsupported channel binding type is requested.
+
+ .. versionadded:: 3.3
.. method:: SSLSocket.unwrap()
@@ -502,7 +597,7 @@ to speed up repeated connections from the same clients.
:class:`SSLContext` objects have the following methods and attributes:
-.. method:: SSLContext.load_cert_chain(certfile, keyfile=None)
+.. method:: SSLContext.load_cert_chain(certfile, keyfile=None, password=None)
Load a private key and the corresponding certificate. The *certfile*
string must be the path to a single file in PEM format containing the
@@ -513,9 +608,25 @@ to speed up repeated connections from the same clients.
:ref:`ssl-certificates` for more information on how the certificate
is stored in the *certfile*.
+ The *password* argument may be a function to call to get the password for
+ decrypting the private key. It will only be called if the private key is
+ encrypted and a password is necessary. It will be called with no arguments,
+ and it should return a string, bytes, or bytearray. If the return value is
+ a string it will be encoded as UTF-8 before using it to decrypt the key.
+ Alternatively a string, bytes, or bytearray value may be supplied directly
+ as the *password* argument. It will be ignored if the private key is not
+ encrypted and no password is needed.
+
+ If the *password* argument is not specified and a password is required,
+ OpenSSL's built-in password prompting mechanism will be used to
+ interactively prompt the user for a password.
+
An :class:`SSLError` is raised if the private key doesn't
match with the certificate.
+ .. versionchanged:: 3.3
+ New optional argument *password*.
+
.. method:: SSLContext.load_verify_locations(cafile=None, capath=None)
Load a set of "certification authority" (CA) certificates used to validate
@@ -933,13 +1044,10 @@ to be aware of:
try:
sock.do_handshake()
break
- except ssl.SSLError as err:
- if err.args[0] == ssl.SSL_ERROR_WANT_READ:
- select.select([sock], [], [])
- elif err.args[0] == ssl.SSL_ERROR_WANT_WRITE:
- select.select([], [sock], [])
- else:
- raise
+ except ssl.SSLWantReadError:
+ select.select([sock], [], [])
+ except ssl.SSLWantWriteError:
+ select.select([], [sock], [])
.. _ssl-security:
diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index 942ecba..b3bd8fb 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -15,6 +15,10 @@ interpreter.
The principal built-in types are numerics, sequences, mappings, classes,
instances and exceptions.
+Some collection classes are mutable. The methods that add, subtract, or
+rearrange their members in place, and don't return a specific item, never return
+the collection instance itself but ``None``.
+
Some operations are supported by several object types; in particular,
practically all objects can be compared, tested for truth value, and converted
to a string (with the :func:`repr` function or the slightly different
@@ -1664,6 +1668,8 @@ Note that while lists allow their items to be of any type, bytearray object
single: append() (sequence method)
single: extend() (sequence method)
single: count() (sequence method)
+ single: clear() (sequence method)
+ single: copy() (sequence method)
single: index() (sequence method)
single: insert() (sequence method)
single: pop() (sequence method)
@@ -1695,6 +1701,12 @@ Note that while lists allow their items to be of any type, bytearray object
| ``s.extend(x)`` | same as ``s[len(s):len(s)] = | \(2) |
| | x`` | |
+------------------------------+--------------------------------+---------------------+
+| ``s.clear()`` | remove all items from ``s`` | |
+| | | |
++------------------------------+--------------------------------+---------------------+
+| ``s.copy()`` | return a shallow copy of ``s`` | |
+| | | |
++------------------------------+--------------------------------+---------------------+
| ``s.count(x)`` | return number of *i*'s for | |
| | which ``s[i] == x`` | |
+------------------------------+--------------------------------+---------------------+
@@ -1773,6 +1785,9 @@ Notes:
(8)
:meth:`sort` is not supported by :class:`bytearray` objects.
+ .. versionadded:: 3.3
+ :meth:`clear` and :meth:`!copy` methods.
+
.. _bytes-methods:
@@ -1790,6 +1805,12 @@ the objects to strings, they have a :func:`decode` method.
Wherever one of these methods needs to interpret the bytes as characters
(e.g. the :func:`is...` methods), the ASCII character set is assumed.
+.. versionadded:: 3.3
+ The functions :func:`count`, :func:`find`, :func:`index`,
+ :func:`rfind` and :func:`rindex` have additional semantics compared to
+ the corresponding string functions: They also accept an integer in
+ range 0 to 255 (a byte) as their first argument.
+
.. note::
The methods on bytes and bytearray objects don't accept strings as their
@@ -2695,7 +2716,7 @@ The Null Object
This object is returned by functions that don't explicitly return a value. It
supports no special operations. There is exactly one null object, named
-``None`` (a built-in name).
+``None`` (a built-in name). ``type(None)()`` produces the same singleton.
It is written as ``None``.
@@ -2707,7 +2728,8 @@ The Ellipsis Object
This object is commonly used by slicing (see :ref:`slicings`). It supports no
special operations. There is exactly one ellipsis object, named
-:const:`Ellipsis` (a built-in name).
+:const:`Ellipsis` (a built-in name). ``type(Ellipsis)()`` produces the
+:const:`Ellipsis` singleton.
It is written as ``Ellipsis`` or ``...``.
@@ -2719,7 +2741,8 @@ The NotImplemented Object
This object is returned from comparisons and binary operations when they are
asked to operate on types they don't support. See :ref:`comparisons` for more
-information.
+information. There is exactly one ``NotImplemented`` object.
+``type(NotImplemented)()`` produces the singleton instance.
It is written as ``NotImplemented``.
diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst
index 12820e0..994506c 100644
--- a/Doc/library/struct.rst
+++ b/Doc/library/struct.rst
@@ -187,17 +187,24 @@ platform-dependent.
| ``Q`` | :c:type:`unsigned long | integer | 8 | \(2), \(3) |
| | long` | | | |
+--------+--------------------------+--------------------+----------------+------------+
-| ``f`` | :c:type:`float` | float | 4 | \(4) |
+| ``n`` | :c:type:`ssize_t` | integer | | \(4) |
+--------+--------------------------+--------------------+----------------+------------+
-| ``d`` | :c:type:`double` | float | 8 | \(4) |
+| ``N`` | :c:type:`size_t` | integer | | \(4) |
++--------+--------------------------+--------------------+----------------+------------+
+| ``f`` | :c:type:`float` | float | 4 | \(5) |
++--------+--------------------------+--------------------+----------------+------------+
+| ``d`` | :c:type:`double` | float | 8 | \(5) |
+--------+--------------------------+--------------------+----------------+------------+
| ``s`` | :c:type:`char[]` | bytes | | |
+--------+--------------------------+--------------------+----------------+------------+
| ``p`` | :c:type:`char[]` | bytes | | |
+--------+--------------------------+--------------------+----------------+------------+
-| ``P`` | :c:type:`void \*` | integer | | \(5) |
+| ``P`` | :c:type:`void \*` | integer | | \(6) |
+--------+--------------------------+--------------------+----------------+------------+
+.. versionchanged:: 3.3
+ Added support for the ``'n'`` and ``'N'`` formats.
+
Notes:
(1)
@@ -219,11 +226,17 @@ Notes:
Use of the :meth:`__index__` method for non-integers is new in 3.2.
(4)
+ The ``'n'`` and ``'N'`` conversion codes are only available for the native
+ size (selected as the default or with the ``'@'`` byte order character).
+ For the standard size, you can use whichever of the other integer formats
+ fits your application.
+
+(5)
For the ``'f'`` and ``'d'`` conversion codes, the packed representation uses
the IEEE 754 binary32 (for ``'f'``) or binary64 (for ``'d'``) format,
regardless of the floating-point format used by the platform.
-(5)
+(6)
The ``'P'`` format character is only available for the native byte ordering
(selected as the default or with the ``'@'`` byte order character). The byte
order character ``'='`` chooses to use little- or big-endian ordering based
diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst
index 16c2605..2c76130 100644
--- a/Doc/library/subprocess.rst
+++ b/Doc/library/subprocess.rst
@@ -92,7 +92,8 @@ This module defines one class called :class:`Popen`:
>>> call("cat " + filename, shell=True) # Uh-oh. This will end badly...
*shell=False* does not suffer from this vulnerability; the above Note may be
- helpful in getting code using *shell=False* to work.
+ helpful in getting code using *shell=False* to work. See also
+ :func:`shlex.quote` for a function useful to quote filenames and commands.
On Windows: the :class:`Popen` class uses CreateProcess() to execute the
child program, which operates on strings. If *args* is a sequence, it will
@@ -123,12 +124,14 @@ This module defines one class called :class:`Popen`:
*stdin*, *stdout* and *stderr* specify the executed programs' standard input,
standard output and standard error file handles, respectively. Valid values
- are :data:`PIPE`, an existing file descriptor (a positive integer), an
- existing :term:`file object`, and ``None``. :data:`PIPE` indicates that a
- new pipe to the child should be created. With ``None``, no redirection will
- occur; the child's file handles will be inherited from the parent. Additionally,
- *stderr* can be :data:`STDOUT`, which indicates that the stderr data from the
- applications should be captured into the same file handle as for stdout.
+ are :data:`PIPE`, :data:`DEVNULL`, an existing file descriptor (a positive
+ integer), an existing :term:`file object`, and ``None``. :data:`PIPE`
+ indicates that a new pipe to the child should be created. :data:`DEVNULL`
+ indicates that the special file :data:`os.devnull` will be used. With ``None``,
+ no redirection will occur; the child's file handles will be inherited from
+ the parent. Additionally, *stderr* can be :data:`STDOUT`, which indicates
+ that the stderr data from the applications should be captured into the same
+ file handle as for stdout.
If *preexec_fn* is set to a callable object, this object will be called in the
child process just before the child is executed.
@@ -228,6 +231,15 @@ This module defines one class called :class:`Popen`:
Added context manager support.
+.. data:: DEVNULL
+
+ Special value that can be used as the *stdin*, *stdout* or *stderr* argument
+ to :class:`Popen` and indicates that the special file :data:`os.devnull`
+ will be used.
+
+ .. versionadded:: 3.3
+
+
.. data:: PIPE
Special value that can be used as the *stdin*, *stdout* or *stderr* argument
@@ -248,15 +260,21 @@ Convenience Functions
This module also defines the following shortcut functions:
-.. function:: call(*popenargs, **kwargs)
+.. function:: call(*popenargs, timeout=None, **kwargs)
Run command with arguments. Wait for command to complete, then return the
:attr:`returncode` attribute.
- The arguments are the same as for the :class:`Popen` constructor. Example::
+ The arguments are the same as for the :class:`Popen` constructor, with the
+ exception of the *timeout* argument, which is given to :meth:`Popen.wait`.
+ Example::
>>> retcode = subprocess.call(["ls", "-l"])
+ If the timeout expires, the child process will be killed and then waited for
+ again. The :exc:`TimeoutExpired` exception will be re-raised after the child
+ process has terminated.
+
.. warning::
Like :meth:`Popen.wait`, this will deadlock when using
@@ -264,34 +282,43 @@ This module also defines the following shortcut functions:
generates enough output to a pipe such that it blocks waiting
for the OS pipe buffer to accept more data.
+ .. versionchanged:: 3.3
+ *timeout* was added.
-.. function:: check_call(*popenargs, **kwargs)
+
+.. function:: check_call(*popenargs, timeout=None, **kwargs)
Run command with arguments. Wait for command to complete. If the exit code was
zero then return, otherwise raise :exc:`CalledProcessError`. The
:exc:`CalledProcessError` object will have the return code in the
:attr:`returncode` attribute.
- The arguments are the same as for the :class:`Popen` constructor. Example::
+ The arguments are the same as for the :func:`call` function. Example::
>>> subprocess.check_call(["ls", "-l"])
0
+ As in the :func:`call` function, if the timeout expires, the child process
+ will be killed and the wait retried. The :exc:`TimeoutExpired` exception
+ will be re-raised after the child process has terminated.
+
.. warning::
See the warning for :func:`call`.
+ .. versionchanged:: 3.3
+ *timeout* was added.
+
-.. function:: check_output(*popenargs, **kwargs)
+.. function:: check_output(*popenargs, timeout=None, **kwargs)
- Run command with arguments and return its output as a byte string.
+ Run command with arguments and return its output as a bytes object.
If the exit code was non-zero it raises a :exc:`CalledProcessError`. The
:exc:`CalledProcessError` object will have the return code in the
- :attr:`returncode`
- attribute and output in the :attr:`output` attribute.
+ :attr:`returncode` attribute and output in the :attr:`output` attribute.
- The arguments are the same as for the :class:`Popen` constructor. Example::
+ The arguments are the same as for the :func:`call` function. Example::
>>> subprocess.check_output(["ls", "-l", "/dev/null"])
b'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n'
@@ -304,8 +331,17 @@ This module also defines the following shortcut functions:
... stderr=subprocess.STDOUT)
b'ls: non_existent_file: No such file or directory\n'
+ As in the :func:`call` function, if the timeout expires, the child process
+ will be killed and the wait retried. The :exc:`TimeoutExpired` exception
+ will be re-raised after the child process has terminated. The output from
+ the child process so far will be in the :attr:`output` attribute of the
+ exception.
+
.. versionadded:: 3.1
+ .. versionchanged:: 3.3
+ *timeout* was added.
+
.. function:: getstatusoutput(cmd)
@@ -358,6 +394,15 @@ arguments.
check_call() will raise :exc:`CalledProcessError`, if the called process returns
a non-zero return code.
+All of the functions and methods that accept a *timeout* parameter, such as
+:func:`call` and :meth:`Popen.communicate` will raise :exc:`TimeoutExpired` if
+the timeout expires before the process exits.
+
+Exceptions defined in this module all inherit from :exc:`SubprocessError`.
+
+ .. versionadded:: 3.3
+ The :exc:`SubprocessError` base class was added.
+
Security
^^^^^^^^
@@ -379,11 +424,15 @@ Instances of the :class:`Popen` class have the following methods:
attribute.
-.. method:: Popen.wait()
+.. method:: Popen.wait(timeout=None)
Wait for child process to terminate. Set and return :attr:`returncode`
attribute.
+ If the process does not terminate after *timeout* seconds, raise a
+ :exc:`TimeoutExpired` exception. It is safe to catch this exception and
+ retry the wait.
+
.. warning::
This will deadlock when using ``stdout=PIPE`` and/or
@@ -391,13 +440,17 @@ Instances of the :class:`Popen` class have the following methods:
a pipe such that it blocks waiting for the OS pipe buffer to
accept more data. Use :meth:`communicate` to avoid that.
+ .. versionchanged:: 3.3
+ *timeout* was added.
+
-.. method:: Popen.communicate(input=None)
+.. method:: Popen.communicate(input=None, timeout=None)
Interact with process: Send data to stdin. Read data from stdout and stderr,
- until end-of-file is reached. Wait for process to terminate. The optional
- *input* argument should be a byte string to be sent to the child process, or
- ``None``, if no data should be sent to the child.
+ until end-of-file is reached. Wait for process to terminate. The optional
+ *input* argument should be data to be sent to the child process, or
+ ``None``, if no data should be sent to the child. The type of *input*
+ must be bytes or, if *universal_newlines* was ``True``, a string.
:meth:`communicate` returns a tuple ``(stdoutdata, stderrdata)``.
@@ -406,11 +459,29 @@ Instances of the :class:`Popen` class have the following methods:
``None`` in the result tuple, you need to give ``stdout=PIPE`` and/or
``stderr=PIPE`` too.
+ If the process does not terminate after *timeout* seconds, a
+ :exc:`TimeoutExpired` exception will be raised. Catching this exception and
+ retrying communication will not lose any output.
+
+ The child process is not killed if the timeout expires, so in order to
+ cleanup properly a well-behaved application should kill the child process and
+ finish communication::
+
+ proc = subprocess.Popen(...)
+ try:
+ outs, errs = proc.communicate(timeout=15)
+ except TimeoutExpired:
+ proc.kill()
+ outs, errs = proc.communicate()
+
.. note::
The data read is buffered in memory, so do not use this method if the data
size is large or unlimited.
+ .. versionchanged:: 3.3
+ *timeout* was added.
+
.. method:: Popen.send_signal(signal)
@@ -801,3 +872,7 @@ runtime):
described in rule 3.
+.. seealso::
+
+ :mod:`shlex`
+ Module which provides function to parse and escape command lines.
diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
index fdf188f..a032549 100644
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -233,14 +233,13 @@ always available.
.. data:: flags
- The struct sequence *flags* exposes the status of command line flags. The
- attributes are read only.
+ The :term:`struct sequence` *flags* exposes the status of command line
+ flags. The attributes are read only.
============================= =============================
attribute flag
============================= =============================
:const:`debug` :option:`-d`
- :const:`division_warning` :option:`-Q`
:const:`inspect` :option:`-i`
:const:`interactive` :option:`-i`
:const:`optimize` :option:`-O` or :option:`-OO`
@@ -256,15 +255,18 @@ always available.
.. versionchanged:: 3.2
Added ``quiet`` attribute for the new :option:`-q` flag.
+ .. versionchanged:: 3.3
+ Removed obsolete ``division_warning`` attribute.
+
.. data:: float_info
- A structseq holding information about the float type. It contains low level
- information about the precision and internal representation. The values
- correspond to the various floating-point constants defined in the standard
- header file :file:`float.h` for the 'C' programming language; see section
- 5.2.4.2.2 of the 1999 ISO/IEC C standard [C99]_, 'Characteristics of
- floating types', for details.
+ A :term:`struct sequence` holding information about the float type. It
+ contains low level information about the precision and internal
+ representation. The values correspond to the various floating-point
+ constants defined in the standard header file :file:`float.h` for the 'C'
+ programming language; see section 5.2.4.2.2 of the 1999 ISO/IEC C standard
+ [C99]_, 'Characteristics of floating types', for details.
+---------------------+----------------+--------------------------------------------------+
| attribute | float.h macro | explanation |
@@ -510,8 +512,9 @@ always available.
.. data:: hash_info
- A structseq giving parameters of the numeric hash implementation. For
- more details about hashing of numeric types, see :ref:`numeric-hash`.
+ A :term:`struct sequence` giving parameters of the numeric hash
+ implementation. For more details about hashing of numeric types, see
+ :ref:`numeric-hash`.
+---------------------+--------------------------------------------------+
| attribute | explanation |
@@ -546,8 +549,8 @@ always available.
This is called ``hexversion`` since it only really looks meaningful when viewed
as the result of passing it to the built-in :func:`hex` function. The
- struct sequence :data:`sys.version_info` may be used for a more human-friendly
- encoding of the same information.
+ :term:`struct sequence` :data:`sys.version_info` may be used for a more
+ human-friendly encoding of the same information.
The ``hexversion`` is a 32-bit number with the following layout:
@@ -575,8 +578,8 @@ always available.
.. data:: int_info
- A struct sequence that holds information about Python's
- internal representation of integers. The attributes are read only.
+ A :term:`struct sequence` that holds information about Python's internal
+ representation of integers. The attributes are read only.
+-------------------------+----------------------------------------------+
| Attribute | Explanation |
@@ -631,9 +634,13 @@ always available.
.. data:: maxunicode
- An integer giving the largest supported code point for a Unicode character. The
- value of this depends on the configuration option that specifies whether Unicode
- characters are stored as UCS-2 or UCS-4.
+ An integer giving the value of the largest Unicode code point,
+ i.e. ``1114111`` (``0x10FFFF`` in hexadecimal).
+
+ .. versionchanged:: 3.3
+ Before :pep:`393`, ``sys.maxunicode`` used to be either ``0xFFFF``
+ or ``0x10FFFF``, depending on the configuration option that specified
+ whether Unicode characters were stored as UCS-2 or UCS-4.
.. data:: meta_path
@@ -708,36 +715,35 @@ always available.
This string contains a platform identifier that can be used to append
platform-specific components to :data:`sys.path`, for instance.
- For most Unix systems, this is the lowercased OS name as returned by ``uname
- -s`` with the first part of the version as returned by ``uname -r`` appended,
- e.g. ``'sunos5'``, *at the time when Python was built*. Unless you want to
- test for a specific system version, it is therefore recommended to use the
- following idiom::
+ For Unix systems, except on Linux, this is the lowercased OS name as
+ returned by ``uname -s`` with the first part of the version as returned by
+ ``uname -r`` appended, e.g. ``'sunos5'`` or ``'freebsd8'``, *at the time
+ when Python was built*. Unless you want to test for a specific system
+ version, it is therefore recommended to use the following idiom::
if sys.platform.startswith('freebsd'):
# FreeBSD-specific code here...
elif sys.platform.startswith('linux'):
# Linux-specific code here...
- .. versionchanged:: 3.2.2
- Since lots of code check for ``sys.platform == 'linux2'``, and there is
- no essential change between Linux 2.x and 3.x, ``sys.platform`` is always
- set to ``'linux2'``, even on Linux 3.x. In Python 3.3 and later, the
- value will always be set to ``'linux'``, so it is recommended to always
- use the ``startswith`` idiom presented above.
-
For other systems, the values are:
- ====================== ===========================
- System :data:`platform` value
- ====================== ===========================
- Linux (2.x *and* 3.x) ``'linux2'``
- Windows ``'win32'``
- Windows/Cygwin ``'cygwin'``
- Mac OS X ``'darwin'``
- OS/2 ``'os2'``
- OS/2 EMX ``'os2emx'``
- ====================== ===========================
+ ================ ===========================
+ System :data:`platform` value
+ ================ ===========================
+ Linux ``'linux'``
+ Windows ``'win32'``
+ Windows/Cygwin ``'cygwin'``
+ Mac OS X ``'darwin'``
+ OS/2 ``'os2'``
+ OS/2 EMX ``'os2emx'``
+ ================ ===========================
+
+ .. versionchanged:: 3.3
+ On Linux, :attr:`sys.platform` doesn't contain the major version anymore.
+ It is always ``'linux'``, instead of ``'linux2'`` or ``'linux3'``. Since
+ older Python versions include the version number, it is recommended to
+ always use the ``startswith`` idiom presented above.
.. seealso::
:attr:`os.name` has a coarser granularity. :func:`os.uname` gives
@@ -746,6 +752,7 @@ always available.
The :mod:`platform` module provides detailed checks for the
system's identity.
+
.. data:: prefix
A string giving the site-specific directory prefix where the platform
@@ -794,11 +801,11 @@ always available.
the interpreter loads extension modules. Among other things, this will enable a
lazy resolving of symbols when importing a module, if called as
``sys.setdlopenflags(0)``. To share symbols across extension modules, call as
- ``sys.setdlopenflags(ctypes.RTLD_GLOBAL)``. Symbolic names for the
- flag modules can be either found in the :mod:`ctypes` module, or in the :mod:`DLFCN`
- module. If :mod:`DLFCN` is not available, it can be generated from
- :file:`/usr/include/dlfcn.h` using the :program:`h2py` script. Availability:
- Unix.
+ ``sys.setdlopenflags(os.RTLD_GLOBAL)``. Symbolic names for the flag modules
+ can be found in the :mod:`os` module (``RTLD_xxx`` constants, e.g.
+ :data:`os.RTLD_LAZY`).
+
+ Availability: Unix.
.. function:: setprofile(profilefunc)
@@ -980,22 +987,33 @@ always available.
to a console and Python apps started with :program:`pythonw`.
-.. data:: subversion
+.. data:: thread_info
- A triple (repo, branch, version) representing the Subversion information of the
- Python interpreter. *repo* is the name of the repository, ``'CPython'``.
- *branch* is a string of one of the forms ``'trunk'``, ``'branches/name'`` or
- ``'tags/name'``. *version* is the output of ``svnversion``, if the interpreter
- was built from a Subversion checkout; it contains the revision number (range)
- and possibly a trailing 'M' if there were local modifications. If the tree was
- exported (or svnversion was not available), it is the revision of
- ``Include/patchlevel.h`` if the branch is a tag. Otherwise, it is ``None``.
+ A :term:`struct sequence` holding information about the thread
+ implementation.
- .. deprecated:: 3.2.1
- Python is now `developed <http://docs.python.org/devguide/>`_ using
- Mercurial. In recent Python 3.2 bugfix releases, :data:`subversion`
- therefore contains placeholder information. It is removed in Python
- 3.3.
+ +------------------+---------------------------------------------------------+
+ | Attribute | Explanation |
+ +==================+=========================================================+
+ | :const:`name` | Name of the thread implementation: |
+ | | |
+ | | * ``'nt'``: Windows threads |
+ | | * ``'os2'``: OS/2 threads |
+ | | * ``'pthread'``: POSIX threads |
+ | | * ``'solaris'``: Solaris threads |
+ +------------------+---------------------------------------------------------+
+ | :const:`lock` | Name of the lock implementation: |
+ | | |
+ | | * ``'semaphore'``: a lock uses a semaphore |
+ | | * ``'mutex+cond'``: a lock uses a mutex |
+ | | and a condition variable |
+ | | * ``None`` if this information is unknown |
+ +------------------+---------------------------------------------------------+
+ | :const:`version` | Name and version of the thread library. It is a string, |
+ | | or ``None`` if these informations are unknown. |
+ +------------------+---------------------------------------------------------+
+
+ .. versionadded:: 3.3
.. data:: tracebacklimit
diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst
index 9b7071b..edae7f1 100644
--- a/Doc/library/tarfile.rst
+++ b/Doc/library/tarfile.rst
@@ -262,9 +262,9 @@ be finalized; only the internally used file object will be closed. See the
If *errorlevel* is ``0``, all errors are ignored when using :meth:`TarFile.extract`.
Nevertheless, they appear as error messages in the debug output, when debugging
- is enabled. If ``1``, all *fatal* errors are raised as :exc:`OSError` or
- :exc:`IOError` exceptions. If ``2``, all *non-fatal* errors are raised as
- :exc:`TarError` exceptions as well.
+ is enabled. If ``1``, all *fatal* errors are raised as :exc:`OSError`
+ exceptions. If ``2``, all *non-fatal* errors are raised as :exc:`TarError`
+ exceptions as well.
The *encoding* and *errors* arguments define the character encoding to be
used for reading or writing the archive and how conversion errors are going
diff --git a/Doc/library/telnetlib.rst b/Doc/library/telnetlib.rst
index 646634d..9bc79c5 100644
--- a/Doc/library/telnetlib.rst
+++ b/Doc/library/telnetlib.rst
@@ -162,9 +162,13 @@ Telnet Objects
.. method:: Telnet.write(buffer)
Write a byte string to the socket, doubling any IAC characters. This can
- block if the connection is blocked. May raise :exc:`socket.error` if the
+ block if the connection is blocked. May raise :exc:`OSError` if the
connection is closed.
+ .. versionchanged:: 3.3
+ This method used to raise :exc:`socket.error`, which is now an alias
+ of :exc:`OSError`.
+
.. method:: Telnet.interact()
diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst
index fff6c4e..dfeb250 100644
--- a/Doc/library/tempfile.rst
+++ b/Doc/library/tempfile.rst
@@ -25,7 +25,7 @@ instead a string of six random characters is used.
Also, all the user-callable functions now take additional arguments which
allow direct control over the location and name of temporary files. It is
-no longer necessary to use the global *tempdir* and *template* variables.
+no longer necessary to use the global *tempdir* variable.
To maintain backward compatibility, the argument order is somewhat odd; it
is recommended to use keyword arguments for clarity.
diff --git a/Doc/library/test.rst b/Doc/library/test.rst
index c27ee08..5e4a1cb 100644
--- a/Doc/library/test.rst
+++ b/Doc/library/test.rst
@@ -223,14 +223,14 @@ The :mod:`test.support` module defines the following constants:
.. data:: verbose
- :const:`True` when verbose output is enabled. Should be checked when more
+ ``True`` when verbose output is enabled. Should be checked when more
detailed information is desired about a running test. *verbose* is set by
:mod:`test.regrtest`.
.. data:: is_jython
- :const:`True` if the running interpreter is Jython.
+ ``True`` if the running interpreter is Jython.
.. data:: TESTFN
@@ -249,7 +249,7 @@ The :mod:`test.support` module defines the following functions:
.. function:: is_resource_enabled(resource)
- Return :const:`True` if *resource* is enabled and available. The list of
+ Return ``True`` if *resource* is enabled and available. The list of
available resources is only set when :mod:`test.regrtest` is executing the
tests.
@@ -258,7 +258,7 @@ The :mod:`test.support` module defines the following functions:
Raise :exc:`ResourceDenied` if *resource* is not available. *msg* is the
argument to :exc:`ResourceDenied` if it is raised. Always returns
- :const:`True` if called by a function whose ``__name__`` is ``'__main__'``.
+ ``True`` if called by a function whose ``__name__`` is ``'__main__'``.
Used when tests are executed by :mod:`test.regrtest`.
@@ -286,6 +286,15 @@ The :mod:`test.support` module defines the following functions:
This will run all tests defined in the named module.
+.. function:: run_doctest(module, verbosity=None)
+
+ Run :func:`doctest.testmod` on the given *module*. Return
+ ``(failure_count, test_count)``.
+
+ If *verbosity* is ``None``, :func:`doctest.testmod` is run with verbosity
+ set to :data:`verbose`. Otherwise, it is run with verbosity set to
+ ``None``.
+
.. function:: check_warnings(\*filters, quiet=True)
A convenience wrapper for :func:`warnings.catch_warnings()` that makes it
@@ -296,12 +305,12 @@ The :mod:`test.support` module defines the following functions:
``check_warnings`` accepts 2-tuples of the form ``("message regexp",
WarningCategory)`` as positional arguments. If one or more *filters* are
- provided, or if the optional keyword argument *quiet* is :const:`False`,
+ provided, or if the optional keyword argument *quiet* is ``False``,
it checks to make sure the warnings are as expected: each specified filter
must match at least one of the warnings raised by the enclosed code or the
test fails, and if any warnings are raised that do not match any of the
specified filters the test fails. To disable the first of these checks,
- set *quiet* to :const:`True`.
+ set *quiet* to ``True``.
If no arguments are specified, it defaults to::
@@ -316,7 +325,7 @@ The :mod:`test.support` module defines the following functions:
representing the most recent warning can also be accessed directly through
the recorder object (see example below). If no warning has been raised,
then any of the attributes that would otherwise be expected on an object
- representing a warning will return :const:`None`.
+ representing a warning will return ``None``.
The recorder object also has a :meth:`reset` method, which clears the
warnings list.
@@ -354,7 +363,7 @@ The :mod:`test.support` module defines the following functions:
.. function:: captured_stdout()
- This is a context manager that runs the :keyword:`with` statement body using
+ A context manager that runs the :keyword:`with` statement body using
a :class:`StringIO.StringIO` object as sys.stdout. That object can be
retrieved using the ``as`` clause of the :keyword:`with` statement.
@@ -365,6 +374,57 @@ The :mod:`test.support` module defines the following functions:
assert s.getvalue() == "hello"
+.. function:: temp_cwd(name='tempcwd', quiet=False, path=None)
+
+ A context manager that temporarily changes the current working
+ directory (CWD).
+
+ An existing path may be provided as *path*, in which case this function
+ makes no changes to the file system.
+
+ Otherwise, the new CWD is created in the current directory and it's named
+ *name*. If *quiet* is ``False`` and it's not possible to create or
+ change the CWD, an error is raised. If it's ``True``, only a warning
+ is raised and the original CWD is used.
+
+
+.. function:: temp_umask(umask)
+
+ A context manager that temporarily sets the process umask.
+
+
+.. function:: can_symlink()
+
+ Return ``True`` if the OS supports symbolic links, ``False``
+ otherwise.
+
+
+.. decorator:: skip_unless_symlink()
+
+ A decorator for running tests that require support for symbolic links.
+
+
+.. decorator:: anticipate_failure(condition)
+
+ A decorator to conditionally mark tests with
+ :func:`unittest.expectedFailure`. Any use of this decorator should
+ have an associated comment identifying the relevant tracker issue.
+
+
+.. decorator:: run_with_locale(catstr, *locales)
+
+ A decorator for running a function in a different locale, correctly
+ resetting it after it has finished. *catstr* is the locale category as
+ a string (for example ``"LC_ALL"``). The *locales* passed will be tried
+ sequentially, and the first valid locale will be used.
+
+
+.. function:: make_bad_fd()
+
+ Create an invalid file descriptor by opening and closing a temporary file,
+ and returning its descripor.
+
+
.. function:: import_module(name, deprecated=False)
This function imports and returns the named module. Unlike a normal
@@ -372,7 +432,7 @@ The :mod:`test.support` module defines the following functions:
cannot be imported.
Module and package deprecation messages are suppressed during this import
- if *deprecated* is :const:`True`.
+ if *deprecated* is ``True``.
.. versionadded:: 3.1
@@ -396,9 +456,9 @@ The :mod:`test.support` module defines the following functions:
``sys.modules`` when the fresh import is complete.
Module and package deprecation messages are suppressed during this import
- if *deprecated* is :const:`True`.
+ if *deprecated* is ``True``.
- This function will raise :exc:`unittest.SkipTest` is the named module
+ This function will raise :exc:`unittest.SkipTest` if the named module
cannot be imported.
Example use::
@@ -413,6 +473,48 @@ The :mod:`test.support` module defines the following functions:
.. versionadded:: 3.1
+.. function:: bind_port(sock, host=HOST)
+
+ Bind the socket to a free port and return the port number. Relies on
+ ephemeral ports in order to ensure we are using an unbound port. This is
+ important as many tests may be running simultaneously, especially in a
+ buildbot environment. This method raises an exception if the
+ ``sock.family`` is :const:`~socket.AF_INET` and ``sock.type`` is
+ :const:`~socket.SOCK_STREAM`, and the socket has
+ :const:`~socket.SO_REUSEADDR` or :const:`~socket.SO_REUSEPORT` set on it.
+ Tests should never set these socket options for TCP/IP sockets.
+ The only case for setting these options is testing multicasting via
+ multiple UDP sockets.
+
+ Additionally, if the :const:`~socket.SO_EXCLUSIVEADDRUSE` socket option is
+ available (i.e. on Windows), it will be set on the socket. This will
+ prevent anyone else from binding to our host/port for the duration of the
+ test.
+
+
+.. function:: find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM)
+
+ Returns an unused port that should be suitable for binding. This is
+ achieved by creating a temporary socket with the same family and type as
+ the ``sock`` parameter (default is :const:`~socket.AF_INET`,
+ :const:`~socket.SOCK_STREAM`),
+ and binding it to the specified host address (defaults to ``0.0.0.0``)
+ with the port set to 0, eliciting an unused ephemeral port from the OS.
+ The temporary socket is then closed and deleted, and the ephemeral port is
+ returned.
+
+ Either this method or :func:`bind_port` should be used for any tests
+ where a server socket needs to be bound to a particular port for the
+ duration of the test.
+ Which one to use depends on whether the calling code is creating a python
+ socket, or if an unused port needs to be provided in a constructor
+ or passed to an external program (i.e. the ``-accept`` argument to
+ openssl's s_server mode). Always prefer :func:`bind_port` over
+ :func:`find_unused_port` where possible. Using a hard coded port is
+ discouraged since it can makes multiple instances of the test impossible to
+ run simultaneously, which is a problem for buildbots.
+
+
The :mod:`test.support` module defines the following classes:
.. class:: TransientResource(exc, **kwargs)
diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst
index 1f1d775..66b3501 100644
--- a/Doc/library/threading.rst
+++ b/Doc/library/threading.rst
@@ -20,17 +20,6 @@ The :mod:`dummy_threading` module is provided for situations where
methods and functions in this module in the Python 2.x series are still
supported by this module.
-.. impl-detail::
-
- Due to the :term:`Global Interpreter Lock`, in CPython only one thread
- can execute Python code at once (even though certain performance-oriented
- libraries might overcome this limitation).
- If you want your application to make better of use of the computational
- resources of multi-core machines, you are advised to use
- :mod:`multiprocessing` or :class:`concurrent.futures.ProcessPoolExecutor`.
- However, threading is still an appropriate model if you want to run
- multiple I/O-bound tasks simultaneously.
-
This module defines the following functions and objects:
@@ -59,6 +48,17 @@ This module defines the following functions and objects:
returned.
+.. function:: get_ident()
+
+ Return the 'thread identifier' of the current thread. This is a nonzero
+ integer. Its value has no direct meaning; it is intended as a magic cookie
+ to be used e.g. to index a dictionary of thread-specific data. Thread
+ identifiers may be recycled when a thread exits and another thread is
+ created.
+
+ .. versionadded:: 3.3
+
+
.. function:: enumerate()
Return a list of all :class:`Thread` objects currently alive. The list
@@ -241,7 +241,7 @@ changed through the :attr:`name` attribute.
A thread can be flagged as a "daemon thread". The significance of this flag is
that the entire Python program exits when only daemon threads are left. The
initial value is inherited from the creating thread. The flag can be set
-through the :attr:`daemon` property.
+through the :attr:`daemon` property or the *daemon* constructor argument.
There is a "main thread" object; this corresponds to the initial thread of
control in the Python program. It is not a daemon thread.
@@ -254,7 +254,8 @@ daemonic, and cannot be :meth:`join`\ ed. They are never deleted, since it is
impossible to detect the termination of alien threads.
-.. class:: Thread(group=None, target=None, name=None, args=(), kwargs={})
+.. class:: Thread(group=None, target=None, name=None, args=(), kwargs={},
+ verbose=None, *, daemon=None)
This constructor should always be called with keyword arguments. Arguments
are:
@@ -273,10 +274,19 @@ impossible to detect the termination of alien threads.
*kwargs* is a dictionary of keyword arguments for the target invocation.
Defaults to ``{}``.
+ *verbose* is a flag used for debugging messages.
+
+ If not ``None``, *daemon* explicitly sets whether the thread is daemonic.
+ If ``None`` (the default), the daemonic property is inherited from the
+ current thread.
+
If the subclass overrides the constructor, it must make sure to invoke the
base class constructor (``Thread.__init__()``) before doing anything else to
the thread.
+ .. versionchanged:: 3.3
+ Added the *daemon* argument.
+
.. method:: start()
Start the thread's activity.
@@ -333,10 +343,10 @@ impossible to detect the termination of alien threads.
.. attribute:: ident
The 'thread identifier' of this thread or ``None`` if the thread has not
- been started. This is a nonzero integer. See the
- :func:`thread.get_ident()` function. Thread identifiers may be recycled
- when a thread exits and another thread is created. The identifier is
- available even after the thread has exited.
+ been started. This is a nonzero integer. See the :func:`get_ident()`
+ function. Thread identifiers may be recycled when a thread exits and
+ another thread is created. The identifier is available even after the
+ thread has exited.
.. method:: is_alive()
@@ -364,6 +374,18 @@ impossible to detect the termination of alien threads.
property instead.
+.. impl-detail::
+
+ Due to the :term:`Global Interpreter Lock`, in CPython only one thread
+ can execute Python code at once (even though certain performance-oriented
+ libraries might overcome this limitation).
+ If you want your application to make better of use of the computational
+ resources of multi-core machines, you are advised to use
+ :mod:`multiprocessing` or :class:`concurrent.futures.ProcessPoolExecutor`.
+ However, threading is still an appropriate model if you want to run
+ multiple I/O-bound tasks simultaneously.
+
+
.. _lock-objects:
Lock Objects
diff --git a/Doc/library/time.rst b/Doc/library/time.rst
index 7c464ac..668651a 100644
--- a/Doc/library/time.rst
+++ b/Doc/library/time.rst
@@ -41,25 +41,6 @@ An explanation of some terminology and conventions is in order.
parsed, they are converted according to the POSIX and ISO C standards: values
69--99 are mapped to 1969--1999, and values 0--68 are mapped to 2000--2068.
- For backward compatibility, years with less than 4 digits are treated
- specially by :func:`asctime`, :func:`mktime`, and :func:`strftime` functions
- that operate on a 9-tuple or :class:`struct_time` values. If year (the first
- value in the 9-tuple) is specified with less than 4 digits, its interpretation
- depends on the value of ``accept2dyear`` variable.
-
- If ``accept2dyear`` is true (default), a backward compatibility behavior is
- invoked as follows:
-
- - for 2-digit year, century is guessed according to POSIX rules for
- ``%y`` strptime format. A deprecation warning is issued when century
- information is guessed in this way.
-
- - for 3-digit or negative year, a :exc:`ValueError` exception is raised.
-
- If ``accept2dyear`` is false (set by the program or as a result of a
- non-empty value assigned to ``PYTHONY2K`` environment variable) all year
- values are interpreted as given.
-
.. index::
single: UTC
single: Coordinated Universal Time
@@ -117,24 +98,6 @@ An explanation of some terminology and conventions is in order.
The module defines the following functions and data items:
-
-.. data:: accept2dyear
-
- Boolean value indicating whether two-digit year values will be
- mapped to 1969--2068 range by :func:`asctime`, :func:`mktime`, and
- :func:`strftime` functions. This is true by default, but will be
- set to false if the environment variable :envvar:`PYTHONY2K` has
- been set to a non-empty string. It may also be modified at run
- time.
-
- .. deprecated:: 3.2
- Mapping of 2-digit year values by :func:`asctime`,
- :func:`mktime`, and :func:`strftime` functions to 1969--2068
- range is deprecated. Programs that need to process 2-digit
- years should use ``%y`` code available in :func:`strptime`
- function or convert 2-digit year values to 4-digit themselves.
-
-
.. data:: altzone
The offset of the local DST timezone, in seconds west of UTC, if one is defined.
@@ -173,6 +136,54 @@ The module defines the following functions and data items:
microsecond.
+.. function:: clock_getres(clk_id)
+
+ Return the resolution (precision) of the specified clock *clk_id*.
+
+ .. versionadded:: 3.3
+
+.. function:: clock_gettime(clk_id)
+
+ Return the time of the specified clock *clk_id*.
+
+ .. versionadded:: 3.3
+
+.. data:: CLOCK_REALTIME
+
+ System-wide real-time clock. Setting this clock requires appropriate
+ privileges.
+
+ .. versionadded:: 3.3
+
+.. data:: CLOCK_MONOTONIC
+
+ Clock that cannot be set and represents monotonic time since some
+ unspecified starting point.
+
+ .. versionadded:: 3.3
+
+.. data:: CLOCK_MONOTONIC_RAW
+
+ Similar to :data:`CLOCK_MONOTONIC`, but provides access to a raw
+ hardware-based time that is not subject to NTP adjustments.
+
+ Availability: Linux 2.6.28 or later.
+
+ .. versionadded:: 3.3
+
+.. data:: CLOCK_PROCESS_CPUTIME_ID
+
+ High-resolution per-process timer from the CPU.
+
+ .. versionadded:: 3.3
+
+.. data:: CLOCK_THREAD_CPUTIME_ID
+
+ Thread-specific CPU-time clock.
+
+ .. versionadded:: 3.3
+
+
.. function:: ctime([secs])
Convert a time expressed in seconds since the epoch to a string representing
@@ -308,7 +319,7 @@ The module defines the following functions and data items:
| ``%y`` | Year without century as a decimal number | |
| | [00,99]. | |
+-----------+------------------------------------------------+-------+
- | ``%Y`` | Year with century as a decimal number. | \(4) |
+ | ``%Y`` | Year with century as a decimal number. | |
| | | |
+-----------+------------------------------------------------+-------+
| ``%Z`` | Time zone name (no characters if no time zone | |
@@ -332,12 +343,6 @@ The module defines the following functions and data items:
When used with the :func:`strptime` function, ``%U`` and ``%W`` are only used in
calculations when the day of the week and the year are specified.
- (4)
- Produces different results depending on the value of
- ``time.accept2dyear`` variable. See :ref:`Year 2000 (Y2K)
- issues <time-y2kissues>` for details.
-
-
Here is an example, a format for dates compatible with that specified in the
:rfc:`2822` Internet email standard. [#]_ ::
@@ -418,8 +423,7 @@ The module defines the following functions and data items:
+-------+-------------------+---------------------------------+
Note that unlike the C structure, the month value is a range of [1, 12], not
- [0, 11]. A year value will be handled as described under :ref:`Year 2000
- (Y2K) issues <time-y2kissues>` above. A ``-1`` argument as the daylight
+ [0, 11]. A ``-1`` argument as the daylight
savings flag, passed to :func:`mktime` will usually result in the correct
daylight savings state to be filled in.
diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst
index 577d7cc..050d74c 100644
--- a/Doc/library/tokenize.rst
+++ b/Doc/library/tokenize.rst
@@ -15,6 +15,9 @@ implemented in Python. The scanner in this module returns comments as tokens
as well, making it useful for implementing "pretty-printers," including
colorizers for on-screen displays.
+Tokenizing Input
+----------------
+
The primary entry point is a :term:`generator`:
.. function:: tokenize(readline)
@@ -116,6 +119,26 @@ function it uses to do this is available:
.. versionadded:: 3.2
+.. _tokenize-cli:
+
+Command-Line Usage
+------------------
+
+.. versionadded:: 3.3
+
+The :mod:`tokenize` module can be executed as a script from the command line.
+It is as simple as:
+
+.. code-block:: sh
+
+ python -m tokenize [filename.py]
+
+If :file:`filename.py` is specified its contents are tokenized to stdout.
+Otherwise, tokenization is performed on stdin.
+
+Examples
+------------------
+
Example of a script rewriter that transforms float literals into Decimal
objects::
@@ -158,3 +181,37 @@ objects::
result.append((toknum, tokval))
return untokenize(result).decode('utf-8')
+Example of tokenizing from the command line. The script::
+
+ def say_hello():
+ print("Hello, World!")
+
+ say_hello()
+
+will be tokenized to the following output where the first column is the range
+of the line/column coordinates where the token is found, the second column is
+the name of the token, and the final column is the value of the token (if any)
+
+.. code-block:: sh
+
+ $ python -m tokenize hello.py
+ 0,0-0,0: ENCODING 'utf-8'
+ 1,0-1,3: NAME 'def'
+ 1,4-1,13: NAME 'say_hello'
+ 1,13-1,14: OP '('
+ 1,14-1,15: OP ')'
+ 1,15-1,16: OP ':'
+ 1,16-1,17: NEWLINE '\n'
+ 2,0-2,4: INDENT ' '
+ 2,4-2,9: NAME 'print'
+ 2,9-2,10: OP '('
+ 2,10-2,25: STRING '"Hello, World!"'
+ 2,25-2,26: OP ')'
+ 2,26-2,27: NEWLINE '\n'
+ 3,0-3,1: NL '\n'
+ 4,0-4,0: DEDENT ''
+ 4,0-4,9: NAME 'say_hello'
+ 4,9-4,10: OP '('
+ 4,10-4,11: OP ')'
+ 4,11-4,12: NEWLINE '\n'
+ 5,0-5,0: ENDMARKER ''
diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst
index bcb3da3..d0783a2 100644
--- a/Doc/library/unicodedata.rst
+++ b/Doc/library/unicodedata.rst
@@ -29,6 +29,9 @@ following functions:
Look up character by name. If a character with the given name is found, return
the corresponding character. If not found, :exc:`KeyError` is raised.
+ .. versionchanged:: 3.3
+ Support for name aliases [#]_ and named sequences [#]_ has been added.
+
.. function:: name(chr[, default])
@@ -160,3 +163,9 @@ Examples:
>>> unicodedata.bidirectional('\u0660') # 'A'rabic, 'N'umber
'AN'
+
+.. rubric:: Footnotes
+
+.. [#] http://www.unicode.org/Public/6.0.0/ucd/NameAliases.txt
+
+.. [#] http://www.unicode.org/Public/6.0.0/ucd/NamedSequences.txt
diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst
index 32f66fe..7340588 100644
--- a/Doc/library/unittest.rst
+++ b/Doc/library/unittest.rst
@@ -792,11 +792,14 @@ Test cases
Run the test, collecting the result into the test result object passed as
*result*. If *result* is omitted or ``None``, a temporary result
object is created (by calling the :meth:`defaultTestResult` method) and
- used. The result object is not returned to :meth:`run`'s caller.
+ used. The result object is returned to :meth:`run`'s caller.
The same effect may be had by simply calling the :class:`TestCase`
instance.
+ .. versionchanged:: 3.3
+ Previous versions of ``run`` did not return the result. Neither did
+ calling an instance.
.. method:: skipTest(reason)
@@ -857,10 +860,11 @@ Test cases
| <TestCase.assertNotIsInstance>` | | |
+-----------------------------------------+-----------------------------+---------------+
- All the assert methods (except :meth:`assertRaises`,
- :meth:`assertRaisesRegex`, :meth:`assertWarns`, :meth:`assertWarnsRegex`)
- accept a *msg* argument that, if specified, is used as the error message on
- failure (see also :data:`longMessage`).
+ All the assert methods accept a *msg* argument that, if specified, is used
+ as the error message on failure (see also :data:`longMessage`).
+ Note that the *msg* keyword argument can be passed to :meth:`assertRaises`,
+ :meth:`assertRaisesRegex`, :meth:`assertWarns`, :meth:`assertWarnsRegex`
+ only when they are used as a context manager.
.. method:: assertEqual(first, second, msg=None)
@@ -954,7 +958,7 @@ Test cases
+---------------------------------------------------------+--------------------------------------+------------+
.. method:: assertRaises(exception, callable, *args, **kwds)
- assertRaises(exception)
+ assertRaises(exception, msg=None)
Test that an exception is raised when *callable* is called with any
positional or keyword arguments that are also passed to
@@ -963,12 +967,16 @@ Test cases
To catch any of a group of exceptions, a tuple containing the exception
classes may be passed as *exception*.
- If only the *exception* argument is given, returns a context manager so
- that the code under test can be written inline rather than as a function::
+ If only the *exception* and possibly the *msg* arguments are given,
+ return a context manager so that the code under test can be written
+ inline rather than as a function::
with self.assertRaises(SomeException):
do_something()
+ When used as a context manager, :meth:`assertRaises` accepts the
+ additional keyword argument *msg*.
+
The context manager will store the caught exception object in its
:attr:`exception` attribute. This can be useful if the intention
is to perform additional checks on the exception raised::
@@ -985,9 +993,12 @@ Test cases
.. versionchanged:: 3.2
Added the :attr:`exception` attribute.
+ .. versionchanged:: 3.3
+ Added the *msg* keyword argument when used as a context manager.
+
.. method:: assertRaisesRegex(exception, regex, callable, *args, **kwds)
- assertRaisesRegex(exception, regex)
+ assertRaisesRegex(exception, regex, msg=None)
Like :meth:`assertRaises` but also tests that *regex* matches
on the string representation of the raised exception. *regex* may be
@@ -1004,12 +1015,16 @@ Test cases
.. versionadded:: 3.1
under the name ``assertRaisesRegexp``.
+
.. versionchanged:: 3.2
Renamed to :meth:`assertRaisesRegex`.
+ .. versionchanged:: 3.3
+ Added the *msg* keyword argument when used as a context manager.
+
.. method:: assertWarns(warning, callable, *args, **kwds)
- assertWarns(warning)
+ assertWarns(warning, msg=None)
Test that a warning is triggered when *callable* is called with any
positional or keyword arguments that are also passed to
@@ -1018,12 +1033,16 @@ Test cases
To catch any of a group of warnings, a tuple containing the warning
classes may be passed as *warnings*.
- If only the *warning* argument is given, returns a context manager so
- that the code under test can be written inline rather than as a function::
+ If only the *warning* and possibly the *msg* arguments are given,
+ returns a context manager so that the code under test can be written
+ inline rather than as a function::
with self.assertWarns(SomeWarning):
do_something()
+ When used as a context manager, :meth:`assertRaises` accepts the
+ additional keyword argument *msg*.
+
The context manager will store the caught warning object in its
:attr:`warning` attribute, and the source line which triggered the
warnings in the :attr:`filename` and :attr:`lineno` attributes.
@@ -1041,9 +1060,12 @@ Test cases
.. versionadded:: 3.2
+ .. versionchanged:: 3.3
+ Added the *msg* keyword argument when used as a context manager.
+
.. method:: assertWarnsRegex(warning, regex, callable, *args, **kwds)
- assertWarnsRegex(warning, regex)
+ assertWarnsRegex(warning, regex, msg=None)
Like :meth:`assertWarns` but also tests that *regex* matches on the
message of the triggered warning. *regex* may be a regular expression
@@ -1061,6 +1083,8 @@ Test cases
.. versionadded:: 3.2
+ .. versionchanged:: 3.3
+ Added the *msg* keyword argument when used as a context manager.
There are also other methods used to perform more specific checks, such as:
diff --git a/Doc/library/urllib.error.rst b/Doc/library/urllib.error.rst
index 282329f..e20db27 100644
--- a/Doc/library/urllib.error.rst
+++ b/Doc/library/urllib.error.rst
@@ -8,21 +8,23 @@
The :mod:`urllib.error` module defines the exception classes for exceptions
-raised by :mod:`urllib.request`. The base exception class is :exc:`URLError`,
-which inherits from :exc:`IOError`.
+raised by :mod:`urllib.request`. The base exception class is :exc:`URLError`.
The following exceptions are raised by :mod:`urllib.error` as appropriate:
.. exception:: URLError
The handlers raise this exception (or derived exceptions) when they run into
- a problem. It is a subclass of :exc:`IOError`.
+ a problem. It is a subclass of :exc:`OSError`.
.. attribute:: reason
The reason for this error. It can be a message string or another
- exception instance (:exc:`socket.error` for remote URLs, :exc:`OSError`
- for local URLs).
+ exception instance.
+
+ .. versionchanged:: 3.3
+ :exc:`URLError` has been made a subclass of :exc:`OSError` instead
+ of :exc:`IOError`.
.. exception:: HTTPError
diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst
index 6b2b013..abd9f35 100644
--- a/Doc/library/urllib.request.rst
+++ b/Doc/library/urllib.request.rst
@@ -132,7 +132,7 @@ The :mod:`urllib.request` module defines the following functions:
The following classes are provided:
-.. class:: Request(url, data=None, headers={}, origin_req_host=None, unverifiable=False)
+.. class:: Request(url, data=None, headers={}, origin_req_host=None, unverifiable=False, method=None)
This class is an abstraction of a URL request.
@@ -140,8 +140,8 @@ The following classes are provided:
*data* may be a string specifying additional data to send to the
server, or ``None`` if no such data is needed. Currently HTTP
- requests are the only ones that use *data*; the HTTP request will
- be a POST instead of a GET when the *data* parameter is provided.
+ requests are the only ones that use *data*, in order to choose between
+ ``'GET'`` and ``'POST'`` when *method* is not specified.
*data* should be a buffer in the standard
:mimetype:`application/x-www-form-urlencoded` format. The
:func:`urllib.parse.urlencode` function takes a mapping or sequence
@@ -157,8 +157,8 @@ The following classes are provided:
:mod:`urllib`'s default user agent string is
``"Python-urllib/2.6"`` (on Python 2.6).
- The final two arguments are only of interest for correct handling
- of third-party HTTP cookies:
+ The following two arguments, *origin_req_host* and *unverifiable*,
+ are only of interest for correct handling of third-party HTTP cookies:
*origin_req_host* should be the request-host of the origin
transaction, as defined by :rfc:`2965`. It defaults to
@@ -175,6 +175,13 @@ The following classes are provided:
document, and the user had no option to approve the automatic
fetching of the image, this should be true.
+ *method* should be a string that indicates the HTTP request method that
+ will be used (e.g. ``'HEAD'``). Its value is stored in the
+ :attr:`~Request.method` attribute and is used by :meth:`get_method()`.
+
+ .. versionchanged:: 3.3
+ :attr:`Request.method` argument is added to the Request class.
+
.. class:: OpenerDirector()
@@ -240,10 +247,11 @@ The following classes are provided:
.. class:: HTTPBasicAuthHandler(password_mgr=None)
- Handle authentication with the remote host. *password_mgr*, if given, should be
- something that is compatible with :class:`HTTPPasswordMgr`; refer to section
- :ref:`http-password-mgr` for information on the interface that must be
- supported.
+ Handle authentication with the remote host. *password_mgr*, if given, should
+ be something that is compatible with :class:`HTTPPasswordMgr`; refer to
+ section :ref:`http-password-mgr` for information on the interface that must
+ be supported. HTTPBasicAuthHandler will raise a :exc:`ValueError` when
+ presented with a wrong Authentication scheme.
.. class:: ProxyBasicAuthHandler(password_mgr=None)
@@ -265,10 +273,19 @@ The following classes are provided:
.. class:: HTTPDigestAuthHandler(password_mgr=None)
- Handle authentication with the remote host. *password_mgr*, if given, should be
- something that is compatible with :class:`HTTPPasswordMgr`; refer to section
- :ref:`http-password-mgr` for information on the interface that must be
- supported.
+ Handle authentication with the remote host. *password_mgr*, if given, should
+ be something that is compatible with :class:`HTTPPasswordMgr`; refer to
+ section :ref:`http-password-mgr` for information on the interface that must
+ be supported. When both Digest Authentication Handler and Basic
+ Authentication Handler are both added, Digest Authentication is always tried
+ first. If the Digest Authentication returns a 40x response again, it is sent
+ to Basic Authentication handler to Handle. This Handler method will raise a
+ :exc:`ValueError` when presented with an authentication scheme other than
+ Digest or Basic.
+
+ .. versionchanged:: 3.3
+ Raise :exc:`ValueError` on unsupported Authentication Scheme.
+
.. class:: ProxyDigestAuthHandler(password_mgr=None)
@@ -359,6 +376,15 @@ request.
boolean, indicates whether the request is unverifiable as defined
by RFC 2965.
+.. attribute:: Request.method
+
+ The HTTP request method to use. This value is used by
+ :meth:`~Request.get_method` to override the computed HTTP request
+ method that would otherwise be returned. This attribute is initialized with
+ the value of the *method* argument passed to the constructor.
+
+ .. versionadded:: 3.3
+
.. method:: Request.add_data(data)
Set the :class:`Request` data to *data*. This is ignored by all handlers except
@@ -368,8 +394,13 @@ request.
.. method:: Request.get_method()
- Return a string indicating the HTTP request method. This is only meaningful for
- HTTP requests, and currently always returns ``'GET'`` or ``'POST'``.
+ Return a string indicating the HTTP request method. If
+ :attr:`Request.method` is not ``None``, return its value, otherwise return
+ ``'GET'`` if :attr:`Request.data` is ``None``, or ``'POST'`` if it's not.
+ This is only meaningful for HTTP requests.
+
+ .. versionchanged:: 3.3
+ get_method now looks at the value of :attr:`Request.method`.
.. method:: Request.has_data()
@@ -1162,7 +1193,7 @@ some point in the future.
*key_file* and *cert_file* are supported to provide an SSL key and certificate;
both are needed to support client authentication.
- :class:`URLopener` objects will raise an :exc:`IOError` exception if the server
+ :class:`URLopener` objects will raise an :exc:`OSError` exception if the server
returns an error code.
.. method:: open(fullurl, data=None)
diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst
index 8af19a2..8387f5a 100644
--- a/Doc/library/warnings.rst
+++ b/Doc/library/warnings.rst
@@ -339,8 +339,7 @@ Available Functions
Write a warning to a file. The default implementation calls
``formatwarning(message, category, filename, lineno, line)`` and writes the
resulting string to *file*, which defaults to ``sys.stderr``. You may replace
- this function with an alternative implementation by assigning to
- ``warnings.showwarning``.
+ this function with any callable by assigning to ``warnings.showwarning``.
*line* is a line of source code to be included in the warning
message; if *line* is not supplied, :func:`showwarning` will
try to read the line specified by *filename* and *lineno*.
diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst
index 5cf30ee..376752e 100644
--- a/Doc/library/winreg.rst
+++ b/Doc/library/winreg.rst
@@ -38,7 +38,11 @@ This module offers the following functions:
*key* is the predefined handle to connect to.
The return value is the handle of the opened key. If the function fails, a
- :exc:`WindowsError` exception is raised.
+ :exc:`OSError` exception is raised.
+
+ .. versionchanged:: 3.3
+ This function used to raise a :exc:`WindowsError`, which is now an
+ alias of :exc:`OSError`.
.. function:: CreateKey(key, sub_key)
@@ -57,7 +61,11 @@ This module offers the following functions:
If the key already exists, this function opens the existing key.
The return value is the handle of the opened key. If the function fails, a
- :exc:`WindowsError` exception is raised.
+ :exc:`OSError` exception is raised.
+
+ .. versionchanged:: 3.3
+ This function used to raise a :exc:`WindowsError`, which is now an
+ alias of :exc:`OSError`.
.. function:: CreateKeyEx(key, sub_key, reserved=0, access=KEY_ALL_ACCESS)
@@ -82,10 +90,14 @@ This module offers the following functions:
If the key already exists, this function opens the existing key.
The return value is the handle of the opened key. If the function fails, a
- :exc:`WindowsError` exception is raised.
+ :exc:`OSError` exception is raised.
.. versionadded:: 3.2
+ .. versionchanged:: 3.3
+ This function used to raise a :exc:`WindowsError`, which is now an
+ alias of :exc:`OSError`.
+
.. function:: DeleteKey(key, sub_key)
@@ -100,7 +112,11 @@ This module offers the following functions:
*This method can not delete keys with subkeys.*
If the method succeeds, the entire key, including all of its values, is removed.
- If the method fails, a :exc:`WindowsError` exception is raised.
+ If the method fails, a :exc:`OSError` exception is raised.
+
+ .. versionchanged:: 3.3
+ This function used to raise a :exc:`WindowsError`, which is now an
+ alias of :exc:`OSError`.
.. function:: DeleteKeyEx(key, sub_key, access=KEY_ALL_ACCESS, reserved=0)
@@ -129,12 +145,16 @@ This module offers the following functions:
*This method can not delete keys with subkeys.*
If the method succeeds, the entire key, including all of its values, is
- removed. If the method fails, a :exc:`WindowsError` exception is raised.
+ removed. If the method fails, a :exc:`OSError` exception is raised.
On unsupported Windows versions, :exc:`NotImplementedError` is raised.
.. versionadded:: 3.2
+ .. versionchanged:: 3.3
+ This function used to raise a :exc:`WindowsError`, which is now an
+ alias of :exc:`OSError`.
+
.. function:: DeleteValue(key, value)
@@ -156,9 +176,13 @@ This module offers the following functions:
*index* is an integer that identifies the index of the key to retrieve.
The function retrieves the name of one subkey each time it is called. It is
- typically called repeatedly until a :exc:`WindowsError` exception is
+ typically called repeatedly until a :exc:`OSError` exception is
raised, indicating, no more values are available.
+ .. versionchanged:: 3.3
+ This function used to raise a :exc:`WindowsError`, which is now an
+ alias of :exc:`OSError`.
+
.. function:: EnumValue(key, index)
@@ -170,7 +194,7 @@ This module offers the following functions:
*index* is an integer that identifies the index of the value to retrieve.
The function retrieves the name of one subkey each time it is called. It is
- typically called repeatedly, until a :exc:`WindowsError` exception is
+ typically called repeatedly, until a :exc:`OSError` exception is
raised, indicating no more values.
The result is a tuple of 3 items:
@@ -189,6 +213,10 @@ This module offers the following functions:
| | :meth:`SetValueEx`) |
+-------+--------------------------------------------+
+ .. versionchanged:: 3.3
+ This function used to raise a :exc:`WindowsError`, which is now an
+ alias of :exc:`OSError`.
+
.. function:: ExpandEnvironmentStrings(str)
@@ -260,10 +288,14 @@ This module offers the following functions:
The result is a new handle to the specified key.
- If the function fails, :exc:`WindowsError` is raised.
+ If the function fails, :exc:`OSError` is raised.
.. versionchanged:: 3.2 Allow the use of named arguments.
+ .. versionchanged:: 3.3
+ This function used to raise a :exc:`WindowsError`, which is now an
+ alias of :exc:`OSError`.
+
.. function:: OpenKeyEx()
diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst
index 4f17092..b47c35b 100644
--- a/Doc/library/zipimport.rst
+++ b/Doc/library/zipimport.rst
@@ -85,9 +85,12 @@ zipimporter Objects
.. method:: get_data(pathname)
- Return the data associated with *pathname*. Raise :exc:`IOError` if the
+ Return the data associated with *pathname*. Raise :exc:`OSError` if the
file wasn't found.
+ .. versionchanged:: 3.3
+ :exc:`IOError` used to be raised instead of :exc:`OSError`.
+
.. method:: get_filename(fullname)
diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst
index a7b8343..9f8eb26 100644
--- a/Doc/library/zlib.rst
+++ b/Doc/library/zlib.rst
@@ -19,7 +19,7 @@ consult the zlib manual at http://www.zlib.net/manual.html for authoritative
information.
For reading and writing ``.gz`` files see the :mod:`gzip` module. For
-other archive formats, see the :mod:`bz2`, :mod:`zipfile`, and
+other related file formats, see the :mod:`bz2`, :mod:`zipfile`, and
:mod:`tarfile` modules.
The available exception and functions in this module are:
@@ -122,6 +122,7 @@ The available exception and functions in this module are:
won't fit into memory at once. The *wbits* parameter controls the size of the
window buffer.
+
Compression objects support the following methods:
@@ -152,7 +153,7 @@ Compression objects support the following methods:
compress a set of data that share a common initial prefix.
-Decompression objects support the following methods, and two attributes:
+Decompression objects support the following methods and attributes:
.. attribute:: Decompress.unused_data
@@ -162,13 +163,6 @@ Decompression objects support the following methods, and two attributes:
available. If the whole bytestring turned out to contain compressed data, this is
``b""``, an empty bytes object.
- The only way to determine where a bytestring of compressed data ends is by actually
- decompressing it. This means that when compressed data is contained part of a
- larger file, you can only find the end of it by reading data and feeding it
- followed by some non-empty bytestring into a decompression object's
- :meth:`decompress` method until the :attr:`unused_data` attribute is no longer
- empty.
-
.. attribute:: Decompress.unconsumed_tail
@@ -179,6 +173,17 @@ Decompression objects support the following methods, and two attributes:
:meth:`decompress` method call in order to get correct output.
+.. attribute:: Decompress.eof
+
+ A boolean indicating whether the end of the compressed data stream has been
+ reached.
+
+ This makes it possible to distinguish between a properly-formed compressed
+ stream, and an incomplete or truncated one.
+
+ .. versionadded:: 3.3
+
+
.. method:: Decompress.decompress(data[, max_length])
Decompress *data*, returning a bytes object containing the uncompressed data
@@ -213,6 +218,24 @@ Decompression objects support the following methods, and two attributes:
seeks into the stream at a future point.
+Information about the version of the zlib library in use is available through
+the following constants:
+
+
+.. data:: ZLIB_VERSION
+
+ The version string of the zlib library that was used for building the module.
+ This may be different from the zlib library actually used at runtime, which
+ is available as :const:`ZLIB_RUNTIME_VERSION`.
+
+
+.. data:: ZLIB_RUNTIME_VERSION
+
+ The version string of the zlib library actually loaded by the interpreter.
+
+ .. versionadded:: 3.3
+
+
.. seealso::
Module :mod:`gzip`
diff --git a/Doc/license.rst b/Doc/license.rst
index 8693a0f..5050ff0 100644
--- a/Doc/license.rst
+++ b/Doc/license.rst
@@ -118,6 +118,8 @@ been GPL-compatible; the table below summarizes the various releases.
+----------------+--------------+------------+------------+-----------------+
| 3.2.2 | 3.2.1 | 2011 | PSF | yes |
+----------------+--------------+------------+------------+-----------------+
+| 3.3 | 3.2 | 2012 | PSF | yes |
++----------------+--------------+------------+------------+-----------------+
.. note::
diff --git a/Doc/packaging/builtdist.rst b/Doc/packaging/builtdist.rst
new file mode 100644
index 0000000..1d9a349
--- /dev/null
+++ b/Doc/packaging/builtdist.rst
@@ -0,0 +1,302 @@
+.. _packaging-built-dist:
+
+****************************
+Creating Built Distributions
+****************************
+
+A "built distribution" is what you're probably used to thinking of either as a
+"binary package" or an "installer" (depending on your background). It's not
+necessarily binary, though, because it might contain only Python source code
+and/or byte-code; and we don't call it a package, because that word is already
+spoken for in Python. (And "installer" is a term specific to the world of
+mainstream desktop systems.)
+
+A built distribution is how you make life as easy as possible for installers of
+your module distribution: for users of RPM-based Linux systems, it's a binary
+RPM; for Windows users, it's an executable installer; for Debian-based Linux
+users, it's a Debian package; and so forth. Obviously, no one person will be
+able to create built distributions for every platform under the sun, so the
+Distutils are designed to enable module developers to concentrate on their
+specialty---writing code and creating source distributions---while an
+intermediary species called *packagers* springs up to turn source distributions
+into built distributions for as many platforms as there are packagers.
+
+Of course, the module developer could be his own packager; or the packager could
+be a volunteer "out there" somewhere who has access to a platform which the
+original developer does not; or it could be software periodically grabbing new
+source distributions and turning them into built distributions for as many
+platforms as the software has access to. Regardless of who they are, a packager
+uses the setup script and the :command:`bdist` command family to generate built
+distributions.
+
+As a simple example, if I run the following command in the Distutils source
+tree::
+
+ python setup.py bdist
+
+then the Distutils builds my module distribution (the Distutils itself in this
+case), does a "fake" installation (also in the :file:`build` directory), and
+creates the default type of built distribution for my platform. The default
+format for built distributions is a "dumb" tar file on Unix, and a simple
+executable installer on Windows. (That tar file is considered "dumb" because it
+has to be unpacked in a specific location to work.)
+
+Thus, the above command on a Unix system creates
+:file:`Distutils-1.0.{plat}.tar.gz`; unpacking this tarball from the right place
+installs the Distutils just as though you had downloaded the source distribution
+and run ``python setup.py install``. (The "right place" is either the root of
+the filesystem or Python's :file:`{prefix}` directory, depending on the options
+given to the :command:`bdist_dumb` command; the default is to make dumb
+distributions relative to :file:`{prefix}`.)
+
+Obviously, for pure Python distributions, this isn't any simpler than just
+running ``python setup.py install``\ ---but for non-pure distributions, which
+include extensions that would need to be compiled, it can mean the difference
+between someone being able to use your extensions or not. And creating "smart"
+built distributions, such as an executable installer for
+Windows, is far more convenient for users even if your distribution doesn't
+include any extensions.
+
+The :command:`bdist` command has a :option:`--formats` option, similar to the
+:command:`sdist` command, which you can use to select the types of built
+distribution to generate: for example, ::
+
+ python setup.py bdist --format=zip
+
+would, when run on a Unix system, create :file:`Distutils-1.0.{plat}.zip`\
+---again, this archive would be unpacked from the root directory to install the
+Distutils.
+
+The available formats for built distributions are:
+
++-------------+------------------------------+---------+
+| Format | Description | Notes |
++=============+==============================+=========+
+| ``gztar`` | gzipped tar file | (1),(3) |
+| | (:file:`.tar.gz`) | |
++-------------+------------------------------+---------+
+| ``tar`` | tar file (:file:`.tar`) | \(3) |
++-------------+------------------------------+---------+
+| ``zip`` | zip file (:file:`.zip`) | (2),(4) |
++-------------+------------------------------+---------+
+| ``wininst`` | self-extracting ZIP file for | \(4) |
+| | Windows | |
++-------------+------------------------------+---------+
+| ``msi`` | Microsoft Installer. | |
++-------------+------------------------------+---------+
+
+
+Notes:
+
+(1)
+ default on Unix
+
+(2)
+ default on Windows
+
+(3)
+ requires external utilities: :program:`tar` and possibly one of :program:`gzip`
+ or :program:`bzip2`
+
+(4)
+ requires either external :program:`zip` utility or :mod:`zipfile` module (part
+ of the standard Python library since Python 1.6)
+
+You don't have to use the :command:`bdist` command with the :option:`--formats`
+option; you can also use the command that directly implements the format you're
+interested in. Some of these :command:`bdist` "sub-commands" actually generate
+several similar formats; for instance, the :command:`bdist_dumb` command
+generates all the "dumb" archive formats (``tar``, ``gztar``, and
+``zip``). The :command:`bdist` sub-commands, and the formats generated by
+each, are:
+
++--------------------------+-----------------------+
+| Command | Formats |
++==========================+=======================+
+| :command:`bdist_dumb` | tar, gztar, zip |
++--------------------------+-----------------------+
+| :command:`bdist_wininst` | wininst |
++--------------------------+-----------------------+
+| :command:`bdist_msi` | msi |
++--------------------------+-----------------------+
+
+The following sections give details on the individual :command:`bdist_\*`
+commands.
+
+
+.. _packaging-creating-dumb:
+
+Creating dumb built distributions
+=================================
+
+.. XXX Need to document absolute vs. prefix-relative packages here, but first
+ I have to implement it!
+
+
+.. _packaging-creating-wininst:
+
+Creating Windows Installers
+===========================
+
+Executable installers are the natural format for binary distributions on
+Windows. They display a nice graphical user interface, display some information
+about the module distribution to be installed taken from the metadata in the
+setup script, let the user select a few options, and start or cancel the
+installation.
+
+Since the metadata is taken from the setup script, creating Windows installers
+is usually as easy as running::
+
+ python setup.py bdist_wininst
+
+or the :command:`bdist` command with the :option:`--formats` option::
+
+ python setup.py bdist --formats=wininst
+
+If you have a pure module distribution (only containing pure Python modules and
+packages), the resulting installer will be version independent and have a name
+like :file:`foo-1.0.win32.exe`. These installers can even be created on Unix
+platforms or Mac OS X.
+
+If you have a non-pure distribution, the extensions can only be created on a
+Windows platform, and will be Python version dependent. The installer filename
+will reflect this and now has the form :file:`foo-1.0.win32-py2.0.exe`. You
+have to create a separate installer for every Python version you want to
+support.
+
+The installer will try to compile pure modules into :term:`bytecode` after installation
+on the target system in normal and optimizing mode. If you don't want this to
+happen for some reason, you can run the :command:`bdist_wininst` command with
+the :option:`--no-target-compile` and/or the :option:`--no-target-optimize`
+option.
+
+By default the installer will display the cool "Python Powered" logo when it is
+run, but you can also supply your own 152x261 bitmap which must be a Windows
+:file:`.bmp` file with the :option:`--bitmap` option.
+
+The installer will also display a large title on the desktop background window
+when it is run, which is constructed from the name of your distribution and the
+version number. This can be changed to another text by using the
+:option:`--title` option.
+
+The installer file will be written to the "distribution directory" --- normally
+:file:`dist/`, but customizable with the :option:`--dist-dir` option.
+
+.. _packaging-cross-compile-windows:
+
+Cross-compiling on Windows
+==========================
+
+Starting with Python 2.6, packaging is capable of cross-compiling between
+Windows platforms. In practice, this means that with the correct tools
+installed, you can use a 32bit version of Windows to create 64bit extensions
+and vice-versa.
+
+To build for an alternate platform, specify the :option:`--plat-name` option
+to the build command. Valid values are currently 'win32', 'win-amd64' and
+'win-ia64'. For example, on a 32bit version of Windows, you could execute::
+
+ python setup.py build --plat-name=win-amd64
+
+to build a 64bit version of your extension. The Windows Installers also
+support this option, so the command::
+
+ python setup.py build --plat-name=win-amd64 bdist_wininst
+
+would create a 64bit installation executable on your 32bit version of Windows.
+
+To cross-compile, you must download the Python source code and cross-compile
+Python itself for the platform you are targetting - it is not possible from a
+binary installtion of Python (as the .lib etc file for other platforms are
+not included.) In practice, this means the user of a 32 bit operating
+system will need to use Visual Studio 2008 to open the
+:file:`PCBuild/PCbuild.sln` solution in the Python source tree and build the
+"x64" configuration of the 'pythoncore' project before cross-compiling
+extensions is possible.
+
+Note that by default, Visual Studio 2008 does not install 64bit compilers or
+tools. You may need to reexecute the Visual Studio setup process and select
+these tools (using Control Panel->[Add/Remove] Programs is a convenient way to
+check or modify your existing install.)
+
+.. _packaging-postinstallation-script:
+
+The Postinstallation script
+---------------------------
+
+Starting with Python 2.3, a postinstallation script can be specified with the
+:option:`--install-script` option. The basename of the script must be
+specified, and the script filename must also be listed in the scripts argument
+to the setup function.
+
+This script will be run at installation time on the target system after all the
+files have been copied, with ``argv[1]`` set to :option:`-install`, and again at
+uninstallation time before the files are removed with ``argv[1]`` set to
+:option:`-remove`.
+
+The installation script runs embedded in the windows installer, every output
+(``sys.stdout``, ``sys.stderr``) is redirected into a buffer and will be
+displayed in the GUI after the script has finished.
+
+Some functions especially useful in this context are available as additional
+built-in functions in the installation script.
+
+.. currentmodule:: bdist_wininst-postinst-script
+
+.. function:: directory_created(path)
+ file_created(path)
+
+ These functions should be called when a directory or file is created by the
+ postinstall script at installation time. It will register *path* with the
+ uninstaller, so that it will be removed when the distribution is uninstalled.
+ To be safe, directories are only removed if they are empty.
+
+
+.. function:: get_special_folder_path(csidl_string)
+
+ This function can be used to retrieve special folder locations on Windows like
+ the Start Menu or the Desktop. It returns the full path to the folder.
+ *csidl_string* must be one of the following strings::
+
+ "CSIDL_APPDATA"
+
+ "CSIDL_COMMON_STARTMENU"
+ "CSIDL_STARTMENU"
+
+ "CSIDL_COMMON_DESKTOPDIRECTORY"
+ "CSIDL_DESKTOPDIRECTORY"
+
+ "CSIDL_COMMON_STARTUP"
+ "CSIDL_STARTUP"
+
+ "CSIDL_COMMON_PROGRAMS"
+ "CSIDL_PROGRAMS"
+
+ "CSIDL_FONTS"
+
+ If the folder cannot be retrieved, :exc:`OSError` is raised.
+
+ Which folders are available depends on the exact Windows version, and probably
+ also the configuration. For details refer to Microsoft's documentation of the
+ :c:func:`SHGetSpecialFolderPath` function.
+
+
+.. function:: create_shortcut(target, description, filename[, arguments[, workdir[, iconpath[, iconindex]]]])
+
+ This function creates a shortcut. *target* is the path to the program to be
+ started by the shortcut. *description* is the description of the shortcut.
+ *filename* is the title of the shortcut that the user will see. *arguments*
+ specifies the command-line arguments, if any. *workdir* is the working directory
+ for the program. *iconpath* is the file containing the icon for the shortcut,
+ and *iconindex* is the index of the icon in the file *iconpath*. Again, for
+ details consult the Microsoft documentation for the :class:`IShellLink`
+ interface.
+
+
+Vista User Access Control (UAC)
+===============================
+
+Starting with Python 2.6, bdist_wininst supports a :option:`--user-access-control`
+option. The default is 'none' (meaning no UAC handling is done), and other
+valid values are 'auto' (meaning prompt for UAC elevation if Python was
+installed for all users) and 'force' (meaning always prompt for elevation).
diff --git a/Doc/packaging/commandhooks.rst b/Doc/packaging/commandhooks.rst
new file mode 100644
index 0000000..b261d00
--- /dev/null
+++ b/Doc/packaging/commandhooks.rst
@@ -0,0 +1,47 @@
+.. TODO integrate this in commandref and configfile
+
+.. _packaging-command-hooks:
+
+=============
+Command hooks
+=============
+
+Packaging provides a way of extending its commands by the use of pre- and
+post-command hooks. Hooks are Python functions (or any callable object) that
+take a command object as argument. They're specified in :ref:`config files
+<packaging-config-filenames>` using their fully qualified names. After a
+command is finalized (its options are processed), the pre-command hooks are
+executed, then the command itself is run, and finally the post-command hooks are
+executed.
+
+See also global setup hooks in :ref:`setupcfg-spec`.
+
+
+.. _packaging-finding-hooks:
+
+Finding hooks
+=============
+
+As a hook is configured with a Python dotted name, it must either be defined in
+a module installed on the system, or in a module present in the project
+directory, where the :file:`setup.cfg` file lives::
+
+ # file: _setuphooks.py
+
+ def hook(install_cmd):
+ metadata = install_cmd.dist.metadata
+ print('Hooked while installing %r %s!' % (metadata['Name'],
+ metadata['Version']))
+
+Then you need to configure it in :file:`setup.cfg`::
+
+ [install_dist]
+ pre-hook.a = _setuphooks.hook
+
+Packaging will add the project directory to :data:`sys.path` and find the
+``_setuphooks`` module.
+
+Hooks defined in different config files (system-wide, user-wide and
+project-wide) do not override each other as long as they are specified with
+different aliases (additional names after the dot). The alias in the example
+above is ``a``.
diff --git a/Doc/packaging/commandref.rst b/Doc/packaging/commandref.rst
new file mode 100644
index 0000000..2165b56
--- /dev/null
+++ b/Doc/packaging/commandref.rst
@@ -0,0 +1,374 @@
+.. _packaging-command-reference:
+
+*****************
+Command Reference
+*****************
+
+This reference briefly documents all standard Packaging commands and some of
+their options.
+
+.. FIXME does not work: Use pysetup run --help-commands to list all
+ standard and extra commands availavble on your system, with their
+ description. Use pysetup run <command> --help to get help about the options
+ of one command.
+
+.. XXX sections from this document should be merged with other docs (e.g. check
+ and upload with uploading.rst, install_* with install/install.rst, etc.);
+ there is no value in partially duplicating information. this file could
+ however serve as an index, i.e. just a list of all commands with links to
+ every section that describes options or usage
+
+
+Preparing distributions
+=======================
+
+:command:`check`
+----------------
+
+Perform some tests on the metadata of a distribution.
+
+For example, it verifies that all required metadata fields are provided in the
+:file:`setup.cfg` file.
+
+.. TODO document reST checks
+
+
+:command:`test`
+---------------
+
+Run a test suite.
+
+When doing test-driven development, or running automated builds that need
+testing before they are installed for downloading or use, it's often useful to
+be able to run a project's unit tests without actually installing the project
+anywhere. The :command:`test` command runs project's unit tests without
+actually installing it, by temporarily putting the project's source on
+:data:`sys.path`, after first running :command:`build_ext -i` to ensure that any
+C extensions are built.
+
+You can use this command in one of two ways: either by specifying a
+unittest-compatible test suite for your project (or any callable that returns
+it) or by passing a test runner function that will run your tests and display
+results in the console. Both options take a Python dotted name in the form
+``package.module.callable`` to specify the object to use.
+
+If none of these options are specified, Packaging will try to perform test
+discovery using either unittest (for Python 3.2 and higher) or unittest2 (for
+older versions, if installed).
+
+.. this is a pseudo-command name used to disambiguate the options in indexes and
+ links
+.. program:: packaging test
+
+.. cmdoption:: --suite=NAME, -s NAME
+
+ Specify the test suite (or module, class, or method) to be run. The default
+ for this option can be set by in the project's :file:`setup.cfg` file:
+
+ .. code-block:: cfg
+
+ [test]
+ suite = mypackage.tests.get_all_tests
+
+.. cmdoption:: --runner=NAME, -r NAME
+
+ Specify the test runner to be called.
+
+
+:command:`config`
+-----------------
+
+Perform distribution configuration.
+
+
+The build step
+==============
+
+This step is mainly useful to compile C/C++ libraries or extension modules. The
+build commands can be run manually to check for syntax errors or packaging
+issues (for example if the addition of a new source file was forgotten in the
+:file:`setup.cfg` file), and is also run automatically by commands which need
+it. Packaging checks the mtime of source and built files to avoid re-building
+if it's not necessary.
+
+
+:command:`build`
+----------------
+
+Build all files of a distribution, delegating to the other :command:`build_*`
+commands to do the work.
+
+
+:command:`build_clib`
+---------------------
+
+Build C libraries.
+
+
+:command:`build_ext`
+--------------------
+
+Build C/C++ extension modules.
+
+
+:command:`build_py`
+-------------------
+
+Build the Python modules (just copy them to the build directory) and
+:term:`byte-compile <bytecode>` them to :file:`.pyc` and/or :file:`.pyo` files.
+
+The byte compilation is controlled by two sets of options:
+
+- ``--compile`` and ``--no-compile`` are used to control the creation of
+ :file:`.pyc` files; the default is ``--no-compile``.
+
+- ``--optimize N`` (or ``-ON``) is used to control the creation of :file:`.pyo`
+ files: ``-O1`` turns on basic optimizations, ``-O2`` also discards docstrings,
+ ``-O0`` does not create :file:`.pyo` files; the default is ``-O0``.
+
+You can mix and match these options: for example, ``--no-compile --optimize 2``
+will create :file:`.pyo` files but no :file:`.pyc` files.
+
+.. XXX these option roles do not work
+
+Calling Python with :option:`-O` or :option:`-B` does not control the creation
+of bytecode files, only the options described above do.
+
+
+:command:`build_scripts`
+------------------------
+Build the scripts (just copy them to the build directory and adjust their
+shebang if they're Python scripts).
+
+
+:command:`clean`
+----------------
+
+Clean the build tree of the release.
+
+.. program:: packaging clean
+
+.. cmdoption:: --all, -a
+
+ Remove build directories for modules, scripts, etc., not only temporary build
+ by-products.
+
+
+Creating source and built distributions
+=======================================
+
+:command:`sdist`
+----------------
+
+Build a source distribution for a release.
+
+It is recommended that you always build and upload a source distribution. Users
+of OSes with easy access to compilers and users of advanced packaging tools will
+prefer to compile from source rather than using pre-built distributions. For
+Windows users, providing a binary installer is also recommended practice.
+
+
+:command:`bdist`
+----------------
+
+Build a binary distribution for a release.
+
+This command will call other :command:`bdist_*` commands to create one or more
+distributions depending on the options given. The default is to create a
+.tar.gz archive on Unix and a zip archive on Windows or OS/2.
+
+.. program:: packaging bdist
+
+.. cmdoption:: --formats
+
+ Binary formats to build (comma-separated list).
+
+.. cmdoption:: --show-formats
+
+ Dump list of available formats.
+
+
+:command:`bdist_dumb`
+---------------------
+
+Build a "dumb" installer, a simple archive of files that could be unpacked under
+``$prefix`` or ``$exec_prefix``.
+
+
+:command:`bdist_wininst`
+------------------------
+
+Build a Windows installer.
+
+
+:command:`bdist_msi`
+--------------------
+
+Build a `Microsoft Installer`_ (.msi) file.
+
+.. _Microsoft Installer: http://msdn.microsoft.com/en-us/library/cc185688(VS.85).aspx
+
+In most cases, the :command:`bdist_msi` installer is a better choice than the
+:command:`bdist_wininst` installer, because it provides better support for Win64
+platforms, allows administrators to perform non-interactive installations, and
+allows installation through group policies.
+
+
+Publishing distributions
+========================
+
+:command:`register`
+-------------------
+
+This command registers the current release with the Python Package Index. This
+is described in more detail in :PEP:`301`.
+
+.. TODO explain user and project registration with the web UI
+
+
+:command:`upload`
+-----------------
+
+Upload source and/or binary distributions to PyPI.
+
+The distributions have to be built on the same command line as the
+:command:`upload` command; see :ref:`packaging-package-upload` for more info.
+
+.. program:: packaging upload
+
+.. cmdoption:: --sign, -s
+
+ Sign each uploaded file using GPG (GNU Privacy Guard). The ``gpg`` program
+ must be available for execution on the system ``PATH``.
+
+.. cmdoption:: --identity=NAME, -i NAME
+
+ Specify the identity or key name for GPG to use when signing. The value of
+ this option will be passed through the ``--local-user`` option of the
+ ``gpg`` program.
+
+.. cmdoption:: --show-response
+
+ Display the full response text from server; this is useful for debugging
+ PyPI problems.
+
+.. cmdoption:: --repository=URL, -r URL
+
+ The URL of the repository to upload to. Defaults to
+ http://pypi.python.org/pypi (i.e., the main PyPI installation).
+
+.. cmdoption:: --upload-docs
+
+ Also run :command:`upload_docs`. Mainly useful as a default value in
+ :file:`setup.cfg` (on the command line, it's shorter to just type both
+ commands).
+
+
+:command:`upload_docs`
+----------------------
+
+Upload HTML documentation to PyPI.
+
+PyPI now supports publishing project documentation at a URI of the form
+``http://packages.python.org/<project>``. :command:`upload_docs` will create
+the necessary zip file out of a documentation directory and will post to the
+repository.
+
+Note that to upload the documentation of a project, the corresponding version
+must already be registered with PyPI, using the :command:`register` command ---
+just like with :command:`upload`.
+
+Assuming there is an ``Example`` project with documentation in the subdirectory
+:file:`docs`, for example::
+
+ Example/
+ example.py
+ setup.cfg
+ docs/
+ build/
+ html/
+ index.html
+ tips_tricks.html
+ conf.py
+ index.txt
+ tips_tricks.txt
+
+You can simply specify the directory with the HTML files in your
+:file:`setup.cfg` file:
+
+.. code-block:: cfg
+
+ [upload_docs]
+ upload-dir = docs/build/html
+
+
+.. program:: packaging upload_docs
+
+.. cmdoption:: --upload-dir
+
+ The directory to be uploaded to the repository. By default documentation
+ is searched for in ``docs`` (or ``doc``) directory in project root.
+
+.. cmdoption:: --show-response
+
+ Display the full response text from server; this is useful for debugging
+ PyPI problems.
+
+.. cmdoption:: --repository=URL, -r URL
+
+ The URL of the repository to upload to. Defaults to
+ http://pypi.python.org/pypi (i.e., the main PyPI installation).
+
+
+The install step
+================
+
+These commands are used by end-users of a project using :program:`pysetup` or
+another compatible installer. Each command will run the corresponding
+:command:`build_*` command and then move the built files to their destination on
+the target system.
+
+
+:command:`install_dist`
+-----------------------
+
+Install a distribution, delegating to the other :command:`install_*` commands to
+do the work. See :ref:`packaging-how-install-works` for complete usage
+instructions.
+
+
+:command:`install_data`
+-----------------------
+
+Install data files.
+
+
+:command:`install_distinfo`
+---------------------------
+
+Install files recording details of the installation as specified in :PEP:`376`.
+
+
+:command:`install_headers`
+--------------------------
+
+Install C/C++ header files.
+
+
+:command:`install_lib`
+----------------------
+
+Install all modules (extensions and pure Python).
+
+.. XXX what about C libraries created with build_clib?
+
+Similarly to ``build_py``, there are options to control the compilation of
+Python code to :term:`bytecode` files (see above). By default, :file:`.pyc`
+files will be created (``--compile``) and :file:`.pyo` files will not
+(``--optimize 0``).
+
+
+:command:`install_scripts`
+--------------------------
+
+Install scripts.
diff --git a/Doc/packaging/configfile.rst b/Doc/packaging/configfile.rst
new file mode 100644
index 0000000..825b5cb
--- /dev/null
+++ b/Doc/packaging/configfile.rst
@@ -0,0 +1,125 @@
+.. _packaging-setup-config:
+
+************************************
+Writing the Setup Configuration File
+************************************
+
+Often, it's not possible to write down everything needed to build a distribution
+*a priori*: you may need to get some information from the user, or from the
+user's system, in order to proceed. As long as that information is fairly
+simple---a list of directories to search for C header files or libraries, for
+example---then providing a configuration file, :file:`setup.cfg`, for users to
+edit is a cheap and easy way to solicit it. Configuration files also let you
+provide default values for any command option, which the installer can then
+override either on the command line or by editing the config file.
+
+The setup configuration file is a useful middle-ground between the setup script
+---which, ideally, would be opaque to installers [#]_---and the command line to
+the setup script, which is outside of your control and entirely up to the
+installer. In fact, :file:`setup.cfg` (and any other Distutils configuration
+files present on the target system) are processed after the contents of the
+setup script, but before the command line. This has several useful
+consequences:
+
+.. If you have more advanced needs, such as determining which extensions to
+ build based on what capabilities are present on the target system, then you
+ need the Distutils auto-configuration facility. This started to appear in
+ Distutils 0.9 but, as of this writing, isn't mature or stable enough yet
+ for real-world use.
+
+* installers can override some of what you put in :file:`setup.py` by editing
+ :file:`setup.cfg`
+
+* you can provide non-standard defaults for options that are not easily set in
+ :file:`setup.py`
+
+* installers can override anything in :file:`setup.cfg` using the command-line
+ options to :file:`setup.py`
+
+The basic syntax of the configuration file is simple::
+
+ [command]
+ option = value
+ ...
+
+where *command* is one of the Distutils commands (e.g. :command:`build_py`,
+:command:`install_dist`), and *option* is one of the options that command supports.
+Any number of options can be supplied for each command, and any number of
+command sections can be included in the file. Blank lines are ignored, as are
+comments, which run from a ``'#'`` character until the end of the line. Long
+option values can be split across multiple lines simply by indenting the
+continuation lines.
+
+You can find out the list of options supported by a particular command with the
+universal :option:`--help` option, e.g. ::
+
+ > python setup.py --help build_ext
+ [...]
+ Options for 'build_ext' command:
+ --build-lib (-b) directory for compiled extension modules
+ --build-temp (-t) directory for temporary files (build by-products)
+ --inplace (-i) ignore build-lib and put compiled extensions into the
+ source directory alongside your pure Python modules
+ --include-dirs (-I) list of directories to search for header files
+ --define (-D) C preprocessor macros to define
+ --undef (-U) C preprocessor macros to undefine
+ --swig-opts list of SWIG command-line options
+ [...]
+
+.. XXX do we want to support ``setup.py --help metadata``?
+
+Note that an option spelled :option:`--foo-bar` on the command line is spelled
+:option:`foo_bar` in configuration files.
+
+For example, say you want your extensions to be built "in-place"---that is, you
+have an extension :mod:`pkg.ext`, and you want the compiled extension file
+(:file:`ext.so` on Unix, say) to be put in the same source directory as your
+pure Python modules :mod:`pkg.mod1` and :mod:`pkg.mod2`. You can always use the
+:option:`--inplace` option on the command line to ensure this::
+
+ python setup.py build_ext --inplace
+
+But this requires that you always specify the :command:`build_ext` command
+explicitly, and remember to provide :option:`--inplace`. An easier way is to
+"set and forget" this option, by encoding it in :file:`setup.cfg`, the
+configuration file for this distribution::
+
+ [build_ext]
+ inplace = 1
+
+This will affect all builds of this module distribution, whether or not you
+explicitly specify :command:`build_ext`. If you include :file:`setup.cfg` in
+your source distribution, it will also affect end-user builds---which is
+probably a bad idea for this option, since always building extensions in-place
+would break installation of the module distribution. In certain peculiar cases,
+though, modules are built right in their installation directory, so this is
+conceivably a useful ability. (Distributing extensions that expect to be built
+in their installation directory is almost always a bad idea, though.)
+
+Another example: certain commands take options that vary from project to
+project but not depending on the installation system, for example,
+:command:`test` needs to know where your test suite is located and what test
+runner to use; likewise, :command:`upload_docs` can find HTML documentation in
+a :file:`doc` or :file:`docs` directory, but needs an option to find files in
+:file:`docs/build/html`. Instead of having to type out these options each
+time you want to run the command, you can put them in the project's
+:file:`setup.cfg`::
+
+ [test]
+ suite = packaging.tests
+
+ [upload_docs]
+ upload-dir = docs/build/html
+
+
+.. seealso::
+
+ :ref:`packaging-config-syntax` in "Installing Python Projects"
+ More information on the configuration files is available in the manual for
+ system administrators.
+
+
+.. rubric:: Footnotes
+
+.. [#] This ideal probably won't be achieved until auto-configuration is fully
+ supported by the Distutils.
diff --git a/Doc/packaging/examples.rst b/Doc/packaging/examples.rst
new file mode 100644
index 0000000..594ade0
--- /dev/null
+++ b/Doc/packaging/examples.rst
@@ -0,0 +1,334 @@
+.. _packaging-examples:
+
+********
+Examples
+********
+
+This chapter provides a number of basic examples to help get started with
+Packaging.
+
+
+.. _packaging-pure-mod:
+
+Pure Python distribution (by module)
+====================================
+
+If you're just distributing a couple of modules, especially if they don't live
+in a particular package, you can specify them individually using the
+:option:`py_modules` option in the setup script.
+
+In the simplest case, you'll have two files to worry about: a setup script and
+the single module you're distributing, :file:`foo.py` in this example::
+
+ <root>/
+ setup.py
+ foo.py
+
+(In all diagrams in this section, *<root>* will refer to the distribution root
+directory.) A minimal setup script to describe this situation would be::
+
+ from packaging.core import setup
+ setup(name='foo',
+ version='1.0',
+ py_modules=['foo'])
+
+Note that the name of the distribution is specified independently with the
+:option:`name` option, and there's no rule that says it has to be the same as
+the name of the sole module in the distribution (although that's probably a good
+convention to follow). However, the distribution name is used to generate
+filenames, so you should stick to letters, digits, underscores, and hyphens.
+
+Since :option:`py_modules` is a list, you can of course specify multiple
+modules, e.g. if you're distributing modules :mod:`foo` and :mod:`bar`, your
+setup might look like this::
+
+ <root>/
+ setup.py
+ foo.py
+ bar.py
+
+and the setup script might be ::
+
+ from packaging.core import setup
+ setup(name='foobar',
+ version='1.0',
+ py_modules=['foo', 'bar'])
+
+You can put module source files into another directory, but if you have enough
+modules to do that, it's probably easier to specify modules by package rather
+than listing them individually.
+
+
+.. _packaging-pure-pkg:
+
+Pure Python distribution (by package)
+=====================================
+
+If you have more than a couple of modules to distribute, especially if they are
+in multiple packages, it's probably easier to specify whole packages rather than
+individual modules. This works even if your modules are not in a package; you
+can just tell the Distutils to process modules from the root package, and that
+works the same as any other package (except that you don't have to have an
+:file:`__init__.py` file).
+
+The setup script from the last example could also be written as ::
+
+ from packaging.core import setup
+ setup(name='foobar',
+ version='1.0',
+ packages=[''])
+
+(The empty string stands for the root package.)
+
+If those two files are moved into a subdirectory, but remain in the root
+package, e.g.::
+
+ <root>/
+ setup.py
+ src/
+ foo.py
+ bar.py
+
+then you would still specify the root package, but you have to tell the
+Distutils where source files in the root package live::
+
+ from packaging.core import setup
+ setup(name='foobar',
+ version='1.0',
+ package_dir={'': 'src'},
+ packages=[''])
+
+More typically, though, you will want to distribute multiple modules in the same
+package (or in sub-packages). For example, if the :mod:`foo` and :mod:`bar`
+modules belong in package :mod:`foobar`, one way to lay out your source tree is
+
+::
+
+ <root>/
+ setup.py
+ foobar/
+ __init__.py
+ foo.py
+ bar.py
+
+This is in fact the default layout expected by the Distutils, and the one that
+requires the least work to describe in your setup script::
+
+ from packaging.core import setup
+ setup(name='foobar',
+ version='1.0',
+ packages=['foobar'])
+
+If you want to put modules in directories not named for their package, then you
+need to use the :option:`package_dir` option again. For example, if the
+:file:`src` directory holds modules in the :mod:`foobar` package::
+
+ <root>/
+ setup.py
+ src/
+ __init__.py
+ foo.py
+ bar.py
+
+an appropriate setup script would be ::
+
+ from packaging.core import setup
+ setup(name='foobar',
+ version='1.0',
+ package_dir={'foobar': 'src'},
+ packages=['foobar'])
+
+Or, you might put modules from your main package right in the distribution
+root::
+
+ <root>/
+ setup.py
+ __init__.py
+ foo.py
+ bar.py
+
+in which case your setup script would be ::
+
+ from packaging.core import setup
+ setup(name='foobar',
+ version='1.0',
+ package_dir={'foobar': ''},
+ packages=['foobar'])
+
+(The empty string also stands for the current directory.)
+
+If you have sub-packages, they must be explicitly listed in :option:`packages`,
+but any entries in :option:`package_dir` automatically extend to sub-packages.
+(In other words, the Distutils does *not* scan your source tree, trying to
+figure out which directories correspond to Python packages by looking for
+:file:`__init__.py` files.) Thus, if the default layout grows a sub-package::
+
+ <root>/
+ setup.py
+ foobar/
+ __init__.py
+ foo.py
+ bar.py
+ subfoo/
+ __init__.py
+ blah.py
+
+then the corresponding setup script would be ::
+
+ from packaging.core import setup
+ setup(name='foobar',
+ version='1.0',
+ packages=['foobar', 'foobar.subfoo'])
+
+(Again, the empty string in :option:`package_dir` stands for the current
+directory.)
+
+
+.. _packaging-single-ext:
+
+Single extension module
+=======================
+
+Extension modules are specified using the :option:`ext_modules` option.
+:option:`package_dir` has no effect on where extension source files are found;
+it only affects the source for pure Python modules. The simplest case, a
+single extension module in a single C source file, is::
+
+ <root>/
+ setup.py
+ foo.c
+
+If the :mod:`foo` extension belongs in the root package, the setup script for
+this could be ::
+
+ from packaging.core import setup, Extension
+ setup(name='foobar',
+ version='1.0',
+ ext_modules=[Extension('foo', ['foo.c'])])
+
+If the extension actually belongs in a package, say :mod:`foopkg`, then
+
+With exactly the same source tree layout, this extension can be put in the
+:mod:`foopkg` package simply by changing the name of the extension::
+
+ from packaging.core import setup, Extension
+ setup(name='foobar',
+ version='1.0',
+ packages=['foopkg'],
+ ext_modules=[Extension('foopkg.foo', ['foo.c'])])
+
+
+Checking metadata
+=================
+
+The ``check`` command allows you to verify if your project's metadata
+meets the minimum requirements to build a distribution.
+
+To run it, just call it using your :file:`setup.py` script. If something is
+missing, ``check`` will display a warning.
+
+Let's take an example with a simple script::
+
+ from packaging.core import setup
+
+ setup(name='foobar')
+
+.. TODO configure logging StreamHandler to match this output
+
+Running the ``check`` command will display some warnings::
+
+ $ python setup.py check
+ running check
+ warning: check: missing required metadata: version, home_page
+ warning: check: missing metadata: either (author and author_email) or
+ (maintainer and maintainer_email) must be supplied
+
+
+If you use the reStructuredText syntax in the ``long_description`` field and
+`Docutils <http://docutils.sourceforge.net/>`_ is installed you can check if
+the syntax is fine with the ``check`` command, using the ``restructuredtext``
+option.
+
+For example, if the :file:`setup.py` script is changed like this::
+
+ from packaging.core import setup
+
+ desc = """\
+ Welcome to foobar!
+ ===============
+
+ This is the description of the ``foobar`` project.
+ """
+
+ setup(name='foobar',
+ version='1.0',
+ author=u'Tarek Ziadé',
+ author_email='tarek@ziade.org',
+ summary='Foobar utilities'
+ description=desc,
+ home_page='http://example.com')
+
+Where the long description is broken, ``check`` will be able to detect it
+by using the :mod:`docutils` parser::
+
+ $ python setup.py check --restructuredtext
+ running check
+ warning: check: Title underline too short. (line 2)
+ warning: check: Could not finish the parsing.
+
+
+.. _packaging-reading-metadata:
+
+Reading the metadata
+====================
+
+The :func:`packaging.core.setup` function provides a command-line interface
+that allows you to query the metadata fields of a project through the
+:file:`setup.py` script of a given project::
+
+ $ python setup.py --name
+ foobar
+
+This call reads the ``name`` metadata by running the
+:func:`packaging.core.setup` function. When a source or binary
+distribution is created with Distutils, the metadata fields are written
+in a static file called :file:`PKG-INFO`. When a Distutils-based project is
+installed in Python, the :file:`PKG-INFO` file is copied alongside the modules
+and packages of the distribution under :file:`NAME-VERSION-pyX.X.egg-info`,
+where ``NAME`` is the name of the project, ``VERSION`` its version as defined
+in the Metadata, and ``pyX.X`` the major and minor version of Python like
+``2.7`` or ``3.2``.
+
+You can read back this static file, by using the
+:class:`packaging.dist.Metadata` class and its
+:func:`read_pkg_file` method::
+
+ >>> from packaging.metadata import Metadata
+ >>> metadata = Metadata()
+ >>> metadata.read_pkg_file(open('distribute-0.6.8-py2.7.egg-info'))
+ >>> metadata.name
+ 'distribute'
+ >>> metadata.version
+ '0.6.8'
+ >>> metadata.description
+ 'Easily download, build, install, upgrade, and uninstall Python packages'
+
+Notice that the class can also be instantiated with a metadata file path to
+loads its values::
+
+ >>> pkg_info_path = 'distribute-0.6.8-py2.7.egg-info'
+ >>> Metadata(pkg_info_path).name
+ 'distribute'
+
+
+.. XXX These comments have been here for at least ten years. Write the
+ sections or delete the comments (we can maybe ask Greg Ward about
+ the planned contents). (Unindent to make them section titles)
+
+ .. multiple-ext::
+
+ Multiple extension modules
+ ==========================
+
+ Putting it all together
+ =======================
diff --git a/Doc/packaging/extending.rst b/Doc/packaging/extending.rst
new file mode 100644
index 0000000..f2d3863
--- /dev/null
+++ b/Doc/packaging/extending.rst
@@ -0,0 +1,95 @@
+.. _extending-packaging:
+
+*******************
+Extending Distutils
+*******************
+
+Distutils can be extended in various ways. Most extensions take the form of new
+commands or replacements for existing commands. New commands may be written to
+support new types of platform-specific packaging, for example, while
+replacements for existing commands may be made to modify details of how the
+command operates on a package.
+
+Most extensions of the packaging are made within :file:`setup.py` scripts that
+want to modify existing commands; many simply add a few file extensions that
+should be copied into packages in addition to :file:`.py` files as a
+convenience.
+
+Most packaging command implementations are subclasses of the
+:class:`packaging.cmd.Command` class. New commands may directly inherit from
+:class:`Command`, while replacements often derive from :class:`Command`
+indirectly, directly subclassing the command they are replacing. Commands are
+required to derive from :class:`Command`.
+
+.. .. _extend-existing:
+ Extending existing commands
+ ===========================
+
+
+.. .. _new-commands:
+ Writing new commands
+ ====================
+
+
+Integrating new commands
+========================
+
+There are different ways to integrate new command implementations into
+packaging. The most difficult is to lobby for the inclusion of the new features
+in packaging itself, and wait for (and require) a version of Python that
+provides that support. This is really hard for many reasons.
+
+The most common, and possibly the most reasonable for most needs, is to include
+the new implementations with your :file:`setup.py` script, and cause the
+:func:`packaging.core.setup` function use them::
+
+ from packaging.core import setup
+ from packaging.command.build_py import build_py as _build_py
+
+ class build_py(_build_py):
+ """Specialized Python source builder."""
+
+ # implement whatever needs to be different...
+
+ setup(..., cmdclass={'build_py': build_py})
+
+This approach is most valuable if the new implementations must be used to use a
+particular package, as everyone interested in the package will need to have the
+new command implementation.
+
+Beginning with Python 2.4, a third option is available, intended to allow new
+commands to be added which can support existing :file:`setup.py` scripts without
+requiring modifications to the Python installation. This is expected to allow
+third-party extensions to provide support for additional packaging systems, but
+the commands can be used for anything packaging commands can be used for. A new
+configuration option, :option:`command_packages` (command-line option
+:option:`--command-packages`), can be used to specify additional packages to be
+searched for modules implementing commands. Like all packaging options, this
+can be specified on the command line or in a configuration file. This option
+can only be set in the ``[global]`` section of a configuration file, or before
+any commands on the command line. If set in a configuration file, it can be
+overridden from the command line; setting it to an empty string on the command
+line causes the default to be used. This should never be set in a configuration
+file provided with a package.
+
+This new option can be used to add any number of packages to the list of
+packages searched for command implementations; multiple package names should be
+separated by commas. When not specified, the search is only performed in the
+:mod:`packaging.command` package. When :file:`setup.py` is run with the option
+:option:`--command-packages` :option:`distcmds,buildcmds`, however, the packages
+:mod:`packaging.command`, :mod:`distcmds`, and :mod:`buildcmds` will be searched
+in that order. New commands are expected to be implemented in modules of the
+same name as the command by classes sharing the same name. Given the example
+command-line option above, the command :command:`bdist_openpkg` could be
+implemented by the class :class:`distcmds.bdist_openpkg.bdist_openpkg` or
+:class:`buildcmds.bdist_openpkg.bdist_openpkg`.
+
+
+Adding new distribution types
+=============================
+
+Commands that create distributions (files in the :file:`dist/` directory) need
+to add ``(command, filename)`` pairs to ``self.distribution.dist_files`` so that
+:command:`upload` can upload it to PyPI. The *filename* in the pair contains no
+path information, only the name of the file itself. In dry-run mode, pairs
+should still be added to represent what would have been created.
diff --git a/Doc/packaging/index.rst b/Doc/packaging/index.rst
new file mode 100644
index 0000000..d3d0dec
--- /dev/null
+++ b/Doc/packaging/index.rst
@@ -0,0 +1,45 @@
+.. _packaging-index:
+
+##############################
+ Distributing Python Projects
+##############################
+
+:Authors: The Fellowship of the Packaging
+:Email: distutils-sig@python.org
+:Release: |version|
+:Date: |today|
+
+This document describes Packaging for Python authors, describing how to use the
+module to make Python applications, packages or modules easily available to a
+wider audience with very little overhead for build/release/install mechanics.
+
+.. toctree::
+ :maxdepth: 2
+ :numbered:
+
+ tutorial
+ setupcfg
+ introduction
+ setupscript
+ configfile
+ sourcedist
+ builtdist
+ packageindex
+ uploading
+ examples
+ extending
+ commandhooks
+ commandref
+
+
+.. seealso::
+
+ :ref:`packaging-install-index`
+ A user-centered manual which includes information on adding projects
+ into an existing Python installation. You do not need to be a Python
+ programmer to read this manual.
+
+ :mod:`packaging`
+ A library reference for developers of packaging tools wanting to use
+ standalone building blocks like :mod:`~packaging.version` or
+ :mod:`~packaging.metadata`, or extend Packaging itself.
diff --git a/Doc/packaging/introduction.rst b/Doc/packaging/introduction.rst
new file mode 100644
index 0000000..a757ffc
--- /dev/null
+++ b/Doc/packaging/introduction.rst
@@ -0,0 +1,193 @@
+.. _packaging-intro:
+
+*****************************
+An Introduction to Packaging
+*****************************
+
+This document covers using Packaging to distribute your Python modules,
+concentrating on the role of developer/distributor. If you're looking for
+information on installing Python modules you should refer to the
+:ref:`packaging-install-index` chapter.
+
+Throughout this documentation, the terms "Distutils", "the Distutils" and
+"Packaging" will be used interchangeably.
+
+.. _packaging-concepts:
+
+Concepts & Terminology
+======================
+
+Using Distutils is quite simple both for module developers and for
+users/administrators installing third-party modules. As a developer, your
+responsibilities (apart from writing solid, well-documented and well-tested
+code, of course!) are:
+
+* writing a setup script (:file:`setup.py` by convention)
+
+* (optional) writing a setup configuration file
+
+* creating a source distribution
+
+* (optional) creating one or more "built" (binary) distributions of your
+ project
+
+All of these tasks are covered in this document.
+
+Not all module developers have access to multiple platforms, so one cannot
+expect them to create buildt distributions for every platform. To remedy
+this, it is hoped that intermediaries called *packagers* will arise to address
+this need. Packagers take source distributions released by module developers,
+build them on one or more platforms and release the resulting built
+distributions. Thus, users on a greater range of platforms will be able to
+install the most popular Python modules in the most natural way for their
+platform without having to run a setup script or compile a single line of code.
+
+
+.. _packaging-simple-example:
+
+A Simple Example
+================
+
+A setup script is usually quite simple, although since it's written in Python
+there are no arbitrary limits to what you can do with it, though you should be
+careful about putting expensive operations in your setup script.
+Unlike, say, Autoconf-style configure scripts the setup script may be run
+multiple times in the course of building and installing a module
+distribution.
+
+If all you want to do is distribute a module called :mod:`foo`, contained in a
+file :file:`foo.py`, then your setup script can be as simple as::
+
+ from packaging.core import setup
+ setup(name='foo',
+ version='1.0',
+ py_modules=['foo'])
+
+Some observations:
+
+* most information that you supply to the Distutils is supplied as keyword
+ arguments to the :func:`setup` function
+
+* those keyword arguments fall into two categories: package metadata (name,
+ version number, etc.) and information about what's in the package (a list
+ of pure Python modules in this case)
+
+* modules are specified by module name, not filename (the same will hold true
+ for packages and extensions)
+
+* it's recommended that you supply a little more metadata than we have in the
+ example. In particular your name, email address and a URL for the
+ project if appropriate (see section :ref:`packaging-setup-script` for an example)
+
+To create a source distribution for this module you would create a setup
+script, :file:`setup.py`, containing the above code and run::
+
+ python setup.py sdist
+
+which will create an archive file (e.g., tarball on Unix, ZIP file on Windows)
+containing your setup script :file:`setup.py`, and your module :file:`foo.py`.
+The archive file will be named :file:`foo-1.0.tar.gz` (or :file:`.zip`), and
+will unpack into a directory :file:`foo-1.0`.
+
+If an end-user wishes to install your :mod:`foo` module all he has to do is
+download :file:`foo-1.0.tar.gz` (or :file:`.zip`), unpack it, and from the
+:file:`foo-1.0` directory run ::
+
+ python setup.py install
+
+which will copy :file:`foo.py` to the appropriate directory for
+third-party modules in their Python installation.
+
+This simple example demonstrates some fundamental concepts of Distutils.
+First, both developers and installers have the same basic user interface, i.e.
+the setup script. The difference is which Distutils *commands* they use: the
+:command:`sdist` command is almost exclusively for module developers, while
+:command:`install` is more often used by installers (although some developers
+will want to install their own code occasionally).
+
+If you want to make things really easy for your users, you can create more
+than one built distributions for them. For instance, if you are running on a
+Windows machine and want to make things easy for other Windows users, you can
+create an executable installer (the most appropriate type of built distribution
+for this platform) with the :command:`bdist_wininst` command. For example::
+
+ python setup.py bdist_wininst
+
+will create an executable installer, :file:`foo-1.0.win32.exe`, in the current
+directory. You can find out what distribution formats are available at any time
+by running ::
+
+ python setup.py bdist --help-formats
+
+
+.. _packaging-python-terms:
+
+General Python terminology
+==========================
+
+If you're reading this document, you probably have a good idea of what Python
+modules, extensions and so forth are. Nevertheless, just to be sure that
+everyone is on the same page, here's a quick overview of Python terms:
+
+module
+ The basic unit of code reusability in Python: a block of code imported by
+ some other code. Three types of modules are important to us here: pure
+ Python modules, extension modules and packages.
+
+pure Python module
+ A module written in Python and contained in a single :file:`.py` file (and
+ possibly associated :file:`.pyc` and/or :file:`.pyo` files). Sometimes
+ referred to as a "pure module."
+
+extension module
+ A module written in the low-level language of the Python implementation: C/C++
+ for Python, Java for Jython. Typically contained in a single dynamically
+ loaded pre-compiled file, e.g. a shared object (:file:`.so`) file for Python
+ extensions on Unix, a DLL (given the :file:`.pyd` extension) for Python
+ extensions on Windows, or a Java class file for Jython extensions. Note that
+ currently Distutils only handles C/C++ extensions for Python.
+
+package
+ A module that contains other modules, typically contained in a directory of
+ the filesystem and distinguished from other directories by the presence of a
+ file :file:`__init__.py`.
+
+root package
+ The root of the hierarchy of packages. (This isn't really a package,
+ since it doesn't have an :file:`__init__.py` file. But... we have to
+ call it something, right?) The vast majority of the standard library is
+ in the root package, as are many small standalone third-party modules that
+ don't belong to a larger module collection. Unlike regular packages,
+ modules in the root package can be found in many directories: in fact,
+ every directory listed in ``sys.path`` contributes modules to the root
+ package.
+
+
+.. _packaging-term:
+
+Distutils-specific terminology
+==============================
+
+The following terms apply more specifically to the domain of distributing Python
+modules using Distutils:
+
+module distribution
+ A collection of Python modules distributed together as a single downloadable
+ resource and meant to be installed all as one. Examples of some well-known
+ module distributions are NumPy, SciPy, PIL (the Python Imaging
+ Library) or mxBase. (Module distributions would be called a *package*,
+ except that term is already taken in the Python context: a single module
+ distribution may contain zero, one, or many Python packages.)
+
+pure module distribution
+ A module distribution that contains only pure Python modules and packages.
+ Sometimes referred to as a "pure distribution."
+
+non-pure module distribution
+ A module distribution that contains at least one extension module. Sometimes
+ referred to as a "non-pure distribution."
+
+distribution root
+ The top-level directory of your source tree (or source distribution). The
+ directory where :file:`setup.py` exists. Generally :file:`setup.py` will
+ be run from this directory.
diff --git a/Doc/packaging/packageindex.rst b/Doc/packaging/packageindex.rst
new file mode 100644
index 0000000..cd1d598
--- /dev/null
+++ b/Doc/packaging/packageindex.rst
@@ -0,0 +1,104 @@
+.. _packaging-package-index:
+
+**********************************
+Registering with the Package Index
+**********************************
+
+The Python Package Index (PyPI) holds metadata describing distributions
+packaged with packaging. The packaging command :command:`register` is used to
+submit your distribution's metadata to the index. It is invoked as follows::
+
+ python setup.py register
+
+Distutils will respond with the following prompt::
+
+ running register
+ We need to know who you are, so please choose either:
+ 1. use your existing login,
+ 2. register as a new user,
+ 3. have the server generate a new password for you (and email it to you), or
+ 4. quit
+ Your selection [default 1]:
+
+Note: if your username and password are saved locally, you will not see this
+menu.
+
+If you have not registered with PyPI, then you will need to do so now. You
+should choose option 2, and enter your details as required. Soon after
+submitting your details, you will receive an email which will be used to confirm
+your registration.
+
+Once you are registered, you may choose option 1 from the menu. You will be
+prompted for your PyPI username and password, and :command:`register` will then
+submit your metadata to the index.
+
+You may submit any number of versions of your distribution to the index. If you
+alter the metadata for a particular version, you may submit it again and the
+index will be updated.
+
+PyPI holds a record for each (name, version) combination submitted. The first
+user to submit information for a given name is designated the Owner of that
+name. They may submit changes through the :command:`register` command or through
+the web interface. They may also designate other users as Owners or Maintainers.
+Maintainers may edit the package information, but not designate other Owners or
+Maintainers.
+
+By default PyPI will list all versions of a given package. To hide certain
+versions, the Hidden property should be set to yes. This must be edited through
+the web interface.
+
+
+.. _packaging-pypirc:
+
+The .pypirc file
+================
+
+The format of the :file:`.pypirc` file is as follows::
+
+ [packaging]
+ index-servers =
+ pypi
+
+ [pypi]
+ repository: <repository-url>
+ username: <username>
+ password: <password>
+
+The *packaging* section defines a *index-servers* variable that lists the
+name of all sections describing a repository.
+
+Each section describing a repository defines three variables:
+
+- *repository*, that defines the url of the PyPI server. Defaults to
+ ``http://www.python.org/pypi``.
+- *username*, which is the registered username on the PyPI server.
+- *password*, that will be used to authenticate. If omitted the user
+ will be prompt to type it when needed.
+
+If you want to define another server a new section can be created and
+listed in the *index-servers* variable::
+
+ [packaging]
+ index-servers =
+ pypi
+ other
+
+ [pypi]
+ repository: <repository-url>
+ username: <username>
+ password: <password>
+
+ [other]
+ repository: http://example.com/pypi
+ username: <username>
+ password: <password>
+
+:command:`register` can then be called with the -r option to point the
+repository to work with::
+
+ python setup.py register -r http://example.com/pypi
+
+For convenience, the name of the section that describes the repository
+may also be used::
+
+ python setup.py register -r other
diff --git a/Doc/packaging/setupcfg.rst b/Doc/packaging/setupcfg.rst
new file mode 100644
index 0000000..9b09a4c
--- /dev/null
+++ b/Doc/packaging/setupcfg.rst
@@ -0,0 +1,890 @@
+.. highlightlang:: cfg
+
+.. _setupcfg-spec:
+
+*******************************************
+Specification of the :file:`setup.cfg` file
+*******************************************
+
+:version: 0.9
+
+This document describes the :file:`setup.cfg`, an ini-style configuration file
+used by Packaging to replace the :file:`setup.py` file used by Distutils.
+This specification is language-agnostic, and will therefore repeat some
+information that's already documented for Python in the
+:class:`configparser.RawConfigParser` documentation.
+
+.. contents::
+ :depth: 3
+ :local:
+
+
+.. _setupcfg-syntax:
+
+Syntax
+======
+
+The ini-style format used in the configuration file is a simple collection of
+sections that group sets of key-value fields separated by ``=`` or ``:`` and
+optional whitespace. Lines starting with ``#`` or ``;`` are comments and will
+be ignored. Empty lines are also ignored. Example::
+
+ [section1]
+ # comment
+ name = value
+ name2 = "other value"
+
+ [section2]
+ foo = bar
+
+
+Parsing values
+---------------
+
+Here are a set of rules to parse values:
+
+- If a value is quoted with ``"`` chars, it's a string. If a quote character is
+ present in the quoted value, it can be escaped as ``\"`` or left as-is.
+
+- If the value is ``true``, ``t``, ``yes``, ``y`` (case-insensitive) or ``1``,
+ it's converted to the language equivalent of a ``True`` value; if it's
+ ``false``, ``f``, ``no``, ``n`` (case-insensitive) or ``0``, it's converted to
+ the equivalent of ``False``.
+
+- A value can contain multiple lines. When read, lines are converted into a
+ sequence of values. Each line after the first must start with a least one
+ space or tab character; this leading indentation will be stripped.
+
+- All other values are considered strings.
+
+Examples::
+
+ [section]
+ foo = one
+ two
+ three
+
+ bar = false
+ baz = 1.3
+ boo = "ok"
+ beee = "wqdqw pojpj w\"ddq"
+
+
+Extending files
+---------------
+
+A configuration file can be extended (i.e. included) by other files. For this,
+a ``DEFAULT`` section must contain an ``extends`` key whose value points to one
+or more files which will be merged into the current files by adding new sections
+and fields. If a file loaded by ``extends`` contains sections or keys that
+already exist in the original file, they will not override the previous values.
+
+Contents of :file:`one.cfg`::
+
+ [section1]
+ name = value
+
+ [section2]
+ foo = foo from one.cfg
+
+Contents of :file:`two.cfg`::
+
+ [DEFAULT]
+ extends = one.cfg
+
+ [section2]
+ foo = foo from two.cfg
+ baz = baz from two.cfg
+
+The result of parsing :file:`two.cfg` is equivalent to this file::
+
+ [section1]
+ name = value
+
+ [section2]
+ foo = foo from one.cfg
+ baz = baz from two.cfg
+
+Example use of multi-line notation to include more than one file::
+
+ [DEFAULT]
+ extends = one.cfg
+ two.cfg
+
+When several files are provided, they are processed sequentially, following the
+precedence rules explained above. This means that the list of files should go
+from most specialized to most common.
+
+**Tools will need to provide a way to produce a merged version of the
+file**. This will be useful to let users publish a single file.
+
+
+.. _setupcfg-sections:
+
+Description of sections and fields
+==================================
+
+Each section contains a description of its options.
+
+- Options that are marked *multi* can have multiple values, one value per
+ line.
+- Options that are marked *optional* can be omitted.
+- Options that are marked *environ* can use environment markers, as described
+ in :PEP:`345`.
+
+
+The sections are:
+
+global
+ Global options not related to one command.
+
+metadata
+ Name, version and other information defined by :PEP:`345`.
+
+files
+ Modules, scripts, data, documentation and other files to include in the
+ distribution.
+
+extension sections
+ Options used to build extension modules.
+
+command sections
+ Options given for specific commands, identical to those that can be given
+ on the command line.
+
+
+.. _setupcfg-section-global:
+
+Global options
+--------------
+
+Contains global options for Packaging. This section is shared with Distutils.
+
+
+commands
+ Defined Packaging command. A command is defined by its fully
+ qualified name. *optional*, *multi*
+
+ Examples::
+
+ [global]
+ commands =
+ package.setup.CustomSdistCommand
+ package.setup.BdistDeb
+
+compilers
+ Defined Packaging compiler. A compiler is defined by its fully
+ qualified name. *optional*, *multi*
+
+ Example::
+
+ [global]
+ compilers =
+ hotcompiler.SmartCCompiler
+
+setup_hooks
+ Defines a list of callables to be called right after the :file:`setup.cfg`
+ file is read, before any other processing. Each value is a Python dotted
+ name to an object, which has to be defined in a module present in the project
+ directory alonside :file:`setup.cfg` or on Python's :data:`sys.path` (see
+ :ref:`packaging-finding-hooks`). The callables are executed in the
+ order they're found in the file; if one of them cannot be found, tools should
+ not stop, but for example produce a warning and continue with the next line.
+ Each callable receives the configuration as a dictionary (keys are
+ :file:`setup.cfg` sections, values are dictionaries of fields) and can make
+ any change to it. *optional*, *multi*
+
+ Example::
+
+ [global]
+ setup_hooks = _setuphooks.customize_config
+
+
+
+.. _setupcfg-section-metadata:
+
+Metadata
+--------
+
+The metadata section contains the metadata for the project as described in
+:PEP:`345`. Field names are case-insensitive.
+
+Fields:
+
+name
+ Name of the project.
+
+version
+ Version of the project. Must comply with :PEP:`386`.
+
+platform
+ Platform specification describing an operating system
+ supported by the distribution which is not listed in the "Operating System"
+ Trove classifiers (:PEP:`301`). *optional*, *multi*
+
+supported-platform
+ Binary distributions containing a PKG-INFO file will
+ use the Supported-Platform field in their metadata to specify the OS and
+ CPU for which the binary distribution was compiled. The semantics of
+ the Supported-Platform field are free form. *optional*, *multi*
+
+summary
+ A one-line summary of what the distribution does.
+ (Used to be called *description* in Distutils1.)
+
+description
+ A longer description. (Used to be called *long_description*
+ in Distutils1.) A file can be provided in the *description-file* field.
+ *optional*
+
+keywords
+ A list of additional keywords to be used to assist searching
+ for the distribution in a larger catalog. Comma or space-separated.
+ *optional*
+
+home-page
+ The URL for the distribution's home page.
+
+download-url
+ The URL from which this version of the distribution
+ can be downloaded. *optional*
+
+author
+ Author's name. *optional*
+
+author-email
+ Author's e-mail. *optional*
+
+maintainer
+ Maintainer's name. *optional*
+
+maintainer-email
+ Maintainer's e-mail. *optional*
+
+license
+ A text indicating the term of uses, when a trove classifier does
+ not match. *optional*.
+
+classifiers
+ Classification for the distribution, as described in PEP 301.
+ *optional*, *multi*, *environ*
+
+requires-dist
+ name of another packaging project required as a dependency.
+ The format is *name (version)* where version is an optional
+ version declaration, as described in PEP 345. *optional*, *multi*, *environ*
+
+provides-dist
+ name of another packaging project contained within this
+ distribution. Same format than *requires-dist*. *optional*, *multi*,
+ *environ*
+
+obsoletes-dist
+ name of another packaging project this version obsoletes.
+ Same format than *requires-dist*. *optional*, *multi*, *environ*
+
+requires-python
+ Specifies the Python version the distribution requires. The value is a
+ comma-separated list of version predicates, as described in PEP 345.
+ *optional*, *environ*
+
+requires-externals
+ a dependency in the system. This field is free-form,
+ and just a hint for downstream maintainers. *optional*, *multi*,
+ *environ*
+
+project-url
+ A label, followed by a browsable URL for the project.
+ "label, url". The label is limited to 32 signs. *optional*, *multi*
+
+One extra field not present in PEP 345 is supported:
+
+description-file
+ Path to a text file that will be used to fill the ``description`` field.
+ Multiple values are accepted; they must be separated by whitespace.
+ ``description-file`` and ``description`` are mutually exclusive. *optional*
+
+
+
+Example::
+
+ [metadata]
+ name = pypi2rpm
+ version = 0.1
+ author = Tarek Ziadé
+ author-email = tarek@ziade.org
+ summary = Script that transforms an sdist archive into a RPM package
+ description-file = README
+ home-page = http://bitbucket.org/tarek/pypi2rpm/wiki/Home
+ project-url:
+ Repository, http://bitbucket.org/tarek/pypi2rpm/
+ RSS feed, https://bitbucket.org/tarek/pypi2rpm/rss
+ classifier =
+ Development Status :: 3 - Alpha
+ License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)
+
+You should not give any explicit value for metadata-version: it will be guessed
+from the fields present in the file.
+
+
+.. _setupcfg-section-files:
+
+Files
+-----
+
+This section describes the files included in the project.
+
+packages_root
+ the root directory containing all packages and modules
+ (default: current directory, i.e. the project's top-level
+ directory where :file:`setup.cfg` lives). *optional*
+
+packages
+ a list of packages the project includes *optional*, *multi*
+
+modules
+ a list of packages the project includes *optional*, *multi*
+
+scripts
+ a list of scripts the project includes *optional*, *multi*
+
+extra_files
+ a list of patterns for additional files to include in source distributions
+ (see :ref:`packaging-manifest`) *optional*, *multi*
+
+Example::
+
+ [files]
+ packages_root = src
+ packages =
+ pypi2rpm
+ pypi2rpm.command
+
+ scripts =
+ pypi2rpm/pypi2rpm.py
+
+ extra_files =
+ setup.py
+ README
+
+
+.. Note::
+ The :file:`setup.cfg` configuration file is included by default. Contrary to
+ Distutils, :file:`README` (or :file:`README.txt`) and :file:`setup.py` are
+ not included by default.
+
+
+Resources
+^^^^^^^^^
+
+This section describes the files used by the project which must not be installed
+in the same place that python modules or libraries, they are called
+**resources**. They are for example documentation files, script files,
+databases, etc...
+
+For declaring resources, you must use this notation::
+
+ source = destination
+
+Data-files are declared in the **resources** field in the **file** section, for
+example::
+
+ [files]
+ resources =
+ source1 = destination1
+ source2 = destination2
+
+The **source** part of the declaration are relative paths of resources files
+(using unix path separator **/**). For example, if you've this source tree::
+
+ foo/
+ doc/
+ doc.man
+ scripts/
+ foo.sh
+
+Your setup.cfg will look like::
+
+ [files]
+ resources =
+ doc/doc.man = destination_doc
+ scripts/foo.sh = destination_scripts
+
+The final paths where files will be placed are composed by : **source** +
+**destination**. In the previous example, **doc/doc.man** will be placed in
+**destination_doc/doc/doc.man** and **scripts/foo.sh** will be placed in
+**destination_scripts/scripts/foo.sh**. (If you want more control on the final
+path, take a look at :ref:`setupcfg-resources-base-prefix`).
+
+The **destination** part of resources declaration are paths with categories.
+Indeed, it's generally a bad idea to give absolute path as it will be cross
+incompatible. So, you must use resources categories in your **destination**
+declaration. Categories will be replaced by their real path at the installation
+time. Using categories is all benefit, your declaration will be simpler, cross
+platform and it will allow packager to place resources files where they want
+without breaking your code.
+
+Categories can be specified by using this syntax::
+
+ {category}
+
+Default categories are:
+
+* config
+* appdata
+* appdata.arch
+* appdata.persistent
+* appdata.disposable
+* help
+* icon
+* scripts
+* doc
+* info
+* man
+
+A special category also exists **{distribution.name}** that will be replaced by
+the name of the distribution, but as most of the defaults categories use them,
+so it's not necessary to add **{distribution.name}** into your destination.
+
+If you use categories in your declarations, and you are encouraged to do, final
+path will be::
+
+ source + destination_expanded
+
+.. _example_final_path:
+
+For example, if you have this setup.cfg::
+
+ [metadata]
+ name = foo
+
+ [files]
+ resources =
+ doc/doc.man = {doc}
+
+And if **{doc}** is replaced by **{datadir}/doc/{distribution.name}**, final
+path will be::
+
+ {datadir}/doc/foo/doc/doc.man
+
+Where {datafir} category will be platform-dependent.
+
+
+More control on source part
+"""""""""""""""""""""""""""
+
+Glob syntax
+'''''''''''
+
+When you declare source file, you can use a glob-like syntax to match multiples file, for example::
+
+ scripts/* = {script}
+
+Will match all the files in the scripts directory and placed them in the script category.
+
+Glob tokens are:
+
+ * ``*``: match all files.
+ * ``?``: match any character.
+ * ``**``: match any level of tree recursion (even 0).
+ * ``{}``: will match any part separated by comma (example: ``{sh,bat}``).
+
+.. TODO Add examples
+
+Order of declaration
+''''''''''''''''''''
+
+The order of declaration is important if one file match multiple rules. The last
+rules matched by file is used, this is useful if you have this source tree::
+
+ foo/
+ doc/
+ index.rst
+ setup.rst
+ documentation.txt
+ doc.tex
+ README
+
+And you want all the files in the doc directory to be placed in {doc} category,
+but README must be placed in {help} category, instead of listing all the files
+one by one, you can declare them in this way::
+
+ [files]
+ resources =
+ doc/* = {doc}
+ doc/README = {help}
+
+Exclude
+'''''''
+
+You can exclude some files of resources declaration by giving no destination, it
+can be useful if you have a non-resources file in the same directory of
+resources files::
+
+ foo/
+ doc/
+ RELEASES
+ doc.tex
+ documentation.txt
+ docu.rst
+
+Your **files** section will be::
+
+ [files]
+ resources =
+ doc/* = {doc}
+ doc/RELEASES =
+
+More control on destination part
+""""""""""""""""""""""""""""""""
+
+.. _setupcfg-resources-base-prefix:
+
+Defining a base prefix
+''''''''''''''''''''''
+
+When you define your resources, you can have more control of how the final path
+is computed.
+
+By default, the final path is::
+
+ destination + source
+
+This can generate long paths, for example (example_final_path_)::
+
+ {datadir}/doc/foo/doc/doc.man
+
+When you declare your source, you can use whitespace to split the source in
+**prefix** **suffix**. So, for example, if you have this source::
+
+ docs/ doc.man
+
+The **prefix** is "docs/" and the **suffix** is "doc.html".
+
+.. note::
+
+ Separator can be placed after a path separator or replace it. So these two
+ sources are equivalent::
+
+ docs/ doc.man
+ docs doc.man
+
+.. note::
+
+ Glob syntax is working the same way with standard source and split source.
+ So these rules::
+
+ docs/*
+ docs/ *
+ docs *
+
+ Will match all the files in the docs directory.
+
+When you use split source, the final path is computed this way::
+
+ destination + prefix
+
+So for example, if you have this setup.cfg::
+
+ [metadata]
+ name = foo
+
+ [files]
+ resources =
+ doc/ doc.man = {doc}
+
+And if **{doc}** is replaced by **{datadir}/doc/{distribution.name}**, final
+path will be::
+
+ {datadir}/doc/foo/doc.man
+
+
+Overwriting paths for categories
+""""""""""""""""""""""""""""""""
+
+This part is intended for system administrators or downstream OS packagers.
+
+The real paths of categories are registered in the *sysconfig.cfg* file
+installed in your python installation. This file uses an ini format too.
+The content of the file is organized into several sections:
+
+* globals: Standard categories's paths.
+* posix_prefix: Standard paths for categories and installation paths for posix
+ system.
+* other ones XXX
+
+Standard categories paths are platform independent, they generally refers to
+other categories, which are platform dependent. :mod:`sysconfig` will choose
+these category from sections matching os.name. For example::
+
+ doc = {datadir}/doc/{distribution.name}
+
+It refers to datadir category, which can be different between platforms. In
+posix system, it may be::
+
+ datadir = /usr/share
+
+So the final path will be::
+
+ doc = /usr/share/doc/{distribution.name}
+
+The platform-dependent categories are:
+
+* confdir
+* datadir
+* libdir
+* base
+
+
+Defining extra categories
+"""""""""""""""""""""""""
+
+.. TODO
+
+
+Examples
+""""""""
+
+These examples are incremental but work unitarily.
+
+Resources in root dir
+'''''''''''''''''''''
+
+Source tree::
+
+ babar-1.0/
+ README
+ babar.sh
+ launch.sh
+ babar.py
+
+:file:`setup.cfg`::
+
+ [files]
+ resources =
+ README = {doc}
+ *.sh = {scripts}
+
+So babar.sh and launch.sh will be placed in {scripts} directory.
+
+Now let's move all the scripts into a scripts directory.
+
+Resources in sub-directory
+''''''''''''''''''''''''''
+
+Source tree::
+
+ babar-1.1/
+ README
+ scripts/
+ babar.sh
+ launch.sh
+ LAUNCH
+ babar.py
+
+:file:`setup.cfg`::
+
+ [files]
+ resources =
+ README = {doc}
+ scripts/ LAUNCH = {doc}
+ scripts/ *.sh = {scripts}
+
+It's important to use the separator after scripts/ to install all the shell
+scripts into {scripts} instead of {scripts}/scripts.
+
+Now let's add some docs.
+
+Resources in multiple sub-directories
+'''''''''''''''''''''''''''''''''''''
+
+Source tree::
+
+ babar-1.2/
+ README
+ scripts/
+ babar.sh
+ launch.sh
+ LAUNCH
+ docs/
+ api
+ man
+ babar.py
+
+:file:`setup.cfg`::
+
+ [files]
+ resources =
+ README = {doc}
+ scripts/ LAUNCH = {doc}
+ scripts/ *.sh = {scripts}
+ doc/ * = {doc}
+ doc/ man = {man}
+
+You want to place all the file in the docs script into {doc} category, instead
+of man, which must be placed into {man} category, we will use the order of
+declaration of globs to choose the destination, the last glob that match the
+file is used.
+
+Now let's add some scripts for windows users.
+
+Complete example
+''''''''''''''''
+
+Source tree::
+
+ babar-1.3/
+ README
+ doc/
+ api
+ man
+ scripts/
+ babar.sh
+ launch.sh
+ babar.bat
+ launch.bat
+ LAUNCH
+
+:file:`setup.cfg`::
+
+ [files]
+ resources =
+ README = {doc}
+ scripts/ LAUNCH = {doc}
+ scripts/ *.{sh,bat} = {scripts}
+ doc/ * = {doc}
+ doc/ man = {man}
+
+We use brace expansion syntax to place all the shell and batch scripts into
+{scripts} category.
+
+
+.. _setupcfg-section-extensions:
+
+Extension modules sections
+--------------------------
+
+If a project includes extension modules written in C or C++, each one of them
+needs to have its options defined in a dedicated section. Here's an example::
+
+ [files]
+ packages = coconut
+
+ [extension: coconut._fastcoconut]
+ language = cxx
+ sources = cxx_src/cononut_utils.cxx
+ cxx_src/python_module.cxx
+ include_dirs = /usr/include/gecode
+ /usr/include/blitz
+ extra_compile_args =
+ -fPIC -O2
+ -DGECODE_VERSION=$(./gecode_version) -- sys.platform != 'win32'
+ /DGECODE_VERSION='win32' -- sys.platform == 'win32'
+
+The section name must start with ``extension:``; the right-hand part is used as
+the full name (including a parent package, if any) of the extension. Whitespace
+around the extension name is allowed. If the extension module is not standalone
+(e.g. ``_bisect``) but part of a package (e.g. ``thing._speedups``), the parent
+package must be listed in the ``packages`` field.
+Valid fields and their values are listed in the documentation of the
+:class:`packaging.compiler.extension.Extension` class; values documented as
+Python lists translate to multi-line values in the configuration file. In
+addition, multi-line values accept environment markers on each line, after a
+``--``.
+
+
+.. _setupcfg-section-commands:
+
+Commands sections
+-----------------
+
+To pass options to commands without having to type them on the command line
+for each invocation, you can write them in the :file:`setup.cfg` file, in a
+section named after the command. Example::
+
+ [sdist]
+ # special function to add custom files
+ manifest-builders = package.setup.list_extra_files
+
+ [build]
+ use-2to3 = True
+
+ [build_ext]
+ inplace = on
+
+ [check]
+ strict = on
+ all = on
+
+Option values given in the configuration file can be overriden on the command
+line. See :ref:`packaging-setup-config` for more information.
+
+These sections are also used to define :ref:`command hooks
+<packaging-command-hooks>`.
+
+
+.. _setupcfg-extensibility:
+
+Extensibility
+=============
+
+Every section can have fields that are not part of this specification. They are
+called **extensions**.
+
+An extension field starts with ``X-``. Example::
+
+ [metadata]
+ name = Distribute
+ X-Debian-Name = python-distribute
+
+
+.. _setupcfg-changes:
+
+Changes in the specification
+============================
+
+The versioning scheme for this specification is **MAJOR.MINOR**. Changes in the
+specification will cause the version number to be updated.
+
+Changes to the minor number reflect backwards-compatible changes:
+
+- New fields and sections (optional or mandatory) can be added.
+- Optional fields can be removed.
+
+The major number will be incremented for backwards-incompatible changes:
+
+- Mandatory fields or sections are removed.
+- Fields change their meaning.
+
+As a consequence, a tool written to consume 1.5 has these properties:
+
+- Can read 1.1, 1.2 and all versions < 1.5, since the tool knows what
+ optional fields weren't there.
+
+ .. XXX clarify
+
+- Can also read 1.6 and other 1.x versions: The tool will just ignore fields it
+ doesn't know about, even if they are mandatory in the new version. If
+ optional fields were removed, the tool will just consider them absent.
+
+- Cannot read 2.x and should refuse to interpret such files.
+
+A tool written to produce 1.x should have these properties:
+
+- Writes all mandatory fields.
+- May write optional fields.
+
+
+.. _setupcfg-acks:
+
+Acknowledgments
+===============
+
+This specification includes work and feedback from these people:
+
+- Tarek Ziadé
+- Julien Jehannet
+- Boris Feld
+- Éric Araujo
+
+(If your name is missing, please :ref:`let us know <reporting-bugs>`.)
diff --git a/Doc/packaging/setupscript.rst b/Doc/packaging/setupscript.rst
new file mode 100644
index 0000000..cafde20
--- /dev/null
+++ b/Doc/packaging/setupscript.rst
@@ -0,0 +1,693 @@
+.. _packaging-setup-script:
+
+************************
+Writing the Setup Script
+************************
+
+The setup script is the center of all activity in building, distributing, and
+installing modules using Distutils. The main purpose of the setup script is
+to describe your module distribution to Distutils, so that the various
+commands that operate on your modules do the right thing. As we saw in section
+:ref:`packaging-simple-example`, the setup script consists mainly of a
+call to :func:`setup` where the most information is supplied as
+keyword arguments to :func:`setup`.
+
+Here's a slightly more involved example, which we'll follow for the next couple
+of sections: a setup script that could be used for Packaging itself::
+
+ #!/usr/bin/env python
+
+ from packaging.core import setup, find_packages
+
+ setup(name='Packaging',
+ version='1.0',
+ summary='Python Distribution Utilities',
+ keywords=['packaging', 'packaging'],
+ author=u'Tarek Ziadé',
+ author_email='tarek@ziade.org',
+ home_page='http://bitbucket.org/tarek/packaging/wiki/Home',
+ license='PSF',
+ packages=find_packages())
+
+
+There are only two differences between this and the trivial one-file
+distribution presented in section :ref:`packaging-simple-example`: more
+metadata and the specification of pure Python modules by package rather than
+by module. This is important since Ristutils consist of a couple of dozen
+modules split into (so far) two packages; an explicit list of every module
+would be tedious to generate and difficult to maintain. For more information
+on the additional metadata, see section :ref:`packaging-metadata`.
+
+Note that any pathnames (files or directories) supplied in the setup script
+should be written using the Unix convention, i.e. slash-separated. The
+Distutils will take care of converting this platform-neutral representation into
+whatever is appropriate on your current platform before actually using the
+pathname. This makes your setup script portable across operating systems, which
+of course is one of the major goals of the Distutils. In this spirit, all
+pathnames in this document are slash-separated.
+
+This, of course, only applies to pathnames given to Distutils functions. If
+you, for example, use standard Python functions such as :func:`glob.glob` or
+:func:`os.listdir` to specify files, you should be careful to write portable
+code instead of hardcoding path separators::
+
+ glob.glob(os.path.join('mydir', 'subdir', '*.html'))
+ os.listdir(os.path.join('mydir', 'subdir'))
+
+
+.. _packaging-listing-packages:
+
+Listing whole packages
+======================
+
+The :option:`packages` option tells the Distutils to process (build, distribute,
+install, etc.) all pure Python modules found in each package mentioned in the
+:option:`packages` list. In order to do this, of course, there has to be a
+correspondence between package names and directories in the filesystem. The
+default correspondence is the most obvious one, i.e. package :mod:`packaging` is
+found in the directory :file:`packaging` relative to the distribution root.
+Thus, when you say ``packages = ['foo']`` in your setup script, you are
+promising that the Distutils will find a file :file:`foo/__init__.py` (which
+might be spelled differently on your system, but you get the idea) relative to
+the directory where your setup script lives. If you break this promise, the
+Distutils will issue a warning but still process the broken package anyway.
+
+If you use a different convention to lay out your source directory, that's no
+problem: you just have to supply the :option:`package_dir` option to tell the
+Distutils about your convention. For example, say you keep all Python source
+under :file:`lib`, so that modules in the "root package" (i.e., not in any
+package at all) are in :file:`lib`, modules in the :mod:`foo` package are in
+:file:`lib/foo`, and so forth. Then you would put ::
+
+ package_dir = {'': 'lib'}
+
+in your setup script. The keys to this dictionary are package names, and an
+empty package name stands for the root package. The values are directory names
+relative to your distribution root. In this case, when you say ``packages =
+['foo']``, you are promising that the file :file:`lib/foo/__init__.py` exists.
+
+Another possible convention is to put the :mod:`foo` package right in
+:file:`lib`, the :mod:`foo.bar` package in :file:`lib/bar`, etc. This would be
+written in the setup script as ::
+
+ package_dir = {'foo': 'lib'}
+
+A ``package: dir`` entry in the :option:`package_dir` dictionary implicitly
+applies to all packages below *package*, so the :mod:`foo.bar` case is
+automatically handled here. In this example, having ``packages = ['foo',
+'foo.bar']`` tells the Distutils to look for :file:`lib/__init__.py` and
+:file:`lib/bar/__init__.py`. (Keep in mind that although :option:`package_dir`
+applies recursively, you must explicitly list all packages in
+:option:`packages`: the Distutils will *not* recursively scan your source tree
+looking for any directory with an :file:`__init__.py` file.)
+
+
+.. _packaging-listing-modules:
+
+Listing individual modules
+==========================
+
+For a small module distribution, you might prefer to list all modules rather
+than listing packages---especially the case of a single module that goes in the
+"root package" (i.e., no package at all). This simplest case was shown in
+section :ref:`packaging-simple-example`; here is a slightly more involved
+example::
+
+ py_modules = ['mod1', 'pkg.mod2']
+
+This describes two modules, one of them in the "root" package, the other in the
+:mod:`pkg` package. Again, the default package/directory layout implies that
+these two modules can be found in :file:`mod1.py` and :file:`pkg/mod2.py`, and
+that :file:`pkg/__init__.py` exists as well. And again, you can override the
+package/directory correspondence using the :option:`package_dir` option.
+
+
+.. _packaging-describing-extensions:
+
+Describing extension modules
+============================
+
+Just as writing Python extension modules is a bit more complicated than writing
+pure Python modules, describing them to the Distutils is a bit more complicated.
+Unlike pure modules, it's not enough just to list modules or packages and expect
+the Distutils to go out and find the right files; you have to specify the
+extension name, source file(s), and any compile/link requirements (include
+directories, libraries to link with, etc.).
+
+.. XXX read over this section
+
+All of this is done through another keyword argument to :func:`setup`, the
+:option:`ext_modules` option. :option:`ext_modules` is just a list of
+:class:`Extension` instances, each of which describes a single extension module.
+Suppose your distribution includes a single extension, called :mod:`foo` and
+implemented by :file:`foo.c`. If no additional instructions to the
+compiler/linker are needed, describing this extension is quite simple::
+
+ Extension('foo', ['foo.c'])
+
+The :class:`Extension` class can be imported from :mod:`packaging.core` along
+with :func:`setup`. Thus, the setup script for a module distribution that
+contains only this one extension and nothing else might be::
+
+ from packaging.core import setup, Extension
+ setup(name='foo',
+ version='1.0',
+ ext_modules=[Extension('foo', ['foo.c'])])
+
+The :class:`Extension` class (actually, the underlying extension-building
+machinery implemented by the :command:`build_ext` command) supports a great deal
+of flexibility in describing Python extensions, which is explained in the
+following sections.
+
+
+Extension names and packages
+----------------------------
+
+The first argument to the :class:`Extension` constructor is always the name of
+the extension, including any package names. For example, ::
+
+ Extension('foo', ['src/foo1.c', 'src/foo2.c'])
+
+describes an extension that lives in the root package, while ::
+
+ Extension('pkg.foo', ['src/foo1.c', 'src/foo2.c'])
+
+describes the same extension in the :mod:`pkg` package. The source files and
+resulting object code are identical in both cases; the only difference is where
+in the filesystem (and therefore where in Python's namespace hierarchy) the
+resulting extension lives.
+
+If your distribution contains only one or more extension modules in a package,
+you need to create a :file:`{package}/__init__.py` file anyway, otherwise Python
+won't be able to import anything.
+
+If you have a number of extensions all in the same package (or all under the
+same base package), use the :option:`ext_package` keyword argument to
+:func:`setup`. For example, ::
+
+ setup(...,
+ ext_package='pkg',
+ ext_modules=[Extension('foo', ['foo.c']),
+ Extension('subpkg.bar', ['bar.c'])])
+
+will compile :file:`foo.c` to the extension :mod:`pkg.foo`, and :file:`bar.c` to
+:mod:`pkg.subpkg.bar`.
+
+
+Extension source files
+----------------------
+
+The second argument to the :class:`Extension` constructor is a list of source
+files. Since the Distutils currently only support C, C++, and Objective-C
+extensions, these are normally C/C++/Objective-C source files. (Be sure to use
+appropriate extensions to distinguish C++\ source files: :file:`.cc` and
+:file:`.cpp` seem to be recognized by both Unix and Windows compilers.)
+
+However, you can also include SWIG interface (:file:`.i`) files in the list; the
+:command:`build_ext` command knows how to deal with SWIG extensions: it will run
+SWIG on the interface file and compile the resulting C/C++ file into your
+extension.
+
+.. XXX SWIG support is rough around the edges and largely untested!
+
+This warning notwithstanding, options to SWIG can be currently passed like
+this::
+
+ setup(...,
+ ext_modules=[Extension('_foo', ['foo.i'],
+ swig_opts=['-modern', '-I../include'])],
+ py_modules=['foo'])
+
+Or on the command line like this::
+
+ > python setup.py build_ext --swig-opts="-modern -I../include"
+
+On some platforms, you can include non-source files that are processed by the
+compiler and included in your extension. Currently, this just means Windows
+message text (:file:`.mc`) files and resource definition (:file:`.rc`) files for
+Visual C++. These will be compiled to binary resource (:file:`.res`) files and
+linked into the executable.
+
+
+Preprocessor options
+--------------------
+
+Three optional arguments to :class:`Extension` will help if you need to specify
+include directories to search or preprocessor macros to define/undefine:
+``include_dirs``, ``define_macros``, and ``undef_macros``.
+
+For example, if your extension requires header files in the :file:`include`
+directory under your distribution root, use the ``include_dirs`` option::
+
+ Extension('foo', ['foo.c'], include_dirs=['include'])
+
+You can specify absolute directories there; if you know that your extension will
+only be built on Unix systems with X11R6 installed to :file:`/usr`, you can get
+away with ::
+
+ Extension('foo', ['foo.c'], include_dirs=['/usr/include/X11'])
+
+You should avoid this sort of non-portable usage if you plan to distribute your
+code: it's probably better to write C code like ::
+
+ #include <X11/Xlib.h>
+
+If you need to include header files from some other Python extension, you can
+take advantage of the fact that header files are installed in a consistent way
+by the Distutils :command:`install_header` command. For example, the Numerical
+Python header files are installed (on a standard Unix installation) to
+:file:`/usr/local/include/python1.5/Numerical`. (The exact location will differ
+according to your platform and Python installation.) Since the Python include
+directory---\ :file:`/usr/local/include/python1.5` in this case---is always
+included in the search path when building Python extensions, the best approach
+is to write C code like ::
+
+ #include <Numerical/arrayobject.h>
+
+.. TODO check if it's d2.sysconfig or the new sysconfig module now
+
+If you must put the :file:`Numerical` include directory right into your header
+search path, though, you can find that directory using the Distutils
+:mod:`packaging.sysconfig` module::
+
+ from packaging.sysconfig import get_python_inc
+ incdir = os.path.join(get_python_inc(plat_specific=1), 'Numerical')
+ setup(...,
+ Extension(..., include_dirs=[incdir]))
+
+Even though this is quite portable---it will work on any Python installation,
+regardless of platform---it's probably easier to just write your C code in the
+sensible way.
+
+You can define and undefine preprocessor macros with the ``define_macros`` and
+``undef_macros`` options. ``define_macros`` takes a list of ``(name, value)``
+tuples, where ``name`` is the name of the macro to define (a string) and
+``value`` is its value: either a string or ``None``. (Defining a macro ``FOO``
+to ``None`` is the equivalent of a bare ``#define FOO`` in your C source: with
+most compilers, this sets ``FOO`` to the string ``1``.) ``undef_macros`` is
+just a list of macros to undefine.
+
+For example::
+
+ Extension(...,
+ define_macros=[('NDEBUG', '1'),
+ ('HAVE_STRFTIME', None)],
+ undef_macros=['HAVE_FOO', 'HAVE_BAR'])
+
+is the equivalent of having this at the top of every C source file::
+
+ #define NDEBUG 1
+ #define HAVE_STRFTIME
+ #undef HAVE_FOO
+ #undef HAVE_BAR
+
+
+Library options
+---------------
+
+You can also specify the libraries to link against when building your extension,
+and the directories to search for those libraries. The ``libraries`` option is
+a list of libraries to link against, ``library_dirs`` is a list of directories
+to search for libraries at link-time, and ``runtime_library_dirs`` is a list of
+directories to search for shared (dynamically loaded) libraries at run-time.
+
+For example, if you need to link against libraries known to be in the standard
+library search path on target systems ::
+
+ Extension(...,
+ libraries=['gdbm', 'readline'])
+
+If you need to link with libraries in a non-standard location, you'll have to
+include the location in ``library_dirs``::
+
+ Extension(...,
+ library_dirs=['/usr/X11R6/lib'],
+ libraries=['X11', 'Xt'])
+
+(Again, this sort of non-portable construct should be avoided if you intend to
+distribute your code.)
+
+.. XXX Should mention clib libraries here or somewhere else!
+
+
+Other options
+-------------
+
+There are still some other options which can be used to handle special cases.
+
+The :option:`optional` option is a boolean; if it is true,
+a build failure in the extension will not abort the build process, but
+instead simply not install the failing extension.
+
+The :option:`extra_objects` option is a list of object files to be passed to the
+linker. These files must not have extensions, as the default extension for the
+compiler is used.
+
+:option:`extra_compile_args` and :option:`extra_link_args` can be used to
+specify additional command-line options for the respective compiler and linker
+command lines.
+
+:option:`export_symbols` is only useful on Windows. It can contain a list of
+symbols (functions or variables) to be exported. This option is not needed when
+building compiled extensions: Distutils will automatically add ``initmodule``
+to the list of exported symbols.
+
+The :option:`depends` option is a list of files that the extension depends on
+(for example header files). The build command will call the compiler on the
+sources to rebuild extension if any on this files has been modified since the
+previous build.
+
+Relationships between Distributions and Packages
+================================================
+
+.. FIXME rewrite to update to PEP 345 (but without dist/release confusion)
+
+A distribution may relate to packages in three specific ways:
+
+#. It can require packages or modules.
+
+#. It can provide packages or modules.
+
+#. It can obsolete packages or modules.
+
+These relationships can be specified using keyword arguments to the
+:func:`packaging.core.setup` function.
+
+Dependencies on other Python modules and packages can be specified by supplying
+the *requires* keyword argument to :func:`setup`. The value must be a list of
+strings. Each string specifies a package that is required, and optionally what
+versions are sufficient.
+
+To specify that any version of a module or package is required, the string
+should consist entirely of the module or package name. Examples include
+``'mymodule'`` and ``'xml.parsers.expat'``.
+
+If specific versions are required, a sequence of qualifiers can be supplied in
+parentheses. Each qualifier may consist of a comparison operator and a version
+number. The accepted comparison operators are::
+
+ < > ==
+ <= >= !=
+
+These can be combined by using multiple qualifiers separated by commas (and
+optional whitespace). In this case, all of the qualifiers must be matched; a
+logical AND is used to combine the evaluations.
+
+Let's look at a bunch of examples:
+
++-------------------------+----------------------------------------------+
+| Requires Expression | Explanation |
++=========================+==============================================+
+| ``==1.0`` | Only version ``1.0`` is compatible |
++-------------------------+----------------------------------------------+
+| ``>1.0, !=1.5.1, <2.0`` | Any version after ``1.0`` and before ``2.0`` |
+| | is compatible, except ``1.5.1`` |
++-------------------------+----------------------------------------------+
+
+Now that we can specify dependencies, we also need to be able to specify what we
+provide that other distributions can require. This is done using the *provides*
+keyword argument to :func:`setup`. The value for this keyword is a list of
+strings, each of which names a Python module or package, and optionally
+identifies the version. If the version is not specified, it is assumed to match
+that of the distribution.
+
+Some examples:
+
++---------------------+----------------------------------------------+
+| Provides Expression | Explanation |
++=====================+==============================================+
+| ``mypkg`` | Provide ``mypkg``, using the distribution |
+| | version |
++---------------------+----------------------------------------------+
+| ``mypkg (1.1)`` | Provide ``mypkg`` version 1.1, regardless of |
+| | the distribution version |
++---------------------+----------------------------------------------+
+
+A package can declare that it obsoletes other packages using the *obsoletes*
+keyword argument. The value for this is similar to that of the *requires*
+keyword: a list of strings giving module or package specifiers. Each specifier
+consists of a module or package name optionally followed by one or more version
+qualifiers. Version qualifiers are given in parentheses after the module or
+package name.
+
+The versions identified by the qualifiers are those that are obsoleted by the
+distribution being described. If no qualifiers are given, all versions of the
+named module or package are understood to be obsoleted.
+
+.. _packaging-installing-scripts:
+
+Installing Scripts
+==================
+
+So far we have been dealing with pure and non-pure Python modules, which are
+usually not run by themselves but imported by scripts.
+
+Scripts are files containing Python source code, intended to be started from the
+command line. Scripts don't require Distutils to do anything very complicated.
+The only clever feature is that if the first line of the script starts with
+``#!`` and contains the word "python", the Distutils will adjust the first line
+to refer to the current interpreter location. By default, it is replaced with
+the current interpreter location. The :option:`--executable` (or :option:`-e`)
+option will allow the interpreter path to be explicitly overridden.
+
+The :option:`scripts` option simply is a list of files to be handled in this
+way. From the PyXML setup script::
+
+ setup(...,
+ scripts=['scripts/xmlproc_parse', 'scripts/xmlproc_val'])
+
+All the scripts will also be added to the ``MANIFEST`` file if no template is
+provided. See :ref:`packaging-manifest`.
+
+.. _packaging-installing-package-data:
+
+Installing Package Data
+=======================
+
+Often, additional files need to be installed into a package. These files are
+often data that's closely related to the package's implementation, or text files
+containing documentation that might be of interest to programmers using the
+package. These files are called :dfn:`package data`.
+
+Package data can be added to packages using the ``package_data`` keyword
+argument to the :func:`setup` function. The value must be a mapping from
+package name to a list of relative path names that should be copied into the
+package. The paths are interpreted as relative to the directory containing the
+package (information from the ``package_dir`` mapping is used if appropriate);
+that is, the files are expected to be part of the package in the source
+directories. They may contain glob patterns as well.
+
+The path names may contain directory portions; any necessary directories will be
+created in the installation.
+
+For example, if a package should contain a subdirectory with several data files,
+the files can be arranged like this in the source tree::
+
+ setup.py
+ src/
+ mypkg/
+ __init__.py
+ module.py
+ data/
+ tables.dat
+ spoons.dat
+ forks.dat
+
+The corresponding call to :func:`setup` might be::
+
+ setup(...,
+ packages=['mypkg'],
+ package_dir={'mypkg': 'src/mypkg'},
+ package_data={'mypkg': ['data/*.dat']})
+
+
+All the files that match ``package_data`` will be added to the ``MANIFEST``
+file if no template is provided. See :ref:`packaging-manifest`.
+
+
+.. _packaging-additional-files:
+
+Installing Additional Files
+===========================
+
+The :option:`data_files` option can be used to specify additional files needed
+by the module distribution: configuration files, message catalogs, data files,
+anything which doesn't fit in the previous categories.
+
+:option:`data_files` specifies a sequence of (*directory*, *files*) pairs in the
+following way::
+
+ setup(...,
+ data_files=[('bitmaps', ['bm/b1.gif', 'bm/b2.gif']),
+ ('config', ['cfg/data.cfg']),
+ ('/etc/init.d', ['init-script'])])
+
+Note that you can specify the directory names where the data files will be
+installed, but you cannot rename the data files themselves.
+
+Each (*directory*, *files*) pair in the sequence specifies the installation
+directory and the files to install there. If *directory* is a relative path, it
+is interpreted relative to the installation prefix (Python's ``sys.prefix`` for
+pure-Python packages, ``sys.exec_prefix`` for packages that contain extension
+modules). Each file name in *files* is interpreted relative to the
+:file:`setup.py` script at the top of the package source distribution. No
+directory information from *files* is used to determine the final location of
+the installed file; only the name of the file is used.
+
+You can specify the :option:`data_files` options as a simple sequence of files
+without specifying a target directory, but this is not recommended, and the
+:command:`install_dist` command will print a warning in this case. To install data
+files directly in the target directory, an empty string should be given as the
+directory.
+
+All the files that match ``data_files`` will be added to the ``MANIFEST`` file
+if no template is provided. See :ref:`packaging-manifest`.
+
+
+
+.. _packaging-metadata:
+
+Metadata reference
+==================
+
+The setup script may include additional metadata beyond the name and version.
+This table describes required and additional information:
+
+.. TODO synchronize with setupcfg; link to it (but don't remove it, it's a
+ useful summary)
+
++----------------------+---------------------------+-----------------+--------+
+| Meta-Data | Description | Value | Notes |
++======================+===========================+=================+========+
+| ``name`` | name of the project | short string | \(1) |
++----------------------+---------------------------+-----------------+--------+
+| ``version`` | version of this release | short string | (1)(2) |
++----------------------+---------------------------+-----------------+--------+
+| ``author`` | project author's name | short string | \(3) |
++----------------------+---------------------------+-----------------+--------+
+| ``author_email`` | email address of the | email address | \(3) |
+| | project author | | |
++----------------------+---------------------------+-----------------+--------+
+| ``maintainer`` | project maintainer's name | short string | \(3) |
++----------------------+---------------------------+-----------------+--------+
+| ``maintainer_email`` | email address of the | email address | \(3) |
+| | project maintainer | | |
++----------------------+---------------------------+-----------------+--------+
+| ``home_page`` | home page for the project | URL | \(1) |
++----------------------+---------------------------+-----------------+--------+
+| ``summary`` | short description of the | short string | |
+| | project | | |
++----------------------+---------------------------+-----------------+--------+
+| ``description`` | longer description of the | long string | \(5) |
+| | project | | |
++----------------------+---------------------------+-----------------+--------+
+| ``download_url`` | location where the | URL | |
+| | project may be downloaded | | |
++----------------------+---------------------------+-----------------+--------+
+| ``classifiers`` | a list of classifiers | list of strings | \(4) |
++----------------------+---------------------------+-----------------+--------+
+| ``platforms`` | a list of platforms | list of strings | |
++----------------------+---------------------------+-----------------+--------+
+| ``license`` | license for the release | short string | \(6) |
++----------------------+---------------------------+-----------------+--------+
+
+Notes:
+
+(1)
+ These fields are required.
+
+(2)
+ It is recommended that versions take the form *major.minor[.patch[.sub]]*.
+
+(3)
+ Either the author or the maintainer must be identified.
+
+(4)
+ The list of classifiers is available from the `PyPI website
+ <http://pypi.python.org/pypi>`_. See also :mod:`packaging.create`.
+
+(5)
+ The ``description`` field is used by PyPI when you are registering a
+ release, to build its PyPI page.
+
+(6)
+ The ``license`` field is a text indicating the license covering the
+ distribution where the license is not a selection from the "License" Trove
+ classifiers. See the ``Classifier`` field. Notice that
+ there's a ``licence`` distribution option which is deprecated but still
+ acts as an alias for ``license``.
+
+'short string'
+ A single line of text, not more than 200 characters.
+
+'long string'
+ Multiple lines of plain text in reStructuredText format (see
+ http://docutils.sf.net/).
+
+'list of strings'
+ See below.
+
+In Python 2.x, "string value" means a unicode object. If a byte string (str or
+bytes) is given, it has to be valid ASCII.
+
+.. TODO move this section to the version document, keep a summary, add a link
+
+Encoding the version information is an art in itself. Python projects generally
+adhere to the version format *major.minor[.patch][sub]*. The major number is 0
+for initial, experimental releases of software. It is incremented for releases
+that represent major milestones in a project. The minor number is incremented
+when important new features are added to the project. The patch number
+increments when bug-fix releases are made. Additional trailing version
+information is sometimes used to indicate sub-releases. These are
+"a1,a2,...,aN" (for alpha releases, where functionality and API may change),
+"b1,b2,...,bN" (for beta releases, which only fix bugs) and "pr1,pr2,...,prN"
+(for final pre-release release testing). Some examples:
+
+0.1.0
+ the first, experimental release of a project
+
+1.0.1a2
+ the second alpha release of the first patch version of 1.0
+
+:option:`classifiers` are specified in a Python list::
+
+ setup(...,
+ classifiers=[
+ 'Development Status :: 4 - Beta',
+ 'Environment :: Console',
+ 'Environment :: Web Environment',
+ 'Intended Audience :: End Users/Desktop',
+ 'Intended Audience :: Developers',
+ 'Intended Audience :: System Administrators',
+ 'License :: OSI Approved :: Python Software Foundation License',
+ 'Operating System :: MacOS :: MacOS X',
+ 'Operating System :: Microsoft :: Windows',
+ 'Operating System :: POSIX',
+ 'Programming Language :: Python',
+ 'Topic :: Communications :: Email',
+ 'Topic :: Office/Business',
+ 'Topic :: Software Development :: Bug Tracking',
+ ])
+
+
+Debugging the setup script
+==========================
+
+Sometimes things go wrong, and the setup script doesn't do what the developer
+wants.
+
+Distutils catches any exceptions when running the setup script, and print a
+simple error message before the script is terminated. The motivation for this
+behaviour is to not confuse administrators who don't know much about Python and
+are trying to install a project. If they get a big long traceback from deep
+inside the guts of Distutils, they may think the project or the Python
+installation is broken because they don't read all the way down to the bottom
+and see that it's a permission problem.
+
+.. FIXME DISTUTILS_DEBUG is dead, document logging/warnings here
+
+On the other hand, this doesn't help the developer to find the cause of the
+failure. For this purpose, the DISTUTILS_DEBUG environment variable can be set
+to anything except an empty string, and Packaging will now print detailed
+information about what it is doing, and prints the full traceback in case an
+exception occurs.
diff --git a/Doc/packaging/sourcedist.rst b/Doc/packaging/sourcedist.rst
new file mode 100644
index 0000000..2cedc15
--- /dev/null
+++ b/Doc/packaging/sourcedist.rst
@@ -0,0 +1,266 @@
+.. _packaging-source-dist:
+
+******************************
+Creating a Source Distribution
+******************************
+
+As shown in section :ref:`packaging-simple-example`, you use the :command:`sdist` command
+to create a source distribution. In the simplest case, ::
+
+ python setup.py sdist
+
+(assuming you haven't specified any :command:`sdist` options in the setup script
+or config file), :command:`sdist` creates the archive of the default format for
+the current platform. The default format is a gzip'ed tar file
+(:file:`.tar.gz`) on Unix, and ZIP file on Windows.
+
+You can specify as many formats as you like using the :option:`--formats`
+option, for example::
+
+ python setup.py sdist --formats=gztar,zip
+
+to create a gzipped tarball and a zip file. The available formats are:
+
++-----------+-------------------------+---------+
+| Format | Description | Notes |
++===========+=========================+=========+
+| ``zip`` | zip file (:file:`.zip`) | (1),(3) |
++-----------+-------------------------+---------+
+| ``gztar`` | gzip'ed tar file | \(2) |
+| | (:file:`.tar.gz`) | |
++-----------+-------------------------+---------+
+| ``bztar`` | bzip2'ed tar file | |
+| | (:file:`.tar.bz2`) | |
++-----------+-------------------------+---------+
+| ``tar`` | tar file (:file:`.tar`) | |
++-----------+-------------------------+---------+
+
+Notes:
+
+(1)
+ default on Windows
+
+(2)
+ default on Unix
+
+(3)
+ requires either external :program:`zip` utility or :mod:`zipfile` module (part
+ of the standard Python library since Python 1.6)
+
+When using any ``tar`` format (``gztar``, ``bztar`` or
+``tar``) under Unix, you can specify the ``owner`` and ``group`` names
+that will be set for each member of the archive.
+
+For example, if you want all files of the archive to be owned by root::
+
+ python setup.py sdist --owner=root --group=root
+
+
+.. _packaging-manifest:
+
+Specifying the files to distribute
+==================================
+
+If you don't supply an explicit list of files (or instructions on how to
+generate one), the :command:`sdist` command puts a minimal default set into the
+source distribution:
+
+* all Python source files implied by the :option:`py_modules` and
+ :option:`packages` options
+
+* all C source files mentioned in the :option:`ext_modules` or
+ :option:`libraries` options
+
+* scripts identified by the :option:`scripts` option
+ See :ref:`packaging-installing-scripts`.
+
+* anything that looks like a test script: :file:`test/test\*.py` (currently, the
+ Packaging don't do anything with test scripts except include them in source
+ distributions, but in the future there will be a standard for testing Python
+ module distributions)
+
+* the configuration file :file:`setup.cfg`
+
+* all files that matches the ``package_data`` metadata.
+ See :ref:`packaging-installing-package-data`.
+
+* all files that matches the ``data_files`` metadata.
+ See :ref:`packaging-additional-files`.
+
+Contrary to Distutils, :file:`README` (or :file:`README.txt`) and
+:file:`setup.py` are not included by default.
+
+Sometimes this is enough, but usually you will want to specify additional files
+to distribute. The typical way to do this is to write a *manifest template*,
+called :file:`MANIFEST.in` by default. The manifest template is just a list of
+instructions for how to generate your manifest file, :file:`MANIFEST`, which is
+the exact list of files to include in your source distribution. The
+:command:`sdist` command processes this template and generates a manifest based
+on its instructions and what it finds in the filesystem.
+
+If you prefer to roll your own manifest file, the format is simple: one filename
+per line, regular files (or symlinks to them) only. If you do supply your own
+:file:`MANIFEST`, you must specify everything: the default set of files
+described above does not apply in this case.
+
+:file:`MANIFEST` files start with a comment indicating they are generated.
+Files without this comment are not overwritten or removed.
+
+See :ref:`packaging-manifest-template` section for a syntax reference.
+
+
+.. _packaging-manifest-options:
+
+Manifest-related options
+========================
+
+The normal course of operations for the :command:`sdist` command is as follows:
+
+* if the manifest file, :file:`MANIFEST` doesn't exist, read :file:`MANIFEST.in`
+ and create the manifest
+
+* if neither :file:`MANIFEST` nor :file:`MANIFEST.in` exist, create a manifest
+ with just the default file set
+
+* if either :file:`MANIFEST.in` or the setup script (:file:`setup.py`) are more
+ recent than :file:`MANIFEST`, recreate :file:`MANIFEST` by reading
+ :file:`MANIFEST.in`
+
+* use the list of files now in :file:`MANIFEST` (either just generated or read
+ in) to create the source distribution archive(s)
+
+There are a couple of options that modify this behaviour. First, use the
+:option:`--no-defaults` and :option:`--no-prune` to disable the standard
+"include" and "exclude" sets.
+
+Second, you might just want to (re)generate the manifest, but not create a
+source distribution::
+
+ python setup.py sdist --manifest-only
+
+:option:`-o` is a shortcut for :option:`--manifest-only`.
+
+
+.. _packaging-manifest-template:
+
+The MANIFEST.in template
+========================
+
+A :file:`MANIFEST.in` file can be added in a project to define the list of
+files to include in the distribution built by the :command:`sdist` command.
+
+When :command:`sdist` is run, it will look for the :file:`MANIFEST.in` file
+and interpret it to generate the :file:`MANIFEST` file that contains the
+list of files that will be included in the package.
+
+This mechanism can be used when the default list of files is not enough.
+(See :ref:`packaging-manifest`).
+
+Principle
+---------
+
+The manifest template has one command per line, where each command specifies a
+set of files to include or exclude from the source distribution. For an
+example, let's look at the Packaging' own manifest template::
+
+ include *.txt
+ recursive-include examples *.txt *.py
+ prune examples/sample?/build
+
+The meanings should be fairly clear: include all files in the distribution root
+matching :file:`\*.txt`, all files anywhere under the :file:`examples` directory
+matching :file:`\*.txt` or :file:`\*.py`, and exclude all directories matching
+:file:`examples/sample?/build`. All of this is done *after* the standard
+include set, so you can exclude files from the standard set with explicit
+instructions in the manifest template. (Or, you can use the
+:option:`--no-defaults` option to disable the standard set entirely.)
+
+The order of commands in the manifest template matters: initially, we have the
+list of default files as described above, and each command in the template adds
+to or removes from that list of files. Once we have fully processed the
+manifest template, we remove files that should not be included in the source
+distribution:
+
+* all files in the Packaging "build" tree (default :file:`build/`)
+
+* all files in directories named :file:`RCS`, :file:`CVS`, :file:`.svn`,
+ :file:`.hg`, :file:`.git`, :file:`.bzr` or :file:`_darcs`
+
+Now we have our complete list of files, which is written to the manifest for
+future reference, and then used to build the source distribution archive(s).
+
+You can disable the default set of included files with the
+:option:`--no-defaults` option, and you can disable the standard exclude set
+with :option:`--no-prune`.
+
+Following the Packaging' own manifest template, let's trace how the
+:command:`sdist` command builds the list of files to include in the Packaging
+source distribution:
+
+#. include all Python source files in the :file:`packaging` and
+ :file:`packaging/command` subdirectories (because packages corresponding to
+ those two directories were mentioned in the :option:`packages` option in the
+ setup script---see section :ref:`packaging-setup-script`)
+
+#. include :file:`README.txt`, :file:`setup.py`, and :file:`setup.cfg` (standard
+ files)
+
+#. include :file:`test/test\*.py` (standard files)
+
+#. include :file:`\*.txt` in the distribution root (this will find
+ :file:`README.txt` a second time, but such redundancies are weeded out later)
+
+#. include anything matching :file:`\*.txt` or :file:`\*.py` in the sub-tree
+ under :file:`examples`,
+
+#. exclude all files in the sub-trees starting at directories matching
+ :file:`examples/sample?/build`\ ---this may exclude files included by the
+ previous two steps, so it's important that the ``prune`` command in the manifest
+ template comes after the ``recursive-include`` command
+
+#. exclude the entire :file:`build` tree, and any :file:`RCS`, :file:`CVS`,
+ :file:`.svn`, :file:`.hg`, :file:`.git`, :file:`.bzr` and :file:`_darcs`
+ directories
+
+Just like in the setup script, file and directory names in the manifest template
+should always be slash-separated; the Packaging will take care of converting
+them to the standard representation on your platform. That way, the manifest
+template is portable across operating systems.
+
+Commands
+--------
+
+The manifest template commands are:
+
++-------------------------------------------+-----------------------------------------------+
+| Command | Description |
++===========================================+===============================================+
+| :command:`include pat1 pat2 ...` | include all files matching any of the listed |
+| | patterns |
++-------------------------------------------+-----------------------------------------------+
+| :command:`exclude pat1 pat2 ...` | exclude all files matching any of the listed |
+| | patterns |
++-------------------------------------------+-----------------------------------------------+
+| :command:`recursive-include dir pat1 pat2 | include all files under *dir* matching any of |
+| ...` | the listed patterns |
++-------------------------------------------+-----------------------------------------------+
+| :command:`recursive-exclude dir pat1 pat2 | exclude all files under *dir* matching any of |
+| ...` | the listed patterns |
++-------------------------------------------+-----------------------------------------------+
+| :command:`global-include pat1 pat2 ...` | include all files anywhere in the source tree |
+| | matching --- & any of the listed patterns |
++-------------------------------------------+-----------------------------------------------+
+| :command:`global-exclude pat1 pat2 ...` | exclude all files anywhere in the source tree |
+| | matching --- & any of the listed patterns |
++-------------------------------------------+-----------------------------------------------+
+| :command:`prune dir` | exclude all files under *dir* |
++-------------------------------------------+-----------------------------------------------+
+| :command:`graft dir` | include all files under *dir* |
++-------------------------------------------+-----------------------------------------------+
+
+The patterns here are Unix-style "glob" patterns: ``*`` matches any sequence of
+regular filename characters, ``?`` matches any single regular filename
+character, and ``[range]`` matches any of the characters in *range* (e.g.,
+``a-z``, ``a-zA-Z``, ``a-f0-9_.``). The definition of "regular filename
+character" is platform-specific: on Unix it is anything except slash; on Windows
+anything except backslash or colon.
diff --git a/Doc/packaging/tutorial.rst b/Doc/packaging/tutorial.rst
new file mode 100644
index 0000000..04f41e5
--- /dev/null
+++ b/Doc/packaging/tutorial.rst
@@ -0,0 +1,112 @@
+==================
+Packaging tutorial
+==================
+
+Welcome to the Packaging tutorial! We will learn how to use Packaging
+to package your project.
+
+.. TODO merge with introduction.rst
+
+
+Getting started
+---------------
+
+Packaging works with the *setup.cfg* file. It contains all the metadata for
+your project, as defined in PEP 345, but also declare what your project
+contains.
+
+Let's say you have a project called *CLVault* containing one package called
+*clvault*, and a few scripts inside. You can use the *pysetup* script to create
+a *setup.cfg* file for the project. The script will ask you a few questions::
+
+ $ mkdir CLVault
+ $ cd CLVault
+ $ pysetup create
+ Project name [CLVault]:
+ Current version number: 0.1
+ Package description:
+ >Command-line utility to store and retrieve passwords
+ Author name: Tarek Ziade
+ Author e-mail address: tarek@ziade.org
+ Project Home Page: http://bitbucket.org/tarek/clvault
+ Do you want to add a package ? (y/n): y
+ Package name: clvault
+ Do you want to add a package ? (y/n): n
+ Do you want to set Trove classifiers? (y/n): y
+ Please select the project status:
+
+ 1 - Planning
+ 2 - Pre-Alpha
+ 3 - Alpha
+ 4 - Beta
+ 5 - Production/Stable
+ 6 - Mature
+ 7 - Inactive
+
+ Status: 3
+ What license do you use: GPL
+ Matching licenses:
+
+ 1) License :: OSI Approved :: GNU General Public License (GPL)
+ 2) License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)
+
+ Type the number of the license you wish to use or ? to try again:: 1
+ Do you want to set other trove identifiers (y/n) [n]: n
+ Wrote "setup.cfg".
+
+
+A setup.cfg file is created, containing the metadata of your project and the
+list of the packages it contains::
+
+ $ cat setup.cfg
+ [metadata]
+ name = CLVault
+ version = 0.1
+ author = Tarek Ziade
+ author_email = tarek@ziade.org
+ description = Command-line utility to store and retrieve passwords
+ home_page = http://bitbucket.org/tarek/clvault
+
+ classifier = Development Status :: 3 - Alpha
+ License :: OSI Approved :: GNU General Public License (GPL)
+
+ [files]
+ packages = clvault
+
+
+Our project will depend on the *keyring* project. Let's add it in the
+[metadata] section::
+
+ [metadata]
+ ...
+ requires_dist =
+ keyring
+
+
+Running commands
+----------------
+
+You can run useful commands on your project once the setup.cfg file is ready:
+
+- sdist: creates a source distribution
+- register: register your project to PyPI
+- upload: upload the distribution to PyPI
+- install_dist: install it
+
+All commands are run using the run script::
+
+ $ pysetup run install_dist
+ $ pysetup run sdist
+ $ pysetup run upload
+
+If you want to push a source distribution of your project to PyPI, do::
+
+ $ pysetup run sdist register upload
+
+
+Installing the project
+----------------------
+
+The project can be installed by manually running the packaging install command::
+
+ $ pysetup run install_dist
diff --git a/Doc/packaging/uploading.rst b/Doc/packaging/uploading.rst
new file mode 100644
index 0000000..297518b
--- /dev/null
+++ b/Doc/packaging/uploading.rst
@@ -0,0 +1,80 @@
+.. _packaging-package-upload:
+
+***************************************
+Uploading Packages to the Package Index
+***************************************
+
+The Python Package Index (PyPI) not only stores the package info, but also the
+package data if the author of the package wishes to. The packaging command
+:command:`upload` pushes the distribution files to PyPI.
+
+The command is invoked immediately after building one or more distribution
+files. For example, the command ::
+
+ python setup.py sdist bdist_wininst upload
+
+will cause the source distribution and the Windows installer to be uploaded to
+PyPI. Note that these will be uploaded even if they are built using an earlier
+invocation of :file:`setup.py`, but that only distributions named on the command
+line for the invocation including the :command:`upload` command are uploaded.
+
+The :command:`upload` command uses the username, password, and repository URL
+from the :file:`$HOME/.pypirc` file (see section :ref:`packaging-pypirc` for more on this
+file). If a :command:`register` command was previously called in the same
+command, and if the password was entered in the prompt, :command:`upload` will
+reuse the entered password. This is useful if you do not want to store a clear
+text password in the :file:`$HOME/.pypirc` file.
+
+You can specify another PyPI server with the :option:`--repository=*url*`
+option::
+
+ python setup.py sdist bdist_wininst upload -r http://example.com/pypi
+
+See section :ref:`packaging-pypirc` for more on defining several servers.
+
+You can use the :option:`--sign` option to tell :command:`upload` to sign each
+uploaded file using GPG (GNU Privacy Guard). The :program:`gpg` program must
+be available for execution on the system :envvar:`PATH`. You can also specify
+which key to use for signing using the :option:`--identity=*name*` option.
+
+Other :command:`upload` options include :option:`--repository=<url>` or
+:option:`--repository=<section>` where *url* is the url of the server and
+*section* the name of the section in :file:`$HOME/.pypirc`, and
+:option:`--show-response` (which displays the full response text from the PyPI
+server for help in debugging upload problems).
+
+PyPI package display
+====================
+
+The ``description`` field plays a special role at PyPI. It is used by
+the server to display a home page for the registered package.
+
+If you use the `reStructuredText <http://docutils.sourceforge.net/rst.html>`_
+syntax for this field, PyPI will parse it and display an HTML output for
+the package home page.
+
+The ``description`` field can be filled from a text file located in the
+project::
+
+ from packaging.core import setup
+
+ fp = open('README.txt')
+ try:
+ description = fp.read()
+ finally:
+ fp.close()
+
+ setup(name='Packaging',
+ description=description)
+
+In that case, :file:`README.txt` is a regular reStructuredText text file located
+in the root of the package besides :file:`setup.py`.
+
+To prevent registering broken reStructuredText content, you can use the
+:program:`rst2html` program that is provided by the :mod:`docutils` package
+and check the ``description`` from the command line::
+
+ $ python setup.py --description | rst2html.py > output.html
+
+:mod:`docutils` will display a warning if there's something wrong with your
+syntax.
diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst
index d01b1f2..a93c09a 100644
--- a/Doc/reference/datamodel.rst
+++ b/Doc/reference/datamodel.rst
@@ -276,16 +276,16 @@ Sequences
single: integer
single: Unicode
- The items of a string object are Unicode code units. A Unicode code
- unit is represented by a string object of one item and can hold either
- a 16-bit or 32-bit value representing a Unicode ordinal (the maximum
- value for the ordinal is given in ``sys.maxunicode``, and depends on
- how Python is configured at compile time). Surrogate pairs may be
- present in the Unicode object, and will be reported as two separate
- items. The built-in functions :func:`chr` and :func:`ord` convert
- between code units and nonnegative integers representing the Unicode
- ordinals as defined in the Unicode Standard 3.0. Conversion from and to
- other encodings are possible through the string method :meth:`encode`.
+ A string is a sequence of values that represent Unicode codepoints.
+ All the codepoints in range ``U+0000 - U+10FFFF`` can be represented
+ in a string. Python doesn't have a :c:type:`chr` type, and
+ every character in the string is represented as a string object
+ with length ``1``. The built-in function :func:`ord` converts a
+ character to its codepoint (as an integer); :func:`chr` converts
+ an integer in range ``0 - 10FFFF`` to the corresponding character.
+ :meth:`str.encode` can be used to convert a :class:`str` to
+ :class:`bytes` using the given encoding, and :meth:`bytes.decode` can
+ be used to achieve the opposite.
Tuples
.. index::
@@ -1351,7 +1351,8 @@ access (use of, assignment to, or deletion of ``x.name``) for class instances.
.. method:: object.__dir__(self)
- Called when :func:`dir` is called on the object. A list must be returned.
+ Called when :func:`dir` is called on the object. A sequence must be
+ returned. :func:`dir` converts the returned sequence to a list and sorts it.
.. _descriptors:
diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst
index 4b49738..5900daa 100644
--- a/Doc/reference/lexical_analysis.rst
+++ b/Doc/reference/lexical_analysis.rst
@@ -492,13 +492,13 @@ Escape sequences only recognized in string literals are:
+-----------------+---------------------------------+-------+
| Escape Sequence | Meaning | Notes |
+=================+=================================+=======+
-| ``\N{name}`` | Character named *name* in the | |
+| ``\N{name}`` | Character named *name* in the | \(4) |
| | Unicode database | |
+-----------------+---------------------------------+-------+
-| ``\uxxxx`` | Character with 16-bit hex value | \(4) |
+| ``\uxxxx`` | Character with 16-bit hex value | \(5) |
| | *xxxx* | |
+-----------------+---------------------------------+-------+
-| ``\Uxxxxxxxx`` | Character with 32-bit hex value | \(5) |
+| ``\Uxxxxxxxx`` | Character with 32-bit hex value | \(6) |
| | *xxxxxxxx* | |
+-----------------+---------------------------------+-------+
@@ -516,10 +516,14 @@ Notes:
with the given value.
(4)
+ .. versionchanged:: 3.3
+ Support for name aliases [#]_ has been added.
+
+(5)
Individual code units which form parts of a surrogate pair can be encoded using
this escape sequence. Exactly four hex digits are required.
-(5)
+(6)
Any Unicode character can be encoded this way, but characters outside the Basic
Multilingual Plane (BMP) will be encoded using a surrogate pair if Python is
compiled to use 16-bit code units (the default). Exactly eight hex digits
@@ -706,3 +710,8 @@ The following printing ASCII characters are not used in Python. Their
occurrence outside string literals and comments is an unconditional error::
$ ? `
+
+
+.. rubric:: Footnotes
+
+.. [#] http://www.unicode.org/Public/6.0.0/ucd/NameAliases.txt
diff --git a/Doc/tools/sphinxext/indexcontent.html b/Doc/tools/sphinxext/indexcontent.html
index d5e17cd..778346f 100644
--- a/Doc/tools/sphinxext/indexcontent.html
+++ b/Doc/tools/sphinxext/indexcontent.html
@@ -20,10 +20,10 @@
<span class="linkdescr">tutorial for C/C++ programmers</span></p>
<p class="biglink"><a class="biglink" href="{{ pathto("c-api/index") }}">Python/C API</a><br/>
<span class="linkdescr">reference for C/C++ programmers</span></p>
- <p class="biglink"><a class="biglink" href="{{ pathto("install/index") }}">Installing Python Modules</a><br/>
- <span class="linkdescr">information for installers &amp; sys-admins</span></p>
- <p class="biglink"><a class="biglink" href="{{ pathto("distutils/index") }}">Distributing Python Modules</a><br/>
- <span class="linkdescr">sharing modules with others</span></p>
+ <p class="biglink"><a class="biglink" href="{{ pathto("install/index") }}">Installing Python Projects</a><br/>
+ <span class="linkdescr">finding and installing modules and applications</span></p>
+ <p class="biglink"><a class="biglink" href="{{ pathto("packaging/index") }}">Distributing Python Projects</a><br/>
+ <span class="linkdescr">packaging and distributing modules and applications</span></p>
<p class="biglink"><a class="biglink" href="{{ pathto("documenting/index") }}">Documenting Python</a><br/>
<span class="linkdescr">guide for documentation authors</span></p>
<p class="biglink"><a class="biglink" href="{{ pathto("faq/index") }}">FAQs</a><br/>
diff --git a/Doc/tools/sphinxext/indexsidebar.html b/Doc/tools/sphinxext/indexsidebar.html
index 672492e..3ad24f9 100644
--- a/Doc/tools/sphinxext/indexsidebar.html
+++ b/Doc/tools/sphinxext/indexsidebar.html
@@ -3,7 +3,7 @@
<h3>Docs for other versions</h3>
<ul>
<li><a href="http://docs.python.org/2.7/">Python 2.7 (stable)</a></li>
- <li><a href="http://docs.python.org/3.1/">Python 3.1 (stable)</a></li>
+ <li><a href="http://docs.python.org/3.2/">Python 3.2 (stable)</a></li>
<li><a href="http://www.python.org/doc/versions/">Old versions</a></li>
</ul>
diff --git a/Doc/tools/sphinxext/pyspecific.py b/Doc/tools/sphinxext/pyspecific.py
index 4329281..d928cfd 100644
--- a/Doc/tools/sphinxext/pyspecific.py
+++ b/Doc/tools/sphinxext/pyspecific.py
@@ -10,7 +10,7 @@
"""
ISSUE_URI = 'http://bugs.python.org/issue%s'
-SOURCE_URI = 'http://hg.python.org/cpython/file/3.2/%s'
+SOURCE_URI = 'http://hg.python.org/cpython/file/default/%s'
from docutils import nodes, utils
from sphinx.util.nodes import split_explicit_title
diff --git a/Doc/tools/sphinxext/susp-ignored.csv b/Doc/tools/sphinxext/susp-ignored.csv
index 350af37..c8fd9c4 100644
--- a/Doc/tools/sphinxext/susp-ignored.csv
+++ b/Doc/tools/sphinxext/susp-ignored.csv
@@ -45,8 +45,8 @@ library/functions,,:stop,"a[start:stop, i]"
library/functions,,:stop,a[start:stop:step]
library/hotshot,,:lineno,"ncalls tottime percall cumtime percall filename:lineno(function)"
library/httplib,,:port,host:port
-library/imaplib,,:MM,"""DD-Mmm-YYYY HH:MM:SS +HHMM"""
-library/imaplib,,:SS,"""DD-Mmm-YYYY HH:MM:SS +HHMM"""
+library/imaplib,,:MM,"""DD-Mmm-YYYY HH:MM:SS"
+library/imaplib,,:SS,"""DD-Mmm-YYYY HH:MM:SS"
library/itertools,,:stop,elements from seq[start:stop:step]
library/itertools,,:step,elements from seq[start:stop:step]
library/linecache,,:sys,"sys:x:3:3:sys:/dev:/bin/sh"
@@ -185,6 +185,99 @@ documenting/fromlatex,152,:noindex,:noindex:
documenting/fromlatex,162,.. describe:,.. describe:: a == b
documenting/fromlatex,168,.. cmdoption:,.. cmdoption:: -O
documenting/fromlatex,168,.. envvar:,.. envvar:: PYTHONINSPECT
+documenting/markup,33,.. sectionauthor:,.. sectionauthor:: Guido van Rossum <guido@python.org>
+documenting/markup,42,:mod,:mod:`parrot` -- Dead parrot access
+documenting/markup,42,`,:mod:`parrot` -- Dead parrot access
+documenting/markup,42,.. module:,.. module:: parrot
+documenting/markup,42,:platform,":platform: Unix, Windows"
+documenting/markup,42,:synopsis,:synopsis: Analyze and reanimate dead parrots.
+documenting/markup,42,.. moduleauthor:,.. moduleauthor:: Eric Cleese <eric@python.invalid>
+documenting/markup,42,.. moduleauthor:,.. moduleauthor:: John Idle <john@python.invalid>
+documenting/markup,88,:noindex,:noindex:
+documenting/markup,95,.. function:,.. function:: spam(eggs)
+documenting/markup,95,:noindex,:noindex:
+documenting/markup,101,.. method:,.. method:: FileInput.input(...)
+documenting/markup,121,:function,c:function
+documenting/markup,121,.. c:,".. c:function:: PyObject* PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems)"
+documenting/markup,121,::,".. c:function:: PyObject* PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems)"
+documenting/markup,131,:member,c:member
+documenting/markup,131,.. c:,.. c:member:: PyObject* PyTypeObject.tp_bases
+documenting/markup,131,::,.. c:member:: PyObject* PyTypeObject.tp_bases
+documenting/markup,139,:macro,c:macro
+documenting/markup,143,:type,c:type
+documenting/markup,150,:var,c:var
+documenting/markup,150,.. cvar:,.. cvar:: PyObject* PyClass_Type
+documenting/markup,179,.. function:,".. function:: Timer.repeat([repeat=3[, number=1000000]])"
+documenting/markup,210,.. decorator:,.. decorator:: removename
+documenting/markup,210,.. decorator:,.. decorator:: setnewname(name)
+documenting/markup,210,:func,:func:
+documenting/markup,233,:meth,:meth:
+documenting/markup,246,.. cmdoption:,.. cmdoption:: -m <module>
+documenting/markup,264,.. describe:,.. describe:: opcode
+documenting/markup,293,.. highlightlang:,.. highlightlang:: c
+documenting/markup,313,.. literalinclude:,.. literalinclude:: example.py
+documenting/markup,328,:rolename,:rolename:`content`
+documenting/markup,328,`,:rolename:`content`
+documenting/markup,333,:role,:role:`title <target>`
+documenting/markup,333,`,:role:`title <target>`
+documenting/markup,339,:meth,:meth:`~Queue.Queue.get`
+documenting/markup,339,`,:meth:`~Queue.Queue.get`
+documenting/markup,387,:func,:func:`filter`
+documenting/markup,387,`,:func:`filter`
+documenting/markup,387,:func,:func:`foo.filter`
+documenting/markup,387,`,:func:`foo.filter`
+documenting/markup,393,:func,:func:`open`
+documenting/markup,393,`,:func:`open`
+documenting/markup,393,:func,:func:`.open`
+documenting/markup,393,`,:func:`.open`
+documenting/markup,409,:data,c:data
+documenting/markup,413,:func,c:func
+documenting/markup,417,:macro,c:macro
+documenting/markup,421,:type,c:type
+documenting/markup,426,:member,c:member
+documenting/markup,476,:file,... is installed in :file:`/usr/lib/python2.{x}/site-packages` ...
+documenting/markup,476,`,... is installed in :file:`/usr/lib/python2.{x}/site-packages` ...
+documenting/markup,495,:kbd,:kbd:`C-x C-f`
+documenting/markup,495,`,:kbd:`C-x C-f`
+documenting/markup,495,:kbd,:kbd:`Control-x Control-f`
+documenting/markup,495,`,:kbd:`Control-x Control-f`
+documenting/markup,509,:mailheader,:mailheader:`Content-Type`
+documenting/markup,509,`,:mailheader:`Content-Type`
+documenting/markup,518,:manpage,:manpage:`ls(1)`
+documenting/markup,518,`,:manpage:`ls(1)`
+documenting/markup,534,:menuselection,:menuselection:`Start --> Programs`
+documenting/markup,534,`,:menuselection:`Start --> Programs`
+documenting/markup,549,`,``code``
+documenting/markup,567,:file,:file:
+documenting/markup,567,`,``code``
+documenting/markup,602,:ref,:ref:`label-name`
+documenting/markup,602,`,:ref:`label-name`
+documenting/markup,606,:ref,"It refers to the section itself, see :ref:`my-reference-label`."
+documenting/markup,606,`,"It refers to the section itself, see :ref:`my-reference-label`."
+documenting/markup,615,:ref,:ref:
+documenting/markup,636,.. note:,.. note::
+documenting/markup,663,.. versionadded:,.. versionadded:: 3.1
+documenting/markup,688,::,.. impl-detail::
+documenting/markup,688,::,.. impl-detail:: This shortly mentions an implementation detail.
+documenting/markup,708,.. seealso:,.. seealso::
+documenting/markup,708,:mod,Module :mod:`zipfile`
+documenting/markup,708,`,Module :mod:`zipfile`
+documenting/markup,708,:mod,Documentation of the :mod:`zipfile` standard module.
+documenting/markup,708,`,Documentation of the :mod:`zipfile` standard module.
+documenting/markup,708,`,"`GNU tar manual, Basic Tar Format <http://link>`_"
+documenting/markup,722,.. centered:,.. centered::
+documenting/markup,767,.. toctree:,.. toctree::
+documenting/markup,767,:maxdepth,:maxdepth: 2
+documenting/markup,783,.. index:,.. index::
+documenting/markup,813,.. index:,".. index:: BNF, grammar, syntax, notation"
+documenting/markup,844,`,"unaryneg ::= ""-"" `integer`"
+documenting/markup,849,.. productionlist:,.. productionlist::
+documenting/markup,849,`,"try1_stmt: ""try"" "":"" `suite`"
+documenting/markup,849,`,": (""except"" [`expression` ["","" `target`]] "":"" `suite`)+"
+documenting/markup,849,`,": [""else"" "":"" `suite`]"
+documenting/markup,849,`,": [""finally"" "":"" `suite`]"
+documenting/markup,849,`,"try2_stmt: ""try"" "":"" `suite`"
+documenting/markup,849,`,": ""finally"" "":"" `suite`"
documenting/rest,33,`,``text``
documenting/rest,47,:rolename,:rolename:`content`
documenting/rest,47,`,:rolename:`content`
@@ -401,3 +494,25 @@ library/pprint,209,::,"'Programming Language :: Python :: 2.6',"
library/pprint,209,::,"'Programming Language :: Python :: 2.7',"
library/pprint,209,::,"'Topic :: Software Development :: Libraries',"
library/pprint,209,::,"'Topic :: Software Development :: Libraries :: Python Modules'],"
+packaging/examples,,`,This is the description of the ``foobar`` project.
+packaging/setupcfg,,::,Development Status :: 3 - Alpha
+packaging/setupcfg,,::,License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)
+packaging/setupscript,,::,"'Development Status :: 4 - Beta',"
+packaging/setupscript,,::,"'Environment :: Console',"
+packaging/setupscript,,::,"'Environment :: Web Environment',"
+packaging/setupscript,,::,"'Intended Audience :: End Users/Desktop',"
+packaging/setupscript,,::,"'Intended Audience :: Developers',"
+packaging/setupscript,,::,"'Intended Audience :: System Administrators',"
+packaging/setupscript,,::,"'License :: OSI Approved :: Python Software Foundation License',"
+packaging/setupscript,,::,"'Operating System :: MacOS :: MacOS X',"
+packaging/setupscript,,::,"'Operating System :: Microsoft :: Windows',"
+packaging/setupscript,,::,"'Operating System :: POSIX',"
+packaging/setupscript,,::,"'Programming Language :: Python',"
+packaging/setupscript,,::,"'Topic :: Communications :: Email',"
+packaging/setupscript,,::,"'Topic :: Office/Business',"
+packaging/setupscript,,::,"'Topic :: Software Development :: Bug Tracking',"
+packaging/tutorial,,::,1) License :: OSI Approved :: GNU General Public License (GPL)
+packaging/tutorial,,::,2) License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)
+packaging/tutorial,,::,Type the number of the license you wish to use or ? to try again:: 1
+packaging/tutorial,,::,classifier = Development Status :: 3 - Alpha
+packaging/tutorial,,::,License :: OSI Approved :: GNU General Public License (GPL)
diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst
index 6ee2e94..4926280 100644
--- a/Doc/tutorial/classes.rst
+++ b/Doc/tutorial/classes.rst
@@ -695,9 +695,9 @@ example, the following code will print B, C, D in that order::
class D(C):
pass
- for c in [B, C, D]:
+ for cls in [B, C, D]:
try:
- raise c()
+ raise cls()
except D:
print("D")
except C:
diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst
index defb47c..44c09c3 100644
--- a/Doc/tutorial/datastructures.rst
+++ b/Doc/tutorial/datastructures.rst
@@ -19,13 +19,13 @@ objects:
.. method:: list.append(x)
:noindex:
- Add an item to the end of the list; equivalent to ``a[len(a):] = [x]``.
+ Add an item to the end of the list. Equivalent to ``a[len(a):] = [x]``.
.. method:: list.extend(L)
:noindex:
- Extend the list by appending all the items in the given list; equivalent to
+ Extend the list by appending all the items in the given list. Equivalent to
``a[len(a):] = L``.
@@ -40,8 +40,8 @@ objects:
.. method:: list.remove(x)
:noindex:
- Remove the first item from the list whose value is *x*. It is an error if there
- is no such item.
+ Remove the first item from the list whose value is *x*. It is an error if
+ there is no such item.
.. method:: list.pop([i])
@@ -70,13 +70,14 @@ objects:
.. method:: list.sort()
:noindex:
- Sort the items of the list, in place.
+ Sort the items of the list in place.
.. method:: list.reverse()
:noindex:
- Reverse the elements of the list, in place.
+ Reverse the elements of the list in place.
+
An example that uses most of the list methods::
@@ -99,6 +100,10 @@ An example that uses most of the list methods::
>>> a
[-1, 1, 66.25, 333, 333, 1234.5]
+You might have noticed that methods like ``insert``, ``remove`` or ``sort`` that
+modify the list have no return value printed -- they return ``None``. [1]_ This
+is a design principle for all mutable data structures in Python.
+
.. _tut-lists-as-stacks:
@@ -438,7 +443,7 @@ using a non-existent key.
Performing ``list(d.keys())`` on a dictionary returns a list of all the keys
used in the dictionary, in arbitrary order (if you want it sorted, just use
-``sorted(d.keys())`` instead). [1]_ To check whether a single key is in the
+``sorted(d.keys())`` instead). [2]_ To check whether a single key is in the
dictionary, use the :keyword:`in` keyword.
Here is a small example using a dictionary::
@@ -622,6 +627,9 @@ interpreter will raise a :exc:`TypeError` exception.
.. rubric:: Footnotes
-.. [1] Calling ``d.keys()`` will return a :dfn:`dictionary view` object. It
+.. [1] Other languages may return the mutated object, which allows method
+ chaining, such as ``d->insert("a")->remove("b")->sort();``.
+
+.. [2] Calling ``d.keys()`` will return a :dfn:`dictionary view` object. It
supports operations like membership test and iteration, but its contents
are not independent of the original dictionary -- it is only a *view*.
diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst
index 2338465..8f08cd8 100644
--- a/Doc/tutorial/interpreter.rst
+++ b/Doc/tutorial/interpreter.rst
@@ -10,11 +10,11 @@ Using the Python Interpreter
Invoking the Interpreter
========================
-The Python interpreter is usually installed as :file:`/usr/local/bin/python3.2`
+The Python interpreter is usually installed as :file:`/usr/local/bin/python3.3`
on those machines where it is available; putting :file:`/usr/local/bin` in your
Unix shell's search path makes it possible to start it by typing the command ::
- python3.2
+ python3.3
to the shell. [#]_ Since the choice of the directory where the interpreter lives
is an installation option, other places are possible; check with your local
@@ -22,11 +22,11 @@ Python guru or system administrator. (E.g., :file:`/usr/local/python` is a
popular alternative location.)
On Windows machines, the Python installation is usually placed in
-:file:`C:\\Python32`, though you can change this when you're running the
+:file:`C:\\Python33`, though you can change this when you're running the
installer. To add this directory to your path, you can type the following
command into the command prompt in a DOS box::
- set path=%path%;C:\python32
+ set path=%path%;C:\python33
Typing an end-of-file character (:kbd:`Control-D` on Unix, :kbd:`Control-Z` on
Windows) at the primary prompt causes the interpreter to exit with a zero exit
@@ -93,8 +93,8 @@ with the *secondary prompt*, by default three dots (``...``). The interpreter
prints a welcome message stating its version number and a copyright notice
before printing the first prompt::
- $ python3.2
- Python 3.2 (py3k, Sep 12 2007, 12:21:02)
+ $ python3.3
+ Python 3.3 (py3k, Sep 12 2007, 12:21:02)
[GCC 3.4.6 20060404 (Red Hat 3.4.6-8)] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>>
@@ -147,7 +147,7 @@ Executable Python Scripts
On BSD'ish Unix systems, Python scripts can be made directly executable, like
shell scripts, by putting the line ::
- #! /usr/bin/env python3.2
+ #! /usr/bin/env python3.3
(assuming that the interpreter is on the user's :envvar:`PATH`) at the beginning
of the script and giving the file an executable mode. The ``#!`` must be the
diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst
index 9729743..500ca7f 100644
--- a/Doc/tutorial/stdlib.rst
+++ b/Doc/tutorial/stdlib.rst
@@ -15,7 +15,7 @@ operating system::
>>> import os
>>> os.getcwd() # Return the current working directory
- 'C:\\Python31'
+ 'C:\\Python33'
>>> os.chdir('/server/accesslogs') # Change current working directory
>>> os.system('mkdir today') # Run the command mkdir in the system shell
0
diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst
index fe7f027..85c88dc 100644
--- a/Doc/tutorial/stdlib2.rst
+++ b/Doc/tutorial/stdlib2.rst
@@ -141,7 +141,9 @@ standard size and in little-endian byte order::
import struct
- data = open('myfile.zip', 'rb').read()
+ with open('myfile.zip', 'rb') as f:
+ data = f.read()
+
start = 0
for i in range(3): # show the first 3 file headers
start += 14
@@ -271,7 +273,7 @@ applications include caching objects that are expensive to create::
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
d['primary'] # entry was automatically removed
- File "C:/python31/lib/weakref.py", line 46, in __getitem__
+ File "C:/python33/lib/weakref.py", line 46, in __getitem__
o = self.data[key]()
KeyError: 'primary'
diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst
index c73f1e0..bc38542 100644
--- a/Doc/using/cmdline.rst
+++ b/Doc/using/cmdline.rst
@@ -240,7 +240,9 @@ Miscellaneous options
.. cmdoption:: -S
Disable the import of the module :mod:`site` and the site-dependent
- manipulations of :data:`sys.path` that it entails.
+ manipulations of :data:`sys.path` that it entails. Also disable these
+ manipulations if :mod:`site` is explicitly imported later (call
+ :func:`site.main` if you want them to be triggered).
.. cmdoption:: -u
@@ -482,8 +484,8 @@ These environment variables influence Python's behavior.
Defines the :data:`user base directory <site.USER_BASE>`, which is used to
compute the path of the :data:`user site-packages directory <site.USER_SITE>`
- and :ref:`Distutils installation paths <inst-alt-install-user>` for ``python
- setup.py install --user``.
+ and :ref:`Packaging installation paths <packaging-alt-install-user>` for
+ ``pysetup run install_dist --user``.
.. seealso::
@@ -502,6 +504,14 @@ These environment variables influence Python's behavior.
separated string, it is equivalent to specifying :option:`-W` multiple
times.
+.. envvar:: PYTHONFAULTHANDLER
+
+ If this environment variable is set, :func:`faulthandler.enable` is called
+ at startup: install a handler for :const:`SIGSEGV`, :const:`SIGFPE`,
+ :const:`SIGABRT`, :const:`SIGBUS` and :const:`SIGILL` signals to dump the
+ Python traceback. This is equivalent to :option:`-X` ``faulthandler``
+ option.
+
Debug-mode variables
~~~~~~~~~~~~~~~~~~~~
diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst
index 0bd2c51..48751e8 100644
--- a/Doc/whatsnew/3.2.rst
+++ b/Doc/whatsnew/3.2.rst
@@ -270,7 +270,7 @@ launch of four parallel threads for copying files::
e.submit(shutil.copy, 'src1.txt', 'dest1.txt')
e.submit(shutil.copy, 'src2.txt', 'dest2.txt')
e.submit(shutil.copy, 'src3.txt', 'dest3.txt')
- e.submit(shutil.copy, 'src4.txt', 'dest4.txt')
+ e.submit(shutil.copy, 'src3.txt', 'dest4.txt')
.. seealso::
@@ -2354,7 +2354,7 @@ A number of small performance enhancements have been added:
(Contributed by Antoine Pitrou; :issue:`3001`.)
* The fast-search algorithm in stringlib is now used by the :meth:`split`,
- :meth:`splitlines` and :meth:`replace` methods on
+ :meth:`rsplit`, :meth:`splitlines` and :meth:`replace` methods on
:class:`bytes`, :class:`bytearray` and :class:`str` objects. Likewise, the
algorithm is also used by :meth:`rfind`, :meth:`rindex`, :meth:`rsplit` and
:meth:`rpartition`.
@@ -2471,14 +2471,14 @@ Code Repository
In addition to the existing Subversion code repository at http://svn.python.org
there is now a `Mercurial <http://mercurial.selenic.com/>`_ repository at
-http://hg.python.org/\.
+http://hg.python.org/ .
After the 3.2 release, there are plans to switch to Mercurial as the primary
repository. This distributed version control system should make it easier for
members of the community to create and share external changesets. See
:pep:`385` for details.
-To learn the new version control system, see the `tutorial by Joel
+To learn to use the new version control system, see the `tutorial by Joel
Spolsky <http://hginit.com>`_ or the `Guide to Mercurial Workflows
<http://mercurial.selenic.com/guide/>`_.
diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst
new file mode 100644
index 0000000..bfc5ed6
--- /dev/null
+++ b/Doc/whatsnew/3.3.rst
@@ -0,0 +1,597 @@
+****************************
+ What's New In Python 3.3
+****************************
+
+:Author: Raymond Hettinger
+:Release: |release|
+:Date: |today|
+
+.. Rules for maintenance:
+
+ * Anyone can add text to this document. Do not spend very much time
+ on the wording of your changes, because your text will probably
+ get rewritten to some degree.
+
+ * The maintainer will go through Misc/NEWS periodically and add
+ changes; it's therefore more important to add your changes to
+ Misc/NEWS than to this file.
+
+ * This is not a complete list of every single change; completeness
+ is the purpose of Misc/NEWS. Some changes I consider too small
+ or esoteric to include. If such a change is added to the text,
+ I'll just remove it. (This is another reason you shouldn't spend
+ too much time on writing your addition.)
+
+ * If you want to draw your new text to the attention of the
+ maintainer, add 'XXX' to the beginning of the paragraph or
+ section.
+
+ * It's OK to just add a fragmentary note about a change. For
+ example: "XXX Describe the transmogrify() function added to the
+ socket module." The maintainer will research the change and
+ write the necessary text.
+
+ * You can comment out your additions if you like, but it's not
+ necessary (especially when a final release is some months away).
+
+ * Credit the author of a patch or bugfix. Just the name is
+ sufficient; the e-mail address isn't necessary.
+
+ * It's helpful to add the bug/patch number as a comment:
+
+ XXX Describe the transmogrify() function added to the socket
+ module.
+ (Contributed by P.Y. Developer in :issue:`12345`.)
+
+ This saves the maintainer the effort of going through the Mercurial log
+ when researching a change.
+
+This article explains the new features in Python 3.3, compared to 3.2.
+
+
+.. _pep-393:
+
+PEP 393: Flexible String Representation
+=======================================
+
+The Unicode string type is changed to support multiple internal
+representations, depending on the character with the largest Unicode ordinal
+(1, 2, or 4 bytes) in the represented string. This allows a space-efficient
+representation in common cases, but gives access to full UCS-4 on all
+systems. For compatibility with existing APIs, several representations may
+exist in parallel; over time, this compatibility should be phased out.
+
+On the Python side, there should be no downside to this change.
+
+On the C API side, PEP 393 is fully backward compatible. The legacy API
+should remain available at least five years. Applications using the legacy
+API will not fully benefit of the memory reduction, or - worse - may use
+a bit more memory, because Python may have to maintain two versions of each
+string (in the legacy format and in the new efficient storage).
+
+Changes introduced by :pep:`393` are the following:
+
+* Python now always supports the full range of Unicode codepoints, including
+ non-BMP ones (i.e. from ``U+0000`` to ``U+10FFFF``). The distinction between
+ narrow and wide builds no longer exists and Python now behaves like a wide
+ build, even under Windows.
+
+* The storage of Unicode strings now depends on the highest codepoint in the string:
+
+ * pure ASCII and Latin1 strings (``U+0000-U+00FF``) use 1 byte per codepoint;
+
+ * BMP strings (``U+0000-U+FFFF``) use 2 bytes per codepoint;
+
+ * non-BMP strings (``U+10000-U+10FFFF``) use 4 bytes per codepoint.
+
+.. The memory usage of Python 3.3 is two to three times smaller than Python 3.2,
+ and a little bit better than Python 2.7, on a `Django benchmark
+ <http://mail.python.org/pipermail/python-dev/2011-September/113714.html>`_.
+ XXX The result should be moved in the PEP and a small summary about
+ performances and a link to the PEP should be added here.
+
+* With the death of narrow builds, the problems specific to narrow builds have
+ also been fixed, for example:
+
+ * :func:`len` now always returns 1 for non-BMP characters,
+ so ``len('\U0010FFFF') == 1``;
+
+ * surrogate pairs are not recombined in string literals,
+ so ``'\uDBFF\uDFFF' != '\U0010FFFF'``;
+
+ * indexing or slicing non-BMP characters returns the expected value,
+ so ``'\U0010FFFF'[0]`` now returns ``'\U0010FFFF'`` and not ``'\uDBFF'``;
+
+ * several other functions in the standard library now handle correctly
+ non-BMP codepoints.
+
+* The value of :data:`sys.maxunicode` is now always ``1114111`` (``0x10FFFF``
+ in hexadecimal). The :c:func:`PyUnicode_GetMax` function still returns
+ either ``0xFFFF`` or ``0x10FFFF`` for backward compatibility, and it should
+ not be used with the new Unicode API (see :issue:`13054`).
+
+* The :file:`./configure` flag ``--with-wide-unicode`` has been removed.
+
+XXX mention new and deprecated functions and macros
+
+
+PEP 3151: Reworking the OS and IO exception hierarchy
+=====================================================
+
+:pep:`3151` - Reworking the OS and IO exception hierarchy
+ PEP written and implemented by Antoine Pitrou.
+
+The hierarchy of exceptions raised by operating system errors is now both
+simplified and finer-grained.
+
+You don't have to worry anymore about choosing the appropriate exception
+type between :exc:`OSError`, :exc:`IOError`, :exc:`EnvironmentError`,
+:exc:`WindowsError`, :exc:`mmap.error`, :exc:`socket.error` or
+:exc:`select.error`. All these exception types are now only one:
+:exc:`OSError`. The other names are kept as aliases for compatibility
+reasons.
+
+Also, it is now easier to catch a specific error condition. Instead of
+inspecting the ``errno`` attribute (or ``args[0]``) for a particular
+constant from the :mod:`errno` module, you can catch the adequate
+:exc:`OSError` subclass. The available subclasses are the following:
+
+* :exc:`BlockingIOError`
+* :exc:`ChildProcessError`
+* :exc:`ConnectionError`
+* :exc:`FileExistsError`
+* :exc:`FileNotFoundError`
+* :exc:`InterruptedError`
+* :exc:`IsADirectoryError`
+* :exc:`NotADirectoryError`
+* :exc:`PermissionError`
+* :exc:`ProcessLookupError`
+* :exc:`TimeoutError`
+
+And the :exc:`ConnectionError` itself has finer-grained subclasses:
+
+* :exc:`BrokenPipeError`
+* :exc:`ConnectionAbortedError`
+* :exc:`ConnectionRefusedError`
+* :exc:`ConnectionResetError`
+
+Thanks to the new exceptions, common usages of the :mod:`errno` can now be
+avoided. For example, the following code written for Python 3.2::
+
+ from errno import ENOENT, EACCES, EPERM
+
+ try:
+ with open("document.txt") as f:
+ content = f.read()
+ except IOError as err:
+ if err.errno == ENOENT:
+ print("document.txt file is missing")
+ elif err.errno in (EACCES, EPERM):
+ print("You are not allowed to read document.txt")
+ else:
+ raise
+
+can now be written without the :mod:`errno` import and without manual
+inspection of exception attributes::
+
+ try:
+ with open("document.txt") as f:
+ content = f.read()
+ except FileNotFoundError:
+ print("document.txt file is missing")
+ except PermissionError:
+ print("You are not allowed to read document.txt")
+
+
+Other Language Changes
+======================
+
+Some smaller changes made to the core Python language are:
+
+* Stub
+
+Added support for Unicode name aliases and named sequences.
+Both :func:`unicodedata.lookup()` and ``'\N{...}'`` now resolve name aliases,
+and :func:`unicodedata.lookup()` resolves named sequences too.
+
+(Contributed by Ezio Melotti in :issue:`12753`)
+
+
+Equality comparisons on :func:`range` objects now return a result reflecting
+the equality of the underlying sequences generated by those range objects.
+
+(:issue:`13021`)
+
+
+New, Improved, and Deprecated Modules
+=====================================
+
+* Stub
+
+array
+-----
+
+The :mod:`array` module supports the :c:type:`long long` type using ``q`` and
+``Q`` type codes.
+
+(Contributed by Oren Tirosh and Hirokazu Yamamoto in :issue:`1172711`)
+
+
+codecs
+------
+
+The :mod:`~encodings.mbcs` codec has be rewritten to handle correclty
+``replace`` and ``ignore`` error handlers on all Windows versions. The
+:mod:`~encodings.mbcs` codec is now supporting all error handlers, instead of
+only ``replace`` to encode and ``ignore`` to decode.
+
+A new Windows-only codec has been added: ``cp65001`` (:issue:`13216`). It is
+the Windows code page 65001 (Windows UTF-8, ``CP_UTF8``). For example, it is
+used by ``sys.stdout`` if the console output code page is set to cp65001 (e.g.
+using ``chcp 65001`` command).
+
+Multibyte CJK decoders now resynchronize faster. They only ignore the first
+byte of an invalid byte sequence. For example, ``b'\xff\n'.decode('gb2312',
+'replace')`` now returns a ``\n`` after the replacement character.
+
+(:issue:`12016`)
+
+Don't reset incremental encoders of CJK codecs at each call to their encode()
+method anymore. For example::
+
+ $ ./python -q
+ >>> import codecs
+ >>> encoder = codecs.getincrementalencoder('hz')('strict')
+ >>> b''.join(encoder.encode(x) for x in '\u52ff\u65bd\u65bc\u4eba\u3002 Bye.')
+ b'~{NpJ)l6HK!#~} Bye.'
+
+This example gives ``b'~{Np~}~{J)~}~{l6~}~{HK~}~{!#~} Bye.'`` with older Python
+versions.
+
+(:issue:`12100`)
+
+crypt
+-----
+
+Addition of salt and modular crypt format and the :func:`~crypt.mksalt`
+function to the :mod:`crypt` module.
+
+(:issue:`10924`)
+
+curses
+------
+
+ * The :class:`curses.window` class has a new :meth:`~curses.window.get_wch`
+ method to get a wide character
+ * The :mod:`curses` module has a new :meth:`~curses.unget_wch` function to
+ push a wide character so the next :meth:`~curses.window.get_wch` will return
+ it
+
+(Contributed by Iñigo Serna in :issue:`6755`)
+
+faulthandler
+------------
+
+New module: :mod:`faulthandler`.
+
+ * :envvar:`PYTHONFAULTHANDLER`
+ * :option:`-X` ``faulthandler``
+
+time
+----
+
+* The :mod:`time` module has new :func:`~time.clock_getres` and
+ :func:`~time.clock_gettime` functions and ``CLOCK_xxx`` constants.
+ :func:`~time.clock_gettime` can be used with :data:`time.CLOCK_MONOTONIC` to
+ get a monotonic clock.
+
+ (Contributed by Victor Stinner in :issue:`10278`)
+
+
+ftplib
+------
+
+The :class:`~ftplib.FTP_TLS` class now provides a new
+:func:`~ftplib.FTP_TLS.ccc` function to revert control channel back to
+plaintext. This can be useful to take advantage of firewalls that know how to
+handle NAT with non-secure FTP without opening fixed ports.
+
+(Contributed by Giampaolo Rodolà in :issue:`12139`)
+
+
+math
+----
+
+The :mod:`math` module has a new function:
+
+ * :func:`~math.log2`: return the base-2 logarithm of *x*
+ (Written by Mark Dickinson in :issue:`11888`).
+
+
+nntplib
+-------
+
+The :class:`nntplib.NNTP` class now supports the context manager protocol to
+unconditionally consume :exc:`socket.error` exceptions and to close the NNTP
+connection when done::
+
+ >>> from nntplib import NNTP
+ >>> with NNTP('news.gmane.org') as n:
+ ... n.group('gmane.comp.python.committers')
+ ...
+ ('211 1755 1 1755 gmane.comp.python.committers', 1755, 1, 1755, 'gmane.comp.python.committers')
+ >>>
+
+(Contributed by Giampaolo Rodolà in :issue:`9795`)
+
+
+os
+--
+
+* The :mod:`os` module has a new :func:`~os.pipe2` function that makes it
+ possible to create a pipe with :data:`~os.O_CLOEXEC` or
+ :data:`~os.O_NONBLOCK` flags set atomically. This is especially useful to
+ avoid race conditions in multi-threaded programs.
+
+* The :mod:`os` module has a new :func:`~os.sendfile` function which provides
+ an efficent "zero-copy" way for copying data from one file (or socket)
+ descriptor to another. The phrase "zero-copy" refers to the fact that all of
+ the copying of data between the two descriptors is done entirely by the
+ kernel, with no copying of data into userspace buffers. :func:`~os.sendfile`
+ can be used to efficiently copy data from a file on disk to a network socket,
+ e.g. for downloading a file.
+
+ (Patch submitted by Ross Lagerwall and Giampaolo Rodolà in :issue:`10882`.)
+
+* The :mod:`os` module has two new functions: :func:`~os.getpriority` and
+ :func:`~os.setpriority`. They can be used to get or set process
+ niceness/priority in a fashion similar to :func:`os.nice` but extended to all
+ processes instead of just the current one.
+
+ (Patch submitted by Giampaolo Rodolà in :issue:`10784`.)
+
+* "at" functions (:issue:`4761`):
+
+ * :func:`~os.faccessat`
+ * :func:`~os.fchmodat`
+ * :func:`~os.fchownat`
+ * :func:`~os.fstatat`
+ * :func:`~os.futimesat`
+ * :func:`~os.futimesat`
+ * :func:`~os.linkat`
+ * :func:`~os.mkdirat`
+ * :func:`~os.mkfifoat`
+ * :func:`~os.mknodat`
+ * :func:`~os.openat`
+ * :func:`~os.readlinkat`
+ * :func:`~os.renameat`
+ * :func:`~os.symlinkat`
+ * :func:`~os.unlinkat`
+ * :func:`~os.utimensat`
+ * :func:`~os.utimensat`
+
+* extended attributes (:issue:`12720`):
+
+ * :func:`~os.fgetxattr`
+ * :func:`~os.flistxattr`
+ * :func:`~os.fremovexattr`
+ * :func:`~os.fsetxattr`
+ * :func:`~os.getxattr`
+ * :func:`~os.lgetxattr`
+ * :func:`~os.listxattr`
+ * :func:`~os.llistxattr`
+ * :func:`~os.lremovexattr`
+ * :func:`~os.lsetxattr`
+ * :func:`~os.removexattr`
+ * :func:`~os.setxattr`
+
+* Scheduler functions (:issue:`12655`):
+
+ * :func:`~os.sched_get_priority_max`
+ * :func:`~os.sched_get_priority_min`
+ * :func:`~os.sched_getaffinity`
+ * :func:`~os.sched_getparam`
+ * :func:`~os.sched_getscheduler`
+ * :func:`~os.sched_rr_get_interval`
+ * :func:`~os.sched_setaffinity`
+ * :func:`~os.sched_setparam`
+ * :func:`~os.sched_setscheduler`
+ * :func:`~os.sched_yield`
+
+* Add some extra posix functions to the os module (:issue:`10812`):
+
+ * :func:`~os.fexecve`
+ * :func:`~os.futimens`
+ * :func:`~os.futimens`
+ * :func:`~os.futimes`
+ * :func:`~os.futimes`
+ * :func:`~os.lockf`
+ * :func:`~os.lutimes`
+ * :func:`~os.lutimes`
+ * :func:`~os.posix_fadvise`
+ * :func:`~os.posix_fallocate`
+ * :func:`~os.pread`
+ * :func:`~os.pwrite`
+ * :func:`~os.readv`
+ * :func:`~os.sync`
+ * :func:`~os.truncate`
+ * :func:`~os.waitid`
+ * :func:`~os.writev`
+
+* Other new functions:
+
+ * :func:`~os.fdlistdir` (:issue:`10755`)
+ * :func:`~os.getgrouplist` (:issue:`9344`)
+
+
+packaging
+---------
+
+:mod:`distutils` has undergone additions and refactoring under a new name,
+:mod:`packaging`, to allow developers to break backward compatibility.
+:mod:`distutils` is still provided in the standard library, but users are
+encouraged to transition to :mod:`packaging`. For older versions of Python, a
+backport compatible with 2.4+ and 3.1+ will be made available on PyPI under the
+name :mod:`distutils2`.
+
+.. TODO add examples and howto to the packaging docs and link to them
+
+
+pydoc
+-----
+
+The Tk GUI and the :func:`~pydoc.serve` function have been removed from the
+:mod:`pydoc` module: ``pydoc -g`` and :func:`~pydoc.serve` have been deprecated
+in Python 3.2.
+
+
+sys
+---
+
+* The :mod:`sys` module has a new :data:`~sys.thread_info` :term:`struct
+ sequence` holding informations about the thread implementation.
+
+ (:issue:`11223`)
+
+
+signal
+------
+
+* The :mod:`signal` module has new functions:
+
+ * :func:`~signal.pthread_sigmask`: fetch and/or change the signal mask of the
+ calling thread (Contributed by Jean-Paul Calderone in :issue:`8407`) ;
+ * :func:`~signal.pthread_kill`: send a signal to a thread ;
+ * :func:`~signal.sigpending`: examine pending functions ;
+ * :func:`~signal.sigwait`: wait a signal.
+ * :func:`~signal.sigwaitinfo`: wait for a signal, returning detailed
+ information about it.
+ * :func:`~signal.sigtimedwait`: like :func:`~signal.sigwaitinfo` but with a
+ timeout.
+
+* The signal handler writes the signal number as a single byte instead of
+ a nul byte into the wakeup file descriptor. So it is possible to wait more
+ than one signal and know which signals were raised.
+
+* :func:`signal.signal` and :func:`signal.siginterrupt` raise an OSError,
+ instead of a RuntimeError: OSError has an errno attribute.
+
+socket
+------
+
+* The :class:`~socket.socket` class now exposes additional methods to process
+ ancillary data when supported by the underlying platform:
+
+ * :func:`~socket.socket.sendmsg`
+ * :func:`~socket.socket.recvmsg`
+ * :func:`~socket.socket.recvmsg_into`
+
+ (Contributed by David Watson in :issue:`6560`, based on an earlier patch by
+ Heiko Wundram)
+
+* The :class:`~socket.socket` class now supports the PF_CAN protocol family
+ (http://en.wikipedia.org/wiki/Socketcan), on Linux
+ (http://lwn.net/Articles/253425).
+
+ (Contributed by Matthias Fuchs, updated by Tiago Gonçalves in :issue:`10141`)
+
+
+ssl
+---
+
+The :mod:`ssl` module has new functions:
+
+ * :func:`~ssl.RAND_bytes`: generate cryptographically strong
+ pseudo-random bytes.
+ * :func:`~ssl.RAND_pseudo_bytes`: generate pseudo-random bytes.
+
+
+shutil
+------
+
+* The :mod:`shutil` module has these new fuctions:
+
+ * :func:`~shutil.disk_usage`: provides total, used and free disk space
+ statistics. (Contributed by Giampaolo Rodolà in :issue:`12442`)
+ * :func:`~shutil.chown`: allows one to change user and/or group of the given
+ path also specifying the user/group names and not only their numeric
+ ids. (Contributed by Sandro Tosi in :issue:`12191`)
+
+urllib
+------
+
+The :class:`~urllib.request.Request` class, now accepts a *method* argument
+used by :meth:`~urllib.request.Request.get_method` to determine what HTTP method
+should be used. For example, this will send a ``'HEAD'`` request::
+
+ >>> urlopen(Request('http://www.python.org', method='HEAD'))
+
+(:issue:`1673007`)
+
+Optimizations
+=============
+
+Major performance enhancements have been added:
+
+* Stub
+
+
+Build and C API Changes
+=======================
+
+Changes to Python's build process and to the C API include:
+
+* Stub
+
+
+Unsupported Operating Systems
+=============================
+
+OS/2 and VMS are no longer supported due to the lack of a maintainer.
+
+Windows 2000 and Windows platforms which set ``COMSPEC`` to ``command.com``
+are no longer supported due to maintenance burden.
+
+
+Porting to Python 3.3
+=====================
+
+This section lists previously described changes and other bugfixes
+that may require changes to your code.
+
+Porting Python code
+-------------------
+
+* Issue #12326: On Linux, sys.platform doesn't contain the major version
+ anymore. It is now always 'linux', instead of 'linux2' or 'linux3' depending
+ on the Linux version used to build Python. Replace sys.platform == 'linux2'
+ with sys.platform.startswith('linux'), or directly sys.platform == 'linux' if
+ you don't need to support older Python versions.
+
+Porting C code
+--------------
+
+* Due to :ref:`PEP 393 <pep-393>`, the :c:type:`Py_UNICODE` type and all
+ functions using this type are deprecated (but will stay available for
+ at least five years). If you were using low-level Unicode APIs to
+ construct and access unicode objects and you want to benefit of the
+ memory footprint reduction provided by the PEP 393, you have to convert
+ your code to the new :doc:`Unicode API <../c-api/unicode>`.
+
+ However, if you only have been using high-level functions such as
+ :c:func:`PyUnicode_Concat()`, :c:func:`PyUnicode_Join` or
+ :c:func:`PyUnicode_FromFormat()`, your code will automatically take
+ advantage of the new unicode representations.
+
+Other issues
+------------
+
+.. Issue #11591: When :program:`python` was started with :option:`-S`,
+ ``import site`` will not add site-specific paths to the module search
+ paths. In previous versions, it did. See changeset for doc changes in
+ various files. Contributed by Carl Meyer with editions by Éric Araujo.
+
+.. Issue #10998: -Q command-line flags are related artifacts have been
+ removed. Code checking sys.flags.division_warning will need updating.
+ Contributed by Éric Araujo.
+
diff --git a/Doc/whatsnew/index.rst b/Doc/whatsnew/index.rst
index 8220bd2..c60818a 100644
--- a/Doc/whatsnew/index.rst
+++ b/Doc/whatsnew/index.rst
@@ -11,6 +11,7 @@ anyone wishing to stay up-to-date after a new release.
.. toctree::
:maxdepth: 2
+ 3.3.rst
3.2.rst
3.1.rst
3.0.rst
diff --git a/Include/Python-ast.h b/Include/Python-ast.h
index 0ad788b..9389049 100644
--- a/Include/Python-ast.h
+++ b/Include/Python-ast.h
@@ -36,6 +36,8 @@ typedef struct _keyword *keyword_ty;
typedef struct _alias *alias_ty;
+typedef struct _withitem *withitem_ty;
+
enum _mod_kind {Module_kind=1, Interactive_kind=2, Expression_kind=3,
Suite_kind=4};
@@ -64,10 +66,9 @@ struct _mod {
enum _stmt_kind {FunctionDef_kind=1, ClassDef_kind=2, Return_kind=3,
Delete_kind=4, Assign_kind=5, AugAssign_kind=6, For_kind=7,
While_kind=8, If_kind=9, With_kind=10, Raise_kind=11,
- TryExcept_kind=12, TryFinally_kind=13, Assert_kind=14,
- Import_kind=15, ImportFrom_kind=16, Global_kind=17,
- Nonlocal_kind=18, Expr_kind=19, Pass_kind=20, Break_kind=21,
- Continue_kind=22};
+ Try_kind=12, Assert_kind=13, Import_kind=14,
+ ImportFrom_kind=15, Global_kind=16, Nonlocal_kind=17,
+ Expr_kind=18, Pass_kind=19, Break_kind=20, Continue_kind=21};
struct _stmt {
enum _stmt_kind kind;
union {
@@ -128,8 +129,7 @@ struct _stmt {
} If;
struct {
- expr_ty context_expr;
- expr_ty optional_vars;
+ asdl_seq *items;
asdl_seq *body;
} With;
@@ -142,12 +142,8 @@ struct _stmt {
asdl_seq *body;
asdl_seq *handlers;
asdl_seq *orelse;
- } TryExcept;
-
- struct {
- asdl_seq *body;
asdl_seq *finalbody;
- } TryFinally;
+ } Try;
struct {
expr_ty test;
@@ -275,7 +271,7 @@ struct _expr {
} Str;
struct {
- string s;
+ bytes s;
} Bytes;
struct {
@@ -383,6 +379,11 @@ struct _alias {
identifier asname;
};
+struct _withitem {
+ expr_ty context_expr;
+ expr_ty optional_vars;
+};
+
#define Module(a0, a1) _Py_Module(a0, a1)
mod_ty _Py_Module(asdl_seq * body, PyArena *arena);
@@ -421,18 +422,16 @@ stmt_ty _Py_While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
#define If(a0, a1, a2, a3, a4, a5) _Py_If(a0, a1, a2, a3, a4, a5)
stmt_ty _Py_If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
int col_offset, PyArena *arena);
-#define With(a0, a1, a2, a3, a4, a5) _Py_With(a0, a1, a2, a3, a4, a5)
-stmt_ty _Py_With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body,
- int lineno, int col_offset, PyArena *arena);
+#define With(a0, a1, a2, a3, a4) _Py_With(a0, a1, a2, a3, a4)
+stmt_ty _Py_With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset,
+ PyArena *arena);
#define Raise(a0, a1, a2, a3, a4) _Py_Raise(a0, a1, a2, a3, a4)
stmt_ty _Py_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset,
PyArena *arena);
-#define TryExcept(a0, a1, a2, a3, a4, a5) _Py_TryExcept(a0, a1, a2, a3, a4, a5)
-stmt_ty _Py_TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse,
- int lineno, int col_offset, PyArena *arena);
-#define TryFinally(a0, a1, a2, a3, a4) _Py_TryFinally(a0, a1, a2, a3, a4)
-stmt_ty _Py_TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, int
- col_offset, PyArena *arena);
+#define Try(a0, a1, a2, a3, a4, a5, a6) _Py_Try(a0, a1, a2, a3, a4, a5, a6)
+stmt_ty _Py_Try(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse,
+ asdl_seq * finalbody, int lineno, int col_offset, PyArena
+ *arena);
#define Assert(a0, a1, a2, a3, a4) _Py_Assert(a0, a1, a2, a3, a4)
stmt_ty _Py_Assert(expr_ty test, expr_ty msg, int lineno, int col_offset,
PyArena *arena);
@@ -502,7 +501,7 @@ expr_ty _Py_Num(object n, int lineno, int col_offset, PyArena *arena);
#define Str(a0, a1, a2, a3) _Py_Str(a0, a1, a2, a3)
expr_ty _Py_Str(string s, int lineno, int col_offset, PyArena *arena);
#define Bytes(a0, a1, a2, a3) _Py_Bytes(a0, a1, a2, a3)
-expr_ty _Py_Bytes(string s, int lineno, int col_offset, PyArena *arena);
+expr_ty _Py_Bytes(bytes s, int lineno, int col_offset, PyArena *arena);
#define Ellipsis(a0, a1, a2) _Py_Ellipsis(a0, a1, a2)
expr_ty _Py_Ellipsis(int lineno, int col_offset, PyArena *arena);
#define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5)
@@ -547,6 +546,9 @@ arg_ty _Py_arg(identifier arg, expr_ty annotation, PyArena *arena);
keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena);
#define alias(a0, a1, a2) _Py_alias(a0, a1, a2)
alias_ty _Py_alias(identifier name, identifier asname, PyArena *arena);
+#define withitem(a0, a1, a2) _Py_withitem(a0, a1, a2)
+withitem_ty _Py_withitem(expr_ty context_expr, expr_ty optional_vars, PyArena
+ *arena);
PyObject* PyAST_mod2obj(mod_ty t);
mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);
diff --git a/Include/Python.h b/Include/Python.h
index 5972ffa..01b98f9 100644
--- a/Include/Python.h
+++ b/Include/Python.h
@@ -48,6 +48,7 @@
#include <assert.h>
#include "pyport.h"
+#include "pymacro.h"
#include "pyatomic.h"
@@ -126,43 +127,6 @@
#include "pystrcmp.h"
#include "dtoa.h"
#include "fileutils.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* _Py_Mangle is defined in compile.c */
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name);
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-/* Argument must be a char or an int in [-128, 127] or [0, 255]. */
-#define Py_CHARMASK(c) ((unsigned char)((c) & 0xff))
-
#include "pyfpe.h"
-/* These definitions must match corresponding definitions in graminit.h.
- There's code in compile.c that checks that they are the same. */
-#define Py_single_input 256
-#define Py_file_input 257
-#define Py_eval_input 258
-
-#ifdef HAVE_PTH
-/* GNU pth user-space thread support */
-#include <pth.h>
-#endif
-
-/* Define macros for inline documentation. */
-#define PyDoc_VAR(name) static char name[]
-#define PyDoc_STRVAR(name,str) PyDoc_VAR(name) = PyDoc_STR(str)
-#ifdef WITH_DOC_STRINGS
-#define PyDoc_STR(str) str
-#else
-#define PyDoc_STR(str) ""
-#endif
-
#endif /* !Py_PYTHON_H */
diff --git a/Include/abstract.h b/Include/abstract.h
index 0fe0956..3946ec5 100644
--- a/Include/abstract.h
+++ b/Include/abstract.h
@@ -7,6 +7,7 @@ extern "C" {
#ifdef PY_SSIZE_T_CLEAN
#define PyObject_CallFunction _PyObject_CallFunction_SizeT
#define PyObject_CallMethod _PyObject_CallMethod_SizeT
+#define _PyObject_CallMethodId _PyObject_CallMethodId_SizeT
#endif
/* Abstract Object Interface (many thanks to Jim Fulton) */
@@ -307,11 +308,22 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
Python expression: o.method(args).
*/
+ PyAPI_FUNC(PyObject *) _PyObject_CallMethodId(PyObject *o, _Py_Identifier *method,
+ char *format, ...);
+
+ /*
+ Like PyObject_CallMethod, but expect a _Py_Identifier* as the
+ method name.
+ */
+
PyAPI_FUNC(PyObject *) _PyObject_CallFunction_SizeT(PyObject *callable,
char *format, ...);
PyAPI_FUNC(PyObject *) _PyObject_CallMethod_SizeT(PyObject *o,
char *name,
char *format, ...);
+ PyAPI_FUNC(PyObject *) _PyObject_CallMethodId_SizeT(PyObject *o,
+ _Py_Identifier *name,
+ char *format, ...);
PyAPI_FUNC(PyObject *) PyObject_CallFunctionObjArgs(PyObject *callable,
...);
diff --git a/Include/asdl.h b/Include/asdl.h
index 9bb0697..42bbbf8 100644
--- a/Include/asdl.h
+++ b/Include/asdl.h
@@ -3,6 +3,7 @@
typedef PyObject * identifier;
typedef PyObject * string;
+typedef PyObject * bytes;
typedef PyObject * object;
/* It would be nice if the code generated by asdl_c.py was completely
diff --git a/Include/ast.h b/Include/ast.h
index a015336..055e8dc 100644
--- a/Include/ast.h
+++ b/Include/ast.h
@@ -4,6 +4,7 @@
extern "C" {
#endif
+PyAPI_FUNC(int) PyAST_Validate(mod_ty);
PyAPI_FUNC(mod_ty) PyAST_FromNode(
const node *n,
PyCompilerFlags *flags,
diff --git a/Include/code.h b/Include/code.h
index e773b6a..7c7e5bf 100644
--- a/Include/code.h
+++ b/Include/code.h
@@ -22,6 +22,7 @@ typedef struct {
PyObject *co_freevars; /* tuple of strings (free variable names) */
PyObject *co_cellvars; /* tuple of strings (cell variable names) */
/* The rest doesn't count for hash or comparisons */
+ unsigned char *co_cell2arg; /* Maps cell vars which are arguments. */
PyObject *co_filename; /* unicode (where it was loaded from) */
PyObject *co_name; /* unicode (name, for reference) */
int co_firstlineno; /* first source line number */
@@ -57,6 +58,11 @@ typedef struct {
#define CO_FUTURE_BARRY_AS_BDFL 0x40000
+/* This value is found in the co_cell2arg array when the associated cell
+ variable does not correspond to an argument. The maximum number of
+ arguments is 255 (indexed up to 254), so 255 work as a special flag.*/
+#define CO_CELL_NOT_AN_ARG 255
+
/* This should be defined if a future statement modifies the syntax.
For example, when a keyword is added.
*/
diff --git a/Include/codecs.h b/Include/codecs.h
index dff09e7..0d9e9b4 100644
--- a/Include/codecs.h
+++ b/Include/codecs.h
@@ -174,6 +174,8 @@ PyAPI_FUNC(PyObject *) PyCodec_XMLCharRefReplaceErrors(PyObject *exc);
/* replace the unicode encode error with backslash escapes (\x, \u and \U) */
PyAPI_FUNC(PyObject *) PyCodec_BackslashReplaceErrors(PyObject *exc);
+PyAPI_DATA(const char *) Py_hexdigits;
+
#ifdef __cplusplus
}
#endif
diff --git a/Include/compile.h b/Include/compile.h
index bc53b39..ac2636d 100644
--- a/Include/compile.h
+++ b/Include/compile.h
@@ -1,7 +1,7 @@
-#ifndef Py_LIMITED_API
#ifndef Py_COMPILE_H
#define Py_COMPILE_H
+#ifndef Py_LIMITED_API
#include "code.h"
#ifdef __cplusplus
@@ -38,9 +38,19 @@ PyAPI_FUNC(PyCodeObject *) PyAST_CompileEx(
PyArena *arena);
PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *);
+/* _Py_Mangle is defined in compile.c */
+PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name);
#ifdef __cplusplus
}
#endif
-#endif /* !Py_COMPILE_H */
+
#endif /* !Py_LIMITED_API */
+
+/* These definitions must match corresponding definitions in graminit.h.
+ There's code in compile.c that checks that they are the same. */
+#define Py_single_input 256
+#define Py_file_input 257
+#define Py_eval_input 258
+
+#endif /* !Py_COMPILE_H */
diff --git a/Include/complexobject.h b/Include/complexobject.h
index c379b08..3e4ecff 100644
--- a/Include/complexobject.h
+++ b/Include/complexobject.h
@@ -64,8 +64,9 @@ PyAPI_FUNC(Py_complex) PyComplex_AsCComplex(PyObject *op);
(Advanced String Formatting). */
#ifndef Py_LIMITED_API
PyAPI_FUNC(PyObject *) _PyComplex_FormatAdvanced(PyObject *obj,
- Py_UNICODE *format_spec,
- Py_ssize_t format_spec_len);
+ PyObject *format_spec,
+ Py_ssize_t start,
+ Py_ssize_t end);
#endif
#ifdef __cplusplus
diff --git a/Include/floatobject.h b/Include/floatobject.h
index 90f0a45..0ca4881 100644
--- a/Include/floatobject.h
+++ b/Include/floatobject.h
@@ -27,12 +27,12 @@ PyAPI_DATA(PyTypeObject) PyFloat_Type;
#define Py_RETURN_NAN return PyFloat_FromDouble(Py_NAN)
#endif
-#define Py_RETURN_INF(sign) do \
- if (copysign(1., sign) == 1.) { \
- return PyFloat_FromDouble(Py_HUGE_VAL); \
- } else { \
- return PyFloat_FromDouble(-Py_HUGE_VAL); \
- } while(0)
+#define Py_RETURN_INF(sign) do \
+ if (copysign(1., sign) == 1.) { \
+ return PyFloat_FromDouble(Py_HUGE_VAL); \
+ } else { \
+ return PyFloat_FromDouble(-Py_HUGE_VAL); \
+ } while(0)
PyAPI_FUNC(double) PyFloat_GetMax(void);
PyAPI_FUNC(double) PyFloat_GetMin(void);
@@ -113,8 +113,9 @@ PyAPI_FUNC(int) PyFloat_ClearFreeList(void);
/* Format the object based on the format_spec, as defined in PEP 3101
(Advanced String Formatting). */
PyAPI_FUNC(PyObject *) _PyFloat_FormatAdvanced(PyObject *obj,
- Py_UNICODE *format_spec,
- Py_ssize_t format_spec_len);
+ PyObject *format_spec,
+ Py_ssize_t start,
+ Py_ssize_t end);
#endif /* Py_LIMITED_API */
#ifdef __cplusplus
diff --git a/Include/import.h b/Include/import.h
index 400e97c..45544111 100644
--- a/Include/import.h
+++ b/Include/import.h
@@ -24,7 +24,16 @@ PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleWithPathnames(
char *pathname, /* decoded from the filesystem encoding */
char *cpathname /* decoded from the filesystem encoding */
);
+PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject(
+ PyObject *name,
+ PyObject *co,
+ PyObject *pathname,
+ PyObject *cpathname
+ );
PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void);
+PyAPI_FUNC(PyObject *) PyImport_AddModuleObject(
+ PyObject *name
+ );
PyAPI_FUNC(PyObject *) PyImport_AddModule(
const char *name /* UTF-8 encoded string */
);
@@ -35,7 +44,14 @@ PyAPI_FUNC(PyObject *) PyImport_ImportModuleNoBlock(
const char *name /* UTF-8 encoded string */
);
PyAPI_FUNC(PyObject *) PyImport_ImportModuleLevel(
- char *name, /* UTF-8 encoded string */
+ const char *name, /* UTF-8 encoded string */
+ PyObject *globals,
+ PyObject *locals,
+ PyObject *fromlist,
+ int level
+ );
+PyAPI_FUNC(PyObject *) PyImport_ImportModuleLevelObject(
+ PyObject *name,
PyObject *globals,
PyObject *locals,
PyObject *fromlist,
@@ -49,6 +65,9 @@ PyAPI_FUNC(PyObject *) PyImport_GetImporter(PyObject *path);
PyAPI_FUNC(PyObject *) PyImport_Import(PyObject *name);
PyAPI_FUNC(PyObject *) PyImport_ReloadModule(PyObject *m);
PyAPI_FUNC(void) PyImport_Cleanup(void);
+PyAPI_FUNC(int) PyImport_ImportFrozenModuleObject(
+ PyObject *name
+ );
PyAPI_FUNC(int) PyImport_ImportFrozenModule(
char *name /* UTF-8 encoded string */
);
@@ -65,17 +84,17 @@ PyAPI_FUNC(int) _PyImport_ReleaseLock(void);
PyAPI_FUNC(void) _PyImport_ReInitLock(void);
PyAPI_FUNC(PyObject *)_PyImport_FindBuiltin(
- char *name /* UTF-8 encoded string */
+ const char *name /* UTF-8 encoded string */
);
-PyAPI_FUNC(PyObject *)_PyImport_FindExtensionUnicode(char *, PyObject *);
+PyAPI_FUNC(PyObject *)_PyImport_FindExtensionObject(PyObject *, PyObject *);
PyAPI_FUNC(int)_PyImport_FixupBuiltin(
PyObject *mod,
char *name /* UTF-8 encoded string */
);
-PyAPI_FUNC(int)_PyImport_FixupExtensionUnicode(PyObject*, char *, PyObject *);
+PyAPI_FUNC(int)_PyImport_FixupExtensionObject(PyObject*, PyObject *, PyObject *);
struct _inittab {
- char *name;
+ char *name; /* ASCII encoded string */
PyObject* (*initfunc)(void);
};
PyAPI_DATA(struct _inittab *) PyImport_Inittab;
diff --git a/Include/longobject.h b/Include/longobject.h
index c09565a..c58ddf4 100644
--- a/Include/longobject.h
+++ b/Include/longobject.h
@@ -12,7 +12,7 @@ typedef struct _longobject PyLongObject; /* Revealed in longintrepr.h */
PyAPI_DATA(PyTypeObject) PyLong_Type;
#define PyLong_Check(op) \
- PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_LONG_SUBCLASS)
+ PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_LONG_SUBCLASS)
#define PyLong_CheckExact(op) (Py_TYPE(op) == &PyLong_Type)
PyAPI_FUNC(PyObject *) PyLong_FromLong(long);
@@ -80,6 +80,7 @@ PyAPI_FUNC(PY_LONG_LONG) PyLong_AsLongLongAndOverflow(PyObject *, int *);
PyAPI_FUNC(PyObject *) PyLong_FromString(char *, char **, int);
#ifndef Py_LIMITED_API
PyAPI_FUNC(PyObject *) PyLong_FromUnicode(Py_UNICODE*, Py_ssize_t, int);
+PyAPI_FUNC(PyObject *) PyLong_FromUnicodeObject(PyObject *u, int base);
#endif
#ifndef Py_LIMITED_API
@@ -121,8 +122,8 @@ PyAPI_FUNC(PyObject *) _PyLong_DivmodNear(PyObject *, PyObject *);
enough memory to create the Python long.
*/
PyAPI_FUNC(PyObject *) _PyLong_FromByteArray(
- const unsigned char* bytes, size_t n,
- int little_endian, int is_signed);
+ const unsigned char* bytes, size_t n,
+ int little_endian, int is_signed);
/* _PyLong_AsByteArray: Convert the least-significant 8*n bits of long
v to a base-256 integer, stored in array bytes. Normally return 0,
@@ -144,8 +145,8 @@ PyAPI_FUNC(PyObject *) _PyLong_FromByteArray(
case, but bytes holds the least-signficant n bytes of the true value.
*/
PyAPI_FUNC(int) _PyLong_AsByteArray(PyLongObject* v,
- unsigned char* bytes, size_t n,
- int little_endian, int is_signed);
+ unsigned char* bytes, size_t n,
+ int little_endian, int is_signed);
/* _PyLong_Format: Convert the long to a string object with given base,
@@ -155,8 +156,9 @@ PyAPI_FUNC(PyObject *) _PyLong_Format(PyObject *aa, int base);
/* Format the object based on the format_spec, as defined in PEP 3101
(Advanced String Formatting). */
PyAPI_FUNC(PyObject *) _PyLong_FormatAdvanced(PyObject *obj,
- Py_UNICODE *format_spec,
- Py_ssize_t format_spec_len);
+ PyObject *format_spec,
+ Py_ssize_t start,
+ Py_ssize_t end);
#endif /* Py_LIMITED_API */
/* These aren't really part of the long object, but they're handy. The
diff --git a/Include/moduleobject.h b/Include/moduleobject.h
index 7b2bf1c..8013dd9 100644
--- a/Include/moduleobject.h
+++ b/Include/moduleobject.h
@@ -12,10 +12,14 @@ PyAPI_DATA(PyTypeObject) PyModule_Type;
#define PyModule_Check(op) PyObject_TypeCheck(op, &PyModule_Type)
#define PyModule_CheckExact(op) (Py_TYPE(op) == &PyModule_Type)
+PyAPI_FUNC(PyObject *) PyModule_NewObject(
+ PyObject *name
+ );
PyAPI_FUNC(PyObject *) PyModule_New(
const char *name /* UTF-8 encoded string */
);
PyAPI_FUNC(PyObject *) PyModule_GetDict(PyObject *);
+PyAPI_FUNC(PyObject *) PyModule_GetNameObject(PyObject *);
PyAPI_FUNC(const char *) PyModule_GetName(PyObject *);
PyAPI_FUNC(const char *) PyModule_GetFilename(PyObject *);
PyAPI_FUNC(PyObject *) PyModule_GetFilenameObject(PyObject *);
diff --git a/Include/object.h b/Include/object.h
index 2528841..648c9cb 100644
--- a/Include/object.h
+++ b/Include/object.h
@@ -455,6 +455,7 @@ PyAPI_FUNC(unsigned int) PyType_ClearCache(void);
PyAPI_FUNC(void) PyType_Modified(PyTypeObject *);
/* Generic operations on objects */
+struct _Py_Identifier;
#ifndef Py_LIMITED_API
PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int);
PyAPI_FUNC(void) _Py_BreakPoint(void);
@@ -472,6 +473,9 @@ PyAPI_FUNC(int) PyObject_HasAttrString(PyObject *, const char *);
PyAPI_FUNC(PyObject *) PyObject_GetAttr(PyObject *, PyObject *);
PyAPI_FUNC(int) PyObject_SetAttr(PyObject *, PyObject *, PyObject *);
PyAPI_FUNC(int) PyObject_HasAttr(PyObject *, PyObject *);
+PyAPI_FUNC(PyObject *) _PyObject_GetAttrId(PyObject *, struct _Py_Identifier *);
+PyAPI_FUNC(int) _PyObject_SetAttrId(PyObject *, struct _Py_Identifier *, PyObject *);
+PyAPI_FUNC(int) _PyObject_HasAttrId(PyObject *, struct _Py_Identifier *);
#ifndef Py_LIMITED_API
PyAPI_FUNC(PyObject **) _PyObject_GetDictPtr(PyObject *);
#endif
@@ -793,6 +797,10 @@ not implemented for a given type combination.
PyAPI_DATA(PyObject) _Py_NotImplementedStruct; /* Don't use this directly */
#define Py_NotImplemented (&_Py_NotImplementedStruct)
+/* Macro for returning Py_NotImplemented from a function */
+#define Py_RETURN_NOTIMPLEMENTED \
+ return Py_INCREF(Py_NotImplemented), Py_NotImplemented
+
/* Rich comparison opcodes */
#define Py_LT 0
#define Py_LE 1
diff --git a/Include/opcode.h b/Include/opcode.h
index 6b10944..ece713e 100644
--- a/Include/opcode.h
+++ b/Include/opcode.h
@@ -7,7 +7,6 @@ extern "C" {
/* Instruction opcodes for compiled code */
-#define STOP_CODE 0
#define POP_TOP 1
#define ROT_TWO 2
#define ROT_THREE 3
diff --git a/Include/parsetok.h b/Include/parsetok.h
index 4b7694f..911dfc1 100644
--- a/Include/parsetok.h
+++ b/Include/parsetok.h
@@ -9,7 +9,10 @@ extern "C" {
typedef struct {
int error;
- const char *filename; /* decoded from the filesystem encoding */
+#ifndef PGEN
+ /* The filename is useless for pgen, see comment in tok_state structure */
+ PyObject *filename;
+#endif
int lineno;
int offset;
char *text; /* UTF-8-encoded string */
@@ -66,8 +69,10 @@ PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx(
perrdetail *err_ret,
int *flags);
-/* Note that he following function is defined in pythonrun.c not parsetok.c. */
+/* Note that the following functions are defined in pythonrun.c,
+ not in parsetok.c */
PyAPI_FUNC(void) PyParser_SetError(perrdetail *);
+PyAPI_FUNC(void) PyParser_ClearError(perrdetail *);
#ifdef __cplusplus
}
diff --git a/Include/patchlevel.h b/Include/patchlevel.h
index 1b8e6d4..4996e8e 100644
--- a/Include/patchlevel.h
+++ b/Include/patchlevel.h
@@ -17,19 +17,15 @@
/* Version parsed out into numeric values */
/*--start constants--*/
#define PY_MAJOR_VERSION 3
-#define PY_MINOR_VERSION 2
-#define PY_MICRO_VERSION 2
-#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL
+#define PY_MINOR_VERSION 3
+#define PY_MICRO_VERSION 0
+#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA
#define PY_RELEASE_SERIAL 0
/* Version as a string */
-#define PY_VERSION "3.2.2+"
+#define PY_VERSION "3.3.0a0"
/*--end constants--*/
-/* Subversion Revision number of this file (not of the repository). Empty
- since Mercurial migration. */
-#define PY_PATCHLEVEL_REVISION ""
-
/* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2.
Use this for numeric comparisons, e.g. #if PY_VERSION_HEX >= ... */
#define PY_VERSION_HEX ((PY_MAJOR_VERSION << 24) | \
diff --git a/Include/pydebug.h b/Include/pydebug.h
index 70c88f6..7173fe3 100644
--- a/Include/pydebug.h
+++ b/Include/pydebug.h
@@ -16,7 +16,6 @@ PyAPI_DATA(int) Py_BytesWarningFlag;
PyAPI_DATA(int) Py_UseClassExceptionsFlag;
PyAPI_DATA(int) Py_FrozenFlag;
PyAPI_DATA(int) Py_IgnoreEnvironmentFlag;
-PyAPI_DATA(int) Py_DivisionWarningFlag;
PyAPI_DATA(int) Py_DontWriteBytecodeFlag;
PyAPI_DATA(int) Py_NoUserSiteDirectory;
PyAPI_DATA(int) Py_UnbufferedStdioFlag;
@@ -26,8 +25,6 @@ PyAPI_DATA(int) Py_UnbufferedStdioFlag;
PYTHONPATH and PYTHONHOME from the environment */
#define Py_GETENV(s) (Py_IgnoreEnvironmentFlag ? NULL : getenv(s))
-PyAPI_FUNC(void) Py_FatalError(const char *message);
-
#ifdef __cplusplus
}
#endif
diff --git a/Include/pyerrors.h b/Include/pyerrors.h
index 4bb3c01..44eb3d9 100644
--- a/Include/pyerrors.h
+++ b/Include/pyerrors.h
@@ -45,18 +45,18 @@ typedef struct {
PyObject *myerrno;
PyObject *strerror;
PyObject *filename;
-} PyEnvironmentErrorObject;
-
#ifdef MS_WINDOWS
-typedef struct {
- PyException_HEAD
- PyObject *myerrno;
- PyObject *strerror;
- PyObject *filename;
PyObject *winerror;
-} PyWindowsErrorObject;
#endif
+ Py_ssize_t written; /* only for BlockingIOError, -1 otherwise */
+} PyOSErrorObject;
+
+/* Compatibility typedefs */
+typedef PyOSErrorObject PyEnvironmentErrorObject;
+#ifdef MS_WINDOWS
+typedef PyOSErrorObject PyWindowsErrorObject;
#endif
+#endif /* !Py_LIMITED_API */
/* Error handling definitions */
@@ -70,7 +70,17 @@ PyAPI_FUNC(PyObject *) PyErr_Occurred(void);
PyAPI_FUNC(void) PyErr_Clear(void);
PyAPI_FUNC(void) PyErr_Fetch(PyObject **, PyObject **, PyObject **);
PyAPI_FUNC(void) PyErr_Restore(PyObject *, PyObject *, PyObject *);
-PyAPI_FUNC(void) Py_FatalError(const char *message);
+
+#if defined(__clang__) || \
+ (defined(__GNUC__) && \
+ ((__GNUC_MAJOR__ >= 3) || \
+ (__GNUC_MAJOR__ == 2) && (__GNUC_MINOR__ >= 5)))
+#define _Py_NO_RETURN __attribute__((__noreturn__))
+#else
+#define _Py_NO_RETURN
+#endif
+
+PyAPI_FUNC(void) Py_FatalError(const char *message) _Py_NO_RETURN;
#if defined(Py_DEBUG) || defined(Py_LIMITED_API)
#define _PyErr_OCCURRED() PyErr_Occurred()
@@ -122,10 +132,9 @@ PyAPI_DATA(PyObject *) PyExc_LookupError;
PyAPI_DATA(PyObject *) PyExc_AssertionError;
PyAPI_DATA(PyObject *) PyExc_AttributeError;
+PyAPI_DATA(PyObject *) PyExc_BufferError;
PyAPI_DATA(PyObject *) PyExc_EOFError;
PyAPI_DATA(PyObject *) PyExc_FloatingPointError;
-PyAPI_DATA(PyObject *) PyExc_EnvironmentError;
-PyAPI_DATA(PyObject *) PyExc_IOError;
PyAPI_DATA(PyObject *) PyExc_OSError;
PyAPI_DATA(PyObject *) PyExc_ImportError;
PyAPI_DATA(PyObject *) PyExc_IndexError;
@@ -150,6 +159,27 @@ PyAPI_DATA(PyObject *) PyExc_UnicodeDecodeError;
PyAPI_DATA(PyObject *) PyExc_UnicodeTranslateError;
PyAPI_DATA(PyObject *) PyExc_ValueError;
PyAPI_DATA(PyObject *) PyExc_ZeroDivisionError;
+
+PyAPI_DATA(PyObject *) PyExc_BlockingIOError;
+PyAPI_DATA(PyObject *) PyExc_BrokenPipeError;
+PyAPI_DATA(PyObject *) PyExc_ChildProcessError;
+PyAPI_DATA(PyObject *) PyExc_ConnectionError;
+PyAPI_DATA(PyObject *) PyExc_ConnectionAbortedError;
+PyAPI_DATA(PyObject *) PyExc_ConnectionRefusedError;
+PyAPI_DATA(PyObject *) PyExc_ConnectionResetError;
+PyAPI_DATA(PyObject *) PyExc_FileExistsError;
+PyAPI_DATA(PyObject *) PyExc_FileNotFoundError;
+PyAPI_DATA(PyObject *) PyExc_InterruptedError;
+PyAPI_DATA(PyObject *) PyExc_IsADirectoryError;
+PyAPI_DATA(PyObject *) PyExc_NotADirectoryError;
+PyAPI_DATA(PyObject *) PyExc_PermissionError;
+PyAPI_DATA(PyObject *) PyExc_ProcessLookupError;
+PyAPI_DATA(PyObject *) PyExc_TimeoutError;
+
+
+/* Compatibility aliases */
+PyAPI_DATA(PyObject *) PyExc_EnvironmentError;
+PyAPI_DATA(PyObject *) PyExc_IOError;
#ifdef MS_WINDOWS
PyAPI_DATA(PyObject *) PyExc_WindowsError;
#endif
@@ -157,8 +187,6 @@ PyAPI_DATA(PyObject *) PyExc_WindowsError;
PyAPI_DATA(PyObject *) PyExc_VMSError;
#endif
-PyAPI_DATA(PyObject *) PyExc_BufferError;
-
PyAPI_DATA(PyObject *) PyExc_RecursionErrorInst;
/* Predefined warning categories */
@@ -198,8 +226,6 @@ PyAPI_FUNC(PyObject *) PyErr_Format(
);
#ifdef MS_WINDOWS
-PyAPI_FUNC(PyObject *) PyErr_SetFromWindowsErrWithFilenameObject(
- int, const char *);
PyAPI_FUNC(PyObject *) PyErr_SetFromWindowsErrWithFilename(
int ierr,
const char *filename /* decoded from the filesystem encoding */
@@ -293,6 +319,12 @@ PyAPI_FUNC(PyObject *) PyUnicodeTranslateError_Create(
Py_ssize_t end,
const char *reason /* UTF-8 encoded string */
);
+PyAPI_FUNC(PyObject *) _PyUnicodeTranslateError_Create(
+ PyObject *object,
+ Py_ssize_t start,
+ Py_ssize_t end,
+ const char *reason /* UTF-8 encoded string */
+ );
#endif
/* get the encoding attribute */
diff --git a/Include/pymacro.h b/Include/pymacro.h
new file mode 100644
index 0000000..1dc0c61
--- /dev/null
+++ b/Include/pymacro.h
@@ -0,0 +1,55 @@
+#ifndef Py_PYMACRO_H
+#define Py_PYMACRO_H
+
+#define Py_MIN(x, y) (((x) > (y)) ? (y) : (x))
+#define Py_MAX(x, y) (((x) > (y)) ? (x) : (y))
+
+/* Argument must be a char or an int in [-128, 127] or [0, 255]. */
+#define Py_CHARMASK(c) ((unsigned char)((c) & 0xff))
+
+
+/* Assert a build-time dependency, as an expression.
+
+ Your compile will fail if the condition isn't true, or can't be evaluated
+ by the compiler. This can be used in an expression: its value is 0.
+
+ Example:
+
+ #define foo_to_char(foo) \
+ ((char *)(foo) \
+ + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0))
+
+ Written by Rusty Russell, public domain, http://ccodearchive.net/ */
+#define Py_BUILD_ASSERT_EXPR(cond) \
+ (sizeof(char [1 - 2*!(cond)]) - 1)
+
+/* Get the number of elements in a visible array
+
+ This does not work on pointers, or arrays declared as [], or function
+ parameters. With correct compiler support, such usage will cause a build
+ error (see Py_BUILD_ASSERT_EXPR).
+
+ Written by Rusty Russell, public domain, http://ccodearchive.net/ */
+#if defined(__GNUC__)
+/* Two gcc extensions.
+ &a[0] degrades to a pointer: a different type from an array */
+#define Py_ARRAY_LENGTH(array) \
+ (sizeof(array) / sizeof((array)[0]) \
+ + Py_BUILD_ASSERT_EXPR(!__builtin_types_compatible_p(typeof(array), \
+ typeof(&(array)[0]))))
+#else
+#define Py_ARRAY_LENGTH(array) \
+ (sizeof(array) / sizeof((array)[0]))
+#endif
+
+
+/* Define macros for inline documentation. */
+#define PyDoc_VAR(name) static char name[]
+#define PyDoc_STRVAR(name,str) PyDoc_VAR(name) = PyDoc_STR(str)
+#ifdef WITH_DOC_STRINGS
+#define PyDoc_STR(str) str
+#else
+#define PyDoc_STR(str) ""
+#endif
+
+#endif /* Py_PYMACRO_H */
diff --git a/Include/pymath.h b/Include/pymath.h
index b4eda66..62a6c42 100644
--- a/Include/pymath.h
+++ b/Include/pymath.h
@@ -37,12 +37,6 @@ extern double pow(double, double);
#endif /* __STDC__ */
#endif /* _MSC_VER */
-#ifdef _OSF_SOURCE
-/* OSF1 5.1 doesn't make these available with XOPEN_SOURCE_EXTENDED defined */
-extern int finite(double);
-extern double copysign(double, double);
-#endif
-
/* High precision defintion of pi and e (Euler)
* The values are taken from libc6's math.h.
*/
diff --git a/Include/pyport.h b/Include/pyport.h
index 62aa53a..5309de6 100644
--- a/Include/pyport.h
+++ b/Include/pyport.h
@@ -132,7 +132,7 @@ Used in: PY_LONG_LONG
#endif
/* Parameters used for the numeric hash implementation. See notes for
- _PyHash_Double in Objects/object.c. Numeric hashes are based on
+ _Py_HashDouble in Objects/object.c. Numeric hashes are based on
reduction modulo the prime 2**_PyHASH_BITS - 1. */
#if SIZEOF_VOID_P >= 8
diff --git a/Include/pystate.h b/Include/pystate.h
index b5fe1ad..1bbb4e2 100644
--- a/Include/pystate.h
+++ b/Include/pystate.h
@@ -74,9 +74,9 @@ typedef struct _ts {
struct _frame *frame;
int recursion_depth;
char overflowed; /* The stack has overflowed. Allow 50 more calls
- to handle the runtime error. */
- char recursion_critical; /* The current calls must not cause
- a stack overflow. */
+ to handle the runtime error. */
+ char recursion_critical; /* The current calls must not cause
+ a stack overflow. */
/* 'tracing' keeps track of the execution depth when tracing/profiling.
This is to prevent the actual trace/profile code from being recorded in
the trace/profile. */
@@ -160,6 +160,8 @@ typedef
enum {PyGILState_LOCKED, PyGILState_UNLOCKED}
PyGILState_STATE;
+#ifdef WITH_THREAD
+
/* Ensure that the current thread is ready to call the Python
C API, regardless of the current state of Python, or of its
thread lock. This may be called as many times as desired
@@ -201,6 +203,8 @@ PyAPI_FUNC(void) PyGILState_Release(PyGILState_STATE);
*/
PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void);
+#endif /* #ifdef WITH_THREAD */
+
/* The implementation of sys._current_frames() Returns a dict mapping
thread id to that thread's current frame.
*/
diff --git a/Include/pythonrun.h b/Include/pythonrun.h
index 00b4972..bdad15c 100644
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -179,9 +179,6 @@ PyAPI_FUNC(const char *) Py_GetCopyright(void);
PyAPI_FUNC(const char *) Py_GetCompiler(void);
PyAPI_FUNC(const char *) Py_GetBuildInfo(void);
#ifndef Py_LIMITED_API
-PyAPI_FUNC(const char *) _Py_svnversion(void);
-PyAPI_FUNC(const char *) Py_SubversionRevision(void);
-PyAPI_FUNC(const char *) Py_SubversionShortBranch(void);
PyAPI_FUNC(const char *) _Py_hgidentifier(void);
PyAPI_FUNC(const char *) _Py_hgversion(void);
#endif
diff --git a/Include/pythread.h b/Include/pythread.h
index 9806c61..6e9f303 100644
--- a/Include/pythread.h
+++ b/Include/pythread.h
@@ -32,7 +32,7 @@ PyAPI_FUNC(int) PyThread_acquire_lock(PyThread_type_lock, int);
on a lock (see PyThread_acquire_lock_timed() below).
PY_TIMEOUT_MAX is the highest usable value (in microseconds) of that
type, and depends on the system threading API.
-
+
NOTE: this isn't the same value as `_thread.TIMEOUT_MAX`. The _thread
module exposes a higher-level API, with timeouts expressed in seconds
and floating-point numbers allowed.
@@ -74,6 +74,8 @@ PyAPI_FUNC(void) PyThread_release_lock(PyThread_type_lock);
PyAPI_FUNC(size_t) PyThread_get_stacksize(void);
PyAPI_FUNC(int) PyThread_set_stacksize(size_t);
+PyAPI_FUNC(PyObject*) PyThread_GetInfo(void);
+
/* Thread Local Storage (TLS) API */
PyAPI_FUNC(int) PyThread_create_key(void);
PyAPI_FUNC(void) PyThread_delete_key(int);
diff --git a/Include/symtable.h b/Include/symtable.h
index fd7de04..82f6269 100644
--- a/Include/symtable.h
+++ b/Include/symtable.h
@@ -23,10 +23,13 @@ struct symtable {
PyObject *st_blocks; /* dict: map AST node addresses
* to symbol table entries */
PyObject *st_stack; /* list: stack of namespace info */
- PyObject *st_global; /* borrowed ref to st_top->st_symbols */
- int st_nblocks; /* number of blocks used */
+ PyObject *st_global; /* borrowed ref to st_top->ste_symbols */
+ int st_nblocks; /* number of blocks used. kept for
+ consistency with the corresponding
+ compiler structure */
PyObject *st_private; /* name of current class or NULL */
- PyFutureFeatures *st_future; /* module's future features */
+ PyFutureFeatures *st_future; /* module's future features that affect
+ the symbol table */
};
typedef struct _symtable_entry {
@@ -34,7 +37,7 @@ typedef struct _symtable_entry {
PyObject *ste_id; /* int: key in ste_table->st_blocks */
PyObject *ste_symbols; /* dict: variable names to flags */
PyObject *ste_name; /* string: name of current block */
- PyObject *ste_varnames; /* list of variable names */
+ PyObject *ste_varnames; /* list of function parameters */
PyObject *ste_children; /* list of child blocks */
_Py_block_ty ste_type; /* module, class, or function */
int ste_unoptimized; /* false if namespace is optimized */
diff --git a/Include/traceback.h b/Include/traceback.h
index 69e3d05..7734707 100644
--- a/Include/traceback.h
+++ b/Include/traceback.h
@@ -5,6 +5,8 @@
extern "C" {
#endif
+#include "pystate.h"
+
struct _frame;
/* Traceback interface */
@@ -28,6 +30,42 @@ PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int);
PyAPI_DATA(PyTypeObject) PyTraceBack_Type;
#define PyTraceBack_Check(v) (Py_TYPE(v) == &PyTraceBack_Type)
+/* Write the Python traceback into the file 'fd'. For example:
+
+ Traceback (most recent call first):
+ File "xxx", line xxx in <xxx>
+ File "xxx", line xxx in <xxx>
+ ...
+ File "xxx", line xxx in <xxx>
+
+ This function is written for debug purpose only, to dump the traceback in
+ the worst case: after a segmentation fault, at fatal error, etc. That's why,
+ it is very limited. Strings are truncated to 100 characters and encoded to
+ ASCII with backslashreplace. It doesn't write the source code, only the
+ function name, filename and line number of each frame. Write only the first
+ 100 frames: if the traceback is truncated, write the line " ...".
+
+ This function is signal safe. */
+
+PyAPI_DATA(void) _Py_DumpTraceback(
+ int fd,
+ PyThreadState *tstate);
+
+/* Write the traceback of all threads into the file 'fd'. current_thread can be
+ NULL. Return NULL on success, or an error message on error.
+
+ This function is written for debug purpose only. It calls
+ _Py_DumpTraceback() for each thread, and so has the same limitations. It
+ only write the traceback of the first 100 threads: write "..." if there are
+ more threads.
+
+ This function is signal safe. */
+
+PyAPI_DATA(const char*) _Py_DumpTracebackThreads(
+ int fd, PyInterpreterState *interp,
+ PyThreadState *current_thread);
+
+
#ifdef __cplusplus
}
#endif
diff --git a/Include/ucnhash.h b/Include/ucnhash.h
index 70fdf13..8de9ba0 100644
--- a/Include/ucnhash.h
+++ b/Include/ucnhash.h
@@ -19,11 +19,13 @@ typedef struct {
success, zero if not. Does not set Python exceptions.
If self is NULL, data come from the default version of the database.
If it is not NULL, it should be a unicodedata.ucd_X_Y_Z object */
- int (*getname)(PyObject *self, Py_UCS4 code, char* buffer, int buflen);
+ int (*getname)(PyObject *self, Py_UCS4 code, char* buffer, int buflen,
+ int with_alias_and_seq);
/* Get character code for a given name. Same error handling
as for getname. */
- int (*getcode)(PyObject *self, const char* name, int namelen, Py_UCS4* code);
+ int (*getcode)(PyObject *self, const char* name, int namelen, Py_UCS4* code,
+ int with_named_seq);
} _PyUnicode_Name_CAPI;
diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h
index 477f526..58c1f55 100644
--- a/Include/unicodeobject.h
+++ b/Include/unicodeobject.h
@@ -64,16 +64,15 @@ Copyright (c) Corporation for National Research Initiatives.
/* Python 3.x requires unicode */
#define Py_USING_UNICODE
-/* FIXME: MvL's new implementation assumes that Py_UNICODE_SIZE is
- properly set, but the default rules below doesn't set it. I'll
- sort this out some other day -- fredrik@pythonware.com */
-
-#ifndef Py_UNICODE_SIZE
-#error Must define Py_UNICODE_SIZE
+#ifndef SIZEOF_WCHAR_T
+#error Must define SIZEOF_WCHAR_T
#endif
-/* Setting Py_UNICODE_WIDE enables UCS-4 storage. Otherwise, Unicode
- strings are stored as UCS-2 (with limited support for UTF-16) */
+#define Py_UNICODE_SIZE SIZEOF_WCHAR_T
+
+/* If wchar_t can be used for UCS-4 storage, set Py_UNICODE_WIDE.
+ Otherwise, Unicode strings are stored as UCS-2 (with limited support
+ for UTF-16) */
#if Py_UNICODE_SIZE >= 4
#define Py_UNICODE_WIDE
@@ -84,19 +83,14 @@ Copyright (c) Corporation for National Research Initiatives.
/* #define HAVE_WCHAR_H */
/* #define HAVE_USABLE_WCHAR_T */
-/* Defaults for various platforms */
-#ifndef PY_UNICODE_TYPE
-
-/* Windows has a usable wchar_t type (unless we're using UCS-4) */
-# if defined(MS_WIN32) && Py_UNICODE_SIZE == 2
-# define HAVE_USABLE_WCHAR_T
-# define PY_UNICODE_TYPE wchar_t
-# endif
-
-# if defined(Py_UNICODE_WIDE)
-# define PY_UNICODE_TYPE Py_UCS4
-# endif
+/* Py_UNICODE was the native Unicode storage format (code unit) used by
+ Python and represents a single Unicode element in the Unicode type.
+ With PEP 393, Py_UNICODE is deprecated and replaced with a
+ typedef to wchar_t. */
+#ifndef Py_LIMITED_API
+#define PY_UNICODE_TYPE wchar_t
+typedef wchar_t Py_UNICODE;
#endif
/* If the compiler provides a wchar_t type we try to support it
@@ -109,6 +103,10 @@ Copyright (c) Corporation for National Research Initiatives.
# endif
#endif
+#if defined(MS_WINDOWS)
+# define HAVE_MBCS
+#endif
+
#ifdef HAVE_WCHAR_H
/* Work around a cosmetic bug in BSDI 4.x wchar.h; thanks to Thomas Wouters */
# ifdef _HAVE_BSDI
@@ -117,201 +115,19 @@ Copyright (c) Corporation for National Research Initiatives.
# include <wchar.h>
#endif
-/*
- * Use this typedef when you need to represent a UTF-16 surrogate pair
- * as single unsigned integer.
- */
+/* Py_UCS4 and Py_UCS2 are typedefs for the respective
+ unicode representations. */
#if SIZEOF_INT >= 4
typedef unsigned int Py_UCS4;
#elif SIZEOF_LONG >= 4
typedef unsigned long Py_UCS4;
-#endif
-
-/* Py_UNICODE is the native Unicode storage format (code unit) used by
- Python and represents a single Unicode element in the Unicode
- type. */
-
-#ifndef Py_LIMITED_API
-typedef PY_UNICODE_TYPE Py_UNICODE;
-#endif
-
-/* --- UCS-2/UCS-4 Name Mangling ------------------------------------------ */
-
-/* Unicode API names are mangled to assure that UCS-2 and UCS-4 builds
- produce different external names and thus cause import errors in
- case Python interpreters and extensions with mixed compiled in
- Unicode width assumptions are combined. */
-
-#ifndef Py_UNICODE_WIDE
-
-# define PyUnicode_AsASCIIString PyUnicodeUCS2_AsASCIIString
-# define PyUnicode_AsCharmapString PyUnicodeUCS2_AsCharmapString
-# define PyUnicode_AsDecodedObject PyUnicodeUCS2_AsDecodedObject
-# define PyUnicode_AsDecodedUnicode PyUnicodeUCS2_AsDecodedUnicode
-# define PyUnicode_AsEncodedObject PyUnicodeUCS2_AsEncodedObject
-# define PyUnicode_AsEncodedString PyUnicodeUCS2_AsEncodedString
-# define PyUnicode_AsEncodedUnicode PyUnicodeUCS2_AsEncodedUnicode
-# define PyUnicode_AsLatin1String PyUnicodeUCS2_AsLatin1String
-# define PyUnicode_AsRawUnicodeEscapeString PyUnicodeUCS2_AsRawUnicodeEscapeString
-# define PyUnicode_AsUTF32String PyUnicodeUCS2_AsUTF32String
-# define PyUnicode_AsUTF16String PyUnicodeUCS2_AsUTF16String
-# define PyUnicode_AsUTF8String PyUnicodeUCS2_AsUTF8String
-# define PyUnicode_AsUnicode PyUnicodeUCS2_AsUnicode
-# define PyUnicode_AsUnicodeEscapeString PyUnicodeUCS2_AsUnicodeEscapeString
-# define PyUnicode_AsWideChar PyUnicodeUCS2_AsWideChar
-# define PyUnicode_AsWideCharString PyUnicodeUCS2_AsWideCharString
-# define PyUnicode_ClearFreeList PyUnicodeUCS2_ClearFreelist
-# define PyUnicode_Compare PyUnicodeUCS2_Compare
-# define PyUnicode_CompareWithASCIIString PyUnicodeUCS2_CompareWithASCIIString
-# define PyUnicode_Concat PyUnicodeUCS2_Concat
-# define PyUnicode_Append PyUnicodeUCS2_Append
-# define PyUnicode_AppendAndDel PyUnicodeUCS2_AppendAndDel
-# define PyUnicode_Contains PyUnicodeUCS2_Contains
-# define PyUnicode_Count PyUnicodeUCS2_Count
-# define PyUnicode_Decode PyUnicodeUCS2_Decode
-# define PyUnicode_DecodeASCII PyUnicodeUCS2_DecodeASCII
-# define PyUnicode_DecodeCharmap PyUnicodeUCS2_DecodeCharmap
-# define PyUnicode_DecodeLatin1 PyUnicodeUCS2_DecodeLatin1
-# define PyUnicode_DecodeFSDefault PyUnicodeUCS2_DecodeFSDefault
-# define PyUnicode_DecodeFSDefaultAndSize PyUnicodeUCS2_DecodeFSDefaultAndSize
-# define PyUnicode_DecodeRawUnicodeEscape PyUnicodeUCS2_DecodeRawUnicodeEscape
-# define PyUnicode_DecodeUTF32 PyUnicodeUCS2_DecodeUTF32
-# define PyUnicode_DecodeUTF32Stateful PyUnicodeUCS2_DecodeUTF32Stateful
-# define PyUnicode_DecodeUTF16 PyUnicodeUCS2_DecodeUTF16
-# define PyUnicode_DecodeUTF16Stateful PyUnicodeUCS2_DecodeUTF16Stateful
-# define PyUnicode_DecodeUTF8 PyUnicodeUCS2_DecodeUTF8
-# define PyUnicode_DecodeUTF8Stateful PyUnicodeUCS2_DecodeUTF8Stateful
-# define PyUnicode_DecodeUnicodeEscape PyUnicodeUCS2_DecodeUnicodeEscape
-# define PyUnicode_Encode PyUnicodeUCS2_Encode
-# define PyUnicode_EncodeASCII PyUnicodeUCS2_EncodeASCII
-# define PyUnicode_EncodeCharmap PyUnicodeUCS2_EncodeCharmap
-# define PyUnicode_EncodeDecimal PyUnicodeUCS2_EncodeDecimal
-# define PyUnicode_EncodeLatin1 PyUnicodeUCS2_EncodeLatin1
-# define PyUnicode_EncodeRawUnicodeEscape PyUnicodeUCS2_EncodeRawUnicodeEscape
-# define PyUnicode_EncodeUTF32 PyUnicodeUCS2_EncodeUTF32
-# define PyUnicode_EncodeUTF16 PyUnicodeUCS2_EncodeUTF16
-# define PyUnicode_EncodeUTF8 PyUnicodeUCS2_EncodeUTF8
-# define PyUnicode_EncodeUnicodeEscape PyUnicodeUCS2_EncodeUnicodeEscape
-# define PyUnicode_Find PyUnicodeUCS2_Find
-# define PyUnicode_Format PyUnicodeUCS2_Format
-# define PyUnicode_FromEncodedObject PyUnicodeUCS2_FromEncodedObject
-# define PyUnicode_FromFormat PyUnicodeUCS2_FromFormat
-# define PyUnicode_FromFormatV PyUnicodeUCS2_FromFormatV
-# define PyUnicode_FromObject PyUnicodeUCS2_FromObject
-# define PyUnicode_FromOrdinal PyUnicodeUCS2_FromOrdinal
-# define PyUnicode_FromString PyUnicodeUCS2_FromString
-# define PyUnicode_FromStringAndSize PyUnicodeUCS2_FromStringAndSize
-# define PyUnicode_FromUnicode PyUnicodeUCS2_FromUnicode
-# define PyUnicode_FromWideChar PyUnicodeUCS2_FromWideChar
-# define PyUnicode_FSConverter PyUnicodeUCS2_FSConverter
-# define PyUnicode_FSDecoder PyUnicodeUCS2_FSDecoder
-# define PyUnicode_GetDefaultEncoding PyUnicodeUCS2_GetDefaultEncoding
-# define PyUnicode_GetMax PyUnicodeUCS2_GetMax
-# define PyUnicode_GetSize PyUnicodeUCS2_GetSize
-# define PyUnicode_IsIdentifier PyUnicodeUCS2_IsIdentifier
-# define PyUnicode_Join PyUnicodeUCS2_Join
-# define PyUnicode_Partition PyUnicodeUCS2_Partition
-# define PyUnicode_RPartition PyUnicodeUCS2_RPartition
-# define PyUnicode_RSplit PyUnicodeUCS2_RSplit
-# define PyUnicode_Replace PyUnicodeUCS2_Replace
-# define PyUnicode_Resize PyUnicodeUCS2_Resize
-# define PyUnicode_RichCompare PyUnicodeUCS2_RichCompare
-# define PyUnicode_Split PyUnicodeUCS2_Split
-# define PyUnicode_Splitlines PyUnicodeUCS2_Splitlines
-# define PyUnicode_Tailmatch PyUnicodeUCS2_Tailmatch
-# define PyUnicode_Translate PyUnicodeUCS2_Translate
-# define PyUnicode_TranslateCharmap PyUnicodeUCS2_TranslateCharmap
-# define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS2_AsDefaultEncodedString
-# define _PyUnicode_Fini _PyUnicodeUCS2_Fini
-# define _PyUnicode_Init _PyUnicodeUCS2_Init
-# define PyUnicode_strdup PyUnicodeUCS2_strdup
-
#else
-
-# define PyUnicode_AsASCIIString PyUnicodeUCS4_AsASCIIString
-# define PyUnicode_AsCharmapString PyUnicodeUCS4_AsCharmapString
-# define PyUnicode_AsDecodedObject PyUnicodeUCS4_AsDecodedObject
-# define PyUnicode_AsDecodedUnicode PyUnicodeUCS4_AsDecodedUnicode
-# define PyUnicode_AsEncodedObject PyUnicodeUCS4_AsEncodedObject
-# define PyUnicode_AsEncodedString PyUnicodeUCS4_AsEncodedString
-# define PyUnicode_AsEncodedUnicode PyUnicodeUCS4_AsEncodedUnicode
-# define PyUnicode_AsLatin1String PyUnicodeUCS4_AsLatin1String
-# define PyUnicode_AsRawUnicodeEscapeString PyUnicodeUCS4_AsRawUnicodeEscapeString
-# define PyUnicode_AsUTF32String PyUnicodeUCS4_AsUTF32String
-# define PyUnicode_AsUTF16String PyUnicodeUCS4_AsUTF16String
-# define PyUnicode_AsUTF8String PyUnicodeUCS4_AsUTF8String
-# define PyUnicode_AsUnicode PyUnicodeUCS4_AsUnicode
-# define PyUnicode_AsUnicodeEscapeString PyUnicodeUCS4_AsUnicodeEscapeString
-# define PyUnicode_AsWideChar PyUnicodeUCS4_AsWideChar
-# define PyUnicode_AsWideCharString PyUnicodeUCS4_AsWideCharString
-# define PyUnicode_ClearFreeList PyUnicodeUCS4_ClearFreelist
-# define PyUnicode_Compare PyUnicodeUCS4_Compare
-# define PyUnicode_CompareWithASCIIString PyUnicodeUCS4_CompareWithASCIIString
-# define PyUnicode_Concat PyUnicodeUCS4_Concat
-# define PyUnicode_Append PyUnicodeUCS4_Append
-# define PyUnicode_AppendAndDel PyUnicodeUCS4_AppendAndDel
-# define PyUnicode_Contains PyUnicodeUCS4_Contains
-# define PyUnicode_Count PyUnicodeUCS4_Count
-# define PyUnicode_Decode PyUnicodeUCS4_Decode
-# define PyUnicode_DecodeASCII PyUnicodeUCS4_DecodeASCII
-# define PyUnicode_DecodeCharmap PyUnicodeUCS4_DecodeCharmap
-# define PyUnicode_DecodeLatin1 PyUnicodeUCS4_DecodeLatin1
-# define PyUnicode_DecodeFSDefault PyUnicodeUCS4_DecodeFSDefault
-# define PyUnicode_DecodeFSDefaultAndSize PyUnicodeUCS4_DecodeFSDefaultAndSize
-# define PyUnicode_DecodeRawUnicodeEscape PyUnicodeUCS4_DecodeRawUnicodeEscape
-# define PyUnicode_DecodeUTF32 PyUnicodeUCS4_DecodeUTF32
-# define PyUnicode_DecodeUTF32Stateful PyUnicodeUCS4_DecodeUTF32Stateful
-# define PyUnicode_DecodeUTF16 PyUnicodeUCS4_DecodeUTF16
-# define PyUnicode_DecodeUTF16Stateful PyUnicodeUCS4_DecodeUTF16Stateful
-# define PyUnicode_DecodeUTF8 PyUnicodeUCS4_DecodeUTF8
-# define PyUnicode_DecodeUTF8Stateful PyUnicodeUCS4_DecodeUTF8Stateful
-# define PyUnicode_DecodeUnicodeEscape PyUnicodeUCS4_DecodeUnicodeEscape
-# define PyUnicode_Encode PyUnicodeUCS4_Encode
-# define PyUnicode_EncodeASCII PyUnicodeUCS4_EncodeASCII
-# define PyUnicode_EncodeCharmap PyUnicodeUCS4_EncodeCharmap
-# define PyUnicode_EncodeDecimal PyUnicodeUCS4_EncodeDecimal
-# define PyUnicode_EncodeLatin1 PyUnicodeUCS4_EncodeLatin1
-# define PyUnicode_EncodeRawUnicodeEscape PyUnicodeUCS4_EncodeRawUnicodeEscape
-# define PyUnicode_EncodeUTF32 PyUnicodeUCS4_EncodeUTF32
-# define PyUnicode_EncodeUTF16 PyUnicodeUCS4_EncodeUTF16
-# define PyUnicode_EncodeUTF8 PyUnicodeUCS4_EncodeUTF8
-# define PyUnicode_EncodeUnicodeEscape PyUnicodeUCS4_EncodeUnicodeEscape
-# define PyUnicode_Find PyUnicodeUCS4_Find
-# define PyUnicode_Format PyUnicodeUCS4_Format
-# define PyUnicode_FromEncodedObject PyUnicodeUCS4_FromEncodedObject
-# define PyUnicode_FromFormat PyUnicodeUCS4_FromFormat
-# define PyUnicode_FromFormatV PyUnicodeUCS4_FromFormatV
-# define PyUnicode_FromObject PyUnicodeUCS4_FromObject
-# define PyUnicode_FromOrdinal PyUnicodeUCS4_FromOrdinal
-# define PyUnicode_FromString PyUnicodeUCS4_FromString
-# define PyUnicode_FromStringAndSize PyUnicodeUCS4_FromStringAndSize
-# define PyUnicode_FromUnicode PyUnicodeUCS4_FromUnicode
-# define PyUnicode_FromWideChar PyUnicodeUCS4_FromWideChar
-# define PyUnicode_FSConverter PyUnicodeUCS4_FSConverter
-# define PyUnicode_FSDecoder PyUnicodeUCS4_FSDecoder
-# define PyUnicode_GetDefaultEncoding PyUnicodeUCS4_GetDefaultEncoding
-# define PyUnicode_GetMax PyUnicodeUCS4_GetMax
-# define PyUnicode_GetSize PyUnicodeUCS4_GetSize
-# define PyUnicode_IsIdentifier PyUnicodeUCS4_IsIdentifier
-# define PyUnicode_Join PyUnicodeUCS4_Join
-# define PyUnicode_Partition PyUnicodeUCS4_Partition
-# define PyUnicode_RPartition PyUnicodeUCS4_RPartition
-# define PyUnicode_RSplit PyUnicodeUCS4_RSplit
-# define PyUnicode_Replace PyUnicodeUCS4_Replace
-# define PyUnicode_Resize PyUnicodeUCS4_Resize
-# define PyUnicode_RichCompare PyUnicodeUCS4_RichCompare
-# define PyUnicode_Split PyUnicodeUCS4_Split
-# define PyUnicode_Splitlines PyUnicodeUCS4_Splitlines
-# define PyUnicode_Tailmatch PyUnicodeUCS4_Tailmatch
-# define PyUnicode_Translate PyUnicodeUCS4_Translate
-# define PyUnicode_TranslateCharmap PyUnicodeUCS4_TranslateCharmap
-# define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS4_AsDefaultEncodedString
-# define _PyUnicode_Fini _PyUnicodeUCS4_Fini
-# define _PyUnicode_Init _PyUnicodeUCS4_Init
-# define PyUnicode_strdup PyUnicodeUCS4_strdup
-
+#error "Could not find a proper typedef for Py_UCS4"
#endif
+typedef unsigned short Py_UCS2;
+typedef unsigned char Py_UCS1;
+
/* --- Internal Unicode Operations ---------------------------------------- */
/* Since splitting on whitespace is an important use case, and
@@ -350,7 +166,7 @@ typedef PY_UNICODE_TYPE Py_UNICODE;
Py_UNICODE_ISDIGIT(ch) || \
Py_UNICODE_ISNUMERIC(ch))
-#define Py_UNICODE_COPY(target, source, length) \
+#define Py_UNICODE_COPY(target, source, length) \
Py_MEMCPY((target), (source), (length)*sizeof(Py_UNICODE))
#define Py_UNICODE_FILL(target, value, length) \
@@ -358,13 +174,23 @@ typedef PY_UNICODE_TYPE Py_UNICODE;
for (i_ = 0; i_ < (length); i_++) t_[i_] = v_;\
} while (0)
+/* macros to work with surrogates */
+#define Py_UNICODE_IS_SURROGATE(ch) (0xD800 <= ch && ch <= 0xDFFF)
+#define Py_UNICODE_IS_HIGH_SURROGATE(ch) (0xD800 <= ch && ch <= 0xDBFF)
+#define Py_UNICODE_IS_LOW_SURROGATE(ch) (0xDC00 <= ch && ch <= 0xDFFF)
+/* Join two surrogate characters and return a single Py_UCS4 value. */
+#define Py_UNICODE_JOIN_SURROGATES(high, low) \
+ (((((Py_UCS4)(high) & 0x03FF) << 10) | \
+ ((Py_UCS4)(low) & 0x03FF)) + 0x10000)
+
/* Check if substring matches at given offset. The offset must be
valid, and the substring must not be empty. */
#define Py_UNICODE_MATCH(string, offset, substring) \
- ((*((string)->str + (offset)) == *((substring)->str)) && \
- ((*((string)->str + (offset) + (substring)->length-1) == *((substring)->str + (substring)->length-1))) && \
- !memcmp((string)->str + (offset), (substring)->str, (substring)->length*sizeof(Py_UNICODE)))
+ ((*((string)->wstr + (offset)) == *((substring)->wstr)) && \
+ ((*((string)->wstr + (offset) + (substring)->wstr_length-1) == *((substring)->wstr + (substring)->wstr_length-1))) && \
+ !memcmp((string)->wstr + (offset), (substring)->wstr, (substring)->wstr_length*sizeof(Py_UNICODE)))
+
#endif /* Py_LIMITED_API */
#ifdef __cplusplus
@@ -374,41 +200,364 @@ extern "C" {
/* --- Unicode Type ------------------------------------------------------- */
#ifndef Py_LIMITED_API
+
+/* ASCII-only strings created through PyUnicode_New use the PyASCIIObject
+ structure. state.ascii and state.compact are set, and the data
+ immediately follow the structure. utf8_length and wstr_length can be found
+ in the length field; the utf8 pointer is equal to the data pointer. */
typedef struct {
+ /* There are 4 forms of Unicode strings:
+
+ - compact ascii:
+
+ * structure = PyASCIIObject
+ * kind = PyUnicode_1BYTE_KIND
+ * compact = 1
+ * ascii = 1
+ * ready = 1
+ * (length is the length of the utf8 and wstr strings)
+ * (data starts just after the structure)
+ * (since ASCII is decoded from UTF-8, the utf8 string are the data)
+
+ - compact:
+
+ * structure = PyCompactUnicodeObject
+ * kind = PyUnicode_1BYTE_KIND, PyUnicode_2BYTE_KIND or
+ PyUnicode_4BYTE_KIND
+ * compact = 1
+ * ready = 1
+ * ascii = 0
+ * utf8 is not shared with data
+ * utf8_length = 0 if utf8 is NULL
+ * wstr is shared with data and wstr_length=length
+ if kind=PyUnicode_2BYTE_KIND and sizeof(wchar_t)=2
+ or if kind=PyUnicode_4BYTE_KIND and sizeof(wchar_t)=4
+ * wstr_length = 0 if wstr is NULL
+ * (data starts just after the structure)
+
+ - legacy string, not ready:
+
+ * structure = PyUnicodeObject
+ * length = 0 (use wstr_length)
+ * hash = -1
+ * kind = PyUnicode_WCHAR_KIND
+ * compact = 0
+ * ascii = 0
+ * ready = 0
+ * interned = SSTATE_NOT_INTERNED
+ * wstr is not NULL
+ * data.any is NULL
+ * utf8 is NULL
+ * utf8_length = 0
+
+ - legacy string, ready:
+
+ * structure = PyUnicodeObject structure
+ * kind = PyUnicode_1BYTE_KIND, PyUnicode_2BYTE_KIND or
+ PyUnicode_4BYTE_KIND
+ * compact = 0
+ * ready = 1
+ * data.any is not NULL
+ * utf8 is shared and utf8_length = length with data.any if ascii = 1
+ * utf8_length = 0 if utf8 is NULL
+ * wstr is shared with data.any and wstr_length = length
+ if kind=PyUnicode_2BYTE_KIND and sizeof(wchar_t)=2
+ or if kind=PyUnicode_4BYTE_KIND and sizeof(wchar_4)=4
+ * wstr_length = 0 if wstr is NULL
+
+ Compact strings use only one memory block (structure + characters),
+ whereas legacy strings use one block for the structure and one block
+ for characters.
+
+ Legacy strings are created by PyUnicode_FromUnicode() and
+ PyUnicode_FromStringAndSize(NULL, size) functions. They become ready
+ when PyUnicode_READY() is called.
+
+ See also _PyUnicode_CheckConsistency().
+ */
PyObject_HEAD
- Py_ssize_t length; /* Length of raw Unicode data in buffer */
- Py_UNICODE *str; /* Raw Unicode buffer */
+ Py_ssize_t length; /* Number of code points in the string */
Py_hash_t hash; /* Hash value; -1 if not set */
- int state; /* != 0 if interned. In this case the two
- * references from the dictionary to this object
- * are *not* counted in ob_refcnt. */
- PyObject *defenc; /* (Default) Encoded version as Python
- string, or NULL; this is used for
- implementing the buffer protocol */
+ struct {
+ /*
+ SSTATE_NOT_INTERNED (0)
+ SSTATE_INTERNED_MORTAL (1)
+ SSTATE_INTERNED_IMMORTAL (2)
+
+ If interned != SSTATE_NOT_INTERNED, the two references from the
+ dictionary to this object are *not* counted in ob_refcnt.
+ */
+ unsigned int interned:2;
+ /* Character size:
+
+ - PyUnicode_WCHAR_KIND (0):
+
+ * character type = wchar_t (16 or 32 bits, depending on the
+ platform)
+
+ - PyUnicode_1BYTE_KIND (1):
+
+ * character type = Py_UCS1 (8 bits, unsigned)
+ * if ascii is set, all characters must be in range
+ U+0000-U+007F, otherwise at least one character must be in range
+ U+0080-U+00FF
+
+ - PyUnicode_2BYTE_KIND (2):
+
+ * character type = Py_UCS2 (16 bits, unsigned)
+ * at least one character must be in range U+0100-U+FFFF
+
+ - PyUnicode_4BYTE_KIND (4):
+
+ * character type = Py_UCS4 (32 bits, unsigned)
+ * at least one character must be in range U+10000-U+10FFFF
+ */
+ unsigned int kind:3;
+ /* Compact is with respect to the allocation scheme. Compact unicode
+ objects only require one memory block while non-compact objects use
+ one block for the PyUnicodeObject struct and another for its data
+ buffer. */
+ unsigned int compact:1;
+ /* The string only contains characters in range U+0000-U+007F (ASCII)
+ and the kind is PyUnicode_1BYTE_KIND. If ascii is set and compact is
+ set, use the PyASCIIObject structure. */
+ unsigned int ascii:1;
+ /* The ready flag indicates whether the object layout is initialized
+ completely. This means that this is either a compact object, or
+ the data pointer is filled out. The bit is redundant, and helps
+ to minimize the test in PyUnicode_IS_READY(). */
+ unsigned int ready:1;
+ } state;
+ wchar_t *wstr; /* wchar_t representation (null-terminated) */
+} PyASCIIObject;
+
+/* Non-ASCII strings allocated through PyUnicode_New use the
+ PyCompactUnicodeObject structure. state.compact is set, and the data
+ immediately follow the structure. */
+typedef struct {
+ PyASCIIObject _base;
+ Py_ssize_t utf8_length; /* Number of bytes in utf8, excluding the
+ * terminating \0. */
+ char *utf8; /* UTF-8 representation (null-terminated) */
+ Py_ssize_t wstr_length; /* Number of code points in wstr, possible
+ * surrogates count as two code points. */
+} PyCompactUnicodeObject;
+
+/* Strings allocated through PyUnicode_FromUnicode(NULL, len) use the
+ PyUnicodeObject structure. The actual string data is initially in the wstr
+ block, and copied into the data block using _PyUnicode_Ready. */
+typedef struct {
+ PyCompactUnicodeObject _base;
+ union {
+ void *any;
+ Py_UCS1 *latin1;
+ Py_UCS2 *ucs2;
+ Py_UCS4 *ucs4;
+ } data; /* Canonical, smallest-form Unicode buffer */
} PyUnicodeObject;
#endif
PyAPI_DATA(PyTypeObject) PyUnicode_Type;
PyAPI_DATA(PyTypeObject) PyUnicodeIter_Type;
-#define SSTATE_NOT_INTERNED 0
-#define SSTATE_INTERNED_MORTAL 1
-#define SSTATE_INTERNED_IMMORTAL 2
-
#define PyUnicode_Check(op) \
PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_UNICODE_SUBCLASS)
#define PyUnicode_CheckExact(op) (Py_TYPE(op) == &PyUnicode_Type)
/* Fast access macros */
#ifndef Py_LIMITED_API
+
+#define PyUnicode_WSTR_LENGTH(op) \
+ (PyUnicode_IS_COMPACT_ASCII(op) ? \
+ ((PyASCIIObject*)op)->length : \
+ ((PyCompactUnicodeObject*)op)->wstr_length)
+
+/* Returns the deprecated Py_UNICODE representation's size in code units
+ (this includes surrogate pairs as 2 units).
+ If the Py_UNICODE representation is not available, it will be computed
+ on request. Use PyUnicode_GET_LENGTH() for the length in code points. */
+
#define PyUnicode_GET_SIZE(op) \
- (assert(PyUnicode_Check(op)),(((PyUnicodeObject *)(op))->length))
+ (assert(PyUnicode_Check(op)), \
+ (((PyASCIIObject *)(op))->wstr) ? \
+ PyUnicode_WSTR_LENGTH(op) : \
+ ((void)PyUnicode_AsUnicode((PyObject *)(op)), \
+ PyUnicode_WSTR_LENGTH(op)))
+
#define PyUnicode_GET_DATA_SIZE(op) \
- (assert(PyUnicode_Check(op)),(((PyUnicodeObject *)(op))->length * sizeof(Py_UNICODE)))
+ (PyUnicode_GET_SIZE(op) * Py_UNICODE_SIZE)
+
+/* Alias for PyUnicode_AsUnicode(). This will create a wchar_t/Py_UNICODE
+ representation on demand. Using this macro is very inefficient now,
+ try to port your code to use the new PyUnicode_*BYTE_DATA() macros or
+ use PyUnicode_WRITE() and PyUnicode_READ(). */
+
#define PyUnicode_AS_UNICODE(op) \
- (assert(PyUnicode_Check(op)),(((PyUnicodeObject *)(op))->str))
+ (assert(PyUnicode_Check(op)), \
+ (((PyASCIIObject *)(op))->wstr) ? (((PyASCIIObject *)(op))->wstr) : \
+ PyUnicode_AsUnicode((PyObject *)(op)))
+
#define PyUnicode_AS_DATA(op) \
- (assert(PyUnicode_Check(op)),((const char *)((PyUnicodeObject *)(op))->str))
+ ((const char *)(PyUnicode_AS_UNICODE(op)))
+
+
+/* --- Flexible String Representation Helper Macros (PEP 393) -------------- */
+
+/* Values for PyUnicodeObject.state: */
+
+/* Interning state. */
+#define SSTATE_NOT_INTERNED 0
+#define SSTATE_INTERNED_MORTAL 1
+#define SSTATE_INTERNED_IMMORTAL 2
+
+/* Return true if the string contains only ASCII characters, or 0 if not. The
+ string may be compact (PyUnicode_IS_COMPACT_ASCII) or not. No type checks
+ or Ready calls are performed. */
+#define PyUnicode_IS_ASCII(op) \
+ (((PyASCIIObject*)op)->state.ascii)
+
+/* Return true if the string is compact or 0 if not.
+ No type checks or Ready calls are performed. */
+#define PyUnicode_IS_COMPACT(op) \
+ (((PyASCIIObject*)(op))->state.compact)
+
+/* Return true if the string is a compact ASCII string (use PyASCIIObject
+ structure), or 0 if not. No type checks or Ready calls are performed. */
+#define PyUnicode_IS_COMPACT_ASCII(op) \
+ (PyUnicode_IS_ASCII(op) && PyUnicode_IS_COMPACT(op))
+
+enum PyUnicode_Kind {
+/* String contains only wstr byte characters. This is only possible
+ when the string was created with a legacy API and _PyUnicode_Ready()
+ has not been called yet. */
+ PyUnicode_WCHAR_KIND = 0,
+/* Return values of the PyUnicode_KIND() macro: */
+ PyUnicode_1BYTE_KIND = 1,
+ PyUnicode_2BYTE_KIND = 2,
+ PyUnicode_4BYTE_KIND = 4
+};
+
+/* Return pointers to the canonical representation cast to unsigned char,
+ Py_UCS2, or Py_UCS4 for direct character access.
+ No checks are performed, use PyUnicode_KIND() before to ensure
+ these will work correctly. */
+
+#define PyUnicode_1BYTE_DATA(op) ((Py_UCS1*)PyUnicode_DATA(op))
+#define PyUnicode_2BYTE_DATA(op) ((Py_UCS2*)PyUnicode_DATA(op))
+#define PyUnicode_4BYTE_DATA(op) ((Py_UCS4*)PyUnicode_DATA(op))
+
+/* Return one of the PyUnicode_*_KIND values defined above. */
+#define PyUnicode_KIND(op) \
+ (assert(PyUnicode_Check(op)), \
+ assert(PyUnicode_IS_READY(op)), \
+ ((PyASCIIObject *)(op))->state.kind)
+
+/* Return a void pointer to the raw unicode buffer. */
+#define _PyUnicode_COMPACT_DATA(op) \
+ (PyUnicode_IS_ASCII(op) ? \
+ ((void*)((PyASCIIObject*)(op) + 1)) : \
+ ((void*)((PyCompactUnicodeObject*)(op) + 1)))
+
+#define _PyUnicode_NONCOMPACT_DATA(op) \
+ (assert(((PyUnicodeObject*)(op))->data.any), \
+ ((((PyUnicodeObject *)(op))->data.any)))
+
+#define PyUnicode_DATA(op) \
+ (assert(PyUnicode_Check(op)), \
+ PyUnicode_IS_COMPACT(op) ? _PyUnicode_COMPACT_DATA(op) : \
+ _PyUnicode_NONCOMPACT_DATA(op))
+
+/* In the access macros below, "kind" may be evaluated more than once.
+ All other macro parameters are evaluated exactly once, so it is safe
+ to put side effects into them (such as increasing the index). */
+
+/* Write into the canonical representation, this macro does not do any sanity
+ checks and is intended for usage in loops. The caller should cache the
+ kind and data pointers obtained from other macro calls.
+ index is the index in the string (starts at 0) and value is the new
+ code point value which should be written to that location. */
+#define PyUnicode_WRITE(kind, data, index, value) \
+ do { \
+ switch ((kind)) { \
+ case PyUnicode_1BYTE_KIND: { \
+ ((Py_UCS1 *)(data))[(index)] = (Py_UCS1)(value); \
+ break; \
+ } \
+ case PyUnicode_2BYTE_KIND: { \
+ ((Py_UCS2 *)(data))[(index)] = (Py_UCS2)(value); \
+ break; \
+ } \
+ default: { \
+ assert((kind) == PyUnicode_4BYTE_KIND); \
+ ((Py_UCS4 *)(data))[(index)] = (Py_UCS4)(value); \
+ } \
+ } \
+ } while (0)
+
+/* Read a code point from the string's canonical representation. No checks
+ or ready calls are performed. */
+#define PyUnicode_READ(kind, data, index) \
+ ((Py_UCS4) \
+ ((kind) == PyUnicode_1BYTE_KIND ? \
+ ((const Py_UCS1 *)(data))[(index)] : \
+ ((kind) == PyUnicode_2BYTE_KIND ? \
+ ((const Py_UCS2 *)(data))[(index)] : \
+ ((const Py_UCS4 *)(data))[(index)] \
+ ) \
+ ))
+
+/* PyUnicode_READ_CHAR() is less efficient than PyUnicode_READ() because it
+ calls PyUnicode_KIND() and might call it twice. For single reads, use
+ PyUnicode_READ_CHAR, for multiple consecutive reads callers should
+ cache kind and use PyUnicode_READ instead. */
+#define PyUnicode_READ_CHAR(unicode, index) \
+ (assert(PyUnicode_Check(unicode)), \
+ assert(PyUnicode_IS_READY(unicode)), \
+ (Py_UCS4) \
+ (PyUnicode_KIND((unicode)) == PyUnicode_1BYTE_KIND ? \
+ ((const Py_UCS1 *)(PyUnicode_DATA((unicode))))[(index)] : \
+ (PyUnicode_KIND((unicode)) == PyUnicode_2BYTE_KIND ? \
+ ((const Py_UCS2 *)(PyUnicode_DATA((unicode))))[(index)] : \
+ ((const Py_UCS4 *)(PyUnicode_DATA((unicode))))[(index)] \
+ ) \
+ ))
+
+/* Returns the length of the unicode string. The caller has to make sure that
+ the string has it's canonical representation set before calling
+ this macro. Call PyUnicode_(FAST_)Ready to ensure that. */
+#define PyUnicode_GET_LENGTH(op) \
+ (assert(PyUnicode_Check(op)), \
+ assert(PyUnicode_IS_READY(op)), \
+ ((PyASCIIObject *)(op))->length)
+
+
+/* Fast check to determine whether an object is ready. Equivalent to
+ PyUnicode_IS_COMPACT(op) || ((PyUnicodeObject*)(op))->data.any) */
+
+#define PyUnicode_IS_READY(op) (((PyASCIIObject*)op)->state.ready)
+
+/* PyUnicode_READY() does less work than _PyUnicode_Ready() in the best
+ case. If the canonical representation is not yet set, it will still call
+ _PyUnicode_Ready().
+ Returns 0 on success and -1 on errors. */
+#define PyUnicode_READY(op) \
+ (assert(PyUnicode_Check(op)), \
+ (PyUnicode_IS_READY(op) ? \
+ 0 : _PyUnicode_Ready((PyObject *)(op))))
+
+/* Return a maximum character value which is suitable for creating another
+ string based on op. This is always an approximation but more efficient
+ than iterating over the string. */
+#define PyUnicode_MAX_CHAR_VALUE(op) \
+ (assert(PyUnicode_IS_READY(op)), \
+ (PyUnicode_IS_ASCII(op) ? \
+ (0x7f) : \
+ (PyUnicode_KIND(op) == PyUnicode_1BYTE_KIND ? \
+ (0xffU) : \
+ (PyUnicode_KIND(op) == PyUnicode_2BYTE_KIND ? \
+ (0xffffU) : \
+ (0x10ffffU)))))
+
#endif
/* --- Constants ---------------------------------------------------------- */
@@ -418,12 +567,68 @@ PyAPI_DATA(PyTypeObject) PyUnicodeIter_Type;
Unicode character U+FFFD is the official REPLACEMENT CHARACTER in
Unicode 3.0. */
-#define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UNICODE) 0xFFFD)
+#define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UCS4) 0xFFFD)
/* === Public API ========================================================= */
/* --- Plain Py_UNICODE --------------------------------------------------- */
+/* With PEP 393, this is the recommended way to allocate a new unicode object.
+ This function will allocate the object and its buffer in a single memory
+ block. Objects created using this function are not resizable. */
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject*) PyUnicode_New(
+ Py_ssize_t size, /* Number of code points in the new string */
+ Py_UCS4 maxchar /* maximum code point value in the string */
+ );
+#endif
+
+/* Initializes the canonical string representation from a the deprecated
+ wstr/Py_UNICODE representation. This function is used to convert Unicode
+ objects which were created using the old API to the new flexible format
+ introduced with PEP 393.
+
+ Don't call this function directly, use the public PyUnicode_READY() macro
+ instead. */
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(int) _PyUnicode_Ready(
+ PyObject *unicode /* Unicode object */
+ );
+#endif
+
+/* Get a copy of a Unicode string. */
+PyAPI_FUNC(PyObject*) PyUnicode_Copy(
+ PyObject *unicode
+ );
+
+/* Copy character from one unicode object into another, this function performs
+ character conversion when necessary and falls back to memcpy if possible.
+
+ Fail if to is too small (smaller than how_many or smaller than
+ len(from)-from_start), or if kind(from[from_start:from_start+how_many]) >
+ kind(to), or if to has more than 1 reference.
+
+ Return the number of written character, or return -1 and raise an exception
+ on error.
+
+ Pseudo-code:
+
+ how_many = min(how_many, len(from) - from_start)
+ to[to_start:to_start+how_many] = from[from_start:from_start+how_many]
+ return how_many
+
+ Note: The function doesn't write a terminating null character.
+ */
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(Py_ssize_t) PyUnicode_CopyCharacters(
+ PyObject *to,
+ Py_ssize_t to_start,
+ PyObject *from,
+ Py_ssize_t from_start,
+ Py_ssize_t how_many
+ );
+#endif
+
/* Create a Unicode Object from the Py_UNICODE buffer u of the given
size.
@@ -448,13 +653,45 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromStringAndSize(
);
/* Similar to PyUnicode_FromUnicode(), but u points to null-terminated
- UTF-8 encoded bytes */
+ UTF-8 encoded bytes. The size is determined with strlen(). */
PyAPI_FUNC(PyObject*) PyUnicode_FromString(
const char *u /* UTF-8 encoded string */
);
+/* Create a new string from a buffer of Py_UCS1, Py_UCS2 or Py_UCS4 characters.
+ Scan the string to find the maximum character. */
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData(
+ int kind,
+ const void *buffer,
+ Py_ssize_t size);
+#endif
+
+PyAPI_FUNC(PyObject*) PyUnicode_Substring(
+ PyObject *str,
+ Py_ssize_t start,
+ Py_ssize_t end);
+
+/* Copy the string into a UCS4 buffer including the null character if copy_null
+ is set. Return NULL and raise an exception on error. Raise a ValueError if
+ the buffer is smaller than the string. Return buffer on success.
+
+ buflen is the length of the buffer in (Py_UCS4) characters. */
+PyAPI_FUNC(Py_UCS4*) PyUnicode_AsUCS4(
+ PyObject *unicode,
+ Py_UCS4* buffer,
+ Py_ssize_t buflen,
+ int copy_null);
+
+/* Copy the string into a UCS4 buffer. A new buffer is allocated using
+ * PyMem_Malloc; if this fails, NULL is returned with a memory error
+ exception set. */
+PyAPI_FUNC(Py_UCS4*) PyUnicode_AsUCS4Copy(PyObject *unicode);
+
/* Return a read-only pointer to the Unicode object's internal
- Py_UNICODE buffer. */
+ Py_UNICODE buffer.
+ If the wchar_t/Py_UNICODE representation is not yet available, this
+ function will calculate it. */
#ifndef Py_LIMITED_API
PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicode(
@@ -462,30 +699,70 @@ PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicode(
);
#endif
+/* Return a read-only pointer to the Unicode object's internal
+ Py_UNICODE buffer and save the length at size.
+ If the wchar_t/Py_UNICODE representation is not yet available, this
+ function will calculate it. */
+
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicodeAndSize(
+ PyObject *unicode, /* Unicode object */
+ Py_ssize_t *size /* location where to save the length */
+ );
+#endif
+
/* Get the length of the Unicode object. */
+PyAPI_FUNC(Py_ssize_t) PyUnicode_GetLength(
+ PyObject *unicode
+);
+
+/* Get the number of Py_UNICODE units in the
+ string representation. */
+
PyAPI_FUNC(Py_ssize_t) PyUnicode_GetSize(
PyObject *unicode /* Unicode object */
);
+/* Read a character from the string. */
+
+PyAPI_FUNC(Py_UCS4) PyUnicode_ReadChar(
+ PyObject *unicode,
+ Py_ssize_t index
+ );
+
+/* Write a character to the string. The string must have been created through
+ PyUnicode_New, must not be shared, and must not have been hashed yet.
+
+ Return 0 on success, -1 on error. */
+
+PyAPI_FUNC(int) PyUnicode_WriteChar(
+ PyObject *unicode,
+ Py_ssize_t index,
+ Py_UCS4 character
+ );
+
#ifndef Py_LIMITED_API
/* Get the maximum ordinal for a Unicode character. */
PyAPI_FUNC(Py_UNICODE) PyUnicode_GetMax(void);
#endif
-/* Resize an already allocated Unicode object to the new size length.
+/* Resize an Unicode object allocated by the legacy API (e.g.
+ PyUnicode_FromUnicode). Unicode objects allocated by the new API (e.g.
+ PyUnicode_New) cannot be resized by this function.
+
+ The length is a number of Py_UNICODE characters (and not the number of code
+ points).
*unicode is modified to point to the new (resized) object and 0
returned on success.
- This API may only be called by the function which also called the
- Unicode constructor. The refcount on the object must be 1. Otherwise,
- an error is returned.
+ If the refcount on the object is 1, the function resizes the string in
+ place, which is usually faster than allocating a new string (and copy
+ characters).
Error handling is implemented as follows: an exception is set, -1
- is returned and *unicode left untouched.
-
-*/
+ is returned and *unicode left untouched. */
PyAPI_FUNC(int) PyUnicode_Resize(
PyObject **unicode, /* Pointer to the Unicode object */
@@ -545,8 +822,9 @@ PyAPI_FUNC(PyObject *) PyUnicode_FromFormat(
/* Format the object based on the format_spec, as defined in PEP 3101
(Advanced String Formatting). */
PyAPI_FUNC(PyObject *) _PyUnicode_FormatAdvanced(PyObject *obj,
- Py_UNICODE *format_spec,
- Py_ssize_t format_spec_len);
+ PyObject *format_spec,
+ Py_ssize_t start,
+ Py_ssize_t end);
#endif
PyAPI_FUNC(void) PyUnicode_InternInPlace(PyObject **);
@@ -559,7 +837,8 @@ PyAPI_FUNC(void) _Py_ReleaseInternedUnicodeStrings(void);
#endif
/* Use only if you know it's a string */
-#define PyUnicode_CHECK_INTERNED(op) (((PyUnicodeObject *)(op))->state)
+#define PyUnicode_CHECK_INTERNED(op) \
+ (((PyASCIIObject *)(op))->state.interned)
/* --- wchar_t support for platforms which support it --------------------- */
@@ -606,6 +885,10 @@ PyAPI_FUNC(wchar_t*) PyUnicode_AsWideCharString(
Py_ssize_t *size /* number of characters of the result */
);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void*) _PyUnicode_AsKind(PyObject *s, unsigned int kind);
+#endif
+
#endif
/* --- Unicode ordinals --------------------------------------------------- */
@@ -651,50 +934,42 @@ PyAPI_FUNC(int) PyUnicode_ClearFreeList(void);
/* --- Manage the default encoding ---------------------------------------- */
-/* Return a Python string holding the default encoded value of the
- Unicode object.
-
- Same as PyUnicode_AsUTF8String() except
- the resulting string is cached in the Unicode object for subsequent
- usage by this function. The cached version is needed to implement
- the character buffer interface and will live (at least) as long as
- the Unicode object itself.
-
- The refcount of the string is *not* incremented.
-
- *** Exported for internal use by the interpreter only !!! ***
-
-*/
-
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject *) _PyUnicode_AsDefaultEncodedString(
- PyObject *unicode,
- const char *errors);
-#endif
-
/* Returns a pointer to the default encoding (UTF-8) of the
Unicode object unicode and the size of the encoded representation
in bytes stored in *size.
In case of an error, no *size is set.
+ This function caches the UTF-8 encoded string in the unicodeobject
+ and subsequent calls will return the same string. The memory is released
+ when the unicodeobject is deallocated.
+
+ _PyUnicode_AsStringAndSize is a #define for PyUnicode_AsUTF8AndSize to
+ support the previous internal function with the same behaviour.
+
*** This API is for interpreter INTERNAL USE ONLY and will likely
*** be removed or changed in the future.
*** If you need to access the Unicode object as UTF-8 bytes string,
*** please use PyUnicode_AsUTF8String() instead.
-
*/
#ifndef Py_LIMITED_API
-PyAPI_FUNC(char *) _PyUnicode_AsStringAndSize(
+PyAPI_FUNC(char *) PyUnicode_AsUTF8AndSize(
PyObject *unicode,
Py_ssize_t *size);
+#define _PyUnicode_AsStringAndSize PyUnicode_AsUTF8AndSize
#endif
/* Returns a pointer to the default encoding (UTF-8) of the
Unicode object unicode.
+ Like PyUnicode_AsUTF8AndSize(), this also caches the UTF-8 representation
+ in the unicodeobject.
+
+ _PyUnicode_AsString is a #define for PyUnicode_AsUTF8 to
+ support the previous internal function with the same behaviour.
+
Use of this API is DEPRECATED since no size information can be
extracted from the returned data.
@@ -707,7 +982,8 @@ PyAPI_FUNC(char *) _PyUnicode_AsStringAndSize(
*/
#ifndef Py_LIMITED_API
-PyAPI_FUNC(char *) _PyUnicode_AsString(PyObject *unicode);
+PyAPI_FUNC(char *) PyUnicode_AsUTF8(PyObject *unicode);
+#define _PyUnicode_AsString PyUnicode_AsUTF8
#endif
/* Returns "utf-8". */
@@ -834,6 +1110,10 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsUTF8String(
);
#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject*) _PyUnicode_AsUTF8String(
+ PyObject *unicode,
+ const char *errors);
+
PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF8(
const Py_UNICODE *data, /* Unicode char buffer */
Py_ssize_t length, /* number of Py_UNICODE chars to encode */
@@ -1064,6 +1344,10 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsLatin1String(
);
#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject*) _PyUnicode_AsLatin1String(
+ PyObject* unicode,
+ const char* errors);
+
PyAPI_FUNC(PyObject*) PyUnicode_EncodeLatin1(
const Py_UNICODE *data, /* Unicode char buffer */
Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
@@ -1088,6 +1372,10 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsASCIIString(
);
#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject*) _PyUnicode_AsASCIIString(
+ PyObject* unicode,
+ const char* errors);
+
PyAPI_FUNC(PyObject*) PyUnicode_EncodeASCII(
const Py_UNICODE *data, /* Unicode char buffer */
Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
@@ -1139,6 +1427,12 @@ PyAPI_FUNC(PyObject*) PyUnicode_EncodeCharmap(
(unicode ordinal -> char ordinal) */
const char *errors /* error handling */
);
+PyAPI_FUNC(PyObject*) _PyUnicode_EncodeCharmap(
+ PyObject *unicode, /* Unicode object */
+ PyObject *mapping, /* character mapping
+ (unicode ordinal -> char ordinal) */
+ const char *errors /* error handling */
+ );
#endif
/* Translate a Py_UNICODE buffer of the given length by applying a
@@ -1163,7 +1457,7 @@ PyAPI_FUNC(PyObject *) PyUnicode_TranslateCharmap(
);
#endif
-#ifdef MS_WIN32
+#ifdef HAVE_MBCS
/* --- MBCS codecs for Windows -------------------------------------------- */
@@ -1180,6 +1474,14 @@ PyAPI_FUNC(PyObject*) PyUnicode_DecodeMBCSStateful(
Py_ssize_t *consumed /* bytes consumed */
);
+PyAPI_FUNC(PyObject*) PyUnicode_DecodeCodePageStateful(
+ int code_page, /* code page number */
+ const char *string, /* encoded string */
+ Py_ssize_t length, /* size of string */
+ const char *errors, /* error handling */
+ Py_ssize_t *consumed /* bytes consumed */
+ );
+
PyAPI_FUNC(PyObject*) PyUnicode_AsMBCSString(
PyObject *unicode /* Unicode object */
);
@@ -1187,12 +1489,18 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsMBCSString(
#ifndef Py_LIMITED_API
PyAPI_FUNC(PyObject*) PyUnicode_EncodeMBCS(
const Py_UNICODE *data, /* Unicode char buffer */
- Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
+ Py_ssize_t length, /* number of Py_UNICODE chars to encode */
const char *errors /* error handling */
);
#endif
-#endif /* MS_WIN32 */
+PyAPI_FUNC(PyObject*) PyUnicode_EncodeCodePage(
+ int code_page, /* code page number */
+ PyObject *unicode, /* Unicode object */
+ const char *errors /* error handling */
+ );
+
+#endif /* HAVE_MBCS */
/* --- Decimal Encoder ---------------------------------------------------- */
@@ -1240,6 +1548,17 @@ PyAPI_FUNC(PyObject*) PyUnicode_TransformDecimalToASCII(
);
#endif
+/* Similar to PyUnicode_TransformDecimalToASCII(), but takes a PyUnicodeObject
+ as argument instead of a raw buffer and length. This function additionally
+ transforms spaces to ASCII because this is what the callers in longobject,
+ floatobject, and complexobject did anyways. */
+
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII(
+ PyObject *unicode /* Unicode object */
+ );
+#endif
+
/* --- File system encoding ---------------------------------------------- */
/* ParseTuple converter: encode str objects to bytes using
@@ -1292,7 +1611,7 @@ PyAPI_FUNC(PyObject*) PyUnicode_EncodeFSDefault(
These are capable of handling Unicode objects and strings on input
(we refer to them as strings in the descriptions) and return
- Unicode objects or integers as apporpriate. */
+ Unicode objects or integers as appropriate. */
/* Concat two strings giving a new Unicode string. */
@@ -1427,6 +1746,15 @@ PyAPI_FUNC(Py_ssize_t) PyUnicode_Find(
int direction /* Find direction: +1 forward, -1 backward */
);
+/* Like PyUnicode_Find, but search for single character only. */
+PyAPI_FUNC(Py_ssize_t) PyUnicode_FindChar(
+ PyObject *str,
+ Py_UCS4 ch,
+ Py_ssize_t start,
+ Py_ssize_t end,
+ int direction
+ );
+
/* Count the number of occurrences of substr in str[start:end]. */
PyAPI_FUNC(Py_ssize_t) PyUnicode_Count(
@@ -1463,7 +1791,7 @@ PyAPI_FUNC(int) PyUnicode_CompareWithASCIIString(
/* Rich compare two strings and return one of the following:
- NULL in case an exception was raised
- - Py_True or Py_False for successfuly comparisons
+ - Py_True or Py_False for successfully comparisons
- Py_NotImplemented in case the type combination is unknown
Note that Py_EQ and Py_NE comparisons can cause a UnicodeWarning in
@@ -1508,7 +1836,7 @@ PyAPI_FUNC(int) PyUnicode_IsIdentifier(PyObject *s);
#ifndef Py_LIMITED_API
/* Externally visible for str.strip(unicode) */
PyAPI_FUNC(PyObject *) _PyUnicode_XStrip(
- PyUnicodeObject *self,
+ PyObject *self,
int striptype,
PyObject *sepobj
);
@@ -1530,13 +1858,16 @@ PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGroupingLocale(Py_UNICODE *buff
into the string pointed to by buffer. For the argument descriptions,
see Objects/stringlib/localeutil.h */
#ifndef Py_LIMITED_API
-PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGrouping(Py_UNICODE *buffer,
- Py_ssize_t n_buffer,
- Py_UNICODE *digits,
- Py_ssize_t n_digits,
- Py_ssize_t min_width,
- const char *grouping,
- const char *thousands_sep);
+PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGrouping(
+ PyObject *unicode,
+ int kind,
+ void *buffer,
+ Py_ssize_t n_buffer,
+ void *digits,
+ Py_ssize_t n_digits,
+ Py_ssize_t min_width,
+ const char *grouping,
+ const char *thousands_sep);
#endif
/* === Characters Type APIs =============================================== */
@@ -1670,6 +2001,46 @@ PyAPI_FUNC(Py_UNICODE*) PyUnicode_AsUnicodeCopy(
);
#endif /* Py_LIMITED_API */
+#if defined(Py_DEBUG) && !defined(Py_LIMITED_API)
+PyAPI_FUNC(int) _PyUnicode_CheckConsistency(
+ PyObject *op,
+ int check_content);
+#endif
+
+/********************* String Literals ****************************************/
+/* This structure helps managing static strings. The basic usage goes like this:
+ Instead of doing
+
+ r = PyObject_CallMethod(o, "foo", "args", ...);
+
+ do
+
+ _Py_IDENTIFIER(foo);
+ ...
+ r = _PyObject_CallMethodId(o, &PyId_foo, "args", ...);
+
+ PyId_foo is a static variable, either on block level or file level. On first
+ usage, the string "foo" is interned, and the structures are linked. On interpreter
+ shutdown, all strings are released (through _PyUnicode_ClearStaticStrings).
+
+ Alternatively, _Py_static_string allows to choose the variable name.
+ _PyUnicode_FromId returns a new reference to the interned string.
+ _PyObject_{Get,Set,Has}AttrId are __getattr__ versions using _Py_Identifier*.
+*/
+typedef struct _Py_Identifier {
+ struct _Py_Identifier *next;
+ const char* string;
+ PyObject *object;
+} _Py_Identifier;
+
+#define _Py_static_string(varname, value) static _Py_Identifier varname = { 0, value, 0 }
+#define _Py_IDENTIFIER(varname) _Py_static_string(PyId_##varname, #varname)
+
+/* Return an interned Unicode object for an Identifier; may fail if there is no memory.*/
+PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*);
+/* Clear all static strings. */
+PyAPI_FUNC(void) _PyUnicode_ClearStaticStrings(void);
+
#ifdef __cplusplus
}
#endif
diff --git a/LICENSE b/LICENSE
index 48dbba7..3a37dd6 100644
--- a/LICENSE
+++ b/LICENSE
@@ -73,6 +73,7 @@ the various releases.
3.2 3.1 2011 PSF yes
3.2.1 3.2 2011 PSF yes
3.2.2 3.2.1 2011 PSF yes
+ 3.3 3.2 2012 PSF yes
Footnotes:
diff --git a/Lib/_dummy_thread.py b/Lib/_dummy_thread.py
index ed50520..13b1f26 100644
--- a/Lib/_dummy_thread.py
+++ b/Lib/_dummy_thread.py
@@ -24,11 +24,7 @@ TIMEOUT_MAX = 2**31
# imports are done when needed on a function-by-function basis. Since threads
# are disabled, the import lock should not be an issue anyway (??).
-class error(Exception):
- """Dummy implementation of _thread.error."""
-
- def __init__(self, *args):
- self.args = args
+error = RuntimeError
def start_new_thread(function, args, kwargs={}):
"""Dummy implementation of _thread.start_new_thread().
diff --git a/Lib/_pyio.py b/Lib/_pyio.py
index 3fa9325..fec1c6f 100644
--- a/Lib/_pyio.py
+++ b/Lib/_pyio.py
@@ -14,7 +14,6 @@ except ImportError:
import io
from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END)
-from errno import EINTR
# open() uses st_blksize whenever we can
DEFAULT_BUFFER_SIZE = 8 * 1024 # bytes
@@ -23,20 +22,12 @@ DEFAULT_BUFFER_SIZE = 8 * 1024 # bytes
# defined in io.py. We don't use real inheritance though, because we don't
# want to inherit the C implementations.
-
-class BlockingIOError(IOError):
-
- """Exception raised when I/O would block on a non-blocking I/O stream."""
-
- def __init__(self, errno, strerror, characters_written=0):
- super().__init__(errno, strerror)
- if not isinstance(characters_written, int):
- raise TypeError("characters_written must be a integer")
- self.characters_written = characters_written
+# Rebind for compatibility
+BlockingIOError = BlockingIOError
def open(file, mode="r", buffering=-1, encoding=None, errors=None,
- newline=None, closefd=True):
+ newline=None, closefd=True, opener=None):
r"""Open file and return a stream. Raise IOError upon failure.
@@ -131,6 +122,12 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
be kept open when the file is closed. This does not work when a file name is
given and must be True in that case.
+ A custom opener can be used by passing a callable as *opener*. The
+ underlying file descriptor for the file object is then obtained by calling
+ *opener* with (*file*, *flags*). *opener* must return an open file
+ descriptor (passing os.open as *opener* results in functionality similar to
+ passing None).
+
open() returns a file object whose type depends on the mode, and
through which the standard file operations such as reading and writing
are performed. When open() is used to open a file in a text mode ('w',
@@ -185,7 +182,7 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
(writing and "w" or "") +
(appending and "a" or "") +
(updating and "+" or ""),
- closefd)
+ closefd, opener=opener)
line_buffering = False
if buffering == 1 or buffering < 0 and raw.isatty():
buffering = -1
@@ -944,15 +941,19 @@ class BufferedReader(_BufferedIOMixin):
# Special case for when the number of bytes to read is unspecified.
if n is None or n == -1:
self._reset_read_buf()
+ if hasattr(self.raw, 'readall'):
+ chunk = self.raw.readall()
+ if chunk is None:
+ return buf[pos:] or None
+ else:
+ return buf[pos:] + chunk
chunks = [buf[pos:]] # Strip the consumed bytes.
current_size = 0
while True:
# Read until EOF or until read() would block.
try:
chunk = self.raw.read()
- except IOError as e:
- if e.errno != EINTR:
- raise
+ except InterruptedError:
continue
if chunk in empty_values:
nodata_val = chunk
@@ -974,9 +975,7 @@ class BufferedReader(_BufferedIOMixin):
while avail < n:
try:
chunk = self.raw.read(wanted)
- except IOError as e:
- if e.errno != EINTR:
- raise
+ except InterruptedError:
continue
if chunk in empty_values:
nodata_val = chunk
@@ -1009,9 +1008,7 @@ class BufferedReader(_BufferedIOMixin):
while True:
try:
current = self.raw.read(to_read)
- except IOError as e:
- if e.errno != EINTR:
- raise
+ except InterruptedError:
continue
break
if current:
@@ -1122,9 +1119,7 @@ class BufferedWriter(_BufferedIOMixin):
while self._write_buf:
try:
n = self.raw.write(self._write_buf)
- except IOError as e:
- if e.errno != EINTR:
- raise
+ except InterruptedError:
continue
if n > len(self._write_buf) or n < 0:
raise IOError("write() returned incorrect number of bytes")
@@ -1516,6 +1511,7 @@ class TextIOWrapper(TextIOBase):
self._snapshot = None # info for reconstructing decoder state
self._seekable = self._telling = self.buffer.seekable()
self._has_read1 = hasattr(self.buffer, 'read1')
+ self._b2cratio = 0.0
if self._seekable and self.writable():
position = self.buffer.tell()
@@ -1686,7 +1682,12 @@ class TextIOWrapper(TextIOBase):
else:
input_chunk = self.buffer.read(self._CHUNK_SIZE)
eof = not input_chunk
- self._set_decoded_chars(self._decoder.decode(input_chunk, eof))
+ decoded_chars = self._decoder.decode(input_chunk, eof)
+ self._set_decoded_chars(decoded_chars)
+ if decoded_chars:
+ self._b2cratio = len(input_chunk) / len(self._decoded_chars)
+ else:
+ self._b2cratio = 0.0
if self._telling:
# At the snapshot point, len(dec_buffer) bytes before the read,
@@ -1740,20 +1741,56 @@ class TextIOWrapper(TextIOBase):
# forward until it gives us enough decoded characters.
saved_state = decoder.getstate()
try:
+ # Fast search for an acceptable start point, close to our
+ # current pos.
+ # Rationale: calling decoder.decode() has a large overhead
+ # regardless of chunk size; we want the number of such calls to
+ # be O(1) in most situations (common decoders, non-crazy input).
+ # Actually, it will be exactly 1 for fixed-size codecs (all
+ # 8-bit codecs, also UTF-16 and UTF-32).
+ skip_bytes = int(self._b2cratio * chars_to_skip)
+ skip_back = 1
+ assert skip_bytes <= len(next_input)
+ while skip_bytes > 0:
+ decoder.setstate((b'', dec_flags))
+ # Decode up to temptative start point
+ n = len(decoder.decode(next_input[:skip_bytes]))
+ if n <= chars_to_skip:
+ b, d = decoder.getstate()
+ if not b:
+ # Before pos and no bytes buffered in decoder => OK
+ dec_flags = d
+ chars_to_skip -= n
+ break
+ # Skip back by buffered amount and reset heuristic
+ skip_bytes -= len(b)
+ skip_back = 1
+ else:
+ # We're too far ahead, skip back a bit
+ skip_bytes -= skip_back
+ skip_back = skip_back * 2
+ else:
+ skip_bytes = 0
+ decoder.setstate((b'', dec_flags))
+
# Note our initial start point.
- decoder.setstate((b'', dec_flags))
- start_pos = position
- start_flags, bytes_fed, chars_decoded = dec_flags, 0, 0
- need_eof = 0
+ start_pos = position + skip_bytes
+ start_flags = dec_flags
+ if chars_to_skip == 0:
+ # We haven't moved from the start point.
+ return self._pack_cookie(start_pos, start_flags)
# Feed the decoder one byte at a time. As we go, note the
# nearest "safe start point" before the current location
# (a point where the decoder has nothing buffered, so seek()
# can safely start from there and advance to this location).
- next_byte = bytearray(1)
- for next_byte[0] in next_input:
+ bytes_fed = 0
+ need_eof = 0
+ # Chars decoded since `start_pos`
+ chars_decoded = 0
+ for i in range(skip_bytes, len(next_input)):
bytes_fed += 1
- chars_decoded += len(decoder.decode(next_byte))
+ chars_decoded += len(decoder.decode(next_input[i:i+1]))
dec_buffer, dec_flags = decoder.getstate()
if not dec_buffer and chars_decoded <= chars_to_skip:
# Decoder buffer is empty, so this is a safe start point.
diff --git a/Lib/abc.py b/Lib/abc.py
index a6c2dc4..40f88b9 100644
--- a/Lib/abc.py
+++ b/Lib/abc.py
@@ -133,11 +133,14 @@ class ABCMeta(type):
return cls
def register(cls, subclass):
- """Register a virtual subclass of an ABC."""
+ """Register a virtual subclass of an ABC.
+
+ Returns the subclass, to allow usage as a class decorator.
+ """
if not isinstance(subclass, type):
raise TypeError("Can only register classes")
if issubclass(subclass, cls):
- return # Already a subclass
+ return subclass # Already a subclass
# Subtle: test for cycles *after* testing for "already a subclass";
# this means we allow X.register(X) and interpret it as a no-op.
if issubclass(cls, subclass):
@@ -145,6 +148,7 @@ class ABCMeta(type):
raise RuntimeError("Refusing to create an inheritance cycle")
cls._abc_registry.add(subclass)
ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache
+ return subclass
def _dump_registry(cls, file=None):
"""Debug helper to print the ABC registry."""
diff --git a/Lib/argparse.py b/Lib/argparse.py
index 2202b57..87d0cef 100644
--- a/Lib/argparse.py
+++ b/Lib/argparse.py
@@ -71,6 +71,7 @@ __all__ = [
'ArgumentDefaultsHelpFormatter',
'RawDescriptionHelpFormatter',
'RawTextHelpFormatter',
+ 'MetavarTypeHelpFormatter',
'Namespace',
'Action',
'ONE_OR_MORE',
@@ -419,7 +420,8 @@ class HelpFormatter(object):
# produce all arg strings
elif not action.option_strings:
- part = self._format_args(action, action.dest)
+ default = self._get_default_metavar_for_positional(action)
+ part = self._format_args(action, default)
# if it's in a group, strip the outer []
if action in group_actions:
@@ -441,7 +443,7 @@ class HelpFormatter(object):
# if the Optional takes a value, format is:
# -s ARGS or --long ARGS
else:
- default = action.dest.upper()
+ default = self._get_default_metavar_for_optional(action)
args_string = self._format_args(action, default)
part = '%s %s' % (option_string, args_string)
@@ -527,7 +529,8 @@ class HelpFormatter(object):
def _format_action_invocation(self, action):
if not action.option_strings:
- metavar, = self._metavar_formatter(action, action.dest)(1)
+ default = self._get_default_metavar_for_positional(action)
+ metavar, = self._metavar_formatter(action, default)(1)
return metavar
else:
@@ -541,7 +544,7 @@ class HelpFormatter(object):
# if the Optional takes a value, format is:
# -s ARGS, --long ARGS
else:
- default = action.dest.upper()
+ default = self._get_default_metavar_for_optional(action)
args_string = self._format_args(action, default)
for option_string in action.option_strings:
parts.append('%s %s' % (option_string, args_string))
@@ -619,6 +622,12 @@ class HelpFormatter(object):
def _get_help_string(self, action):
return action.help
+ def _get_default_metavar_for_optional(self, action):
+ return action.dest.upper()
+
+ def _get_default_metavar_for_positional(self, action):
+ return action.dest
+
class RawDescriptionHelpFormatter(HelpFormatter):
"""Help message formatter which retains any formatting in descriptions.
@@ -628,7 +637,7 @@ class RawDescriptionHelpFormatter(HelpFormatter):
"""
def _fill_text(self, text, width, indent):
- return ''.join([indent + line for line in text.splitlines(True)])
+ return ''.join(indent + line for line in text.splitlines(keepends=True))
class RawTextHelpFormatter(RawDescriptionHelpFormatter):
@@ -659,6 +668,22 @@ class ArgumentDefaultsHelpFormatter(HelpFormatter):
return help
+class MetavarTypeHelpFormatter(HelpFormatter):
+ """Help message formatter which uses the argument 'type' as the default
+ metavar value (instead of the argument 'dest')
+
+ Only the name of this class is considered a public API. All the methods
+ provided by the class are considered an implementation detail.
+ """
+
+ def _get_default_metavar_for_optional(self, action):
+ return action.type.__name__
+
+ def _get_default_metavar_for_positional(self, action):
+ return action.type.__name__
+
+
+
# =====================
# Options and Arguments
# =====================
@@ -1940,17 +1965,12 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
# if we didn't consume all the argument strings, there were extras
extras.extend(arg_strings[stop_index:])
- # if we didn't use all the Positional objects, there were too few
- # arg strings supplied.
- if positionals:
- self.error(_('too few arguments'))
-
# make sure all required actions were present
- for action in self._actions:
- if action.required:
- if action not in seen_actions:
- name = _get_action_name(action)
- self.error(_('argument %s is required') % name)
+ required_actions = [_get_action_name(action) for action in self._actions
+ if action.required and action not in seen_actions]
+ if required_actions:
+ self.error(_('the following arguments are required: %s') %
+ ', '.join(required_actions))
# make sure all required groups had one option present
for group in self._mutually_exclusive_groups:
diff --git a/Lib/ast.py b/Lib/ast.py
index fb5adac..13f59f9 100644
--- a/Lib/ast.py
+++ b/Lib/ast.py
@@ -25,7 +25,6 @@
:license: Python License.
"""
from _ast import *
-from _ast import __version__
def parse(source, filename='<unknown>', mode='exec'):
diff --git a/Lib/asynchat.py b/Lib/asynchat.py
index 6558512..2199d1b 100644
--- a/Lib/asynchat.py
+++ b/Lib/asynchat.py
@@ -75,7 +75,7 @@ class async_chat (asyncore.dispatcher):
# sign of an application bug that we don't want to pass silently
use_encoding = 0
- encoding = 'latin1'
+ encoding = 'latin-1'
def __init__ (self, sock=None, map=None):
# for string terminator matching
diff --git a/Lib/asyncore.py b/Lib/asyncore.py
index 7f42d39..6d4bbbe 100644
--- a/Lib/asyncore.py
+++ b/Lib/asyncore.py
@@ -54,7 +54,7 @@ import warnings
import os
from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \
- ENOTCONN, ESHUTDOWN, EINTR, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \
+ ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \
errorcode
_DISCONNECTED = frozenset((ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE,
@@ -143,11 +143,8 @@ def poll(timeout=0.0, map=None):
try:
r, w, e = select.select(r, w, e, timeout)
- except select.error as err:
- if err.args[0] != EINTR:
- raise
- else:
- return
+ except InterruptedError:
+ return
for fd in r:
obj = map.get(fd)
@@ -190,9 +187,7 @@ def poll2(timeout=0.0, map=None):
pollster.register(fd, flags)
try:
r = pollster.poll(timeout)
- except select.error as err:
- if err.args[0] != EINTR:
- raise
+ except InterruptedError:
r = []
for fd, flags in r:
obj = map.get(fd)
@@ -291,7 +286,7 @@ class dispatcher:
del map[fd]
self._fileno = None
- def create_socket(self, family, type):
+ def create_socket(self, family=socket.AF_INET, type=socket.SOCK_STREAM):
self.family_and_type = family, type
sock = socket.socket(family, type)
sock.setblocking(0)
diff --git a/Lib/binhex.py b/Lib/binhex.py
index 999a675..7bf9278 100644
--- a/Lib/binhex.py
+++ b/Lib/binhex.py
@@ -23,7 +23,6 @@ hexbin(inputfilename, outputfilename)
#
import io
import os
-import sys
import struct
import binascii
diff --git a/Lib/bz2.py b/Lib/bz2.py
new file mode 100644
index 0000000..cbf5233
--- /dev/null
+++ b/Lib/bz2.py
@@ -0,0 +1,413 @@
+"""Interface to the libbzip2 compression library.
+
+This module provides a file interface, classes for incremental
+(de)compression, and functions for one-shot (de)compression.
+"""
+
+__all__ = ["BZ2File", "BZ2Compressor", "BZ2Decompressor", "compress",
+ "decompress"]
+
+__author__ = "Nadeem Vawda <nadeem.vawda@gmail.com>"
+
+import io
+import threading
+import warnings
+
+from _bz2 import BZ2Compressor, BZ2Decompressor
+
+
+_MODE_CLOSED = 0
+_MODE_READ = 1
+_MODE_READ_EOF = 2
+_MODE_WRITE = 3
+
+_BUFFER_SIZE = 8192
+
+
+class BZ2File(io.BufferedIOBase):
+
+ """A file object providing transparent bzip2 (de)compression.
+
+ A BZ2File can act as a wrapper for an existing file object, or refer
+ directly to a named file on disk.
+
+ Note that BZ2File provides a *binary* file interface - data read is
+ returned as bytes, and data to be written should be given as bytes.
+ """
+
+ def __init__(self, filename=None, mode="r", buffering=None,
+ compresslevel=9, fileobj=None):
+ """Open a bzip2-compressed file.
+
+ If filename is given, open the named file. Otherwise, operate on
+ the file object given by fileobj. Exactly one of these two
+ parameters should be provided.
+
+ mode can be 'r' for reading (default), or 'w' for writing.
+
+ buffering is ignored. Its use is deprecated.
+
+ If mode is 'w', compresslevel can be a number between 1 and 9
+ specifying the level of compression: 1 produces the least
+ compression, and 9 (default) produces the most compression.
+ """
+ # This lock must be recursive, so that BufferedIOBase's
+ # readline(), readlines() and writelines() don't deadlock.
+ self._lock = threading.RLock()
+ self._fp = None
+ self._closefp = False
+ self._mode = _MODE_CLOSED
+ self._pos = 0
+ self._size = -1
+
+ if buffering is not None:
+ warnings.warn("Use of 'buffering' argument is deprecated",
+ DeprecationWarning)
+
+ if not (1 <= compresslevel <= 9):
+ raise ValueError("compresslevel must be between 1 and 9")
+
+ if mode in ("", "r", "rb"):
+ mode = "rb"
+ mode_code = _MODE_READ
+ self._decompressor = BZ2Decompressor()
+ self._buffer = None
+ elif mode in ("w", "wb"):
+ mode = "wb"
+ mode_code = _MODE_WRITE
+ self._compressor = BZ2Compressor(compresslevel)
+ elif mode in ("a", "ab"):
+ mode = "ab"
+ mode_code = _MODE_WRITE
+ self._compressor = BZ2Compressor(compresslevel)
+ else:
+ raise ValueError("Invalid mode: {!r}".format(mode))
+
+ if filename is not None and fileobj is None:
+ self._fp = open(filename, mode)
+ self._closefp = True
+ self._mode = mode_code
+ elif fileobj is not None and filename is None:
+ self._fp = fileobj
+ self._mode = mode_code
+ else:
+ raise ValueError("Must give exactly one of filename and fileobj")
+
+ def close(self):
+ """Flush and close the file.
+
+ May be called more than once without error. Once the file is
+ closed, any other operation on it will raise a ValueError.
+ """
+ with self._lock:
+ if self._mode == _MODE_CLOSED:
+ return
+ try:
+ if self._mode in (_MODE_READ, _MODE_READ_EOF):
+ self._decompressor = None
+ elif self._mode == _MODE_WRITE:
+ self._fp.write(self._compressor.flush())
+ self._compressor = None
+ finally:
+ try:
+ if self._closefp:
+ self._fp.close()
+ finally:
+ self._fp = None
+ self._closefp = False
+ self._mode = _MODE_CLOSED
+ self._buffer = None
+
+ @property
+ def closed(self):
+ """True if this file is closed."""
+ return self._mode == _MODE_CLOSED
+
+ def fileno(self):
+ """Return the file descriptor for the underlying file."""
+ return self._fp.fileno()
+
+ def seekable(self):
+ """Return whether the file supports seeking."""
+ return self.readable()
+
+ def readable(self):
+ """Return whether the file was opened for reading."""
+ return self._mode in (_MODE_READ, _MODE_READ_EOF)
+
+ def writable(self):
+ """Return whether the file was opened for writing."""
+ return self._mode == _MODE_WRITE
+
+ # Mode-checking helper functions.
+
+ def _check_not_closed(self):
+ if self.closed:
+ raise ValueError("I/O operation on closed file")
+
+ def _check_can_read(self):
+ if not self.readable():
+ self._check_not_closed()
+ raise io.UnsupportedOperation("File not open for reading")
+
+ def _check_can_write(self):
+ if not self.writable():
+ self._check_not_closed()
+ raise io.UnsupportedOperation("File not open for writing")
+
+ def _check_can_seek(self):
+ if not self.seekable():
+ self._check_not_closed()
+ raise io.UnsupportedOperation("Seeking is only supported "
+ "on files open for reading")
+
+ # Fill the readahead buffer if it is empty. Returns False on EOF.
+ def _fill_buffer(self):
+ if self._buffer:
+ return True
+
+ if self._decompressor.unused_data:
+ rawblock = self._decompressor.unused_data
+ else:
+ rawblock = self._fp.read(_BUFFER_SIZE)
+
+ if not rawblock:
+ if self._decompressor.eof:
+ self._mode = _MODE_READ_EOF
+ self._size = self._pos
+ return False
+ else:
+ raise EOFError("Compressed file ended before the "
+ "end-of-stream marker was reached")
+
+ # Continue to next stream.
+ if self._decompressor.eof:
+ self._decompressor = BZ2Decompressor()
+
+ self._buffer = self._decompressor.decompress(rawblock)
+ return True
+
+ # Read data until EOF.
+ # If return_data is false, consume the data without returning it.
+ def _read_all(self, return_data=True):
+ blocks = []
+ while self._fill_buffer():
+ if return_data:
+ blocks.append(self._buffer)
+ self._pos += len(self._buffer)
+ self._buffer = None
+ if return_data:
+ return b"".join(blocks)
+
+ # Read a block of up to n bytes.
+ # If return_data is false, consume the data without returning it.
+ def _read_block(self, n, return_data=True):
+ blocks = []
+ while n > 0 and self._fill_buffer():
+ if n < len(self._buffer):
+ data = self._buffer[:n]
+ self._buffer = self._buffer[n:]
+ else:
+ data = self._buffer
+ self._buffer = None
+ if return_data:
+ blocks.append(data)
+ self._pos += len(data)
+ n -= len(data)
+ if return_data:
+ return b"".join(blocks)
+
+ def peek(self, n=0):
+ """Return buffered data without advancing the file position.
+
+ Always returns at least one byte of data, unless at EOF.
+ The exact number of bytes returned is unspecified.
+ """
+ with self._lock:
+ self._check_can_read()
+ if self._mode == _MODE_READ_EOF or not self._fill_buffer():
+ return b""
+ return self._buffer
+
+ def read(self, size=-1):
+ """Read up to size uncompressed bytes from the file.
+
+ If size is negative or omitted, read until EOF is reached.
+ Returns b'' if the file is already at EOF.
+ """
+ with self._lock:
+ self._check_can_read()
+ if self._mode == _MODE_READ_EOF or size == 0:
+ return b""
+ elif size < 0:
+ return self._read_all()
+ else:
+ return self._read_block(size)
+
+ def read1(self, size=-1):
+ """Read up to size uncompressed bytes with at most one read
+ from the underlying stream.
+
+ Returns b'' if the file is at EOF.
+ """
+ with self._lock:
+ self._check_can_read()
+ if (size == 0 or self._mode == _MODE_READ_EOF or
+ not self._fill_buffer()):
+ return b""
+ if 0 < size < len(self._buffer):
+ data = self._buffer[:size]
+ self._buffer = self._buffer[size:]
+ else:
+ data = self._buffer
+ self._buffer = None
+ self._pos += len(data)
+ return data
+
+ def readinto(self, b):
+ """Read up to len(b) bytes into b.
+
+ Returns the number of bytes read (0 for EOF).
+ """
+ with self._lock:
+ return io.BufferedIOBase.readinto(self, b)
+
+ def readline(self, size=-1):
+ """Read a line of uncompressed bytes from the file.
+
+ The terminating newline (if present) is retained. If size is
+ non-negative, no more than size bytes will be read (in which
+ case the line may be incomplete). Returns b'' if already at EOF.
+ """
+ if not hasattr(size, "__index__"):
+ raise TypeError("Integer argument expected")
+ size = size.__index__()
+ with self._lock:
+ return io.BufferedIOBase.readline(self, size)
+
+ def readlines(self, size=-1):
+ """Read a list of lines of uncompressed bytes from the file.
+
+ size can be specified to control the number of lines read: no
+ further lines will be read once the total size of the lines read
+ so far equals or exceeds size.
+ """
+ if not hasattr(size, "__index__"):
+ raise TypeError("Integer argument expected")
+ size = size.__index__()
+ with self._lock:
+ return io.BufferedIOBase.readlines(self, size)
+
+ def write(self, data):
+ """Write a byte string to the file.
+
+ Returns the number of uncompressed bytes written, which is
+ always len(data). Note that due to buffering, the file on disk
+ may not reflect the data written until close() is called.
+ """
+ with self._lock:
+ self._check_can_write()
+ compressed = self._compressor.compress(data)
+ self._fp.write(compressed)
+ self._pos += len(data)
+ return len(data)
+
+ def writelines(self, seq):
+ """Write a sequence of byte strings to the file.
+
+ Returns the number of uncompressed bytes written.
+ seq can be any iterable yielding byte strings.
+
+ Line separators are not added between the written byte strings.
+ """
+ with self._lock:
+ return io.BufferedIOBase.writelines(self, seq)
+
+ # Rewind the file to the beginning of the data stream.
+ def _rewind(self):
+ self._fp.seek(0, 0)
+ self._mode = _MODE_READ
+ self._pos = 0
+ self._decompressor = BZ2Decompressor()
+ self._buffer = None
+
+ def seek(self, offset, whence=0):
+ """Change the file position.
+
+ The new position is specified by offset, relative to the
+ position indicated by whence. Values for whence are:
+
+ 0: start of stream (default); offset must not be negative
+ 1: current stream position
+ 2: end of stream; offset must not be positive
+
+ Returns the new file position.
+
+ Note that seeking is emulated, so depending on the parameters,
+ this operation may be extremely slow.
+ """
+ with self._lock:
+ self._check_can_seek()
+
+ # Recalculate offset as an absolute file position.
+ if whence == 0:
+ pass
+ elif whence == 1:
+ offset = self._pos + offset
+ elif whence == 2:
+ # Seeking relative to EOF - we need to know the file's size.
+ if self._size < 0:
+ self._read_all(return_data=False)
+ offset = self._size + offset
+ else:
+ raise ValueError("Invalid value for whence: {}".format(whence))
+
+ # Make it so that offset is the number of bytes to skip forward.
+ if offset < self._pos:
+ self._rewind()
+ else:
+ offset -= self._pos
+
+ # Read and discard data until we reach the desired position.
+ if self._mode != _MODE_READ_EOF:
+ self._read_block(offset, return_data=False)
+
+ return self._pos
+
+ def tell(self):
+ """Return the current file position."""
+ with self._lock:
+ self._check_not_closed()
+ return self._pos
+
+
+def compress(data, compresslevel=9):
+ """Compress a block of data.
+
+ compresslevel, if given, must be a number between 1 and 9.
+
+ For incremental compression, use a BZ2Compressor object instead.
+ """
+ comp = BZ2Compressor(compresslevel)
+ return comp.compress(data) + comp.flush()
+
+
+def decompress(data):
+ """Decompress a block of data.
+
+ For incremental decompression, use a BZ2Decompressor object instead.
+ """
+ if len(data) == 0:
+ return b""
+
+ results = []
+ while True:
+ decomp = BZ2Decompressor()
+ results.append(decomp.decompress(data))
+ if not decomp.eof:
+ raise ValueError("Compressed data ended before the "
+ "end-of-stream marker was reached")
+ if not decomp.unused_data:
+ return b"".join(results)
+ # There is unused data left over. Proceed to next stream.
+ data = decomp.unused_data
diff --git a/Lib/cgi.py b/Lib/cgi.py
index 90a3345..b3e32f1 100755
--- a/Lib/cgi.py
+++ b/Lib/cgi.py
@@ -76,7 +76,7 @@ def initlog(*allargs):
send an error message).
"""
- global logfp, log
+ global log, logfile, logfp
if logfile and not logfp:
try:
logfp = open(logfile, "a")
@@ -96,6 +96,15 @@ def nolog(*allargs):
"""Dummy function, assigned to log when logging is disabled."""
pass
+def closelog():
+ """Close the log file."""
+ global log, logfile, logfp
+ logfile = ''
+ if logfp:
+ logfp.close()
+ logfp = None
+ log = initlog
+
log = initlog # The current logging function
diff --git a/Lib/cgitb.py b/Lib/cgitb.py
index 7b52c8e..e3ce2cb 100644
--- a/Lib/cgitb.py
+++ b/Lib/cgitb.py
@@ -31,7 +31,6 @@ import tempfile
import time
import tokenize
import traceback
-import types
def reset():
"""Return a string that resets the CGI and browser to a known state."""
diff --git a/Lib/codecs.py b/Lib/codecs.py
index b150d64..e63a0c6 100644
--- a/Lib/codecs.py
+++ b/Lib/codecs.py
@@ -484,7 +484,7 @@ class StreamReader(Codec):
if firstline:
newchars, decodedbytes = \
self.decode(data[:exc.start], self.errors)
- lines = newchars.splitlines(True)
+ lines = newchars.splitlines(keepends=True)
if len(lines)<=1:
raise
else:
@@ -526,7 +526,7 @@ class StreamReader(Codec):
self.charbuffer = self.linebuffer[0]
self.linebuffer = None
if not keepends:
- line = line.splitlines(False)[0]
+ line = line.splitlines(keepends=False)[0]
return line
readsize = size or 72
@@ -543,7 +543,7 @@ class StreamReader(Codec):
data += self.read(size=1, chars=1)
line += data
- lines = line.splitlines(True)
+ lines = line.splitlines(keepends=True)
if lines:
if len(lines) > 1:
# More than one line result; the first line is a full line
@@ -559,10 +559,10 @@ class StreamReader(Codec):
# only one remaining line, put it back into charbuffer
self.charbuffer = lines[0] + self.charbuffer
if not keepends:
- line = line.splitlines(False)[0]
+ line = line.splitlines(keepends=False)[0]
break
line0withend = lines[0]
- line0withoutend = lines[0].splitlines(False)[0]
+ line0withoutend = lines[0].splitlines(keepends=False)[0]
if line0withend != line0withoutend: # We really have a line end
# Put the rest back together and keep it until the next call
self.charbuffer = self._empty_charbuffer.join(lines[1:]) + \
@@ -575,7 +575,7 @@ class StreamReader(Codec):
# we didn't get anything or this was our only try
if not data or size is not None:
if line and not keepends:
- line = line.splitlines(False)[0]
+ line = line.splitlines(keepends=False)[0]
break
if readsize < 8000:
readsize *= 2
@@ -803,7 +803,7 @@ class StreamRecoder:
data = self.reader.read()
data, bytesencoded = self.encode(data, self.errors)
- return data.splitlines(1)
+ return data.splitlines(keepends=True)
def __next__(self):
diff --git a/Lib/collections.py b/Lib/collections/__init__.py
index 2b6abd8..68b63a8 100644
--- a/Lib/collections.py
+++ b/Lib/collections/__init__.py
@@ -1,10 +1,11 @@
__all__ = ['deque', 'defaultdict', 'namedtuple', 'UserDict', 'UserList',
- 'UserString', 'Counter', 'OrderedDict']
-# For bootstrapping reasons, the collection ABCs are defined in _abcoll.py.
-# They should however be considered an integral part of collections.py.
-from _abcoll import *
-import _abcoll
-__all__ += _abcoll.__all__
+ 'UserString', 'Counter', 'OrderedDict', 'ChainMap']
+
+# For backwards compatibility, continue to make the collections ABCs
+# available through the collections module.
+from collections.abc import *
+import collections.abc
+__all__ += collections.abc.__all__
from _collections import deque, defaultdict
from operator import itemgetter as _itemgetter
@@ -364,8 +365,9 @@ def namedtuple(typename, field_names, verbose=False, rename=False):
except SyntaxError as e:
raise SyntaxError(e.msg + ':\n\n' + class_definition)
result = namespace[typename]
+ result._source = class_definition
if verbose:
- print(class_definition)
+ print(result._source)
# For pickling to work, the __module__ variable needs to be set to the frame
# where the named tuple is created. Bypass this step in enviroments where
@@ -670,12 +672,86 @@ class Counter(dict):
result[elem] = newcount
return result
+ def __pos__(self):
+ 'Adds an empty counter, effectively stripping negative and zero counts'
+ return self + Counter()
+
+ def __neg__(self):
+ '''Subtracts from an empty counter. Strips positive and zero counts,
+ and flips the sign on negative counts.
+
+ '''
+ return Counter() - self
+
+ def _keep_positive(self):
+ '''Internal method to strip elements with a negative or zero count'''
+ nonpositive = [elem for elem, count in self.items() if not count > 0]
+ for elem in nonpositive:
+ del self[elem]
+ return self
+
+ def __iadd__(self, other):
+ '''Inplace add from another counter, keeping only positive counts.
+
+ >>> c = Counter('abbb')
+ >>> c += Counter('bcc')
+ >>> c
+ Counter({'b': 4, 'c': 2, 'a': 1})
+
+ '''
+ for elem, count in other.items():
+ self[elem] += count
+ return self._keep_positive()
+
+ def __isub__(self, other):
+ '''Inplace subtract counter, but keep only results with positive counts.
+
+ >>> c = Counter('abbbc')
+ >>> c -= Counter('bccd')
+ >>> c
+ Counter({'b': 2, 'a': 1})
+
+ '''
+ for elem, count in other.items():
+ self[elem] -= count
+ return self._keep_positive()
+
+ def __ior__(self, other):
+ '''Inplace union is the maximum of value from either counter.
+
+ >>> c = Counter('abbb')
+ >>> c |= Counter('bcc')
+ >>> c
+ Counter({'b': 3, 'c': 2, 'a': 1})
+
+ '''
+ for elem, other_count in other.items():
+ count = self[elem]
+ if other_count > count:
+ self[elem] = other_count
+ return self._keep_positive()
+
+ def __iand__(self, other):
+ '''Inplace intersection is the minimum of corresponding counts.
+
+ >>> c = Counter('abbb')
+ >>> c &= Counter('bcc')
+ >>> c
+ Counter({'b': 1})
+
+ '''
+ for elem, count in self.items():
+ other_count = other[elem]
+ if other_count < count:
+ self[elem] = other_count
+ return self._keep_positive()
+
########################################################################
-### ChainMap (helper for configparser)
+### ChainMap (helper for configparser and string.Template)
########################################################################
-class _ChainMap(MutableMapping):
+class ChainMap(MutableMapping):
''' A ChainMap groups multiple dicts (or other mappings) together
to create a single, updateable view.
@@ -886,6 +962,8 @@ class UserList(MutableSequence):
def insert(self, i, item): self.data.insert(i, item)
def pop(self, i=-1): return self.data.pop(i)
def remove(self, item): self.data.remove(item)
+ def clear(self): self.data.clear()
+ def copy(self): return self.__class__(self)
def count(self, item): return self.data.count(item)
def index(self, item, *args): return self.data.index(item, *args)
def reverse(self): self.data.reverse()
@@ -1030,7 +1108,7 @@ class UserString(Sequence):
return self.data.split(sep, maxsplit)
def rsplit(self, sep=None, maxsplit=-1):
return self.data.rsplit(sep, maxsplit)
- def splitlines(self, keepends=0): return self.data.splitlines(keepends)
+ def splitlines(self, keepends=False): return self.data.splitlines(keepends)
def startswith(self, prefix, start=0, end=_sys.maxsize):
return self.data.startswith(prefix, start, end)
def strip(self, chars=None): return self.__class__(self.data.strip(chars))
diff --git a/Lib/_abcoll.py b/Lib/collections/abc.py
index 2417d18..7fbe84d 100644
--- a/Lib/_abcoll.py
+++ b/Lib/collections/abc.py
@@ -3,9 +3,7 @@
"""Abstract Base Classes (ABCs) for collections, according to PEP 3119.
-DON'T USE THIS MODULE DIRECTLY! The classes here should be imported
-via collections; they are defined here only to alleviate certain
-bootstrapping issues. Unit tests are in test_collections.
+Unit tests are in test_collections.
"""
from abc import ABCMeta, abstractmethod
@@ -48,6 +46,8 @@ dict_proxy = type(type.__dict__)
class Hashable(metaclass=ABCMeta):
+ __slots__ = ()
+
@abstractmethod
def __hash__(self):
return 0
@@ -65,6 +65,8 @@ class Hashable(metaclass=ABCMeta):
class Iterable(metaclass=ABCMeta):
+ __slots__ = ()
+
@abstractmethod
def __iter__(self):
while False:
@@ -80,6 +82,8 @@ class Iterable(metaclass=ABCMeta):
class Iterator(Iterable):
+ __slots__ = ()
+
@abstractmethod
def __next__(self):
raise StopIteration
@@ -111,6 +115,8 @@ Iterator.register(zip_iterator)
class Sized(metaclass=ABCMeta):
+ __slots__ = ()
+
@abstractmethod
def __len__(self):
return 0
@@ -125,6 +131,8 @@ class Sized(metaclass=ABCMeta):
class Container(metaclass=ABCMeta):
+ __slots__ = ()
+
@abstractmethod
def __contains__(self, x):
return False
@@ -139,6 +147,8 @@ class Container(metaclass=ABCMeta):
class Callable(metaclass=ABCMeta):
+ __slots__ = ()
+
@abstractmethod
def __call__(self, *args, **kwds):
return False
@@ -166,6 +176,8 @@ class Set(Sized, Iterable, Container):
then the other operations will automatically follow suit.
"""
+ __slots__ = ()
+
def __le__(self, other):
if not isinstance(other, Set):
return NotImplemented
@@ -277,6 +289,8 @@ Set.register(frozenset)
class MutableSet(Set):
+ __slots__ = ()
+
@abstractmethod
def add(self, value):
"""Add an element."""
@@ -350,6 +364,8 @@ MutableSet.register(set)
class Mapping(Sized, Iterable, Container):
+ __slots__ = ()
+
@abstractmethod
def __getitem__(self, key):
raise KeyError
@@ -453,6 +469,8 @@ ValuesView.register(dict_values)
class MutableMapping(Mapping):
+ __slots__ = ()
+
@abstractmethod
def __setitem__(self, key, value):
raise KeyError
@@ -532,6 +550,8 @@ class Sequence(Sized, Iterable, Container):
__getitem__, and __len__.
"""
+ __slots__ = ()
+
@abstractmethod
def __getitem__(self, index):
raise IndexError
@@ -577,12 +597,16 @@ class ByteString(Sequence):
XXX Should add all their methods.
"""
+ __slots__ = ()
+
ByteString.register(bytes)
ByteString.register(bytearray)
class MutableSequence(Sequence):
+ __slots__ = ()
+
@abstractmethod
def __setitem__(self, index, value):
raise IndexError
@@ -598,6 +622,13 @@ class MutableSequence(Sequence):
def append(self, value):
self.insert(len(self), value)
+ def clear(self):
+ try:
+ while True:
+ self.pop()
+ except IndexError:
+ pass
+
def reverse(self):
n = len(self)
for i in range(n//2):
diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py
index 79b91d4..6cfded3 100644
--- a/Lib/concurrent/futures/_base.py
+++ b/Lib/concurrent/futures/_base.py
@@ -536,15 +536,19 @@ class Executor(object):
fs = [self.submit(fn, *args) for args in zip(*iterables)]
- try:
- for future in fs:
- if timeout is None:
- yield future.result()
- else:
- yield future.result(end_time - time.time())
- finally:
- for future in fs:
- future.cancel()
+ # Yield must be hidden in closure so that the futures are submitted
+ # before the first iterator value is required.
+ def result_iterator():
+ try:
+ for future in fs:
+ if timeout is None:
+ yield future.result()
+ else:
+ yield future.result(end_time - time.time())
+ finally:
+ for future in fs:
+ future.cancel()
+ return result_iterator()
def shutdown(self, wait=True):
"""Clean-up the resources associated with the Executor.
diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py
index 8082940..9b2e0f3 100644
--- a/Lib/concurrent/futures/process.py
+++ b/Lib/concurrent/futures/process.py
@@ -46,9 +46,11 @@ Process #1..n:
__author__ = 'Brian Quinlan (brian@sweetapp.com)'
import atexit
+import os
from concurrent.futures import _base
import queue
import multiprocessing
+from multiprocessing.queues import SimpleQueue, SentinelReady, Full
import threading
import weakref
@@ -121,7 +123,7 @@ def _process_worker(call_queue, result_queue):
call_item = call_queue.get(block=True)
if call_item is None:
# Wake up queue management thread
- result_queue.put(None)
+ result_queue.put(os.getpid())
return
try:
r = call_item.fn(*call_item.args, **call_item.kwargs)
@@ -193,51 +195,92 @@ def _queue_management_worker(executor_reference,
result_queue: A multiprocessing.Queue of _ResultItems generated by the
process workers.
"""
- nb_shutdown_processes = 0
- def shutdown_one_process():
- """Tell a worker to terminate, which will in turn wake us again"""
- nonlocal nb_shutdown_processes
- call_queue.put(None)
- nb_shutdown_processes += 1
+ executor = None
+
+ def shutting_down():
+ return _shutdown or executor is None or executor._shutdown_thread
+
+ def shutdown_worker():
+ # This is an upper bound
+ nb_children_alive = sum(p.is_alive() for p in processes.values())
+ for i in range(0, nb_children_alive):
+ call_queue.put_nowait(None)
+ # Release the queue's resources as soon as possible.
+ call_queue.close()
+ # If .join() is not called on the created processes then
+ # some multiprocessing.Queue methods may deadlock on Mac OS X.
+ for p in processes.values():
+ p.join()
+
while True:
_add_call_item_to_queue(pending_work_items,
work_ids_queue,
call_queue)
- result_item = result_queue.get(block=True)
- if result_item is not None:
- work_item = pending_work_items[result_item.work_id]
- del pending_work_items[result_item.work_id]
-
- if result_item.exception:
- work_item.future.set_exception(result_item.exception)
- else:
- work_item.future.set_result(result_item.result)
- continue
- # If we come here, we either got a timeout or were explicitly woken up.
- # In either case, check whether we should start shutting down.
+ sentinels = [p.sentinel for p in processes.values()]
+ assert sentinels
+ try:
+ result_item = result_queue.get(sentinels=sentinels)
+ except SentinelReady as e:
+ # Mark the process pool broken so that submits fail right now.
+ executor = executor_reference()
+ if executor is not None:
+ executor._broken = True
+ executor._shutdown_thread = True
+ executor = None
+ # All futures in flight must be marked failed
+ for work_id, work_item in pending_work_items.items():
+ work_item.future.set_exception(
+ BrokenProcessPool(
+ "A process in the process pool was "
+ "terminated abruptly while the future was "
+ "running or pending."
+ ))
+ pending_work_items.clear()
+ # Terminate remaining workers forcibly: the queues or their
+ # locks may be in a dirty state and block forever.
+ for p in processes.values():
+ p.terminate()
+ shutdown_worker()
+ return
+ if isinstance(result_item, int):
+ # Clean shutdown of a worker using its PID
+ # (avoids marking the executor broken)
+ assert shutting_down()
+ p = processes.pop(result_item)
+ p.join()
+ if not processes:
+ shutdown_worker()
+ return
+ elif result_item is not None:
+ work_item = pending_work_items.pop(result_item.work_id, None)
+ # work_item can be None if another process terminated (see above)
+ if work_item is not None:
+ if result_item.exception:
+ work_item.future.set_exception(result_item.exception)
+ else:
+ work_item.future.set_result(result_item.result)
+ # Check whether we should start shutting down.
executor = executor_reference()
# No more work items can be added if:
# - The interpreter is shutting down OR
# - The executor that owns this worker has been collected OR
# - The executor that owns this worker has been shutdown.
- if _shutdown or executor is None or executor._shutdown_thread:
- # Since no new work items can be added, it is safe to shutdown
- # this thread if there are no pending work items.
- if not pending_work_items:
- while nb_shutdown_processes < len(processes):
- shutdown_one_process()
- # If .join() is not called on the created processes then
- # some multiprocessing.Queue methods may deadlock on Mac OS
- # X.
- for p in processes:
- p.join()
- call_queue.close()
- return
- else:
- # Start shutting down by telling a process it can exit.
- shutdown_one_process()
- del executor
+ if shutting_down():
+ try:
+ # Since no new work items can be added, it is safe to shutdown
+ # this thread if there are no pending work items.
+ if not pending_work_items:
+ shutdown_worker()
+ return
+ else:
+ # Start shutting down by telling a process it can exit.
+ call_queue.put_nowait(None)
+ except Full:
+ # This is not a problem: we will eventually be woken up (in
+ # result_queue.get()) and be able to send a sentinel again.
+ pass
+ executor = None
_system_limits_checked = False
_system_limited = None
@@ -264,6 +307,14 @@ def _check_system_limits():
_system_limited = "system provides too few semaphores (%d available, 256 necessary)" % nsems_max
raise NotImplementedError(_system_limited)
+
+class BrokenProcessPool(RuntimeError):
+ """
+ Raised when a process in a ProcessPoolExecutor terminated abruptly
+ while a future was in the running state.
+ """
+
+
class ProcessPoolExecutor(_base.Executor):
def __init__(self, max_workers=None):
"""Initializes a new ProcessPoolExecutor instance.
@@ -285,14 +336,20 @@ class ProcessPoolExecutor(_base.Executor):
# because futures in the call queue cannot be cancelled.
self._call_queue = multiprocessing.Queue(self._max_workers +
EXTRA_QUEUED_CALLS)
- self._result_queue = multiprocessing.Queue()
+ # Killed worker processes can produce spurious "broken pipe"
+ # tracebacks in the queue's own worker thread. But we detect killed
+ # processes anyway, so silence the tracebacks.
+ self._call_queue._ignore_epipe = True
+ self._result_queue = SimpleQueue()
self._work_ids = queue.Queue()
self._queue_management_thread = None
- self._processes = set()
+ # Map of pids to processes
+ self._processes = {}
# Shutdown is a two-step process.
self._shutdown_thread = False
self._shutdown_lock = threading.Lock()
+ self._broken = False
self._queue_count = 0
self._pending_work_items = {}
@@ -302,6 +359,8 @@ class ProcessPoolExecutor(_base.Executor):
def weakref_cb(_, q=self._result_queue):
q.put(None)
if self._queue_management_thread is None:
+ # Start the processes so that their sentinels are known.
+ self._adjust_process_count()
self._queue_management_thread = threading.Thread(
target=_queue_management_worker,
args=(weakref.ref(self, weakref_cb),
@@ -321,10 +380,13 @@ class ProcessPoolExecutor(_base.Executor):
args=(self._call_queue,
self._result_queue))
p.start()
- self._processes.add(p)
+ self._processes[p.pid] = p
def submit(self, fn, *args, **kwargs):
with self._shutdown_lock:
+ if self._broken:
+ raise BrokenProcessPool('A child process terminated '
+ 'abruptly, the process pool is not usable anymore')
if self._shutdown_thread:
raise RuntimeError('cannot schedule new futures after shutdown')
@@ -338,7 +400,6 @@ class ProcessPoolExecutor(_base.Executor):
self._result_queue.put(None)
self._start_queue_management_thread()
- self._adjust_process_count()
return f
submit.__doc__ = _base.Executor.submit.__doc__
diff --git a/Lib/configparser.py b/Lib/configparser.py
index d148b88..7bc4398 100644
--- a/Lib/configparser.py
+++ b/Lib/configparser.py
@@ -119,7 +119,8 @@ ConfigParser -- responsible for parsing a list of
between keys and values are surrounded by spaces.
"""
-from collections import MutableMapping, OrderedDict as _default_dict, _ChainMap
+from collections.abc import MutableMapping
+from collections import OrderedDict as _default_dict, ChainMap as _ChainMap
import functools
import io
import itertools
diff --git a/Lib/contextlib.py b/Lib/contextlib.py
index 5ebbbc6..2f8f00d 100644
--- a/Lib/contextlib.py
+++ b/Lib/contextlib.py
@@ -2,7 +2,6 @@
import sys
from functools import wraps
-from warnings import warn
__all__ = ["contextmanager", "closing", "ContextDecorator"]
diff --git a/Lib/copy.py b/Lib/copy.py
index 089d101..d96201e 100644
--- a/Lib/copy.py
+++ b/Lib/copy.py
@@ -173,8 +173,10 @@ def deepcopy(x, memo=None, _nil=[]):
"un(deep)copyable object of type %s" % cls)
y = _reconstruct(x, rv, 1, memo)
- memo[d] = y
- _keep_alive(x, memo) # Make sure x lives at least as long as d
+ # If is its own copy, don't memoize.
+ if y is not x:
+ memo[d] = y
+ _keep_alive(x, memo) # Make sure x lives at least as long as d
return y
_deepcopy_dispatch = d = {}
@@ -214,9 +216,10 @@ def _deepcopy_tuple(x, memo):
y = []
for a in x:
y.append(deepcopy(a, memo))
- d = id(x)
+ # We're not going to put the tuple in the memo, but it's still important we
+ # check for it, in case the tuple contains recursive mutable structures.
try:
- return memo[d]
+ return memo[id(x)]
except KeyError:
pass
for i in range(len(x)):
@@ -225,7 +228,6 @@ def _deepcopy_tuple(x, memo):
break
else:
y = x
- memo[d] = y
return y
d[tuple] = _deepcopy_tuple
@@ -321,68 +323,3 @@ del types
# Helper for instance creation without calling __init__
class _EmptyClass:
pass
-
-def _test():
- l = [None, 1, 2, 3.14, 'xyzzy', (1, 2), [3.14, 'abc'],
- {'abc': 'ABC'}, (), [], {}]
- l1 = copy(l)
- print(l1==l)
- l1 = map(copy, l)
- print(l1==l)
- l1 = deepcopy(l)
- print(l1==l)
- class C:
- def __init__(self, arg=None):
- self.a = 1
- self.arg = arg
- if __name__ == '__main__':
- import sys
- file = sys.argv[0]
- else:
- file = __file__
- self.fp = open(file)
- self.fp.close()
- def __getstate__(self):
- return {'a': self.a, 'arg': self.arg}
- def __setstate__(self, state):
- for key, value in state.items():
- setattr(self, key, value)
- def __deepcopy__(self, memo=None):
- new = self.__class__(deepcopy(self.arg, memo))
- new.a = self.a
- return new
- c = C('argument sketch')
- l.append(c)
- l2 = copy(l)
- print(l == l2)
- print(l)
- print(l2)
- l2 = deepcopy(l)
- print(l == l2)
- print(l)
- print(l2)
- l.append({l[1]: l, 'xyz': l[2]})
- l3 = copy(l)
- import reprlib
- print(map(reprlib.repr, l))
- print(map(reprlib.repr, l1))
- print(map(reprlib.repr, l2))
- print(map(reprlib.repr, l3))
- l3 = deepcopy(l)
- print(map(reprlib.repr, l))
- print(map(reprlib.repr, l1))
- print(map(reprlib.repr, l2))
- print(map(reprlib.repr, l3))
- class odict(dict):
- def __init__(self, d = {}):
- self.a = 99
- dict.__init__(self, d)
- def __setitem__(self, k, i):
- dict.__setitem__(self, k, i)
- self.a
- o = odict({"A" : "B"})
- x = deepcopy(o)
- print(o, x)
-
-if __name__ == '__main__':
- _test()
diff --git a/Lib/crypt.py b/Lib/crypt.py
new file mode 100644
index 0000000..e65b0cb
--- /dev/null
+++ b/Lib/crypt.py
@@ -0,0 +1,62 @@
+"""Wrapper to the POSIX crypt library call and associated functionality."""
+
+import _crypt
+import string
+from random import choice
+from collections import namedtuple
+
+
+_saltchars = string.ascii_letters + string.digits + './'
+
+
+class _Method(namedtuple('_Method', 'name ident salt_chars total_size')):
+
+ """Class representing a salt method per the Modular Crypt Format or the
+ legacy 2-character crypt method."""
+
+ def __repr__(self):
+ return '<crypt.METHOD_{}>'.format(self.name)
+
+
+
+def mksalt(method=None):
+ """Generate a salt for the specified method.
+
+ If not specified, the strongest available method will be used.
+
+ """
+ if method is None:
+ method = methods[0]
+ s = '${}$'.format(method.ident) if method.ident else ''
+ s += ''.join(choice(_saltchars) for _ in range(method.salt_chars))
+ return s
+
+
+def crypt(word, salt=None):
+ """Return a string representing the one-way hash of a password, with a salt
+ prepended.
+
+ If ``salt`` is not specified or is ``None``, the strongest
+ available method will be selected and a salt generated. Otherwise,
+ ``salt`` may be one of the ``crypt.METHOD_*`` values, or a string as
+ returned by ``crypt.mksalt()``.
+
+ """
+ if salt is None or isinstance(salt, _Method):
+ salt = mksalt(salt)
+ return _crypt.crypt(word, salt)
+
+
+# available salting/crypto methods
+METHOD_CRYPT = _Method('CRYPT', None, 2, 13)
+METHOD_MD5 = _Method('MD5', '1', 8, 34)
+METHOD_SHA256 = _Method('SHA256', '5', 16, 63)
+METHOD_SHA512 = _Method('SHA512', '6', 16, 106)
+
+methods = []
+for _method in (METHOD_SHA512, METHOD_SHA256, METHOD_MD5):
+ _result = crypt('', _method)
+ if _result and len(_result) == _method.total_size:
+ methods.append(_method)
+methods.append(METHOD_CRYPT)
+del _result, _method
diff --git a/Lib/ctypes/test/test_callbacks.py b/Lib/ctypes/test/test_callbacks.py
index c7207ea..5600b43 100644
--- a/Lib/ctypes/test/test_callbacks.py
+++ b/Lib/ctypes/test/test_callbacks.py
@@ -140,7 +140,7 @@ class Callbacks(unittest.TestCase):
def __del__(self):
gc.collect()
CFUNCTYPE(None)(lambda x=Nasty(): None)
-
+
try:
WINFUNCTYPE
diff --git a/Lib/ctypes/test/test_memfunctions.py b/Lib/ctypes/test/test_memfunctions.py
index aa2113b..aec4aaa 100644
--- a/Lib/ctypes/test/test_memfunctions.py
+++ b/Lib/ctypes/test/test_memfunctions.py
@@ -1,4 +1,5 @@
import sys
+from test import support
import unittest
from ctypes import *
@@ -49,6 +50,7 @@ class MemFunctionsTest(unittest.TestCase):
self.assertEqual(cast(a, POINTER(c_byte))[:7:7],
[97])
+ @support.refcount_test
def test_string_at(self):
s = string_at(b"foo bar")
# XXX The following may be wrong, depending on how Python
diff --git a/Lib/ctypes/test/test_parameters.py b/Lib/ctypes/test/test_parameters.py
index e83fd9a..9762fb9 100644
--- a/Lib/ctypes/test/test_parameters.py
+++ b/Lib/ctypes/test/test_parameters.py
@@ -73,13 +73,10 @@ class SimpleTypesTestCase(unittest.TestCase):
except ImportError:
## print "(No c_wchar_p)"
return
- s = "123"
- if sys.platform == "win32":
- self.assertTrue(c_wchar_p.from_param(s)._obj is s)
- self.assertRaises(TypeError, c_wchar_p.from_param, 42)
- # new in 0.9.1: convert (decode) ascii to unicode
- self.assertEqual(c_wchar_p.from_param("123")._obj, "123")
+ c_wchar_p.from_param("123")
+
+ self.assertRaises(TypeError, c_wchar_p.from_param, 42)
self.assertRaises(TypeError, c_wchar_p.from_param, b"123\377")
pa = c_wchar_p.from_param(c_wchar_p("123"))
diff --git a/Lib/ctypes/test/test_python_api.py b/Lib/ctypes/test/test_python_api.py
index 1f4c603..9de3980 100644
--- a/Lib/ctypes/test/test_python_api.py
+++ b/Lib/ctypes/test/test_python_api.py
@@ -1,5 +1,6 @@
from ctypes import *
import unittest, sys
+from test import support
from ctypes.test import is_resource_enabled
################################################################
@@ -25,6 +26,7 @@ class PythonAPITestCase(unittest.TestCase):
self.assertEqual(PyBytes_FromStringAndSize(b"abcdefghi", 3), b"abc")
+ @support.refcount_test
def test_PyString_FromString(self):
pythonapi.PyBytes_FromString.restype = py_object
pythonapi.PyBytes_FromString.argtypes = (c_char_p,)
@@ -56,6 +58,7 @@ class PythonAPITestCase(unittest.TestCase):
del res
self.assertEqual(grc(42), ref42)
+ @support.refcount_test
def test_PyObj_FromPtr(self):
s = "abc def ghi jkl"
ref = grc(s)
diff --git a/Lib/ctypes/test/test_refcounts.py b/Lib/ctypes/test/test_refcounts.py
index 35a81aa..5613e7a 100644
--- a/Lib/ctypes/test/test_refcounts.py
+++ b/Lib/ctypes/test/test_refcounts.py
@@ -1,4 +1,5 @@
import unittest
+from test import support
import ctypes
import gc
@@ -10,6 +11,7 @@ dll = ctypes.CDLL(_ctypes_test.__file__)
class RefcountTestCase(unittest.TestCase):
+ @support.refcount_test
def test_1(self):
from sys import getrefcount as grc
@@ -34,6 +36,7 @@ class RefcountTestCase(unittest.TestCase):
self.assertEqual(grc(callback), 2)
+ @support.refcount_test
def test_refcount(self):
from sys import getrefcount as grc
def func(*args):
diff --git a/Lib/ctypes/test/test_stringptr.py b/Lib/ctypes/test/test_stringptr.py
index 3d25fa5..95cd161 100644
--- a/Lib/ctypes/test/test_stringptr.py
+++ b/Lib/ctypes/test/test_stringptr.py
@@ -1,4 +1,5 @@
import unittest
+from test import support
from ctypes import *
import _ctypes_test
@@ -7,6 +8,7 @@ lib = CDLL(_ctypes_test.__file__)
class StringPtrTestCase(unittest.TestCase):
+ @support.refcount_test
def test__POINTER_c_char(self):
class X(Structure):
_fields_ = [("str", POINTER(c_char))]
diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py
index 1881e89..97d0c2f 100644
--- a/Lib/ctypes/util.py
+++ b/Lib/ctypes/util.py
@@ -1,5 +1,6 @@
import sys, os
import contextlib
+import subprocess
# find_library(name) returns the pathname of a library, or None.
if os.name == "nt":
@@ -136,16 +137,12 @@ elif os.name == "posix":
rv = f.close()
if rv == 10:
raise OSError('objdump command not found')
- with contextlib.closing(os.popen(cmd)) as f:
- data = f.read()
- res = re.search(r'\sSONAME\s+([^\s]+)', data)
+ res = re.search(r'\sSONAME\s+([^\s]+)', dump)
if not res:
return None
return res.group(1)
- if (sys.platform.startswith("freebsd")
- or sys.platform.startswith("openbsd")
- or sys.platform.startswith("dragonfly")):
+ if sys.platform.startswith(("freebsd", "openbsd", "dragonfly")):
def _num_version(libname):
# "libxyz.so.MAJOR.MINOR" => [ MAJOR, MINOR ]
@@ -171,22 +168,6 @@ elif os.name == "posix":
else:
- def _findLib_ldconfig(name):
- # XXX assuming GLIBC's ldconfig (with option -p)
- expr = r'/[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name)
- with contextlib.closing(os.popen('/sbin/ldconfig -p 2>/dev/null')) as f:
- data = f.read()
- res = re.search(expr, data)
- if not res:
- # Hm, this works only for libs needed by the python executable.
- cmd = 'ldd %s 2>/dev/null' % sys.executable
- with contextlib.closing(os.popen(cmd)) as f:
- data = f.read()
- res = re.search(expr, data)
- if not res:
- return None
- return res.group(0)
-
def _findSoname_ldconfig(name):
import struct
if struct.calcsize('l') == 4:
@@ -203,14 +184,19 @@ elif os.name == "posix":
abi_type = mach_map.get(machine, 'libc6')
# XXX assuming GLIBC's ldconfig (with option -p)
- expr = r'(\S+)\s+\((%s(?:, OS ABI:[^\)]*)?)\)[^/]*(/[^\(\)\s]*lib%s\.[^\(\)\s]*)' \
- % (abi_type, re.escape(name))
- with contextlib.closing(os.popen('LC_ALL=C LANG=C /sbin/ldconfig -p 2>/dev/null')) as f:
- data = f.read()
- res = re.search(expr, data)
- if not res:
- return None
- return res.group(1)
+ regex = os.fsencode(
+ '\s+(lib%s\.[^\s]+)\s+\(%s' % (re.escape(name), abi_type))
+ try:
+ with subprocess.Popen(['/sbin/ldconfig', '-p'],
+ stdin=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ stdout=subprocess.PIPE,
+ env={'LC_ALL': 'C', 'LANG': 'C'}) as p:
+ res = re.search(regex, p.stdout.read())
+ if res:
+ return os.fsdecode(res.group(1))
+ except OSError:
+ pass
def find_library(name):
return _findSoname_ldconfig(name) or _get_soname(_findLib_gcc(name))
diff --git a/Lib/curses/__init__.py b/Lib/curses/__init__.py
index 303ea3e..61ce443 100644
--- a/Lib/curses/__init__.py
+++ b/Lib/curses/__init__.py
@@ -11,7 +11,6 @@ the package, and perhaps a particular module inside it.
"""
from _curses import *
-from curses.wrapper import wrapper
import os as _os
import sys as _sys
@@ -55,3 +54,48 @@ try:
has_key
except NameError:
from has_key import has_key
+
+# Wrapper for the entire curses-based application. Runs a function which
+# should be the rest of your curses-based application. If the application
+# raises an exception, wrapper() will restore the terminal to a sane state so
+# you can read the resulting traceback.
+
+def wrapper(func, *args, **kwds):
+ """Wrapper function that initializes curses and calls another function,
+ restoring normal keyboard/screen behavior on error.
+ The callable object 'func' is then passed the main window 'stdscr'
+ as its first argument, followed by any other arguments passed to
+ wrapper().
+ """
+
+ try:
+ # Initialize curses
+ stdscr = initscr()
+
+ # Turn off echoing of keys, and enter cbreak mode,
+ # where no buffering is performed on keyboard input
+ noecho()
+ cbreak()
+
+ # In keypad mode, escape sequences for special keys
+ # (like the cursor keys) will be interpreted and
+ # a special value like curses.KEY_LEFT will be returned
+ stdscr.keypad(1)
+
+ # Start color, too. Harmless if the terminal doesn't have
+ # color; user can test with has_color() later on. The try/catch
+ # works around a minor bit of over-conscientiousness in the curses
+ # module -- the error return from C start_color() is ignorable.
+ try:
+ start_color()
+ except:
+ pass
+
+ return func(stdscr, *args, **kwds)
+ finally:
+ # Set everything back to normal
+ if 'stdscr' in locals():
+ stdscr.keypad(0)
+ echo()
+ nocbreak()
+ endwin()
diff --git a/Lib/curses/wrapper.py b/Lib/curses/wrapper.py
deleted file mode 100644
index 5183ce7..0000000
--- a/Lib/curses/wrapper.py
+++ /dev/null
@@ -1,50 +0,0 @@
-"""curses.wrapper
-
-Contains one function, wrapper(), which runs another function which
-should be the rest of your curses-based application. If the
-application raises an exception, wrapper() will restore the terminal
-to a sane state so you can read the resulting traceback.
-
-"""
-
-import curses
-
-def wrapper(func, *args, **kwds):
- """Wrapper function that initializes curses and calls another function,
- restoring normal keyboard/screen behavior on error.
- The callable object 'func' is then passed the main window 'stdscr'
- as its first argument, followed by any other arguments passed to
- wrapper().
- """
-
- try:
- # Initialize curses
- stdscr = curses.initscr()
-
- # Turn off echoing of keys, and enter cbreak mode,
- # where no buffering is performed on keyboard input
- curses.noecho()
- curses.cbreak()
-
- # In keypad mode, escape sequences for special keys
- # (like the cursor keys) will be interpreted and
- # a special value like curses.KEY_LEFT will be returned
- stdscr.keypad(1)
-
- # Start color, too. Harmless if the terminal doesn't have
- # color; user can test with has_color() later on. The try/catch
- # works around a minor bit of over-conscientiousness in the curses
- # module -- the error return from C start_color() is ignorable.
- try:
- curses.start_color()
- except:
- pass
-
- return func(stdscr, *args, **kwds)
- finally:
- # Set everything back to normal
- if 'stdscr' in locals():
- stdscr.keypad(0)
- curses.echo()
- curses.nocbreak()
- curses.endwin()
diff --git a/Lib/datetime.py b/Lib/datetime.py
index 65f95d2..c5eeca4 100644
--- a/Lib/datetime.py
+++ b/Lib/datetime.py
@@ -172,10 +172,6 @@ def _format_time(hh, mm, ss, us):
# Correctly substitute for %z and %Z escapes in strftime formats.
def _wrap_strftime(object, format, timetuple):
- year = timetuple[0]
- if year < 1000:
- raise ValueError("year=%d is before 1000; the datetime strftime() "
- "methods require year >= 1000" % year)
# Don't call utcoffset() or tzname() unless actually needed.
freplace = None # the string to use for %f
zreplace = None # the string to use for %z
diff --git a/Lib/decimal.py b/Lib/decimal.py
index f5277c5..e946182 100644
--- a/Lib/decimal.py
+++ b/Lib/decimal.py
@@ -1871,6 +1871,7 @@ class Decimal(object):
"""
other = _convert_other(other, raiseit=True)
+ third = _convert_other(third, raiseit=True)
# compute product; raise InvalidOperation if either operand is
# a signaling NaN or if the product is zero times infinity.
@@ -1900,7 +1901,6 @@ class Decimal(object):
str(int(self._int) * int(other._int)),
self._exp + other._exp)
- third = _convert_other(third, raiseit=True)
return product.__add__(third, context)
def _power_modulo(self, other, modulo, context=None):
@@ -2001,9 +2001,9 @@ class Decimal(object):
nonzero. For efficiency, other._exp should not be too large,
so that 10**abs(other._exp) is a feasible calculation."""
- # In the comments below, we write x for the value of self and
- # y for the value of other. Write x = xc*10**xe and y =
- # yc*10**ye.
+ # In the comments below, we write x for the value of self and y for the
+ # value of other. Write x = xc*10**xe and abs(y) = yc*10**ye, with xc
+ # and yc positive integers not divisible by 10.
# The main purpose of this method is to identify the *failure*
# of x**y to be exactly representable with as little effort as
@@ -2011,13 +2011,12 @@ class Decimal(object):
# eliminate the possibility of x**y being exact. Only if all
# these tests are passed do we go on to actually compute x**y.
- # Here's the main idea. First normalize both x and y. We
- # express y as a rational m/n, with m and n relatively prime
- # and n>0. Then for x**y to be exactly representable (at
- # *any* precision), xc must be the nth power of a positive
- # integer and xe must be divisible by n. If m is negative
- # then additionally xc must be a power of either 2 or 5, hence
- # a power of 2**n or 5**n.
+ # Here's the main idea. Express y as a rational number m/n, with m and
+ # n relatively prime and n>0. Then for x**y to be exactly
+ # representable (at *any* precision), xc must be the nth power of a
+ # positive integer and xe must be divisible by n. If y is negative
+ # then additionally xc must be a power of either 2 or 5, hence a power
+ # of 2**n or 5**n.
#
# There's a limit to how small |y| can be: if y=m/n as above
# then:
@@ -2089,21 +2088,43 @@ class Decimal(object):
return None
# now xc is a power of 2; e is its exponent
e = _nbits(xc)-1
- # find e*y and xe*y; both must be integers
- if ye >= 0:
- y_as_int = yc*10**ye
- e = e*y_as_int
- xe = xe*y_as_int
- else:
- ten_pow = 10**-ye
- e, remainder = divmod(e*yc, ten_pow)
- if remainder:
- return None
- xe, remainder = divmod(xe*yc, ten_pow)
- if remainder:
- return None
-
- if e*65 >= p*93: # 93/65 > log(10)/log(5)
+
+ # We now have:
+ #
+ # x = 2**e * 10**xe, e > 0, and y < 0.
+ #
+ # The exact result is:
+ #
+ # x**y = 5**(-e*y) * 10**(e*y + xe*y)
+ #
+ # provided that both e*y and xe*y are integers. Note that if
+ # 5**(-e*y) >= 10**p, then the result can't be expressed
+ # exactly with p digits of precision.
+ #
+ # Using the above, we can guard against large values of ye.
+ # 93/65 is an upper bound for log(10)/log(5), so if
+ #
+ # ye >= len(str(93*p//65))
+ #
+ # then
+ #
+ # -e*y >= -y >= 10**ye > 93*p/65 > p*log(10)/log(5),
+ #
+ # so 5**(-e*y) >= 10**p, and the coefficient of the result
+ # can't be expressed in p digits.
+
+ # emax >= largest e such that 5**e < 10**p.
+ emax = p*93//65
+ if ye >= len(str(emax)):
+ return None
+
+ # Find -e*y and -xe*y; both must be integers
+ e = _decimal_lshift_exact(e * yc, ye)
+ xe = _decimal_lshift_exact(xe * yc, ye)
+ if e is None or xe is None:
+ return None
+
+ if e > emax:
return None
xc = 5**e
@@ -2117,19 +2138,20 @@ class Decimal(object):
while xc % 5 == 0:
xc //= 5
e -= 1
- if ye >= 0:
- y_as_integer = yc*10**ye
- e = e*y_as_integer
- xe = xe*y_as_integer
- else:
- ten_pow = 10**-ye
- e, remainder = divmod(e*yc, ten_pow)
- if remainder:
- return None
- xe, remainder = divmod(xe*yc, ten_pow)
- if remainder:
- return None
- if e*3 >= p*10: # 10/3 > log(10)/log(2)
+
+ # Guard against large values of ye, using the same logic as in
+ # the 'xc is a power of 2' branch. 10/3 is an upper bound for
+ # log(10)/log(2).
+ emax = p*10//3
+ if ye >= len(str(emax)):
+ return None
+
+ e = _decimal_lshift_exact(e * yc, ye)
+ xe = _decimal_lshift_exact(xe * yc, ye)
+ if e is None or xe is None:
+ return None
+
+ if e > emax:
return None
xc = 2**e
else:
@@ -3881,28 +3903,6 @@ class Context(object):
return nc
__copy__ = copy
- # _clamp is provided for backwards compatibility with third-party
- # code. May be removed in Python >= 3.3.
- def _get_clamp(self):
- "_clamp mirrors the clamp attribute. Its use is deprecated."
- import warnings
- warnings.warn('Use of the _clamp attribute is deprecated. '
- 'Please use clamp instead.',
- DeprecationWarning)
- return self.clamp
-
- def _set_clamp(self, clamp):
- "_clamp mirrors the clamp attribute. Its use is deprecated."
- import warnings
- warnings.warn('Use of the _clamp attribute is deprecated. '
- 'Please use clamp instead.',
- DeprecationWarning)
- self.clamp = clamp
-
- # don't bother with _del_clamp; no sane 3rd party code should
- # be deleting the _clamp attribute
- _clamp = property(_get_clamp, _set_clamp)
-
def _raise_error(self, condition, explanation = None, *args):
"""Handles an error
@@ -5529,6 +5529,27 @@ def _normalize(op1, op2, prec = 0):
_nbits = int.bit_length
+def _decimal_lshift_exact(n, e):
+ """ Given integers n and e, return n * 10**e if it's an integer, else None.
+
+ The computation is designed to avoid computing large powers of 10
+ unnecessarily.
+
+ >>> _decimal_lshift_exact(3, 4)
+ 30000
+ >>> _decimal_lshift_exact(300, -999999999) # returns None
+
+ """
+ if n == 0:
+ return 0
+ elif e >= 0:
+ return n * 10**e
+ else:
+ # val_n = largest power of 10 dividing n.
+ str_n = str(abs(n))
+ val_n = len(str_n) - len(str_n.rstrip('0'))
+ return None if val_n < -e else n // 10**-e
+
def _sqrt_nearest(n, a):
"""Closest integer to the square root of the positive integer n. a is
an initial approximation to the square root. Any positive integer
diff --git a/Lib/difflib.py b/Lib/difflib.py
index e6cc6ee..d34bbca 100644
--- a/Lib/difflib.py
+++ b/Lib/difflib.py
@@ -800,7 +800,7 @@ class Differ:
... 2. Explicit is better than implicit.
... 3. Simple is better than complex.
... 4. Complex is better than complicated.
- ... '''.splitlines(1)
+ ... '''.splitlines(keepends=True)
>>> len(text1)
4
>>> text1[0][-1]
@@ -809,7 +809,7 @@ class Differ:
... 3. Simple is better than complex.
... 4. Complicated is better than complex.
... 5. Flat is better than nested.
- ... '''.splitlines(1)
+ ... '''.splitlines(keepends=True)
Next we instantiate a Differ object:
@@ -896,8 +896,8 @@ class Differ:
Example:
- >>> print(''.join(Differ().compare('one\ntwo\nthree\n'.splitlines(1),
- ... 'ore\ntree\nemu\n'.splitlines(1))),
+ >>> print(''.join(Differ().compare('one\ntwo\nthree\n'.splitlines(True),
+ ... 'ore\ntree\nemu\n'.splitlines(True))),
... end="")
- one
? ^
@@ -1269,8 +1269,8 @@ def context_diff(a, b, fromfile='', tofile='',
Example:
- >>> print(''.join(context_diff('one\ntwo\nthree\nfour\n'.splitlines(1),
- ... 'zero\none\ntree\nfour\n'.splitlines(1), 'Original', 'Current')),
+ >>> print(''.join(context_diff('one\ntwo\nthree\nfour\n'.splitlines(True),
+ ... 'zero\none\ntree\nfour\n'.splitlines(True), 'Original', 'Current')),
... end="")
*** Original
--- Current
@@ -1339,8 +1339,8 @@ def ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK):
Example:
- >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(1),
- ... 'ore\ntree\nemu\n'.splitlines(1))
+ >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(keepends=True),
+ ... 'ore\ntree\nemu\n'.splitlines(keepends=True))
>>> print(''.join(diff), end="")
- one
? ^
@@ -2034,8 +2034,8 @@ def restore(delta, which):
Examples:
- >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(1),
- ... 'ore\ntree\nemu\n'.splitlines(1))
+ >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(keepends=True),
+ ... 'ore\ntree\nemu\n'.splitlines(keepends=True))
>>> diff = list(diff)
>>> print(''.join(restore(diff, 1)), end="")
one
diff --git a/Lib/distutils/__init__.py b/Lib/distutils/__init__.py
index 9ec6165..f883916 100644
--- a/Lib/distutils/__init__.py
+++ b/Lib/distutils/__init__.py
@@ -13,5 +13,5 @@ used from a setup script as
# Updated automatically by the Python release process.
#
#--start constants--
-__version__ = "3.2.2"
+__version__ = "3.3a0"
#--end constants--
diff --git a/Lib/distutils/command/bdist_wininst.py b/Lib/distutils/command/bdist_wininst.py
index e3ed3ad..959a8bf 100644
--- a/Lib/distutils/command/bdist_wininst.py
+++ b/Lib/distutils/command/bdist_wininst.py
@@ -265,11 +265,11 @@ class bdist_wininst(Command):
cfgdata = cfgdata + b"\0"
if self.pre_install_script:
# We need to normalize newlines, so we open in text mode and
- # convert back to bytes. "latin1" simply avoids any possible
+ # convert back to bytes. "latin-1" simply avoids any possible
# failures.
with open(self.pre_install_script, "r",
- encoding="latin1") as script:
- script_data = script.read().encode("latin1")
+ encoding="latin-1") as script:
+ script_data = script.read().encode("latin-1")
cfgdata = cfgdata + script_data + b"\n\0"
else:
# empty pre-install script
diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py
index 8d843d6..8baf538 100644
--- a/Lib/distutils/command/build_ext.py
+++ b/Lib/distutils/command/build_ext.py
@@ -240,8 +240,7 @@ class build_ext(Command):
# for extensions under Linux or Solaris with a shared Python library,
# Python's library directory must be appended to library_dirs
sysconfig.get_config_var('Py_ENABLE_SHARED')
- if ((sys.platform.startswith('linux') or sys.platform.startswith('gnu')
- or sys.platform.startswith('sunos'))
+ if (sys.platform.startswith(('linux', 'gnu', 'sunos'))
and sysconfig.get_config_var('Py_ENABLE_SHARED')):
if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")):
# building third party extensions
diff --git a/Lib/distutils/command/build_scripts.py b/Lib/distutils/command/build_scripts.py
index ec43477..4b5b22e 100644
--- a/Lib/distutils/command/build_scripts.py
+++ b/Lib/distutils/command/build_scripts.py
@@ -126,10 +126,9 @@ class build_scripts(Command):
"The shebang ({!r}) is not decodable "
"from the script encoding ({})"
.format(shebang, encoding))
- outf = open(outfile, "wb")
- outf.write(shebang)
- outf.writelines(f.readlines())
- outf.close()
+ with open(outfile, "wb") as outf:
+ outf.write(shebang)
+ outf.writelines(f.readlines())
if f:
f.close()
else:
diff --git a/Lib/distutils/tests/test_bdist_rpm.py b/Lib/distutils/tests/test_bdist_rpm.py
index 804fb13..9b0639a 100644
--- a/Lib/distutils/tests/test_bdist_rpm.py
+++ b/Lib/distutils/tests/test_bdist_rpm.py
@@ -28,6 +28,11 @@ class BuildRpmTestCase(support.TempdirManager,
unittest.TestCase):
def setUp(self):
+ try:
+ sys.executable.encode("UTF-8")
+ except UnicodeEncodeError:
+ raise unittest.SkipTest("sys.executable is not encodable to UTF-8")
+
super(BuildRpmTestCase, self).setUp()
self.old_location = os.getcwd()
self.old_sys_argv = sys.argv, sys.argv[:]
@@ -42,7 +47,7 @@ class BuildRpmTestCase(support.TempdirManager,
# XXX I am unable yet to make this test work without
# spurious sdtout/stderr output under Mac OS X
- if sys.platform != 'linux2':
+ if not sys.platform.startswith('linux'):
return
# this test will run only if the rpm commands are found
@@ -82,7 +87,7 @@ class BuildRpmTestCase(support.TempdirManager,
# XXX I am unable yet to make this test work without
# spurious sdtout/stderr output under Mac OS X
- if sys.platform != 'linux2':
+ if not sys.platform.startswith('linux'):
return
# http://bugs.python.org/issue1533164
diff --git a/Lib/doctest.py b/Lib/doctest.py
index f60b06d..9d1501c 100644
--- a/Lib/doctest.py
+++ b/Lib/doctest.py
@@ -1332,7 +1332,7 @@ class DocTestRunner:
m = self.__LINECACHE_FILENAME_RE.match(filename)
if m and m.group('name') == self.test.name:
example = self.test.examples[int(m.group('examplenum'))]
- return example.source.splitlines(True)
+ return example.source.splitlines(keepends=True)
else:
return self.save_linecache_getlines(filename, module_globals)
@@ -1378,6 +1378,7 @@ class DocTestRunner:
# Note that the interactive output will go to *our*
# save_stdout, even if that's not the real sys.stdout; this
# allows us to write test cases for the set_trace behavior.
+ save_trace = sys.gettrace()
save_set_trace = pdb.set_trace
self.debugger = _OutputRedirectingPdb(save_stdout)
self.debugger.reset()
@@ -1397,6 +1398,7 @@ class DocTestRunner:
finally:
sys.stdout = save_stdout
pdb.set_trace = save_set_trace
+ sys.settrace(save_trace)
linecache.getlines = self.save_linecache_getlines
sys.displayhook = save_displayhook
if clear_globs:
@@ -1593,8 +1595,8 @@ class OutputChecker:
# Check if we should use diff.
if self._do_a_fancy_diff(want, got, optionflags):
# Split want & got into lines.
- want_lines = want.splitlines(True) # True == keep line ends
- got_lines = got.splitlines(True)
+ want_lines = want.splitlines(keepends=True)
+ got_lines = got.splitlines(keepends=True)
# Use difflib to find their differences.
if optionflags & REPORT_UDIFF:
diff = difflib.unified_diff(want_lines, got_lines, n=2)
diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py
index 41694f9..c455e05 100644
--- a/Lib/email/_parseaddr.py
+++ b/Lib/email/_parseaddr.py
@@ -47,6 +47,21 @@ def parsedate_tz(data):
Accounts for military timezones.
"""
+ res = _parsedate_tz(data)
+ if res[9] is None:
+ res[9] = 0
+ return tuple(res)
+
+def _parsedate_tz(data):
+ """Convert date to extended time tuple.
+
+ The last (additional) element is the time zone offset in seconds, except if
+ the timezone was specified as -0000. In that case the last element is
+ None. This indicates a UTC timestamp that explicitly declaims knowledge of
+ the source timezone, as opposed to a +0000 timestamp that indicates the
+ source timezone really was UTC.
+
+ """
data = data.split()
# The FWS after the comma after the day-of-week is optional, so search and
# adjust for this.
@@ -99,6 +114,14 @@ def parsedate_tz(data):
tss = '0'
elif len(tm) == 3:
[thh, tmm, tss] = tm
+ elif len(tm) == 1 and '.' in tm[0]:
+ # Some non-compliant MUAs use '.' to separate time elements.
+ tm = tm[0].split('.')
+ if len(tm) == 2:
+ [thh, tmm] = tm
+ tss = 0
+ elif len(tm) == 3:
+ [thh, tmm, tss] = tm
else:
return None
try:
@@ -130,6 +153,8 @@ def parsedate_tz(data):
tzoffset = int(tz)
except ValueError:
pass
+ if tzoffset==0 and tz.startswith('-'):
+ tzoffset = None
# Convert a timezone offset into seconds ; -0500 -> -18000
if tzoffset:
if tzoffset < 0:
@@ -139,7 +164,7 @@ def parsedate_tz(data):
tzsign = 1
tzoffset = tzsign * ( (tzoffset//100)*3600 + (tzoffset % 100)*60)
# Daylight Saving Time flag is set to -1, since DST is unknown.
- return yy, mm, dd, thh, tmm, tss, 0, 1, -1, tzoffset
+ return [yy, mm, dd, thh, tmm, tss, 0, 1, -1, tzoffset]
def parsedate(data):
diff --git a/Lib/email/errors.py b/Lib/email/errors.py
index d52a624..c04deb4 100644
--- a/Lib/email/errors.py
+++ b/Lib/email/errors.py
@@ -32,7 +32,7 @@ class CharsetError(MessageError):
# These are parsing defects which the parser was able to work around.
-class MessageDefect:
+class MessageDefect(Exception):
"""Base class for a message defect."""
def __init__(self, line=None):
@@ -55,3 +55,6 @@ class MalformedHeaderDefect(MessageDefect):
class MultipartInvariantViolationDefect(MessageDefect):
"""A message claimed to be a multipart but no subparts were found."""
+
+class InvalidMultipartContentTransferEncodingDefect(MessageDefect):
+ """An invalid content transfer encoding was set on the multipart itself."""
diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py
index 60a8325..e754d89 100644
--- a/Lib/email/feedparser.py
+++ b/Lib/email/feedparser.py
@@ -25,6 +25,7 @@ import re
from email import errors
from email import message
+from email import policy
NLCRE = re.compile('\r\n|\r|\n')
NLCRE_bol = re.compile('(\r\n|\r|\n)')
@@ -120,9 +121,6 @@ class BufferedSubFile(object):
# Reverse and insert at the front of the lines.
self._lines[:0] = lines[::-1]
- def is_closed(self):
- return self._closed
-
def __iter__(self):
return self
@@ -137,9 +135,16 @@ class BufferedSubFile(object):
class FeedParser:
"""A feed-style parser of email."""
- def __init__(self, _factory=message.Message):
- """_factory is called with no arguments to create a new message obj"""
+ def __init__(self, _factory=message.Message, *, policy=policy.default):
+ """_factory is called with no arguments to create a new message obj
+
+ The policy keyword specifies a policy object that controls a number of
+ aspects of the parser's operation. The default policy maintains
+ backward compatibility.
+
+ """
self._factory = _factory
+ self.policy = policy
self._input = BufferedSubFile()
self._msgstack = []
self._parse = self._parsegen().__next__
@@ -171,7 +176,8 @@ class FeedParser:
# Look for final set of defects
if root.get_content_maintype() == 'multipart' \
and not root.is_multipart():
- root.defects.append(errors.MultipartInvariantViolationDefect())
+ defect = errors.MultipartInvariantViolationDefect()
+ self.policy.handle_defect(root, defect)
return root
def _new_message(self):
@@ -284,7 +290,8 @@ class FeedParser:
# defined a boundary. That's a problem which we'll handle by
# reading everything until the EOF and marking the message as
# defective.
- self._cur.defects.append(errors.NoBoundaryInMultipartDefect())
+ defect = errors.NoBoundaryInMultipartDefect()
+ self.policy.handle_defect(self._cur, defect)
lines = []
for line in self._input:
if line is NeedMoreData:
@@ -293,6 +300,11 @@ class FeedParser:
lines.append(line)
self._cur.set_payload(EMPTYSTRING.join(lines))
return
+ # Make sure a valid content type was specified per RFC 2045:6.4.
+ if (self._cur.get('content-transfer-encoding', '8bit').lower()
+ not in ('7bit', '8bit', 'binary')):
+ defect = errors.InvalidMultipartContentTransferEncodingDefect()
+ self.policy.handle_defect(self._cur, defect)
# Create a line match predicate which matches the inter-part
# boundary as well as the end-of-multipart boundary. Don't push
# this onto the input stream until we've scanned past the
@@ -388,7 +400,8 @@ class FeedParser:
# that as a defect and store the captured text as the payload.
# Everything from here to the EOF is epilogue.
if capturing_preamble:
- self._cur.defects.append(errors.StartBoundaryNotFoundDefect())
+ defect = errors.StartBoundaryNotFoundDefect()
+ self.policy.handle_defect(self._cur, defect)
self._cur.set_payload(EMPTYSTRING.join(preamble))
epilogue = []
for line in self._input:
@@ -440,7 +453,7 @@ class FeedParser:
# is illegal, so let's note the defect, store the illegal
# line, and ignore it for purposes of headers.
defect = errors.FirstHeaderLineIsContinuationDefect(line)
- self._cur.defects.append(defect)
+ self.policy.handle_defect(self._cur, defect)
continue
lastvalue.append(line)
continue
diff --git a/Lib/email/generator.py b/Lib/email/generator.py
index f0e7a95..d8b8fa9 100644
--- a/Lib/email/generator.py
+++ b/Lib/email/generator.py
@@ -13,8 +13,10 @@ import random
import warnings
from io import StringIO, BytesIO
+from email import policy
from email.header import Header
from email.message import _has_surrogates
+import email.charset as _charset
UNDERSCORE = '_'
NL = '\n' # XXX: no longer used by the code below.
@@ -33,7 +35,8 @@ class Generator:
# Public interface
#
- def __init__(self, outfp, mangle_from_=True, maxheaderlen=78):
+ def __init__(self, outfp, mangle_from_=True, maxheaderlen=None, *,
+ policy=policy.default):
"""Create the generator for message flattening.
outfp is the output file-like object for writing the message to. It
@@ -49,16 +52,23 @@ class Generator:
defined in the Header class. Set maxheaderlen to zero to disable
header wrapping. The default is 78, as recommended (but not required)
by RFC 2822.
+
+ The policy keyword specifies a policy object that controls a number of
+ aspects of the generator's operation. The default policy maintains
+ backward compatibility.
+
"""
self._fp = outfp
self._mangle_from_ = mangle_from_
- self._maxheaderlen = maxheaderlen
+ self._maxheaderlen = (maxheaderlen if maxheaderlen is not None else
+ policy.max_line_length)
+ self.policy = policy
def write(self, s):
# Just delegate to the file object
self._fp.write(s)
- def flatten(self, msg, unixfrom=False, linesep='\n'):
+ def flatten(self, msg, unixfrom=False, linesep=None):
r"""Print the message object tree rooted at msg to the output file
specified when the Generator instance was created.
@@ -70,17 +80,15 @@ class Generator:
Note that for subobjects, no From_ line is printed.
linesep specifies the characters used to indicate a new line in
- the output. The default value is the most useful for typical
- Python applications, but it can be set to \r\n to produce RFC-compliant
- line separators when needed.
+ the output. The default value is determined by the policy.
"""
# We use the _XXX constants for operating on data that comes directly
# from the msg, and _encoded_XXX constants for operating on data that
# has already been converted (to bytes in the BytesGenerator) and
# inserted into a temporary buffer.
- self._NL = linesep
- self._encoded_NL = self._encode(linesep)
+ self._NL = linesep if linesep is not None else self.policy.linesep
+ self._encoded_NL = self._encode(self._NL)
self._EMPTY = ''
self._encoded_EMTPY = self._encode('')
if unixfrom:
@@ -297,10 +305,12 @@ class Generator:
# message/rfc822. Such messages are generated by, for example,
# Groupwise when forwarding unadorned messages. (Issue 7970.) So
# in that case we just emit the string body.
- payload = msg.get_payload()
+ payload = msg._payload
if isinstance(payload, list):
g.flatten(msg.get_payload(0), unixfrom=False, linesep=self._NL)
payload = s.getvalue()
+ else:
+ payload = self._encode(payload)
self._fp.write(payload)
# This used to be a module level function; we use a classmethod for this
@@ -336,7 +346,10 @@ class BytesGenerator(Generator):
Functionally identical to the base Generator except that the output is
bytes and not string. When surrogates were used in the input to encode
- bytes, these are decoded back to bytes for output.
+ bytes, these are decoded back to bytes for output. If the policy has
+ must_be_7bit set true, then the message is transformed such that the
+ non-ASCII bytes are properly content transfer encoded, using the
+ charset unknown-8bit.
The outfp object must accept bytes in its write method.
"""
@@ -359,21 +372,22 @@ class BytesGenerator(Generator):
# strings with 8bit bytes.
for h, v in msg._headers:
self.write('%s: ' % h)
- if isinstance(v, Header):
- self.write(v.encode(maxlinelen=self._maxheaderlen)+NL)
- elif _has_surrogates(v):
- # If we have raw 8bit data in a byte string, we have no idea
- # what the encoding is. There is no safe way to split this
- # string. If it's ascii-subset, then we could do a normal
- # ascii split, but if it's multibyte then we could break the
- # string. There's no way to know so the least harm seems to
- # be to not split the string and risk it being too long.
- self.write(v+NL)
- else:
- # Header's got lots of smarts and this string is safe...
- header = Header(v, maxlinelen=self._maxheaderlen,
- header_name=h)
- self.write(header.encode(linesep=self._NL)+self._NL)
+ if isinstance(v, str):
+ if _has_surrogates(v):
+ if not self.policy.must_be_7bit:
+ # If we have raw 8bit data in a byte string, we have no idea
+ # what the encoding is. There is no safe way to split this
+ # string. If it's ascii-subset, then we could do a normal
+ # ascii split, but if it's multibyte then we could break the
+ # string. There's no way to know so the least harm seems to
+ # be to not split the string and risk it being too long.
+ self.write(v+NL)
+ continue
+ h = Header(v, charset=_charset.UNKNOWN8BIT, header_name=h)
+ else:
+ h = Header(v, header_name=h)
+ self.write(h.encode(linesep=self._NL,
+ maxlinelen=self._maxheaderlen)+self._NL)
# A blank line always separates headers from body
self.write(self._NL)
@@ -382,7 +396,7 @@ class BytesGenerator(Generator):
# just write it back out.
if msg._payload is None:
return
- if _has_surrogates(msg._payload):
+ if _has_surrogates(msg._payload) and not self.policy.must_be_7bit:
self.write(msg._payload)
else:
super(BytesGenerator,self)._handle_text(msg)
diff --git a/Lib/email/parser.py b/Lib/email/parser.py
index 6caaff5..0f92160 100644
--- a/Lib/email/parser.py
+++ b/Lib/email/parser.py
@@ -4,18 +4,19 @@
"""A parser of RFC 2822 and MIME email messages."""
-__all__ = ['Parser', 'HeaderParser']
+__all__ = ['Parser', 'HeaderParser', 'BytesParser', 'BytesHeaderParser']
import warnings
from io import StringIO, TextIOWrapper
from email.feedparser import FeedParser
from email.message import Message
+from email import policy
class Parser:
- def __init__(self, *args, **kws):
+ def __init__(self, _class=Message, *, policy=policy.default):
"""Parser of RFC 2822 and MIME email messages.
Creates an in-memory object tree representing the email message, which
@@ -30,28 +31,14 @@ class Parser:
_class is the class to instantiate for new message objects when they
must be created. This class must have a constructor that can take
zero arguments. Default is Message.Message.
+
+ The policy keyword specifies a policy object that controls a number of
+ aspects of the parser's operation. The default policy maintains
+ backward compatibility.
+
"""
- if len(args) >= 1:
- if '_class' in kws:
- raise TypeError("Multiple values for keyword arg '_class'")
- kws['_class'] = args[0]
- if len(args) == 2:
- if 'strict' in kws:
- raise TypeError("Multiple values for keyword arg 'strict'")
- kws['strict'] = args[1]
- if len(args) > 2:
- raise TypeError('Too many arguments')
- if '_class' in kws:
- self._class = kws['_class']
- del kws['_class']
- else:
- self._class = Message
- if 'strict' in kws:
- warnings.warn("'strict' argument is deprecated (and ignored)",
- DeprecationWarning, 2)
- del kws['strict']
- if kws:
- raise TypeError('Unexpected keyword arguments')
+ self._class = _class
+ self.policy = policy
def parse(self, fp, headersonly=False):
"""Create a message structure from the data in a file.
@@ -61,7 +48,7 @@ class Parser:
parsing after reading the headers or not. The default is False,
meaning it parses the entire contents of the file.
"""
- feedparser = FeedParser(self._class)
+ feedparser = FeedParser(self._class, policy=self.policy)
if headersonly:
feedparser._set_headersonly()
while True:
@@ -134,3 +121,11 @@ class BytesParser:
"""
text = text.decode('ASCII', errors='surrogateescape')
return self.parser.parsestr(text, headersonly)
+
+
+class BytesHeaderParser(BytesParser):
+ def parse(self, fp, headersonly=True):
+ return BytesParser.parse(self, fp, headersonly=True)
+
+ def parsebytes(self, text, headersonly=True):
+ return BytesParser.parsebytes(self, text, headersonly=True)
diff --git a/Lib/email/policy.py b/Lib/email/policy.py
new file mode 100644
index 0000000..88877a2
--- /dev/null
+++ b/Lib/email/policy.py
@@ -0,0 +1,174 @@
+"""Policy framework for the email package.
+
+Allows fine grained feature control of how the package parses and emits data.
+"""
+
+__all__ = [
+ 'Policy',
+ 'default',
+ 'strict',
+ 'SMTP',
+ 'HTTP',
+ ]
+
+
+class _PolicyBase:
+
+ """Policy Object basic framework.
+
+ This class is useless unless subclassed. A subclass should define
+ class attributes with defaults for any values that are to be
+ managed by the Policy object. The constructor will then allow
+ non-default values to be set for these attributes at instance
+ creation time. The instance will be callable, taking these same
+ attributes keyword arguments, and returning a new instance
+ identical to the called instance except for those values changed
+ by the keyword arguments. Instances may be added, yielding new
+ instances with any non-default values from the right hand
+ operand overriding those in the left hand operand. That is,
+
+ A + B == A(<non-default values of B>)
+
+ The repr of an instance can be used to reconstruct the object
+ if and only if the repr of the values can be used to reconstruct
+ those values.
+
+ """
+
+ def __init__(self, **kw):
+ """Create new Policy, possibly overriding some defaults.
+
+ See class docstring for a list of overridable attributes.
+
+ """
+ for name, value in kw.items():
+ if hasattr(self, name):
+ super(_PolicyBase,self).__setattr__(name, value)
+ else:
+ raise TypeError(
+ "{!r} is an invalid keyword argument for {}".format(
+ name, self.__class__.__name__))
+
+ def __repr__(self):
+ args = [ "{}={!r}".format(name, value)
+ for name, value in self.__dict__.items() ]
+ return "{}({})".format(self.__class__.__name__, args if args else '')
+
+ def clone(self, **kw):
+ """Return a new instance with specified attributes changed.
+
+ The new instance has the same attribute values as the current object,
+ except for the changes passed in as keyword arguments.
+
+ """
+ for attr, value in self.__dict__.items():
+ if attr not in kw:
+ kw[attr] = value
+ return self.__class__(**kw)
+
+ def __setattr__(self, name, value):
+ if hasattr(self, name):
+ msg = "{!r} object attribute {!r} is read-only"
+ else:
+ msg = "{!r} object has no attribute {!r}"
+ raise AttributeError(msg.format(self.__class__.__name__, name))
+
+ def __add__(self, other):
+ """Non-default values from right operand override those from left.
+
+ The object returned is a new instance of the subclass.
+
+ """
+ return self.clone(**other.__dict__)
+
+
+class Policy(_PolicyBase):
+
+ """Controls for how messages are interpreted and formatted.
+
+ Most of the classes and many of the methods in the email package
+ accept Policy objects as parameters. A Policy object contains a set
+ of values and functions that control how input is interpreted and how
+ output is rendered. For example, the parameter 'raise_on_defect'
+ controls whether or not an RFC violation throws an error or not,
+ while 'max_line_length' controls the maximum length of output lines
+ when a Message is serialized.
+
+ Any valid attribute may be overridden when a Policy is created by
+ passing it as a keyword argument to the constructor. Policy
+ objects are immutable, but a new Policy object can be created
+ with only certain values changed by calling the Policy instance
+ with keyword arguments. Policy objects can also be added,
+ producing a new Policy object in which the non-default attributes
+ set in the right hand operand overwrite those specified in the
+ left operand.
+
+ Settable attributes:
+
+ raise_on_defect -- If true, then defects should be raised
+ as errors. Default False.
+
+ linesep -- string containing the value to use as
+ separation between output lines. Default '\n'.
+
+ must_be_7bit -- output must contain only 7bit clean data.
+ Default False.
+
+ max_line_length -- maximum length of lines, excluding 'linesep',
+ during serialization. None means no line
+ wrapping is done. Default is 78.
+
+ Methods:
+
+ register_defect(obj, defect)
+ defect is a Defect instance. The default implementation appends defect
+ to the objs 'defects' attribute.
+
+ handle_defect(obj, defect)
+ intended to be called by parser code that finds a defect. If
+ raise_on_defect is True, defect is raised as an error, otherwise
+ register_defect is called.
+
+ """
+
+ raise_on_defect = False
+ linesep = '\n'
+ must_be_7bit = False
+ max_line_length = 78
+
+ def handle_defect(self, obj, defect):
+ """Based on policy, either raise defect or call register_defect.
+
+ handle_defect(obj, defect)
+
+ defect should be a Defect subclass, but in any case must be an
+ Exception subclass. obj is the object on which the defect should be
+ registered if it is not raised. If the raise_on_defect is True, the
+ defect is raised as an error, otherwise the object and the defect are
+ passed to register_defect.
+
+ This class is intended to be called by parsers that discover defects,
+ and will not be called from code using the library unless that code is
+ implementing an alternate parser.
+
+ """
+ if self.raise_on_defect:
+ raise defect
+ self.register_defect(obj, defect)
+
+ def register_defect(self, obj, defect):
+ """Record 'defect' on 'obj'.
+
+ Called by handle_defect if raise_on_defect is False. This method is
+ part of the Policy API so that Policy subclasses can implement custom
+ defect handling. The default implementation calls the append method
+ of the defects attribute of obj.
+
+ """
+ obj.defects.append(defect)
+
+
+default = Policy()
+strict = default.clone(raise_on_defect=True)
+SMTP = default.clone(linesep='\r\n')
+HTTP = default.clone(linesep='\r\n', max_line_length=None)
diff --git a/Lib/email/utils.py b/Lib/email/utils.py
index ac4da37..aecea65 100644
--- a/Lib/email/utils.py
+++ b/Lib/email/utils.py
@@ -11,12 +11,14 @@ __all__ = [
'encode_rfc2231',
'formataddr',
'formatdate',
+ 'format_datetime',
'getaddresses',
'make_msgid',
'mktime_tz',
'parseaddr',
'parsedate',
'parsedate_tz',
+ 'parsedate_to_datetime',
'unquote',
]
@@ -26,6 +28,7 @@ import time
import base64
import random
import socket
+import datetime
import urllib.parse
import warnings
from io import StringIO
@@ -37,11 +40,13 @@ from email._parseaddr import mktime_tz
# We need wormarounds for bugs in these methods in older Pythons (see below)
from email._parseaddr import parsedate as _parsedate
from email._parseaddr import parsedate_tz as _parsedate_tz
+from email._parseaddr import _parsedate_tz as __parsedate_tz
from quopri import decodestring as _qdecode
# Intrapackage imports
from email.encoders import _bencode, _qencode
+from email.charset import Charset
COMMASPACE = ', '
EMPTYSTRING = ''
@@ -56,21 +61,36 @@ escapesre = re.compile(r'[][\\()"]')
# Helpers
-def formataddr(pair):
+def formataddr(pair, charset='utf-8'):
"""The inverse of parseaddr(), this takes a 2-tuple of the form
(realname, email_address) and returns the string value suitable
for an RFC 2822 From, To or Cc header.
If the first element of pair is false, then the second element is
returned unmodified.
+
+ Optional charset if given is the character set that is used to encode
+ realname in case realname is not ASCII safe. Can be an instance of str or
+ a Charset-like object which has a header_encode method. Default is
+ 'utf-8'.
"""
name, address = pair
+ # The address MUST (per RFC) be ascii, so throw a UnicodeError if it isn't.
+ address.encode('ascii')
if name:
- quotes = ''
- if specialsre.search(name):
- quotes = '"'
- name = escapesre.sub(r'\\\g<0>', name)
- return '%s%s%s <%s>' % (quotes, name, quotes, address)
+ try:
+ name.encode('ascii')
+ except UnicodeEncodeError:
+ if isinstance(charset, str):
+ charset = Charset(charset)
+ encoded_name = charset.header_encode(name)
+ return "%s <%s>" % (encoded_name, address)
+ else:
+ quotes = ''
+ if specialsre.search(name):
+ quotes = '"'
+ name = escapesre.sub(r'\\\g<0>', name)
+ return '%s%s%s <%s>' % (quotes, name, quotes, address)
return address
@@ -94,6 +114,14 @@ ecre = re.compile(r'''
''', re.VERBOSE | re.IGNORECASE)
+def _format_timetuple_and_zone(timetuple, zone):
+ return '%s, %02d %s %04d %02d:%02d:%02d %s' % (
+ ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'][timetuple[6]],
+ timetuple[2],
+ ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
+ 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'][timetuple[1] - 1],
+ timetuple[0], timetuple[3], timetuple[4], timetuple[5],
+ zone)
def formatdate(timeval=None, localtime=False, usegmt=False):
"""Returns a date string as specified by RFC 2822, e.g.:
@@ -138,14 +166,25 @@ def formatdate(timeval=None, localtime=False, usegmt=False):
zone = 'GMT'
else:
zone = '-0000'
- return '%s, %02d %s %04d %02d:%02d:%02d %s' % (
- ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'][now[6]],
- now[2],
- ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
- 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'][now[1] - 1],
- now[0], now[3], now[4], now[5],
- zone)
+ return _format_timetuple_and_zone(now, zone)
+
+def format_datetime(dt, usegmt=False):
+ """Turn a datetime into a date string as specified in RFC 2822.
+ If usegmt is True, dt must be an aware datetime with an offset of zero. In
+ this case 'GMT' will be rendered instead of the normal +0000 required by
+ RFC2822. This is to support HTTP headers involving date stamps.
+ """
+ now = dt.timetuple()
+ if usegmt:
+ if dt.tzinfo is None or dt.tzinfo != datetime.timezone.utc:
+ raise ValueError("usegmt option requires a UTC datetime")
+ zone = 'GMT'
+ elif dt.tzinfo is None:
+ zone = '-0000'
+ else:
+ zone = dt.strftime("%z")
+ return _format_timetuple_and_zone(now, zone)
def make_msgid(idstring=None, domain=None):
@@ -187,6 +226,15 @@ def parsedate_tz(data):
return None
return _parsedate_tz(data)
+def parsedate_to_datetime(data):
+ if not data:
+ return None
+ *dtuple, tz = __parsedate_tz(data)
+ if tz is None:
+ return datetime.datetime(*dtuple[:6])
+ return datetime.datetime(*dtuple[:6],
+ tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
+
def parseaddr(addr):
addrs = _AddressList(addr).addresslist
diff --git a/Lib/encodings/cp65001.py b/Lib/encodings/cp65001.py
new file mode 100644
index 0000000..287eb87
--- /dev/null
+++ b/Lib/encodings/cp65001.py
@@ -0,0 +1,40 @@
+"""
+Code page 65001: Windows UTF-8 (CP_UTF8).
+"""
+
+import codecs
+import functools
+
+if not hasattr(codecs, 'code_page_encode'):
+ raise LookupError("cp65001 encoding is only available on Windows")
+
+### Codec APIs
+
+encode = functools.partial(codecs.code_page_encode, 65001)
+decode = functools.partial(codecs.code_page_decode, 65001)
+
+class IncrementalEncoder(codecs.IncrementalEncoder):
+ def encode(self, input, final=False):
+ return encode(input, self.errors)[0]
+
+class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
+ _buffer_decode = decode
+
+class StreamWriter(codecs.StreamWriter):
+ encode = encode
+
+class StreamReader(codecs.StreamReader):
+ decode = decode
+
+### encodings module API
+
+def getregentry():
+ return codecs.CodecInfo(
+ name='cp65001',
+ encode=encode,
+ decode=decode,
+ incrementalencoder=IncrementalEncoder,
+ incrementaldecoder=IncrementalDecoder,
+ streamreader=StreamReader,
+ streamwriter=StreamWriter,
+ )
diff --git a/Lib/fnmatch.py b/Lib/fnmatch.py
index 726fbe5..3df20d8 100644
--- a/Lib/fnmatch.py
+++ b/Lib/fnmatch.py
@@ -36,8 +36,8 @@ def fnmatch(name, pat):
return fnmatchcase(name, pat)
@functools.lru_cache(maxsize=250)
-def _compile_pattern(pat, is_bytes=False):
- if is_bytes:
+def _compile_pattern(pat):
+ if isinstance(pat, bytes):
pat_str = str(pat, 'ISO-8859-1')
res_str = translate(pat_str)
res = bytes(res_str, 'ISO-8859-1')
@@ -49,7 +49,7 @@ def filter(names, pat):
"""Return the subset of the list NAMES that match PAT."""
result = []
pat = os.path.normcase(pat)
- match = _compile_pattern(pat, isinstance(pat, bytes))
+ match = _compile_pattern(pat)
if os.path is posixpath:
# normcase on posix is NOP. Optimize it away from the loop.
for name in names:
@@ -67,7 +67,7 @@ def fnmatchcase(name, pat):
This is a version of fnmatch() which doesn't case-normalize
its arguments.
"""
- match = _compile_pattern(pat, isinstance(pat, bytes))
+ match = _compile_pattern(pat)
return match(name) is not None
diff --git a/Lib/ftplib.py b/Lib/ftplib.py
index 8e53023..b470216 100644
--- a/Lib/ftplib.py
+++ b/Lib/ftplib.py
@@ -100,14 +100,15 @@ class FTP:
file = None
welcome = None
passiveserver = 1
- encoding = "latin1"
+ encoding = "latin-1"
# Initialization method (called by class instantiation).
# Initialize host to localhost, port to standard ftp port
# Optional arguments are host (for connect()),
# and user, passwd, acct (for login())
def __init__(self, host='', user='', passwd='', acct='',
- timeout=_GLOBAL_DEFAULT_TIMEOUT):
+ timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+ self.source_address = source_address
self.timeout = timeout
if host:
self.connect(host)
@@ -128,10 +129,12 @@ class FTP:
if self.sock is not None:
self.close()
- def connect(self, host='', port=0, timeout=-999):
+ def connect(self, host='', port=0, timeout=-999, source_address=None):
'''Connect to host. Arguments are:
- host: hostname to connect to (string, default previous host)
- port: port to connect to (integer, default previous port)
+ - source_address: a 2-tuple (host, port) for the socket to bind
+ to as its source address before connecting.
'''
if host != '':
self.host = host
@@ -139,7 +142,10 @@ class FTP:
self.port = port
if timeout != -999:
self.timeout = timeout
- self.sock = socket.create_connection((self.host, self.port), self.timeout)
+ if source_address is not None:
+ self.source_address = source_address
+ self.sock = socket.create_connection((self.host, self.port), self.timeout,
+ source_address=self.source_address)
self.af = self.sock.family
self.file = self.sock.makefile('r', encoding=self.encoding)
self.welcome = self.getresp()
@@ -169,10 +175,8 @@ class FTP:
# Internal: "sanitize" a string for printing
def sanitize(self, s):
- if s[:5] == 'pass ' or s[:5] == 'PASS ':
- i = len(s)
- while i > 5 and s[i-1] in {'\r', '\n'}:
- i = i-1
+ if s[:5] in {'pass ', 'PASS '}:
+ i = len(s.rstrip('\r\n'))
s = s[:5] + '*'*(i-5) + s[i:]
return repr(s)
@@ -335,7 +339,8 @@ class FTP:
size = None
if self.passiveserver:
host, port = self.makepasv()
- conn = socket.create_connection((host, port), self.timeout)
+ conn = socket.create_connection((host, port), self.timeout,
+ source_address=self.source_address)
try:
if rest is not None:
self.sendcmd("REST %s" % rest)
@@ -426,7 +431,7 @@ class FTP:
"""Retrieve data in line mode. A new port is created for you.
Args:
- cmd: A RETR, LIST, NLST, or MLSD command.
+ cmd: A RETR, LIST, or NLST command.
callback: An optional single parameter callable that is called
for each line with the trailing CRLF stripped.
[default: print_line()]
@@ -527,6 +532,34 @@ class FTP:
cmd = cmd + (' ' + arg)
self.retrlines(cmd, func)
+ def mlsd(self, path="", facts=[]):
+ '''List a directory in a standardized format by using MLSD
+ command (RFC-3659). If path is omitted the current directory
+ is assumed. "facts" is a list of strings representing the type
+ of information desired (e.g. ["type", "size", "perm"]).
+
+ Return a generator object yielding a tuple of two elements
+ for every file found in path.
+ First element is the file name, the second one is a dictionary
+ including a variable number of "facts" depending on the server
+ and whether "facts" argument has been provided.
+ '''
+ if facts:
+ self.sendcmd("OPTS MLST " + ";".join(facts) + ";")
+ if path:
+ cmd = "MLSD %s" % path
+ else:
+ cmd = "MLSD"
+ lines = []
+ self.retrlines(cmd, lines.append)
+ for line in lines:
+ facts_found, _, name = line.rstrip(CRLF).partition(' ')
+ entry = {}
+ for fact in facts_found[:-1].split(";"):
+ key, _, value = fact.partition("=")
+ entry[key.lower()] = value
+ yield (name, entry)
+
def rename(self, fromname, toname):
'''Rename a file.'''
resp = self.sendcmd('RNFR ' + fromname)
@@ -561,10 +594,7 @@ class FTP:
resp = self.sendcmd('SIZE ' + filename)
if resp[:3] == '213':
s = resp[3:].strip()
- try:
- return int(s)
- except (OverflowError, ValueError):
- return int(s)
+ return int(s)
def mkd(self, dirname):
'''Make a directory, return its full pathname.'''
@@ -596,11 +626,11 @@ class FTP:
def close(self):
'''Close the connection without assuming anything about it.'''
- if self.file:
+ if self.file is not None:
self.file.close()
+ if self.sock is not None:
self.sock.close()
- self.file = self.sock = None
-
+ self.file = self.sock = None
try:
import ssl
@@ -644,7 +674,7 @@ else:
def __init__(self, host='', user='', passwd='', acct='', keyfile=None,
certfile=None, context=None,
- timeout=_GLOBAL_DEFAULT_TIMEOUT):
+ timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None):
if context is not None and keyfile is not None:
raise ValueError("context and keyfile arguments are mutually "
"exclusive")
@@ -655,7 +685,7 @@ else:
self.certfile = certfile
self.context = context
self._prot_p = False
- FTP.__init__(self, host, user, passwd, acct, timeout)
+ FTP.__init__(self, host, user, passwd, acct, timeout, source_address)
def login(self, user='', passwd='', acct='', secure=True):
if secure and not isinstance(self.sock, ssl.SSLSocket):
@@ -679,6 +709,14 @@ else:
self.file = self.sock.makefile(mode='r', encoding=self.encoding)
return resp
+ def ccc(self):
+ '''Switch back to a clear-text control connection.'''
+ if not isinstance(self.sock, ssl.SSLSocket):
+ raise ValueError("not using TLS")
+ resp = self.voidcmd('CCC')
+ self.sock = self.sock.unwrap()
+ return resp
+
def prot_p(self):
'''Set up secure data connection.'''
# PROT defines whether or not the data channel is to be protected.
@@ -818,11 +856,7 @@ def parse150(resp):
m = _150_re.match(resp)
if not m:
return None
- s = m.group(1)
- try:
- return int(s)
- except (OverflowError, ValueError):
- return int(s)
+ return int(m.group(1))
_227_re = None
diff --git a/Lib/functools.py b/Lib/functools.py
index 85ea257..1abb37a 100644
--- a/Lib/functools.py
+++ b/Lib/functools.py
@@ -114,14 +114,23 @@ def cmp_to_key(mycmp):
__hash__ = None
return K
+try:
+ from _functools import cmp_to_key
+except ImportError:
+ pass
+
_CacheInfo = namedtuple("CacheInfo", "hits misses maxsize currsize")
-def lru_cache(maxsize=100):
+def lru_cache(maxsize=100, typed=False):
"""Least-recently-used cache decorator.
If *maxsize* is set to None, the LRU features are disabled and the cache
can grow without bound.
+ If *typed* is True, arguments of different types will be cached separately.
+ For example, f(3.0) and f(3) will be treated as distinct calls with
+ distinct results.
+
Arguments to the cached function must be hashable.
View the cache statistics named tuple (hits, misses, maxsize, currsize) with
@@ -137,7 +146,7 @@ def lru_cache(maxsize=100):
# to allow the implementation to change (including a possible C version).
def decorating_function(user_function,
- tuple=tuple, sorted=sorted, len=len, KeyError=KeyError):
+ *, tuple=tuple, sorted=sorted, map=map, len=len, type=type, KeyError=KeyError):
hits = misses = 0
kwd_mark = (object(),) # separates positional and keyword args
@@ -151,7 +160,12 @@ def lru_cache(maxsize=100):
nonlocal hits, misses
key = args
if kwds:
- key += kwd_mark + tuple(sorted(kwds.items()))
+ sorted_items = tuple(sorted(kwds.items()))
+ key += kwd_mark + sorted_items
+ if typed:
+ key += tuple(map(type, args))
+ if kwds:
+ key += tuple(type(v) for k, v in sorted_items)
try:
result = cache[key]
hits += 1
@@ -172,7 +186,12 @@ def lru_cache(maxsize=100):
nonlocal hits, misses
key = args
if kwds:
- key += kwd_mark + tuple(sorted(kwds.items()))
+ sorted_items = tuple(sorted(kwds.items()))
+ key += kwd_mark + sorted_items
+ if typed:
+ key += tuple(map(type, args))
+ if kwds:
+ key += tuple(type(v) for k, v in sorted_items)
with lock:
try:
result = cache[key]
diff --git a/Lib/getopt.py b/Lib/getopt.py
index 980861d..3d6ecbd 100644
--- a/Lib/getopt.py
+++ b/Lib/getopt.py
@@ -19,7 +19,7 @@ option involved with the exception.
# Gerrit Holl <gerrit@nl.linux.org> moved the string-based exceptions
# to class-based exceptions.
#
-# Peter Åstrand <astrand@lysator.liu.se> added gnu_getopt().
+# Peter Åstrand <astrand@lysator.liu.se> added gnu_getopt().
#
# TODO for gnu_getopt():
#
@@ -34,6 +34,11 @@ option involved with the exception.
__all__ = ["GetoptError","error","getopt","gnu_getopt"]
import os
+try:
+ from gettext import gettext as _
+except ImportError:
+ # Bootstrapping Python: gettext's dependencies not built yet
+ def _(s): return s
class GetoptError(Exception):
opt = ''
@@ -153,10 +158,10 @@ def do_longs(opts, opt, longopts, args):
if has_arg:
if optarg is None:
if not args:
- raise GetoptError('option --%s requires argument' % opt, opt)
+ raise GetoptError(_('option --%s requires argument') % opt, opt)
optarg, args = args[0], args[1:]
elif optarg is not None:
- raise GetoptError('option --%s must not have an argument' % opt, opt)
+ raise GetoptError(_('option --%s must not have an argument') % opt, opt)
opts.append(('--' + opt, optarg or ''))
return opts, args
@@ -166,7 +171,7 @@ def do_longs(opts, opt, longopts, args):
def long_has_args(opt, longopts):
possibilities = [o for o in longopts if o.startswith(opt)]
if not possibilities:
- raise GetoptError('option --%s not recognized' % opt, opt)
+ raise GetoptError(_('option --%s not recognized') % opt, opt)
# Is there an exact match?
if opt in possibilities:
return False, opt
@@ -176,7 +181,7 @@ def long_has_args(opt, longopts):
if len(possibilities) > 1:
# XXX since possibilities contains all valid continuations, might be
# nice to work them into the error msg
- raise GetoptError('option --%s not a unique prefix' % opt, opt)
+ raise GetoptError(_('option --%s not a unique prefix') % opt, opt)
assert len(possibilities) == 1
unique_match = possibilities[0]
has_arg = unique_match.endswith('=')
@@ -190,7 +195,7 @@ def do_shorts(opts, optstring, shortopts, args):
if short_has_arg(opt, shortopts):
if optstring == '':
if not args:
- raise GetoptError('option -%s requires argument' % opt,
+ raise GetoptError(_('option -%s requires argument') % opt,
opt)
optstring, args = args[0], args[1:]
optarg, optstring = optstring, ''
@@ -203,7 +208,7 @@ def short_has_arg(opt, shortopts):
for i in range(len(shortopts)):
if opt == shortopts[i] != ':':
return shortopts.startswith(':', i+1)
- raise GetoptError('option -%s not recognized' % opt, opt)
+ raise GetoptError(_('option -%s not recognized') % opt, opt)
if __name__ == '__main__':
import sys
diff --git a/Lib/getpass.py b/Lib/getpass.py
index dc02bd1..0044742 100644
--- a/Lib/getpass.py
+++ b/Lib/getpass.py
@@ -72,7 +72,7 @@ def unix_getpass(prompt='Password: ', stream=None):
finally:
termios.tcsetattr(fd, tcsetattr_flags, old)
stream.flush() # issue7208
- except termios.error as e:
+ except termios.error:
if passwd is not None:
# _raw_input succeeded. The final tcsetattr failed. Reraise
# instead of leaving the terminal in an unknown state.
@@ -145,8 +145,6 @@ def getuser():
"""
- import os
-
for name in ('LOGNAME', 'USER', 'LNAME', 'USERNAME'):
user = os.environ.get(name)
if user:
diff --git a/Lib/gzip.py b/Lib/gzip.py
index ba2149e..e6b8193 100644
--- a/Lib/gzip.py
+++ b/Lib/gzip.py
@@ -16,18 +16,6 @@ FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16
READ, WRITE = 1, 2
-def U32(i):
- """Return i as an unsigned integer, assuming it fits in 32 bits.
- If it's >= 2GB when viewed as a 32-bit unsigned int, return a long.
- """
- if i < 0:
- i += 1 << 32
- return i
-
-def LOWU32(i):
- """Return the low-order 32 bits, as a non-negative int"""
- return i & 0xFFFFFFFF
-
def write32u(output, value):
# The L format writes the bit pattern correctly whether signed
# or unsigned.
@@ -348,6 +336,28 @@ class GzipFile(io.BufferedIOBase):
self.offset += size
return chunk
+ def read1(self, size=-1):
+ self._check_closed()
+ if self.mode != READ:
+ import errno
+ raise IOError(errno.EBADF, "read1() on write-only GzipFile object")
+
+ if self.extrasize <= 0 and self.fileobj is None:
+ return b''
+
+ try:
+ self._read()
+ except EOFError:
+ pass
+ if size < 0 or size > self.extrasize:
+ size = self.extrasize
+
+ offset = self.offset - self.extrastart
+ chunk = self.extrabuf[offset: offset + size]
+ self.extrasize -= size
+ self.offset += size
+ return chunk
+
def peek(self, n):
if self.mode != READ:
import errno
diff --git a/Lib/http/client.py b/Lib/http/client.py
index 745b999..88da550 100644
--- a/Lib/http/client.py
+++ b/Lib/http/client.py
@@ -700,7 +700,7 @@ class HTTPConnection:
self.send(connect_bytes)
for header, value in self._tunnel_headers.items():
header_str = "%s: %s\r\n" % (header, value)
- header_bytes = header_str.encode("latin1")
+ header_bytes = header_str.encode("latin-1")
self.send(header_bytes)
self.send(b'\r\n')
@@ -940,7 +940,7 @@ class HTTPConnection:
values = list(values)
for i, one_value in enumerate(values):
if hasattr(one_value, 'encode'):
- values[i] = one_value.encode('latin1')
+ values[i] = one_value.encode('latin-1')
elif isinstance(one_value, int):
values[i] = str(one_value).encode('ascii')
value = b'\r\n\t'.join(values)
diff --git a/Lib/http/server.py b/Lib/http/server.py
index 86fa37f..b79d191 100644
--- a/Lib/http/server.py
+++ b/Lib/http/server.py
@@ -105,6 +105,7 @@ import copy
DEFAULT_ERROR_MESSAGE = """\
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
"http://www.w3.org/TR/html4/strict.dtd">
+<html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8">
<title>Error response</title>
@@ -355,6 +356,7 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
"""
self.send_response_only(100)
+ self.flush_headers()
return True
def handle_one_request(self):
@@ -432,7 +434,8 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
self.wfile.write(content.encode('UTF-8', 'replace'))
def send_response(self, code, message=None):
- """Send the response header and log the response code.
+ """Add the response header to the headers buffer and log the
+ response code.
Also send two standard headers with the server software
version and the current date.
@@ -451,16 +454,19 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
else:
message = ''
if self.request_version != 'HTTP/0.9':
- self.wfile.write(("%s %d %s\r\n" %
- (self.protocol_version, code, message)).encode('latin1', 'strict'))
+ if not hasattr(self, '_headers_buffer'):
+ self._headers_buffer = []
+ self._headers_buffer.append(("%s %d %s\r\n" %
+ (self.protocol_version, code, message)).encode(
+ 'latin-1', 'strict'))
def send_header(self, keyword, value):
- """Send a MIME header."""
+ """Send a MIME header to the headers buffer."""
if self.request_version != 'HTTP/0.9':
if not hasattr(self, '_headers_buffer'):
self._headers_buffer = []
self._headers_buffer.append(
- ("%s: %s\r\n" % (keyword, value)).encode('latin1', 'strict'))
+ ("%s: %s\r\n" % (keyword, value)).encode('latin-1', 'strict'))
if keyword.lower() == 'connection':
if value.lower() == 'close':
@@ -472,6 +478,10 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
"""Send the blank line ending the MIME headers."""
if self.request_version != 'HTTP/0.9':
self._headers_buffer.append(b"\r\n")
+ self.flush_headers()
+
+ def flush_headers(self):
+ if hasattr(self, '_headers_buffer'):
self.wfile.write(b"".join(self._headers_buffer))
self._headers_buffer = []
@@ -725,10 +735,16 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
list.sort(key=lambda a: a.lower())
r = []
displaypath = html.escape(urllib.parse.unquote(self.path))
- r.append('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
- r.append("<html>\n<title>Directory listing for %s</title>\n" % displaypath)
- r.append("<body>\n<h2>Directory listing for %s</h2>\n" % displaypath)
- r.append("<hr>\n<ul>\n")
+ enc = sys.getfilesystemencoding()
+ title = 'Directory listing for %s' % displaypath
+ r.append('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" '
+ '"http://www.w3.org/TR/html4/strict.dtd">')
+ r.append('<html>\n<head>')
+ r.append('<meta http-equiv="Content-Type" '
+ 'content="text/html; charset=%s">' % enc)
+ r.append('<title>%s</title>\n</head>' % title)
+ r.append('<body>\n<h1>%s</h1>' % title)
+ r.append('<hr>\n<ul>')
for name in list:
fullname = os.path.join(path, name)
displayname = linkname = name
@@ -739,11 +755,10 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
if os.path.islink(fullname):
displayname = name + "@"
# Note: a link to a directory displays with @ and links with /
- r.append('<li><a href="%s">%s</a>\n'
+ r.append('<li><a href="%s">%s</a></li>'
% (urllib.parse.quote(linkname), html.escape(displayname)))
- r.append("</ul>\n<hr>\n</body>\n</html>\n")
- enc = sys.getfilesystemencoding()
- encoded = ''.join(r).encode(enc)
+ r.append('</ul>\n<hr>\n</body>\n</html>\n')
+ encoded = '\n'.join(r).encode(enc)
f = io.BytesIO()
f.write(encoded)
f.seek(0)
@@ -888,11 +903,7 @@ def nobody_uid():
def executable(path):
"""Test for executable file."""
- try:
- st = os.stat(path)
- except os.error:
- return False
- return st.st_mode & 0o111 != 0
+ return os.access(path, os.X_OK)
class CGIHTTPRequestHandler(SimpleHTTPRequestHandler):
@@ -1006,7 +1017,7 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler):
scriptname)
return
ispy = self.is_python(scriptname)
- if not ispy:
+ if self.have_fork or not ispy:
if not self.is_executable(scriptfile):
self.send_error(403, "CGI script is not executable (%r)" %
scriptname)
@@ -1081,6 +1092,7 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler):
env.setdefault(k, "")
self.send_response(200, "Script output follows")
+ self.flush_headers()
decoded_query = query.replace('+', ' ')
diff --git a/Lib/idlelib/configHandler.py b/Lib/idlelib/configHandler.py
index 73b8db5..79315ef 100644
--- a/Lib/idlelib/configHandler.py
+++ b/Lib/idlelib/configHandler.py
@@ -145,7 +145,8 @@ class IdleUserConfParser(IdleConfParser):
except IOError:
os.unlink(fname)
cfgFile = open(fname, 'w')
- self.write(cfgFile)
+ with cfgFile:
+ self.write(cfgFile)
else:
self.RemoveFile()
diff --git a/Lib/idlelib/idlever.py b/Lib/idlelib/idlever.py
index 97bf87b..5e9afb1 100644
--- a/Lib/idlelib/idlever.py
+++ b/Lib/idlelib/idlever.py
@@ -1 +1 @@
-IDLE_VERSION = "3.2.2"
+IDLE_VERSION = "3.3a0"
diff --git a/Lib/idlelib/rpc.py b/Lib/idlelib/rpc.py
index def4394..ddb36bd 100644
--- a/Lib/idlelib/rpc.py
+++ b/Lib/idlelib/rpc.py
@@ -534,6 +534,10 @@ class RPCClient(SocketIO):
def get_remote_proxy(self, oid):
return RPCProxy(self, oid)
+ def close(self):
+ self.listening_sock.close()
+ SocketIO.close(self)
+
class RPCProxy(object):
__methods = None
diff --git a/Lib/imaplib.py b/Lib/imaplib.py
index 1022e77..1b54065 100644
--- a/Lib/imaplib.py
+++ b/Lib/imaplib.py
@@ -249,15 +249,7 @@ class IMAP4:
def read(self, size):
"""Read 'size' bytes from remote."""
- chunks = []
- read = 0
- while read < size:
- data = self.file.read(min(size-read, 4096))
- if not data:
- break
- read += len(data)
- chunks.append(data)
- return b''.join(chunks)
+ return self.file.read(size)
def readline(self):
@@ -1177,25 +1169,40 @@ if HAVE_SSL:
"""IMAP4 client class over SSL connection
- Instantiate with: IMAP4_SSL([host[, port[, keyfile[, certfile]]]])
+ Instantiate with: IMAP4_SSL([host[, port[, keyfile[, certfile[, ssl_context]]]]])
host - host's name (default: localhost);
- port - port number (default: standard IMAP4 SSL port).
+ port - port number (default: standard IMAP4 SSL port);
keyfile - PEM formatted file that contains your private key (default: None);
certfile - PEM formatted certificate chain file (default: None);
+ ssl_context - a SSLContext object that contains your certificate chain
+ and private key (default: None)
+ Note: if ssl_context is provided, then parameters keyfile or
+ certfile should not be set otherwise ValueError is thrown.
for more documentation see the docstring of the parent class IMAP4.
"""
- def __init__(self, host = '', port = IMAP4_SSL_PORT, keyfile = None, certfile = None):
+ def __init__(self, host='', port=IMAP4_SSL_PORT, keyfile=None, certfile=None, ssl_context=None):
+ if ssl_context is not None and keyfile is not None:
+ raise ValueError("ssl_context and keyfile arguments are mutually "
+ "exclusive")
+ if ssl_context is not None and certfile is not None:
+ raise ValueError("ssl_context and certfile arguments are mutually "
+ "exclusive")
+
self.keyfile = keyfile
self.certfile = certfile
+ self.ssl_context = ssl_context
IMAP4.__init__(self, host, port)
def _create_socket(self):
sock = IMAP4._create_socket(self)
- return ssl.wrap_socket(sock, self.keyfile, self.certfile)
+ if self.ssl_context:
+ return self.ssl_context.wrap_socket(sock)
+ else:
+ return ssl.wrap_socket(sock, self.keyfile, self.certfile)
def open(self, host='', port=IMAP4_SSL_PORT):
"""Setup connection to remote server on "host:port".
diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py
index 2baaf93..9b20367 100644
--- a/Lib/importlib/__init__.py
+++ b/Lib/importlib/__init__.py
@@ -81,11 +81,10 @@ except ImportError:
except ImportError:
raise ImportError('posix, nt, or os2 module required for importlib')
_bootstrap._os = _os
-import imp, sys, marshal, errno, _io
+import imp, sys, marshal, _io
_bootstrap.imp = imp
_bootstrap.sys = sys
_bootstrap.marshal = marshal
-_bootstrap.errno = errno
_bootstrap._io = _io
import _warnings
_bootstrap._warnings = _warnings
diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
index 425b8bf..775fa85 100644
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -7,7 +7,7 @@ work. One should use importlib as the public-facing version of this module.
"""
-# Injected modules are '_warnings', 'imp', 'sys', 'marshal', 'errno', '_io',
+# Injected modules are '_warnings', 'imp', 'sys', 'marshal', '_io',
# and '_os' (a.k.a. 'posix', 'nt' or 'os2').
# Injected attribute is path_sep.
#
@@ -80,6 +80,30 @@ def _path_absolute(path):
return _path_join(_os.getcwd(), path)
+def _write_atomic(path, data):
+ """Best-effort function to write data to a path atomically.
+ Be prepared to handle a FileExistsError if concurrent writing of the
+ temporary file is attempted."""
+ if not sys.platform.startswith('win'):
+ # On POSIX-like platforms, renaming is atomic. id() is used to generate
+ # a pseudo-random filename.
+ path_tmp = '{}.{}'.format(path, id(path))
+ fd = _os.open(path_tmp, _os.O_EXCL | _os.O_CREAT | _os.O_WRONLY)
+ try:
+ with _io.FileIO(fd, 'wb') as file:
+ file.write(data)
+ _os.rename(path_tmp, path)
+ except OSError:
+ try:
+ _os.unlink(path_tmp)
+ except OSError:
+ pass
+ raise
+ else:
+ with _io.FileIO(path, 'wb') as file:
+ file.write(data)
+
+
def _wrap(new, old):
"""Simple substitute for functools.wraps."""
for replace in ['__module__', '__name__', '__doc__']:
@@ -240,7 +264,7 @@ class BuiltinImporter:
@classmethod
@_requires_builtin
def is_package(cls, fullname):
- """Return None as built-in module are never packages."""
+ """Return None as built-in modules are never packages."""
return False
@@ -404,6 +428,7 @@ class SourceLoader(_LoaderBasics):
else:
found = marshal.loads(bytes_data)
if isinstance(found, code_type):
+ imp._fix_co_filename(found, source_path)
return found
else:
msg = "Non-code object in {}"
@@ -479,28 +504,19 @@ class _SourceFileLoader(_FileLoader, SourceLoader):
parent = _path_join(parent, part)
try:
_os.mkdir(parent)
- except OSError as exc:
+ except FileExistsError:
# Probably another Python process already created the dir.
- if exc.errno == errno.EEXIST:
- continue
- else:
- raise
- except IOError as exc:
+ continue
+ except PermissionError:
# If can't get proper access, then just forget about writing
# the data.
- if exc.errno == errno.EACCES:
- return
- else:
- raise
- try:
- with _io.FileIO(path, 'wb') as file:
- file.write(data)
- except IOError as exc:
- # Don't worry if you can't write bytecode.
- if exc.errno == errno.EACCES:
return
- else:
- raise
+ try:
+ _write_atomic(path, data)
+ except (PermissionError, FileExistsError):
+ # Don't worry if you can't write bytecode or someone is writing
+ # it at the same time.
+ pass
class _SourcelessFileLoader(_FileLoader, _LoaderBasics):
@@ -758,14 +774,14 @@ class _ImportLockContext:
_IMPLICIT_META_PATH = [BuiltinImporter, FrozenImporter, _DefaultPathFinder]
-_ERR_MSG = 'No module named {}'
+_ERR_MSG = 'No module named {!r}'
def _gcd_import(name, package=None, level=0):
"""Import and return the module based on its name, the package the call is
being made from, and the level adjustment.
This function represents the greatest common denominator of functionality
- between import_module and __import__. This includes settting __package__ if
+ between import_module and __import__. This includes setting __package__ if
the loader did not.
"""
@@ -855,7 +871,7 @@ def __import__(name, globals={}, locals={}, fromlist=[], level=0):
module = _gcd_import(name)
else:
# __package__ is not guaranteed to be defined or could be set to None
- # to represent that it's proper value is unknown
+ # to represent that its proper value is unknown
package = globals.get('__package__')
if package is None:
package = globals['__name__']
diff --git a/Lib/importlib/test/__main__.py b/Lib/importlib/test/__main__.py
index decc53d..a1990b1 100644
--- a/Lib/importlib/test/__main__.py
+++ b/Lib/importlib/test/__main__.py
@@ -4,7 +4,6 @@ Specifying the ``--builtin`` flag will run tests, where applicable, with
builtins.__import__ instead of importlib.__import__.
"""
-import importlib
from importlib.test.import_ import util
import os.path
from test.support import run_unittest
@@ -13,11 +12,7 @@ import unittest
def test_main():
- if '__pycache__' in __file__:
- parts = __file__.split(os.path.sep)
- start_dir = sep.join(parts[:-2])
- else:
- start_dir = os.path.dirname(__file__)
+ start_dir = os.path.dirname(__file__)
top_dir = os.path.dirname(os.path.dirname(start_dir))
test_loader = unittest.TestLoader()
if '--builtin' in sys.argv:
diff --git a/Lib/importlib/test/regrtest.py b/Lib/importlib/test/regrtest.py
index b103ae7d..dc0eb97 100644
--- a/Lib/importlib/test/regrtest.py
+++ b/Lib/importlib/test/regrtest.py
@@ -5,13 +5,6 @@ invalidates are automatically skipped if the entire test suite is run.
Otherwise all command-line options valid for test.regrtest are also valid for
this script.
-XXX FAILING
- * test_import
- - test_incorrect_code_name
- file name differing between __file__ and co_filename (r68360 on trunk)
- - test_import_by_filename
- exception for trying to import by file name does not match
-
"""
import importlib
import sys
diff --git a/Lib/inspect.py b/Lib/inspect.py
index bb46ea6..9898b11 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -33,7 +33,6 @@ import sys
import os
import types
import itertools
-import string
import re
import imp
import tokenize
@@ -918,6 +917,43 @@ def formatargvalues(args, varargs, varkw, locals,
specs.append(formatvarkw(varkw) + formatvalue(locals[varkw]))
return '(' + ', '.join(specs) + ')'
+def _missing_arguments(f_name, argnames, pos, values):
+ names = [repr(name) for name in argnames if name not in values]
+ missing = len(names)
+ if missing == 1:
+ s = names[0]
+ elif missing == 2:
+ s = "{} and {}".format(*names)
+ else:
+ tail = ", {} and {}".format(names[-2:])
+ del names[-2:]
+ s = ", ".join(names) + tail
+ raise TypeError("%s() missing %i required %s argument%s: %s" %
+ (f_name, missing,
+ "positional" if pos else "keyword-only",
+ "" if missing == 1 else "s", s))
+
+def _too_many(f_name, args, kwonly, varargs, defcount, given, values):
+ atleast = len(args) - defcount
+ kwonly_given = len([arg for arg in kwonly if arg in values])
+ if varargs:
+ plural = atleast != 1
+ sig = "at least %d" % (atleast,)
+ elif defcount:
+ plural = True
+ sig = "from %d to %d" % (atleast, len(args))
+ else:
+ plural = len(args) != 1
+ sig = str(len(args))
+ kwonly_sig = ""
+ if kwonly_given:
+ msg = " positional argument%s (and %d keyword-only argument%s)"
+ kwonly_sig = (msg % ("s" if given != 1 else "", kwonly_given,
+ "s" if kwonly_given != 1 else ""))
+ raise TypeError("%s() takes %s positional argument%s but %d%s %s given" %
+ (f_name, sig, "s" if plural else "", given, kwonly_sig,
+ "was" if given == 1 and not kwonly_given else "were"))
+
def getcallargs(func, *positional, **named):
"""Get the mapping of arguments to values.
@@ -929,64 +965,53 @@ def getcallargs(func, *positional, **named):
f_name = func.__name__
arg2value = {}
+
if ismethod(func) and func.__self__ is not None:
# implicit 'self' (or 'cls' for classmethods) argument
positional = (func.__self__,) + positional
num_pos = len(positional)
- num_total = num_pos + len(named)
num_args = len(args)
num_defaults = len(defaults) if defaults else 0
- for arg, value in zip(args, positional):
- arg2value[arg] = value
+
+ n = min(num_pos, num_args)
+ for i in range(n):
+ arg2value[args[i]] = positional[i]
if varargs:
- if num_pos > num_args:
- arg2value[varargs] = positional[-(num_pos-num_args):]
- else:
- arg2value[varargs] = ()
- elif 0 < num_args < num_pos:
- raise TypeError('%s() takes %s %d positional %s (%d given)' % (
- f_name, 'at most' if defaults else 'exactly', num_args,
- 'arguments' if num_args > 1 else 'argument', num_total))
- elif num_args == 0 and num_total:
- if varkw or kwonlyargs:
- if num_pos:
- # XXX: We should use num_pos, but Python also uses num_total:
- raise TypeError('%s() takes exactly 0 positional arguments '
- '(%d given)' % (f_name, num_total))
- else:
- raise TypeError('%s() takes no arguments (%d given)' %
- (f_name, num_total))
-
- for arg in itertools.chain(args, kwonlyargs):
- if arg in named:
- if arg in arg2value:
- raise TypeError("%s() got multiple values for keyword "
- "argument '%s'" % (f_name, arg))
- else:
- arg2value[arg] = named.pop(arg)
- for kwonlyarg in kwonlyargs:
- if kwonlyarg not in arg2value:
- try:
- arg2value[kwonlyarg] = kwonlydefaults[kwonlyarg]
- except KeyError:
- raise TypeError("%s() needs keyword-only argument %s" %
- (f_name, kwonlyarg))
- if defaults: # fill in any missing values with the defaults
- for arg, value in zip(args[-num_defaults:], defaults):
- if arg not in arg2value:
- arg2value[arg] = value
+ arg2value[varargs] = tuple(positional[n:])
+ possible_kwargs = set(args + kwonlyargs)
if varkw:
- arg2value[varkw] = named
- elif named:
- unexpected = next(iter(named))
- raise TypeError("%s() got an unexpected keyword argument '%s'" %
- (f_name, unexpected))
- unassigned = num_args - len([arg for arg in args if arg in arg2value])
- if unassigned:
- num_required = num_args - num_defaults
- raise TypeError('%s() takes %s %d %s (%d given)' % (
- f_name, 'at least' if defaults else 'exactly', num_required,
- 'arguments' if num_required > 1 else 'argument', num_total))
+ arg2value[varkw] = {}
+ for kw, value in named.items():
+ if kw not in possible_kwargs:
+ if not varkw:
+ raise TypeError("%s() got an unexpected keyword argument %r" %
+ (f_name, kw))
+ arg2value[varkw][kw] = value
+ continue
+ if kw in arg2value:
+ raise TypeError("%s() got multiple values for argument %r" %
+ (f_name, kw))
+ arg2value[kw] = value
+ if num_pos > num_args and not varargs:
+ _too_many(f_name, args, kwonlyargs, varargs, num_defaults,
+ num_pos, arg2value)
+ if num_pos < num_args:
+ req = args[:num_args - num_defaults]
+ for arg in req:
+ if arg not in arg2value:
+ _missing_arguments(f_name, req, True, arg2value)
+ for i, arg in enumerate(args[num_args - num_defaults:]):
+ if arg not in arg2value:
+ arg2value[arg] = defaults[i]
+ missing = 0
+ for kwarg in kwonlyargs:
+ if kwarg not in arg2value:
+ if kwarg in kwonlydefaults:
+ arg2value[kwarg] = kwonlydefaults[kwarg]
+ else:
+ missing += 1
+ if missing:
+ _missing_arguments(f_name, kwonlyargs, False, arg2value)
return arg2value
# -------------------------------------------------- stack frame extraction
diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py
index 3174e31..e7c0539 100644
--- a/Lib/json/decoder.py
+++ b/Lib/json/decoder.py