summaryrefslogtreecommitdiffstats
path: root/Source/CPack/cmCPackPKGGenerator.h
Commit message (Expand)AuthorAgeFilesLines
* CPack/productbuild: add options to control domains elementDavid Wosk2022-01-061-0/+4
* Source: Cleanup and simplify some codeRose2021-11-041-1/+2
* Modernize: Use #pragma once in all header filesKitware Robot2020-09-031-4/+1
* CPack: Set background image in macOS installerSergey Larin2019-11-081-1/+7
* Use C++11 override instead of CM_OVERRIDEBrad King2017-09-151-4/+4
* IWYU: Mark cmConfigure.h with pragma: keepDaniel Pfeifer2017-08-261-1/+1
* Add CM_OVERRIDE to CPack generator destructorsBrad King2017-06-081-1/+1
* Use quotes for non-system includesDaniel Pfeifer2017-04-111-1/+1
* cmConfigure: Ensure separate include block in headersDaniel Pfeifer2017-04-111-0/+1
* iwyu: Fix OSX specific issuesDaniel Pfeifer2016-11-221-1/+3
* Remove cmObject.h and cmTypeMacro.hDaniel Pfeifer2016-10-241-0/+3
* Simplify CMake per-source license noticesBrad King2016-09-271-12/+2
* Source/CPack: Add CM_OVERRIDE on MacOS cmCPackGenerator overridesBrad King2016-07-221-3/+3
* CPack/PackageMaker: port to cmXMLWriterDaniel Pfeifer2016-06-091-9/+4
* Source/CPack: Run clang-format to fix styleBrad King2016-06-071-2/+0
* PackageMaker: factor out common code for creating pkg files.Clinton Stimpson2016-06-021-0/+103
: 0.0%;'/> -rw-r--r--Doc/c-api/import.rst6
-rw-r--r--Doc/c-api/init.rst17
-rw-r--r--Doc/c-api/memory.rst44
-rw-r--r--Doc/c-api/memoryview.rst2
-rw-r--r--Doc/c-api/module.rst348
-rw-r--r--Doc/c-api/number.rst17
-rw-r--r--Doc/c-api/sys.rst54
-rw-r--r--Doc/c-api/typeobj.rst67
-rw-r--r--Doc/c-api/unicode.rst35
-rw-r--r--Doc/conf.py3
-rw-r--r--Doc/data/refcounts.dat11
-rw-r--r--Doc/distutils/apiref.rst43
-rw-r--r--Doc/distutils/builtdist.rst47
-rw-r--r--Doc/distutils/introduction.rst6
-rw-r--r--Doc/distutils/sourcedist.rst8
-rw-r--r--Doc/extending/building.rst63
-rw-r--r--Doc/extending/embedding.rst20
-rw-r--r--Doc/extending/extending.rst19
-rw-r--r--Doc/extending/newtypes.rst2
-rw-r--r--Doc/extending/windows.rst5
-rw-r--r--Doc/glossary.rst115
-rw-r--r--Doc/howto/clinic.rst71
-rw-r--r--Doc/howto/functional.rst8
-rw-r--r--Doc/howto/logging-cookbook.rst11
-rw-r--r--Doc/howto/pyporting.rst15
-rw-r--r--Doc/howto/regex.rst2
-rw-r--r--Doc/howto/unicode.rst14
-rw-r--r--Doc/includes/noddy.c2
-rw-r--r--Doc/includes/run-func.c2
-rw-r--r--Doc/includes/typestruct.h3
-rw-r--r--Doc/install/index.rst2
-rw-r--r--Doc/library/__future__.rst3
-rw-r--r--Doc/library/argparse.rst35
-rw-r--r--Doc/library/asynchat.rst34
-rw-r--r--Doc/library/asyncio-dev.rst12
-rw-r--r--Doc/library/asyncio-eventloop.rst2
-rw-r--r--Doc/library/asyncio-eventloops.rst6
-rw-r--r--Doc/library/asyncio-protocol.rst6
-rw-r--r--Doc/library/asyncio-queue.rst11
-rw-r--r--Doc/library/asyncio-task.rst92
-rw-r--r--Doc/library/bz2.rst40
-rw-r--r--Doc/library/cgi.rst7
-rw-r--r--Doc/library/cmath.rst32
-rw-r--r--Doc/library/code.rst5
-rw-r--r--Doc/library/codecs.rst49
-rw-r--r--Doc/library/collections.abc.rst84
-rw-r--r--Doc/library/collections.rst42
-rw-r--r--Doc/library/compileall.rst70
-rw-r--r--Doc/library/concurrent.futures.rst25
-rw-r--r--Doc/library/configparser.rst120
-rw-r--r--Doc/library/constants.rst1
-rw-r--r--Doc/library/contextlib.rst10
-rw-r--r--Doc/library/csv.rst5
-rw-r--r--Doc/library/curses.rst7
-rw-r--r--Doc/library/datetime.rst27
-rw-r--r--Doc/library/dbm.rst9
-rw-r--r--Doc/library/decimal.rst15
-rw-r--r--Doc/library/development.rst1
-rw-r--r--Doc/library/difflib.rst101
-rw-r--r--Doc/library/dis.rst87
-rw-r--r--Doc/library/distribution.rst1
-rw-r--r--Doc/library/doctest.rst12
-rw-r--r--Doc/library/email.message.rst9
-rw-r--r--Doc/library/email.mime.rst6
-rw-r--r--Doc/library/email.policy.rst43
-rw-r--r--Doc/library/enum.rst22
-rw-r--r--Doc/library/errno.rst5
-rw-r--r--Doc/library/exceptions.rst38
-rw-r--r--Doc/library/faulthandler.rst24
-rw-r--r--Doc/library/fcntl.rst64
-rw-r--r--Doc/library/formatter.rst2
-rw-r--r--Doc/library/fractions.rst3
-rw-r--r--Doc/library/functions.rst46
-rw-r--r--Doc/library/gc.rst6
-rw-r--r--Doc/library/gettext.rst6
-rw-r--r--Doc/library/glob.rst24
-rw-r--r--Doc/library/gzip.rst37
-rw-r--r--Doc/library/heapq.rst14
-rw-r--r--Doc/library/html.parser.rst47
-rw-r--r--Doc/library/http.client.rst232
-rw-r--r--Doc/library/http.cookies.rst41
-rw-r--r--Doc/library/http.rst113
-rw-r--r--Doc/library/imaplib.rst43
-rw-r--r--Doc/library/imghdr.rst10
-rw-r--r--Doc/library/imp.rst11
-rw-r--r--Doc/library/importlib.rst172
-rw-r--r--Doc/library/inspect.rst218
-rw-r--r--Doc/library/io.rst57
-rw-r--r--Doc/library/ipaddress.rst34
-rw-r--r--Doc/library/itertools.rst38
-rw-r--r--Doc/library/json.rst103
-rw-r--r--Doc/library/linecache.rst8
-rw-r--r--Doc/library/locale.rst8
-rw-r--r--Doc/library/logging.handlers.rst13
-rw-r--r--Doc/library/logging.rst13
-rw-r--r--Doc/library/lzma.rst41
-rw-r--r--Doc/library/math.rst58
-rw-r--r--Doc/library/mmap.rst9
-rw-r--r--Doc/library/multiprocessing.rst7
-rw-r--r--Doc/library/operator.rst18
-rw-r--r--Doc/library/os.path.rst32
-rw-r--r--Doc/library/os.rst314
-rw-r--r--Doc/library/ossaudiodev.rst29
-rw-r--r--Doc/library/pathlib.rst112
-rw-r--r--Doc/library/pickle.rst4
-rw-r--r--Doc/library/pkgutil.rst4
-rw-r--r--Doc/library/platform.rst4
-rw-r--r--Doc/library/poplib.rst9
-rw-r--r--Doc/library/pprint.rst35
-rw-r--r--Doc/library/py_compile.rst9
-rw-r--r--Doc/library/queue.rst36
-rw-r--r--Doc/library/random.rst3
-rw-r--r--Doc/library/re.rst93
-rw-r--r--Doc/library/readline.rst29
-rw-r--r--Doc/library/select.rst31
-rw-r--r--Doc/library/selectors.rst17
-rw-r--r--Doc/library/shutil.rst50
-rw-r--r--Doc/library/signal.rst35
-rw-r--r--Doc/library/site.rst18
-rw-r--r--Doc/library/smtpd.rst88
-rw-r--r--Doc/library/smtplib.rst113
-rw-r--r--Doc/library/sndhdr.rst15
-rw-r--r--Doc/library/socket.rst141
-rw-r--r--Doc/library/socketserver.rst4
-rw-r--r--Doc/library/sqlite3.rst3
-rw-r--r--Doc/library/ssl.rst296
-rw-r--r--Doc/library/stat.rst28
-rw-r--r--Doc/library/stdtypes.rst297
-rw-r--r--Doc/library/string.rst4
-rw-r--r--Doc/library/subprocess.rst308
-rw-r--r--Doc/library/symtable.rst4
-rw-r--r--Doc/library/sys.rst63
-rw-r--r--Doc/library/tarfile.rst64
-rw-r--r--Doc/library/tempfile.rst191
-rw-r--r--Doc/library/test.rst11
-rw-r--r--Doc/library/time.rst9
-rw-r--r--Doc/library/timeit.rst44
-rw-r--r--Doc/library/token.rst1
-rw-r--r--Doc/library/traceback.rst208
-rw-r--r--Doc/library/tracemalloc.rst7
-rw-r--r--Doc/library/tulip_coro.diabin4461 -> 4459 bytes-rw-r--r--Doc/library/tulip_coro.pngbin45565 -> 45021 bytes-rw-r--r--Doc/library/turtle.rst5
-rw-r--r--Doc/library/types.rst34
-rw-r--r--Doc/library/typing.rst433
-rw-r--r--Doc/library/unicodedata.rst8
-rw-r--r--Doc/library/unittest.mock.rst52
-rw-r--r--Doc/library/unittest.rst116
-rw-r--r--Doc/library/urllib.parse.rst25
-rw-r--r--Doc/library/urllib.request.rst63
-rw-r--r--Doc/library/weakref.rst6
-rw-r--r--Doc/library/wsgiref.rst9
-rw-r--r--Doc/library/xml.sax.reader.rst12
-rw-r--r--Doc/library/xml.sax.rst6
-rw-r--r--Doc/library/xmlrpc.client.rst27
-rw-r--r--Doc/library/zipapp.rst259
-rw-r--r--Doc/library/zipfile.rst44
-rw-r--r--Doc/library/zipimport.rst5
-rw-r--r--Doc/make.bat32
-rw-r--r--Doc/reference/compound_stmts.rst127
-rw-r--r--Doc/reference/datamodel.rst219
-rw-r--r--Doc/reference/expressions.rst48
-rw-r--r--Doc/reference/import.rst22
-rw-r--r--Doc/reference/lexical_analysis.rst8
-rw-r--r--Doc/reference/simple_stmts.rst2
-rw-r--r--Doc/tools/extensions/pyspecific.py2
-rw-r--r--Doc/tools/susp-ignored.csv48
-rw-r--r--Doc/tools/templates/indexsidebar.html3
-rw-r--r--Doc/tutorial/appendix.rst4
-rw-r--r--Doc/tutorial/datastructures.rst25
-rw-r--r--Doc/tutorial/interpreter.rst12
-rw-r--r--Doc/tutorial/modules.rst11
-rw-r--r--Doc/tutorial/stdlib.rst2
-rw-r--r--Doc/tutorial/stdlib2.rst4
-rw-r--r--Doc/using/cmdline.rst7
-rw-r--r--Doc/using/venv-create.inc4
-rw-r--r--Doc/using/win_installer.pngbin0 -> 48994 bytes-rw-r--r--Doc/using/windows.rst669
-rw-r--r--Doc/whatsnew/3.5.rst1249
-rw-r--r--Doc/whatsnew/index.rst1
-rw-r--r--Grammar/Grammar41
-rw-r--r--Include/Python-ast.h83
-rw-r--r--Include/Python.h2
-rw-r--r--Include/abstract.h18
-rw-r--r--Include/bytes_methods.h4
-rw-r--r--Include/bytesobject.h1
-rw-r--r--Include/ceval.h8
-rw-r--r--Include/code.h13
-rw-r--r--Include/codecs.h9
-rw-r--r--Include/compile.h1
-rw-r--r--Include/complexobject.h22
-rw-r--r--Include/dictobject.h27
-rw-r--r--Include/fileobject.h11
-rw-r--r--Include/fileutils.h81
-rw-r--r--Include/genobject.h48
-rw-r--r--Include/graminit.h155
-rw-r--r--Include/grammar.h2
-rw-r--r--Include/listobject.h1
-rw-r--r--Include/longobject.h3
-rw-r--r--Include/memoryobject.h4
-rw-r--r--Include/methodobject.h5
-rw-r--r--Include/modsupport.h69
-rw-r--r--Include/moduleobject.h26
-rw-r--r--Include/node.h2
-rw-r--r--Include/object.h26
-rw-r--r--Include/objimpl.h4
-rw-r--r--Include/odictobject.h43
-rw-r--r--Include/opcode.h257
-rw-r--r--Include/osdefs.h5
-rw-r--r--Include/patchlevel.h10
-rw-r--r--Include/pyatomic.h88
-rw-r--r--Include/pydebug.h2
-rw-r--r--Include/pyerrors.h9
-rw-r--r--Include/pylifecycle.h124
-rw-r--r--Include/pymacro.h15
-rw-r--r--Include/pymem.h13
-rw-r--r--Include/pyport.h61
-rw-r--r--Include/pystate.h13
-rw-r--r--Include/pystrhex.h17
-rw-r--r--Include/pythonrun.h113
-rw-r--r--Include/pytime.h176
-rw-r--r--Include/setobject.h96
-rw-r--r--Include/sliceobject.h2
-rw-r--r--Include/symtable.h5
-rw-r--r--Include/token.h15
-rw-r--r--Include/typeslots.h9
-rw-r--r--Include/ucnhash.h2
-rw-r--r--Include/unicodeobject.h8
-rw-r--r--Lib/__future__.py6
-rw-r--r--Lib/_collections_abc.py203
-rw-r--r--Lib/_compression.py152
-rw-r--r--Lib/_dummy_thread.py8
-rw-r--r--Lib/_pydecimal.py6380
-rw-r--r--Lib/_pyio.py471
-rw-r--r--Lib/_strptime.py8
-rw-r--r--Lib/abc.py2
-rw-r--r--Lib/argparse.py45
-rw-r--r--Lib/ast.py2
-rw-r--r--Lib/asynchat.py3
-rw-r--r--Lib/asyncore.py39
-rw-r--r--Lib/binhex.py28
-rw-r--r--Lib/bz2.py237
-rwxr-xr-xLib/cgi.py6
-rw-r--r--Lib/cgitb.py5
-rw-r--r--Lib/code.py38
-rw-r--r--Lib/codecs.py18
-rw-r--r--Lib/collections/__init__.py83
-rw-r--r--Lib/compileall.py135
-rw-r--r--Lib/concurrent/futures/_base.py21
-rw-r--r--Lib/concurrent/futures/process.py80
-rw-r--r--Lib/concurrent/futures/thread.py10
-rw-r--r--Lib/configparser.py182
-rw-r--r--Lib/contextlib.py51
-rw-r--r--Lib/copy.py12
-rw-r--r--Lib/csv.py14
-rw-r--r--Lib/ctypes/__init__.py14
-rw-r--r--Lib/ctypes/_endian.py2
-rw-r--r--Lib/ctypes/test/test_as_parameter.py2
-rw-r--r--Lib/ctypes/test/test_byteswap.py20
-rw-r--r--Lib/ctypes/test/test_loading.py4
-rw-r--r--Lib/ctypes/test/test_pointers.py5
-rw-r--r--Lib/ctypes/test/test_prototypes.py5
-rw-r--r--Lib/ctypes/test/test_values.py19
-rw-r--r--Lib/ctypes/util.py8
-rw-r--r--Lib/datetime.py363
-rw-r--r--Lib/dbm/__init__.py6
-rw-r--r--Lib/dbm/dumb.py27
-rw-r--r--Lib/decimal.py6418
-rw-r--r--Lib/difflib.py122
-rw-r--r--Lib/dis.py10
-rw-r--r--Lib/distutils/_msvccompiler.py494
-rw-r--r--Lib/distutils/archive_util.py21
-rw-r--r--Lib/distutils/ccompiler.py2
-rw-r--r--Lib/distutils/command/bdist.py3
-rw-r--r--Lib/distutils/command/bdist_dumb.py3
-rw-r--r--Lib/distutils/command/bdist_wininst.py36
-rw-r--r--Lib/distutils/command/build.py9
-rw-r--r--Lib/distutils/command/build_ext.py107
-rw-r--r--Lib/distutils/command/build_py.py4
-rw-r--r--Lib/distutils/command/install.py2
-rw-r--r--Lib/distutils/command/install_lib.py16
-rw-r--r--Lib/distutils/command/upload.py40
-rw-r--r--Lib/distutils/command/wininst-14.0-amd64.exebin0 -> 136192 bytes-rw-r--r--Lib/distutils/command/wininst-14.0.exebin0 -> 129024 bytes-rw-r--r--Lib/distutils/core.py2
-rw-r--r--Lib/distutils/dist.py69
-rw-r--r--Lib/distutils/extension.py8
-rw-r--r--Lib/distutils/msvc9compiler.py3
-rw-r--r--Lib/distutils/msvccompiler.py3
-rw-r--r--Lib/distutils/spawn.py3
-rw-r--r--Lib/distutils/sysconfig.py27
-rw-r--r--Lib/distutils/tests/test_archive_util.py154
-rw-r--r--Lib/distutils/tests/test_bdist.py2
-rw-r--r--Lib/distutils/tests/test_build_ext.py56
-rw-r--r--Lib/distutils/tests/test_build_py.py4
-rw-r--r--Lib/distutils/tests/test_install.py2
-rw-r--r--Lib/distutils/tests/test_install_lib.py13
-rw-r--r--Lib/distutils/tests/test_msvccompiler.py39
-rw-r--r--Lib/distutils/util.py9
-rw-r--r--Lib/distutils/version.py6
-rw-r--r--Lib/doctest.py21
-rw-r--r--Lib/email/__init__.py2
-rw-r--r--Lib/email/_header_value_parser.py11
-rw-r--r--Lib/email/_policybase.py8
-rw-r--r--Lib/email/charset.py3
-rw-r--r--Lib/email/feedparser.py16
-rw-r--r--Lib/email/generator.py13
-rw-r--r--Lib/email/header.py3
-rw-r--r--Lib/email/headerregistry.py6
-rw-r--r--Lib/email/message.py28
-rw-r--r--Lib/email/mime/text.py3
-rw-r--r--Lib/email/policy.py15
-rw-r--r--Lib/encodings/aliases.py5
-rw-r--r--Lib/encodings/cp65001.py9
-rw-r--r--Lib/encodings/koi8_t.py308
-rw-r--r--Lib/encodings/kz1048.py307
-rw-r--r--Lib/enum.py34
-rw-r--r--Lib/fileinput.py8
-rw-r--r--Lib/formatter.py2
-rw-r--r--Lib/fractions.py67
-rw-r--r--Lib/ftplib.py117
-rw-r--r--Lib/functools.py298
-rw-r--r--Lib/genericpath.py13
-rw-r--r--Lib/gettext.py13
-rw-r--r--Lib/glob.py56
-rw-r--r--Lib/gzip.py500
-rw-r--r--Lib/heapq.py349
-rw-r--r--Lib/html/entities.py3
-rw-r--r--Lib/html/parser.py114
-rw-r--r--Lib/http/__init__.py135
-rw-r--r--Lib/http/client.py358
-rw-r--r--Lib/http/cookiejar.py3
-rw-r--r--Lib/http/cookies.py253
-rw-r--r--Lib/http/server.py156
-rw-r--r--Lib/idlelib/ChangeLog10
-rw-r--r--Lib/idlelib/CodeContext.py8
-rw-r--r--Lib/idlelib/HISTORY.txt8
-rw-r--r--Lib/idlelib/NEWS.txt30
-rw-r--r--Lib/idlelib/README.txt2
-rw-r--r--Lib/idlelib/WidgetRedirector.py8
-rw-r--r--Lib/idlelib/idle_test/test_searchengine.py2
-rw-r--r--Lib/idlelib/idlever.py1
-rw-r--r--Lib/imaplib.py85
-rw-r--r--Lib/imghdr.py12
-rw-r--r--Lib/imp.py70
-rw-r--r--Lib/importlib/__init__.py29
-rw-r--r--Lib/importlib/_bootstrap.py1725
-rw-r--r--Lib/importlib/_bootstrap_external.py1426
-rw-r--r--Lib/importlib/abc.py29
-rw-r--r--Lib/importlib/machinery.py18
-rw-r--r--Lib/importlib/util.py111
-rw-r--r--Lib/inspect.py739
-rw-r--r--Lib/ipaddress.py568
-rw-r--r--Lib/json/__init__.py7
-rw-r--r--Lib/json/decoder.py86
-rw-r--r--Lib/json/encoder.py9
-rw-r--r--Lib/json/tool.py39
-rw-r--r--Lib/lib2to3/Grammar.txt10
-rwxr-xr-xLib/lib2to3/pgen2/token.py6
-rw-r--r--Lib/lib2to3/pgen2/tokenize.py78
-rw-r--r--Lib/lib2to3/tests/test_parser.py54
-rw-r--r--Lib/linecache.py89
-rw-r--r--Lib/locale.py17
-rw-r--r--Lib/logging/__init__.py27
-rw-r--r--Lib/logging/config.py10
-rw-r--r--Lib/logging/handlers.py14
-rw-r--r--Lib/lzma.py224
-rw-r--r--Lib/macpath.py34
-rw-r--r--Lib/mailbox.py14
-rw-r--r--Lib/modulefinder.py6
-rw-r--r--Lib/msilib/__init__.py7
-rw-r--r--Lib/multiprocessing/connection.py33
-rw-r--r--Lib/multiprocessing/dummy/__init__.py2
-rw-r--r--Lib/multiprocessing/dummy/connection.py5
-rw-r--r--Lib/multiprocessing/forkserver.py20
-rw-r--r--Lib/multiprocessing/heap.py20
-rw-r--r--Lib/multiprocessing/managers.py35
-rw-r--r--Lib/multiprocessing/pool.py27
-rw-r--r--Lib/multiprocessing/popen_fork.py3
-rw-r--r--Lib/multiprocessing/queues.py27
-rw-r--r--Lib/multiprocessing/sharedctypes.py26
-rw-r--r--Lib/multiprocessing/synchronize.py32
-rw-r--r--Lib/multiprocessing/util.py14
-rw-r--r--Lib/ntpath.py323
-rw-r--r--Lib/opcode.py21
-rw-r--r--Lib/operator.py69
-rw-r--r--Lib/os.py80
-rw-r--r--Lib/pathlib.py118
-rwxr-xr-xLib/pdb.py2
-rw-r--r--Lib/pickle.py33
-rw-r--r--Lib/pkgutil.py2
-rwxr-xr-xLib/platform.py98
-rw-r--r--Lib/poplib.py7
-rw-r--r--Lib/posixpath.py73
-rw-r--r--Lib/pprint.py457
-rw-r--r--Lib/py_compile.py19
-rwxr-xr-xLib/pydoc.py38
-rw-r--r--Lib/pydoc_data/topics.py40
-rw-r--r--Lib/queue.py5
-rw-r--r--Lib/random.py2
-rw-r--r--Lib/re.py48
-rw-r--r--Lib/reprlib.py12
-rw-r--r--Lib/runpy.py2
-rw-r--r--Lib/sched.py6
-rw-r--r--Lib/selectors.py77
-rw-r--r--Lib/shlex.py3
-rw-r--r--Lib/shutil.py102
-rw-r--r--Lib/signal.py79
-rw-r--r--Lib/site.py25
-rwxr-xr-xLib/smtpd.py287
-rwxr-xr-xLib/smtplib.py239
-rw-r--r--Lib/sndhdr.py7
-rw-r--r--Lib/socket.py203
-rw-r--r--Lib/socketserver.py97
-rw-r--r--Lib/sqlite3/test/factory.py18
-rw-r--r--Lib/sre_compile.py184
-rw-r--r--Lib/sre_constants.py259
-rw-r--r--Lib/sre_parse.py606
-rw-r--r--Lib/ssl.py310
-rw-r--r--Lib/stat.py23
-rw-r--r--Lib/statistics.py4
-rw-r--r--Lib/string.py3
-rw-r--r--Lib/subprocess.py272
-rwxr-xr-xLib/symbol.py155
-rw-r--r--Lib/symtable.py9
-rw-r--r--Lib/sysconfig.py19
-rwxr-xr-xLib/tarfile.py95
-rw-r--r--Lib/telnetlib.py9
-rw-r--r--Lib/tempfile.py143
-rw-r--r--Lib/test/_test_multiprocessing.py10
-rw-r--r--Lib/test/badsyntax_async1.py2
-rw-r--r--Lib/test/badsyntax_async2.py2
-rw-r--r--Lib/test/badsyntax_async3.py2
-rw-r--r--Lib/test/badsyntax_async4.py2
-rw-r--r--Lib/test/badsyntax_async5.py2
-rw-r--r--Lib/test/badsyntax_async6.py2
-rw-r--r--Lib/test/badsyntax_async7.py2
-rw-r--r--Lib/test/badsyntax_async8.py2
-rw-r--r--Lib/test/datetimetester.py120
-rw-r--r--Lib/test/eintrdata/eintr_tester.py376
-rw-r--r--Lib/test/exception_hierarchy.txt2
-rw-r--r--Lib/test/fork_wait.py7
-rw-r--r--Lib/test/imghdrdata/python.exrbin0 -> 2635 bytes-rw-r--r--Lib/test/imghdrdata/python.webpbin0 -> 432 bytes-rw-r--r--Lib/test/inspect_fodder.py15
-rw-r--r--Lib/test/inspect_fodder2.py28
-rw-r--r--Lib/test/list_tests.py12
-rw-r--r--Lib/test/lock_tests.py8
-rw-r--r--Lib/test/mock_socket.py15
-rw-r--r--Lib/test/pickletester.py116
-rwxr-xr-xLib/test/pystone.py10
-rwxr-xr-xLib/test/re_tests.py6
-rwxr-xr-xLib/test/regrtest.py6
-rw-r--r--Lib/test/ssl_servers.py8
-rw-r--r--Lib/test/string_tests.py9
-rw-r--r--Lib/test/support/__init__.py67
-rw-r--r--Lib/test/support/script_helper.py (renamed from Lib/test/script_helper.py)34
-rw-r--r--Lib/test/test___future__.py6
-rw-r--r--Lib/test/test__opcode.py7
-rw-r--r--Lib/test/test__osx_support.py6
-rw-r--r--Lib/test/test_argparse.py138
-rw-r--r--Lib/test/test_array.py9
-rw-r--r--Lib/test/test_asdl_parser.py122
-rw-r--r--Lib/test/test_ast.py92
-rw-r--r--Lib/test/test_asynchat.py13
-rw-r--r--Lib/test/test_asyncio/test_pep492.py208
-rw-r--r--Lib/test/test_asyncore.py27
-rw-r--r--Lib/test/test_atexit.py6
-rw-r--r--Lib/test/test_augassign.py21
-rw-r--r--Lib/test/test_base64.py26
-rw-r--r--Lib/test/test_binascii.py9
-rw-r--r--Lib/test/test_binop.py6
-rw-r--r--Lib/test/test_buffer.py102
-rw-r--r--Lib/test/test_builtin.py8
-rw-r--r--Lib/test/test_bytes.py113
-rw-r--r--Lib/test/test_bz2.py133
-rw-r--r--Lib/test/test_calendar.py2
-rw-r--r--Lib/test/test_call.py7
-rw-r--r--Lib/test/test_capi.py129
-rw-r--r--Lib/test/test_cgi.py18
-rw-r--r--Lib/test/test_cgitb.py9
-rw-r--r--Lib/test/test_charmapcodec.py5
-rw-r--r--Lib/test/test_class.py22
-rw-r--r--Lib/test/test_cmath.py47
-rw-r--r--Lib/test/test_cmd_line.py11
-rw-r--r--Lib/test/test_cmd_line_script.py47
-rw-r--r--Lib/test/test_code_module.py36
-rw-r--r--Lib/test/test_codeccallbacks.py122
-rw-r--r--Lib/test/test_codecencodings_cn.py5
-rw-r--r--Lib/test/test_codecencodings_hk.py5
-rw-r--r--Lib/test/test_codecencodings_iso2022.py5
-rw-r--r--Lib/test/test_codecencodings_jp.py5
-rw-r--r--Lib/test/test_codecencodings_kr.py5
-rw-r--r--Lib/test/test_codecencodings_tw.py5
-rw-r--r--Lib/test/test_codecs.py98
-rw-r--r--Lib/test/test_codeop.py8
-rw-r--r--Lib/test/test_collections.py655
-rw-r--r--Lib/test/test_compare.py6
-rw-r--r--Lib/test/test_compile.py18
-rw-r--r--Lib/test/test_compileall.py122
-rw-r--r--Lib/test/test_concurrent_futures.py57
-rw-r--r--Lib/test/test_configparser.py277
-rw-r--r--Lib/test/test_contains.py6
-rw-r--r--Lib/test/test_contextlib.py94
-rw-r--r--Lib/test/test_copy.py82
-rw-r--r--Lib/test/test_copyreg.py7
-rw-r--r--Lib/test/test_coroutines.py1471
-rw-r--r--Lib/test/test_cprofile.py10
-rw-r--r--Lib/test/test_crashers.py7
-rw-r--r--Lib/test/test_csv.py32
-rw-r--r--Lib/test/test_curses.py7
-rw-r--r--Lib/test/test_datetime.py10
-rw-r--r--Lib/test/test_dbm_dumb.py8
-rw-r--r--Lib/test/test_decimal.py157
-rw-r--r--Lib/test/test_decorators.py9
-rw-r--r--Lib/test/test_defaultdict.py11
-rw-r--r--Lib/test/test_deque.py182
-rw-r--r--Lib/test/test_descr.py156
-rw-r--r--Lib/test/test_dict.py9
-rw-r--r--Lib/test/test_dictviews.py8
-rw-r--r--Lib/test/test_difflib.py182
-rw-r--r--Lib/test/test_difflib_expect.html2
-rw-r--r--Lib/test/test_dis.py202
-rw-r--r--Lib/test/test_doctest.py68
-rw-r--r--Lib/test/test_docxmlrpc.py14
-rw-r--r--Lib/test/test_dummy_threading.py6
-rw-r--r--Lib/test/test_dynamic.py8
-rw-r--r--Lib/test/test_dynamicclassattribute.py6
-rw-r--r--Lib/test/test_eintr.py21
-rw-r--r--Lib/test/test_email/test_email.py15
-rw-r--r--Lib/test/test_email/test_generator.py44
-rw-r--r--Lib/test/test_email/test_message.py10
-rw-r--r--Lib/test/test_email/test_policy.py6
-rw-r--r--Lib/test/test_ensurepip.py2
-rw-r--r--Lib/test/test_enum.py135
-rw-r--r--Lib/test/test_enumerate.py13
-rw-r--r--Lib/test/test_eof.py5
-rw-r--r--Lib/test/test_epoll.py7
-rw-r--r--Lib/test/test_errno.py7
-rw-r--r--Lib/test/test_exception_variations.py6
-rw-r--r--Lib/test/test_exceptions.py15
-rw-r--r--Lib/test/test_extcall.py22
-rw-r--r--Lib/test/test_faulthandler.py196
-rw-r--r--Lib/test/test_file_eintr.py46
-rw-r--r--Lib/test/test_fileio.py192
-rw-r--r--Lib/test/test_finalization.py5
-rw-r--r--Lib/test/test_float.py23
-rw-r--r--Lib/test/test_flufl.py7
-rw-r--r--Lib/test/test_fnmatch.py9
-rw-r--r--Lib/test/test_fork1.py12
-rw-r--r--Lib/test/test_format.py417
-rw-r--r--Lib/test/test_fractions.py43
-rw-r--r--Lib/test/test_frame.py5
-rw-r--r--Lib/test/test_ftplib.py29
-rw-r--r--Lib/test/test_funcattrs.py10
-rw-r--r--Lib/test/test_functools.py226
-rw-r--r--Lib/test/test_gc.py25
-rw-r--r--Lib/test/test_gdb.py14
-rw-r--r--Lib/test/test_generators.py138
-rw-r--r--Lib/test/test_genericpath.py38
-rw-r--r--Lib/test/test_getargs2.py14
-rw-r--r--Lib/test/test_getopt.py7
-rw-r--r--Lib/test/test_gettext.py70
-rw-r--r--Lib/test/test_glob.py128
-rw-r--r--Lib/test/test_grammar.py113
-rw-r--r--Lib/test/test_grp.py5
-rw-r--r--Lib/test/test_gzip.py31
-rw-r--r--Lib/test/test_hash.py2
-rw-r--r--Lib/test/test_heapq.py25
-rw-r--r--Lib/test/test_hmac.py11
-rw-r--r--Lib/test/test_html.py1
-rw-r--r--Lib/test/test_htmlparser.py76
-rw-r--r--Lib/test/test_http_cookies.py213
-rw-r--r--Lib/test/test_httplib.py503
-rw-r--r--Lib/test/test_httpservers.py174
-rw-r--r--Lib/test/test_imaplib.py224
-rw-r--r--Lib/test/test_imghdr.py4
-rw-r--r--Lib/test/test_imp.py115
-rw-r--r--Lib/test/test_import/__init__.py (renamed from Lib/test/test_import.py)122
-rw-r--r--Lib/test/test_import/__main__.py3
-rw-r--r--Lib/test/test_import/data/circular_imports/basic.py2
-rw-r--r--Lib/test/test_import/data/circular_imports/basic2.py1
-rw-r--r--Lib/test/test_import/data/circular_imports/indirect.py1
-rw-r--r--Lib/test/test_import/data/circular_imports/rebinding.py3
-rw-r--r--Lib/test/test_import/data/circular_imports/rebinding2.py3
-rw-r--r--Lib/test/test_import/data/circular_imports/subpackage.py2
-rw-r--r--Lib/test/test_import/data/circular_imports/subpkg/subpackage2.py2
-rw-r--r--Lib/test/test_import/data/circular_imports/subpkg/util.py2
-rw-r--r--Lib/test/test_import/data/circular_imports/util.py2
-rw-r--r--Lib/test/test_importlib/builtin/test_finder.py33
-rw-r--r--Lib/test/test_importlib/builtin/test_loader.py38
-rw-r--r--Lib/test/test_importlib/builtin/util.py7
-rw-r--r--Lib/test/test_importlib/extension/test_case_sensitivity.py22
-rw-r--r--Lib/test/test_importlib/extension/test_finder.py15
-rw-r--r--Lib/test/test_importlib/extension/test_loader.py218
-rw-r--r--Lib/test/test_importlib/extension/test_path_hook.py13
-rw-r--r--Lib/test/test_importlib/extension/util.py19
-rw-r--r--Lib/test/test_importlib/frozen/test_finder.py12
-rw-r--r--Lib/test/test_importlib/frozen/test_loader.py17
-rw-r--r--Lib/test/test_importlib/import_/test___loader__.py15
-rw-r--r--Lib/test/test_importlib/import_/test___package__.py19
-rw-r--r--Lib/test/test_importlib/import_/test_api.py17
-rw-r--r--Lib/test/test_importlib/import_/test_caching.py9
-rw-r--r--Lib/test/test_importlib/import_/test_fromlist.py13
-rw-r--r--Lib/test/test_importlib/import_/test_meta_path.py21
-rw-r--r--Lib/test/test_importlib/import_/test_packages.py7
-rw-r--r--Lib/test/test_importlib/import_/test_path.py85
-rw-r--r--Lib/test/test_importlib/import_/test_relative_imports.py7
-rw-r--r--Lib/test/test_importlib/import_/util.py20
-rw-r--r--Lib/test/test_importlib/source/test_case_sensitivity.py23
-rw-r--r--Lib/test/test_importlib/source/test_file_loader.py111
-rw-r--r--Lib/test/test_importlib/source/test_finder.py28
-rw-r--r--Lib/test/test_importlib/source/test_path_hook.py8
-rw-r--r--Lib/test/test_importlib/source/test_source_encoding.py33
-rw-r--r--Lib/test/test_importlib/source/util.py96
-rw-r--r--Lib/test/test_importlib/test_abc.py292
-rw-r--r--Lib/test/test_importlib/test_api.py99
-rw-r--r--Lib/test/test_importlib/test_lazy.py132
-rw-r--r--Lib/test/test_importlib/test_locks.py178
-rw-r--r--Lib/test/test_importlib/test_spec.py256
-rw-r--r--Lib/test/test_importlib/test_util.py338
-rw-r--r--Lib/test/test_importlib/test_windows.py100
-rw-r--r--Lib/test/test_importlib/util.py187
-rw-r--r--Lib/test/test_inspect.py513
-rw-r--r--Lib/test/test_int.py5
-rw-r--r--Lib/test/test_int_literal.py6
-rw-r--r--Lib/test/test_io.py154
-rw-r--r--Lib/test/test_ioctl.py2
-rw-r--r--Lib/test/test_ipaddress.py172
-rw-r--r--Lib/test/test_isinstance.py21
-rw-r--r--Lib/test/test_itertools.py2
-rw-r--r--Lib/test/test_json/__init__.py4
-rw-r--r--Lib/test/test_json/test_decode.py8
-rw-r--r--Lib/test/test_json/test_encode_basestring_ascii.py3
-rw-r--r--Lib/test/test_json/test_fail.py59
-rw-r--r--Lib/test/test_json/test_recursion.py12
-rw-r--r--Lib/test/test_json/test_scanstring.py2
-rw-r--r--Lib/test/test_json/test_tool.py43
-rw-r--r--Lib/test/test_keywordonlyarg.py6
-rw-r--r--Lib/test/test_kqueue.py8
-rw-r--r--Lib/test/test_linecache.py43
-rw-r--r--Lib/test/test_list.py17
-rw-r--r--Lib/test/test_locale.py54
-rw-r--r--Lib/test/test_logging.py212
-rw-r--r--Lib/test/test_long.py7
-rw-r--r--Lib/test/test_longexp.py8
-rw-r--r--Lib/test/test_lzma.py116
-rw-r--r--Lib/test/test_macpath.py2
-rw-r--r--Lib/test/test_mailcap.py6
-rw-r--r--Lib/test/test_marshal.py2
-rw-r--r--Lib/test/test_math.py195
-rw-r--r--Lib/test/test_memoryio.py71
-rw-r--r--Lib/test/test_memoryview.py28
-rw-r--r--Lib/test/test_mimetypes.py8
-rw-r--r--Lib/test/test_minidom.py22
-rw-r--r--Lib/test/test_mmap.py8
-rw-r--r--Lib/test/test_module.py34
-rw-r--r--Lib/test/test_msilib.py7
-rw-r--r--Lib/test/test_multiprocessing_main_handling.py31
-rw-r--r--Lib/test/test_nis.py5
-rw-r--r--Lib/test/test_normalization.py7
-rw-r--r--Lib/test/test_ntpath.py69
-rw-r--r--Lib/test/test_numeric_tower.py6
-rw-r--r--Lib/test/test_opcodes.py6
-rw-r--r--Lib/test/test_openpty.py6
-rw-r--r--Lib/test/test_operator.py116
-rw-r--r--Lib/test/test_os.py490
-rw-r--r--Lib/test/test_osx_env.py2
-rw-r--r--Lib/test/test_parser.py45
-rw-r--r--Lib/test/test_pathlib.py216
-rw-r--r--Lib/test/test_peepholer.py18
-rw-r--r--Lib/test/test_pep247.py6
-rw-r--r--Lib/test/test_pep292.py8
-rw-r--r--Lib/test/test_pep3120.py8
-rw-r--r--Lib/test/test_pep3131.py8
-rw-r--r--Lib/test/test_pep3151.py8
-rw-r--r--Lib/test/test_pep380.py8
-rw-r--r--Lib/test/test_pep479.py34
-rw-r--r--Lib/test/test_pickle.py5
-rw-r--r--Lib/test/test_pkg.py6
-rw-r--r--Lib/test/test_pkgimport.py8
-rw-r--r--Lib/test/test_pkgutil.py3
-rw-r--r--Lib/test/test_platform.py46
-rw-r--r--Lib/test/test_popen.py5
-rw-r--r--Lib/test/test_poplib.py33
-rw-r--r--Lib/test/test_posix.py13
-rw-r--r--Lib/test/test_posixpath.py66
-rw-r--r--Lib/test/test_pow.py5
-rw-r--r--Lib/test/test_pprint.py456
-rw-r--r--Lib/test/test_property.py28
-rw-r--r--Lib/test/test_pstats.py9
-rw-r--r--Lib/test/test_pty.py34
-rw-r--r--Lib/test/test_pulldom.py8
-rw-r--r--Lib/test/test_pwd.py5
-rw-r--r--Lib/test/test_py_compile.py5
-rw-r--r--Lib/test/test_pyclbr.py7
-rw-r--r--Lib/test/test_pydoc.py47
-rw-r--r--Lib/test/test_pyexpat.py18
-rw-r--r--Lib/test/test_queue.py7
-rw-r--r--Lib/test/test_quopri.py7
-rw-r--r--Lib/test/test_raise.py3
-rw-r--r--Lib/test/test_range.py5
-rw-r--r--Lib/test/test_re.py566
-rw-r--r--Lib/test/test_readline.py49
-rw-r--r--Lib/test/test_reprlib.py48
-rw-r--r--Lib/test/test_richcmp.py29
-rw-r--r--Lib/test/test_rlcompleter.py1
-rw-r--r--Lib/test/test_runpy.py12
-rw-r--r--Lib/test/test_sax.py40
-rw-r--r--Lib/test/test_scope.py7
-rw-r--r--[-rwxr-xr-x]Lib/test/test_script_helper.py28
-rw-r--r--Lib/test/test_select.py5
-rw-r--r--Lib/test/test_selectors.py62
-rw-r--r--Lib/test/test_set.py15
-rw-r--r--Lib/test/test_shlex.py6
-rw-r--r--Lib/test/test_shutil.py26
-rw-r--r--Lib/test/test_signal.py342
-rw-r--r--Lib/test/test_site.py18
-rw-r--r--Lib/test/test_slice.py6
-rw-r--r--Lib/test/test_smtpd.py486
-rw-r--r--Lib/test/test_smtplib.py343
-rw-r--r--Lib/test/test_smtpnet.py5
-rw-r--r--Lib/test/test_sndhdr.py14
-rw-r--r--Lib/test/test_socket.py436
-rw-r--r--Lib/test/test_socketserver.py32
-rw-r--r--Lib/test/test_sort.py21
-rw-r--r--Lib/test/test_ssl.py501
-rw-r--r--Lib/test/test_startfile.py7
-rw-r--r--Lib/test/test_stat.py29
-rw-r--r--Lib/test/test_string.py33
-rw-r--r--Lib/test/test_stringprep.py6
-rw-r--r--Lib/test/test_strlit.py6
-rw-r--r--Lib/test/test_strptime.py15
-rw-r--r--Lib/test/test_strtod.py5
-rw-r--r--Lib/test/test_struct.py5
-rw-r--r--Lib/test/test_structmembers.py5
-rw-r--r--Lib/test/test_structseq.py6
-rw-r--r--Lib/test/test_subprocess.py132
-rw-r--r--Lib/test/test_sundry.py2
-rw-r--r--Lib/test/test_super.py5
-rw-r--r--Lib/test/test_support.py34
-rw-r--r--Lib/test/test_symtable.py6
-rw-r--r--Lib/test/test_syntax.py16
-rw-r--r--Lib/test/test_sys.py88
-rw-r--r--Lib/test/test_sys_setprofile.py12
-rw-r--r--Lib/test/test_sysconfig.py19
-rw-r--r--Lib/test/test_syslog.py5
-rw-r--r--Lib/test/test_tarfile.py244
-rw-r--r--Lib/test/test_tcl.py26
-rw-r--r--Lib/test/test_telnetlib.py8
-rw-r--r--Lib/test/test_tempfile.py177
-rw-r--r--Lib/test/test_textwrap.py16
-rw-r--r--Lib/test/test_thread.py6
-rw-r--r--Lib/test/test_threaded_import.py10
-rw-r--r--Lib/test/test_threading.py4
-rw-r--r--Lib/test/test_time.py432
-rw-r--r--Lib/test/test_timeit.py52
-rw-r--r--Lib/test/test_timeout.py8
-rw-r--r--Lib/test/test_tix.py32
-rw-r--r--Lib/test/test_tk.py3
-rw-r--r--Lib/test/test_tokenize.py286
-rw-r--r--Lib/test/test_tools/test_i18n.py68
-rw-r--r--Lib/test/test_tools/test_md5sum.py2
-rw-r--r--Lib/test/test_tools/test_pindent.py2
-rw-r--r--Lib/test/test_tools/test_reindent.py2
-rw-r--r--Lib/test/test_trace.py10
-rw-r--r--Lib/test/test_traceback.py443
-rw-r--r--Lib/test/test_tracemalloc.py18
-rw-r--r--Lib/test/test_ttk_guionly.py4
-rw-r--r--Lib/test/test_ttk_textonly.py3
-rw-r--r--Lib/test/test_tuple.py18
-rw-r--r--Lib/test/test_typechecks.py5
-rw-r--r--Lib/test/test_types.py327
-rw-r--r--Lib/test/test_typing.py1257
-rw-r--r--Lib/test/test_ucn.py5
-rw-r--r--Lib/test/test_unary.py7
-rw-r--r--Lib/test/test_unicode.py54
-rw-r--r--Lib/test/test_unicodedata.py16
-rw-r--r--Lib/test/test_unpack.py2
-rw-r--r--Lib/test/test_unpack_ex.py195
-rw-r--r--Lib/test/test_urllib.py8
-rw-r--r--Lib/test/test_urllib2.py234
-rw-r--r--Lib/test/test_urllib2net.py2
-rw-r--r--Lib/test/test_urlparse.py81
-rw-r--r--Lib/test/test_userdict.py7
-rw-r--r--Lib/test/test_userlist.py5
-rw-r--r--Lib/test/test_uuid.py43
-rw-r--r--Lib/test/test_venv.py7
-rw-r--r--Lib/test/test_wait3.py10
-rw-r--r--Lib/test/test_wait4.py12
-rw-r--r--Lib/test/test_warnings.py54
-rw-r--r--Lib/test/test_weakref.py23
-rw-r--r--Lib/test/test_weakset.py6
-rw-r--r--Lib/test/test_winsound.py7
-rw-r--r--Lib/test/test_with.py18
-rw-r--r--Lib/test/test_wsgiref.py13
-rw-r--r--Lib/test/test_xdrlib.py7
-rw-r--r--Lib/test/test_xml_etree.py2
-rw-r--r--Lib/test/test_xml_etree_c.py2
-rw-r--r--Lib/test/test_xmlrpc.py20
-rw-r--r--Lib/test/test_zipapp.py349
-rw-r--r--Lib/test/test_zipfile.py237
-rw-r--r--Lib/test/test_zipimport.py6
-rw-r--r--Lib/test/test_zipimport_support.py19
-rw-r--r--Lib/test/test_zlib.py13
-rw-r--r--Lib/test/tf_inherit_check.py32
-rw-r--r--Lib/textwrap.py23
-rw-r--r--Lib/threading.py15
-rwxr-xr-xLib/timeit.py113
-rw-r--r--Lib/tkinter/__init__.py235
-rw-r--r--Lib/tkinter/_fix.py78
-rw-r--r--Lib/tkinter/font.py8
-rw-r--r--Lib/tkinter/simpledialog.py4
-rw-r--r--Lib/tkinter/test/runtktests.py2
-rw-r--r--Lib/tkinter/test/test_tkinter/test_misc.py5
-rw-r--r--Lib/tkinter/test/test_tkinter/test_variables.py6
-rw-r--r--Lib/tkinter/test/test_tkinter/test_widgets.py7
-rw-r--r--Lib/tkinter/test/test_ttk/test_extensions.py6
-rw-r--r--Lib/token.py25
-rw-r--r--Lib/tokenize.py78
-rwxr-xr-xLib/trace.py66
-rw-r--r--Lib/traceback.py527
-rw-r--r--Lib/tracemalloc.py2
-rw-r--r--[-rwxr-xr-x]Lib/turtledemo/__main__.py0
-rw-r--r--Lib/turtledemo/sorting_animate.py204
-rw-r--r--Lib/types.py102
-rw-r--r--Lib/typing.py1648
-rw-r--r--Lib/unittest/__init__.py9
-rw-r--r--Lib/unittest/case.py117
-rw-r--r--Lib/unittest/loader.py256
-rw-r--r--Lib/unittest/main.py25
-rw-r--r--Lib/unittest/mock.py71
-rw-r--r--Lib/unittest/result.py7
-rw-r--r--Lib/unittest/runner.py10
-rw-r--r--Lib/unittest/test/support.py4
-rw-r--r--Lib/unittest/test/test_assertions.py15
-rw-r--r--Lib/unittest/test/test_break.py3
-rw-r--r--Lib/unittest/test/test_case.py113
-rw-r--r--Lib/unittest/test/test_discovery.py320
-rw-r--r--Lib/unittest/test/test_loader.py423
-rw-r--r--Lib/unittest/test/test_program.py26
-rw-r--r--Lib/unittest/test/test_result.py43
-rw-r--r--Lib/unittest/test/test_runner.py10
-rw-r--r--Lib/unittest/test/test_setups.py7
-rw-r--r--Lib/unittest/test/testmock/testmagicmethods.py11
-rw-r--r--Lib/unittest/test/testmock/testmock.py45
-rw-r--r--Lib/unittest/test/testmock/testpatch.py24
-rw-r--r--Lib/unittest/util.py2
-rw-r--r--Lib/urllib/error.py7
-rw-r--r--Lib/urllib/parse.py189
-rw-r--r--Lib/urllib/request.py63
-rw-r--r--Lib/urllib/robotparser.py2
-rw-r--r--Lib/uuid.py110
-rw-r--r--Lib/warnings.py6
-rw-r--r--Lib/weakref.py4
-rwxr-xr-xLib/webbrowser.py120
-rw-r--r--Lib/wsgiref/headers.py6
-rw-r--r--Lib/xml/dom/minidom.py8
-rw-r--r--Lib/xml/dom/xmlbuilder.py26
-rw-r--r--Lib/xml/etree/ElementPath.py20
-rw-r--r--Lib/xml/etree/ElementTree.py8
-rw-r--r--Lib/xml/sax/__init__.py8
-rw-r--r--Lib/xml/sax/expatreader.py11
-rw-r--r--Lib/xml/sax/saxutils.py7
-rw-r--r--Lib/xml/sax/xmlreader.py4
-rw-r--r--Lib/xmlrpc/client.py60
-rw-r--r--Lib/zipapp.py200
-rw-r--r--Lib/zipfile.py651
-rw-r--r--Mac/BuildScript/README.txt71
-rwxr-xr-xMac/BuildScript/build-installer.py4
-rw-r--r--Mac/BuildScript/resources/ReadMe.rtf90
-rw-r--r--Mac/BuildScript/resources/Welcome.rtf8
-rwxr-xr-xMac/BuildScript/scripts/postflight.ensurepip10
-rwxr-xr-xMac/BuildScript/scripts/postflight.framework16
-rw-r--r--Mac/Makefile.in6
-rw-r--r--Mac/PythonLauncher/Info.plist.in4
-rw-r--r--Mac/PythonLauncher/Makefile.in29
-rw-r--r--Mac/README14
-rwxr-xr-xMac/Tools/bundlebuilder.py934
-rw-r--r--Makefile.pre.in91
-rw-r--r--Misc/ACKS50
-rw-r--r--Misc/HISTORY1115
-rw-r--r--Misc/NEWS6738
-rw-r--r--Misc/Porting42
-rw-r--r--Misc/coverity_model.c60
-rw-r--r--Misc/gdbinit4
-rw-r--r--Misc/python.man25
-rw-r--r--Modules/README2
-rw-r--r--Modules/Setup.config.in2
-rw-r--r--Modules/Setup.dist2
-rw-r--r--Modules/_bz2module.c263
-rw-r--r--Modules/_codecsmodule.c1270
-rw-r--r--Modules/_collectionsmodule.c643
-rw-r--r--Modules/_cryptmodule.c41
-rw-r--r--Modules/_csv.c85
-rw-r--r--Modules/_ctypes/_ctypes.c7
-rw-r--r--Modules/_ctypes/callproc.c2
-rw-r--r--Modules/_ctypes/cfield.c24
-rw-r--r--Modules/_ctypes/libffi_msvc/ffi.c37
-rw-r--r--Modules/_cursesmodule.c85
-rw-r--r--Modules/_datetimemodule.c102
-rw-r--r--Modules/_dbmmodule.c203
-rw-r--r--Modules/_decimal/_decimal.c4
-rw-r--r--Modules/_decimal/docstrings.h860
-rw-r--r--Modules/_decimal/libmpdec/mpdecimal.c1
-rw-r--r--Modules/_decimal/tests/deccheck.py7
-rw-r--r--Modules/_elementtree.c1030
-rw-r--r--Modules/_functoolsmodule.c626
-rw-r--r--Modules/_gdbmmodule.c306
-rw-r--r--Modules/_hashopenssl.c34
-rw-r--r--Modules/_heapqmodule.c432
-rw-r--r--Modules/_io/_iomodule.c306
-rw-r--r--Modules/_io/_iomodule.h2
-rw-r--r--Modules/_io/bufferedio.c1023
-rw-r--r--Modules/_io/bytesio.c661
-rw-r--r--Modules/_io/clinic/_iomodule.c.h159
-rw-r--r--Modules/_io/clinic/bufferedio.c.h454
-rw-r--r--Modules/_io/clinic/bytesio.c.h422
-rw-r--r--Modules/_io/clinic/fileio.c.h367
-rw-r--r--Modules/_io/clinic/iobase.c.h279
-rw-r--r--Modules/_io/clinic/stringio.c.h286
-rw-r--r--Modules/_io/clinic/textio.c.h456
-rw-r--r--Modules/_io/fileio.c650
-rw-r--r--Modules/_io/iobase.c264
-rw-r--r--Modules/_io/stringio.c269
-rw-r--r--Modules/_io/textio.c539
-rw-r--r--Modules/_json.c170
-rw-r--r--Modules/_lsprof.c11
-rw-r--r--Modules/_lzmamodule.c226
-rw-r--r--Modules/_opcode.c37
-rw-r--r--Modules/_operator.c285
-rw-r--r--Modules/_pickle.c298
-rw-r--r--Modules/_posixsubprocess.c23
-rw-r--r--Modules/_randommodule.c138
-rw-r--r--Modules/_scproxy.c2
-rw-r--r--Modules/_sqlite/connection.c3
-rw-r--r--Modules/_sqlite/connection.h2
-rw-r--r--Modules/_sqlite/cursor.c6
-rw-r--r--Modules/_sqlite/microprotocols.h2
-rw-r--r--Modules/_sqlite/row.c5
-rw-r--r--Modules/_sre.c1053
-rw-r--r--Modules/_ssl.c1794
-rw-r--r--Modules/_stat.c36
-rw-r--r--Modules/_struct.c14
-rw-r--r--Modules/_testbuffer.c61
-rw-r--r--Modules/_testcapimodule.c552
-rw-r--r--Modules/_testmultiphase.c584
-rw-r--r--Modules/_threadmodule.c155
-rw-r--r--Modules/_tkinter.c1085
-rw-r--r--Modules/_tracemalloc.c86
-rw-r--r--Modules/_weakref.c31
-rw-r--r--Modules/_winapi.c991
-rw-r--r--Modules/arraymodule.c768
-rw-r--r--Modules/atexitmodule.c10
-rw-r--r--Modules/audioop.c69
-rw-r--r--Modules/binascii.c39
-rw-r--r--Modules/cjkcodecs/_codecs_iso2022.c7
-rw-r--r--Modules/cjkcodecs/cjkcodecs.h4
-rw-r--r--Modules/cjkcodecs/clinic/multibytecodec.c.h320
-rw-r--r--Modules/cjkcodecs/multibytecodec.c345
-rw-r--r--Modules/clinic/_bz2module.c.h48
-rw-r--r--Modules/clinic/_codecsmodule.c.h1396
-rw-r--r--Modules/clinic/_cryptmodule.c.h37
-rw-r--r--Modules/clinic/_cursesmodule.c.h71
-rw-r--r--Modules/clinic/_datetimemodule.c.h37
-rw-r--r--Modules/clinic/_dbmmodule.c.h141
-rw-r--r--Modules/clinic/_elementtree.c.h679
-rw-r--r--Modules/clinic/_gdbmmodule.c.h253
-rw-r--r--Modules/clinic/_lzmamodule.c.h73
-rw-r--r--Modules/clinic/_opcode.c.h36
-rw-r--r--Modules/clinic/_pickle.c.h43
-rw-r--r--Modules/clinic/_sre.c.h693
-rw-r--r--Modules/clinic/_ssl.c.h1105
-rw-r--r--Modules/clinic/_tkinter.c.h624
-rw-r--r--Modules/clinic/_weakref.c.h31
-rw-r--r--Modules/clinic/_winapi.c.h854
-rw-r--r--Modules/clinic/arraymodule.c.h499
-rw-r--r--Modules/clinic/audioop.c.h120
-rw-r--r--Modules/clinic/binascii.c.h117
-rw-r--r--Modules/clinic/cmathmodule.c.h860
-rw-r--r--Modules/clinic/fcntlmodule.c.h185
-rw-r--r--Modules/clinic/grpmodule.c.h85
-rw-r--r--Modules/clinic/md5module.c.h95
-rw-r--r--Modules/clinic/posixmodule.c.h5791
-rw-r--r--Modules/clinic/pwdmodule.c.h71
-rw-r--r--Modules/clinic/pyexpat.c.h284
-rw-r--r--Modules/clinic/sha1module.c.h95
-rw-r--r--Modules/clinic/sha256module.c.h123
-rw-r--r--Modules/clinic/sha512module.c.h171
-rw-r--r--Modules/clinic/signalmodule.c.h432
-rw-r--r--Modules/clinic/spwdmodule.c.h68
-rw-r--r--Modules/clinic/unicodedata.c.h368
-rw-r--r--Modules/clinic/zlibmodule.c.h58
-rw-r--r--Modules/cmathmodule.c558
-rw-r--r--Modules/config.c.in2
-rw-r--r--Modules/faulthandler.c216
-rw-r--r--Modules/fcntlmodule.c443
-rw-r--r--Modules/gcmodule.c82
-rw-r--r--Modules/getpath.c38
-rw-r--r--Modules/grpmodule.c78
-rw-r--r--Modules/itertoolsmodule.c2
-rw-r--r--Modules/ld_so_aix.in6
-rw-r--r--Modules/main.c39
-rwxr-xr-xModules/makesetup2
-rw-r--r--Modules/mathmodule.c160
-rw-r--r--Modules/md5module.c123
-rw-r--r--Modules/mmapmodule.c62
-rw-r--r--Modules/nismodule.c4
-rw-r--r--Modules/ossaudiodev.c92
-rw-r--r--Modules/parsermodule.c375
-rw-r--r--Modules/posixmodule.c8211
-rw-r--r--Modules/pwdmodule.c71
-rw-r--r--Modules/pyexpat.c383
-rw-r--r--Modules/readline.c41
-rw-r--r--Modules/selectmodule.c465
-rw-r--r--Modules/sha1module.c114
-rw-r--r--Modules/sha256module.c146
-rw-r--r--Modules/sha512module.c156
-rw-r--r--Modules/signalmodule.c744
-rw-r--r--Modules/socketmodule.c1319
-rw-r--r--Modules/socketmodule.h9
-rw-r--r--Modules/spwdmodule.c48
-rw-r--r--Modules/sre.h9
-rw-r--r--Modules/sre_constants.h3
-rw-r--r--Modules/sre_lib.h28
-rw-r--r--Modules/symtablemodule.c3
-rw-r--r--Modules/timemodule.c381
-rw-r--r--Modules/tkappinit.c19
-rw-r--r--Modules/tkinter.h8
-rw-r--r--Modules/unicodedata.c463
-rw-r--r--Modules/unicodedata_db.h7130
-rw-r--r--Modules/unicodename_db.h42270
-rw-r--r--Modules/winreparse.h53
-rw-r--r--Modules/xxlimited.c83
-rw-r--r--Modules/xxmodule.c54
-rw-r--r--Modules/xxsubtype.c61
-rw-r--r--Modules/zipimport.c32
-rw-r--r--Modules/zlibmodule.c18
-rw-r--r--Objects/README1
-rw-r--r--Objects/abstract.c188
-rw-r--r--Objects/bytearrayobject.c941
-rw-r--r--Objects/bytes_methods.c27
-rw-r--r--Objects/bytesobject.c1322
-rw-r--r--Objects/classobject.c43
-rw-r--r--Objects/clinic/bytearrayobject.c.h698
-rw-r--r--Objects/clinic/bytesobject.c.h487
-rw-r--r--Objects/clinic/dictobject.c.h42
-rw-r--r--Objects/clinic/unicodeobject.c.h41
-rw-r--r--Objects/complexobject.c34
-rw-r--r--Objects/descrobject.c39
-rw-r--r--Objects/dict-common.h22
-rw-r--r--Objects/dictobject.c443
-rw-r--r--Objects/exceptions.c30
-rw-r--r--Objects/fileobject.c26
-rw-r--r--Objects/floatobject.c9
-rw-r--r--Objects/frameobject.c8
-rw-r--r--Objects/genobject.c483
-rw-r--r--Objects/iterobject.c2
-rw-r--r--Objects/listobject.c4
-rw-r--r--Objects/longobject.c290
-rw-r--r--Objects/memoryobject.c199
-rw-r--r--Objects/methodobject.c107
-rw-r--r--Objects/moduleobject.c385
-rw-r--r--Objects/object.c52
-rw-r--r--Objects/obmalloc.c142
-rw-r--r--Objects/odictobject.c2493
-rw-r--r--Objects/rangeobject.c4
-rw-r--r--Objects/setobject.c445
-rw-r--r--Objects/stringlib/codecs.h87
-rw-r--r--Objects/stringlib/fastsearch.h4
-rw-r--r--Objects/stringlib/find.h23
-rw-r--r--Objects/stringlib/transmogrify.h2
-rw-r--r--Objects/tupleobject.c4
-rw-r--r--Objects/typeobject.c489
-rw-r--r--Objects/typeslots.inc6
-rwxr-xr-xObjects/typeslots.py2
-rw-r--r--Objects/unicodectype.c2
-rw-r--r--Objects/unicodeobject.c1129
-rw-r--r--Objects/unicodetype_db.h3856
-rw-r--r--PC/VS9.0/_bz2.vcproj581
-rw-r--r--PC/VS9.0/_ctypes.vcproj705
-rw-r--r--PC/VS9.0/_ctypes_test.vcproj521
-rw-r--r--PC/VS9.0/_decimal.vcproj743
-rw-r--r--PC/VS9.0/_elementtree.vcproj613
-rw-r--r--PC/VS9.0/_hashlib.vcproj537
-rw-r--r--PC/VS9.0/_lzma.vcproj537
-rw-r--r--PC/VS9.0/_msi.vcproj529
-rw-r--r--PC/VS9.0/_multiprocessing.vcproj541
-rw-r--r--PC/VS9.0/_socket.vcproj537
-rw-r--r--PC/VS9.0/_sqlite3.vcproj609
-rw-r--r--PC/VS9.0/_ssl.vcproj537
-rw-r--r--PC/VS9.0/_testbuffer.vcproj521
-rw-r--r--PC/VS9.0/_testcapi.vcproj521
-rw-r--r--PC/VS9.0/_testimportmultiple.vcproj521
-rw-r--r--PC/VS9.0/_tkinter.vcproj541
-rw-r--r--PC/VS9.0/bdist_wininst.vcproj270
-rw-r--r--PC/VS9.0/debug.vsprops15
-rw-r--r--PC/VS9.0/kill_python.c178
-rw-r--r--PC/VS9.0/kill_python.vcproj279
-rw-r--r--PC/VS9.0/make_buildinfo.c195
-rw-r--r--PC/VS9.0/make_buildinfo.vcproj101
-rw-r--r--PC/VS9.0/make_versioninfo.vcproj324
-rw-r--r--PC/VS9.0/pcbuild.sln690
-rw-r--r--PC/VS9.0/pginstrument.vsprops34
-rw-r--r--PC/VS9.0/pgupdate.vsprops14
-rw-r--r--PC/VS9.0/pyd.vsprops28
-rw-r--r--PC/VS9.0/pyd_d.vsprops36
-rw-r--r--PC/VS9.0/pyexpat.vcproj553
-rw-r--r--PC/VS9.0/pyproject.vsprops91
-rw-r--r--PC/VS9.0/python.vcproj637
-rw-r--r--PC/VS9.0/python3dll.vcproj246
-rw-r--r--PC/VS9.0/pythoncore.vcproj1877
-rw-r--r--PC/VS9.0/pythonw.vcproj618
-rw-r--r--PC/VS9.0/release.vsprops15
-rw-r--r--PC/VS9.0/select.vcproj537
-rw-r--r--PC/VS9.0/sqlite3.vcproj537
-rw-r--r--PC/VS9.0/sqlite3.vsprops14
-rw-r--r--PC/VS9.0/ssl.vcproj189
-rw-r--r--PC/VS9.0/unicodedata.vcproj533
-rw-r--r--PC/VS9.0/winsound.vcproj523
-rw-r--r--PC/VS9.0/x64.vsprops22
-rw-r--r--PC/VS9.0/xxlimited.vcproj417
-rw-r--r--PC/bdist_wininst/archive.h9
-rw-r--r--PC/bdist_wininst/bdist_wininst.vcxproj103
-rw-r--r--PC/bdist_wininst/bdist_wininst.vcxproj.filters (renamed from PCbuild/bdist_wininst.vcxproj.filters)0
-rw-r--r--PC/bdist_wininst/build.bat22
-rw-r--r--PC/bdist_wininst/extract.c9
-rw-r--r--PC/bdist_wininst/install.c38
-rw-r--r--PC/bdist_wininst/install.rc162
-rw-r--r--PC/bdist_wininst/resource.h24
-rw-r--r--PC/bdist_wininst/wininst-7.1.sln21
-rw-r--r--PC/bdist_wininst/wininst-7.1.vcproj214
-rw-r--r--PC/bdist_wininst/wininst-8.sln19
-rw-r--r--PC/bdist_wininst/wininst-8.vcproj320
-rw-r--r--PC/bdist_wininst/wininst.dsp123
-rw-r--r--PC/bdist_wininst/wininst.dsw29
-rw-r--r--PC/clinic/msvcrtmodule.c.h554
-rw-r--r--PC/clinic/winreg.c.h1059
-rw-r--r--PC/clinic/winsound.c.h100
-rw-r--r--PC/config.c6
-rw-r--r--PC/dl_nt.c8
-rw-r--r--PC/example_nt/example.vcproj4
-rw-r--r--PC/example_nt/readme.txt20
-rw-r--r--PC/getpathp.c171
-rw-r--r--PC/invalid_parameter_handler.c22
-rw-r--r--PC/launcher.c166
-rw-r--r--PC/msvcrtmodule.c604
-rw-r--r--PC/pyconfig.h58
-rw-r--r--PC/pylauncher.rc2
-rw-r--r--PC/python.manifest12
-rw-r--r--PC/python3.def1410
-rw-r--r--PC/python3.mak14
-rw-r--r--PC/python34gen.py26
-rw-r--r--PC/python34stub.def700
-rw-r--r--PC/python_exe.rc50
-rw-r--r--PC/python_nt.rc42
-rw-r--r--PC/python_ver_rc.h35
-rw-r--r--PC/readme.txt5
-rw-r--r--PC/winreg.c1401
-rw-r--r--PC/winsound.c129
-rw-r--r--PCbuild/_bz2.vcxproj184
-rw-r--r--PCbuild/_ctypes.vcxproj211
-rw-r--r--PCbuild/_ctypes_test.vcxproj131
-rw-r--r--PCbuild/_decimal.vcxproj212
-rw-r--r--PCbuild/_elementtree.vcxproj183
-rw-r--r--PCbuild/_freeze_importlib.vcxproj174
-rw-r--r--PCbuild/_freeze_importlib.vcxproj.filters4
-rw-r--r--PCbuild/_hashlib.vcxproj222
-rw-r--r--PCbuild/_lzma.vcxproj189
-rw-r--r--PCbuild/_msi.vcxproj164
-rw-r--r--PCbuild/_multiprocessing.vcxproj162
-rw-r--r--PCbuild/_overlapped.vcxproj170
-rw-r--r--PCbuild/_socket.vcxproj162
-rw-r--r--PCbuild/_sqlite3.vcxproj183
-rw-r--r--PCbuild/_ssl.vcxproj222
-rw-r--r--PCbuild/_testbuffer.vcxproj156
-rw-r--r--PCbuild/_testcapi.vcxproj156
-rw-r--r--PCbuild/_testembed.vcxproj132
-rw-r--r--PCbuild/_testembed.vcxproj.filters2
-rw-r--r--PCbuild/_testimportmultiple.vcxproj156
-rw-r--r--PCbuild/_testmultiphase.vcxproj80
-rw-r--r--PCbuild/_testmultiphase.vcxproj.filters22
-rw-r--r--PCbuild/_tkinter.vcxproj192
-rw-r--r--PCbuild/bdist_wininst.vcxproj158
-rw-r--r--PCbuild/build.bat163
-rw-r--r--PCbuild/build_pgo.bat37
-rw-r--r--PCbuild/build_ssl.py269
-rw-r--r--PCbuild/build_tkinter.py78
-rw-r--r--PCbuild/clean.bat5
-rw-r--r--PCbuild/debug.props27
-rw-r--r--PCbuild/env.bat16
-rw-r--r--PCbuild/get_externals.bat21
-rw-r--r--PCbuild/idle.bat6
-rw-r--r--PCbuild/installer.bmpbin58806 -> 0 bytes-rw-r--r--PCbuild/kill_python.c178
-rw-r--r--PCbuild/kill_python.vcxproj120
-rw-r--r--PCbuild/kill_python.vcxproj.filters13
-rw-r--r--PCbuild/libeay.vcxproj907
-rw-r--r--PCbuild/make_buildinfo.c194
-rw-r--r--PCbuild/make_buildinfo.vcxproj52
-rw-r--r--PCbuild/make_buildinfo.vcxproj.filters14
-rw-r--r--PCbuild/make_versioninfo.vcxproj200
-rw-r--r--PCbuild/make_versioninfo.vcxproj.filters13
-rw-r--r--PCbuild/openssl.props77
-rw-r--r--PCbuild/pcbuild.proj90
-rw-r--r--PCbuild/pcbuild.sln317
-rw-r--r--PCbuild/pginstrument.props38
-rw-r--r--PCbuild/pgupdate.props17
-rw-r--r--PCbuild/prepare_ssl.bat (renamed from PCbuild/build_ssl.bat)8
-rw-r--r--PCbuild/prepare_ssl.py248
-rw-r--r--PCbuild/pyd.props25
-rw-r--r--PCbuild/pyd_d.props31
-rw-r--r--PCbuild/pyexpat.vcxproj173
-rw-r--r--PCbuild/pylauncher.vcxproj250
-rw-r--r--PCbuild/pyproject.props212
-rw-r--r--PCbuild/python.props152
-rw-r--r--PCbuild/python.vcxproj306
-rw-r--r--PCbuild/python.vcxproj.filters4
-rw-r--r--PCbuild/python3dll.vcxproj206
-rw-r--r--PCbuild/pythoncore.vcxproj437
-rw-r--r--PCbuild/pythoncore.vcxproj.filters15
-rw-r--r--PCbuild/pythonw.vcxproj278
-rw-r--r--PCbuild/pywlauncher.vcxproj190
-rw-r--r--PCbuild/readme.txt234
-rw-r--r--PCbuild/release.props19
-rw-r--r--PCbuild/rt.bat14
-rw-r--r--PCbuild/select.vcxproj171
-rw-r--r--PCbuild/sqlite3.props16
-rw-r--r--PCbuild/sqlite3.vcxproj181
-rw-r--r--PCbuild/ssl.vcxproj221
-rw-r--r--PCbuild/ssleay.vcxproj119
-rw-r--r--PCbuild/tcl.vcxproj91
-rw-r--r--PCbuild/tcltk.props45
-rw-r--r--PCbuild/tix.vcxproj92
-rw-r--r--PCbuild/tk.vcxproj95
-rw-r--r--PCbuild/unicodedata.vcxproj155
-rw-r--r--PCbuild/vs9to10.py56
-rw-r--r--PCbuild/vs9to8.py34
-rw-r--r--PCbuild/winsound.vcxproj155
-rw-r--r--PCbuild/x64.props20
-rw-r--r--PCbuild/xxlimited.vcxproj152
-rw-r--r--Parser/Python.asdl33
-rw-r--r--Parser/asdl.py587
-rwxr-xr-xParser/asdl_c.py54
-rw-r--r--Parser/node.c4
-rw-r--r--Parser/pgen.c5
-rw-r--r--Parser/pgenmain.c7
-rw-r--r--Parser/printgrammar.c5
-rw-r--r--Parser/spark.py849
-rw-r--r--Parser/tokenizer.c72
-rw-r--r--Parser/tokenizer.h7
-rw-r--r--Programs/README1
-rw-r--r--Programs/_freeze_importlib.c (renamed from Modules/_freeze_importlib.c)30
-rw-r--r--Programs/_testembed.c (renamed from Modules/_testembed.c)4
-rw-r--r--Programs/python.c (renamed from Modules/python.c)2
-rw-r--r--Python/Python-ast.c602
-rw-r--r--Python/README1
-rw-r--r--Python/_warnings.c18
-rw-r--r--Python/asdl.c16
-rw-r--r--Python/ast.c523
-rw-r--r--Python/bltinmodule.c956
-rw-r--r--Python/ceval.c549
-rw-r--r--Python/ceval_gil.h16
-rw-r--r--Python/clinic/bltinmodule.c.h663
-rw-r--r--Python/clinic/import.c.h355
-rw-r--r--Python/codecs.c197
-rw-r--r--Python/compile.c660
-rw-r--r--Python/condvar.h6
-rw-r--r--Python/dtoa.c4
-rw-r--r--Python/dynload_aix.c5
-rw-r--r--Python/dynload_dl.c7
-rw-r--r--Python/dynload_hpux.c12
-rw-r--r--Python/dynload_next.c7
-rw-r--r--Python/dynload_shlib.c22
-rw-r--r--Python/dynload_win.c28
-rw-r--r--Python/errors.c44
-rw-r--r--Python/fileutils.c761
-rw-r--r--Python/formatter_unicode.c7
-rw-r--r--Python/frozen.c3
-rw-r--r--Python/frozenmain.c2
-rw-r--r--Python/future.c2
-rw-r--r--Python/getargs.c32
-rw-r--r--Python/graminit.c2525
-rw-r--r--Python/import.c590
-rw-r--r--Python/importdl.c226
-rw-r--r--Python/importdl.h3
-rw-r--r--Python/importlib.h6258
-rw-r--r--Python/importlib_external.h2596
-rw-r--r--Python/marshal.c221
-rw-r--r--Python/opcode_targets.h30
-rw-r--r--Python/peephole.c15
-rw-r--r--Python/pyfpe.c4
-rw-r--r--Python/pylifecycle.c1581
-rw-r--r--Python/pystate.c28
-rw-r--r--Python/pystrhex.c61
-rw-r--r--Python/pythonrun.c1572
-rw-r--r--Python/pytime.c662
-rw-r--r--Python/random.c142
-rw-r--r--Python/symtable.c97
-rw-r--r--Python/sysmodule.c71
-rw-r--r--Python/thread.c2
-rw-r--r--Python/thread_foobar.h63
-rw-r--r--Python/thread_pthread.h10
-rw-r--r--Python/traceback.c41
-rw-r--r--README20
-rw-r--r--Tools/buildbot/buildmsi.bat22
-rw-r--r--Tools/buildbot/test.bat26
-rwxr-xr-xTools/clinic/clinic.py555
-rwxr-xr-xTools/demo/ss1.py8
-rw-r--r--Tools/freeze/README18
-rw-r--r--Tools/freeze/bkfile.py67
-rw-r--r--Tools/freeze/extensions_win32.ini8
-rwxr-xr-xTools/freeze/freeze.py21
-rw-r--r--Tools/freeze/makefreeze.py54
-rwxr-xr-xTools/i18n/makelocalealias.py62
-rwxr-xr-xTools/i18n/pygettext.py4
-rw-r--r--Tools/msi/README.txt515
-rw-r--r--Tools/msi/build.bat70
-rw-r--r--Tools/msi/buildrelease.bat166
-rw-r--r--Tools/msi/bundle/Default.thm136
-rw-r--r--Tools/msi/bundle/Default.wxl123
-rw-r--r--Tools/msi/bundle/SideBar.pngbin0 -> 19170 bytes-rw-r--r--Tools/msi/bundle/bootstrap/LICENSE.txt25
-rw-r--r--Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp3132
-rw-r--r--Tools/msi/bundle/bootstrap/pch.cpp1
-rw-r--r--Tools/msi/bundle/bootstrap/pch.h59
-rw-r--r--Tools/msi/bundle/bootstrap/pythonba.cpp76
-rw-r--r--Tools/msi/bundle/bootstrap/pythonba.def18
-rw-r--r--Tools/msi/bundle/bootstrap/pythonba.sln22
-rw-r--r--Tools/msi/bundle/bootstrap/pythonba.vcxproj69
-rw-r--r--Tools/msi/bundle/bootstrap/resource.h25
-rw-r--r--Tools/msi/bundle/bundle.icobin0 -> 19790 bytes-rw-r--r--Tools/msi/bundle/bundle.targets111
-rw-r--r--Tools/msi/bundle/bundle.wxl5
-rw-r--r--Tools/msi/bundle/bundle.wxs91
-rw-r--r--Tools/msi/bundle/full.wixproj21
-rw-r--r--Tools/msi/bundle/packagegroups/core.wxs62
-rw-r--r--Tools/msi/bundle/packagegroups/crt.wxs49
-rw-r--r--Tools/msi/bundle/packagegroups/dev.wxs44
-rw-r--r--Tools/msi/bundle/packagegroups/doc.wxs28
-rw-r--r--Tools/msi/bundle/packagegroups/exe.wxs64
-rw-r--r--Tools/msi/bundle/packagegroups/launcher.wxs23
-rw-r--r--Tools/msi/bundle/packagegroups/lib.wxs62
-rw-r--r--Tools/msi/bundle/packagegroups/postinstall.wxs66
-rw-r--r--Tools/msi/bundle/packagegroups/tcltk.wxs68
-rw-r--r--Tools/msi/bundle/packagegroups/test.wxs62
-rw-r--r--Tools/msi/bundle/packagegroups/tools.wxs26
-rw-r--r--Tools/msi/bundle/releaselocal.wixproj21
-rw-r--r--Tools/msi/bundle/releaseweb.wixproj21
-rw-r--r--Tools/msi/bundle/snapshot.wixproj26
-rw-r--r--Tools/msi/common.wxs110
-rw-r--r--Tools/msi/common_en-US.wxl_template17
-rw-r--r--Tools/msi/core/core.wixproj19
-rw-r--r--Tools/msi/core/core.wxs13
-rw-r--r--Tools/msi/core/core_d.wixproj19
-rw-r--r--Tools/msi/core/core_d.wxs14
-rw-r--r--Tools/msi/core/core_en-US.wxl5
-rw-r--r--Tools/msi/core/core_files.wxs31
-rw-r--r--Tools/msi/core/core_pdb.wixproj19
-rw-r--r--Tools/msi/core/core_pdb.wxs14
-rw-r--r--Tools/msi/csv_to_wxs.py127
-rw-r--r--Tools/msi/dev/dev.wixproj49
-rw-r--r--Tools/msi/dev/dev.wxs19
-rw-r--r--Tools/msi/dev/dev_d.wixproj19
-rw-r--r--Tools/msi/dev/dev_d.wxs13
-rw-r--r--Tools/msi/dev/dev_en-US.wxl5
-rw-r--r--Tools/msi/dev/dev_files.wxs42
-rw-r--r--Tools/msi/doc/doc.wixproj30
-rw-r--r--Tools/msi/doc/doc.wxs32
-rw-r--r--Tools/msi/doc/doc_en-US.wxl_template7
-rw-r--r--Tools/msi/doc/doc_files.wxs15
-rw-r--r--Tools/msi/doc/doc_no_files.wxs17
-rw-r--r--Tools/msi/exe/crtlicense.txt (renamed from Tools/msi/crtlicense.txt)7
-rw-r--r--Tools/msi/exe/exe.wixproj43
-rw-r--r--Tools/msi/exe/exe.wxs32
-rw-r--r--Tools/msi/exe/exe_d.wixproj20
-rw-r--r--Tools/msi/exe/exe_d.wxs13
-rw-r--r--Tools/msi/exe/exe_en-US.wxl_template7
-rw-r--r--Tools/msi/exe/exe_files.wxs73
-rw-r--r--Tools/msi/exe/exe_pdb.wixproj20
-rw-r--r--Tools/msi/exe/exe_pdb.wxs13
-rw-r--r--Tools/msi/generate_md5.py27
-rw-r--r--Tools/msi/get_wix.py49
-rw-r--r--Tools/msi/launcher/launcher.wixproj20
-rw-r--r--Tools/msi/launcher/launcher.wxs31
-rw-r--r--Tools/msi/launcher/launcher_en-US.wxl10
-rw-r--r--Tools/msi/launcher/launcher_files.wxs25
-rw-r--r--Tools/msi/launcher/launcher_reg.wxs46
-rw-r--r--Tools/msi/lib/lib.wixproj34
-rw-r--r--Tools/msi/lib/lib.wxs17
-rw-r--r--Tools/msi/lib/lib_d.wixproj19
-rw-r--r--Tools/msi/lib/lib_d.wxs13
-rw-r--r--Tools/msi/lib/lib_en-US.wxl5
-rw-r--r--Tools/msi/lib/lib_files.wxs79
-rw-r--r--Tools/msi/lib/lib_pdb.wixproj19
-rw-r--r--Tools/msi/lib/lib_pdb.wxs13
-rw-r--r--Tools/msi/make_zip.proj38
-rw-r--r--Tools/msi/make_zip.py172
-rw-r--r--Tools/msi/msi.props165
-rw-r--r--Tools/msi/msi.py1454
-rw-r--r--Tools/msi/msi.targets62
-rw-r--r--Tools/msi/msilib.py679
-rw-r--r--Tools/msi/msisupport.c93
-rw-r--r--Tools/msi/msisupport.mak9
-rw-r--r--Tools/msi/path/path.wixproj19
-rw-r--r--Tools/msi/path/path.wxs39
-rw-r--r--Tools/msi/path/path_en-US.wxl6
-rw-r--r--Tools/msi/pip/pip.wixproj19
-rw-r--r--Tools/msi/pip/pip.wxs39
-rw-r--r--Tools/msi/pip/pip_en-US.wxl6
-rw-r--r--Tools/msi/schema.py1007
-rw-r--r--Tools/msi/sequence.py126
-rw-r--r--Tools/msi/tcltk/tcltk.wixproj50
-rw-r--r--Tools/msi/tcltk/tcltk.wxs64
-rw-r--r--Tools/msi/tcltk/tcltk_d.wixproj28
-rw-r--r--Tools/msi/tcltk/tcltk_d.wxs14
-rw-r--r--Tools/msi/tcltk/tcltk_en-US.wxl_template12
-rw-r--r--Tools/msi/tcltk/tcltk_files.wxs35
-rw-r--r--Tools/msi/tcltk/tcltk_pdb.wixproj19
-rw-r--r--Tools/msi/tcltk/tcltk_pdb.wxs13
-rw-r--r--Tools/msi/tcltk/tcltk_reg.wxs48
-rw-r--r--Tools/msi/test/test.wixproj29
-rw-r--r--Tools/msi/test/test.wxs16
-rw-r--r--Tools/msi/test/test_d.wixproj19
-rw-r--r--Tools/msi/test/test_d.wxs13
-rw-r--r--Tools/msi/test/test_en-US.wxl7
-rw-r--r--Tools/msi/test/test_files.wxs77
-rw-r--r--Tools/msi/test/test_pdb.wixproj19
-rw-r--r--Tools/msi/test/test_pdb.wxs13
-rw-r--r--Tools/msi/testrelease.bat117
-rw-r--r--Tools/msi/tools/tools.wixproj43
-rw-r--r--Tools/msi/tools/tools.wxs15
-rw-r--r--Tools/msi/tools/tools_en-US.wxl5
-rw-r--r--Tools/msi/tools/tools_files.wxs16
-rw-r--r--Tools/msi/uisample.py1400
-rw-r--r--Tools/msi/uploadrelease.bat56
-rw-r--r--Tools/msi/uploadrelease.proj80
-rw-r--r--Tools/msi/wix.props12
-rw-r--r--Tools/parser/unparse.py61
-rw-r--r--Tools/pybench/README14
-rw-r--r--Tools/pynche/README14
-rwxr-xr-xTools/scripts/diff.py34
-rw-r--r--Tools/scripts/dutree.doc6
-rwxr-xr-xTools/scripts/eptags.py2
-rwxr-xr-xTools/scripts/find_recursionlimit.py4
-rw-r--r--Tools/scripts/generate_opcode_h.py54
-rw-r--r--Tools/scripts/run_tests.py8
-rw-r--r--Tools/scripts/win_add2path.py3
-rw-r--r--Tools/ssl/sslspeed.vcxproj70
-rw-r--r--Tools/unicode/gencodec.py2
-rw-r--r--Tools/unicode/makeunicodedata.py7
-rw-r--r--Tools/unittestgui/README.txt4
-rwxr-xr-xconfigure1055
-rw-r--r--configure.ac343
-rw-r--r--pyconfig.h.in21
-rw-r--r--setup.py52
1469 files changed, 161603 insertions, 113282 deletions
diff --git a/.gitignore b/.gitignore
index 8809435..cc1cdcd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,11 +13,8 @@
*~
.gdb_history
Doc/build/
-Doc/tools/docutils/
-Doc/tools/jinja/
-Doc/tools/jinja2/
-Doc/tools/pygments/
-Doc/tools/sphinx/
+Doc/venv/
+Lib/distutils/command/*.pdb
Lib/lib2to3/*.pickle
Lib/test/data/*
Lib/_sysconfigdata.py
@@ -31,18 +28,31 @@ Modules/Setup.config
Modules/Setup.local
Modules/config.c
Modules/ld_so_aix
-Modules/_freeze_importlib
-Modules/_testembed
-PCbuild/*.bsc
-PCbuild/*.dll
-PCbuild/*.exe
-PCbuild/*.exp
-PCbuild/*.lib
-PCbuild/*.ncb
-PCbuild/*.o
-PCbuild/*.pdb
-PCbuild/Win32-temp-*
+Programs/_freeze_importlib
+Programs/_testembed
+PC/python_nt*.h
+PC/pythonnt_rc*.h
+PC/*/*.exe
+PC/*/*.exp
+PC/*/*.lib
+PC/*/*.bsc
+PC/*/*.dll
+PC/*/*.pdb
+PC/*/*.user
+PC/*/*.ncb
+PC/*/*.suo
+PC/*/Win32-temp-*
+PC/*/x64-temp-*
+PC/*/amd64
+PCbuild/*.user
+PCbuild/*.suo
+PCbuild/*.*sdf
+PCbuild/*-pgi
+PCbuild/*-pgo
+PCbuild/.vs/
PCbuild/amd64/
+PCbuild/obj/
+PCBuild/win32/
.purify
Parser/pgen
__pycache__
@@ -64,6 +74,7 @@ pybuilddir.txt
pyconfig.h
python-config
python-config.py
+python.bat
python.exe
python-gdb.py
python.exe-gdb.py
@@ -75,3 +86,6 @@ TAGS
coverage/
externals/
htmlcov/
+Tools/msi/obj
+Tools/ssl/amd64
+Tools/ssl/win32
diff --git a/.hgignore b/.hgignore
index 9e5a583..806e156 100644
--- a/.hgignore
+++ b/.hgignore
@@ -9,6 +9,7 @@ TAGS$
autom4te.cache$
^build/
^Doc/build/
+^Doc/venv/
buildno$
config.cache
config.log
@@ -18,6 +19,7 @@ db_home
platform$
pyconfig.h$
python$
+python.bat$
python.exe$
python-config$
python-config.py$
@@ -48,13 +50,12 @@ libpython*.so*
*.pyd
*.cover
*~
+Lib/distutils/command/*.pdb
Lib/lib2to3/*.pickle
Lib/test/data/*
Misc/*.wpu
PC/python_nt*.h
PC/pythonnt_rc*.h
-PC/*.obj
-PC/*.exe
PC/*/*.exe
PC/*/*.exp
PC/*/*.lib
@@ -67,29 +68,21 @@ PC/*/*.suo
PC/*/Win32-temp-*
PC/*/x64-temp-*
PC/*/amd64
-PCbuild/*.exe
-PCbuild/*.dll
-PCbuild/*.pdb
-PCbuild/*.lib
-PCbuild/*.exp
-PCbuild/*.o
-PCbuild/*.ncb
-PCbuild/*.bsc
PCbuild/*.user
PCbuild/*.suo
PCbuild/*.*sdf
-PCbuild/Win32-temp-*
-PCbuild/x64-temp-*
PCbuild/*-pgi
PCbuild/*-pgo
+PCbuild/.vs
PCbuild/amd64
-PCbuild/ipch
+PCbuild/obj
+PCbuild/win32
Tools/unicode/build/
Tools/unicode/MAPPINGS/
BuildLog.htm
__pycache__
-Modules/_freeze_importlib
-Modules/_testembed
+Programs/_freeze_importlib
+Programs/_testembed
.coverage
coverage/
externals/
@@ -98,3 +91,6 @@ htmlcov/
*.gcno
*.gcov
coverage.info
+Tools/msi/obj
+Tools/ssl/amd64
+Tools/ssl/win32
diff --git a/.hgtags b/.hgtags
index d5eea24..b1aa75d 100644
--- a/.hgtags
+++ b/.hgtags
@@ -144,3 +144,13 @@ c0e311e010fcb5bae8d87ca22051cd0845ea0ca0 v3.4.1
ab2c023a9432f16652e89c404bbc84aa91bf55af v3.4.2
69dd528ca6255a66c37cc5cf680e8357d108b036 v3.4.3rc1
b4cbecbc0781e89a309d03b60a1f75f8499250e6 v3.4.3
+5d4b6a57d5fd7564bf73f3db0e46fe5eeb00bcd8 v3.5.0a1
+0337bd7ebcb6559d69679bc7025059ad1ce4f432 v3.5.0a2
+82656e28b5e5c4ae48d8dd8b5f0d7968908a82b6 v3.5.0a3
+413e0e0004f4f954331cb8122aa55fe208984955 v3.5.0a4
+071fefbb5e3db770c6c19fba9994699f121b1cea v3.5.0b1
+7a088af5615bf04024e9912068f4bd8f43ed3917 v3.5.0b2
+0035fcd9b9243ae52c2e830204fd9c1f7d528534 v3.5.0b3
+c0d64105463581f85d0e368e8d6e59b7fd8f12b1 v3.5.0b4
+1a58b1227501e046eee13d90f113417b60843301 v3.5.0rc1
+cc15d736d860303b9da90d43cd32db39bab048df v3.5.0rc2
diff --git a/.hgtouch b/.hgtouch
index 7e3a5e7..b9be0f1 100644
--- a/.hgtouch
+++ b/.hgtouch
@@ -2,7 +2,9 @@
# Define dependencies of generated files that are checked into hg.
# The syntax of this file uses make rule dependencies, without actions
-Python/importlib.h: Lib/importlib/_bootstrap.py Modules/_freeze_importlib.c
+Python/importlib.h: Lib/importlib/_bootstrap.py Programs/_freeze_importlib.c
+
+Include/opcode.h: Lib/opcode.py Tools/scripts/generate_opcode_h.py
Include/Python-ast.h: Parser/Python.asdl Parser/asdl.py Parser/asdl_c.py
Python/Python-ast.c: Include/Python-ast.h
diff --git a/Doc/Makefile b/Doc/Makefile
index ea30231..a42e98b 100644
--- a/Doc/Makefile
+++ b/Doc/Makefile
@@ -15,11 +15,12 @@ ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_paper_size=$(PAPER) \
.PHONY: help build html htmlhelp latex text changes linkcheck \
suspicious coverage doctest pydoc-topics htmlview clean dist check serve \
- autobuild-dev autobuild-stable
+ autobuild-dev autobuild-stable venv
help:
@echo "Please use \`make <target>' where <target> is one of"
@echo " clean to remove build files"
+ @echo " venv to create a venv with necessary tools"
@echo " html to make standalone HTML files"
@echo " htmlview to open the index page built by the html target in your browser"
@echo " htmlhelp to make HTML files and a HTML help project"
@@ -95,14 +96,18 @@ doctest:
pydoc-topics: BUILDER = pydoc-topics
pydoc-topics: build
- @echo "Building finished; now copy build/pydoc-topics/topics.py" \
- "to ../Lib/pydoc_data/topics.py"
+ @echo "Building finished; now run this:" \
+ "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py"
htmlview: html
$(PYTHON) -c "import webbrowser; webbrowser.open('build/html/index.html')"
clean:
- -rm -rf build/*
+ -rm -rf build/* venv/*
+
+venv:
+ $(PYTHON) -m venv venv
+ ./venv/bin/python3 -m pip install -U Sphinx
dist:
rm -rf dist
@@ -172,4 +177,3 @@ autobuild-stable:
exit 1;; \
esac
@make autobuild-dev
-
diff --git a/Doc/README.txt b/Doc/README.txt
index 58bda62..580d102 100644
--- a/Doc/README.txt
+++ b/Doc/README.txt
@@ -3,7 +3,7 @@ Python Documentation README
This directory contains the reStructuredText (reST) sources to the Python
documentation. You don't need to build them yourself, prebuilt versions are
-available at <https://docs.python.org/3.4/download.html>.
+available at <https://docs.python.org/dev/download.html>.
Documentation on authoring Python documentation, including information about
both style and markup, is available in the "Documenting Python" chapter of the
diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst
index 3c0f4b9..ed62dea 100644
--- a/Doc/c-api/arg.rst
+++ b/Doc/c-api/arg.rst
@@ -152,7 +152,7 @@ Unless otherwise stated, buffers are not NUL-terminated.
any conversion. Raises :exc:`TypeError` if the object is not a Unicode
object. The C variable may also be declared as :c:type:`PyObject\*`.
-``w*`` (:class:`bytearray` or read-write byte-oriented buffer) [Py_buffer]
+``w*`` (read-write :term:`bytes-like object`) [Py_buffer]
This format accepts any object which implements the read-write buffer
interface. It fills a :c:type:`Py_buffer` structure provided by the caller.
The buffer may contain embedded null bytes. The caller have to call
diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst
index d099ace..46c19d3 100644
--- a/Doc/c-api/buffer.rst
+++ b/Doc/c-api/buffer.rst
@@ -96,8 +96,8 @@ a buffer, see :c:func:`PyObject_GetBuffer`.
block of the exporter. For example, with negative :c:member:`~Py_buffer.strides`
the value may point to the end of the memory block.
- For contiguous arrays, the value points to the beginning of the memory
- block.
+ For :term:`contiguous` arrays, the value points to the beginning of
+ the memory block.
.. c:member:: void \*obj
@@ -281,11 +281,14 @@ of the flags below it.
+-----------------------------+-------+---------+------------+
+.. index:: contiguous, C-contiguous, Fortran contiguous
+
contiguity requests
~~~~~~~~~~~~~~~~~~~
-C or Fortran contiguity can be explicitly requested, with and without stride
-information. Without stride information, the buffer must be C-contiguous.
+C or Fortran :term:`contiguity <contiguous>` can be explicitly requested,
+with and without stride information. Without stride information, the buffer
+must be C-contiguous.
.. tabularcolumns:: |p{0.35\linewidth}|l|l|l|l|
@@ -466,13 +469,13 @@ Buffer-related functions
.. c:function:: int PyBuffer_IsContiguous(Py_buffer *view, char order)
Return 1 if the memory defined by the *view* is C-style (*order* is
- ``'C'``) or Fortran-style (*order* is ``'F'``) contiguous or either one
+ ``'C'``) or Fortran-style (*order* is ``'F'``) :term:`contiguous` or either one
(*order* is ``'A'``). Return 0 otherwise.
.. c:function:: void PyBuffer_FillContiguousStrides(int ndim, Py_ssize_t *shape, Py_ssize_t *strides, Py_ssize_t itemsize, char order)
- Fill the *strides* array with byte-strides of a contiguous (C-style if
+ Fill the *strides* array with byte-strides of a :term:`contiguous` (C-style if
*order* is ``'C'`` or Fortran-style if *order* is ``'F'``) array of the
given shape with the given number of bytes per element.
diff --git a/Doc/c-api/codec.rst b/Doc/c-api/codec.rst
index 83252af..dfe3d43 100644
--- a/Doc/c-api/codec.rst
+++ b/Doc/c-api/codec.rst
@@ -116,3 +116,8 @@ Registry API for Unicode encoding error handlers
Replace the unicode encode error with backslash escapes (``\x``, ``\u`` and
``\U``).
+.. c:function:: PyObject* PyCodec_NameReplaceErrors(PyObject *exc)
+
+ Replace the unicode encode error with ``\N{...}`` escapes.
+
+ .. versionadded:: 3.5
diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst
index 2d56386..47dab81 100644
--- a/Doc/c-api/concrete.rst
+++ b/Doc/c-api/concrete.rst
@@ -112,5 +112,6 @@ Other Objects
weakref.rst
capsule.rst
gen.rst
+ coro.rst
datetime.rst
diff --git a/Doc/c-api/coro.rst b/Doc/c-api/coro.rst
new file mode 100644
index 0000000..2fe50b5
--- /dev/null
+++ b/Doc/c-api/coro.rst
@@ -0,0 +1,34 @@
+.. highlightlang:: c
+
+.. _coro-objects:
+
+Coroutine Objects
+-----------------
+
+.. versionadded:: 3.5
+
+Coroutine objects are what functions declared with an ``async`` keyword
+return.
+
+
+.. c:type:: PyCoroObject
+
+ The C structure used for coroutine objects.
+
+
+.. c:var:: PyTypeObject PyCoro_Type
+
+ The type object corresponding to coroutine objects.
+
+
+.. c:function:: int PyCoro_CheckExact(PyObject *ob)
+
+ Return true if *ob*'s type is *PyCoro_Type*; *ob* must not be *NULL*.
+
+
+.. c:function:: PyObject* PyCoro_New(PyFrameObject *frame, PyObject *name, PyObject *qualname)
+
+ Create and return a new coroutine object based on the *frame* object,
+ with ``__name__`` and ``__qualname__`` set to *name* and *qualname*.
+ A reference to *frame* is stolen by this function. The *frame* argument
+ must not be *NULL*.
diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst
index c2df767..3fd69ba 100644
--- a/Doc/c-api/exceptions.rst
+++ b/Doc/c-api/exceptions.rst
@@ -9,13 +9,19 @@ Exception Handling
The functions described in this chapter will let you handle and raise Python
exceptions. It is important to understand some of the basics of Python
-exception handling. It works somewhat like the Unix :c:data:`errno` variable:
+exception handling. It works somewhat like the POSIX :c:data:`errno` variable:
there is a global indicator (per thread) of the last error that occurred. Most
-functions don't clear this on success, but will set it to indicate the cause of
-the error on failure. Most functions also return an error indicator, usually
-*NULL* if they are supposed to return a pointer, or ``-1`` if they return an
-integer (exception: the :c:func:`PyArg_\*` functions return ``1`` for success and
-``0`` for failure).
+C API functions don't clear this on success, but will set it to indicate the
+cause of the error on failure. Most C API functions also return an error
+indicator, usually *NULL* if they are supposed to return a pointer, or ``-1``
+if they return an integer (exception: the :c:func:`PyArg_\*` functions
+return ``1`` for success and ``0`` for failure).
+
+Concretely, the error indicator consists of three object pointers: the
+exception's type, the exception's value, and the traceback object. Any
+of those pointers can be NULL if non-set (although some combinations are
+forbidden, for example you can't have a non-NULL traceback if the exception
+type is NULL).
When a function must fail because some function it called failed, it generally
doesn't set the error indicator; the function it called already set it. It is
@@ -27,12 +33,21 @@ the caller that an error has been set. If the error is not handled or carefully
propagated, additional calls into the Python/C API may not behave as intended
and may fail in mysterious ways.
-The error indicator consists of three Python objects corresponding to the result
-of ``sys.exc_info()``. API functions exist to interact with the error indicator
-in various ways. There is a separate error indicator for each thread.
+.. note::
+ The error indicator is **not** the result of :func:`sys.exc_info()`.
+ The former corresponds to an exception that is not yet caught (and is
+ therefore still propagating), while the latter returns an exception after
+ it is caught (and has therefore stopped propagating).
-.. XXX Order of these should be more thoughtful.
- Either alphabetical or some kind of structure.
+
+Printing and clearing
+=====================
+
+
+.. c:function:: void PyErr_Clear()
+
+ Clear the error indicator. If the error indicator is not set, there is no
+ effect.
.. c:function:: void PyErr_PrintEx(int set_sys_last_vars)
@@ -51,127 +66,24 @@ in various ways. There is a separate error indicator for each thread.
Alias for ``PyErr_PrintEx(1)``.
-.. c:function:: PyObject* PyErr_Occurred()
-
- Test whether the error indicator is set. If set, return the exception *type*
- (the first argument to the last call to one of the :c:func:`PyErr_Set\*`
- functions or to :c:func:`PyErr_Restore`). If not set, return *NULL*. You do not
- own a reference to the return value, so you do not need to :c:func:`Py_DECREF`
- it.
-
- .. note::
-
- Do not compare the return value to a specific exception; use
- :c:func:`PyErr_ExceptionMatches` instead, shown below. (The comparison could
- easily fail since the exception may be an instance instead of a class, in the
- case of a class exception, or it may be a subclass of the expected exception.)
-
-
-.. c:function:: int PyErr_ExceptionMatches(PyObject *exc)
-
- Equivalent to ``PyErr_GivenExceptionMatches(PyErr_Occurred(), exc)``. This
- should only be called when an exception is actually set; a memory access
- violation will occur if no exception has been raised.
-
-
-.. c:function:: int PyErr_GivenExceptionMatches(PyObject *given, PyObject *exc)
-
- Return true if the *given* exception matches the exception in *exc*. If
- *exc* is a class object, this also returns true when *given* is an instance
- of a subclass. If *exc* is a tuple, all exceptions in the tuple (and
- recursively in subtuples) are searched for a match.
-
-
-.. c:function:: void PyErr_NormalizeException(PyObject**exc, PyObject**val, PyObject**tb)
-
- Under certain circumstances, the values returned by :c:func:`PyErr_Fetch` below
- can be "unnormalized", meaning that ``*exc`` is a class object but ``*val`` is
- not an instance of the same class. This function can be used to instantiate
- the class in that case. If the values are already normalized, nothing happens.
- The delayed normalization is implemented to improve performance.
-
- .. note::
-
- This function *does not* implicitly set the ``__traceback__``
- attribute on the exception value. If setting the traceback
- appropriately is desired, the following additional snippet is needed::
-
- if (tb != NULL) {
- PyException_SetTraceback(val, tb);
- }
-
-
-.. c:function:: void PyErr_Clear()
-
- Clear the error indicator. If the error indicator is not set, there is no
- effect.
-
-
-.. c:function:: void PyErr_Fetch(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback)
-
- Retrieve the error indicator into three variables whose addresses are passed.
- If the error indicator is not set, set all three variables to *NULL*. If it is
- set, it will be cleared and you own a reference to each object retrieved. The
- value and traceback object may be *NULL* even when the type object is not.
-
- .. note::
-
- This function is normally only used by code that needs to handle exceptions or
- by code that needs to save and restore the error indicator temporarily.
-
-
-.. c:function:: void PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback)
-
- Set the error indicator from the three objects. If the error indicator is
- already set, it is cleared first. If the objects are *NULL*, the error
- indicator is cleared. Do not pass a *NULL* type and non-*NULL* value or
- traceback. The exception type should be a class. Do not pass an invalid
- exception type or value. (Violating these rules will cause subtle problems
- later.) This call takes away a reference to each object: you must own a
- reference to each object before the call and after the call you no longer own
- these references. (If you don't understand this, don't use this function. I
- warned you.)
-
- .. note::
-
- This function is normally only used by code that needs to save and restore the
- error indicator temporarily; use :c:func:`PyErr_Fetch` to save the current
- exception state.
-
-
-.. c:function:: void PyErr_GetExcInfo(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback)
-
- Retrieve the exception info, as known from ``sys.exc_info()``. This refers
- to an exception that was already caught, not to an exception that was
- freshly raised. Returns new references for the three objects, any of which
- may be *NULL*. Does not modify the exception info state.
-
- .. note::
-
- This function is not normally used by code that wants to handle exceptions.
- Rather, it can be used when code needs to save and restore the exception
- state temporarily. Use :c:func:`PyErr_SetExcInfo` to restore or clear the
- exception state.
-
- .. versionadded:: 3.3
-
+.. c:function:: void PyErr_WriteUnraisable(PyObject *obj)
-.. c:function:: void PyErr_SetExcInfo(PyObject *type, PyObject *value, PyObject *traceback)
+ This utility function prints a warning message to ``sys.stderr`` when an
+ exception has been set but it is impossible for the interpreter to actually
+ raise the exception. It is used, for example, when an exception occurs in an
+ :meth:`__del__` method.
- Set the exception info, as known from ``sys.exc_info()``. This refers
- to an exception that was already caught, not to an exception that was
- freshly raised. This function steals the references of the arguments.
- To clear the exception state, pass *NULL* for all three arguments.
- For general rules about the three arguments, see :c:func:`PyErr_Restore`.
+ The function is called with a single argument *obj* that identifies the context
+ in which the unraisable exception occurred. The repr of *obj* will be printed in
+ the warning message.
- .. note::
- This function is not normally used by code that wants to handle exceptions.
- Rather, it can be used when code needs to save and restore the exception
- state temporarily. Use :c:func:`PyErr_GetExcInfo` to read the exception
- state.
+Raising exceptions
+==================
- .. versionadded:: 3.3
+These functions help you set the current thread's error indicator.
+For convenience, some of these functions will always return a
+NULL pointer for use in a ``return`` statement.
.. c:function:: void PyErr_SetString(PyObject *type, const char *message)
@@ -197,6 +109,14 @@ in various ways. There is a separate error indicator for each thread.
string.
+.. c:function:: PyObject* PyErr_FormatV(PyObject *exception, const char *format, va_list vargs)
+
+ Same as :c:func:`PyErr_Format`, but taking a :c:type:`va_list` argument rather
+ than a variable number of arguments.
+
+ .. versionadded:: 3.5
+
+
.. c:function:: void PyErr_SetNone(PyObject *type)
This is a shorthand for ``PyErr_SetObject(type, Py_None)``.
@@ -346,6 +266,22 @@ in various ways. There is a separate error indicator for each thread.
use.
+Issuing warnings
+================
+
+Use these functions to issue warnings from C code. They mirror similar
+functions exported by the Python :mod:`warnings` module. They normally
+print a warning message to *sys.stderr*; however, it is
+also possible that the user has specified that warnings are to be turned into
+errors, and in that case they will raise an exception. It is also possible that
+the functions raise an exception because of a problem with the warning machinery.
+The return value is ``0`` if no exception is raised, or ``-1`` if an exception
+is raised. (It is not possible to determine whether a warning message is
+actually printed, nor what the reason is for the exception; this is
+intentional.) If an exception is raised, the caller should do its normal
+exception handling (for example, :c:func:`Py_DECREF` owned references and return
+an error value).
+
.. c:function:: int PyErr_WarnEx(PyObject *category, const char *message, Py_ssize_t stack_level)
Issue a warning message. The *category* argument is a warning category (see
@@ -355,18 +291,6 @@ in various ways. There is a separate error indicator for each thread.
is the function calling :c:func:`PyErr_WarnEx`, 2 is the function above that,
and so forth.
- This function normally prints a warning message to *sys.stderr*; however, it is
- also possible that the user has specified that warnings are to be turned into
- errors, and in that case this will raise an exception. It is also possible that
- the function raises an exception because of a problem with the warning machinery
- (the implementation imports the :mod:`warnings` module to do the heavy lifting).
- The return value is ``0`` if no exception is raised, or ``-1`` if an exception
- is raised. (It is not possible to determine whether a warning message is
- actually printed, nor what the reason is for the exception; this is
- intentional.) If an exception is raised, the caller should do its normal
- exception handling (for example, :c:func:`Py_DECREF` owned references and return
- an error value).
-
Warning categories must be subclasses of :c:data:`Warning`; the default warning
category is :c:data:`RuntimeWarning`. The standard Python warning categories are
available as global variables whose names are ``PyExc_`` followed by the Python
@@ -410,6 +334,139 @@ in various ways. There is a separate error indicator for each thread.
.. versionadded:: 3.2
+Querying the error indicator
+============================
+
+.. c:function:: PyObject* PyErr_Occurred()
+
+ Test whether the error indicator is set. If set, return the exception *type*
+ (the first argument to the last call to one of the :c:func:`PyErr_Set\*`
+ functions or to :c:func:`PyErr_Restore`). If not set, return *NULL*. You do not
+ own a reference to the return value, so you do not need to :c:func:`Py_DECREF`
+ it.
+
+ .. note::
+
+ Do not compare the return value to a specific exception; use
+ :c:func:`PyErr_ExceptionMatches` instead, shown below. (The comparison could
+ easily fail since the exception may be an instance instead of a class, in the
+ case of a class exception, or it may be a subclass of the expected exception.)
+
+
+.. c:function:: int PyErr_ExceptionMatches(PyObject *exc)
+
+ Equivalent to ``PyErr_GivenExceptionMatches(PyErr_Occurred(), exc)``. This
+ should only be called when an exception is actually set; a memory access
+ violation will occur if no exception has been raised.
+
+
+.. c:function:: int PyErr_GivenExceptionMatches(PyObject *given, PyObject *exc)
+
+ Return true if the *given* exception matches the exception type in *exc*. If
+ *exc* is a class object, this also returns true when *given* is an instance
+ of a subclass. If *exc* is a tuple, all exception types in the tuple (and
+ recursively in subtuples) are searched for a match.
+
+
+.. c:function:: void PyErr_Fetch(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback)
+
+ Retrieve the error indicator into three variables whose addresses are passed.
+ If the error indicator is not set, set all three variables to *NULL*. If it is
+ set, it will be cleared and you own a reference to each object retrieved. The
+ value and traceback object may be *NULL* even when the type object is not.
+
+ .. note::
+
+ This function is normally only used by code that needs to catch exceptions or
+ by code that needs to save and restore the error indicator temporarily, e.g.::
+
+ {
+ PyObject **type, **value, **traceback;
+ PyErr_Fetch(&type, &value, &traceback);
+
+ /* ... code that might produce other errors ... */
+
+ PyErr_Restore(type, value, traceback);
+ }
+
+
+.. c:function:: void PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback)
+
+ Set the error indicator from the three objects. If the error indicator is
+ already set, it is cleared first. If the objects are *NULL*, the error
+ indicator is cleared. Do not pass a *NULL* type and non-*NULL* value or
+ traceback. The exception type should be a class. Do not pass an invalid
+ exception type or value. (Violating these rules will cause subtle problems
+ later.) This call takes away a reference to each object: you must own a
+ reference to each object before the call and after the call you no longer own
+ these references. (If you don't understand this, don't use this function. I
+ warned you.)
+
+ .. note::
+
+ This function is normally only used by code that needs to save and restore the
+ error indicator temporarily. Use :c:func:`PyErr_Fetch` to save the current
+ error indicator.
+
+
+.. c:function:: void PyErr_NormalizeException(PyObject**exc, PyObject**val, PyObject**tb)
+
+ Under certain circumstances, the values returned by :c:func:`PyErr_Fetch` below
+ can be "unnormalized", meaning that ``*exc`` is a class object but ``*val`` is
+ not an instance of the same class. This function can be used to instantiate
+ the class in that case. If the values are already normalized, nothing happens.
+ The delayed normalization is implemented to improve performance.
+
+ .. note::
+
+ This function *does not* implicitly set the ``__traceback__``
+ attribute on the exception value. If setting the traceback
+ appropriately is desired, the following additional snippet is needed::
+
+ if (tb != NULL) {
+ PyException_SetTraceback(val, tb);
+ }
+
+
+.. c:function:: void PyErr_GetExcInfo(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback)
+
+ Retrieve the exception info, as known from ``sys.exc_info()``. This refers
+ to an exception that was *already caught*, not to an exception that was
+ freshly raised. Returns new references for the three objects, any of which
+ may be *NULL*. Does not modify the exception info state.
+
+ .. note::
+
+ This function is not normally used by code that wants to handle exceptions.
+ Rather, it can be used when code needs to save and restore the exception
+ state temporarily. Use :c:func:`PyErr_SetExcInfo` to restore or clear the
+ exception state.
+
+ .. versionadded:: 3.3
+
+
+.. c:function:: void PyErr_SetExcInfo(PyObject *type, PyObject *value, PyObject *traceback)
+
+ Set the exception info, as known from ``sys.exc_info()``. This refers
+ to an exception that was *already caught*, not to an exception that was
+ freshly raised. This function steals the references of the arguments.
+ To clear the exception state, pass *NULL* for all three arguments.
+ For general rules about the three arguments, see :c:func:`PyErr_Restore`.
+
+ .. note::
+
+ This function is not normally used by code that wants to handle exceptions.
+ Rather, it can be used when code needs to save and restore the exception
+ state temporarily. Use :c:func:`PyErr_GetExcInfo` to read the exception
+ state.
+
+ .. versionadded:: 3.3
+
+
+Signal Handling
+===============
+
+
.. c:function:: int PyErr_CheckSignals()
.. index::
@@ -443,13 +500,21 @@ in various ways. There is a separate error indicator for each thread.
.. c:function:: int PySignal_SetWakeupFd(int fd)
- This utility function specifies a file descriptor to which a ``'\0'`` byte will
- be written whenever a signal is received. It returns the previous such file
- descriptor. The value ``-1`` disables the feature; this is the initial state.
+ This utility function specifies a file descriptor to which the signal number
+ is written as a single byte whenever a signal is received. *fd* must be
+ non-blocking. It returns the previous such file descriptor.
+
+ The value ``-1`` disables the feature; this is the initial state.
This is equivalent to :func:`signal.set_wakeup_fd` in Python, but without any
error checking. *fd* should be a valid file descriptor. The function should
only be called from the main thread.
+ .. versionchanged:: 3.5
+ On Windows, the function now also supports socket handles.
+
+
+Exception Classes
+=================
.. c:function:: PyObject* PyErr_NewException(const char *name, PyObject *base, PyObject *dict)
@@ -475,18 +540,6 @@ in various ways. There is a separate error indicator for each thread.
.. versionadded:: 3.2
-.. c:function:: void PyErr_WriteUnraisable(PyObject *obj)
-
- This utility function prints a warning message to ``sys.stderr`` when an
- exception has been set but it is impossible for the interpreter to actually
- raise the exception. It is used, for example, when an exception occurs in an
- :meth:`__del__` method.
-
- The function is called with a single argument *obj* that identifies the context
- in which the unraisable exception occurred. The repr of *obj* will be printed in
- the warning message.
-
-
Exception Objects
=================
@@ -630,12 +683,12 @@ recursion depth automatically).
sets a :exc:`MemoryError` and returns a nonzero value.
The function then checks if the recursion limit is reached. If this is the
- case, a :exc:`RuntimeError` is set and a nonzero value is returned.
+ case, a :exc:`RecursionError` is set and a nonzero value is returned.
Otherwise, zero is returned.
*where* should be a string such as ``" in instance check"`` to be
- concatenated to the :exc:`RuntimeError` message caused by the recursion depth
- limit.
+ concatenated to the :exc:`RecursionError` message caused by the recursion
+ depth limit.
.. c:function:: void Py_LeaveRecursiveCall()
@@ -747,6 +800,8 @@ the variables:
+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | |
+-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_RecursionError` | :exc:`RecursionError` | |
++-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | \(2) |
+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | |
@@ -776,6 +831,9 @@ the variables:
:c:data:`PyExc_PermissionError`, :c:data:`PyExc_ProcessLookupError`
and :c:data:`PyExc_TimeoutError` were introduced following :pep:`3151`.
+.. versionadded:: 3.5
+ :c:data:`PyExc_RecursionError`.
+
These are compatibility aliases to :c:data:`PyExc_OSError`:
@@ -824,6 +882,7 @@ These are compatibility aliases to :c:data:`PyExc_OSError`:
single: PyExc_OverflowError
single: PyExc_PermissionError
single: PyExc_ProcessLookupError
+ single: PyExc_RecursionError
single: PyExc_ReferenceError
single: PyExc_RuntimeError
single: PyExc_SyntaxError
diff --git a/Doc/c-api/gen.rst b/Doc/c-api/gen.rst
index 33cd27a..3ab073b 100644
--- a/Doc/c-api/gen.rst
+++ b/Doc/c-api/gen.rst
@@ -7,7 +7,7 @@ Generator Objects
Generator objects are what Python uses to implement generator iterators. They
are normally created by iterating over a function that yields values, rather
-than explicitly calling :c:func:`PyGen_New`.
+than explicitly calling :c:func:`PyGen_New` or :c:func:`PyGen_NewWithQualName`.
.. c:type:: PyGenObject
@@ -20,19 +20,25 @@ than explicitly calling :c:func:`PyGen_New`.
The type object corresponding to generator objects
-.. c:function:: int PyGen_Check(ob)
+.. c:function:: int PyGen_Check(PyObject *ob)
Return true if *ob* is a generator object; *ob* must not be *NULL*.
-.. c:function:: int PyGen_CheckExact(ob)
+.. c:function:: int PyGen_CheckExact(PyObject *ob)
- Return true if *ob*'s type is *PyGen_Type* is a generator object; *ob* must not
- be *NULL*.
+ Return true if *ob*'s type is *PyGen_Type*; *ob* must not be *NULL*.
.. c:function:: PyObject* PyGen_New(PyFrameObject *frame)
- Create and return a new generator object based on the *frame* object. A
- reference to *frame* is stolen by this function. The parameter must not be
+ Create and return a new generator object based on the *frame* object.
+ A reference to *frame* is stolen by this function. The argument must not be
*NULL*.
+
+.. c:function:: PyObject* PyGen_NewWithQualName(PyFrameObject *frame, PyObject *name, PyObject *qualname)
+
+ Create and return a new generator object based on the *frame* object,
+ with ``__name__`` and ``__qualname__`` set to *name* and *qualname*.
+ A reference to *frame* is stolen by this function. The *frame* argument
+ must not be *NULL*.
diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst
index 60865f4..bf1b495 100644
--- a/Doc/c-api/import.rst
+++ b/Doc/c-api/import.rst
@@ -183,9 +183,9 @@ Importing Modules
.. c:function:: long PyImport_GetMagicNumber()
- Return the magic number for Python bytecode files (a.k.a. :file:`.pyc` and
- :file:`.pyo` files). The magic number should be present in the first four bytes
- of the bytecode file, in little-endian byte order. Returns -1 on error.
+ Return the magic number for Python bytecode files (a.k.a. :file:`.pyc` file).
+ The magic number should be present in the first four bytes of the bytecode
+ file, in little-endian byte order. Returns -1 on error.
.. versionchanged:: 3.3
Return value of -1 upon failure.
diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst
index 4bb5064..81823bf 100644
--- a/Doc/c-api/init.rst
+++ b/Doc/c-api/init.rst
@@ -134,6 +134,9 @@ Process-wide parameters
change for the duration of the program's execution. No code in the Python
interpreter will change the contents of this storage.
+ Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a
+ :c:type:`wchar_*` string.
+
.. c:function:: wchar* Py_GetProgramName()
@@ -245,6 +248,9 @@ Process-wide parameters
:data:`sys.exec_prefix` to be empty. It is up to the caller to modify these
if required after calling :c:func:`Py_Initialize`.
+ Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a
+ :c:type:`wchar_*` string.
+
The path argument is copied internally, so the caller may free it after the
call completes.
@@ -344,6 +350,9 @@ Process-wide parameters
:data:`sys.path`, which is the same as prepending the current working
directory (``"."``).
+ Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a
+ :c:type:`wchar_*` string.
+
.. note::
It is recommended that applications embedding the Python interpreter
for purposes other than executing a single script pass 0 as *updatepath*,
@@ -368,6 +377,9 @@ Process-wide parameters
to 1 unless the :program:`python` interpreter was started with the
:option:`-I`.
+ Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a
+ :c:type:`wchar_*` string.
+
.. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`.
@@ -382,6 +394,9 @@ Process-wide parameters
execution. No code in the Python interpreter will change the contents of
this storage.
+ Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a
+ :c:type:`wchar_*` string.
+
.. c:function:: w_char* Py_GetPythonHome()
@@ -858,6 +873,8 @@ been created.
instead.
+.. _sub-interpreter-support:
+
Sub-interpreter support
=======================
diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst
index 7908622..7339006 100644
--- a/Doc/c-api/memory.rst
+++ b/Doc/c-api/memory.rst
@@ -92,8 +92,8 @@ functions are thread-safe, the :term:`GIL <global interpreter lock>` does not
need to be held.
The default raw memory block allocator uses the following functions:
-:c:func:`malloc`, :c:func:`realloc` and :c:func:`free`; call ``malloc(1)`` when
-requesting zero bytes.
+:c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`; call
+``malloc(1)`` (or ``calloc(1, 1)``) when requesting zero bytes.
.. versionadded:: 3.4
@@ -106,6 +106,17 @@ requesting zero bytes.
been initialized in any way.
+.. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize)
+
+ Allocates *nelem* elements each whose size in bytes is *elsize* and returns
+ a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the
+ request fails. The memory is initialized to zeros. Requesting zero elements
+ or elements of size zero bytes returns a distinct non-*NULL* pointer if
+ possible, as if ``PyMem_RawCalloc(1, 1)`` had been called instead.
+
+ .. versionadded:: 3.5
+
+
.. c:function:: void* PyMem_RawRealloc(void *p, size_t n)
Resizes the memory block pointed to by *p* to *n* bytes. The contents will
@@ -136,8 +147,8 @@ behavior when requesting zero bytes, are available for allocating and releasing
memory from the Python heap.
The default memory block allocator uses the following functions:
-:c:func:`malloc`, :c:func:`realloc` and :c:func:`free`; call ``malloc(1)`` when
-requesting zero bytes.
+:c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`; call
+``malloc(1)`` (or ``calloc(1, 1)``) when requesting zero bytes.
.. warning::
@@ -152,6 +163,17 @@ requesting zero bytes.
been called instead. The memory will not have been initialized in any way.
+.. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize)
+
+ Allocates *nelem* elements each whose size in bytes is *elsize* and returns
+ a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the
+ request fails. The memory is initialized to zeros. Requesting zero elements
+ or elements of size zero bytes returns a distinct non-*NULL* pointer if
+ possible, as if ``PyMem_Calloc(1, 1)`` had been called instead.
+
+ .. versionadded:: 3.5
+
+
.. c:function:: void* PyMem_Realloc(void *p, size_t n)
Resizes the memory block pointed to by *p* to *n* bytes. The contents will be
@@ -210,7 +232,7 @@ Customize Memory Allocators
.. versionadded:: 3.4
-.. c:type:: PyMemAllocator
+.. c:type:: PyMemAllocatorEx
Structure used to describe a memory block allocator. The structure has
four fields:
@@ -222,11 +244,19 @@ Customize Memory Allocators
+----------------------------------------------------------+---------------------------------------+
| ``void* malloc(void *ctx, size_t size)`` | allocate a memory block |
+----------------------------------------------------------+---------------------------------------+
+ | ``void* calloc(void *ctx, size_t nelem, size_t elsize)`` | allocate a memory block initialized |
+ | | with zeros |
+ +----------------------------------------------------------+---------------------------------------+
| ``void* realloc(void *ctx, void *ptr, size_t new_size)`` | allocate or resize a memory block |
+----------------------------------------------------------+---------------------------------------+
| ``void free(void *ctx, void *ptr)`` | free a memory block |
+----------------------------------------------------------+---------------------------------------+
+ .. versionchanged:: 3.5
+ The :c:type:`PyMemAllocator` structure was renamed to
+ :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added.
+
+
.. c:type:: PyMemAllocatorDomain
Enum used to identify an allocator domain. Domains:
@@ -239,12 +269,12 @@ Customize Memory Allocators
:c:func:`PyObject_Realloc` and :c:func:`PyObject_Free`
-.. c:function:: void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator)
+.. c:function:: void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator)
Get the memory block allocator of the specified domain.
-.. c:function:: void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator)
+.. c:function:: void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator)
Set the memory block allocator of the specified domain.
diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst
index 5e50977..9f6bfd7 100644
--- a/Doc/c-api/memoryview.rst
+++ b/Doc/c-api/memoryview.rst
@@ -35,7 +35,7 @@ any other object.
.. c:function:: PyObject *PyMemoryView_GetContiguous(PyObject *obj, int buffertype, char order)
- Create a memoryview object to a contiguous chunk of memory (in either
+ Create a memoryview object to a :term:`contiguous` chunk of memory (in either
'C' or 'F'ortran *order*) from an object that defines the buffer
interface. If memory is contiguous, the memoryview object points to the
original memory. Otherwise, a copy is made and the memoryview points to a
diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst
index 985a347..ef778cc 100644
--- a/Doc/c-api/module.rst
+++ b/Doc/c-api/module.rst
@@ -7,8 +7,6 @@ Module Objects
.. index:: object: module
-There are only a few functions special to module objects.
-
.. c:var:: PyTypeObject PyModule_Type
@@ -84,6 +82,18 @@ There are only a few functions special to module objects.
Similar to :c:func:`PyModule_GetNameObject` but return the name encoded to
``'utf-8'``.
+.. c:function:: void* PyModule_GetState(PyObject *module)
+
+ Return the "state" of the module, that is, a pointer to the block of memory
+ allocated at module creation time, or *NULL*. See
+ :c:member:`PyModuleDef.m_size`.
+
+
+.. c:function:: PyModuleDef* PyModule_GetDef(PyObject *module)
+
+ Return a pointer to the :c:type:`PyModuleDef` struct from which the module was
+ created, or *NULL* if the module wasn't created from a definition.
+
.. c:function:: PyObject* PyModule_GetFilenameObject(PyObject *module)
@@ -109,55 +119,109 @@ There are only a few functions special to module objects.
unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead.
-.. c:function:: void* PyModule_GetState(PyObject *module)
+.. _initializing-modules:
- Return the "state" of the module, that is, a pointer to the block of memory
- allocated at module creation time, or *NULL*. See
- :c:member:`PyModuleDef.m_size`.
+Initializing C modules
+^^^^^^^^^^^^^^^^^^^^^^
+Modules objects are usually created from extension modules (shared libraries
+which export an initialization function), or compiled-in modules
+(where the initialization function is added using :c:func:`PyImport_AppendInittab`).
+See :ref:`building` or :ref:`extending-with-embedding` for details.
-.. c:function:: PyModuleDef* PyModule_GetDef(PyObject *module)
+The initialization function can either pass pass a module definition instance
+to :c:func:`PyModule_Create`, and return the resulting module object,
+or request "multi-phase initialization" by returning the definition struct itself.
- Return a pointer to the :c:type:`PyModuleDef` struct from which the module was
- created, or *NULL* if the module wasn't created with
- :c:func:`PyModule_Create`.
+.. c:type:: PyModuleDef
-.. c:function:: PyObject* PyState_FindModule(PyModuleDef *def)
+ The module definition struct, which holds all information needed to create
+ a module object. There is usually only one statically initialized variable
+ of this type for each module.
- Returns the module object that was created from *def* for the current interpreter.
- This method requires that the module object has been attached to the interpreter state with
- :c:func:`PyState_AddModule` beforehand. In case the corresponding module object is not
- found or has not been attached to the interpreter state yet, it returns NULL.
+ .. c:member:: PyModuleDef_Base m_base
-.. c:function:: int PyState_AddModule(PyObject *module, PyModuleDef *def)
+ Always initialize this member to :const:`PyModuleDef_HEAD_INIT`.
- Attaches the module object passed to the function to the interpreter state. This allows
- the module object to be accessible via
- :c:func:`PyState_FindModule`.
+ .. c:member:: char* m_name
- .. versionadded:: 3.3
+ Name for the new module.
-.. c:function:: int PyState_RemoveModule(PyModuleDef *def)
+ .. c:member:: char* m_doc
- Removes the module object created from *def* from the interpreter state.
+ Docstring for the module; usually a docstring variable created with
+ :c:func:`PyDoc_STRVAR` is used.
- .. versionadded:: 3.3
+ .. c:member:: Py_ssize_t m_size
-Initializing C modules
-^^^^^^^^^^^^^^^^^^^^^^
+ Module state may be kept in a per-module memory area that can be
+ retrieved with :c:func:`PyModule_GetState`, rather than in static globals.
+ This makes modules safe for use in multiple sub-interpreters.
+
+ This memory area is allocated based on *m_size* on module creation,
+ and freed when the module object is deallocated, after the
+ :c:member:`m_free` function has been called, if present.
+
+ Setting ``m_size`` to ``-1`` means that the module does not support
+ sub-interpreters, because it has global state.
-These functions are usually used in the module initialization function.
+ Setting it to a non-negative value means that the module can be
+ re-initialized and specifies the additional amount of memory it requires
+ for its state. Non-negative ``m_size`` is required for multi-phase
+ initialization.
-.. c:function:: PyObject* PyModule_Create(PyModuleDef *module)
+ See :PEP:`3121` for more details.
+
+ .. c:member:: PyMethodDef* m_methods
- Create a new module object, given the definition in *module*. This behaves
+ A pointer to a table of module-level functions, described by
+ :c:type:`PyMethodDef` values. Can be *NULL* if no functions are present.
+
+ .. c:member:: PyModuleDef_Slot* m_slots
+
+ An array of slot definitions for multi-phase initialization, terminated by
+ a ``{0, NULL}`` entry.
+ When using single-phase initialization, *m_slots* must be *NULL*.
+
+ .. versionchanged:: 3.5
+
+ Prior to version 3.5, this member was always set to *NULL*,
+ and was defined as:
+
+ .. c:member:: inquiry m_reload
+
+ .. c:member:: traverseproc m_traverse
+
+ A traversal function to call during GC traversal of the module object, or
+ *NULL* if not needed.
+
+ .. c:member:: inquiry m_clear
+
+ A clear function to call during GC clearing of the module object, or
+ *NULL* if not needed.
+
+ .. c:member:: freefunc m_free
+
+ A function to call during deallocation of the module object, or *NULL* if
+ not needed.
+
+Single-phase initialization
+...........................
+
+The module initialization function may create and return the module object
+directly. This is referred to as "single-phase initialization", and uses one
+of the following two module creation functions:
+
+.. c:function:: PyObject* PyModule_Create(PyModuleDef *def)
+
+ Create a new module object, given the definition in *def*. This behaves
like :c:func:`PyModule_Create2` with *module_api_version* set to
:const:`PYTHON_API_VERSION`.
-.. c:function:: PyObject* PyModule_Create2(PyModuleDef *module, int module_api_version)
+.. c:function:: PyObject* PyModule_Create2(PyModuleDef *def, int module_api_version)
- Create a new module object, given the definition in *module*, assuming the
+ Create a new module object, given the definition in *def*, assuming the
API version *module_api_version*. If that version does not match the version
of the running interpreter, a :exc:`RuntimeWarning` is emitted.
@@ -166,69 +230,179 @@ These functions are usually used in the module initialization function.
Most uses of this function should be using :c:func:`PyModule_Create`
instead; only use this if you are sure you need it.
+Before it is returned from in the initialization function, the resulting module
+object is typically populated using functions like :c:func:`PyModule_AddObject`.
-.. c:type:: PyModuleDef
+.. _multi-phase-initialization:
- This struct holds all information that is needed to create a module object.
- There is usually only one static variable of that type for each module, which
- is statically initialized and then passed to :c:func:`PyModule_Create` in the
- module initialization function.
+Multi-phase initialization
+..........................
- .. c:member:: PyModuleDef_Base m_base
+An alternate way to specify extensions is to request "multi-phase initialization".
+Extension modules created this way behave more like Python modules: the
+initialization is split between the *creation phase*, when the module object
+is created, and the *execution phase*, when it is populated.
+The distinction is similar to the :py:meth:`__new__` and :py:meth:`__init__` methods
+of classes.
- Always initialize this member to :const:`PyModuleDef_HEAD_INIT`.
+Unlike modules created using single-phase initialization, these modules are not
+singletons: if the *sys.modules* entry is removed and the module is re-imported,
+a new module object is created, and the old module is subject to normal garbage
+collection -- as with Python modules.
+By default, multiple modules created from the same definition should be
+independent: changes to one should not affect the others.
+This means that all state should be specific to the module object (using e.g.
+using :c:func:`PyModule_GetState`), or its contents (such as the module's
+:attr:`__dict__` or individual classes created with :c:func:`PyType_FromSpec`).
- .. c:member:: char* m_name
+All modules created using multi-phase initialization are expected to support
+:ref:`sub-interpreters <sub-interpreter-support>`. Making sure multiple modules
+are independent is typically enough to achieve this.
- Name for the new module.
+To request multi-phase initialization, the initialization function
+(PyInit_modulename) returns a :c:type:`PyModuleDef` instance with non-empty
+:c:member:`~PyModuleDef.m_slots`. Before it is returned, the ``PyModuleDef``
+instance must be initialized with the following function:
- .. c:member:: char* m_doc
+.. c:function:: PyObject* PyModuleDef_Init(PyModuleDef *def)
- Docstring for the module; usually a docstring variable created with
- :c:func:`PyDoc_STRVAR` is used.
+ Ensures a module definition is a properly initialized Python object that
+ correctly reports its type and reference count.
- .. c:member:: Py_ssize_t m_size
+ Returns *def* cast to ``PyObject*``, or *NULL* if an error occurred.
- Some modules allow re-initialization (calling their ``PyInit_*`` function
- more than once). These modules should keep their state in a per-module
- memory area that can be retrieved with :c:func:`PyModule_GetState`.
+ .. versionadded:: 3.5
- This memory should be used, rather than static globals, to hold per-module
- state, since it is then safe for use in multiple sub-interpreters. It is
- freed when the module object is deallocated, after the :c:member:`m_free`
- function has been called, if present.
+The *m_slots* member of the module definition must point to an array of
+``PyModuleDef_Slot`` structures:
- Setting ``m_size`` to ``-1`` means that the module can not be
- re-initialized because it has global state. Setting it to a non-negative
- value means that the module can be re-initialized and specifies the
- additional amount of memory it requires for its state.
+.. c:type:: PyModuleDef_Slot
- See :PEP:`3121` for more details.
+ .. c:member:: int slot
- .. c:member:: PyMethodDef* m_methods
+ A slot ID, chosen from the available values explained below.
- A pointer to a table of module-level functions, described by
- :c:type:`PyMethodDef` values. Can be *NULL* if no functions are present.
+ .. c:member:: void* value
- .. c:member:: inquiry m_reload
+ Value of the slot, whose meaning depends on the slot ID.
- Currently unused, should be *NULL*.
+ .. versionadded:: 3.5
- .. c:member:: traverseproc m_traverse
+The *m_slots* array must be terminated by a slot with id 0.
- A traversal function to call during GC traversal of the module object, or
- *NULL* if not needed.
+The available slot types are:
- .. c:member:: inquiry m_clear
+.. c:var:: Py_mod_create
- A clear function to call during GC clearing of the module object, or
- *NULL* if not needed.
+ Specifies a function that is called to create the module object itself.
+ The *value* pointer of this slot must point to a function of the signature:
- .. c:member:: freefunc m_free
+ .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def)
- A function to call during deallocation of the module object, or *NULL* if
- not needed.
+ The function receives a :py:class:`~importlib.machinery.ModuleSpec`
+ instance, as defined in :PEP:`451`, and the module definition.
+ It should return a new module object, or set an error
+ and return *NULL*.
+
+ This function should be kept minimal. In particular, it should not
+ call arbitrary Python code, as trying to import the same module again may
+ result in an infinite loop.
+
+ Multiple ``Py_mod_create`` slots may not be specified in one module
+ definition.
+
+ If ``Py_mod_create`` is not specified, the import machinery will create
+ a normal module object using :c:func:`PyModule_New`. The name is taken from
+ *spec*, not the definition, to allow extension modules to dynamically adjust
+ to their place in the module hierarchy and be imported under different
+ names through symlinks, all while sharing a single module definition.
+
+ There is no requirement for the returned object to be an instance of
+ :c:type:`PyModule_Type`. Any type can be used, as long as it supports
+ setting and getting import-related attributes.
+ However, only ``PyModule_Type`` instances may be returned if the
+ ``PyModuleDef`` has non-*NULL* ``m_methods``, ``m_traverse``, ``m_clear``,
+ ``m_free``; non-zero ``m_size``; or slots other than ``Py_mod_create``.
+
+.. c:var:: Py_mod_exec
+
+ Specifies a function that is called to *execute* the module.
+ This is equivalent to executing the code of a Python module: typically,
+ this function adds classes and constants to the module.
+ The signature of the function is:
+
+ .. c:function:: int exec_module(PyObject* module)
+
+ If multiple ``Py_mod_exec`` slots are specified, they are processed in the
+ order they appear in the *m_slots* array.
+
+See :PEP:`489` for more details on multi-phase initialization.
+
+Low-level module creation functions
+...................................
+
+The following functions are called under the hood when using multi-phase
+initialization. They can be used directly, for example when creating module
+objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and
+``PyModule_ExecDef`` must be called to fully initialize a module.
+
+.. c:function:: PyObject * PyModule_FromDefAndSpec(PyModuleDef *def, PyObject *spec)
+
+ Create a new module object, given the definition in *module* and the
+ ModuleSpec *spec*. This behaves like :c:func:`PyModule_FromDefAndSpec2`
+ with *module_api_version* set to :const:`PYTHON_API_VERSION`.
+
+ .. versionadded:: 3.5
+
+.. c:function:: PyObject * PyModule_FromDefAndSpec2(PyModuleDef *def, PyObject *spec, int module_api_version)
+
+ Create a new module object, given the definition in *module* and the
+ ModuleSpec *spec*, assuming the API version *module_api_version*.
+ If that version does not match the version of the running interpreter,
+ a :exc:`RuntimeWarning` is emitted.
+
+ .. note::
+ Most uses of this function should be using :c:func:`PyModule_FromDefAndSpec`
+ instead; only use this if you are sure you need it.
+
+ .. versionadded:: 3.5
+
+.. c:function:: int PyModule_ExecDef(PyObject *module, PyModuleDef *def)
+
+ Process any execution slots (:c:data:`Py_mod_exec`) given in *def*.
+
+ .. versionadded:: 3.5
+
+.. c:function:: int PyModule_SetDocString(PyObject *module, const char *docstring)
+
+ Set the docstring for *module* to *docstring*.
+ This function is called automatically when creating a module from
+ ``PyModuleDef``, using either ``PyModule_Create`` or
+ ``PyModule_FromDefAndSpec``.
+
+ .. versionadded:: 3.5
+
+.. c:function:: int PyModule_AddFunctions(PyObject *module, PyMethodDef *functions)
+
+ Add the functions from the *NULL* terminated *functions* array to *module*.
+ Refer to the :c:type:`PyMethodDef` documentation for details on individual
+ entries (due to the lack of a shared module namespace, module level
+ "functions" implemented in C typically receive the module as their first
+ parameter, making them similar to instance methods on Python classes).
+ This function is called automatically when creating a module from
+ ``PyModuleDef``, using either ``PyModule_Create`` or
+ ``PyModule_FromDefAndSpec``.
+
+ .. versionadded:: 3.5
+
+Support functions
+.................
+
+The module initialization function (if using single phase initialization) or
+a function called from a module execution slot (if using multi-phase
+initialization), can use the following functions to help initialize the module
+state:
.. c:function:: int PyModule_AddObject(PyObject *module, const char *name, PyObject *value)
@@ -236,7 +410,6 @@ These functions are usually used in the module initialization function.
be used from the module's initialization function. This steals a reference to
*value*. Return ``-1`` on error, ``0`` on success.
-
.. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value)
Add an integer constant to *module* as *name*. This convenience function can be
@@ -248,7 +421,7 @@ These functions are usually used in the module initialization function.
Add a string constant to *module* as *name*. This convenience function can be
used from the module's initialization function. The string *value* must be
- null-terminated. Return ``-1`` on error, ``0`` on success.
+ *NULL*-terminated. Return ``-1`` on error, ``0`` on success.
.. c:function:: int PyModule_AddIntMacro(PyObject *module, macro)
@@ -262,3 +435,36 @@ These functions are usually used in the module initialization function.
.. c:function:: int PyModule_AddStringMacro(PyObject *module, macro)
Add a string constant to *module*.
+
+
+Module lookup
+^^^^^^^^^^^^^
+
+Single-phase initialization creates singleton modules that can be looked up
+in the context of the current interpreter. This allows the module object to be
+retrieved later with only a reference to the module definition.
+
+These functions will not work on modules created using multi-phase initialization,
+since multiple such modules can be created from a single definition.
+
+.. c:function:: PyObject* PyState_FindModule(PyModuleDef *def)
+
+ Returns the module object that was created from *def* for the current interpreter.
+ This method requires that the module object has been attached to the interpreter state with
+ :c:func:`PyState_AddModule` beforehand. In case the corresponding module object is not
+ found or has not been attached to the interpreter state yet, it returns *NULL*.
+
+.. c:function:: int PyState_AddModule(PyObject *module, PyModuleDef *def)
+
+ Attaches the module object passed to the function to the interpreter state. This allows
+ the module object to be accessible via :c:func:`PyState_FindModule`.
+
+ Only effective on modules created using single-phase initialization.
+
+ .. versionadded:: 3.3
+
+.. c:function:: int PyState_RemoveModule(PyModuleDef *def)
+
+ Removes the module object created from *def* from the interpreter state.
+
+ .. versionadded:: 3.3
diff --git a/Doc/c-api/number.rst b/Doc/c-api/number.rst
index 21951c3..9bcb649 100644
--- a/Doc/c-api/number.rst
+++ b/Doc/c-api/number.rst
@@ -30,6 +30,14 @@ Number Protocol
the equivalent of the Python expression ``o1 * o2``.
+.. c:function:: PyObject* PyNumber_MatrixMultiply(PyObject *o1, PyObject *o2)
+
+ Returns the result of matrix multiplication on *o1* and *o2*, or *NULL* on
+ failure. This is the equivalent of the Python expression ``o1 @ o2``.
+
+ .. versionadded:: 3.5
+
+
.. c:function:: PyObject* PyNumber_FloorDivide(PyObject *o1, PyObject *o2)
Return the floor of *o1* divided by *o2*, or *NULL* on failure. This is
@@ -146,6 +154,15 @@ Number Protocol
the Python statement ``o1 *= o2``.
+.. c:function:: PyObject* PyNumber_InPlaceMatrixMultiply(PyObject *o1, PyObject *o2)
+
+ Returns the result of matrix multiplication on *o1* and *o2*, or *NULL* on
+ failure. The operation is done *in-place* when *o1* supports it. This is
+ the equivalent of the Python statement ``o1 @= o2``.
+
+ .. versionadded:: 3.5
+
+
.. c:function:: PyObject* PyNumber_InPlaceFloorDivide(PyObject *o1, PyObject *o2)
Returns the mathematical floor of dividing *o1* by *o2*, or *NULL* on failure.
diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst
index 7cead07..3d83b27 100644
--- a/Doc/c-api/sys.rst
+++ b/Doc/c-api/sys.rst
@@ -47,6 +47,60 @@ Operating System Utilities
not call those functions directly! :c:type:`PyOS_sighandler_t` is a typedef
alias for :c:type:`void (\*)(int)`.
+.. c:function:: wchar_t* Py_DecodeLocale(const char* arg, size_t *size)
+
+ Decode a byte string from the locale encoding with the :ref:`surrogateescape
+ error handler <surrogateescape>`: undecodable bytes are decoded as
+ characters in range U+DC80..U+DCFF. If a byte sequence can be decoded as a
+ surrogate character, escape the bytes using the surrogateescape error
+ handler instead of decoding them.
+
+ Return a pointer to a newly allocated wide character string, use
+ :c:func:`PyMem_RawFree` to free the memory. If size is not ``NULL``, write
+ the number of wide characters excluding the null character into ``*size``
+
+ Return ``NULL`` on decoding error or memory allocation error. If *size* is
+ not ``NULL``, ``*size`` is set to ``(size_t)-1`` on memory error or set to
+ ``(size_t)-2`` on decoding error.
+
+ Decoding errors should never happen, unless there is a bug in the C
+ library.
+
+ Use the :c:func:`Py_EncodeLocale` function to encode the character string
+ back to a byte string.
+
+ .. seealso::
+
+ The :c:func:`PyUnicode_DecodeFSDefaultAndSize` and
+ :c:func:`PyUnicode_DecodeLocaleAndSize` functions.
+
+ .. versionadded:: 3.5
+
+
+.. c:function:: char* Py_EncodeLocale(const wchar_t *text, size_t *error_pos)
+
+ Encode a wide character string to the locale encoding with the
+ :ref:`surrogateescape error handler <surrogateescape>`: surrogate characters
+ in the range U+DC80..U+DCFF are converted to bytes 0x80..0xFF.
+
+ Return a pointer to a newly allocated byte string, use :c:func:`PyMem_Free`
+ to free the memory. Return ``NULL`` on encoding error or memory allocation
+ error
+
+ If error_pos is not ``NULL``, ``*error_pos`` is set to the index of the
+ invalid character on encoding error, or set to ``(size_t)-1`` otherwise.
+
+ Use the :c:func:`Py_DecodeLocale` function to decode the bytes string back
+ to a wide character string.
+
+ .. seealso::
+
+ The :c:func:`PyUnicode_EncodeFSDefault` and
+ :c:func:`PyUnicode_EncodeLocale` functions.
+
+ .. versionadded:: 3.5
+
+
.. _systemfunctions:
System Functions
diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst
index 5de8be0..b5113aa 100644
--- a/Doc/c-api/typeobj.rst
+++ b/Doc/c-api/typeobj.rst
@@ -220,9 +220,14 @@ type objects) *must* have the :attr:`ob_size` field.
the subtype's :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` are both *NULL*.
-.. c:member:: void* PyTypeObject.tp_reserved
+.. c:member:: PyAsyncMethods* tp_as_async
- Reserved slot, formerly known as tp_compare.
+ Pointer to an additional structure that contains fields relevant only to
+ objects which implement :term:`awaitable` and :term:`asynchronous iterator`
+ protocols at the C-level. See :ref:`async-structs` for details.
+
+ .. versionadded:: 3.5
+ Formerly known as ``tp_compare`` and ``tp_reserved``.
.. c:member:: reprfunc PyTypeObject.tp_repr
@@ -1118,6 +1123,9 @@ Number Object Structures
binaryfunc nb_inplace_true_divide;
unaryfunc nb_index;
+
+ binaryfunc nb_matrix_multiply;
+ binaryfunc nb_inplace_matrix_multiply;
} PyNumberMethods;
.. note::
@@ -1329,3 +1337,58 @@ Buffer Object Structures
:c:func:`PyBuffer_Release` is the interface for the consumer that
wraps this function.
+
+
+.. _async-structs:
+
+
+Async Object Structures
+=======================
+
+.. sectionauthor:: Yury Selivanov <yselivanov@sprymix.com>
+
+.. versionadded:: 3.5
+
+.. c:type:: PyAsyncMethods
+
+ This structure holds pointers to the functions required to implement
+ :term:`awaitable` and :term:`asynchronous iterator` objects.
+
+ Here is the structure definition::
+
+ typedef struct {
+ unaryfunc am_await;
+ unaryfunc am_aiter;
+ unaryfunc am_anext;
+ } PyAsyncMethods;
+
+.. c:member:: unaryfunc PyAsyncMethods.am_await
+
+ The signature of this function is::
+
+ PyObject *am_await(PyObject *self)
+
+ The returned object must be an iterator, i.e. :c:func:`PyIter_Check` must
+ return ``1`` for it.
+
+ This slot may be set to *NULL* if an object is not an :term:`awaitable`.
+
+.. c:member:: unaryfunc PyAsyncMethods.am_aiter
+
+ The signature of this function is::
+
+ PyObject *am_aiter(PyObject *self)
+
+ Must return an :term:`awaitable` object. See :meth:`__anext__` for details.
+
+ This slot may be set to *NULL* if an object does not implement
+ asynchronous iteration protocol.
+
+.. c:member:: unaryfunc PyAsyncMethods.am_anext
+
+ The signature of this function is::
+
+ PyObject *am_anext(PyObject *self)
+
+ Must return an :term:`awaitable` object. See :meth:`__anext__` for details.
+ This slot may be set to *NULL*.
diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst
index f7e99d6..2eeadb5 100644
--- a/Doc/c-api/unicode.rst
+++ b/Doc/c-api/unicode.rst
@@ -765,11 +765,13 @@ system.
*errors* is ``NULL``. *str* must end with a null character but
cannot contain embedded null characters.
+ Use :c:func:`PyUnicode_DecodeFSDefaultAndSize` to decode a string from
+ :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at
+ Python startup).
+
.. seealso::
- Use :c:func:`PyUnicode_DecodeFSDefaultAndSize` to decode a string from
- :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at
- Python startup).
+ The :c:func:`Py_DecodeLocale` function.
.. versionadded:: 3.3
@@ -790,11 +792,13 @@ system.
*errors* is ``NULL``. Return a :class:`bytes` object. *str* cannot
contain embedded null characters.
+ Use :c:func:`PyUnicode_EncodeFSDefault` to encode a string to
+ :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at
+ Python startup).
+
.. seealso::
- Use :c:func:`PyUnicode_EncodeFSDefault` to encode a string to
- :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at
- Python startup).
+ The :c:func:`Py_EncodeLocale` function.
.. versionadded:: 3.3
@@ -839,12 +843,14 @@ used, passing :c:func:`PyUnicode_FSDecoder` as the conversion function:
If :c:data:`Py_FileSystemDefaultEncoding` is not set, fall back to the
locale encoding.
+ :c:data:`Py_FileSystemDefaultEncoding` is initialized at startup from the
+ locale encoding and cannot be modified later. If you need to decode a string
+ from the current locale encoding, use
+ :c:func:`PyUnicode_DecodeLocaleAndSize`.
+
.. seealso::
- :c:data:`Py_FileSystemDefaultEncoding` is initialized at startup from the
- locale encoding and cannot be modified later. If you need to decode a
- string from the current locale encoding, use
- :c:func:`PyUnicode_DecodeLocaleAndSize`.
+ The :c:func:`Py_DecodeLocale` function.
.. versionchanged:: 3.2
Use ``"strict"`` error handler on Windows.
@@ -874,12 +880,13 @@ used, passing :c:func:`PyUnicode_FSDecoder` as the conversion function:
If :c:data:`Py_FileSystemDefaultEncoding` is not set, fall back to the
locale encoding.
+ :c:data:`Py_FileSystemDefaultEncoding` is initialized at startup from the
+ locale encoding and cannot be modified later. If you need to encode a string
+ to the current locale encoding, use :c:func:`PyUnicode_EncodeLocale`.
+
.. seealso::
- :c:data:`Py_FileSystemDefaultEncoding` is initialized at startup from the
- locale encoding and cannot be modified later. If you need to encode a
- string to the current locale encoding, use
- :c:func:`PyUnicode_EncodeLocale`.
+ The :c:func:`Py_EncodeLocale` function.
.. versionadded:: 3.2
diff --git a/Doc/conf.py b/Doc/conf.py
index f803de2..28dd80f 100644
--- a/Doc/conf.py
+++ b/Doc/conf.py
@@ -36,6 +36,9 @@ highlight_language = 'python3'
# Require Sphinx 1.2 for build.
needs_sphinx = '1.2'
+# Ignore any .rst files in the venv/ directory.
+exclude_patterns = ['venv/*']
+
# Options for HTML output
# -----------------------
diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat
index 6025617..e388195 100644
--- a/Doc/data/refcounts.dat
+++ b/Doc/data/refcounts.dat
@@ -349,6 +349,11 @@ PyErr_Format:PyObject*:exception:+1:
PyErr_Format:const char*:format::
PyErr_Format::...::
+PyErr_FormatV:PyObject*::null:
+PyErr_FormatV:PyObject*:exception:+1:
+PyErr_FormatV:const char*:format::
+PyErr_FormatV:va_list:vargs::
+
PyErr_WarnEx:int:::
PyErr_WarnEx:PyObject*:category:0:
PyErr_WarnEx:const char*:message::
@@ -486,6 +491,12 @@ PyFunction_SetDefaults:PyObject*:defaults:+1:
PyGen_New:PyObject*::+1:
PyGen_New:PyFrameObject*:frame:0:
+PyGen_NewWithQualName:PyObject*::+1:
+PyGen_NewWithQualName:PyFrameObject*:frame:0:
+
+PyCoro_New:PyObject*::+1:
+PyCoro_New:PyFrameObject*:frame:0:
+
Py_InitModule:PyObject*::0:
Py_InitModule:const char*:name::
Py_InitModule:PyMethodDef[]:methods::
diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst
index 53d7527..d0f8947 100644
--- a/Doc/distutils/apiref.rst
+++ b/Doc/distutils/apiref.rst
@@ -868,23 +868,31 @@ tarballs or zipfiles.
Create an archive file (eg. ``zip`` or ``tar``). *base_name* is the name of
the file to create, minus any format-specific extension; *format* is the
- archive format: one of ``zip``, ``tar``, ``ztar``, or ``gztar``. *root_dir* is
- a directory that will be the root directory of the archive; ie. we typically
- ``chdir`` into *root_dir* before creating the archive. *base_dir* is the
- directory where we start archiving from; ie. *base_dir* will be the common
- prefix of all files and directories in the archive. *root_dir* and *base_dir*
- both default to the current directory. Returns the name of the archive file.
+ archive format: one of ``zip``, ``tar``, ``gztar``, ``bztar``, ``xztar``, or
+ ``ztar``. *root_dir* is a directory that will be the root directory of the
+ archive; ie. we typically ``chdir`` into *root_dir* before creating the
+ archive. *base_dir* is the directory where we start archiving from; ie.
+ *base_dir* will be the common prefix of all files and directories in the
+ archive. *root_dir* and *base_dir* both default to the current directory.
+ Returns the name of the archive file.
+
+ .. versionchanged: 3.5
+ Added support for the ``xztar`` format.
.. function:: make_tarball(base_name, base_dir[, compress='gzip', verbose=0, dry_run=0])
'Create an (optional compressed) archive as a tar file from all files in and
- under *base_dir*. *compress* must be ``'gzip'`` (the default), ``'compress'``,
- ``'bzip2'``, or ``None``. Both :program:`tar` and the compression utility named
- by *compress* must be on the default program search path, so this is probably
- Unix-specific. The output tar file will be named :file:`base_dir.tar`,
- possibly plus the appropriate compression extension (:file:`.gz`, :file:`.bz2`
- or :file:`.Z`). Return the output filename.
+ under *base_dir*. *compress* must be ``'gzip'`` (the default),
+ ``'bzip2'``, ``'xz'``, ``'compress'``, or ``None``. For the ``'compress'``
+ method the compression utility named by :program:`compress` must be on the
+ default program search path, so this is probably Unix-specific. The output
+ tar file will be named :file:`base_dir.tar`, possibly plus the appropriate
+ compression extension (``.gz``, ``.bz2``, ``.xz`` or ``.Z``). Return the
+ output filename.
+
+ .. versionchanged: 3.5
+ Added support for the ``xz`` compression.
.. function:: make_zipfile(base_name, base_dir[, verbose=0, dry_run=0])
@@ -1193,12 +1201,12 @@ other utility module.
.. function:: byte_compile(py_files[, optimize=0, force=0, prefix=None, base_dir=None, verbose=1, dry_run=0, direct=None])
- Byte-compile a collection of Python source files to either :file:`.pyc` or
- :file:`.pyo` files in a :file:`__pycache__` subdirectory (see :pep:`3147`).
+ Byte-compile a collection of Python source files to :file:`.pyc` files in a
+ :file:`__pycache__` subdirectory (see :pep:`3147` and :pep:`488`).
*py_files* is a list of files to compile; any files that don't end in
:file:`.py` are silently skipped. *optimize* must be one of the following:
- * ``0`` - don't optimize (generate :file:`.pyc`)
+ * ``0`` - don't optimize
* ``1`` - normal optimization (like ``python -O``)
* ``2`` - extra optimization (like ``python -OO``)
@@ -1222,10 +1230,13 @@ other utility module.
doing, leave it set to ``None``.
.. versionchanged:: 3.2.3
- Create ``.pyc`` or ``.pyo`` files with an :func:`import magic tag
+ Create ``.pyc`` files with an :func:`import magic tag
<imp.get_tag>` in their name, in a :file:`__pycache__` subdirectory
instead of files without tag in the current directory.
+ .. versionchanged: 3.5
+ Create ``.pyc`` files according to :pep:`488`.
+
.. function:: rfc822_escape(header)
diff --git a/Doc/distutils/builtdist.rst b/Doc/distutils/builtdist.rst
index c5827b6..523d1e0 100644
--- a/Doc/distutils/builtdist.rst
+++ b/Doc/distutils/builtdist.rst
@@ -72,13 +72,19 @@ The available formats for built distributions are:
+-------------+------------------------------+---------+
| Format | Description | Notes |
+=============+==============================+=========+
-| ``gztar`` | gzipped tar file | (1),(3) |
+| ``gztar`` | gzipped tar file | \(1) |
| | (:file:`.tar.gz`) | |
+-------------+------------------------------+---------+
+| ``bztar`` | bzipped tar file | |
+| | (:file:`.tar.bz2`) | |
++-------------+------------------------------+---------+
+| ``xztar`` | xzipped tar file | |
+| | (:file:`.tar.xz`) | |
++-------------+------------------------------+---------+
| ``ztar`` | compressed tar file | \(3) |
| | (:file:`.tar.Z`) | |
+-------------+------------------------------+---------+
-| ``tar`` | tar file (:file:`.tar`) | \(3) |
+| ``tar`` | tar file (:file:`.tar`) | |
+-------------+------------------------------+---------+
| ``zip`` | zip file (:file:`.zip`) | (2),(4) |
+-------------+------------------------------+---------+
@@ -94,6 +100,9 @@ The available formats for built distributions are:
| ``msi`` | Microsoft Installer. | |
+-------------+------------------------------+---------+
+.. versionchanged:: 3.5
+ Added support for the ``xztar`` format.
+
Notes:
@@ -104,8 +113,7 @@ Notes:
default on Windows
(3)
- requires external utilities: :program:`tar` and possibly one of :program:`gzip`,
- :program:`bzip2`, or :program:`compress`
+ requires external :program:`compress` utility.
(4)
requires either external :program:`zip` utility or :mod:`zipfile` module (part
@@ -119,21 +127,22 @@ You don't have to use the :command:`bdist` command with the :option:`--formats`
option; you can also use the command that directly implements the format you're
interested in. Some of these :command:`bdist` "sub-commands" actually generate
several similar formats; for instance, the :command:`bdist_dumb` command
-generates all the "dumb" archive formats (``tar``, ``ztar``, ``gztar``, and
-``zip``), and :command:`bdist_rpm` generates both binary and source RPMs. The
-:command:`bdist` sub-commands, and the formats generated by each, are:
-
-+--------------------------+-----------------------+
-| Command | Formats |
-+==========================+=======================+
-| :command:`bdist_dumb` | tar, ztar, gztar, zip |
-+--------------------------+-----------------------+
-| :command:`bdist_rpm` | rpm, srpm |
-+--------------------------+-----------------------+
-| :command:`bdist_wininst` | wininst |
-+--------------------------+-----------------------+
-| :command:`bdist_msi` | msi |
-+--------------------------+-----------------------+
+generates all the "dumb" archive formats (``tar``, ``gztar``, ``bztar``,
+``xztar``, ``ztar``, and ``zip``), and :command:`bdist_rpm` generates both
+binary and source RPMs. The :command:`bdist` sub-commands, and the formats
+generated by each, are:
+
++--------------------------+-------------------------------------+
+| Command | Formats |
++==========================+=====================================+
+| :command:`bdist_dumb` | tar, gztar, bztar, xztar, ztar, zip |
++--------------------------+-------------------------------------+
+| :command:`bdist_rpm` | rpm, srpm |
++--------------------------+-------------------------------------+
+| :command:`bdist_wininst` | wininst |
++--------------------------+-------------------------------------+
+| :command:`bdist_msi` | msi |
++--------------------------+-------------------------------------+
The following sections give details on the individual :command:`bdist_\*`
commands.
diff --git a/Doc/distutils/introduction.rst b/Doc/distutils/introduction.rst
index 0ece646..8f46bd7 100644
--- a/Doc/distutils/introduction.rst
+++ b/Doc/distutils/introduction.rst
@@ -156,8 +156,8 @@ module
pure Python module
a module written in Python and contained in a single :file:`.py` file (and
- possibly associated :file:`.pyc` and/or :file:`.pyo` files). Sometimes referred
- to as a "pure module."
+ possibly associated :file:`.pyc` files). Sometimes referred to as a
+ "pure module."
extension module
a module written in the low-level language of the Python implementation: C/C++
@@ -210,5 +210,3 @@ distribution root
the top-level directory of your source tree (or source distribution); the
directory where :file:`setup.py` exists. Generally :file:`setup.py` will be
run from this directory.
-
-
diff --git a/Doc/distutils/sourcedist.rst b/Doc/distutils/sourcedist.rst
index b9f0cc8..fb70514 100644
--- a/Doc/distutils/sourcedist.rst
+++ b/Doc/distutils/sourcedist.rst
@@ -32,12 +32,18 @@ to create a gzipped tarball and a zip file. The available formats are:
| ``bztar`` | bzip2'ed tar file | |
| | (:file:`.tar.bz2`) | |
+-----------+-------------------------+---------+
+| ``xztar`` | xz'ed tar file | |
+| | (:file:`.tar.xz`) | |
++-----------+-------------------------+---------+
| ``ztar`` | compressed tar file | \(4) |
| | (:file:`.tar.Z`) | |
+-----------+-------------------------+---------+
| ``tar`` | tar file (:file:`.tar`) | |
+-----------+-------------------------+---------+
+.. versionchanged:: 3.5
+ Added support for the ``xztar`` format.
+
Notes:
(1)
@@ -54,7 +60,7 @@ Notes:
requires the :program:`compress` program. Notice that this format is now
pending for deprecation and will be removed in the future versions of Python.
-When using any ``tar`` format (``gztar``, ``bztar``, ``ztar`` or
+When using any ``tar`` format (``gztar``, ``bztar``, ``xztar``, ``ztar`` or
``tar``), under Unix you can specify the ``owner`` and ``group`` names
that will be set for each member of the archive.
diff --git a/Doc/extending/building.rst b/Doc/extending/building.rst
index 656af88..163179d 100644
--- a/Doc/extending/building.rst
+++ b/Doc/extending/building.rst
@@ -1,27 +1,58 @@
.. highlightlang:: c
-
.. _building:
-********************************************
-Building C and C++ Extensions with distutils
-********************************************
+*****************************
+Building C and C++ Extensions
+*****************************
-.. sectionauthor:: Martin v. Löwis <martin@v.loewis.de>
+A C extension for CPython is a shared library (e.g. a ``.so`` file on Linux,
+``.pyd`` on Windows), which exports an *initialization function*.
+
+To be importable, the shared library must be available on :envvar:`PYTHONPATH`,
+and must be named after the module name, with an appropriate extension.
+When using distutils, the correct filename is generated automatically.
+
+The initialization function has the signature:
+
+.. c:function:: PyObject* PyInit_modulename(void)
+
+It returns either a fully-initialized module, or a :c:type:`PyModuleDef`
+instance. See :ref:`initializing-modules` for details.
+
+.. highlightlang:: python
+For modules with ASCII-only names, the function must be named
+``PyInit_<modulename>``, with ``<modulename>`` replaced by the name of the
+module. When using :ref:`multi-phase-initialization`, non-ASCII module names
+are allowed. In this case, the initialization function name is
+``PyInitU_<modulename>``, with ``<modulename>`` encoded using Python's
+*punycode* encoding with hyphens replaced by underscores. In Python::
-Starting in Python 1.4, Python provides, on Unix, a special make file for
-building make files for building dynamically-linked extensions and custom
-interpreters. Starting with Python 2.0, this mechanism (known as related to
-Makefile.pre.in, and Setup files) is no longer supported. Building custom
-interpreters was rarely used, and extension modules can be built using
-distutils.
+ def initfunc_name(name):
+ try:
+ suffix = b'_' + name.encode('ascii')
+ except UnicodeEncodeError:
+ suffix = b'U_' + name.encode('punycode').replace(b'-', b'_')
+ return b'PyInit' + suffix
+
+It is possible to export multiple modules from a single shared library by
+defining multiple initialization functions. However, importing them requires
+using symbolic links or a custom importer, because by default only the
+function corresponding to the filename is found.
+See :PEP:`489#multiple-modules-in-one-library` for details.
+
+
+.. highlightlang:: c
+
+Building C and C++ Extensions with distutils
+============================================
+
+.. sectionauthor:: Martin v. Löwis <martin@v.loewis.de>
-Building an extension module using distutils requires that distutils is
-installed on the build machine, which is included in Python 2.x and available
-separately for Python 1.5. Since distutils also supports creation of binary
-packages, users don't necessarily need a compiler and distutils to install the
-extension.
+Extension modules can be built using distutils, which is included in Python.
+Since distutils also supports creation of binary packages, users don't
+necessarily need a compiler and distutils to install the extension.
A distutils package contains a driver script, :file:`setup.py`. This is a plain
Python file, which, in the most simple case, could look like this::
diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst
index 6cb686a..acd60ae 100644
--- a/Doc/extending/embedding.rst
+++ b/Doc/extending/embedding.rst
@@ -58,12 +58,18 @@ perform some operation on a file. ::
int
main(int argc, char *argv[])
{
- Py_SetProgramName(argv[0]); /* optional but recommended */
- Py_Initialize();
- PyRun_SimpleString("from time import time,ctime\n"
- "print('Today is', ctime(time()))\n");
- Py_Finalize();
- return 0;
+ wchar_t *program = Py_DecodeLocale(argv[0], NULL);
+ if (program == NULL) {
+ fprintf(stderr, "Fatal error: cannot decode argv[0]\n");
+ exit(1);
+ }
+ Py_SetProgramName(program); /* optional but recommended */
+ Py_Initialize();
+ PyRun_SimpleString("from time import time,ctime\n"
+ "print('Today is', ctime(time()))\n");
+ Py_Finalize();
+ PyMem_RawFree(program);
+ return 0;
}
The :c:func:`Py_SetProgramName` function should be called before
@@ -160,7 +166,7 @@ for data conversion between Python and C, and for error reporting. The
interesting part with respect to embedding Python starts with ::
Py_Initialize();
- pName = PyUnicode_FromString(argv[1]);
+ pName = PyUnicode_DecodeFSDefault(argv[1]);
/* Error checking of pName left out */
pModule = PyImport_Import(pName);
diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst
index ba6bfa7..8cc4184 100644
--- a/Doc/extending/extending.rst
+++ b/Doc/extending/extending.rst
@@ -375,11 +375,17 @@ optionally followed by an import of the module::
int
main(int argc, char *argv[])
{
+ wchar_t *program = Py_DecodeLocale(argv[0], NULL);
+ if (program == NULL) {
+ fprintf(stderr, "Fatal error: cannot decode argv[0]\n");
+ exit(1);
+ }
+
/* Add a built-in module, before Py_Initialize */
PyImport_AppendInittab("spam", PyInit_spam);
/* Pass argv[0] to the Python interpreter */
- Py_SetProgramName(argv[0]);
+ Py_SetProgramName(program);
/* Initialize the Python interpreter. Required. */
Py_Initialize();
@@ -391,6 +397,10 @@ optionally followed by an import of the module::
...
+ PyMem_RawFree(program);
+ return 0;
+ }
+
.. note::
Removing entries from ``sys.modules`` or importing compiled modules into
@@ -403,6 +413,13 @@ A more substantial example module is included in the Python source distribution
as :file:`Modules/xxmodule.c`. This file may be used as a template or simply
read as an example.
+.. note::
+
+ Unlike our ``spam`` example, ``xxmodule`` uses *multi-phase initialization*
+ (new in Python 3.5), where a PyModuleDef structure is returned from
+ ``PyInit_spam``, and creation of the module is left to the import machinery.
+ For details on multi-phase initialization, see :PEP:`489`.
+
.. _compilation:
diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst
index 0884430..f60e208 100644
--- a/Doc/extending/newtypes.rst
+++ b/Doc/extending/newtypes.rst
@@ -80,7 +80,7 @@ Moving on, we come to the crunch --- the type object. ::
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
- 0, /* tp_reserved */
+ 0, /* tp_as_async */
0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
diff --git a/Doc/extending/windows.rst b/Doc/extending/windows.rst
index 3fd5e57..f0c69b8 100644
--- a/Doc/extending/windows.rst
+++ b/Doc/extending/windows.rst
@@ -98,9 +98,8 @@ described here are distributed with the Python sources in the
it. Copy your C sources into it. Note that the module source file name does
not necessarily have to match the module name, but the name of the
initialization function should match the module name --- you can only import a
- module :mod:`spam` if its initialization function is called :c:func:`initspam`,
- and it should call :c:func:`Py_InitModule` with the string ``"spam"`` as its
- first argument (use the minimal :file:`example.c` in this directory as a guide).
+ module :mod:`spam` if its initialization function is called :c:func:`PyInit_spam`,
+ (see :ref:`building`, or use the minimal :file:`Modules/xxmodule.c` as a guide).
By convention, it lives in a file called :file:`spam.c` or :file:`spammodule.c`.
The output file should be called :file:`spam.pyd` (in Release mode) or
:file:`spam_d.pyd` (in Debug mode). The extension :file:`.pyd` was chosen
diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index 1de86ef..846c01a 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -69,11 +69,34 @@ Glossary
:ref:`the difference between arguments and parameters
<faq-argument-vs-parameter>`, and :pep:`362`.
+ asynchronous context manager
+ An object which controls the environment seen in an
+ :keyword:`async with` statement by defining :meth:`__aenter__` and
+ :meth:`__aexit__` methods. Introduced by :pep:`492`.
+
+ asynchronous iterable
+ An object, that can be used in an :keyword:`async for` statement.
+ Must return an :term:`awaitable` from its :meth:`__aiter__` method,
+ which should in turn be resolved in an :term:`asynchronous iterator`
+ object. Introduced by :pep:`492`.
+
+ asynchronous iterator
+ An object that implements :meth:`__aiter__` and :meth:`__anext__`
+ methods, that must return :term:`awaitable` objects.
+ :keyword:`async for` resolves awaitable returned from asynchronous
+ iterator's :meth:`__anext__` method until it raises
+ :exc:`StopAsyncIteration` exception. Introduced by :pep:`492`.
+
attribute
A value associated with an object which is referenced by name using
dotted expressions. For example, if an object *o* has an attribute
*a* it would be referenced as *o.a*.
+ awaitable
+ An object that can be used in an :keyword:`await` expression. Can be
+ a :term:`coroutine` or an object with an :meth:`__await__` method.
+ See also :pep:`492`.
+
BDFL
Benevolent Dictator For Life, a.k.a. `Guido van Rossum
<https://www.python.org/~guido/>`_, Python's creator.
@@ -86,12 +109,21 @@ Glossary
A :term:`text file` reads and writes :class:`str` objects.
bytes-like object
- An object that supports the :ref:`bufferobjects`, like :class:`bytes`,
- :class:`bytearray` or :class:`memoryview`. Bytes-like objects can
- be used for various operations that expect binary data, such as
- compression, saving to a binary file or sending over a socket.
- Some operations need the binary data to be mutable, in which case
- not all bytes-like objects can apply.
+ An object that supports the :ref:`bufferobjects` and can
+ export a C-:term:`contiguous` buffer. This includes all :class:`bytes`,
+ :class:`bytearray`, and :class:`array.array` objects, as well as many
+ common :class:`memoryview` objects. Bytes-like objects can
+ be used for various operations that work with binary data; these include
+ compression, saving to a binary file, and sending over a socket.
+
+ Some operations need the binary data to be mutable. The documentation
+ often refers to these as "read-write bytes-like objects". Example
+ mutable buffer objects include :class:`bytearray` and a
+ :class:`memoryview` of a :class:`bytearray`.
+ Other operations require the binary data to be stored in
+ immutable objects ("read-only bytes-like objects"); examples
+ of these include :class:`bytes` and a :class:`memoryview`
+ of a :class:`bytes` object.
bytecode
Python source code is compiled into bytecode, the internal representation
@@ -139,6 +171,32 @@ Glossary
statement by defining :meth:`__enter__` and :meth:`__exit__` methods.
See :pep:`343`.
+ contiguous
+ .. index:: C-contiguous, Fortran contiguous
+
+ A buffer is considered contiguous exactly if it is either
+ *C-contiguous* or *Fortran contiguous*. Zero-dimensional buffers are
+ C and Fortran contiguous. In one-dimensional arrays, the items
+ must be layed out in memory next to each other, in order of
+ increasing indexes starting from zero. In multidimensional
+ C-contiguous arrays, the last index varies the fastest when
+ visiting items in order of memory address. However, in
+ Fortran contiguous arrays, the first index varies the fastest.
+
+ coroutine
+ Coroutines is a more generalized form of subroutines. Subroutines are
+ entered at one point and exited at another point. Coroutines can be
+ entered, exited, and resumed at many different points. They can be
+ implemented with the :keyword:`async def` statement. See also
+ :pep:`492`.
+
+ coroutine function
+ A function which returns a :term:`coroutine` object. A coroutine
+ function may be defined with the :keyword:`async def` statement,
+ and may contain :keyword:`await`, :keyword:`async for`, and
+ :keyword:`async with` keywords. These were introduced
+ by :pep:`492`.
+
CPython
The canonical implementation of the Python programming language, as
distributed on `python.org <https://www.python.org>`_. The term "CPython"
@@ -290,14 +348,23 @@ Glossary
.. index:: single: generator
generator
- A function which returns an iterator. It looks like a normal function
- except that it contains :keyword:`yield` statements for producing a series
- of values usable in a for-loop or that can be retrieved one at a time with
- the :func:`next` function. Each :keyword:`yield` temporarily suspends
- processing, remembering the location execution state (including local
- variables and pending try-statements). When the generator resumes, it
- picks-up where it left-off (in contrast to functions which start fresh on
- every invocation).
+ A function which returns a :term:`generator iterator`. It looks like a
+ normal function except that it contains :keyword:`yield` expressions
+ for producing a series of values usable in a for-loop or that can be
+ retrieved one at a time with the :func:`next` function.
+
+ Usually refers to a generator function, but may refer to a
+ *generator iterator* in some contexts. In cases where the intended
+ meaning isn't clear, using the full terms avoids ambiguity.
+
+ generator iterator
+ An object created by a :term:`generator` function.
+
+ Each :keyword:`yield` temporarily suspends processing, remembering the
+ location execution state (including local variables and pending
+ try-statements). When the *generator iterator* resumes, it picks-up where
+ it left-off (in contrast to functions which start fresh on every
+ invocation).
.. index:: single: generator expression
@@ -402,6 +469,19 @@ Glossary
than compiled ones, though their programs generally also run more
slowly. See also :term:`interactive`.
+ interpreter shutdown
+ When asked to shut down, the Python interpreter enters a special phase
+ where it gradually releases all allocated resources, such as modules
+ and various critical internal structures. It also makes several calls
+ to the :term:`garbage collector <garbage collection>`. This can trigger
+ the execution of code in user-defined destructors or weakref callbacks.
+ Code executed during the shutdown phase can encounter various
+ exceptions as the resources it relies on may not function anymore
+ (common examples are library modules or the warnings machinery).
+
+ The main reason for interpreter shutdown is that the ``__main__`` module
+ or the script being run has finished executing.
+
iterable
An object capable of returning its members one at a time. Examples of
iterables include all sequence types (such as :class:`list`, :class:`str`,
@@ -444,12 +524,13 @@ Glossary
A number of tools in Python accept key functions to control how elements
are ordered or grouped. They include :func:`min`, :func:`max`,
- :func:`sorted`, :meth:`list.sort`, :func:`heapq.nsmallest`,
- :func:`heapq.nlargest`, and :func:`itertools.groupby`.
+ :func:`sorted`, :meth:`list.sort`, :func:`heapq.merge`,
+ :func:`heapq.nsmallest`, :func:`heapq.nlargest`, and
+ :func:`itertools.groupby`.
There are several ways to create a key function. For example. the
:meth:`str.lower` method can serve as a key function for case insensitive
- sorts. Alternatively, an ad-hoc key function can be built from a
+ sorts. Alternatively, a key function can be built from a
:keyword:`lambda` expression such as ``lambda r: (r[0], r[2])``. Also,
the :mod:`operator` module provides three key function constructors:
:func:`~operator.attrgetter`, :func:`~operator.itemgetter`, and
diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst
index e362631..7524c4a 100644
--- a/Doc/howto/clinic.rst
+++ b/Doc/howto/clinic.rst
@@ -758,6 +758,14 @@ All Argument Clinic converters accept the following arguments:
In addition, some converters accept additional arguments. Here is a list
of these arguments, along with their meanings:
+ ``accept``
+ A set of Python types (and possibly pseudo-types);
+ this restricts the allowable Python argument to values of these types.
+ (This is not a general-purpose facility; as a rule it only supports
+ specific lists of types as shown in the legacy converter table.)
+
+ To accept ``None``, add ``NoneType`` to this set.
+
``bitwise``
Only supported for unsigned integers. The native integer value of this
Python argument will be written to the parameter without any range checking,
@@ -772,39 +780,27 @@ of these arguments, along with their meanings:
Only supported for strings. Specifies the encoding to use when converting
this string from a Python str (Unicode) value into a C ``char *`` value.
- ``length``
- Only supported for strings. If true, requests that the length of the
- string be passed in to the impl function, just after the string parameter,
- in a parameter named ``<parameter_name>_length``.
-
- ``nullable``
- Only supported for strings. If true, this parameter may also be set to
- ``None``, in which case the C parameter will be set to ``NULL``.
``subclass_of``
Only supported for the ``object`` converter. Requires that the Python
value be a subclass of a Python type, as expressed in C.
- ``types``
- Only supported for the ``object`` (and ``self``) converter. Specifies
+ ``type``
+ Only supported for the ``object`` and ``self`` converters. Specifies
the C type that will be used to declare the variable. Default value is
``"PyObject *"``.
- ``types``
- A string containing a list of Python types (and possibly pseudo-types);
- this restricts the allowable Python argument to values of these types.
- (This is not a general-purpose facility; as a rule it only supports
- specific lists of types as shown in the legacy converter table.)
-
``zeroes``
Only supported for strings. If true, embedded NUL bytes (``'\\0'``) are
- permitted inside the value.
+ permitted inside the value. The length of the string will be passed in
+ to the impl function, just after the string parameter, as a parameter named
+ ``<parameter_name>_length``.
Please note, not every possible combination of arguments will work.
-Often these arguments are implemented internally by specific ``PyArg_ParseTuple``
+Usually these arguments are implemented by specific ``PyArg_ParseTuple``
*format units*, with specific behavior. For example, currently you cannot
-call ``str`` and pass in ``zeroes=True`` without also specifying an ``encoding``;
-although it's perfectly reasonable to think this would work, these semantics don't
+call ``unsigned_short`` without also specifying ``bitwise=True``.
+Although it's perfectly reasonable to think this would work, these semantics don't
map to any existing format unit. So Argument Clinic doesn't support it. (Or, at
least, not yet.)
@@ -816,13 +812,13 @@ on the right is the text you'd replace it with.
``'B'`` ``unsigned_char(bitwise=True)``
``'b'`` ``unsigned_char``
``'c'`` ``char``
-``'C'`` ``int(types='str')``
+``'C'`` ``int(accept={str})``
``'d'`` ``double``
``'D'`` ``Py_complex``
-``'es#'`` ``str(encoding='name_of_encoding', length=True, zeroes=True)``
``'es'`` ``str(encoding='name_of_encoding')``
-``'et#'`` ``str(encoding='name_of_encoding', types='bytes bytearray str', length=True)``
-``'et'`` ``str(encoding='name_of_encoding', types='bytes bytearray str')``
+``'es#'`` ``str(encoding='name_of_encoding', zeroes=True)``
+``'et'`` ``str(encoding='name_of_encoding', accept={bytes, bytearray, str})``
+``'et#'`` ``str(encoding='name_of_encoding', accept={bytes, bytearray, str}, zeroes=True)``
``'f'`` ``float``
``'h'`` ``short``
``'H'`` ``unsigned_short(bitwise=True)``
@@ -830,29 +826,30 @@ on the right is the text you'd replace it with.
``'I'`` ``unsigned_int(bitwise=True)``
``'k'`` ``unsigned_long(bitwise=True)``
``'K'`` ``unsigned_PY_LONG_LONG(bitwise=True)``
+``'l'`` ``long``
``'L'`` ``PY_LONG_LONG``
``'n'`` ``Py_ssize_t``
+``'O'`` ``object``
``'O!'`` ``object(subclass_of='&PySomething_Type')``
``'O&'`` ``object(converter='name_of_c_function')``
-``'O'`` ``object``
``'p'`` ``bool``
-``'s#'`` ``str(length=True)``
``'S'`` ``PyBytesObject``
``'s'`` ``str``
-``'s*'`` ``Py_buffer(types='str bytes bytearray buffer')``
-``'u#'`` ``Py_UNICODE(length=True)``
-``'u'`` ``Py_UNICODE``
+``'s#'`` ``str(zeroes=True)``
+``'s*'`` ``Py_buffer(accept={buffer, str})``
``'U'`` ``unicode``
-``'w*'`` ``Py_buffer(types='bytearray rwbuffer')``
-``'y#'`` ``str(types='bytes', length=True)``
+``'u'`` ``Py_UNICODE``
+``'u#'`` ``Py_UNICODE(zeroes=True)``
+``'w*'`` ``Py_buffer(accept={rwbuffer})``
``'Y'`` ``PyByteArrayObject``
-``'y'`` ``str(types='bytes')``
+``'y'`` ``str(accept={bytes})``
+``'y#'`` ``str(accept={robuffer}, zeroes=True)``
``'y*'`` ``Py_buffer``
-``'Z#'`` ``Py_UNICODE(nullable=True, length=True)``
-``'z#'`` ``str(nullable=True, length=True)``
-``'Z'`` ``Py_UNICODE(nullable=True)``
-``'z'`` ``str(nullable=True)``
-``'z*'`` ``Py_buffer(types='str bytes bytearray buffer', nullable=True)``
+``'Z'`` ``Py_UNICODE(accept={str, NoneType})``
+``'Z#'`` ``Py_UNICODE(accept={str, NoneType}, zeroes=True)``
+``'z'`` ``str(accept={str, NoneType})``
+``'z#'`` ``str(accept={str, NoneType}, zeroes=True)``
+``'z*'`` ``Py_buffer(accept={buffer, str, NoneType})``
========= =================================================================================
As an example, here's our sample ``pickle.Pickler.dump`` using the proper
diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst
index 1969b32..945a240 100644
--- a/Doc/howto/functional.rst
+++ b/Doc/howto/functional.rst
@@ -481,10 +481,10 @@ Here's a sample usage of the ``generate_ints()`` generator:
You could equally write ``for i in generate_ints(5)``, or ``a,b,c =
generate_ints(3)``.
-Inside a generator function, ``return value`` is semantically equivalent to
-``raise StopIteration(value)``. If no value is returned or the bottom of the
-function is reached, the procession of values ends and the generator cannot
-return any further values.
+Inside a generator function, ``return value`` causes ``StopIteration(value)``
+to be raised from the :meth:`~generator.__next__` method. Once this happens, or
+the bottom of the function is reached, the procession of values ends and the
+generator cannot yield any further values.
You could achieve the effect of generators manually by writing your own class
and storing all the local variables of the generator as instance variables. For
diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst
index 6305d24..813bc0f 100644
--- a/Doc/howto/logging-cookbook.rst
+++ b/Doc/howto/logging-cookbook.rst
@@ -325,6 +325,15 @@ which, when run, will produce::
MainThread: Look out!
+.. versionchanged:: 3.5
+ Prior to Python 3.5, the :class:`QueueListener` always passed every message
+ received from the queue to every handler it was initialized with. (This was
+ because it was assumed that level filtering was all done on the other side,
+ where the queue is filled.) From 3.5 onwards, this behaviour can be changed
+ by passing a keyword argument ``respect_handler_level=True`` to the
+ listener's constructor. When this is done, the listener compares the level
+ of each message with the handler's level, and only passes a message to a
+ handler if it's appropriate to do so.
.. _network-logging:
@@ -1680,7 +1689,7 @@ as in the following complete example::
def main():
logging.basicConfig(level=logging.INFO, format='%(message)s')
- logging.info(_('message 1', set_value=set([1, 2, 3]), snowman='\u2603'))
+ logging.info(_('message 1', set_value={1, 2, 3}, snowman='\u2603'))
if __name__ == '__main__':
main()
diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst
index 5e875cd..a2aaf36 100644
--- a/Doc/howto/pyporting.rst
+++ b/Doc/howto/pyporting.rst
@@ -207,13 +207,12 @@ that's ``str``/``bytes`` in Python 2 and ``bytes`` in Python 3). The following
table lists the **unique** methods of each data type across Python 2 & 3
(e.g., the ``decode()`` method is usable on the equivalent binary data type in
either Python 2 or 3, but it can't be used by the text data type consistently
-between Python 2 and 3 because ``str`` in Python 3 doesn't have the method).
+between Python 2 and 3 because ``str`` in Python 3 doesn't have the method). Do
+note that as of Python 3.5 the ``__mod__`` method was added to the bytes type.
======================== =====================
**Text data** **Binary data**
------------------------ ---------------------
-__mod__ (``%`` operator)
------------------------- ---------------------
\ decode
------------------------ ---------------------
encode
@@ -348,10 +347,12 @@ tox with your continuous integration system so that you never accidentally break
Python 2 or 3 support.
You may also want to use use the ``-bb`` flag with the Python 3 interpreter to
-trigger an exception when you are comparing bytes to strings. Usually it's
-simply ``False``, but if you made a mistake in your separation of text/binary
-data handling you may be accidentally comparing text and binary data. This flag
-will raise an exception when that occurs to help track down such cases.
+trigger an exception when you are comparing bytes to strings or bytes to an int
+(the latter is available starting in Python 3.5). By default type-differing
+comparisons simply return ``False``, but if you made a mistake in your
+separation of text/binary data handling or indexing on bytes you wouldn't easily
+find the mistake. This flag will raise an exception when these kinds of
+comparisons occur, making the mistake much easier to track down.
And that's mostly it! At this point your code base is compatible with both
Python 2 and 3 simultaneously. Your testing will also be set up so that you
diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst
index 9ae04d7..ad2c6ab 100644
--- a/Doc/howto/regex.rst
+++ b/Doc/howto/regex.rst
@@ -1138,7 +1138,7 @@ Empty matches are replaced only when they're not adjacent to a previous match.
If *replacement* is a string, any backslash escapes in it are processed. That
is, ``\n`` is converted to a single newline character, ``\r`` is converted to a
-carriage return, and so forth. Unknown escapes such as ``\j`` are left alone.
+carriage return, and so forth. Unknown escapes such as ``\&`` are left alone.
Backreferences, such as ``\6``, are replaced with the substring matched by the
corresponding group in the RE. This lets you incorporate portions of the
original text in the resulting replacement string.
diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst
index b49ac39..ee31a9c 100644
--- a/Doc/howto/unicode.rst
+++ b/Doc/howto/unicode.rst
@@ -280,8 +280,9 @@ and optionally an *errors* argument.
The *errors* argument specifies the response when the input string can't be
converted according to the encoding's rules. Legal values for this argument are
``'strict'`` (raise a :exc:`UnicodeDecodeError` exception), ``'replace'`` (use
-``U+FFFD``, ``REPLACEMENT CHARACTER``), or ``'ignore'`` (just leave the
-character out of the Unicode result).
+``U+FFFD``, ``REPLACEMENT CHARACTER``), ``'ignore'`` (just leave the
+character out of the Unicode result), or ``'backslashreplace'`` (inserts a
+``\xNN`` escape sequence).
The following examples show the differences::
>>> b'\x80abc'.decode("utf-8", "strict") #doctest: +NORMALIZE_WHITESPACE
@@ -291,6 +292,8 @@ The following examples show the differences::
invalid start byte
>>> b'\x80abc'.decode("utf-8", "replace")
'\ufffdabc'
+ >>> b'\x80abc'.decode("utf-8", "backslashreplace")
+ '\\x80abc'
>>> b'\x80abc'.decode("utf-8", "ignore")
'abc'
@@ -325,8 +328,9 @@ The *errors* parameter is the same as the parameter of the
:meth:`~bytes.decode` method but supports a few more possible handlers. As well as
``'strict'``, ``'ignore'``, and ``'replace'`` (which in this case
inserts a question mark instead of the unencodable character), there is
-also ``'xmlcharrefreplace'`` (inserts an XML character reference) and
-``backslashreplace`` (inserts a ``\uNNNN`` escape sequence).
+also ``'xmlcharrefreplace'`` (inserts an XML character reference),
+``backslashreplace`` (inserts a ``\uNNNN`` escape sequence) and
+``namereplace`` (inserts a ``\N{...}`` escape sequence).
The following example shows the different results::
@@ -346,6 +350,8 @@ The following example shows the different results::
b'&#40960;abcd&#1972;'
>>> u.encode('ascii', 'backslashreplace')
b'\\ua000abcd\\u07b4'
+ >>> u.encode('ascii', 'namereplace')
+ b'\\N{YI SYLLABLE IT}abcd\\u07b4'
The low-level routines for registering and accessing the available
encodings are found in the :mod:`codecs` module. Implementing new
diff --git a/Doc/includes/noddy.c b/Doc/includes/noddy.c
index 8f79fcf..19a27a8 100644
--- a/Doc/includes/noddy.c
+++ b/Doc/includes/noddy.c
@@ -38,7 +38,7 @@ static PyModuleDef noddymodule = {
};
PyMODINIT_FUNC
-PyInit_noddy(void)
+PyInit_noddy(void)
{
PyObject* m;
diff --git a/Doc/includes/run-func.c b/Doc/includes/run-func.c
index 1c9860d..986d670 100644
--- a/Doc/includes/run-func.c
+++ b/Doc/includes/run-func.c
@@ -13,7 +13,7 @@ main(int argc, char *argv[])
}
Py_Initialize();
- pName = PyUnicode_FromString(argv[1]);
+ pName = PyUnicode_DecodeFSDefault(argv[1]);
/* Error checking of pName left out */
pModule = PyImport_Import(pName);
diff --git a/Doc/includes/typestruct.h b/Doc/includes/typestruct.h
index 7265009..9f47899 100644
--- a/Doc/includes/typestruct.h
+++ b/Doc/includes/typestruct.h
@@ -9,7 +9,8 @@ typedef struct _typeobject {
printfunc tp_print;
getattrfunc tp_getattr;
setattrfunc tp_setattr;
- void *tp_reserved; /* formerly known as tp_compare */
+ PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2)
+ or tp_reserved (Python 3) */
reprfunc tp_repr;
/* Method suites for standard classes */
diff --git a/Doc/install/index.rst b/Doc/install/index.rst
index 8f3ad72..876f350 100644
--- a/Doc/install/index.rst
+++ b/Doc/install/index.rst
@@ -361,7 +361,7 @@ And here are the values used on Windows:
Type of file Installation directory
=============== ===========================================================
modules :file:`{userbase}\\Python{XY}\\site-packages`
-scripts :file:`{userbase}\\Scripts`
+scripts :file:`{userbase}\\Python{XY}\\Scripts`
data :file:`{userbase}`
C headers :file:`{userbase}\\Python{XY}\\Include\\{distname}`
=============== ===========================================================
diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst
index 72f2963..73d8b6b 100644
--- a/Doc/library/__future__.rst
+++ b/Doc/library/__future__.rst
@@ -87,6 +87,9 @@ language using this mechanism:
| unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: |
| | | | *Bytes literals in Python 3000* |
+------------------+-------------+--------------+---------------------------------------------+
+| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: |
+| | | | *StopIteration handling inside generators* |
++------------------+-------------+--------------+---------------------------------------------+
.. seealso::
diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst
index 067fb8f..72095a8 100644
--- a/Doc/library/argparse.rst
+++ b/Doc/library/argparse.rst
@@ -135,7 +135,7 @@ ArgumentParser objects
formatter_class=argparse.HelpFormatter, \
prefix_chars='-', fromfile_prefix_chars=None, \
argument_default=None, conflict_handler='error', \
- add_help=True)
+ add_help=True, allow_abbrev=True)
Create a new :class:`ArgumentParser` object. All parameters should be passed
as keyword arguments. Each parameter has its own more detailed description
@@ -169,6 +169,12 @@ ArgumentParser objects
* add_help_ - Add a -h/--help option to the parser (default: ``True``)
+ * allow_abbrev_ - Allows long options to be abbreviated if the
+ abbreviation is unambiguous. (default: ``True``)
+
+ .. versionchanged:: 3.5
+ *allow_abbrev* parameter was added.
+
The following sections describe how each of these are used.
@@ -518,6 +524,26 @@ calls, we supply ``argument_default=SUPPRESS``::
>>> parser.parse_args([])
Namespace()
+.. _allow_abbrev:
+
+allow_abbrev
+^^^^^^^^^^^^
+
+Normally, when you pass an argument list to the
+:meth:`~ArgumentParser.parse_args` method of a :class:`ArgumentParser`,
+it :ref:`recognizes abbreviations <prefix-matching>` of long options.
+
+This feature can be disabled by setting ``allow_abbrev`` to ``False``::
+
+ >>> parser = argparse.ArgumentParser(prog='PROG', allow_abbrev=False)
+ >>> parser.add_argument('--foobar', action='store_true')
+ >>> parser.add_argument('--foonley', action='store_false')
+ >>> parser.parse_args(['--foon'])
+ usage: PROG [-h] [--foobar] [--foonley]
+ PROG: error: unrecognized arguments: --foon
+
+.. versionadded:: 3.5
+
conflict_handler
^^^^^^^^^^^^^^^^
@@ -1410,9 +1436,9 @@ argument::
Argument abbreviations (prefix matching)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-The :meth:`~ArgumentParser.parse_args` method allows long options to be
-abbreviated to a prefix, if the abbreviation is unambiguous (the prefix matches
-a unique option)::
+The :meth:`~ArgumentParser.parse_args` method :ref:`by default <allow_abbrev>`
+allows long options to be abbreviated to a prefix, if the abbreviation is
+unambiguous (the prefix matches a unique option)::
>>> parser = argparse.ArgumentParser(prog='PROG')
>>> parser.add_argument('-bacon')
@@ -1426,6 +1452,7 @@ a unique option)::
PROG: error: ambiguous option: -ba could match -badger, -bacon
An error is produced for arguments that could produce more than one options.
+This feature can be disabled by setting :ref:`allow_abbrev` to ``False``.
Beyond ``sys.argv``
diff --git a/Doc/library/asynchat.rst b/Doc/library/asynchat.rst
index c6fa061..794da8c 100644
--- a/Doc/library/asynchat.rst
+++ b/Doc/library/asynchat.rst
@@ -147,40 +147,6 @@ connection requests.
by the channel after :meth:`found_terminator` is called.
-asynchat - Auxiliary Classes
-------------------------------------------
-
-.. class:: fifo(list=None)
-
- A :class:`fifo` holding data which has been pushed by the application but
- not yet popped for writing to the channel. A :class:`fifo` is a list used
- to hold data and/or producers until they are required. If the *list*
- argument is provided then it should contain producers or data items to be
- written to the channel.
-
-
- .. method:: is_empty()
-
- Returns ``True`` if and only if the fifo is empty.
-
-
- .. method:: first()
-
- Returns the least-recently :meth:`push`\ ed item from the fifo.
-
-
- .. method:: push(data)
-
- Adds the given data (which may be a string or a producer object) to the
- producer fifo.
-
-
- .. method:: pop()
-
- If the fifo is not empty, returns ``True, first()``, deleting the popped
- item. Returns ``False, None`` for an empty fifo.
-
-
.. _asynchat-example:
asynchat Example
diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst
index 3a196db..e27cef8 100644
--- a/Doc/library/asyncio-dev.rst
+++ b/Doc/library/asyncio-dev.rst
@@ -99,7 +99,7 @@ To schedule a callback from a different thread, the
:meth:`BaseEventLoop.call_soon_threadsafe` method should be used. Example to
schedule a coroutine from a different thread::
- loop.call_soon_threadsafe(asyncio.async, coro_func())
+ loop.call_soon_threadsafe(asyncio.ensure_future, coro_func())
Most asyncio objects are not thread safe. You should only worry if you access
objects outside the event loop. For example, to cancel a future, don't call
@@ -162,10 +162,10 @@ Detect coroutine objects never scheduled
----------------------------------------
When a coroutine function is called and its result is not passed to
-:func:`async` or to the :meth:`BaseEventLoop.create_task` method, the execution
-of the coroutine object will never be scheduled which is probably a bug.
-:ref:`Enable the debug mode of asyncio <asyncio-debug-mode>` to :ref:`log a
-warning <asyncio-logger>` to detect it.
+:func:`ensure_future` or to the :meth:`BaseEventLoop.create_task` method,
+the execution of the coroutine object will never be scheduled which is
+probably a bug. :ref:`Enable the debug mode of asyncio <asyncio-debug-mode>`
+to :ref:`log a warning <asyncio-logger>` to detect it.
Example with the bug::
@@ -184,7 +184,7 @@ Output in debug mode::
File "test.py", line 7, in <module>
test()
-The fix is to call the :func:`async` function or the
+The fix is to call the :func:`ensure_future` function or the
:meth:`BaseEventLoop.create_task` method with the coroutine object.
.. seealso::
diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
index dee3286..146c8c9 100644
--- a/Doc/library/asyncio-eventloop.rst
+++ b/Doc/library/asyncio-eventloop.rst
@@ -36,7 +36,7 @@ Run an event loop
Run until the :class:`Future` is done.
If the argument is a :ref:`coroutine object <coroutine>`, it is wrapped by
- :func:`async`.
+ :func:`ensure_future`.
Return the Future's result, or raise its exception.
diff --git a/Doc/library/asyncio-eventloops.rst b/Doc/library/asyncio-eventloops.rst
index afb8b9f..ae3bf90 100644
--- a/Doc/library/asyncio-eventloops.rst
+++ b/Doc/library/asyncio-eventloops.rst
@@ -100,8 +100,6 @@ Common limits of Windows event loops:
:class:`ProactorEventLoop` specific limits:
-- SSL is not supported: :meth:`~BaseEventLoop.create_connection` and
- :meth:`~BaseEventLoop.create_server` cannot be used with SSL for example
- :meth:`~BaseEventLoop.create_datagram_endpoint` (UDP) is not supported
- :meth:`~BaseEventLoop.add_reader` and :meth:`~BaseEventLoop.add_writer` are
not supported
@@ -112,6 +110,10 @@ The best resolution is 0.5 msec. The resolution depends on the hardware
<http://fr.wikipedia.org/wiki/High_Precision_Event_Timer>`_) and on the Windows
configuration. See :ref:`asyncio delayed calls <asyncio-delayed-calls>`.
+.. versionchanged:: 3.5
+
+ :class:`ProactorEventLoop` now supports SSL.
+
Mac OS X
^^^^^^^^
diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst
index 2e671e8..e68c234 100644
--- a/Doc/library/asyncio-protocol.rst
+++ b/Doc/library/asyncio-protocol.rst
@@ -448,9 +448,9 @@ buffer size reaches the low-water mark.
Coroutines and protocols
------------------------
-Coroutines can be scheduled in a protocol method using :func:`async`, but there
-is no guarantee made about the execution order. Protocols are not aware of
-coroutines created in protocol methods and so will not wait for them.
+Coroutines can be scheduled in a protocol method using :func:`ensure_future`,
+but there is no guarantee made about the execution order. Protocols are not
+aware of coroutines created in protocol methods and so will not wait for them.
To have a reliable execution order, use :ref:`stream objects <asyncio-streams>` in a
coroutine with ``yield from``. For example, the :meth:`StreamWriter.drain`
diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst
index 3370672..f11c09a 100644
--- a/Doc/library/asyncio-queue.rst
+++ b/Doc/library/asyncio-queue.rst
@@ -8,7 +8,6 @@ Queues:
* :class:`Queue`
* :class:`PriorityQueue`
* :class:`LifoQueue`
-* :class:`JoinableQueue`
asyncio queue API was designed to be close to classes of the :mod:`queue`
module (:class:`~queue.Queue`, :class:`~queue.PriorityQueue`,
@@ -144,16 +143,6 @@ LifoQueue
first.
-JoinableQueue
-^^^^^^^^^^^^^
-
-.. class:: JoinableQueue
-
- Deprecated alias for :class:`Queue`.
-
- .. deprecated:: 3.4.4
-
-
Exceptions
^^^^^^^^^^
diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
index e7ff7d2..166ab73 100644
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -8,17 +8,23 @@ Tasks and coroutines
Coroutines
----------
-A coroutine is a generator that follows certain conventions. For
-documentation purposes, all coroutines should be decorated with
-``@asyncio.coroutine``, but this cannot be strictly enforced.
-
-Coroutines use the ``yield from`` syntax introduced in :pep:`380`,
+Coroutines used with :mod:`asyncio` may be implemented using the
+:keyword:`async def` statement, or by using :term:`generators <generator>`.
+The :keyword:`async def` type of coroutine was added in Python 3.5, and
+is recommended if there is no need to support older Python versions.
+
+Generator-based coroutines should be decorated with :func:`@asyncio.coroutine
+<asyncio.coroutine>`, although this is not strictly enforced.
+The decorator enables compatibility with :keyword:`async def` coroutines,
+and also serves as documentation. Generator-based
+coroutines use the ``yield from`` syntax introduced in :pep:`380`,
instead of the original ``yield`` syntax.
The word "coroutine", like the word "generator", is used for two
different (though related) concepts:
-- The function that defines a coroutine (a function definition
+- The function that defines a coroutine
+ (a function definition using :keyword:`async def` or
decorated with ``@asyncio.coroutine``). If disambiguation is needed
we will call this a *coroutine function* (:func:`iscoroutinefunction`
returns ``True``).
@@ -30,29 +36,30 @@ different (though related) concepts:
Things a coroutine can do:
-- ``result = yield from future`` -- suspends the coroutine until the
+- ``result = await future`` or ``result = yield from future`` --
+ suspends the coroutine until the
future is done, then returns the future's result, or raises an
exception, which will be propagated. (If the future is cancelled,
it will raise a ``CancelledError`` exception.) Note that tasks are
futures, and everything said about futures also applies to tasks.
-- ``result = yield from coroutine`` -- wait for another coroutine to
+- ``result = await coroutine`` or ``result = yield from coroutine`` --
+ wait for another coroutine to
produce a result (or raise an exception, which will be propagated).
The ``coroutine`` expression must be a *call* to another coroutine.
- ``return expression`` -- produce a result to the coroutine that is
- waiting for this one using ``yield from``.
+ waiting for this one using :keyword:`await` or ``yield from``.
- ``raise exception`` -- raise an exception in the coroutine that is
- waiting for this one using ``yield from``.
+ waiting for this one using :keyword:`await` or ``yield from``.
-Calling a coroutine does not start its code running -- it is just a
-generator, and the coroutine object returned by the call is really a
-generator object, which doesn't do anything until you iterate over it.
-In the case of a coroutine object, there are two basic ways to start
-it running: call ``yield from coroutine`` from another coroutine
+Calling a coroutine does not start its code running --
+the coroutine object returned by the call doesn't do anything until you
+schedule its execution. There are two basic ways to start it running:
+call ``await coroutine`` or ``yield from coroutine`` from another coroutine
(assuming the other coroutine is already running!), or schedule its execution
-using the :func:`async` function or the :meth:`BaseEventLoop.create_task`
+using the :func:`ensure_future` function or the :meth:`BaseEventLoop.create_task`
method.
@@ -60,9 +67,15 @@ Coroutines (and tasks) can only run when the event loop is running.
.. decorator:: coroutine
- Decorator to mark coroutines.
+ Decorator to mark generator-based coroutines. This enables
+ the generator use :keyword:`!yield from` to call :keyword:`async
+ def` coroutines, and also enables the generator to be called by
+ :keyword:`async def` coroutines, for instance using an
+ :keyword:`await` expression.
+
+ There is no need to decorate :keyword:`async def` coroutines themselves.
- If the coroutine is not yielded from before it is destroyed, an error
+ If the generator is not yielded from before it is destroyed, an error
message is logged. See :ref:`Detect coroutines never scheduled
<asyncio-coroutine-not-scheduled>`.
@@ -72,7 +85,7 @@ Coroutines (and tasks) can only run when the event loop is running.
even if they are plain Python functions returning a :class:`Future`.
This is intentional to have a freedom of tweaking the implementation
of these functions in the future. If such a function is needed to be
- used in a callback-style code, wrap its result with :func:`async`.
+ used in a callback-style code, wrap its result with :func:`ensure_future`.
.. _asyncio-hello-world-coroutine:
@@ -84,8 +97,7 @@ Example of coroutine displaying ``"Hello World"``::
import asyncio
- @asyncio.coroutine
- def hello_world():
+ async def hello_world():
print("Hello World!")
loop = asyncio.get_event_loop()
@@ -111,20 +123,30 @@ using the :meth:`sleep` function::
import asyncio
import datetime
- @asyncio.coroutine
- def display_date(loop):
+ async def display_date(loop):
end_time = loop.time() + 5.0
while True:
print(datetime.datetime.now())
if (loop.time() + 1.0) >= end_time:
break
- yield from asyncio.sleep(1)
+ await asyncio.sleep(1)
loop = asyncio.get_event_loop()
# Blocking call which returns when the display_date() coroutine is done
loop.run_until_complete(display_date(loop))
loop.close()
+The same coroutine implemented using a generator::
+
+ @asyncio.coroutine
+ def display_date(loop):
+ end_time = loop.time() + 5.0
+ while True:
+ print(datetime.datetime.now())
+ if (loop.time() + 1.0) >= end_time:
+ break
+ yield from asyncio.sleep(1)
+
.. seealso::
The :ref:`display the current date with call_later()
@@ -139,15 +161,13 @@ Example chaining coroutines::
import asyncio
- @asyncio.coroutine
- def compute(x, y):
+ async def compute(x, y):
print("Compute %s + %s ..." % (x, y))
- yield from asyncio.sleep(1.0)
+ await asyncio.sleep(1.0)
return x + y
- @asyncio.coroutine
- def print_sum(x, y):
- result = yield from compute(x, y)
+ async def print_sum(x, y):
+ result = await compute(x, y)
print("%s + %s = %s" % (x, y, result))
loop = asyncio.get_event_loop()
@@ -374,7 +394,7 @@ Task
<coroutine>` did not complete. It is probably a bug and a warning is
logged: see :ref:`Pending task destroyed <asyncio-pending-task-destroyed>`.
- Don't directly create :class:`Task` instances: use the :func:`async`
+ Don't directly create :class:`Task` instances: use the :func:`ensure_future`
function or the :meth:`BaseEventLoop.create_task` method.
This class is :ref:`not thread safe <asyncio-multithreading>`.
@@ -550,12 +570,14 @@ Task functions
.. function:: iscoroutine(obj)
- Return ``True`` if *obj* is a :ref:`coroutine object <coroutine>`.
+ Return ``True`` if *obj* is a :ref:`coroutine object <coroutine>`,
+ which may be based on a generator or an :keyword:`async def` coroutine.
-.. function:: iscoroutinefunction(obj)
+.. function:: iscoroutinefunction(func)
- Return ``True`` if *func* is a decorated :ref:`coroutine function
- <coroutine>`.
+ Return ``True`` if *func* is determined to be a :ref:`coroutine function
+ <coroutine>`, which may be a decorated generator function or an
+ :keyword:`async def` function.
.. coroutinefunction:: sleep(delay, result=None, \*, loop=None)
diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst
index 488cda5..1b8d9cf 100644
--- a/Doc/library/bz2.rst
+++ b/Doc/library/bz2.rst
@@ -120,6 +120,10 @@ All of the classes in this module may safely be accessed from multiple threads.
.. versionchanged:: 3.4
The ``'x'`` (exclusive creation) mode was added.
+ .. versionchanged:: 3.5
+ The :meth:`~io.BufferedIOBase.read` method now accepts an argument of
+ ``None``.
+
Incremental (de)compression
---------------------------
@@ -162,15 +166,32 @@ Incremental (de)compression
you need to decompress a multi-stream input with :class:`BZ2Decompressor`,
you must use a new decompressor for each stream.
- .. method:: decompress(data)
+ .. method:: decompress(data, max_length=-1)
+
+ Decompress *data* (a :term:`bytes-like object`), returning
+ uncompressed data as bytes. Some of *data* may be buffered
+ internally, for use in later calls to :meth:`decompress`. The
+ returned data should be concatenated with the output of any
+ previous calls to :meth:`decompress`.
+
+ If *max_length* is nonnegative, returns at most *max_length*
+ bytes of decompressed data. If this limit is reached and further
+ output can be produced, the :attr:`~.needs_input` attribute will
+ be set to ``False``. In this case, the next call to
+ :meth:`~.decompress` may provide *data* as ``b''`` to obtain
+ more of the output.
- Provide data to the decompressor object. Returns a chunk of decompressed
- data if possible, or an empty byte string otherwise.
+ If all of the input data was decompressed and returned (either
+ because this was less than *max_length* bytes, or because
+ *max_length* was negative), the :attr:`~.needs_input` attribute
+ will be set to ``True``.
- Attempting to decompress data after the end of the current stream is
- reached raises an :exc:`EOFError`. If any data is found after the end of
- the stream, it is ignored and saved in the :attr:`unused_data` attribute.
+ Attempting to decompress data after the end of stream is reached
+ raises an `EOFError`. Any data found after the end of the
+ stream is ignored and saved in the :attr:`~.unused_data` attribute.
+ .. versionchanged:: 3.5
+ Added the *max_length* parameter.
.. attribute:: eof
@@ -186,6 +207,13 @@ Incremental (de)compression
If this attribute is accessed before the end of the stream has been
reached, its value will be ``b''``.
+ .. attribute:: needs_input
+
+ ``False`` if the :meth:`.decompress` method can provide more
+ decompressed data before requiring new uncompressed input.
+
+ .. versionadded:: 3.5
+
One-shot (de)compression
------------------------
diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst
index fa13145..0a9e884 100644
--- a/Doc/library/cgi.rst
+++ b/Doc/library/cgi.rst
@@ -157,6 +157,9 @@ return bytes)::
if not line: break
linecount = linecount + 1
+:class:`FieldStorage` objects also support being used in a :keyword:`with`
+statement, which will automatically close them when done.
+
If an error is encountered when obtaining the contents of an uploaded file
(for example, when the user interrupts the form submission by clicking on
a Back or Cancel button) the :attr:`~FieldStorage.done` attribute of the
@@ -182,6 +185,10 @@ A form submitted via POST that also has a query string will contain both
The :attr:`~FieldStorage.file` attribute is automatically closed upon the
garbage collection of the creating :class:`FieldStorage` instance.
+.. versionchanged:: 3.5
+ Added support for the context management protocol to the
+ :class:`FieldStorage` class.
+
Higher Level Interface
----------------------
diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst
index a981d94..ab619a0 100644
--- a/Doc/library/cmath.rst
+++ b/Doc/library/cmath.rst
@@ -207,6 +207,38 @@ Classification functions
and ``False`` otherwise.
+.. function:: isclose(a, b, *, rel_tol=1e-09, abs_tol=0.0)
+
+ Return ``True`` if the values *a* and *b* are close to each other and
+ ``False`` otherwise.
+
+ Whether or not two values are considered close is determined according to
+ given absolute and relative tolerances.
+
+ *rel_tol* is the relative tolerance -- it is the maximum allowed difference
+ between *a* and *b*, relative to the larger absolute value of *a* or *b*.
+ For example, to set a tolerance of 5%, pass ``rel_tol=0.05``. The default
+ tolerance is ``1e-09``, which assures that the two values are the same
+ within about 9 decimal digits. *rel_tol* must be greater than zero.
+
+ *abs_tol* is the minimum absolute tolerance -- useful for comparisons near
+ zero. *abs_tol* must be at least zero.
+
+ If no errors occur, the result will be:
+ ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``.
+
+ The IEEE 754 special values of ``NaN``, ``inf``, and ``-inf`` will be
+ handled according to IEEE rules. Specifically, ``NaN`` is not considered
+ close to any other value, including ``NaN``. ``inf`` and ``-inf`` are only
+ considered close to themselves.
+
+ .. versionadded:: 3.5
+
+ .. seealso::
+
+ :pep:`485` -- A function for testing approximate equality
+
+
Constants
---------
diff --git a/Doc/library/code.rst b/Doc/library/code.rst
index 5b5d7cc..275201c 100644
--- a/Doc/library/code.rst
+++ b/Doc/library/code.rst
@@ -4,6 +4,7 @@
.. module:: code
:synopsis: Facilities to implement read-eval-print loops.
+**Source code:** :source:`Lib/code.py`
The ``code`` module provides facilities to implement read-eval-print loops in
Python. Two classes and convenience functions are included which can be used to
@@ -113,6 +114,9 @@ Interactive Interpreter Objects
because it is within the interpreter object implementation. The output is
written by the :meth:`write` method.
+ .. versionchanged:: 3.5 The full chained traceback is displayed instead
+ of just the primary traceback.
+
.. method:: InteractiveInterpreter.write(data)
@@ -165,4 +169,3 @@ interpreter objects as well as the following additions.
newline. When the user enters the EOF key sequence, :exc:`EOFError` is raised.
The base implementation reads from ``sys.stdin``; a subclass may replace this
with a different implementation.
-
diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst
index 9496035..dae556e 100644
--- a/Doc/library/codecs.rst
+++ b/Doc/library/codecs.rst
@@ -7,6 +7,7 @@
.. sectionauthor:: Marc-André Lemburg <mal@lemburg.com>
.. sectionauthor:: Martin v. Löwis <martin@v.loewis.de>
+**Source code:** :source:`Lib/codecs.py`
.. index::
single: Unicode
@@ -29,10 +30,9 @@ module features are restricted to use specifically with
The module defines the following functions for encoding and decoding with
any codec:
-.. function:: encode(obj, [encoding[, errors]])
+.. function:: encode(obj, encoding='utf-8', errors='strict')
- Encodes *obj* using the codec registered for *encoding*. The default
- encoding is ``utf-8``.
+ Encodes *obj* using the codec registered for *encoding*.
*Errors* may be given to set the desired error handling scheme. The
default error handler is ``'strict'`` meaning that encoding errors raise
@@ -40,10 +40,9 @@ any codec:
:exc:`UnicodeEncodeError`). Refer to :ref:`codec-base-classes` for more
information on codec error handling.
-.. function:: decode(obj, [encoding[, errors]])
+.. function:: decode(obj, encoding='utf-8', errors='strict')
- Decodes *obj* using the codec registered for *encoding*. The default
- encoding is ``utf-8``.
+ Decodes *obj* using the codec registered for *encoding*.
*Errors* may be given to set the desired error handling scheme. The
default error handler is ``'strict'`` meaning that decoding errors raise
@@ -104,7 +103,6 @@ The full details for each codec can also be looked up directly:
To simplify access to the various codec components, the module provides
these additional functions which use :func:`lookup` for the codec lookup:
-
.. function:: getencoder(encoding)
Look up the codec for the given encoding and return its encoder function.
@@ -274,6 +272,7 @@ implement the file protocols. Codec authors also need to define how the
codec will handle encoding and decoding errors.
+.. _surrogateescape:
.. _error-handlers:
Error Handlers
@@ -315,10 +314,14 @@ The following error handlers are only applicable to
| | reference (only for encoding). Implemented |
| | in :func:`xmlcharrefreplace_errors`. |
+-------------------------+-----------------------------------------------+
-| ``'backslashreplace'`` | Replace with backslashed escape sequences |
-| | (only for encoding). Implemented in |
+| ``'backslashreplace'`` | Replace with backslashed escape sequences. |
+| | Implemented in |
| | :func:`backslashreplace_errors`. |
+-------------------------+-----------------------------------------------+
+| ``'namereplace'`` | Replace with ``\N{...}`` escape sequences |
+| | (only for encoding). Implemented in |
+| | :func:`namereplace_errors`. |
++-------------------------+-----------------------------------------------+
| ``'surrogateescape'`` | On decoding, replace byte with individual |
| | surrogate code ranging from ``U+DC80`` to |
| | ``U+DCFF``. This code will then be turned |
@@ -344,6 +347,13 @@ In addition, the following error handler is specific to the given codecs:
.. versionchanged:: 3.4
The ``'surrogatepass'`` error handlers now works with utf-16\* and utf-32\* codecs.
+.. versionadded:: 3.5
+ The ``'namereplace'`` error handler.
+
+.. versionchanged:: 3.5
+ The ``'backslashreplace'`` error handlers now works with decoding and
+ translating.
+
The set of allowed values can be extended by registering a new named error
handler:
@@ -411,9 +421,17 @@ functions:
.. function:: backslashreplace_errors(exception)
- Implements the ``'backslashreplace'`` error handling (for encoding with
+ Implements the ``'backslashreplace'`` error handling (for
+ :term:`text encodings <text encoding>` only): malformed data is
+ replaced by a backslashed escape sequence.
+
+.. function:: namereplace_errors(exception)
+
+ Implements the ``'namereplace'`` error handling (for encoding with
:term:`text encodings <text encoding>` only): the
- unencodable character is replaced by a backslashed escape sequence.
+ unencodable character is replaced by a ``\N{...}`` escape sequence.
+
+ .. versionadded:: 3.5
.. _codec-objects:
@@ -1142,8 +1160,16 @@ particular, the following variants typically exist:
+-----------------+--------------------------------+--------------------------------+
| koi8_r | | Russian |
+-----------------+--------------------------------+--------------------------------+
+| koi8_t | | Tajik |
+| | | |
+| | | .. versionadded:: 3.5 |
++-----------------+--------------------------------+--------------------------------+
| koi8_u | | Ukrainian |
+-----------------+--------------------------------+--------------------------------+
+| kz1048 | kz_1048, strk1048_2002, rk1048 | Kazakh |
+| | | |
+| | | .. versionadded:: 3.5 |
++-----------------+--------------------------------+--------------------------------+
| mac_cyrillic | maccyrillic | Bulgarian, Byelorussian, |
| | | Macedonian, Russian, Serbian |
+-----------------+--------------------------------+--------------------------------+
@@ -1444,4 +1470,3 @@ This module implements a variant of the UTF-8 codec: On encoding a UTF-8 encoded
BOM will be prepended to the UTF-8 encoded bytes. For the stateful encoder this
is only done once (on the first write to the byte stream). For decoding an
optional UTF-8 encoded BOM at the start of the data will be skipped.
-
diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst
index 99c4311..563c1bc 100644
--- a/Doc/library/collections.abc.rst
+++ b/Doc/library/collections.abc.rst
@@ -33,13 +33,14 @@ The collections module offers the following :term:`ABCs <abstract base class>`:
.. tabularcolumns:: |l|L|L|L|
-========================= ===================== ====================== ====================================================
+========================== ====================== ======================= ====================================================
ABC Inherits from Abstract Methods Mixin Methods
-========================= ===================== ====================== ====================================================
+========================== ====================== ======================= ====================================================
:class:`Container` ``__contains__``
:class:`Hashable` ``__hash__``
:class:`Iterable` ``__iter__``
:class:`Iterator` :class:`Iterable` ``__next__`` ``__iter__``
+:class:`Generator` :class:`Iterator` ``send``, ``throw`` ``close``, ``__iter__``, ``__next__``
:class:`Sized` ``__len__``
:class:`Callable` ``__call__``
@@ -80,7 +81,11 @@ ABC Inherits from Abstract Methods Mixin
:class:`KeysView` :class:`MappingView`, ``__contains__``,
:class:`Set` ``__iter__``
:class:`ValuesView` :class:`MappingView` ``__contains__``, ``__iter__``
-========================= ===================== ====================== ====================================================
+:class:`Awaitable` ``__await__``
+:class:`Coroutine` :class:`Awaitable` ``send``, ``throw`` ``close``
+:class:`AsyncIterable` ``__aiter__``
+:class:`AsyncIterator` :class:`AsyncIterable` ``__anext__`` ``__aiter__``
+========================== ====================== ======================= ====================================================
.. class:: Container
@@ -102,11 +107,34 @@ ABC Inherits from Abstract Methods Mixin
:meth:`~iterator.__next__` methods. See also the definition of
:term:`iterator`.
+.. class:: Generator
+
+ ABC for generator classes that implement the protocol defined in
+ :pep:`342` that extends iterators with the :meth:`~generator.send`,
+ :meth:`~generator.throw` and :meth:`~generator.close` methods.
+ See also the definition of :term:`generator`.
+
+ .. versionadded:: 3.5
+
.. class:: Sequence
MutableSequence
ABCs for read-only and mutable :term:`sequences <sequence>`.
+ Implementation note: Some of the mixin methods, such as
+ :meth:`__iter__`, :meth:`__reversed__` and :meth:`index`, make
+ repeated calls to the underlying :meth:`__getitem__` method.
+ Consequently, if :meth:`__getitem__` is implemented with constant
+ access speed, the mixin methods will have linear performance;
+ however, if the underlying method is linear (as it would be with a
+ linked list), the mixins will have quadratic performance and will
+ likely need to be overridden.
+
+ .. versionchanged:: 3.5
+ The index() method added support for *stop* and *start*
+ arguments.
+
+
.. class:: Set
MutableSet
@@ -124,6 +152,56 @@ ABC Inherits from Abstract Methods Mixin
ABCs for mapping, items, keys, and values :term:`views <view>`.
+.. class:: Awaitable
+
+ ABC for :term:`awaitable` objects, which can be used in :keyword:`await`
+ expressions. Custom implementations must provide the :meth:`__await__`
+ method.
+
+ :term:`Coroutine` objects and instances of the
+ :class:`~collections.abc.Coroutine` ABC are all instances of this ABC.
+
+ .. note::
+ In CPython, generator-based coroutines (generators decorated with
+ :func:`types.coroutine` or :func:`asyncio.coroutine`) are
+ *awaitables*, even though they do not have an :meth:`__await__` method.
+ Using ``isinstance(gencoro, Awaitable)`` for them will return ``False``.
+ Use :func:`inspect.isawaitable` to detect them.
+
+ .. versionadded:: 3.5
+
+.. class:: Coroutine
+
+ ABC for coroutine compatible classes. These implement the
+ following methods, defined in :ref:`coroutine-objects`:
+ :meth:`~coroutine.send`, :meth:`~coroutine.throw`, and
+ :meth:`~coroutine.close`. Custom implementations must also implement
+ :meth:`__await__`. All :class:`Coroutine` instances are also instances of
+ :class:`Awaitable`. See also the definition of :term:`coroutine`.
+
+ .. note::
+ In CPython, generator-based coroutines (generators decorated with
+ :func:`types.coroutine` or :func:`asyncio.coroutine`) are
+ *awaitables*, even though they do not have an :meth:`__await__` method.
+ Using ``isinstance(gencoro, Coroutine)`` for them will return ``False``.
+ Use :func:`inspect.isawaitable` to detect them.
+
+ .. versionadded:: 3.5
+
+.. class:: AsyncIterable
+
+ ABC for classes that provide ``__aiter__`` method. See also the
+ definition of :term:`asynchronous iterable`.
+
+ .. versionadded:: 3.5
+
+.. class:: AsyncIterator
+
+ ABC for classes that provide ``__aiter__`` and ``__anext__``
+ methods. See also the definition of :term:`asynchronous iterator`.
+
+ .. versionadded:: 3.5
+
These ABCs allow us to ask classes or instances if they provide
particular functionality, for example::
diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst
index 347cf1f..60e8fe8 100644
--- a/Doc/library/collections.rst
+++ b/Doc/library/collections.rst
@@ -387,7 +387,7 @@ or subtracting from an empty counter.
Section 4.6.3, Exercise 19*.
* To enumerate all distinct multisets of a given size over a given set of
- elements, see :func:`itertools.combinations_with_replacement`.
+ elements, see :func:`itertools.combinations_with_replacement`:
map(Counter, combinations_with_replacement('ABC', 2)) --> AA AB AC BB BC CC
@@ -437,6 +437,13 @@ or subtracting from an empty counter.
Remove all elements from the deque leaving it with length 0.
+ .. method:: copy()
+
+ Create a shallow copy of the deque.
+
+ .. versionadded:: 3.5
+
+
.. method:: count(x)
Count the number of deque elements equal to *x*.
@@ -457,6 +464,22 @@ or subtracting from an empty counter.
elements in the iterable argument.
+ .. method:: index(x[, start[, stop]])
+
+ Return the position of *x* in the deque (at or after index *start*
+ and before index *stop*). Returns the first match or raises
+ :exc:`ValueError` if not found.
+
+ .. versionadded:: 3.5
+
+
+ .. method:: insert(i, x)
+
+ Insert *x* into the deque at position *i*.
+
+ .. versionadded:: 3.5
+
+
.. method:: pop()
Remove and return an element from the right side of the deque. If no
@@ -471,7 +494,7 @@ or subtracting from an empty counter.
.. method:: remove(value)
- Removed the first occurrence of *value*. If not found, raises a
+ Remove the first occurrence of *value*. If not found, raises a
:exc:`ValueError`.
@@ -504,6 +527,9 @@ the :keyword:`in` operator, and subscript references such as ``d[-1]``. Indexed
access is O(1) at both ends but slows to O(n) in the middle. For fast random
access, use lists instead.
+Starting in version 3.5, deques support ``__add__()``, ``__mul__()``,
+and ``__imul__()``.
+
Example:
.. doctest::
@@ -899,6 +925,15 @@ create a new named tuple type from the :attr:`_fields` attribute:
>>> Point3D = namedtuple('Point3D', Point._fields + ('z',))
+Docstrings can be customized by making direct assignments to the ``__doc__``
+fields:
+
+ >>> Book = namedtuple('Book', ['id', 'title', 'authors'])
+ >>> Book.__doc__ = 'Hardcover book in active collection'
+ >>> Book.id.__doc__ = '13-digit ISBN'
+ >>> Book.title.__doc__ = 'Title of first printing'
+ >>> Book.author.__doc__ = 'List of authors sorted by last name'
+
Default values can be implemented by using :meth:`_replace` to
customize a prototype instance:
@@ -982,6 +1017,9 @@ The :class:`OrderedDict` constructor and :meth:`update` method both accept
keyword arguments, but their order is lost because Python's function call
semantics pass-in keyword arguments using a regular unordered dictionary.
+.. versionchanged:: 3.5
+ The items, keys, and values :term:`views <view>` of :class:`OrderedDict` now
+ support reverse iteration using :func:`reversed`.
:class:`OrderedDict` Examples and Recipes
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst
index 9712de2..0325f1a 100644
--- a/Doc/library/compileall.rst
+++ b/Doc/library/compileall.rst
@@ -42,7 +42,8 @@ compile Python sources.
.. cmdoption:: -q
- Do not print the list of files compiled, print only error messages.
+ Do not print the list of files compiled. If passed once, error messages will
+ still be printed. If passed twice (``-qq``), all output is suppressed.
.. cmdoption:: -d destdir
@@ -70,9 +71,33 @@ compile Python sources.
is to write files to their :pep:`3147` locations and names, which allows
byte-code files from multiple versions of Python to coexist.
+.. cmdoption:: -r
+
+ Control the maximum recursion level for subdirectories.
+ If this is given, then ``-l`` option will not be taken into account.
+ :program:`python -m compileall <directory> -r 0` is equivalent to
+ :program:`python -m compileall <directory> -l`.
+
+.. cmdoption:: -j N
+
+ Use *N* workers to compile the files within the given directory.
+ If ``0`` is used, then the result of :func:`os.cpu_count()`
+ will be used.
+
.. versionchanged:: 3.2
Added the ``-i``, ``-b`` and ``-h`` options.
+.. versionchanged:: 3.5
+ Added the ``-j`` and ``-r`` options.
+
+.. versionchanged:: 3.5
+ ``-q`` option was changed to a multilevel value.
+
+.. versionchanged:: 3.5
+ ``-b`` will always produce a byte-code file ending in ``.pyc``, never
+ ``.pyo``.
+
+
There is no command-line option to control the optimization level used by the
:func:`compile` function, because the Python interpreter itself already
provides the option: :program:`python -O -m compileall`.
@@ -80,7 +105,7 @@ provides the option: :program:`python -O -m compileall`.
Public functions
----------------
-.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=False, legacy=False, optimize=-1)
+.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1)
Recursively descend the directory tree named by *dir*, compiling all :file:`.py`
files along the way.
@@ -101,8 +126,9 @@ Public functions
file considered for compilation, and if it returns a true value, the file
is skipped.
- If *quiet* is true, nothing is printed to the standard output unless errors
- occur.
+ If *quiet* is ``False`` or ``0`` (the default), the filenames and other
+ information are printed to standard out. Set to ``1``, only errors are
+ printed. Set to ``2``, all output is suppressed.
If *legacy* is true, byte-code files are written to their legacy locations
and names, which may overwrite byte-code files created by another version of
@@ -113,11 +139,26 @@ Public functions
*optimize* specifies the optimization level for the compiler. It is passed to
the built-in :func:`compile` function.
+ The argument *workers* specifies how many workers are used to
+ compile files in parallel. The default is to not use multiple workers.
+ If the platform can't use multiple workers and *workers* argument is given,
+ then sequential compilation will be used as a fallback. If *workers* is
+ lower than ``0``, a :exc:`ValueError` will be raised.
+
.. versionchanged:: 3.2
Added the *legacy* and *optimize* parameter.
+ .. versionchanged:: 3.5
+ Added the *workers* parameter.
+
+ .. versionchanged:: 3.5
+ *quiet* parameter was changed to a multilevel value.
-.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=False, legacy=False, optimize=-1)
+ .. versionchanged:: 3.5
+ The *legacy* parameter only writes out ``.pyc`` files, not ``.pyo`` files
+ no matter what the value of *optimize* is.
+
+.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1)
Compile the file with path *fullname*.
@@ -131,8 +172,9 @@ Public functions
file being compiled, and if it returns a true value, the file is not
compiled and ``True`` is returned.
- If *quiet* is true, nothing is printed to the standard output unless errors
- occur.
+ If *quiet* is ``False`` or ``0`` (the default), the filenames and other
+ information are printed to standard out. Set to ``1``, only errors are
+ printed. Set to ``2``, all output is suppressed.
If *legacy* is true, byte-code files are written to their legacy locations
and names, which may overwrite byte-code files created by another version of
@@ -145,8 +187,14 @@ Public functions
.. versionadded:: 3.2
+ .. versionchanged:: 3.5
+ *quiet* parameter was changed to a multilevel value.
+
+ .. versionchanged:: 3.5
+ The *legacy* parameter only writes out ``.pyc`` files, not ``.pyo`` files
+ no matter what the value of *optimize* is.
-.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, legacy=False, optimize=-1)
+.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, quiet=0, legacy=False, optimize=-1)
Byte-compile all the :file:`.py` files found along ``sys.path``. If
*skip_curdir* is true (the default), the current directory is not included
@@ -157,6 +205,12 @@ Public functions
.. versionchanged:: 3.2
Added the *legacy* and *optimize* parameter.
+ .. versionchanged:: 3.5
+ *quiet* parameter was changed to a multilevel value.
+
+ .. versionchanged:: 3.5
+ The *legacy* parameter only writes out ``.pyc`` files, not ``.pyo`` files
+ no matter what the value of *optimize* is.
To force a recompile of all the :file:`.py` files in the :file:`Lib/`
subdirectory and all its subdirectories::
diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst
index 48b4362..11b3916 100644
--- a/Doc/library/concurrent.futures.rst
+++ b/Doc/library/concurrent.futures.rst
@@ -38,7 +38,7 @@ Executor Objects
future = executor.submit(pow, 323, 1235)
print(future.result())
- .. method:: map(func, *iterables, timeout=None)
+ .. method:: map(func, *iterables, timeout=None, chunksize=1)
Equivalent to :func:`map(func, *iterables) <map>` except *func* is executed
asynchronously and several calls to *func* may be made concurrently. The
@@ -48,7 +48,16 @@ Executor Objects
*timeout* can be an int or a float. If *timeout* is not specified or
``None``, there is no limit to the wait time. If a call raises an
exception, then that exception will be raised when its value is
- retrieved from the iterator.
+ retrieved from the iterator. When using :class:`ProcessPoolExecutor`, this
+ method chops *iterables* into a number of chunks which it submits to the
+ pool as separate tasks. The (approximate) size of these chunks can be
+ specified by setting *chunksize* to a positive integer. For very long
+ iterables, using a large value for *chunksize* can significantly improve
+ performance compared to the default size of 1. With :class:`ThreadPoolExecutor`,
+ *chunksize* has no effect.
+
+ .. versionchanged:: 3.5
+ Added the *chunksize* argument.
.. method:: shutdown(wait=True)
@@ -115,11 +124,19 @@ And::
executor.submit(wait_on_future)
-.. class:: ThreadPoolExecutor(max_workers)
+.. class:: ThreadPoolExecutor(max_workers=None)
An :class:`Executor` subclass that uses a pool of at most *max_workers*
threads to execute calls asynchronously.
+ .. versionchanged:: 3.5
+ If *max_workers* is ``None`` or
+ not given, it will default to the number of processors on the machine,
+ multiplied by ``5``, assuming that :class:`ThreadPoolExecutor` is often
+ used to overlap I/O instead of CPU work and the number of workers
+ should be higher than the number of workers
+ for :class:`ProcessPoolExecutor`.
+
.. _threadpoolexecutor-example:
@@ -175,6 +192,8 @@ to a :class:`ProcessPoolExecutor` will result in deadlock.
An :class:`Executor` subclass that executes calls asynchronously using a pool
of at most *max_workers* processes. If *max_workers* is ``None`` or not
given, it will default to the number of processors on the machine.
+ If *max_workers* is lower or equal to ``0``, then a :exc:`ValueError`
+ will be raised.
.. versionchanged:: 3.3
When one of the worker processes terminates abruptly, a
diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst
index 92551bc..c9187a3 100644
--- a/Doc/library/configparser.rst
+++ b/Doc/library/configparser.rst
@@ -11,6 +11,8 @@
.. sectionauthor:: Christopher G. Petrilli <petrilli@amber.org>
.. sectionauthor:: Łukasz Langa <lukasz@langa.pl>
+**Source code:** :source:`Lib/configparser.py`
+
.. index::
pair: .ini; file
pair: configuration; file
@@ -142,12 +144,13 @@ datatypes, you should convert on your own:
>>> float(topsecret['CompressionLevel'])
9.0
-Extracting Boolean values is not that simple, though. Passing the value
-to ``bool()`` would do no good since ``bool('False')`` is still
-``True``. This is why config parsers also provide :meth:`getboolean`.
-This method is case-insensitive and recognizes Boolean values from
-``'yes'``/``'no'``, ``'on'``/``'off'`` and ``'1'``/``'0'`` [1]_.
-For example:
+Since this task is so common, config parsers provide a range of handy getter
+methods to handle integers, floats and booleans. The last one is the most
+interesting because simply passing the value to ``bool()`` would do no good
+since ``bool('False')`` is still ``True``. This is why config parsers also
+provide :meth:`getboolean`. This method is case-insensitive and recognizes
+Boolean values from ``'yes'``/``'no'``, ``'on'``/``'off'``,
+``'true'``/``'false'`` and ``'1'``/``'0'`` [1]_. For example:
.. doctest::
@@ -159,10 +162,8 @@ For example:
True
Apart from :meth:`getboolean`, config parsers also provide equivalent
-:meth:`getint` and :meth:`getfloat` methods, but these are far less
-useful since conversion using :func:`int` and :func:`float` is
-sufficient for these types.
-
+:meth:`getint` and :meth:`getfloat` methods. You can register your own
+converters and customize the provided ones. [1]_
Fallback Values
---------------
@@ -317,11 +318,11 @@ from ``get()`` calls.
.. class:: ExtendedInterpolation()
An alternative handler for interpolation which implements a more advanced
- syntax, used for instance in ``zc.buildout``. Extended interpolation is
+ syntax, used for instance in ``zc.buildout``. Extended interpolation is
using ``${section:option}`` to denote a value from a foreign section.
- Interpolation can span multiple levels. For convenience, if the ``section:``
- part is omitted, interpolation defaults to the current section (and possibly
- the default values from the special section).
+ Interpolation can span multiple levels. For convenience, if the
+ ``section:`` part is omitted, interpolation defaults to the current section
+ (and possibly the default values from the special section).
For example, the configuration specified above with basic interpolation,
would look like this with extended interpolation:
@@ -399,13 +400,13 @@ However, there are a few differences that should be taken into account:
* ``parser.popitem()`` never returns it.
* ``parser.get(section, option, **kwargs)`` - the second argument is **not**
- a fallback value. Note however that the section-level ``get()`` methods are
+ a fallback value. Note however that the section-level ``get()`` methods are
compatible both with the mapping protocol and the classic configparser API.
* ``parser.items()`` is compatible with the mapping protocol (returns a list of
*section_name*, *section_proxy* pairs including the DEFAULTSECT). However,
this method can also be invoked with arguments: ``parser.items(section, raw,
- vars)``. The latter call returns a list of *option*, *value* pairs for
+ vars)``. The latter call returns a list of *option*, *value* pairs for
a specified ``section``, with all interpolations expanded (unless
``raw=True`` is provided).
@@ -539,9 +540,9 @@ the :meth:`__init__` options:
* *delimiters*, default value: ``('=', ':')``
- Delimiters are substrings that delimit keys from values within a section. The
- first occurrence of a delimiting substring on a line is considered a delimiter.
- This means values (but not keys) can contain the delimiters.
+ Delimiters are substrings that delimit keys from values within a section.
+ The first occurrence of a delimiting substring on a line is considered
+ a delimiter. This means values (but not keys) can contain the delimiters.
See also the *space_around_delimiters* argument to
:meth:`ConfigParser.write`.
@@ -553,7 +554,7 @@ the :meth:`__init__` options:
Comment prefixes are strings that indicate the start of a valid comment within
a config file. *comment_prefixes* are used only on otherwise empty lines
(optionally indented) whereas *inline_comment_prefixes* can be used after
- every valid value (e.g. section names, options and empty lines as well). By
+ every valid value (e.g. section names, options and empty lines as well). By
default inline comments are disabled and ``'#'`` and ``';'`` are used as
prefixes for whole line comments.
@@ -563,10 +564,10 @@ the :meth:`__init__` options:
Please note that config parsers don't support escaping of comment prefixes so
using *inline_comment_prefixes* may prevent users from specifying option
- values with characters used as comment prefixes. When in doubt, avoid setting
- *inline_comment_prefixes*. In any circumstances, the only way of storing
- comment prefix characters at the beginning of a line in multiline values is to
- interpolate the prefix, for example::
+ values with characters used as comment prefixes. When in doubt, avoid
+ setting *inline_comment_prefixes*. In any circumstances, the only way of
+ storing comment prefix characters at the beginning of a line in multiline
+ values is to interpolate the prefix, for example::
>>> from configparser import ConfigParser, ExtendedInterpolation
>>> parser = ConfigParser(interpolation=ExtendedInterpolation())
@@ -611,7 +612,7 @@ the :meth:`__init__` options:
When set to ``True``, the parser will not allow for any section or option
duplicates while reading from a single source (using :meth:`read_file`,
- :meth:`read_string` or :meth:`read_dict`). It is recommended to use strict
+ :meth:`read_string` or :meth:`read_dict`). It is recommended to use strict
parsers in new applications.
.. versionchanged:: 3.2
@@ -646,12 +647,12 @@ the :meth:`__init__` options:
The convention of allowing a special section of default values for other
sections or interpolation purposes is a powerful concept of this library,
- letting users create complex declarative configurations. This section is
+ letting users create complex declarative configurations. This section is
normally called ``"DEFAULT"`` but this can be customized to point to any
- other valid section name. Some typical values include: ``"general"`` or
- ``"common"``. The name provided is used for recognizing default sections when
- reading from any source and is used when writing configuration back to
- a file. Its current value can be retrieved using the
+ other valid section name. Some typical values include: ``"general"`` or
+ ``"common"``. The name provided is used for recognizing default sections
+ when reading from any source and is used when writing configuration back to
+ a file. Its current value can be retrieved using the
``parser_instance.default_section`` attribute and may be modified at runtime
(i.e. to convert files from one format to another).
@@ -660,14 +661,30 @@ the :meth:`__init__` options:
Interpolation behaviour may be customized by providing a custom handler
through the *interpolation* argument. ``None`` can be used to turn off
interpolation completely, ``ExtendedInterpolation()`` provides a more
- advanced variant inspired by ``zc.buildout``. More on the subject in the
+ advanced variant inspired by ``zc.buildout``. More on the subject in the
`dedicated documentation section <#interpolation-of-values>`_.
:class:`RawConfigParser` has a default value of ``None``.
+* *converters*, default value: not set
+
+ Config parsers provide option value getters that perform type conversion. By
+ default :meth:`getint`, :meth:`getfloat`, and :meth:`getboolean` are
+ implemented. Should other getters be desirable, users may define them in
+ a subclass or pass a dictionary where each key is a name of the converter and
+ each value is a callable implementing said conversion. For instance, passing
+ ``{'decimal': decimal.Decimal}`` would add :meth:`getdecimal` on both the
+ parser object and all section proxies. In other words, it will be possible
+ to write both ``parser_instance.getdecimal('section', 'key', fallback=0)``
+ and ``parser_instance['section'].getdecimal('key', 0)``.
+
+ If the converter needs to access the state of the parser, it can be
+ implemented as a method on a config parser subclass. If the name of this
+ method starts with ``get``, it will be available on all section proxies, in
+ the dict-compatible form (see the ``getdecimal()`` example above).
More advanced customization may be achieved by overriding default values of
-these parser attributes. The defaults are defined on the classes, so they
-may be overridden by subclasses or by attribute assignment.
+these parser attributes. The defaults are defined on the classes, so they may
+be overridden by subclasses or by attribute assignment.
.. attribute:: BOOLEAN_STATES
@@ -725,10 +742,11 @@ may be overridden by subclasses or by attribute assignment.
.. attribute:: SECTCRE
- A compiled regular expression used to parse section headers. The default
- matches ``[section]`` to the name ``"section"``. Whitespace is considered part
- of the section name, thus ``[ larch ]`` will be read as a section of name
- ``" larch "``. Override this attribute if that's unsuitable. For example:
+ A compiled regular expression used to parse section headers. The default
+ matches ``[section]`` to the name ``"section"``. Whitespace is considered
+ part of the section name, thus ``[ larch ]`` will be read as a section of
+ name ``" larch "``. Override this attribute if that's unsuitable. For
+ example:
.. doctest::
@@ -859,7 +877,7 @@ interpolation if an option used is not defined elsewhere. ::
ConfigParser Objects
--------------------
-.. class:: ConfigParser(defaults=None, dict_type=collections.OrderedDict, allow_no_value=False, delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=configparser.DEFAULTSECT, interpolation=BasicInterpolation())
+.. class:: ConfigParser(defaults=None, dict_type=collections.OrderedDict, allow_no_value=False, delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=configparser.DEFAULTSECT, interpolation=BasicInterpolation(), converters={})
The main configuration parser. When *defaults* is given, it is initialized
into the dictionary of intrinsic defaults. When *dict_type* is given, it
@@ -869,8 +887,8 @@ ConfigParser Objects
When *delimiters* is given, it is used as the set of substrings that
divide keys from values. When *comment_prefixes* is given, it will be used
as the set of substrings that prefix comments in otherwise empty lines.
- Comments can be indented. When *inline_comment_prefixes* is given, it will be
- used as the set of substrings that prefix comments in non-empty lines.
+ Comments can be indented. When *inline_comment_prefixes* is given, it will
+ be used as the set of substrings that prefix comments in non-empty lines.
When *strict* is ``True`` (the default), the parser won't allow for
any section or option duplicates while reading from a single source (file,
@@ -884,13 +902,13 @@ ConfigParser Objects
When *default_section* is given, it specifies the name for the special
section holding default values for other sections and interpolation purposes
- (normally named ``"DEFAULT"``). This value can be retrieved and changed on
+ (normally named ``"DEFAULT"``). This value can be retrieved and changed on
runtime using the ``default_section`` instance attribute.
Interpolation behaviour may be customized by providing a custom handler
through the *interpolation* argument. ``None`` can be used to turn off
interpolation completely, ``ExtendedInterpolation()`` provides a more
- advanced variant inspired by ``zc.buildout``. More on the subject in the
+ advanced variant inspired by ``zc.buildout``. More on the subject in the
`dedicated documentation section <#interpolation-of-values>`_.
All option names used in interpolation will be passed through the
@@ -899,6 +917,12 @@ ConfigParser Objects
converts option names to lower case), the values ``foo %(bar)s`` and ``foo
%(BAR)s`` are equivalent.
+ When *converters* is given, it should be a dictionary where each key
+ represents the name of a type converter and each value is a callable
+ implementing the conversion from string to the desired datatype. Every
+ converter gets its own corresponding :meth:`get*()` method on the parser
+ object and section proxies.
+
.. versionchanged:: 3.1
The default *dict_type* is :class:`collections.OrderedDict`.
@@ -907,6 +931,9 @@ ConfigParser Objects
*empty_lines_in_values*, *default_section* and *interpolation* were
added.
+ .. versionchanged:: 3.5
+ The *converters* argument was added.
+
.. method:: defaults()
@@ -944,7 +971,7 @@ ConfigParser Objects
.. method:: has_option(section, option)
If the given *section* exists, and contains the given *option*, return
- :const:`True`; otherwise return :const:`False`. If the specified
+ :const:`True`; otherwise return :const:`False`. If the specified
*section* is :const:`None` or an empty string, DEFAULT is assumed.
@@ -1069,7 +1096,7 @@ ConfigParser Objects
:meth:`get` method.
.. versionchanged:: 3.2
- Items present in *vars* no longer appear in the result. The previous
+ Items present in *vars* no longer appear in the result. The previous
behaviour mixed actual parser options with variables provided for
interpolation.
@@ -1170,7 +1197,7 @@ RawConfigParser Objects
.. note::
Consider using :class:`ConfigParser` instead which checks types of
- the values to be stored internally. If you don't want interpolation, you
+ the values to be stored internally. If you don't want interpolation, you
can use ``ConfigParser(interpolation=None)``.
@@ -1181,7 +1208,7 @@ RawConfigParser Objects
*default section* name is passed, :exc:`ValueError` is raised.
Type of *section* is not checked which lets users create non-string named
- sections. This behaviour is unsupported and may cause internal errors.
+ sections. This behaviour is unsupported and may cause internal errors.
.. method:: set(section, option, value)
@@ -1282,3 +1309,4 @@ Exceptions
.. [1] Config parsers allow for heavy customization. If you are interested in
changing the behaviour outlined by the footnote reference, consult the
`Customizing Parser Behaviour`_ section.
+
diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst
index 42b5af2..d5a0f09 100644
--- a/Doc/library/constants.rst
+++ b/Doc/library/constants.rst
@@ -45,7 +45,6 @@ A small number of constants live in the built-in namespace. They are:
for more details.
-
.. data:: Ellipsis
The same as ``...``. Special value used mostly in conjunction with extended
diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst
index 6f36864..550b347 100644
--- a/Doc/library/contextlib.rst
+++ b/Doc/library/contextlib.rst
@@ -172,6 +172,16 @@ Functions and classes provided:
.. versionadded:: 3.4
+.. function:: redirect_stderr(new_target)
+
+ Similar to :func:`~contextlib.redirect_stdout` but redirecting
+ :data:`sys.stderr` to another file or file-like object.
+
+ This context manager is :ref:`reentrant <reentrant-cms>`.
+
+ .. versionadded:: 3.5
+
+
.. class:: ContextDecorator()
A base class that enables a context manager to also be used as a decorator.
diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst
index 313dce0..602e909 100644
--- a/Doc/library/csv.rst
+++ b/Doc/library/csv.rst
@@ -5,6 +5,7 @@
:synopsis: Write and read tabular data to and from delimited files.
.. sectionauthor:: Skip Montanaro <skip@pobox.com>
+**Source code:** :source:`Lib/csv.py`
.. index::
single: csv
@@ -418,7 +419,7 @@ Writer Objects
:class:`Writer` objects (:class:`DictWriter` instances and objects returned by
the :func:`writer` function) have the following public methods. A *row* must be
-a sequence of strings or numbers for :class:`Writer` objects and a dictionary
+an iterable of strings or numbers for :class:`Writer` objects and a dictionary
mapping fieldnames to strings or numbers (by passing them through :func:`str`
first) for :class:`DictWriter` objects. Note that complex numbers are written
out surrounded by parens. This may cause some problems for other programs which
@@ -430,6 +431,8 @@ read CSV files (assuming they support complex numbers at all).
Write the *row* parameter to the writer's file object, formatted according to
the current dialect.
+ .. versionchanged:: 3.5
+ Added support of arbitrary iterables.
.. method:: csvwriter.writerows(rows)
diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst
index f3e60b4..e8dfd83 100644
--- a/Doc/library/curses.rst
+++ b/Doc/library/curses.rst
@@ -599,6 +599,13 @@ The module :mod:`curses` defines the following functions:
Only one *ch* can be pushed before :meth:`getch` is called.
+.. function:: update_lines_cols()
+
+ Update :envvar:`LINES` and :envvar:`COLS`. Useful for detecting manual screen resize.
+
+ .. versionadded:: 3.5
+
+
.. function:: unget_wch(ch)
Push *ch* so the next :meth:`get_wch` will return it.
diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst
index 88f3f6e..976cd49 100644
--- a/Doc/library/datetime.rst
+++ b/Doc/library/datetime.rst
@@ -7,6 +7,8 @@
.. sectionauthor:: Tim Peters <tim@zope.com>
.. sectionauthor:: A.M. Kuchling <amk@amk.ca>
+**Source code:** :source:`Lib/datetime.py`
+
.. XXX what order should the types be discussed in?
The :mod:`datetime` module supplies classes for manipulating dates and times in
@@ -757,13 +759,19 @@ Other constructors, all class methods:
:attr:`tzinfo` ``None``. This may raise :exc:`OverflowError`, if the timestamp is
out of the range of values supported by the platform C :c:func:`gmtime` function,
and :exc:`OSError` on :c:func:`gmtime` failure.
- It's common for this to be restricted to years in 1970 through 2038. See also
- :meth:`fromtimestamp`.
+ It's common for this to be restricted to years in 1970 through 2038.
+
+ To get an aware :class:`.datetime` object, call :meth:`fromtimestamp`::
+
+ datetime.fromtimestamp(timestamp, timezone.utc)
+
+ On the POSIX compliant platforms, it is equivalent to the following
+ expression::
- On the POSIX compliant platforms, ``utcfromtimestamp(timestamp)``
- is equivalent to the following expression::
+ datetime(1970, 1, 1, tzinfo=timezone.utc) + timedelta(seconds=timestamp)
- datetime(1970, 1, 1) + timedelta(seconds=timestamp)
+ except the latter formula always supports the full years range: between
+ :const:`MINYEAR` and :const:`MAXYEAR` inclusive.
.. versionchanged:: 3.3
Raise :exc:`OverflowError` instead of :exc:`ValueError` if the timestamp
@@ -1376,10 +1384,13 @@ Supported operations:
* efficient pickling
-* in Boolean contexts, a :class:`.time` object is considered to be true if and
- only if, after converting it to minutes and subtracting :meth:`utcoffset` (or
- ``0`` if that's ``None``), the result is non-zero.
+In boolean contexts, a :class:`.time` object is always considered to be true.
+.. versionchanged:: 3.5
+ Before Python 3.5, a :class:`.time` object was considered to be false if it
+ represented midnight in UTC. This behavior was considered obscure and
+ error-prone and has been removed in Python 3.5. See :issue:`13936` for full
+ details.
Instance methods:
diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst
index e6a82d6..3f3c43d 100644
--- a/Doc/library/dbm.rst
+++ b/Doc/library/dbm.rst
@@ -325,13 +325,18 @@ The module defines the following:
dumbdbm database is created, files with :file:`.dat` and :file:`.dir` extensions
are created.
- The optional *flag* argument is currently ignored; the database is always opened
- for update, and will be created if it does not exist.
+ The optional *flag* argument supports only the semantics of ``'c'``
+ and ``'n'`` values. Other values will default to database being always
+ opened for update, and will be created if it does not exist.
The optional *mode* argument is the Unix mode of the file, used only when the
database has to be created. It defaults to octal ``0o666`` (and will be modified
by the prevailing umask).
+ .. versionchanged:: 3.5
+ :func:`.open` always creates a new database when the flag has the value
+ ``'n'``.
+
In addition to the methods provided by the
:class:`collections.abc.MutableMapping` class, :class:`dumbdbm` objects
provide the following methods:
diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst
index 759be70..5d7ffff 100644
--- a/Doc/library/decimal.rst
+++ b/Doc/library/decimal.rst
@@ -12,6 +12,8 @@
.. moduleauthor:: Stefan Krah <skrah at bytereef.org>
.. sectionauthor:: Raymond D. Hettinger <python at rcn.com>
+**Source code:** :source:`Lib/decimal.py`
+
.. import modules for testing inline doctests with the Sphinx doctest builder
.. testsetup:: *
@@ -742,7 +744,7 @@ Decimal objects
* ``"NaN"``, indicating that the operand is a quiet NaN (Not a Number).
* ``"sNaN"``, indicating that the operand is a signaling NaN.
- .. method:: quantize(exp, rounding=None, context=None, watchexp=True)
+ .. method:: quantize(exp, rounding=None, context=None)
Return a value equal to the first operand after rounding and having the
exponent of the second operand.
@@ -765,14 +767,8 @@ Decimal objects
``context`` argument; if neither argument is given the rounding mode of
the current thread's context is used.
- If *watchexp* is set (default), then an error is returned whenever the
- resulting exponent is greater than :attr:`Emax` or less than
- :attr:`Etiny`.
-
- .. deprecated:: 3.3
- *watchexp* is an implementation detail from the pure Python version
- and is not present in the C version. It will be removed in version
- 3.4, where it defaults to ``True``.
+ An error is returned whenever the resulting exponent is greater than
+ :attr:`Emax` or less than :attr:`Etiny`.
.. method:: radix()
@@ -2092,4 +2088,3 @@ Alternatively, inputs can be rounded upon creation using the
>>> Context(prec=5, rounding=ROUND_DOWN).create_decimal('1.2345678')
Decimal('1.2345')
-
diff --git a/Doc/library/development.rst b/Doc/library/development.rst
index 06e7048..d2b5fa2 100644
--- a/Doc/library/development.rst
+++ b/Doc/library/development.rst
@@ -16,6 +16,7 @@ The list of modules described in this chapter is:
.. toctree::
+ typing.rst
pydoc.rst
doctest.rst
unittest.rst
diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst
index 5f72ea6..38245e7 100644
--- a/Doc/library/difflib.rst
+++ b/Doc/library/difflib.rst
@@ -7,6 +7,8 @@
.. sectionauthor:: Tim Peters <tim_one@users.sourceforge.net>
.. Markup by Fred L. Drake, Jr. <fdrake@acm.org>
+**Source code:** :source:`Lib/difflib.py`
+
.. testsetup::
import sys
@@ -25,7 +27,9 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module.
little fancier than, an algorithm published in the late 1980's by Ratcliff and
Obershelp under the hyperbolic name "gestalt pattern matching." The idea is to
find the longest contiguous matching subsequence that contains no "junk"
- elements (the Ratcliff and Obershelp algorithm doesn't address junk). The same
+ elements; these "junk" elements are ones that are uninteresting in some
+ sense, such as blank lines or whitespace. (Handling junk is an
+ extension to the Ratcliff and Obershelp algorithm.) The same
idea is then applied recursively to the pieces of the sequences to the left and
to the right of the matching subsequence. This does not yield minimal edit
sequences, but does tend to yield matches that "look right" to people.
@@ -100,7 +104,8 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module.
The following methods are public:
- .. method:: make_file(fromlines, tolines, fromdesc='', todesc='', context=False, numlines=5)
+ .. method:: make_file(fromlines, tolines, fromdesc='', todesc='', context=False, \
+ numlines=5, *, charset='utf-8')
Compares *fromlines* and *tolines* (lists of strings) and returns a string which
is a complete HTML file containing a table showing line by line differences with
@@ -119,6 +124,10 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module.
the next difference highlight at the top of the browser without any leading
context).
+ .. versionchanged:: 3.5
+ *charset* keyword-only argument was added. The default charset of
+ HTML document changed from ``'ISO-8859-1'`` to ``'utf-8'``.
+
.. method:: make_table(fromlines, tolines, fromdesc='', todesc='', context=False, numlines=5)
Compares *fromlines* and *tolines* (lists of strings) and returns a string which
@@ -208,7 +217,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module.
Compare *a* and *b* (lists of strings); return a :class:`Differ`\ -style
delta (a :term:`generator` generating the delta lines).
- Optional keyword parameters *linejunk* and *charjunk* are for filter functions
+ Optional keyword parameters *linejunk* and *charjunk* are filtering functions
(or ``None``):
*linejunk*: A function that accepts a single string argument, and returns
@@ -222,7 +231,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module.
*charjunk*: A function that accepts a character (a string of length 1), and
returns if the character is junk, or false if not. The default is module-level
function :func:`IS_CHARACTER_JUNK`, which filters out whitespace characters (a
- blank or tab; note: bad idea to include newline in this!).
+ blank or tab; it's a bad idea to include newline in this!).
:file:`Tools/scripts/ndiff.py` is a command-line front-end to this function.
@@ -306,6 +315,21 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module.
See :ref:`difflib-interface` for a more detailed example.
+.. function:: diff_bytes(dfunc, a, b, fromfile=b'', tofile=b'', fromfiledate=b'', tofiledate=b'', n=3, lineterm=b'\\n')
+
+ Compare *a* and *b* (lists of bytes objects) using *dfunc*; yield a
+ sequence of delta lines (also bytes) in the format returned by *dfunc*.
+ *dfunc* must be a callable, typically either :func:`unified_diff` or
+ :func:`context_diff`.
+
+ Allows you to compare data with unknown or inconsistent encoding. All
+ inputs except *n* must be bytes objects, not str. Works by losslessly
+ converting all inputs (except *n*) to str, and calling ``dfunc(a, b,
+ fromfile, tofile, fromfiledate, tofiledate, n, lineterm)``. The output of
+ *dfunc* is then converted back to bytes, so the delta lines that you
+ receive have the same unknown/inconsistent encodings as *a* and *b*.
+
+ .. versionadded:: 3.5
.. function:: IS_LINE_JUNK(line)
@@ -622,6 +646,12 @@ The :class:`Differ` class has this constructor:
length 1), and returns true if the character is junk. The default is ``None``,
meaning that no character is considered junk.
+ These junk-filtering functions speed up matching to find
+ differences and do not cause any differing lines or characters to
+ be ignored. Read the description of the
+ :meth:`~SequenceMatcher.find_longest_match` method's *isjunk*
+ parameter for an explanation.
+
:class:`Differ` objects are used (deltas generated) via a single method:
@@ -713,65 +743,4 @@ This example shows how to use difflib to create a ``diff``-like utility.
It is also contained in the Python source distribution, as
:file:`Tools/scripts/diff.py`.
-.. testcode::
-
- """ Command line interface to difflib.py providing diffs in four formats:
-
- * ndiff: lists every line and highlights interline changes.
- * context: highlights clusters of changes in a before/after format.
- * unified: highlights clusters of changes in an inline format.
- * html: generates side by side comparison with change highlights.
-
- """
-
- import sys, os, time, difflib, optparse
-
- def main():
- # Configure the option parser
- usage = "usage: %prog [options] fromfile tofile"
- parser = optparse.OptionParser(usage)
- parser.add_option("-c", action="store_true", default=False,
- help='Produce a context format diff (default)')
- parser.add_option("-u", action="store_true", default=False,
- help='Produce a unified format diff')
- hlp = 'Produce HTML side by side diff (can use -c and -l in conjunction)'
- parser.add_option("-m", action="store_true", default=False, help=hlp)
- parser.add_option("-n", action="store_true", default=False,
- help='Produce a ndiff format diff')
- parser.add_option("-l", "--lines", type="int", default=3,
- help='Set number of context lines (default 3)')
- (options, args) = parser.parse_args()
-
- if len(args) == 0:
- parser.print_help()
- sys.exit(1)
- if len(args) != 2:
- parser.error("need to specify both a fromfile and tofile")
-
- n = options.lines
- fromfile, tofile = args # as specified in the usage string
-
- # we're passing these as arguments to the diff function
- fromdate = time.ctime(os.stat(fromfile).st_mtime)
- todate = time.ctime(os.stat(tofile).st_mtime)
- with open(fromfile) as fromf, open(tofile) as tof:
- fromlines, tolines = list(fromf), list(tof)
-
- if options.u:
- diff = difflib.unified_diff(fromlines, tolines, fromfile, tofile,
- fromdate, todate, n=n)
- elif options.n:
- diff = difflib.ndiff(fromlines, tolines)
- elif options.m:
- diff = difflib.HtmlDiff().make_file(fromlines, tolines, fromfile,
- tofile, context=options.c,
- numlines=n)
- else:
- diff = difflib.context_diff(fromlines, tolines, fromfile, tofile,
- fromdate, todate, n=n)
-
- # we're using writelines because diff is a generator
- sys.stdout.writelines(diff)
-
- if __name__ == '__main__':
- main()
+.. literalinclude:: ../../Tools/scripts/diff.py
diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst
index 273fb20..1bcb3a4 100644
--- a/Doc/library/dis.rst
+++ b/Doc/library/dis.rst
@@ -48,8 +48,9 @@ code.
.. class:: Bytecode(x, *, first_line=None, current_offset=None)
- Analyse the bytecode corresponding to a function, method, string of source
- code, or a code object (as returned by :func:`compile`).
+
+ Analyse the bytecode corresponding to a function, generator, method, string
+ of source code, or a code object (as returned by :func:`compile`).
This is a convenience wrapper around many of the functions listed below, most
notably :func:`get_instructions`, as iterating over a :class:`Bytecode`
@@ -109,7 +110,7 @@ operation is being performed, so the intermediate analysis object isn't useful:
.. function:: code_info(x)
Return a formatted multi-line string with detailed code object information
- for the supplied function, method, source code string or code object.
+ for the supplied function, generator, method, source code string or code object.
Note that the exact contents of code info strings are highly implementation
dependent and they may change arbitrarily across Python VMs or Python
@@ -136,11 +137,11 @@ operation is being performed, so the intermediate analysis object isn't useful:
.. function:: dis(x=None, *, file=None)
Disassemble the *x* object. *x* can denote either a module, a class, a
- method, a function, a code object, a string of source code or a byte sequence
- of raw bytecode. For a module, it disassembles all functions. For a class,
- it disassembles all methods. For a code object or sequence of raw bytecode,
- it prints one line per bytecode instruction. Strings are first compiled to
- code objects with the :func:`compile` built-in function before being
+ method, a function, a generator, a code object, a string of source code or
+ a byte sequence of raw bytecode. For a module, it disassembles all functions.
+ For a class, it disassembles all methods. For a code object or sequence of
+ raw bytecode, it prints one line per bytecode instruction. Strings are first
+ compiled to code objects with the :func:`compile` built-in function before being
disassembled. If no object is provided, this function disassembles the last
traceback.
@@ -345,6 +346,14 @@ result back on the stack.
Implements ``TOS = iter(TOS)``.
+.. opcode:: GET_YIELD_FROM_ITER
+
+ If ``TOS`` is a :term:`generator iterator` or :term:`coroutine` object
+ it is left as is. Otherwise, implements ``TOS = iter(TOS)``.
+
+ .. versionadded:: 3.5
+
+
**Binary operations**
Binary operations remove the top of the stack (TOS) and the second top-most
@@ -361,6 +370,13 @@ result back on the stack.
Implements ``TOS = TOS1 * TOS``.
+.. opcode:: BINARY_MATRIX_MULTIPLY
+
+ Implements ``TOS = TOS1 @ TOS``.
+
+ .. versionadded:: 3.5
+
+
.. opcode:: BINARY_FLOOR_DIVIDE
Implements ``TOS = TOS1 // TOS``.
@@ -433,6 +449,13 @@ the original TOS1.
Implements in-place ``TOS = TOS1 * TOS``.
+.. opcode:: INPLACE_MATRIX_MULTIPLY
+
+ Implements in-place ``TOS = TOS1 @ TOS``.
+
+ .. versionadded:: 3.5
+
+
.. opcode:: INPLACE_FLOOR_DIVIDE
Implements in-place ``TOS = TOS1 // TOS``.
@@ -493,6 +516,40 @@ the original TOS1.
Implements ``del TOS1[TOS]``.
+**Coroutine opcodes**
+
+.. opcode:: GET_AWAITABLE
+
+ Implements ``TOS = get_awaitable(TOS)``, where ``get_awaitable(o)``
+ returns ``o`` if ``o`` is a coroutine object or a generator object with
+ the CO_ITERABLE_COROUTINE flag, or resolves
+ ``o.__await__``.
+
+
+.. opcode:: GET_AITER
+
+ Implements ``TOS = get_awaitable(TOS.__aiter__())``. See ``GET_AWAITABLE``
+ for details about ``get_awaitable``
+
+
+.. opcode:: GET_ANEXT
+
+ Implements ``PUSH(get_awaitable(TOS.__anext__()))``. See ``GET_AWAITABLE``
+ for details about ``get_awaitable``
+
+
+.. opcode:: BEFORE_ASYNC_WITH
+
+ Resolves ``__aenter__`` and ``__aexit__`` from the object on top of the
+ stack. Pushes ``__aexit__`` and result of ``__aenter__()`` to the stack.
+
+
+.. opcode:: SETUP_ASYNC_WITH
+
+ Creates a new frame object.
+
+
+
**Miscellaneous opcodes**
.. opcode:: PRINT_EXPR
@@ -597,7 +654,7 @@ iterations of the loop.
:opcode:`UNPACK_SEQUENCE`).
-.. opcode:: WITH_CLEANUP
+.. opcode:: WITH_CLEANUP_START
Cleans up the stack when a :keyword:`with` statement block exits. TOS is the
context manager's :meth:`__exit__` bound method. Below TOS are 1--3 values
@@ -609,7 +666,13 @@ iterations of the loop.
* (SECOND, THIRD, FOURTH) = exc_info()
In the last case, ``TOS(SECOND, THIRD, FOURTH)`` is called, otherwise
- ``TOS(None, None, None)``. In addition, TOS is removed from the stack.
+ ``TOS(None, None, None)``. Pushes SECOND and result of the call
+ to the stack.
+
+
+.. opcode:: WITH_CLEANUP_FINISH
+
+ Pops exception type and result of 'exit' function call from the stack.
If the stack represents an exception, *and* the function call returns a
'true' value, this information is "zapped" and replaced with a single
@@ -795,10 +858,6 @@ the more significant byte last.
Pushes a try block from a try-except clause onto the block stack. *delta*
points to the finally block.
-.. opcode:: STORE_MAP
-
- Store a key and value pair in a dictionary. Pops the key and value while
- leaving the dictionary on the stack.
.. opcode:: LOAD_FAST (var_num)
diff --git a/Doc/library/distribution.rst b/Doc/library/distribution.rst
index c4954d1..3e6e84b 100644
--- a/Doc/library/distribution.rst
+++ b/Doc/library/distribution.rst
@@ -12,3 +12,4 @@ with a local index server, or without any index server at all.
distutils.rst
ensurepip.rst
venv.rst
+ zipapp.rst
diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst
index 2e372de..9aa9ea6 100644
--- a/Doc/library/doctest.rst
+++ b/Doc/library/doctest.rst
@@ -1058,15 +1058,9 @@ from text files and modules with doctests:
This function uses the same search technique as :func:`testmod`.
- .. note::
- Unlike :func:`testmod` and :class:`DocTestFinder`, this function raises
- a :exc:`ValueError` if *module* contains no docstrings. You can prevent
- this error by passing a :class:`DocTestFinder` instance as the
- *test_finder* argument with its *exclude_empty* keyword argument set
- to ``False``::
-
- >>> finder = doctest.DocTestFinder(exclude_empty=False)
- >>> suite = doctest.DocTestSuite(test_finder=finder)
+ .. versionchanged:: 3.5
+ :func:`DocTestSuite` returns an empty :class:`unittest.TestSuite` if *module*
+ contains no docstrings instead of raising :exc:`ValueError`.
Under the covers, :func:`DocTestSuite` creates a :class:`unittest.TestSuite` out
diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst
index aeea942..b91f26d 100644
--- a/Doc/library/email.message.rst
+++ b/Doc/library/email.message.rst
@@ -578,6 +578,15 @@ Here are the methods of the :class:`Message` class:
will be *failobj*.
+ .. method:: get_content_disposition()
+
+ Return the lowercased value (without parameters) of the message's
+ :mailheader:`Content-Disposition` header if it has one, or ``None``. The
+ possible values for this method are *inline*, *attachment* or ``None``
+ if the message follows :rfc:`2183`.
+
+ .. versionadded:: 3.5
+
.. method:: walk()
The :meth:`walk` method is an all-purpose generator which can be used to
diff --git a/Doc/library/email.mime.rst b/Doc/library/email.mime.rst
index 1d70225..67d0a67 100644
--- a/Doc/library/email.mime.rst
+++ b/Doc/library/email.mime.rst
@@ -195,7 +195,8 @@ Here are the classes:
set of the text and is passed as an argument to the
:class:`~email.mime.nonmultipart.MIMENonMultipart` constructor; it defaults
to ``us-ascii`` if the string contains only ``ascii`` code points, and
- ``utf-8`` otherwise.
+ ``utf-8`` otherwise. The *_charset* parameter accepts either a string or a
+ :class:`~email.charset.Charset` instance.
Unless the *_charset* argument is explicitly set to ``None``, the
MIMEText object created will have both a :mailheader:`Content-Type` header
@@ -206,3 +207,6 @@ Here are the classes:
``Content-Transfer-Encoding`` header, after which a ``set_payload`` call
will automatically encode the new payload (and add a new
:mailheader:`Content-Transfer-Encoding` header).
+
+ .. versionchanged:: 3.5
+ *_charset* also accepts :class:`~email.charset.Charset` instances.
diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst
index d4e3fc1..045b119 100644
--- a/Doc/library/email.policy.rst
+++ b/Doc/library/email.policy.rst
@@ -187,6 +187,18 @@ added matters. To illustrate::
:const:`False` (the default), defects will be passed to the
:meth:`register_defect` method.
+
+
+ .. attribute:: mangle_from\_
+
+ If :const:`True`, lines starting with *"From "* in the body are
+ escaped by putting a ``>`` in front of them. This parameter is used when
+ the message is being serialized by a generator.
+ Default: :const:`False`.
+
+ .. versionadded:: 3.5
+ The *mangle_from_* parameter.
+
The following :class:`Policy` method is intended to be called by code using
the email library to create policy instances with custom settings:
@@ -319,6 +331,13 @@ added matters. To illustrate::
:const:`compat32`, that is used as the default policy. Thus the default
behavior of the email package is to maintain compatibility with Python 3.2.
+ The following attributes have values that are different from the
+ :class:`Policy` default:
+
+ .. attribute:: mangle_from_
+
+ The default is ``True``.
+
The class provides the following concrete implementations of the
abstract methods of :class:`Policy`:
@@ -356,6 +375,14 @@ added matters. To illustrate::
line breaks and any (RFC invalid) binary data it may contain.
+An instance of :class:`Compat32` is provided as a module constant:
+
+.. data:: compat32
+
+ An instance of :class:`Compat32`, providing backward compatibility with the
+ behavior of the email package in Python 3.2.
+
+
.. note::
The documentation below describes new policies that are included in the
@@ -378,6 +405,14 @@ added matters. To illustrate::
In addition to the settable attributes listed above that apply to all
policies, this policy adds the following additional attributes:
+ .. attribute:: utf8
+
+ If ``False``, follow :rfc:`5322`, supporting non-ASCII characters in
+ headers by encoding them as "encoded words". If ``True``, follow
+ :rfc:`6532` and use ``utf-8`` encoding for headers. Messages
+ formatted in this way may be passed to SMTP servers that support
+ the ``SMTPUTF8`` extension (:rfc:`6531`).
+
.. attribute:: refold_source
If the value for a header in the ``Message`` object originated from a
@@ -499,6 +534,14 @@ more closely to the RFCs relevant to their domains.
Like ``default``, but with ``linesep`` set to ``\r\n``, which is RFC
compliant.
+.. data:: SMTPUTF8
+
+ The same as ``SMTP`` except that :attr:`~EmailPolicy.utf8` is ``True``.
+ Useful for serializing messages to a message store without using encoded
+ words in the headers. Should only be used for SMTP trasmission if the
+ sender or recipient addresses have non-ASCII characters (the
+ :meth:`smtplib.SMTP.send_message` method handles this automatically).
+
.. data:: HTTP
Suitable for serializing headers with for use in HTTP traffic. Like
diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst
index cf09559..d3b838c 100644
--- a/Doc/library/enum.rst
+++ b/Doc/library/enum.rst
@@ -314,11 +314,11 @@ Then::
>>> str(Mood.funky)
'my custom str! 1'
-The rules for what is allowed are as follows: _sunder_ names (starting and
-ending with a single underscore) are reserved by enum and cannot be used;
-all other attributes defined within an enumeration will become members of this
-enumeration, with the exception of *__dunder__* names and descriptors (methods
-are also descriptors).
+The rules for what is allowed are as follows: names that start and end with a
+with a single underscore are reserved by enum and cannot be used; all other
+attributes defined within an enumeration will become members of this
+enumeration, with the exception of special methods (:meth:`__str__`,
+:meth:`__add__`, etc.) and descriptors (methods are also descriptors).
Note: if your enumeration defines :meth:`__new__` and/or :meth:`__init__` then
whatever value(s) were given to the enum member will be passed into those
@@ -400,7 +400,8 @@ The second argument is the *source* of enumeration member names. It can be a
whitespace-separated string of names, a sequence of names, a sequence of
2-tuples with key/value pairs, or a mapping (e.g. dictionary) of names to
values. The last two options enable assigning arbitrary values to
-enumerations; the others auto-assign increasing integers starting with 1. A
+enumerations; the others auto-assign increasing integers starting with 1 (use
+the ``start`` parameter to specify a different starting value). A
new class derived from :class:`Enum` is returned. In other words, the above
assignment to :class:`Animal` is equivalent to::
@@ -438,12 +439,12 @@ SomeData in the global scope::
The complete signature is::
- Enum(value='NewEnumName', names=<...>, *, module='...', qualname='...', type=<mixed-in class>)
+ Enum(value='NewEnumName', names=<...>, *, module='...', qualname='...', type=<mixed-in class>, start=1)
:value: What the new Enum class will record as its name.
:names: The Enum members. This can be a whitespace or comma separated string
- (values will start at 1)::
+ (values will start at 1 unless otherwise specified)::
'red green blue' | 'red,green,blue' | 'red, green, blue'
@@ -465,6 +466,11 @@ The complete signature is::
:type: type to mix in to new Enum class.
+:start: number to start counting at if only names are passed in
+
+.. versionchanged:: 3.5
+ The *start* parameter was added.
+
Derived Enumerations
--------------------
diff --git a/Doc/library/errno.rst b/Doc/library/errno.rst
index d2163b6..22a5cbc 100644
--- a/Doc/library/errno.rst
+++ b/Doc/library/errno.rst
@@ -41,7 +41,10 @@ defined by the module. The specific list of defined symbols is available as
.. data:: EINTR
- Interrupted system call
+ Interrupted system call.
+
+ .. seealso::
+ This error is mapped to the exception :exc:`InterruptedError`.
.. data:: EIO
diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst
index 271a5c8..0a422b2 100644
--- a/Doc/library/exceptions.rst
+++ b/Doc/library/exceptions.rst
@@ -162,7 +162,8 @@ The following exceptions are the exceptions that are usually raised.
.. exception:: GeneratorExit
- Raised when a :term:`generator`\'s :meth:`close` method is called. It
+ Raised when a :term:`generator` or :term:`coroutine` is closed;
+ see :meth:`generator.close` and :meth:`coroutine.close`. It
directly inherits from :exc:`BaseException` instead of :exc:`Exception` since
it is technically not an error.
@@ -281,6 +282,16 @@ The following exceptions are the exceptions that are usually raised.
handling in C, most floating point operations are not checked.
+.. exception:: RecursionError
+
+ This exception is derived from :exc:`RuntimeError`. It is raised when the
+ interpreter detects that the maximum recursion depth (see
+ :func:`sys.getrecursionlimit`) is exceeded.
+
+ .. versionadded:: 3.5
+ Previously, a plain :exc:`RuntimeError` was raised.
+
+
.. exception:: ReferenceError
This exception is raised when a weak reference proxy, created by the
@@ -306,14 +317,30 @@ The following exceptions are the exceptions that are usually raised.
given as an argument when constructing the exception, and defaults
to :const:`None`.
- When a generator function returns, a new :exc:`StopIteration` instance is
+ When a :term:`generator` or :term:`coroutine` function
+ returns, a new :exc:`StopIteration` instance is
raised, and the value returned by the function is used as the
:attr:`value` parameter to the constructor of the exception.
+ If a generator function defined in the presence of a ``from __future__
+ import generator_stop`` directive raises :exc:`StopIteration`, it will be
+ converted into a :exc:`RuntimeError` (retaining the :exc:`StopIteration`
+ as the new exception's cause).
+
.. versionchanged:: 3.3
Added ``value`` attribute and the ability for generator functions to
use it to return a value.
+ .. versionchanged:: 3.5
+ Introduced the RuntimeError transformation.
+
+.. exception:: StopAsyncIteration
+
+ Must be raised by :meth:`__anext__` method of an
+ :term:`asynchronous iterator` object to stop the iteration.
+
+ .. versionadded:: 3.5
+
.. exception:: SyntaxError
Raised when the parser encounters a syntax error. This may occur in an
@@ -536,7 +563,12 @@ depending on the system error code.
.. exception:: InterruptedError
Raised when a system call is interrupted by an incoming signal.
- Corresponds to :c:data:`errno` ``EINTR``.
+ Corresponds to :c:data:`errno` :py:data:`~errno.EINTR`.
+
+ .. versionchanged:: 3.5
+ Python now retries system calls when a syscall is interrupted by a
+ signal, except if the signal handler raises an exception (see :pep:`475`
+ for the rationale), instead of raising :exc:`InterruptedError`.
.. exception:: IsADirectoryError
diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst
index eb2016a..3a5badd 100644
--- a/Doc/library/faulthandler.rst
+++ b/Doc/library/faulthandler.rst
@@ -47,6 +47,9 @@ Dumping the traceback
Dump the tracebacks of all threads into *file*. If *all_threads* is
``False``, dump only the current thread.
+ .. versionchanged:: 3.5
+ Added support for passing file descriptor to this function.
+
Fault handler state
-------------------
@@ -59,6 +62,12 @@ Fault handler state
produce tracebacks for every running thread. Otherwise, dump only the current
thread.
+ The *file* must be kept open until the fault handler is disabled: see
+ :ref:`issue with file descriptors <faulthandler-fd>`.
+
+ .. versionchanged:: 3.5
+ Added support for passing file descriptor to this function.
+
.. function:: disable()
Disable the fault handler: uninstall the signal handlers installed by
@@ -82,9 +91,16 @@ Dumping the tracebacks after a timeout
call replaces previous parameters and resets the timeout. The timer has a
sub-second resolution.
+ The *file* must be kept open until the traceback is dumped or
+ :func:`cancel_dump_traceback_later` is called: see :ref:`issue with file
+ descriptors <faulthandler-fd>`.
+
This function is implemented using a watchdog thread and therefore is not
available if Python is compiled with threads disabled.
+ .. versionchanged:: 3.5
+ Added support for passing file descriptor to this function.
+
.. function:: cancel_dump_traceback_later()
Cancel the last call to :func:`dump_traceback_later`.
@@ -99,8 +115,14 @@ Dumping the traceback on a user signal
the traceback of all threads, or of the current thread if *all_threads* is
``False``, into *file*. Call the previous handler if chain is ``True``.
+ The *file* must be kept open until the signal is unregistered by
+ :func:`unregister`: see :ref:`issue with file descriptors <faulthandler-fd>`.
+
Not available on Windows.
+ .. versionchanged:: 3.5
+ Added support for passing file descriptor to this function.
+
.. function:: unregister(signum)
Unregister a user signal: uninstall the handler of the *signum* signal
@@ -110,6 +132,8 @@ Dumping the traceback on a user signal
Not available on Windows.
+.. _faulthandler-fd:
+
Issue with file descriptors
---------------------------
diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst
index 8e932fb..432140f 100644
--- a/Doc/library/fcntl.rst
+++ b/Doc/library/fcntl.rst
@@ -28,41 +28,41 @@ descriptor.
The module defines the following functions:
-.. function:: fcntl(fd, op[, arg])
+.. function:: fcntl(fd, cmd, arg=0)
- Perform the operation *op* on file descriptor *fd* (file objects providing
+ Perform the operation *cmd* on file descriptor *fd* (file objects providing
a :meth:`~io.IOBase.fileno` method are accepted as well). The values used
- for *op* are operating system dependent, and are available as constants
+ for *cmd* are operating system dependent, and are available as constants
in the :mod:`fcntl` module, using the same names as used in the relevant C
- header files. The argument *arg* is optional, and defaults to the integer
- value ``0``. When present, it can either be an integer value, or a string.
- With the argument missing or an integer value, the return value of this function
- is the integer return value of the C :c:func:`fcntl` call. When the argument is
- a string it represents a binary structure, e.g. created by :func:`struct.pack`.
- The binary data is copied to a buffer whose address is passed to the C
- :c:func:`fcntl` call. The return value after a successful call is the contents
- of the buffer, converted to a string object. The length of the returned string
- will be the same as the length of the *arg* argument. This is limited to 1024
- bytes. If the information returned in the buffer by the operating system is
- larger than 1024 bytes, this is most likely to result in a segmentation
- violation or a more subtle data corruption.
+ header files. The argument *arg* can either be an integer value, or a
+ :class:`bytes` object. With an integer value, the return value of this
+ function is the integer return value of the C :c:func:`fcntl` call. When
+ the argument is bytes it represents a binary structure, e.g. created by
+ :func:`struct.pack`. The binary data is copied to a buffer whose address is
+ passed to the C :c:func:`fcntl` call. The return value after a successful
+ call is the contents of the buffer, converted to a :class:`bytes` object.
+ The length of the returned object will be the same as the length of the
+ *arg* argument. This is limited to 1024 bytes. If the information returned
+ in the buffer by the operating system is larger than 1024 bytes, this is
+ most likely to result in a segmentation violation or a more subtle data
+ corruption.
If the :c:func:`fcntl` fails, an :exc:`OSError` is raised.
-.. function:: ioctl(fd, op[, arg[, mutate_flag]])
+.. function:: ioctl(fd, request, arg=0, mutate_flag=True)
This function is identical to the :func:`~fcntl.fcntl` function, except
that the argument handling is even more complicated.
- The op parameter is limited to values that can fit in 32-bits.
- Additional constants of interest for use as the *op* argument can be
+ The *request* parameter is limited to values that can fit in 32-bits.
+ Additional constants of interest for use as the *request* argument can be
found in the :mod:`termios` module, under the same names as used in
the relevant C header files.
- The parameter *arg* can be one of an integer, absent (treated identically to the
- integer ``0``), an object supporting the read-only buffer interface (most likely
- a plain Python string) or an object supporting the read-write buffer interface.
+ The parameter *arg* can be one of an integer, an object supporting the
+ read-only buffer interface (like :class:`bytes`) or an object supporting
+ the read-write buffer interface (like :class:`bytearray`).
In all but the last case, behaviour is as for the :func:`~fcntl.fcntl`
function.
@@ -72,7 +72,7 @@ The module defines the following functions:
If it is false, the buffer's mutability is ignored and behaviour is as for a
read-only buffer, except that the 1024 byte limit mentioned above is avoided --
- so long as the buffer you pass is as least as long as what the operating system
+ so long as the buffer you pass is at least as long as what the operating system
wants to put there, things should work.
If *mutate_flag* is true (the default), then the buffer is (in effect) passed
@@ -97,25 +97,25 @@ The module defines the following functions:
array('h', [13341])
-.. function:: flock(fd, op)
+.. function:: flock(fd, operation)
- Perform the lock operation *op* on file descriptor *fd* (file objects providing
+ Perform the lock operation *operation* on file descriptor *fd* (file objects providing
a :meth:`~io.IOBase.fileno` method are accepted as well). See the Unix manual
:manpage:`flock(2)` for details. (On some systems, this function is emulated
using :c:func:`fcntl`.)
-.. function:: lockf(fd, operation, [length, [start, [whence]]])
+.. function:: lockf(fd, cmd, len=0, start=0, whence=0)
This is essentially a wrapper around the :func:`~fcntl.fcntl` locking calls.
- *fd* is the file descriptor of the file to lock or unlock, and *operation*
+ *fd* is the file descriptor of the file to lock or unlock, and *cmd*
is one of the following values:
* :const:`LOCK_UN` -- unlock
* :const:`LOCK_SH` -- acquire a shared lock
* :const:`LOCK_EX` -- acquire an exclusive lock
- When *operation* is :const:`LOCK_SH` or :const:`LOCK_EX`, it can also be
+ When *cmd* is :const:`LOCK_SH` or :const:`LOCK_EX`, it can also be
bitwise ORed with :const:`LOCK_NB` to avoid blocking on lock acquisition.
If :const:`LOCK_NB` is used and the lock cannot be acquired, an
:exc:`OSError` will be raised and the exception will have an *errno*
@@ -124,7 +124,7 @@ The module defines the following functions:
systems, :const:`LOCK_EX` can only be used if the file descriptor refers to a
file opened for writing.
- *length* is the number of bytes to lock, *start* is the byte offset at
+ *len* is the number of bytes to lock, *start* is the byte offset at
which the lock starts, relative to *whence*, and *whence* is as with
:func:`io.IOBase.seek`, specifically:
@@ -133,7 +133,7 @@ The module defines the following functions:
* :const:`2` -- relative to the end of the file (:data:`os.SEEK_END`)
The default for *start* is 0, which means to start at the beginning of the file.
- The default for *length* is 0 which means to lock to the end of the file. The
+ The default for *len* is 0 which means to lock to the end of the file. The
default for *whence* is also 0.
Examples (all on a SVR4 compliant system)::
@@ -147,9 +147,9 @@ Examples (all on a SVR4 compliant system)::
rv = fcntl.fcntl(f, fcntl.F_SETLKW, lockdata)
Note that in the first example the return value variable *rv* will hold an
-integer value; in the second example it will hold a string value. The structure
-lay-out for the *lockdata* variable is system dependent --- therefore using the
-:func:`flock` call may be better.
+integer value; in the second example it will hold a :class:`bytes` object. The
+structure lay-out for the *lockdata* variable is system dependent --- therefore
+using the :func:`flock` call may be better.
.. seealso::
diff --git a/Doc/library/formatter.rst b/Doc/library/formatter.rst
index 1847a80..a515f74 100644
--- a/Doc/library/formatter.rst
+++ b/Doc/library/formatter.rst
@@ -5,7 +5,7 @@
:synopsis: Generic output formatter and device interface.
:deprecated:
-.. deprecated:: 3.4
+.. deprecated-removed:: 3.4 3.6
Due to lack of usage, the formatter module has been deprecated and is slated
for removal in Python 3.6.
diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst
index c2c7401..e0f0682 100644
--- a/Doc/library/fractions.rst
+++ b/Doc/library/fractions.rst
@@ -172,6 +172,9 @@ another rational number, or from a string.
sign as *b* if *b* is nonzero; otherwise it takes the sign of *a*. ``gcd(0,
0)`` returns ``0``.
+ .. deprecated:: 3.5
+ Use :func:`math.gcd` instead.
+
.. seealso::
diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index 5377335..409f6c4 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -156,11 +156,12 @@ are always available. They are listed here in alphabetical order.
.. function:: chr(i)
- Return the string representing a character whose Unicode code point is the integer
- *i*. For example, ``chr(97)`` returns the string ``'a'``. This is the
- inverse of :func:`ord`. The valid range for the argument is from 0 through
- 1,114,111 (0x10FFFF in base 16). :exc:`ValueError` will be raised if *i* is
- outside that range.
+ Return the string representing a character whose Unicode code point is the
+ integer *i*. For example, ``chr(97)`` returns the string ``'a'``, while
+ ``chr(957)`` returns the string ``'ν'``. This is the inverse of :func:`ord`.
+
+ The valid range for the argument is from 0 through 1,114,111 (0x10FFFF in
+ base 16). :exc:`ValueError` will be raised if *i* is outside that range.
.. function:: classmethod(function)
@@ -972,9 +973,11 @@ are always available. They are listed here in alphabetical order.
Characters not supported by the encoding are replaced with the
appropriate XML character reference ``&#nnn;``.
- * ``'backslashreplace'`` (also only supported when writing)
- replaces unsupported characters with Python's backslashed escape
- sequences.
+ * ``'backslashreplace'`` replaces malformed data by Python's backslashed
+ escape sequences.
+
+ * ``'namereplace'`` (also only supported when writing)
+ replaces unsupported characters with ``\N{...}`` escape sequences.
.. index::
single: universal newlines; open() built-in function
@@ -999,8 +1002,8 @@ are always available. They are listed here in alphabetical order.
If *closefd* is ``False`` and a file descriptor rather than a filename was
given, the underlying file descriptor will be kept open when the file is
- closed. If a filename is given *closefd* has no effect and must be ``True``
- (the default).
+ closed. If a filename is given *closefd* must be ``True`` (the default)
+ otherwise an error will be raised.
A custom opener can be used by passing a callable as *opener*. The underlying
file descriptor for the file object is then obtained by calling *opener* with
@@ -1062,14 +1065,18 @@ are always available. They are listed here in alphabetical order.
The ``'U'`` mode.
+ .. versionchanged:: 3.5
+ If the system call is interrupted and the signal handler does not raise an
+ exception, the function now retries the system call instead of raising an
+ :exc:`InterruptedError` exception (see :pep:`475` for the rationale).
+
-.. XXX works for bytes too, but should it?
.. function:: ord(c)
Given a string representing one Unicode character, return an integer
- representing the Unicode code
- point of that character. For example, ``ord('a')`` returns the integer ``97``
- and ``ord('\u2020')`` returns ``8224``. This is the inverse of :func:`chr`.
+ representing the Unicode code point of that character. For example,
+ ``ord('a')`` returns the integer ``97`` and ``ord('ν')`` returns ``957``.
+ This is the inverse of :func:`chr`.
.. function:: pow(x, y[, z])
@@ -1186,6 +1193,9 @@ are always available. They are listed here in alphabetical order.
The returned property object also has the attributes ``fget``, ``fset``, and
``fdel`` corresponding to the constructor arguments.
+ .. versionchanged:: 3.5
+ The docstrings of property objects are now writeable.
+
.. _func-range:
.. function:: range(stop)
@@ -1218,8 +1228,8 @@ are always available. They are listed here in alphabetical order.
.. function:: round(number[, ndigits])
Return the floating point value *number* rounded to *ndigits* digits after
- the decimal point. If *ndigits* is omitted, it defaults to zero. Delegates
- to ``number.__round__(ndigits)``.
+ the decimal point. If *ndigits* is omitted, it returns the nearest integer
+ to its input. Delegates to ``number.__round__(ndigits)``.
For the built-in types supporting :func:`round`, values are rounded to the
closest multiple of 10 to the power minus *ndigits*; if two multiples are
@@ -1485,7 +1495,9 @@ are always available. They are listed here in alphabetical order.
The left-to-right evaluation order of the iterables is guaranteed. This
makes possible an idiom for clustering a data series into n-length groups
- using ``zip(*[iter(s)]*n)``.
+ using ``zip(*[iter(s)]*n)``. This repeats the *same* iterator ``n`` times
+ so that each output tuple has the result of ``n`` calls to the iterator.
+ This has the effect of dividing the input into n-length chunks.
:func:`zip` should only be used with unequal length inputs when you don't
care about trailing, unmatched values from the longer iterables. If those
diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst
index 8135542..d11c2e1 100644
--- a/Doc/library/gc.rst
+++ b/Doc/library/gc.rst
@@ -186,7 +186,7 @@ values but should not rebind them):
added to this list rather than freed.
.. versionchanged:: 3.2
- If this list is non-empty at interpreter shutdown, a
+ If this list is non-empty at :term:`interpreter shutdown`, a
:exc:`ResourceWarning` is emitted, which is silent by default. If
:const:`DEBUG_UNCOLLECTABLE` is set, in addition all uncollectable objects
are printed.
@@ -252,8 +252,8 @@ The following constants are provided for use with :func:`set_debug`:
to the ``garbage`` list.
.. versionchanged:: 3.2
- Also print the contents of the :data:`garbage` list at interpreter
- shutdown, if it isn't empty.
+ Also print the contents of the :data:`garbage` list at
+ :term:`interpreter shutdown`, if it isn't empty.
.. data:: DEBUG_SAVEALL
diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst
index ff23b59..514cc5a 100644
--- a/Doc/library/gettext.rst
+++ b/Doc/library/gettext.rst
@@ -344,9 +344,9 @@ will assume message ids as Unicode strings, not byte strings.
The entire set of key/value pairs are placed into a dictionary and set as the
"protected" :attr:`_info` instance variable.
-If the :file:`.mo` file's magic number is invalid, or if other problems occur
-while reading the file, instantiating a :class:`GNUTranslations` class can raise
-:exc:`OSError`.
+If the :file:`.mo` file's magic number is invalid, the major version number is
+unexpected, or if other problems occur while reading the file, instantiating a
+:class:`GNUTranslations` class can raise :exc:`OSError`.
The following methods are overridden from the base class implementation:
diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst
index abcbf38..50f38a4 100644
--- a/Doc/library/glob.rst
+++ b/Doc/library/glob.rst
@@ -29,7 +29,7 @@ For example, ``'[?]'`` matches the character ``'?'``.
The :mod:`pathlib` module offers high-level path objects.
-.. function:: glob(pathname)
+.. function:: glob(pathname, *, recursive=False)
Return a possibly-empty list of path names that match *pathname*, which must be
a string containing a path specification. *pathname* can be either absolute
@@ -37,8 +37,19 @@ For example, ``'[?]'`` matches the character ``'?'``.
:file:`../../Tools/\*/\*.gif`), and can contain shell-style wildcards. Broken
symlinks are included in the results (as in the shell).
+ If *recursive* is true, the pattern "``**``" will match any files and zero or
+ more directories and subdirectories. If the pattern is followed by a
+ ``os.sep``, only directories and subdirectories match.
-.. function:: iglob(pathname)
+ .. note::
+ Using the "``**``" pattern in large directory trees may consume
+ an inordinate amount of time.
+
+ .. versionchanged:: 3.5
+ Support for recursive globs using "``**``".
+
+
+.. function:: iglob(pathname, recursive=False)
Return an :term:`iterator` which yields the same values as :func:`glob`
without actually storing them all simultaneously.
@@ -55,8 +66,9 @@ For example, ``'[?]'`` matches the character ``'?'``.
.. versionadded:: 3.4
-For example, consider a directory containing only the following files:
-:file:`1.gif`, :file:`2.txt`, and :file:`card.gif`. :func:`glob` will produce
+For example, consider a directory containing the following files:
+:file:`1.gif`, :file:`2.txt`, :file:`card.gif` and a subdirectory :file:`sub`
+which contains only the file :file:`3.txt`. :func:`glob` will produce
the following results. Notice how any leading components of the path are
preserved. ::
@@ -67,6 +79,10 @@ preserved. ::
['1.gif', 'card.gif']
>>> glob.glob('?.gif')
['1.gif']
+ >>> glob.glob('**/*.txt', recursive=True)
+ ['2.txt', 'sub/3.txt']
+ >>> glob.glob('./**/', recursive=True)
+ ['./', './sub/']
If the directory contains files starting with ``.`` they won't be matched by
default. For example, consider a directory containing :file:`card.gif` and
diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst
index 78536fa..04c41d5 100644
--- a/Doc/library/gzip.rst
+++ b/Doc/library/gzip.rst
@@ -90,13 +90,9 @@ The module defines the following items:
is no compression. The default is ``9``.
The *mtime* argument is an optional numeric timestamp to be written to
- the stream when compressing. All :program:`gzip` compressed streams are
- required to contain a timestamp. If omitted or ``None``, the current
- time is used. This module ignores the timestamp when decompressing;
- however, some programs, such as :program:`gunzip`\ , make use of it.
- The format of the timestamp is the same as that of the return value of
- ``time.time()`` and of the ``st_mtime`` attribute of the object returned
- by ``os.stat()``.
+ the last modification time field in the stream when compressing. It
+ should only be provided in compression mode. If omitted or ``None``, the
+ current time is used. See the :attr:`mtime` attribute for more details.
Calling a :class:`GzipFile` object's :meth:`close` method does not close
*fileobj*, since you might wish to append more material after the compressed
@@ -108,9 +104,9 @@ The module defines the following items:
including iteration and the :keyword:`with` statement. Only the
:meth:`truncate` method isn't implemented.
- :class:`GzipFile` also provides the following method:
+ :class:`GzipFile` also provides the following method and attribute:
- .. method:: peek([n])
+ .. method:: peek(n)
Read *n* uncompressed bytes without advancing the file position.
At most one single read on the compressed stream is done to satisfy
@@ -124,9 +120,21 @@ The module defines the following items:
.. versionadded:: 3.2
+ .. attribute:: mtime
+
+ When decompressing, the value of the last modification time field in
+ the most recently read header may be read from this attribute, as an
+ integer. The initial value before reading any headers is ``None``.
+
+ All :program:`gzip` compressed streams are required to contain this
+ timestamp field. Some programs, such as :program:`gunzip`\ , make use
+ of the timestamp. The format is the same as the return value of
+ :func:`time.time` and the :attr:`~os.stat_result.st_mtime` attribute of
+ the object returned by :func:`os.stat`.
+
.. versionchanged:: 3.1
Support for the :keyword:`with` statement was added, along with the
- *mtime* argument.
+ *mtime* constructor argument and :attr:`mtime` attribute.
.. versionchanged:: 3.2
Support for zero-padded and unseekable files was added.
@@ -137,6 +145,12 @@ The module defines the following items:
.. versionchanged:: 3.4
Added support for the ``'x'`` and ``'xb'`` modes.
+ .. versionchanged:: 3.5
+ Added support for writing arbitrary
+ :term:`bytes-like objects <bytes-like object>`.
+ The :meth:`~io.BufferedIOBase.read` method now accepts an argument of
+ ``None``.
+
.. function:: compress(data, compresslevel=9)
@@ -175,9 +189,10 @@ Example of how to create a compressed GZIP file::
Example of how to GZIP compress an existing file::
import gzip
+ import shutil
with open('/home/joe/file.txt', 'rb') as f_in:
with gzip.open('/home/joe/file.txt.gz', 'wb') as f_out:
- f_out.writelines(f_in)
+ shutil.copyfileobj(f_in, f_out)
Example of how to GZIP compress a binary string::
diff --git a/Doc/library/heapq.rst b/Doc/library/heapq.rst
index f8970be..9fbbcc6 100644
--- a/Doc/library/heapq.rst
+++ b/Doc/library/heapq.rst
@@ -82,7 +82,7 @@ The following functions are provided:
The module also offers three general purpose functions based on heaps.
-.. function:: merge(*iterables)
+.. function:: merge(*iterables, key=None, reverse=False)
Merge multiple sorted inputs into a single sorted output (for example, merge
timestamped entries from multiple log files). Returns an :term:`iterator`
@@ -92,6 +92,18 @@ The module also offers three general purpose functions based on heaps.
not pull the data into memory all at once, and assumes that each of the input
streams is already sorted (smallest to largest).
+ Has two optional arguments which must be specified as keyword arguments.
+
+ *key* specifies a :term:`key function` of one argument that is used to
+ extract a comparison key from each input element. The default value is
+ ``None`` (compare the elements directly).
+
+ *reverse* is a boolean value. If set to ``True``, then the input elements
+ are merged as if each comparison were reversed.
+
+ .. versionchanged:: 3.5
+ Added the optional *key* and *reverse* parameters.
+
.. function:: nlargest(n, iterable, key=None)
diff --git a/Doc/library/html.parser.rst b/Doc/library/html.parser.rst
index fef9c38..824995e 100644
--- a/Doc/library/html.parser.rst
+++ b/Doc/library/html.parser.rst
@@ -16,21 +16,13 @@
This module defines a class :class:`HTMLParser` which serves as the basis for
parsing text files formatted in HTML (HyperText Mark-up Language) and XHTML.
-.. class:: HTMLParser(strict=False, *, convert_charrefs=False)
+.. class:: HTMLParser(*, convert_charrefs=True)
- Create a parser instance.
+ Create a parser instance able to parse invalid markup.
- If *convert_charrefs* is ``True`` (default: ``False``), all character
+ If *convert_charrefs* is ``True`` (the default), all character
references (except the ones in ``script``/``style`` elements) are
automatically converted to the corresponding Unicode characters.
- The use of ``convert_charrefs=True`` is encouraged and will become
- the default in Python 3.5.
-
- If *strict* is ``False`` (the default), the parser will accept and parse
- invalid markup. If *strict* is ``True`` the parser will raise an
- :exc:`~html.parser.HTMLParseError` exception instead [#]_ when it's not
- able to parse the markup. The use of ``strict=True`` is discouraged and
- the *strict* argument is deprecated.
An :class:`.HTMLParser` instance is fed HTML data and calls handler methods
when start tags, end tags, text, comments, and other markup elements are
@@ -40,31 +32,11 @@ parsing text files formatted in HTML (HyperText Mark-up Language) and XHTML.
This parser does not check that end tags match start tags or call the end-tag
handler for elements which are closed implicitly by closing an outer element.
- .. versionchanged:: 3.2
- *strict* argument added.
-
- .. deprecated-removed:: 3.3 3.5
- The *strict* argument and the strict mode have been deprecated.
- The parser is now able to accept and parse invalid markup too.
-
.. versionchanged:: 3.4
*convert_charrefs* keyword argument added.
-An exception is defined as well:
-
-
-.. exception:: HTMLParseError
-
- Exception raised by the :class:`HTMLParser` class when it encounters an error
- while parsing and *strict* is ``True``. This exception provides three
- attributes: :attr:`msg` is a brief message explaining the error,
- :attr:`lineno` is the number of the line on which the broken construct was
- detected, and :attr:`offset` is the number of characters into the line at
- which the construct starts.
-
- .. deprecated-removed:: 3.3 3.5
- This exception has been deprecated because it's never raised by the parser
- (when the default non-strict mode is used).
+ .. versionchanged:: 3.5
+ The default value for argument *convert_charrefs* is now ``True``.
Example HTML Parser Application
@@ -246,8 +218,7 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`):
The *data* parameter will be the entire contents of the declaration inside
the ``<![...]>`` markup. It is sometimes useful to be overridden by a
- derived class. The base class implementation raises an :exc:`HTMLParseError`
- when *strict* is ``True``.
+ derived class. The base class implementation does nothing.
.. _htmlparser-examples:
@@ -358,9 +329,3 @@ Parsing invalid HTML (e.g. unquoted attributes) also works::
Data : tag soup
End tag : p
End tag : a
-
-.. rubric:: Footnotes
-
-.. [#] For backward compatibility reasons *strict* mode does not raise
- exceptions for all non-compliant HTML. That is, some invalid HTML
- is tolerated even in *strict* mode.
diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst
index 807f685..d57649c 100644
--- a/Doc/library/http.client.rst
+++ b/Doc/library/http.client.rst
@@ -180,6 +180,17 @@ The following exceptions are raised as appropriate:
is received in the HTTP protocol from the server.
+.. exception:: RemoteDisconnected
+
+ A subclass of :exc:`ConnectionResetError` and :exc:`BadStatusLine`. Raised
+ by :meth:`HTTPConnection.getresponse` when the attempt to read the response
+ results in no data read from the connection, indicating that the remote end
+ has closed the connection.
+
+ .. versionadded:: 3.5
+ Previously, :exc:`BadStatusLine`\ ``('')`` was raised.
+
+
The constants defined in this module are:
.. data:: HTTP_PORT
@@ -191,221 +202,15 @@ The constants defined in this module are:
The default port for the HTTPS protocol (always ``443``).
-and also the following constants for integer status codes:
-
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| Constant | Value | Definition |
-+==========================================+=========+=======================================================================+
-| :const:`CONTINUE` | ``100`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.1.1 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.1.1>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`SWITCHING_PROTOCOLS` | ``101`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.1.2 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.1.2>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`PROCESSING` | ``102`` | WEBDAV, `RFC 2518, Section 10.1 |
-| | | <http://www.webdav.org/specs/rfc2518.html#STATUS_102>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`OK` | ``200`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.2.1 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.2.1>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`CREATED` | ``201`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.2.2 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.2.2>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`ACCEPTED` | ``202`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.2.3 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.2.3>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`NON_AUTHORITATIVE_INFORMATION` | ``203`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.2.4 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.2.4>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`NO_CONTENT` | ``204`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.2.5 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.2.5>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`RESET_CONTENT` | ``205`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.2.6 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.2.6>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`PARTIAL_CONTENT` | ``206`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.2.7 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.2.7>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`MULTI_STATUS` | ``207`` | WEBDAV `RFC 2518, Section 10.2 |
-| | | <http://www.webdav.org/specs/rfc2518.html#STATUS_207>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`IM_USED` | ``226`` | Delta encoding in HTTP, |
-| | | :rfc:`3229`, Section 10.4.1 |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`MULTIPLE_CHOICES` | ``300`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.3.1 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.1>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`MOVED_PERMANENTLY` | ``301`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.3.2 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.2>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`FOUND` | ``302`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.3.3 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.3>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`SEE_OTHER` | ``303`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.3.4 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`NOT_MODIFIED` | ``304`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.3.5 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.5>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`USE_PROXY` | ``305`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.3.6 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.6>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`TEMPORARY_REDIRECT` | ``307`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.3.8 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.8>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`BAD_REQUEST` | ``400`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.4.1 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.1>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`UNAUTHORIZED` | ``401`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.4.2 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.2>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`PAYMENT_REQUIRED` | ``402`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.4.3 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.3>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`FORBIDDEN` | ``403`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.4.4 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.4>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`NOT_FOUND` | ``404`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.4.5 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.5>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`METHOD_NOT_ALLOWED` | ``405`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.4.6 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.6>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`NOT_ACCEPTABLE` | ``406`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.4.7 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.7>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`PROXY_AUTHENTICATION_REQUIRED` | ``407`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.4.8 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.8>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`REQUEST_TIMEOUT` | ``408`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.4.9 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.9>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`CONFLICT` | ``409`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.4.10 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.10>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`GONE` | ``410`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.4.11 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.11>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`LENGTH_REQUIRED` | ``411`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.4.12 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.12>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`PRECONDITION_FAILED` | ``412`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.4.13 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.13>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`REQUEST_ENTITY_TOO_LARGE` | ``413`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.4.14 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.14>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`REQUEST_URI_TOO_LONG` | ``414`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.4.15 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.15>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`UNSUPPORTED_MEDIA_TYPE` | ``415`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.4.16 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.16>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`REQUESTED_RANGE_NOT_SATISFIABLE` | ``416`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.4.17 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.17>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`EXPECTATION_FAILED` | ``417`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.4.18 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.18>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`UNPROCESSABLE_ENTITY` | ``422`` | WEBDAV, `RFC 2518, Section 10.3 |
-| | | <http://www.webdav.org/specs/rfc2518.html#STATUS_422>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`LOCKED` | ``423`` | WEBDAV `RFC 2518, Section 10.4 |
-| | | <http://www.webdav.org/specs/rfc2518.html#STATUS_423>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`FAILED_DEPENDENCY` | ``424`` | WEBDAV, `RFC 2518, Section 10.5 |
-| | | <http://www.webdav.org/specs/rfc2518.html#STATUS_424>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`UPGRADE_REQUIRED` | ``426`` | HTTP Upgrade to TLS, |
-| | | :rfc:`2817`, Section 6 |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`PRECONDITION_REQUIRED` | ``428`` | Additional HTTP Status Codes, |
-| | | :rfc:`6585`, Section 3 |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`TOO_MANY_REQUESTS` | ``429`` | Additional HTTP Status Codes, |
-| | | :rfc:`6585`, Section 4 |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`REQUEST_HEADER_FIELDS_TOO_LARGE` | ``431`` | Additional HTTP Status Codes, |
-| | | :rfc:`6585`, Section 5 |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`INTERNAL_SERVER_ERROR` | ``500`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.5.1 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.5.1>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`NOT_IMPLEMENTED` | ``501`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.5.2 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.5.2>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`BAD_GATEWAY` | ``502`` | HTTP/1.1 `RFC 2616, Section |
-| | | 10.5.3 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.5.3>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`SERVICE_UNAVAILABLE` | ``503`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.5.4 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.5.4>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`GATEWAY_TIMEOUT` | ``504`` | HTTP/1.1 `RFC 2616, Section |
-| | | 10.5.5 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.5.5>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`HTTP_VERSION_NOT_SUPPORTED` | ``505`` | HTTP/1.1, `RFC 2616, Section |
-| | | 10.5.6 |
-| | | <http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.5.6>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`INSUFFICIENT_STORAGE` | ``507`` | WEBDAV, `RFC 2518, Section 10.6 |
-| | | <http://www.webdav.org/specs/rfc2518.html#STATUS_507>`_ |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`NOT_EXTENDED` | ``510`` | An HTTP Extension Framework, |
-| | | :rfc:`2774`, Section 7 |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-| :const:`NETWORK_AUTHENTICATION_REQUIRED` | ``511`` | Additional HTTP Status Codes, |
-| | | :rfc:`6585`, Section 6 |
-+------------------------------------------+---------+-----------------------------------------------------------------------+
-
-.. versionchanged:: 3.3
- Added codes ``428``, ``429``, ``431`` and ``511`` from :rfc:`6585`.
-
-
.. data:: responses
This dictionary maps the HTTP 1.1 status codes to the W3C names.
Example: ``http.client.responses[http.client.NOT_FOUND]`` is ``'Not Found'``.
+See :ref:`http-status-codes` for a list of HTTP status codes that are
+available in this module as constants.
+
.. _httpconnection-objects:
@@ -458,6 +263,11 @@ HTTPConnection Objects
Note that you must have read the whole response before you can send a new
request to the server.
+ .. versionchanged:: 3.5
+ If a :exc:`ConnectionError` or subclass is raised, the
+ :class:`HTTPConnection` object will be ready to reconnect when
+ a new request is sent.
+
.. method:: HTTPConnection.set_debuglevel(level)
@@ -496,7 +306,9 @@ HTTPConnection Objects
.. method:: HTTPConnection.connect()
- Connect to the server specified when the object was created.
+ Connect to the server specified when the object was created. By default,
+ this is called automatically when making a request if the client does not
+ already have a connection.
.. method:: HTTPConnection.close()
diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst
index 646f2e8..7c85d09 100644
--- a/Doc/library/http.cookies.rst
+++ b/Doc/library/http.cookies.rst
@@ -143,26 +143,43 @@ Morsel Objects
The keys are case-insensitive.
+ .. versionchanged:: 3.5
+ :meth:`~Morsel.__eq__` now takes :attr:`~Morsel.key` and :attr:`~Morsel.value`
+ into account.
+
.. attribute:: Morsel.value
The value of the cookie.
+ .. deprecated:: 3.5
+ assigning to ``value``; use :meth:`~Morsel.set` instead.
+
.. attribute:: Morsel.coded_value
The encoded value of the cookie --- this is what should be sent.
+ .. deprecated:: 3.5
+ assigning to ``coded_value``; use :meth:`~Morsel.set` instead.
+
.. attribute:: Morsel.key
The name of the cookie.
+ .. deprecated:: 3.5
+ assigning to ``key``; use :meth:`~Morsel.set` instead.
+
.. method:: Morsel.set(key, value, coded_value)
Set the *key*, *value* and *coded_value* attributes.
+ .. deprecated:: 3.5
+ The undocumented *LegalChars* parameter is ignored and will be removed in
+ a future version.
+
.. method:: Morsel.isReservedKey(K)
@@ -193,6 +210,30 @@ Morsel Objects
The meaning for *attrs* is the same as in :meth:`output`.
+.. method:: Morsel.update(values)
+
+ Update the values in the Morsel dictionary with the values in the dictionary
+ *values*. Raise an error if any of the keys in the *values* dict is not a
+ valid :rfc:`2109` attribute.
+
+ .. versionchanged:: 3.5
+ an error is raised for invalid keys.
+
+
+.. method:: Morsel.copy(value)
+
+ Return a shallow copy of the Morsel object.
+
+ .. versionchanged:: 3.5
+ return a Morsel object instead of a dict.
+
+
+.. method:: Morsel.setdefault(key, value=None)
+
+ Raise an error if key is not a valid :rfc:`2109` attribute, otherwise
+ behave the same as :meth:`dict.setdefault`.
+
+
.. _cookie-example:
Example
diff --git a/Doc/library/http.rst b/Doc/library/http.rst
index a387a37..b6f2c58 100644
--- a/Doc/library/http.rst
+++ b/Doc/library/http.rst
@@ -1,7 +1,16 @@
:mod:`http` --- HTTP modules
============================
-``http`` is a package that collects several modules for working with the
+.. module:: http
+ :synopsis: HTTP status codes and messages
+
+.. index::
+ pair: HTTP; protocol
+ single: HTTP; http (standard module)
+
+**Source code:** :source:`Lib/http/__init__.py`
+
+:mod:`http` is a package that collects several modules for working with the
HyperText Transfer Protocol:
* :mod:`http.client` is a low-level HTTP protocol client; for high-level URL
@@ -9,3 +18,105 @@ HyperText Transfer Protocol:
* :mod:`http.server` contains basic HTTP server classes based on :mod:`socketserver`
* :mod:`http.cookies` has utilities for implementing state management with cookies
* :mod:`http.cookiejar` provides persistence of cookies
+
+:mod:`http` is also a module that defines a number of HTTP status codes and
+associated messages through the :class:`http.HTTPStatus` enum:
+
+.. class:: HTTPStatus
+
+ .. versionadded:: 3.5
+
+ A subclass of :class:`enum.IntEnum` that defines a set of HTTP status codes,
+ reason phrases and long descriptions written in English.
+
+ Usage::
+
+ >>> from http import HTTPStatus
+ >>> HTTPStatus.OK
+ <HTTPStatus.OK: 200>
+ >>> HTTPStatus.OK == 200
+ True
+ >>> http.HTTPStatus.OK.value
+ 200
+ >>> HTTPStatus.OK.phrase
+ 'OK'
+ >>> HTTPStatus.OK.description
+ 'Request fulfilled, document follows'
+ >>> list(HTTPStatus)
+ [<HTTPStatus.CONTINUE: 100>, <HTTPStatus.SWITCHING_PROTOCOLS: 101>, ...]
+
+.. _http-status-codes:
+
+HTTP status codes
+-----------------
+
+Supported,
+`IANA-registered <http://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml>`_
+status codes available in :class:`http.HTTPStatus` are:
+
+======= =================================== ==================================================================
+Code Enum Name Details
+======= =================================== ==================================================================
+``100`` ``CONTINUE`` HTTP/1.1 :rfc:`7231`, Section 6.2.1
+``101`` ``SWITCHING_PROTOCOLS`` HTTP/1.1 :rfc:`7231`, Section 6.2.2
+``102`` ``PROCESSING`` WebDAV :rfc:`2518`, Section 10.1
+``200`` ``OK`` HTTP/1.1 :rfc:`7231`, Section 6.3.1
+``201`` ``CREATED`` HTTP/1.1 :rfc:`7231`, Section 6.3.2
+``202`` ``ACCEPTED`` HTTP/1.1 :rfc:`7231`, Section 6.3.3
+``203`` ``NON_AUTHORITATIVE_INFORMATION`` HTTP/1.1 :rfc:`7231`, Section 6.3.4
+``204`` ``NO_CONTENT`` HTTP/1.1 :rfc:`7231`, Section 6.3.5
+``205`` ``RESET_CONTENT`` HTTP/1.1 :rfc:`7231`, Section 6.3.6
+``206`` ``PARTIAL_CONTENT`` HTTP/1.1 :rfc:`7233`, Section 4.1
+``207`` ``MULTI_STATUS`` WebDAV :rfc:`4918`, Section 11.1
+``208`` ``ALREADY_REPORTED`` WebDAV Binding Extensions :rfc:`5842`, Section 7.1 (Experimental)
+``226`` ``IM_USED`` Delta Encoding in HTTP :rfc:`3229`, Section 10.4.1
+``300`` ``MULTIPLE_CHOICES`` HTTP/1.1 :rfc:`7231`, Section 6.4.1
+``301`` ``MOVED_PERMANENTLY`` HTTP/1.1 :rfc:`7231`, Section 6.4.2
+``302`` ``FOUND`` HTTP/1.1 :rfc:`7231`, Section 6.4.3
+``303`` ``SEE_OTHER`` HTTP/1.1 :rfc:`7231`, Section 6.4.4
+``304`` ``NOT_MODIFIED`` HTTP/1.1 :rfc:`7232`, Section 4.1
+``305`` ``USE_PROXY`` HTTP/1.1 :rfc:`7231`, Section 6.4.5
+``307`` ``TEMPORARY_REDIRECT`` HTTP/1.1 :rfc:`7231`, Section 6.4.7
+``308`` ``PERMANENT_REDIRECT`` Permanent Redirect :rfc:`7238`, Section 3 (Experimental)
+``400`` ``BAD_REQUEST`` HTTP/1.1 :rfc:`7231`, Section 6.5.1
+``401`` ``UNAUTHORIZED`` HTTP/1.1 Authentication :rfc:`7235`, Section 3.1
+``402`` ``PAYMENT_REQUIRED`` HTTP/1.1 :rfc:`7231`, Section 6.5.2
+``403`` ``FORBIDDEN`` HTTP/1.1 :rfc:`7231`, Section 6.5.3
+``404`` ``NOT_FOUND`` HTTP/1.1 :rfc:`7231`, Section 6.5.4
+``405`` ``METHOD_NOT_ALLOWED`` HTTP/1.1 :rfc:`7231`, Section 6.5.5
+``406`` ``NOT_ACCEPTABLE`` HTTP/1.1 :rfc:`7231`, Section 6.5.6
+``407`` ``PROXY_AUTHENTICATION_REQUIRED`` HTTP/1.1 Authentication :rfc:`7235`, Section 3.2
+``408`` ``REQUEST_TIMEOUT`` HTTP/1.1 :rfc:`7231`, Section 6.5.7
+``409`` ``CONFLICT`` HTTP/1.1 :rfc:`7231`, Section 6.5.8
+``410`` ``GONE`` HTTP/1.1 :rfc:`7231`, Section 6.5.9
+``411`` ``LENGTH_REQUIRED`` HTTP/1.1 :rfc:`7231`, Section 6.5.10
+``412`` ``PRECONDITION_FAILED`` HTTP/1.1 :rfc:`7232`, Section 4.2
+``413`` ``REQUEST_ENTITY_TOO_LARGE`` HTTP/1.1 :rfc:`7231`, Section 6.5.11
+``414`` ``REQUEST_URI_TOO_LONG`` HTTP/1.1 :rfc:`7231`, Section 6.5.12
+``415`` ``UNSUPPORTED_MEDIA_TYPE`` HTTP/1.1 :rfc:`7231`, Section 6.5.13
+``416`` ``REQUEST_RANGE_NOT_SATISFIABLE`` HTTP/1.1 Range Requests :rfc:`7233`, Section 4.4
+``417`` ``EXPECTATION_FAILED`` HTTP/1.1 :rfc:`7231`, Section 6.5.14
+``422`` ``UNPROCESSABLE_ENTITY`` WebDAV :rfc:`4918`, Section 11.2
+``423`` ``LOCKED`` WebDAV :rfc:`4918`, Section 11.3
+``424`` ``FAILED_DEPENDENCY`` WebDAV :rfc:`4918`, Section 11.4
+``426`` ``UPGRADE_REQUIRED`` HTTP/1.1 :rfc:`7231`, Section 6.5.15
+``428`` ``PRECONDITION_REQUIRED`` Additional HTTP Status Codes :rfc:`6585`
+``429`` ``TOO_MANY_REQUESTS`` Additional HTTP Status Codes :rfc:`6585`
+``431`` ``REQUEST_HEADER_FIELDS_TOO_LARGE`` Additional HTTP Status Codes :rfc:`6585`
+``500`` ``INTERNAL_SERVER_ERROR`` HTTP/1.1 :rfc:`7231`, Section 6.6.1
+``501`` ``NOT_IMPLEMENTED`` HTTP/1.1 :rfc:`7231`, Section 6.6.2
+``502`` ``BAD_GATEWAY`` HTTP/1.1 :rfc:`7231`, Section 6.6.3
+``503`` ``SERVICE_UNAVAILABLE`` HTTP/1.1 :rfc:`7231`, Section 6.6.4
+``504`` ``GATEWAY_TIMEOUT`` HTTP/1.1 :rfc:`7231`, Section 6.6.5
+``505`` ``HTTP_VERSION_NOT_SUPPORTED`` HTTP/1.1 :rfc:`7231`, Section 6.6.6
+``506`` ``VARIANT_ALSO_NEGOTIATES`` Transparent Content Negotiation in HTTP :rfc:`2295`, Section 8.1 (Experimental)
+``507`` ``INSUFFICIENT_STORAGE`` WebDAV :rfc:`4918`, Section 11.5
+``508`` ``LOOP_DETECTED`` WebDAV Binding Extensions :rfc:`5842`, Section 7.2 (Experimental)
+``510`` ``NOT_EXTENDED`` An HTTP Extension Framework :rfc:`2774`, Section 7 (Experimental)
+``511`` ``NETWORK_AUTHENTICATION_REQUIRED`` Additional HTTP Status Codes :rfc:`6585`, Section 6
+======= =================================== ==================================================================
+
+In order to preserve backwards compatibility, enum values are also present
+in the :mod:`http.client` module in the form of constants. The enum name is
+equal to the constant name (i.e. ``http.HTTPStatus.OK`` is also available as
+``http.client.OK``).
diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst
index fa736fe..15b0932 100644
--- a/Doc/library/imaplib.rst
+++ b/Doc/library/imaplib.rst
@@ -37,6 +37,19 @@ base class:
initialized. If *host* is not specified, ``''`` (the local host) is used. If
*port* is omitted, the standard IMAP4 port (143) is used.
+ The :class:`IMAP4` class supports the :keyword:`with` statement. When used
+ like this, the IMAP4 ``LOGOUT`` command is issued automatically when the
+ :keyword:`with` statement exits. E.g.::
+
+ >>> from imaplib import IMAP4
+ >>> with IMAP4("domain.org") as M:
+ ... M.noop()
+ ...
+ ('OK', [b'Nothing Accomplished. d25if65hy903weo.87'])
+
+ .. versionchanged:: 3.5
+ Support for the :keyword:`with` statement was added.
+
Three exceptions are defined as attributes of the :class:`IMAP4` class:
@@ -64,7 +77,8 @@ Three exceptions are defined as attributes of the :class:`IMAP4` class:
There's also a subclass for secure connections:
-.. class:: IMAP4_SSL(host='', port=IMAP4_SSL_PORT, keyfile=None, certfile=None, ssl_context=None)
+.. class:: IMAP4_SSL(host='', port=IMAP4_SSL_PORT, keyfile=None, \
+ certfile=None, ssl_context=None)
This is a subclass derived from :class:`IMAP4` that connects over an SSL
encrypted socket (to use this class you need a socket module that was compiled
@@ -198,6 +212,10 @@ An :class:`IMAP4` instance has the following methods:
that will be base64 encoded and sent to the server. It should return
``None`` if the client abort response ``*`` should be sent instead.
+ .. versionchanged:: 3.5
+ string usernames and passwords are now encoded to ``utf-8`` instead of
+ being limited to ASCII.
+
.. method:: IMAP4.check()
@@ -230,6 +248,16 @@ An :class:`IMAP4` instance has the following methods:
Delete the ACLs (remove any rights) set for who on mailbox.
+.. method:: IMAP4.enable(capability)
+
+ Enable *capability* (see :rfc:`5161`). Most capabilities do not need to be
+ enabled. Currently only the ``UTF8=ACCEPT`` capability is supported
+ (see :RFC:`6855`).
+
+ .. versionadded:: 3.5
+ The :meth:`enable` method itself, and :RFC:`6855` support.
+
+
.. method:: IMAP4.expunge()
Permanently remove deleted items from selected mailbox. Generates an ``EXPUNGE``
@@ -367,7 +395,9 @@ An :class:`IMAP4` instance has the following methods:
Search mailbox for matching messages. *charset* may be ``None``, in which case
no ``CHARSET`` will be specified in the request to the server. The IMAP
protocol requires that at least one criterion be specified; an exception will be
- raised when the server returns an error.
+ raised when the server returns an error. *charset* must be ``None`` if
+ the ``UTF8=ACCEPT`` capability was enabled using the :meth:`enable`
+ command.
Example::
@@ -529,6 +559,15 @@ The following attributes are defined on instances of :class:`IMAP4`:
the module variable ``Debug``. Values greater than three trace each command.
+.. attribute:: IMAP4.utf8_enabled
+
+ Boolean value that is normally ``False``, but is set to ``True`` if an
+ :meth:`enable` command is successfully issued for the ``UTF8=ACCEPT``
+ capability.
+
+ .. versionadded:: 3.5
+
+
.. _imap4-example:
IMAP4 Example
diff --git a/Doc/library/imghdr.rst b/Doc/library/imghdr.rst
index 9e89523..c60df24 100644
--- a/Doc/library/imghdr.rst
+++ b/Doc/library/imghdr.rst
@@ -48,6 +48,16 @@ from :func:`what`:
+------------+-----------------------------------+
| ``'png'`` | Portable Network Graphics |
+------------+-----------------------------------+
+| ``'webp'`` | WebP files |
++------------+-----------------------------------+
+| ``'exr'`` | OpenEXR Files |
++------------+-----------------------------------+
+
+.. versionadded:: 3.5
+ The *exr* format was added.
+
+.. versionchanged:: 3.5
+ The *webp* type was added.
You can extend the list of file types :mod:`imghdr` can recognize by appending
to this variable:
diff --git a/Doc/library/imp.rst b/Doc/library/imp.rst
index 83a52e4..68a6b68 100644
--- a/Doc/library/imp.rst
+++ b/Doc/library/imp.rst
@@ -197,11 +197,9 @@ file paths.
value would be ``/foo/bar/__pycache__/baz.cpython-32.pyc`` for Python 3.2.
The ``cpython-32`` string comes from the current magic tag (see
:func:`get_tag`; if :attr:`sys.implementation.cache_tag` is not defined then
- :exc:`NotImplementedError` will be raised). The returned path will end in
- ``.pyc`` when ``__debug__`` is ``True`` or ``.pyo`` for an optimized Python
- (i.e. ``__debug__`` is ``False``). By passing in ``True`` or ``False`` for
- *debug_override* you can override the system's value for ``__debug__`` for
- extension selection.
+ :exc:`NotImplementedError` will be raised). By passing in ``True`` or
+ ``False`` for *debug_override* you can override the system's value for
+ ``__debug__``, leading to optimized bytecode.
*path* need not exist.
@@ -212,6 +210,9 @@ file paths.
.. deprecated:: 3.4
Use :func:`importlib.util.cache_from_source` instead.
+ .. versionchanged:: 3.5
+ The *debug_override* parameter no longer creates a ``.pyo`` file.
+
.. function:: source_from_cache(path)
diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst
index c84d2df..632df75 100644
--- a/Doc/library/importlib.rst
+++ b/Doc/library/importlib.rst
@@ -55,6 +55,12 @@ generically as an :term:`importer`) to participate in the import process.
:pep:`451`
A ModuleSpec Type for the Import System
+ :pep:`488`
+ Elimination of PYO files
+
+ :pep:`489`
+ Multi-phase extension module initialization
+
:pep:`3120`
Using UTF-8 as the Default Source Encoding
@@ -69,6 +75,10 @@ Functions
An implementation of the built-in :func:`__import__` function.
+ .. note::
+ Programmatic importing of modules should use :func:`import_module`
+ instead of this function.
+
.. function:: import_module(name, package=None)
Import a module. The *name* argument specifies what module to
@@ -81,12 +91,15 @@ Functions
The :func:`import_module` function acts as a simplifying wrapper around
:func:`importlib.__import__`. This means all semantics of the function are
- derived from :func:`importlib.__import__`, including requiring the package
- from which an import is occurring to have been previously imported
- (i.e., *package* must already be imported). The most important difference
- is that :func:`import_module` returns the specified package or module
- (e.g. ``pkg.mod``), while :func:`__import__` returns the
- top-level package or module (e.g. ``pkg``).
+ derived from :func:`importlib.__import__`. The most important difference
+ between these two functions is that :func:`import_module` returns the
+ specified package or module (e.g. ``pkg.mod``), while :func:`__import__`
+ returns the top-level package or module (e.g. ``pkg``).
+
+ If you are dynamically importing a module that was created since the
+ interpreter began execution (e.g., created a Python source file), you may
+ need to call :func:`invalidate_caches` in order for the new module to be
+ noticed by the import system.
.. versionchanged:: 3.3
Parent packages are automatically imported.
@@ -341,13 +354,16 @@ ABC hierarchy::
.. method:: create_module(spec)
- An optional method that returns the module object to use when
- importing a module. create_module() may also return ``None``,
- indicating that the default module creation should take place
- instead.
+ A method that returns the module object to use when
+ importing a module. This method may return ``None``,
+ indicating that default module creation semantics should take place.
.. versionadded:: 3.4
+ .. versionchanged:: 3.5
+ Starting in Python 3.6, this method will not be optional when
+ :meth:`exec_module` is defined.
+
.. method:: exec_module(module)
An abstract method that executes the module in its own namespace
@@ -411,7 +427,7 @@ ABC hierarchy::
.. deprecated:: 3.4
The recommended API for loading a module is :meth:`exec_module`
- (and optionally :meth:`create_module`). Loaders should implement
+ (and :meth:`create_module`). Loaders should implement
it instead of load_module(). The import machinery takes care of
all the other responsibilities of load_module() when exec_module()
is implemented.
@@ -493,7 +509,7 @@ ABC hierarchy::
.. versionchanged:: 3.4
Raises :exc:`ImportError` instead of :exc:`NotImplementedError`.
- .. method:: source_to_code(data, path='<string>')
+ .. staticmethod:: source_to_code(data, path='<string>')
Create a code object from Python source.
@@ -502,8 +518,14 @@ ABC hierarchy::
the "path" to where the source code originated from, which can be an
abstract concept (e.g. location in a zip file).
+ With the subsequent code object one can execute it in a module by
+ running ``exec(code, module.__dict__)``.
+
.. versionadded:: 3.4
+ .. versionchanged:: 3.5
+ Made the method static.
+
.. method:: exec_module(module)
Implementation of :meth:`Loader.exec_module`.
@@ -689,6 +711,9 @@ find and load modules.
.. versionadded:: 3.3
+ .. deprecated:: 3.5
+ Use :attr:`BYTECODE_SUFFIXES` instead.
+
.. attribute:: OPTIMIZED_BYTECODE_SUFFIXES
A list of strings representing the file suffixes for optimized bytecode
@@ -696,14 +721,19 @@ find and load modules.
.. versionadded:: 3.3
+ .. deprecated:: 3.5
+ Use :attr:`BYTECODE_SUFFIXES` instead.
+
.. attribute:: BYTECODE_SUFFIXES
A list of strings representing the recognized file suffixes for bytecode
- modules. Set to either :attr:`DEBUG_BYTECODE_SUFFIXES` or
- :attr:`OPTIMIZED_BYTECODE_SUFFIXES` based on whether ``__debug__`` is true.
+ modules (including the leading dot).
.. versionadded:: 3.3
+ .. versionchanged:: 3.5
+ The value is no longer dependent on ``__debug__``.
+
.. attribute:: EXTENSION_SUFFIXES
A list of strings representing the recognized file suffixes for
@@ -732,9 +762,9 @@ find and load modules.
Only class methods are defined by this class to alleviate the need for
instantiation.
- .. note::
- Due to limitations in the extension module C-API, for now
- BuiltinImporter does not implement :meth:`Loader.exec_module`.
+ .. versionchanged:: 3.5
+ As part of :pep:`489`, the builtin importer now implements
+ :meth:`Loader.create_module` and :meth:`Loader.exec_module`
.. class:: FrozenImporter
@@ -782,6 +812,11 @@ find and load modules.
.. versionadded:: 3.4
+ .. versionchanged:: 3.5
+ If the current working directory -- represented by an empty string --
+ is no longer valid then ``None`` is returned but no value is cached
+ in :data:`sys.path_importer_cache`.
+
.. classmethod:: find_module(fullname, path=None)
A legacy wrapper around :meth:`find_spec`.
@@ -944,14 +979,18 @@ find and load modules.
Path to the extension module.
- .. method:: load_module(name=None)
+ .. method:: create_module(spec)
+
+ Creates the module object from the given specification in accordance
+ with :pep:`489`.
+
+ .. versionadded:: 3.5
- Loads the extension module if and only if *fullname* is the same as
- :attr:`name` or is ``None``.
+ .. method:: exec_module(module)
- .. note::
- Due to limitations in the extension module C-API, for now
- ExtensionFileLoader does not implement :meth:`Loader.exec_module`.
+ Initializes the given module object in accordance with :pep:`489`.
+
+ .. versionadded:: 3.5
.. method:: is_package(fullname)
@@ -1047,23 +1086,37 @@ an :term:`importer`.
.. versionadded:: 3.4
-.. function:: cache_from_source(path, debug_override=None)
+.. function:: cache_from_source(path, debug_override=None, *, optimization=None)
- Return the :pep:`3147` path to the byte-compiled file associated with the
- source *path*. For example, if *path* is ``/foo/bar/baz.py`` the return
+ Return the :pep:`3147`/:pep:`488` path to the byte-compiled file associated
+ with the source *path*. For example, if *path* is ``/foo/bar/baz.py`` the return
value would be ``/foo/bar/__pycache__/baz.cpython-32.pyc`` for Python 3.2.
The ``cpython-32`` string comes from the current magic tag (see
:func:`get_tag`; if :attr:`sys.implementation.cache_tag` is not defined then
- :exc:`NotImplementedError` will be raised). The returned path will end in
- ``.pyc`` when ``__debug__`` is ``True`` or ``.pyo`` for an optimized Python
- (i.e. ``__debug__`` is ``False``). By passing in ``True`` or ``False`` for
- *debug_override* you can override the system's value for ``__debug__`` for
- extension selection.
-
- *path* need not exist.
+ :exc:`NotImplementedError` will be raised).
+
+ The *optimization* parameter is used to specify the optimization level of the
+ bytecode file. An empty string represents no optimization, so
+ ``/foo/bar/baz.py`` with an *optimization* of ``''`` will result in a
+ bytecode path of ``/foo/bar/__pycache__/baz.cpython-32.pyc``. ``None`` causes
+ the interpter's optimization level to be used. Any other value's string
+ representation being used, so ``/foo/bar/baz.py`` with an *optimization* of
+ ``2`` will lead to the bytecode path of
+ ``/foo/bar/__pycache__/baz.cpython-32.opt-2.pyc``. The string representation
+ of *optimization* can only be alphanumeric, else :exc:`ValueError` is raised.
+
+ The *debug_override* parameter is deprecated and can be used to override
+ the system's value for ``__debug__``. A ``True`` value is the equivalent of
+ setting *optimization* to the empty string. A ``False`` value is the same as
+ setting *optimization* to ``1``. If both *debug_override* an *optimization*
+ are not ``None`` then :exc:`TypeError` is raised.
.. versionadded:: 3.4
+ .. versionchanged ::3.5
+ The *optimization* parameter was added and the *debug_override* parameter
+ was deprecated.
+
.. function:: source_from_cache(path)
@@ -1071,7 +1124,7 @@ an :term:`importer`.
file path. For example, if *path* is
``/foo/bar/__pycache__/baz.cpython-32.pyc`` the returned path would be
``/foo/bar/baz.py``. *path* need not exist, however if it does not conform
- to :pep:`3147` format, a ``ValueError`` is raised. If
+ to :pep:`3147` or :pep:`488` format, a ``ValueError`` is raised. If
:attr:`sys.implementation.cache_tag` is not defined,
:exc:`NotImplementedError` is raised.
@@ -1117,6 +1170,21 @@ an :term:`importer`.
.. versionadded:: 3.4
+.. function:: module_from_spec(spec)
+
+ Create a new module based on **spec** and ``spec.loader.create_module()``.
+
+ If ``spec.loader.create_module()`` does not return ``None``, then any
+ pre-existing attributes will not be reset. Also, no :exc:`AttributeError`
+ will be raised if triggered while accessing **spec** or setting an attribute
+ on the module.
+
+ This function is preferred over using :class:`types.ModuleType` to create a
+ new module as **spec** is used to set as many import-controlled attributes on
+ the module as possible.
+
+ .. versionadded:: 3.5
+
.. decorator:: module_for_loader
A :term:`decorator` for :meth:`importlib.abc.Loader.load_module`
@@ -1195,3 +1263,39 @@ an :term:`importer`.
module will be file-based.
.. versionadded:: 3.4
+
+.. class:: LazyLoader(loader)
+
+ A class which postpones the execution of the loader of a module until the
+ module has an attribute accessed.
+
+ This class **only** works with loaders that define
+ :meth:`~importlib.abc.Loader.exec_module` as control over what module type
+ is used for the module is required. For those same reasons, the loader's
+ :meth:`~importlib.abc.Loader.create_module` method will be ignored (i.e., the
+ loader's method should only return ``None``). Finally,
+ modules which substitute the object placed into :attr:`sys.modules` will
+ not work as there is no way to properly replace the module references
+ throughout the interpreter safely; :exc:`ValueError` is raised if such a
+ substitution is detected.
+
+ .. note::
+ For projects where startup time is critical, this class allows for
+ potentially minimizing the cost of loading a module if it is never used.
+ For projects where startup time is not essential then use of this class is
+ **heavily** discouraged due to error messages created during loading being
+ postponed and thus occurring out of context.
+
+ .. versionadded:: 3.5
+
+ .. classmethod:: factory(loader)
+
+ A static method which returns a callable that creates a lazy loader. This
+ is meant to be used in situations where the loader is passed by class
+ instead of by instance.
+ ::
+
+ suffixes = importlib.machinery.SOURCE_SUFFIXES
+ loader = importlib.machinery.SourceFileLoader
+ lazy_loader = importlib.util.LazyLoader.factory(loader)
+ finder = importlib.machinery.FileFinder(path, [(lazy_loader, suffixes)])
diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst
index 57eb4ff..d2247e8 100644
--- a/Doc/library/inspect.rst
+++ b/Doc/library/inspect.rst
@@ -28,7 +28,7 @@ Types and members
-----------------
The :func:`getmembers` function retrieves the members of an object such as a
-class or module. The sixteen functions whose names begin with "is" are mainly
+class or module. The functions whose names begin with "is" are mainly
provided as convenient choices for the second argument to :func:`getmembers`.
They also help you determine when you can expect to find the following special
attributes:
@@ -168,6 +168,33 @@ attributes:
| | | arguments and local |
| | | variables |
+-----------+-----------------+---------------------------+
+| generator | __name__ | name |
++-----------+-----------------+---------------------------+
+| | __qualname__ | qualified name |
++-----------+-----------------+---------------------------+
+| | gi_frame | frame |
++-----------+-----------------+---------------------------+
+| | gi_running | is the generator running? |
++-----------+-----------------+---------------------------+
+| | gi_code | code |
++-----------+-----------------+---------------------------+
+| | gi_yieldfrom | object being iterated by |
+| | | ``yield from``, or |
+| | | ``None`` |
++-----------+-----------------+---------------------------+
+| coroutine | __name__ | name |
++-----------+-----------------+---------------------------+
+| | __qualname__ | qualified name |
++-----------+-----------------+---------------------------+
+| | cr_await | object being awaited on, |
+| | | or ``None`` |
++-----------+-----------------+---------------------------+
+| | cr_frame | frame |
++-----------+-----------------+---------------------------+
+| | cr_running | is the coroutine running? |
++-----------+-----------------+---------------------------+
+| | cr_code | code |
++-----------+-----------------+---------------------------+
| builtin | __doc__ | documentation string |
+-----------+-----------------+---------------------------+
| | __name__ | original name of this |
@@ -180,6 +207,13 @@ attributes:
| | | ``None`` |
+-----------+-----------------+---------------------------+
+.. versionchanged:: 3.5
+
+ Add ``__qualname__`` and ``gi_yieldfrom`` attributes to generators.
+
+ The ``__name__`` attribute of generators is now set from the function
+ name, instead of the code name, and it can now be modified.
+
.. function:: getmembers(object[, predicate])
@@ -261,6 +295,41 @@ attributes:
Return true if the object is a generator.
+.. function:: iscoroutinefunction(object)
+
+ Return true if the object is a :term:`coroutine function`
+ (a function defined with an :keyword:`async def` syntax).
+
+ .. versionadded:: 3.5
+
+
+.. function:: iscoroutine(object)
+
+ Return true if the object is a :term:`coroutine` created by an
+ :keyword:`async def` function.
+
+ .. versionadded:: 3.5
+
+
+.. function:: isawaitable(object)
+
+ Return true if the object can be used in :keyword:`await` expression.
+
+ Can also be used to distinguish generator-based coroutines from regular
+ generators::
+
+ def gen():
+ yield
+ @types.coroutine
+ def gen_coro():
+ yield
+
+ assert not isawaitable(gen())
+ assert isawaitable(gen_coro())
+
+ .. versionadded:: 3.5
+
+
.. function:: istraceback(object)
Return true if the object is a traceback.
@@ -351,6 +420,12 @@ Retrieving source code
.. function:: getdoc(object)
Get the documentation string for an object, cleaned up with :func:`cleandoc`.
+ If the documentation string for an object is not provided and the object is
+ a class, a method, a property or a descriptor, retrieve the documentation
+ string from the inheritance hierarchy.
+
+ .. versionchanged:: 3.5
+ Documentation strings are now inherited if not overridden.
.. function:: getcomments(object)
@@ -423,7 +498,7 @@ The Signature object represents the call signature of a callable object and its
return annotation. To retrieve a Signature object, use the :func:`signature`
function.
-.. function:: signature(callable)
+.. function:: signature(callable, \*, follow_wrapped=True)
Return a :class:`Signature` object for the given ``callable``::
@@ -448,6 +523,11 @@ function.
Raises :exc:`ValueError` if no signature can be provided, and
:exc:`TypeError` if that type of object is not supported.
+ .. versionadded:: 3.5
+ ``follow_wrapped`` parameter. Pass ``False`` to get a signature of
+ ``callable`` specifically (``callable.__wrapped__`` will not be used to
+ unwrap decorated callables.)
+
.. note::
Some callables may not be introspectable in certain implementations of
@@ -473,6 +553,9 @@ function.
Signature objects are *immutable*. Use :meth:`Signature.replace` to make a
modified copy.
+ .. versionchanged:: 3.5
+ Signature objects are picklable and hashable.
+
.. attribute:: Signature.empty
A special class-level marker to specify absence of a return annotation.
@@ -517,12 +600,30 @@ function.
>>> str(new_sig)
"(a, b) -> 'new return anno'"
+ .. classmethod:: Signature.from_callable(obj, \*, follow_wrapped=True)
+
+ Return a :class:`Signature` (or its subclass) object for a given callable
+ ``obj``. Pass ``follow_wrapped=False`` to get a signature of ``obj``
+ without unwrapping its ``__wrapped__`` chain.
+
+ This method simplifies subclassing of :class:`Signature`::
+
+ class MySignature(Signature):
+ pass
+ sig = MySignature.from_callable(min)
+ assert isinstance(sig, MySignature)
+
+ .. versionadded:: 3.5
+
.. class:: Parameter(name, kind, \*, default=Parameter.empty, annotation=Parameter.empty)
Parameter objects are *immutable*. Instead of modifying a Parameter object,
you can use :meth:`Parameter.replace` to create a modified copy.
+ .. versionchanged:: 3.5
+ Parameter objects are picklable and hashable.
+
.. attribute:: Parameter.empty
A special class-level marker to specify absence of default values and
@@ -639,27 +740,8 @@ function.
Arguments for which :meth:`Signature.bind` or
:meth:`Signature.bind_partial` relied on a default value are skipped.
- However, if needed, it is easy to include them.
-
- ::
-
- >>> def foo(a, b=10):
- ... pass
-
- >>> sig = signature(foo)
- >>> ba = sig.bind(5)
-
- >>> ba.args, ba.kwargs
- ((5,), {})
-
- >>> for param in sig.parameters.values():
- ... if (param.name not in ba.arguments
- ... and param.default is not param.empty):
- ... ba.arguments[param.name] = param.default
-
- >>> ba.args, ba.kwargs
- ((5, 10), {})
-
+ However, if needed, use :meth:`BoundArguments.apply_defaults` to add
+ them.
.. attribute:: BoundArguments.args
@@ -675,6 +757,26 @@ function.
A reference to the parent :class:`Signature` object.
+ .. method:: BoundArguments.apply_defaults()
+
+ Set default values for missing arguments.
+
+ For variable-positional arguments (``*args``) the default is an
+ empty tuple.
+
+ For variable-keyword arguments (``**kwargs``) the default is an
+ empty dict.
+
+ ::
+
+ >>> def foo(a, b='ham', *args): pass
+ >>> ba = inspect.signature(foo).bind('spam')
+ >>> ba.apply_defaults()
+ >>> ba.arguments
+ OrderedDict([('a', 'spam'), ('b', 'ham'), ('args', ())])
+
+ .. versionadded:: 3.5
+
The :attr:`args` and :attr:`kwargs` properties can be used to invoke
functions::
@@ -719,8 +821,10 @@ Classes and functions
*n* elements listed in *args*.
.. deprecated:: 3.0
- Use :func:`getfullargspec` instead, which provides information about
- keyword-only arguments and annotations.
+ Use :func:`signature` and
+ :ref:`Signature Object <inspect-signature-object>`, which provide a
+ better introspecting API for callables. This function will be removed
+ in Python 3.6.
.. function:: getfullargspec(func)
@@ -741,15 +845,16 @@ Classes and functions
The first four items in the tuple correspond to :func:`getargspec`.
- .. note::
- Consider using the new :ref:`Signature Object <inspect-signature-object>`
- interface, which provides a better way of introspecting functions.
-
.. versionchanged:: 3.4
This function is now based on :func:`signature`, but still ignores
``__wrapped__`` attributes and includes the already bound first
parameter in the signature output for bound methods.
+ .. deprecated:: 3.5
+ Use :func:`signature` and
+ :ref:`Signature Object <inspect-signature-object>`, which provide a
+ better introspecting API for callables.
+
.. function:: getargvalues(frame)
@@ -759,6 +864,11 @@ Classes and functions
are the names of the ``*`` and ``**`` arguments or ``None``. *locals* is the
locals dictionary of the given frame.
+ .. deprecated:: 3.5
+ Use :func:`signature` and
+ :ref:`Signature Object <inspect-signature-object>`, which provide a
+ better introspecting API for callables.
+
.. function:: formatargspec(args[, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations[, formatarg, formatvarargs, formatvarkw, formatvalue, formatreturns, formatannotations]])
@@ -781,6 +891,11 @@ Classes and functions
>>> formatargspec(*getfullargspec(f))
'(a: int, b: float)'
+ .. deprecated:: 3.5
+ Use :func:`signature` and
+ :ref:`Signature Object <inspect-signature-object>`, which provide a
+ better introspecting API for callables.
+
.. function:: formatargvalues(args[, varargs, varkw, locals, formatarg, formatvarargs, formatvarkw, formatvalue])
@@ -788,6 +903,11 @@ Classes and functions
:func:`getargvalues`. The format\* arguments are the corresponding optional
formatting functions that are called to turn names and values into strings.
+ .. deprecated:: 3.5
+ Use :func:`signature` and
+ :ref:`Signature Object <inspect-signature-object>`, which provide a
+ better introspecting API for callables.
+
.. function:: getmro(cls)
@@ -822,8 +942,8 @@ Classes and functions
.. versionadded:: 3.2
- .. note::
- Consider using the new :meth:`Signature.bind` instead.
+ .. deprecated:: 3.5
+ Use :meth:`Signature.bind` and :meth:`Signature.bind_partial` instead.
.. function:: getclosurevars(func)
@@ -864,11 +984,17 @@ Classes and functions
The interpreter stack
---------------------
-When the following functions return "frame records," each record is a tuple of
-six items: the frame object, the filename, the line number of the current line,
+When the following functions return "frame records," each record is a
+:term:`named tuple`
+``FrameInfo(frame, filename, lineno, function, code_context, index)``.
+The tuple contains the frame object, the filename, the line number of the
+current line,
the function name, a list of lines of context from the source code, and the
index of the current line within that list.
+.. versionchanged:: 3.5
+ Return a named tuple instead of a tuple.
+
.. note::
Keeping references to frame objects, as found in the first element of the frame
@@ -1007,8 +1133,8 @@ code execution::
pass
-Current State of a Generator
-----------------------------
+Current State of Generators and Coroutines
+------------------------------------------
When implementing coroutine schedulers and for other advanced uses of
generators, it is useful to determine whether a generator is currently
@@ -1028,6 +1154,21 @@ generator to be determined easily.
.. versionadded:: 3.2
+.. function:: getcoroutinestate(coroutine)
+
+ Get current state of a coroutine object. The function is intended to be
+ used with coroutine objects created by :keyword:`async def` functions, but
+ will accept any coroutine-like object that has ``cr_running`` and
+ ``cr_frame`` attributes.
+
+ Possible states are:
+ * CORO_CREATED: Waiting to start execution.
+ * CORO_RUNNING: Currently being executed by the interpreter.
+ * CORO_SUSPENDED: Currently suspended at an await expression.
+ * CORO_CLOSED: Execution has completed.
+
+ .. versionadded:: 3.5
+
The current internal state of the generator can also be queried. This is
mostly useful for testing purposes, to ensure that internal state is being
updated as expected:
@@ -1052,6 +1193,13 @@ updated as expected:
.. versionadded:: 3.3
+.. function:: getcoroutinelocals(coroutine)
+
+ This function is analogous to :func:`~inspect.getgeneratorlocals`, but
+ works for coroutine objects created by :keyword:`async def` functions.
+
+ .. versionadded:: 3.5
+
.. _inspect-module-cli:
diff --git a/Doc/library/io.rst b/Doc/library/io.rst
index 3adf6e9..f009c65 100644
--- a/Doc/library/io.rst
+++ b/Doc/library/io.rst
@@ -339,8 +339,11 @@ I/O Base Classes
if *size* is not specified). The current stream position isn't changed.
This resizing can extend or reduce the current file size. In case of
extension, the contents of the new file area depend on the platform
- (on most systems, additional bytes are zero-filled, on Windows they're
- undetermined). The new file size is returned.
+ (on most systems, additional bytes are zero-filled). The new file size
+ is returned.
+
+ .. versionchanged:: 3.5
+ Windows will now zero-fill files when extending.
.. method:: writable()
@@ -391,8 +394,8 @@ I/O Base Classes
.. method:: readinto(b)
Read up to ``len(b)`` bytes into :class:`bytearray` *b* and return the
- number of bytes read. If the object is in non-blocking mode and no
- bytes are available, ``None`` is returned.
+ number of bytes read. If the object is in non-blocking mode and no bytes
+ are available, ``None`` is returned.
.. method:: write(b)
@@ -465,9 +468,10 @@ I/O Base Classes
.. method:: read1(size=-1)
- Read and return up to *size* bytes, with at most one call to the underlying
- raw stream's :meth:`~RawIOBase.read` method. This can be useful if you
- are implementing your own buffering on top of a :class:`BufferedIOBase`
+ Read and return up to *size* bytes, with at most one call to the
+ underlying raw stream's :meth:`~RawIOBase.read` (or
+ :meth:`~RawIOBase.readinto`) method. This can be useful if you are
+ implementing your own buffering on top of a :class:`BufferedIOBase`
object.
.. method:: readinto(b)
@@ -478,8 +482,19 @@ I/O Base Classes
Like :meth:`read`, multiple reads may be issued to the underlying raw
stream, unless the latter is interactive.
- A :exc:`BlockingIOError` is raised if the underlying raw stream is in
- non blocking-mode, and has no data available at the moment.
+ A :exc:`BlockingIOError` is raised if the underlying raw stream is in non
+ blocking-mode, and has no data available at the moment.
+
+ .. method:: readinto1(b)
+
+ Read up to ``len(b)`` bytes into bytearray *b*, ,using at most one call to
+ the underlying raw stream's :meth:`~RawIOBase.read` (or
+ :meth:`~RawIOBase.readinto`) method. Return the number of bytes read.
+
+ A :exc:`BlockingIOError` is raised if the underlying raw stream is in non
+ blocking-mode, and has no data available at the moment.
+
+ .. versionadded:: 3.5
.. method:: write(b)
@@ -507,9 +522,12 @@ Raw File I/O
The *name* can be one of two things:
* a character string or :class:`bytes` object representing the path to the
- file which will be opened;
+ file which will be opened. In this case closefd must be True (the default)
+ otherwise an error will be raised.
* an integer representing the number of an existing OS-level file descriptor
- to which the resulting :class:`FileIO` object will give access.
+ to which the resulting :class:`FileIO` object will give access. When the
+ FileIO object is closed this fd will be closed as well, unless *closefd*
+ is set to ``False``.
The *mode* can be ``'r'``, ``'w'``, ``'x'`` or ``'a'`` for reading
(default), writing, exclusive creation or appending. The file will be
@@ -598,6 +616,11 @@ than raw I/O does.
In :class:`BytesIO`, this is the same as :meth:`read`.
+ .. method:: readinto1()
+
+ In :class:`BytesIO`, this is the same as :meth:`readinto`.
+
+ .. versionadded:: 3.5
.. class:: BufferedReader(raw, buffer_size=DEFAULT_BUFFER_SIZE)
@@ -807,11 +830,13 @@ Text I/O
exception if there is an encoding error (the default of ``None`` has the same
effect), or pass ``'ignore'`` to ignore errors. (Note that ignoring encoding
errors can lead to data loss.) ``'replace'`` causes a replacement marker
- (such as ``'?'``) to be inserted where there is malformed data. When
- writing, ``'xmlcharrefreplace'`` (replace with the appropriate XML character
- reference) or ``'backslashreplace'`` (replace with backslashed escape
- sequences) can be used. Any other error handling name that has been
- registered with :func:`codecs.register_error` is also valid.
+ (such as ``'?'``) to be inserted where there is malformed data.
+ ``'backslashreplace'`` causes malformed data to be replaced by a
+ backslashed escape sequence. When writing, ``'xmlcharrefreplace'``
+ (replace with the appropriate XML character reference) or ``'namereplace'``
+ (replace with ``\N{...}`` escape sequences) can be used. Any other error
+ handling name that has been registered with
+ :func:`codecs.register_error` is also valid.
.. index::
single: universal newlines; io.TextIOWrapper class
diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst
index 301048e..90fcc74 100644
--- a/Doc/library/ipaddress.rst
+++ b/Doc/library/ipaddress.rst
@@ -146,6 +146,20 @@ write code that handles both IP versions correctly.
the appropriate length (most significant octet first). This is 4 bytes
for IPv4 and 16 bytes for IPv6.
+ .. attribute:: reverse_pointer
+
+ The name of the reverse DNS PTR record for the IP address, e.g.::
+
+ >>> ipaddress.ip_address("127.0.0.1").reverse_pointer
+ '1.0.0.127.in-addr.arpa'
+ >>> ipaddress.ip_address("2001:db8::1").reverse_pointer
+ '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa'
+
+ This is the name that could be used for performing a PTR lookup, not the
+ resolved hostname itself.
+
+ .. versionadded:: 3.5
+
.. attribute:: is_multicast
``True`` if the address is reserved for multicast use. See
@@ -226,6 +240,7 @@ write code that handles both IP versions correctly.
:class:`IPv4Address` class:
.. attribute:: packed
+ .. attribute:: reverse_pointer
.. attribute:: version
.. attribute:: max_prefixlen
.. attribute:: is_multicast
@@ -377,6 +392,12 @@ so to avoid duplication they are only documented for :class:`IPv4Network`.
3. An integer packed into a :class:`bytes` object of length 4, big-endian.
The interpretation is similar to an integer *address*.
+ 4. A two-tuple of an address description and a netmask, where the address
+ description is either a string, a 32-bits integer, a 4-bytes packed
+ integer, or an existing IPv4Address object; and the netmask is either
+ an integer representing the prefix length (e.g. ``24``) or a string
+ representing the prefix mask (e.g. ``255.255.255.0``).
+
An :exc:`AddressValueError` is raised if *address* is not a valid IPv4
address. A :exc:`NetmaskValueError` is raised if the mask is not valid for
an IPv4 address.
@@ -389,6 +410,10 @@ so to avoid duplication they are only documented for :class:`IPv4Network`.
objects will raise :exc:`TypeError` if the argument's IP version is
incompatible to ``self``
+ .. versionchanged:: 3.5
+
+ Added the two-tuple form for the *address* constructor parameter.
+
.. attribute:: version
.. attribute:: max_prefixlen
@@ -553,6 +578,11 @@ so to avoid duplication they are only documented for :class:`IPv4Network`.
3. An integer packed into a :class:`bytes` object of length 16, big-endian.
The interpretation is similar to an integer *address*.
+ 4. A two-tuple of an address description and a netmask, where the address
+ description is either a string, a 128-bits integer, a 16-bytes packed
+ integer, or an existing IPv4Address object; and the netmask is an
+ integer representing the prefix length.
+
An :exc:`AddressValueError` is raised if *address* is not a valid IPv6
address. A :exc:`NetmaskValueError` is raised if the mask is not valid for
an IPv6 address.
@@ -561,6 +591,10 @@ so to avoid duplication they are only documented for :class:`IPv4Network`.
then :exc:`ValueError` is raised. Otherwise, the host bits are masked out
to determine the appropriate network address.
+ .. versionchanged:: 3.5
+
+ Added the two-tuple form for the *address* constructor parameter.
+
.. attribute:: version
.. attribute:: max_prefixlen
.. attribute:: is_multicast
diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst
index f489535..8c7592d 100644
--- a/Doc/library/itertools.rst
+++ b/Doc/library/itertools.rst
@@ -87,10 +87,15 @@ loops that truncate the stream.
.. function:: accumulate(iterable[, func])
- Make an iterator that returns accumulated sums. Elements may be any addable
- type including :class:`~decimal.Decimal` or :class:`~fractions.Fraction`.
- If the optional *func* argument is supplied, it should be a function of two
- arguments and it will be used instead of addition.
+ Make an iterator that returns accumulated sums, or accumulated
+ results of other binary functions (specified via the optional
+ *func* argument). If *func* is supplied, it should be a function
+ of two arguments. Elements of the input *iterable* may be any type
+ that can be accepted as arguments to *func*. (For example, with
+ the default operation of addition, elements may be any addable
+ type including :class:`~decimal.Decimal` or
+ :class:`~fractions.Fraction`.) If the input iterable is empty, the
+ output iterable will also be empty.
Equivalent to::
@@ -99,7 +104,10 @@ loops that truncate the stream.
# accumulate([1,2,3,4,5]) --> 1 3 6 10 15
# accumulate([1,2,3,4,5], operator.mul) --> 1 2 6 24 120
it = iter(iterable)
- total = next(it)
+ try:
+ total = next(it)
+ except StopIteration:
+ return
yield total
for element in it:
total = func(total, element)
@@ -400,7 +408,10 @@ loops that truncate the stream.
def _grouper(self, tgtkey):
while self.currkey == tgtkey:
yield self.currvalue
- self.currvalue = next(self.it) # Exit on StopIteration
+ try:
+ self.currvalue = next(self.it)
+ except StopIteration:
+ return
self.currkey = self.keyfunc(self.currvalue)
@@ -424,7 +435,10 @@ loops that truncate the stream.
# islice('ABCDEFG', 0, None, 2) --> A C E G
s = slice(*args)
it = iter(range(s.start or 0, s.stop or sys.maxsize, s.step or 1))
- nexti = next(it)
+ try:
+ nexti = next(it)
+ except StopIteration:
+ return
for i, element in enumerate(iterable):
if i == nexti:
yield element
@@ -582,7 +596,10 @@ loops that truncate the stream.
def gen(mydeque):
while True:
if not mydeque: # when the local deque is empty
- newval = next(it) # fetch a new value and
+ try:
+ newval = next(it) # fetch a new value and
+ except StopIteration:
+ return
for d in deques: # load it to all the deques
d.append(newval)
yield mydeque.popleft()
@@ -657,6 +674,11 @@ which incur interpreter overhead.
"Return function(0), function(1), ..."
return map(function, count(start))
+ def tail(n, iterable):
+ "Return an iterator over the last n items"
+ # tail(3, 'ABCDEFG') --> E F G
+ return iter(collections.deque(iterable, maxlen=n))
+
def consume(iterator, n):
"Advance the iterator n-steps ahead. If n is none, consume entirely."
# Use functions that consume iterators at C speed.
diff --git a/Doc/library/json.rst b/Doc/library/json.rst
index 6f5f8b1..49bb090 100644
--- a/Doc/library/json.rst
+++ b/Doc/library/json.rst
@@ -106,6 +106,8 @@ Using json.tool from the shell to validate and pretty-print::
$ echo '{1.2:3.4}' | python -m json.tool
Expecting property name enclosed in double quotes: line 1 column 2 (char 1)
+See :ref:`json-commandline` for detailed documentation.
+
.. highlight:: python3
.. note::
@@ -248,7 +250,7 @@ Basic Usage
will be passed to the constructor of the class.
If the data being deserialized is not a valid JSON document, a
- :exc:`ValueError` will be raised.
+ :exc:`JSONDecodeError` will be raised.
.. function:: loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw)
@@ -259,7 +261,7 @@ Basic Usage
*encoding* which is ignored and deprecated.
If the data being deserialized is not a valid JSON document, a
- :exc:`ValueError` will be raised.
+ :exc:`JSONDecodeError` will be raised.
Encoders and Decoders
---------------------
@@ -332,13 +334,16 @@ Encoders and Decoders
``'\n'``, ``'\r'`` and ``'\0'``.
If the data being deserialized is not a valid JSON document, a
- :exc:`ValueError` will be raised.
+ :exc:`JSONDecodeError` will be raised.
.. method:: decode(s)
Return the Python representation of *s* (a :class:`str` instance
containing a JSON document)
+ :exc:`JSONDecodeError` will be raised if the given JSON document is not
+ valid.
+
.. method:: raw_decode(s)
Decode a JSON document from *s* (a :class:`str` beginning with a
@@ -467,6 +472,36 @@ Encoders and Decoders
mysocket.write(chunk)
+Exceptions
+----------
+
+.. exception:: JSONDecodeError(msg, doc, pos, end=None)
+
+ Subclass of :exc:`ValueError` with the following additional attributes:
+
+ .. attribute:: msg
+
+ The unformatted error message.
+
+ .. attribute:: doc
+
+ The JSON document being parsed.
+
+ .. attribute:: pos
+
+ The start index of *doc* where parsing failed.
+
+ .. attribute:: lineno
+
+ The line corresponding to *pos*.
+
+ .. attribute:: colno
+
+ The column corresponding to *pos*.
+
+ .. versionadded:: 3.5
+
+
Standard Compliance and Interoperability
----------------------------------------
@@ -588,6 +623,68 @@ when serializing Python :class:`int` values of extremely large magnitude, or
when serializing instances of "exotic" numerical types such as
:class:`decimal.Decimal`.
+.. highlight:: bash
+.. module:: json.tool
+
+.. _json-commandline:
+
+Command Line Interface
+----------------------
+
+The :mod:`json.tool` module provides a simple command line interface to validate
+and pretty-print JSON objects.
+
+If the optional ``infile`` and ``outfile`` arguments are not
+specified, :attr:`sys.stdin` and :attr:`sys.stdout` will be used respectively::
+
+ $ echo '{"json": "obj"}' | python -m json.tool
+ {
+ "json": "obj"
+ }
+ $ echo '{1.2:3.4}' | python -m json.tool
+ Expecting property name enclosed in double quotes: line 1 column 2 (char 1)
+
+.. versionchanged:: 3.5
+ The output is now in the same order as the input. Use the
+ :option:`--sort-keys` option to sort the output of dictionaries
+ alphabetically by key.
+
+Command line options
+^^^^^^^^^^^^^^^^^^^^
+
+.. cmdoption:: infile
+
+ The JSON file to be validated or pretty-printed::
+
+ $ python -m json.tool mp_films.json
+ [
+ {
+ "title": "And Now for Something Completely Different",
+ "year": 1971
+ },
+ {
+ "title": "Monty Python and the Holy Grail",
+ "year": 1975
+ }
+ ]
+
+ If *infile* is not specified, read from :attr:`sys.stdin`.
+
+.. cmdoption:: outfile
+
+ Write the output of the *infile* to the given *outfile*. Otherwise, write it
+ to :attr:`sys.stdout`.
+
+.. cmdoption:: --sort-keys
+
+ Sort the output of dictionaries alphabetically by key.
+
+ .. versionadded:: 3.5
+
+.. cmdoption:: -h, --help
+
+ Show the help message.
+
.. rubric:: Footnotes
diff --git a/Doc/library/linecache.rst b/Doc/library/linecache.rst
index f18b1cd..6c92cc5 100644
--- a/Doc/library/linecache.rst
+++ b/Doc/library/linecache.rst
@@ -47,6 +47,14 @@ The :mod:`linecache` module defines the following functions:
changed on disk, and you require the updated version. If *filename* is omitted,
it will check all the entries in the cache.
+.. function:: lazycache(filename, module_globals)
+
+ Capture enough detail about a non-file based module to permit getting its
+ lines later via :func:`getline` even if *module_globals* is None in the later
+ call. This avoids doing I/O until a line is actually needed, without having
+ to carry the module globals around indefinitely.
+
+ .. versionadded:: 3.5
Example::
diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst
index 9600193..bc7f5f9 100644
--- a/Doc/library/locale.rst
+++ b/Doc/library/locale.rst
@@ -387,6 +387,14 @@ The :mod:`locale` module defines the following exception and functions:
``str(float)``, but takes the decimal point into account.
+.. function:: delocalize(string)
+
+ Converts a string into a normalized number string, following the
+ :const:`LC_NUMERIC` settings.
+
+ .. versionadded:: 3.5
+
+
.. function:: atof(string)
Converts a string to a floating point number, following the :const:`LC_NUMERIC`
diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst
index d946892..67403a9 100644
--- a/Doc/library/logging.handlers.rst
+++ b/Doc/library/logging.handlers.rst
@@ -853,7 +853,7 @@ supports sending logging messages to a Web server, using either ``GET`` or
credentials, you should also specify secure=True so that your userid and
password are not passed in cleartext across the wire.
- .. versionchanged:: 3.4.3
+ .. versionchanged:: 3.5
The *context* parameter was added.
.. method:: mapLogRecord(record)
@@ -953,13 +953,20 @@ applications where threads servicing clients need to respond as quickly as
possible, while any potentially slow operations (such as sending an email via
:class:`SMTPHandler`) are done on a separate thread.
-.. class:: QueueListener(queue, *handlers)
+.. class:: QueueListener(queue, *handlers, respect_handler_level=False)
Returns a new instance of the :class:`QueueListener` class. The instance is
initialized with the queue to send messages to and a list of handlers which
will handle entries placed on the queue. The queue can be any queue-
like object; it's passed as-is to the :meth:`dequeue` method, which needs
- to know how to get messages from it.
+ to know how to get messages from it. If ``respect_handler_level`` is ``True``,
+ a handler's level is respected (compared with the level for the message) when
+ deciding whether to pass messages to that handler; otherwise, the behaviour
+ is as in previous Python versions - to always pass each message to each
+ handler.
+
+ .. versionchanged:: 3.5
+ The ``respect_handler_levels`` argument was added.
.. method:: dequeue(block)
diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst
index 90fb9b0..bf821ab 100644
--- a/Doc/library/logging.rst
+++ b/Doc/library/logging.rst
@@ -159,11 +159,13 @@ is the module's name in the Python package namespace.
*msg* using the string formatting operator. (Note that this means that you can
use keywords in the format string, together with a single dictionary argument.)
- There are three keyword arguments in *kwargs* which are inspected: *exc_info*
- which, if it does not evaluate as false, causes exception information to be
+ There are three keyword arguments in *kwargs* which are inspected:
+ *exc_info*, *stack_info*, and *extra*.
+
+ If *exc_info* does not evaluate as false, it causes exception information to be
added to the logging message. If an exception tuple (in the format returned by
- :func:`sys.exc_info`) is provided, it is used; otherwise, :func:`sys.exc_info`
- is called to get the exception information.
+ :func:`sys.exc_info`) or an exception instance is provided, it is used;
+ otherwise, :func:`sys.exc_info` is called to get the exception information.
The second optional keyword argument is *stack_info*, which defaults to
``False``. If true, stack information is added to the logging
@@ -220,6 +222,9 @@ is the module's name in the Python package namespace.
.. versionadded:: 3.2
The *stack_info* parameter was added.
+ .. versionchanged:: 3.5
+ The *exc_info* parameter can now accept exception instances.
+
.. method:: Logger.info(msg, *args, **kwargs)
diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst
index b71051d..0546005 100644
--- a/Doc/library/lzma.rst
+++ b/Doc/library/lzma.rst
@@ -110,6 +110,10 @@ Reading and writing compressed files
.. versionchanged:: 3.4
Added support for the ``"x"`` and ``"xb"`` modes.
+ .. versionchanged:: 3.5
+ The :meth:`~io.BufferedIOBase.read` method now accepts an argument of
+ ``None``.
+
Compressing and decompressing data in memory
--------------------------------------------
@@ -221,13 +225,32 @@ Compressing and decompressing data in memory
decompress a multi-stream input with :class:`LZMADecompressor`, you must
create a new decompressor for each stream.
- .. method:: decompress(data)
+ .. method:: decompress(data, max_length=-1)
+
+ Decompress *data* (a :term:`bytes-like object`), returning
+ uncompressed data as bytes. Some of *data* may be buffered
+ internally, for use in later calls to :meth:`decompress`. The
+ returned data should be concatenated with the output of any
+ previous calls to :meth:`decompress`.
+
+ If *max_length* is nonnegative, returns at most *max_length*
+ bytes of decompressed data. If this limit is reached and further
+ output can be produced, the :attr:`~.needs_input` attribute will
+ be set to ``False``. In this case, the next call to
+ :meth:`~.decompress` may provide *data* as ``b''`` to obtain
+ more of the output.
+
+ If all of the input data was decompressed and returned (either
+ because this was less than *max_length* bytes, or because
+ *max_length* was negative), the :attr:`~.needs_input` attribute
+ will be set to ``True``.
- Decompress *data* (a :class:`bytes` object), returning a :class:`bytes`
- object containing the decompressed data for at least part of the input.
- Some of *data* may be buffered internally, for use in later calls to
- :meth:`decompress`. The returned data should be concatenated with the
- output of any previous calls to :meth:`decompress`.
+ Attempting to decompress data after the end of stream is reached
+ raises an `EOFError`. Any data found after the end of the
+ stream is ignored and saved in the :attr:`~.unused_data` attribute.
+
+ .. versionchanged:: 3.5
+ Added the *max_length* parameter.
.. attribute:: check
@@ -245,6 +268,12 @@ Compressing and decompressing data in memory
Before the end of the stream is reached, this will be ``b""``.
+ .. attribute:: needs_input
+
+ ``False`` if the :meth:`.decompress` method can provide more
+ decompressed data before requiring new uncompressed input.
+
+ .. versionadded:: 3.5
.. function:: compress(data, format=FORMAT_XZ, check=-1, preset=None, filters=None)
diff --git a/Doc/library/math.rst b/Doc/library/math.rst
index 0fc7c7c..244663e 100644
--- a/Doc/library/math.rst
+++ b/Doc/library/math.rst
@@ -100,6 +100,48 @@ Number-theoretic and representation functions
<http://code.activestate.com/recipes/393090/>`_\.
+.. function:: gcd(a, b)
+
+ Return the greatest common divisor of the integers *a* and *b*. If either
+ *a* or *b* is nonzero, then the value of ``gcd(a, b)`` is the largest
+ positive integer that divides both *a* and *b*. ``gcd(0, 0)`` returns
+ ``0``.
+
+ .. versionadded:: 3.5
+
+
+.. function:: isclose(a, b, *, rel_tol=1e-09, abs_tol=0.0)
+
+ Return ``True`` if the values *a* and *b* are close to each other and
+ ``False`` otherwise.
+
+ Whether or not two values are considered close is determined according to
+ given absolute and relative tolerances.
+
+ *rel_tol* is the relative tolerance -- it is the maximum allowed difference
+ between *a* and *b*, relative to the larger absolute value of *a* or *b*.
+ For example, to set a tolerance of 5%, pass ``rel_tol=0.05``. The default
+ tolerance is ``1e-09``, which assures that the two values are the same
+ within about 9 decimal digits. *rel_tol* must be greater than zero.
+
+ *abs_tol* is the minimum absolute tolerance -- useful for comparisons near
+ zero. *abs_tol* must be at least zero.
+
+ If no errors occur, the result will be:
+ ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``.
+
+ The IEEE 754 special values of ``NaN``, ``inf``, and ``-inf`` will be
+ handled according to IEEE rules. Specifically, ``NaN`` is not considered
+ close to any other value, including ``NaN``. ``inf`` and ``-inf`` are only
+ considered close to themselves.
+
+ .. versionadded:: 3.5
+
+ .. seealso::
+
+ :pep:`485` -- A function for testing approximate equality
+
+
.. function:: isfinite(x)
Return ``True`` if *x* is neither an infinity nor a NaN, and
@@ -383,6 +425,22 @@ Constants
The mathematical constant e = 2.718281..., to available precision.
+.. data:: inf
+
+ A floating-point positive infinity. (For negative infinity, use
+ ``-math.inf``.) Equivalent to the output of ``float('inf')``.
+
+ .. versionadded:: 3.5
+
+
+.. data:: nan
+
+ A floating-point "not a number" (NaN) value. Equivalent to the output of
+ ``float('nan')``.
+
+ .. versionadded:: 3.5
+
+
.. impl-detail::
The :mod:`math` module consists mostly of thin wrappers around the platform C
diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst
index 18e05e3..b74a823 100644
--- a/Doc/library/mmap.rst
+++ b/Doc/library/mmap.rst
@@ -174,6 +174,9 @@ To map anonymous memory, -1 should be passed as the fileno along with the length
Optional arguments *start* and *end* are interpreted as in slice notation.
Returns ``-1`` on failure.
+ .. versionchanged: 3.5
+ Writable :term:`bytes-like object` is now accepted.
+
.. method:: flush([offset[, size]])
@@ -234,6 +237,9 @@ To map anonymous memory, -1 should be passed as the fileno along with the length
Optional arguments *start* and *end* are interpreted as in slice notation.
Returns ``-1`` on failure.
+ .. versionchanged: 3.5
+ Writable :term:`bytes-like object` is now accepted.
+
.. method:: seek(pos[, whence])
@@ -261,6 +267,9 @@ To map anonymous memory, -1 should be passed as the fileno along with the length
were written. If the mmap was created with :const:`ACCESS_READ`, then
writing to it will raise a :exc:`TypeError` exception.
+ .. versionchanged: 3.5
+ Writable :term:`bytes-like object` is now accepted.
+
.. method:: write_byte(byte)
diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst
index 4d5f308..8703f8f 100644
--- a/Doc/library/multiprocessing.rst
+++ b/Doc/library/multiprocessing.rst
@@ -1352,6 +1352,9 @@ processes.
Note that accessing the ctypes object through the wrapper can be a lot slower
than accessing the raw ctypes object.
+ .. versionchanged:: 3.5
+ Synchronized objects support the :term:`context manager` protocol.
+
The table below compares the syntax for creating shared ctypes objects from
shared memory with the normal ctypes syntax. (In the table ``MyStruct`` is some
@@ -1839,9 +1842,9 @@ itself. This means, for example, that one shared object can contain a second:
>>> l = manager.list(range(10))
>>> l._callmethod('__len__')
10
- >>> l._callmethod('__getitem__', (slice(2, 7),)) # equiv to `l[2:7]`
+ >>> l._callmethod('__getitem__', (slice(2, 7),)) # equivalent to l[2:7]
[2, 3, 4, 5, 6]
- >>> l._callmethod('__getitem__', (20,)) # equiv to `l[20]`
+ >>> l._callmethod('__getitem__', (20,)) # equivalent to l[20]
Traceback (most recent call last):
...
IndexError: list index out of range
diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst
index f9e2a3d..c01e63b 100644
--- a/Doc/library/operator.rst
+++ b/Doc/library/operator.rst
@@ -138,6 +138,14 @@ The mathematical and bitwise operations are the most numerous:
Return ``a * b``, for *a* and *b* numbers.
+.. function:: matmul(a, b)
+ __matmul__(a, b)
+
+ Return ``a @ b``.
+
+ .. versionadded:: 3.5
+
+
.. function:: neg(obj)
__neg__(obj)
@@ -391,6 +399,8 @@ Python syntax and the functions in the :mod:`operator` module.
+-----------------------+-------------------------+---------------------------------------+
| Multiplication | ``a * b`` | ``mul(a, b)`` |
+-----------------------+-------------------------+---------------------------------------+
+| Matrix Multiplication | ``a @ b`` | ``matmul(a, b)`` |
++-----------------------+-------------------------+---------------------------------------+
| Negation (Arithmetic) | ``- a`` | ``neg(a)`` |
+-----------------------+-------------------------+---------------------------------------+
| Negation (Logical) | ``not a`` | ``not_(a)`` |
@@ -499,6 +509,14 @@ will perform the update, so no subsequent assignment is necessary:
``a = imul(a, b)`` is equivalent to ``a *= b``.
+.. function:: imatmul(a, b)
+ __imatmul__(a, b)
+
+ ``a = imatmul(a, b)`` is equivalent to ``a @= b``.
+
+ .. versionadded:: 3.5
+
+
.. function:: ior(a, b)
__ior__(a, b)
diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst
index 92631b2..a3fe73c 100644
--- a/Doc/library/os.path.rst
+++ b/Doc/library/os.path.rst
@@ -66,11 +66,37 @@ the :mod:`glob` module.)
empty string (``''``).
+.. function:: commonpath(paths)
+
+ Return the longest common sub-path of each pathname in the sequence
+ *paths*. Raise ValueError if *paths* contains both absolute and relative
+ pathnames, or if *paths* is empty. Unlike :func:`commonprefix`, this
+ returns a valid path.
+
+ Availability: Unix, Windows
+
+ .. versionadded:: 3.5
+
+
.. function:: commonprefix(list)
- Return the longest path prefix (taken character-by-character) that is a prefix
- of all paths in *list*. If *list* is empty, return the empty string (``''``).
- Note that this may return invalid paths because it works a character at a time.
+ Return the longest path prefix (taken character-by-character) that is a
+ prefix of all paths in *list*. If *list* is empty, return the empty string
+ (``''``).
+
+ .. note::
+
+ This function may return invalid paths because it works a
+ character at a time. To obtain a valid path, see
+ :func:`commonpath`.
+
+ ::
+
+ >>> os.path.commonprefix(['/usr/lib', '/usr/local/lib'])
+ '/usr/l'
+
+ >>> os.path.commonpath(['/usr/lib', '/usr/local/lib'])
+ '/usr'
.. function:: dirname(path)
diff --git a/Doc/library/os.rst b/Doc/library/os.rst
index d677b39..d4032e0 100644
--- a/Doc/library/os.rst
+++ b/Doc/library/os.rst
@@ -78,9 +78,10 @@ uses the file system encoding to perform this conversion (see
.. versionchanged:: 3.1
On some systems, conversion using the file system encoding may fail. In this
- case, Python uses the ``surrogateescape`` encoding error handler, which means
- that undecodable bytes are replaced by a Unicode character U+DCxx on
- decoding, and these are again translated to the original byte on encoding.
+ case, Python uses the :ref:`surrogateescape encoding error handler
+ <surrogateescape>`, which means that undecodable bytes are replaced by a
+ Unicode character U+DCxx on decoding, and these are again translated to the
+ original byte on encoding.
The file system encoding must guarantee to successfully decode all bytes
@@ -311,8 +312,6 @@ process and user.
Return the current process id.
- Availability: Unix, Windows.
-
.. function:: getppid()
@@ -549,8 +548,6 @@ process and user.
On platforms where :c:func:`strerror` returns ``NULL`` when given an unknown
error number, :exc:`ValueError` is raised.
- Availability: Unix, Windows.
-
.. data:: supports_bytes_environ
@@ -564,8 +561,6 @@ process and user.
Set the current numeric umask and return the previous umask.
- Availability: Unix, Windows.
-
.. function:: uname()
@@ -656,8 +651,6 @@ as internal buffering of data.
Close file descriptor *fd*.
- Availability: Unix, Windows.
-
.. note::
This function is intended for low-level I/O and must be applied to a file
@@ -677,8 +670,6 @@ as internal buffering of data.
except OSError:
pass
- Availability: Unix, Windows.
-
.. function:: device_encoding(fd)
@@ -695,8 +686,6 @@ as internal buffering of data.
2: stderr), the new file descriptor is :ref:`inheritable
<fd_inheritance>`.
- Availability: Unix, Windows.
-
.. versionchanged:: 3.4
The new file descriptor is now non-inheritable.
@@ -707,8 +696,6 @@ as internal buffering of data.
The file descriptor *fd2* is :ref:`inheritable <fd_inheritance>` by default,
or non-inheritable if *inheritable* is ``False``.
- Availability: Unix, Windows.
-
.. versionchanged:: 3.4
Add the optional *inheritable* parameter.
@@ -774,8 +761,6 @@ as internal buffering of data.
The :func:`.stat` function.
- Availability: Unix, Windows.
-
.. function:: fstatvfs(fd)
@@ -804,8 +789,21 @@ as internal buffering of data.
most *length* bytes in size. As of Python 3.3, this is equivalent to
``os.truncate(fd, length)``.
+ Availability: Unix, Windows.
+
+ .. versionchanged:: 3.5
+ Added support for Windows
+
+.. function:: get_blocking(fd)
+
+ Get the blocking mode of the file descriptor: ``False`` if the
+ :data:`O_NONBLOCK` flag is set, ``True`` if the flag is cleared.
+
+ See also :func:`set_blocking` and :meth:`socket.socket.setblocking`.
+
Availability: Unix.
+ .. versionadded:: 3.5
.. function:: isatty(fd)
@@ -846,8 +844,6 @@ as internal buffering of data.
current position; :const:`SEEK_END` or ``2`` to set it relative to the end of
the file. Return the new cursor position in bytes, starting from the beginning.
- Availability: Unix, Windows.
-
.. data:: SEEK_SET
SEEK_CUR
@@ -856,8 +852,6 @@ as internal buffering of data.
Parameters to the :func:`lseek` function. Their values are 0, 1, and 2,
respectively.
- Availability: Unix, Windows.
-
.. versionadded:: 3.3
Some operating systems could support additional values, like
:data:`os.SEEK_HOLE` or :data:`os.SEEK_DATA`.
@@ -878,8 +872,6 @@ as internal buffering of data.
This function can support :ref:`paths relative to directory descriptors
<dir_fd>` with the *dir_fd* parameter.
- Availability: Unix, Windows.
-
.. versionchanged:: 3.4
The new file descriptor is now non-inheritable.
@@ -893,6 +885,11 @@ as internal buffering of data.
.. versionadded:: 3.3
The *dir_fd* argument.
+ .. versionchanged:: 3.5
+ If the system call is interrupted and the signal handler does not raise an
+ exception, the function now retries the system call instead of raising an
+ :exc:`InterruptedError` exception (see :pep:`475` for the rationale).
+
The following constants are options for the *flags* parameter to the
:func:`~os.open` function. They can be combined using the bitwise OR operator
``|``. Some of them are not available on all platforms. For descriptions of
@@ -1060,8 +1057,6 @@ or `the MSDN <http://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Window
bytes read. If the end of the file referred to by *fd* has been reached, an
empty bytes object is returned.
- Availability: Unix, Windows.
-
.. note::
This function is intended for low-level I/O and must be applied to a file
@@ -1070,6 +1065,11 @@ or `the MSDN <http://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Window
:func:`popen` or :func:`fdopen`, or :data:`sys.stdin`, use its
:meth:`~file.read` or :meth:`~file.readline` methods.
+ .. versionchanged:: 3.5
+ If the system call is interrupted and the signal handler does not raise an
+ exception, the function now retries the system call instead of raising an
+ :exc:`InterruptedError` exception (see :pep:`475` for the rationale).
+
.. function:: sendfile(out, in, offset, nbytes)
sendfile(out, in, offset, nbytes, headers=None, trailers=None, flags=0)
@@ -1107,6 +1107,18 @@ or `the MSDN <http://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Window
.. versionadded:: 3.3
+.. function:: set_blocking(fd, blocking)
+
+ Set the blocking mode of the specified file descriptor. Set the
+ :data:`O_NONBLOCK` flag if blocking is ``False``, clear the flag otherwise.
+
+ See also :func:`get_blocking` and :meth:`socket.socket.setblocking`.
+
+ Availability: Unix.
+
+ .. versionadded:: 3.5
+
+
.. data:: SF_NODISKIO
SF_MNOWAIT
SF_SYNC
@@ -1163,8 +1175,6 @@ or `the MSDN <http://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Window
Write the bytestring in *str* to file descriptor *fd*. Return the number of
bytes actually written.
- Availability: Unix, Windows.
-
.. note::
This function is intended for low-level I/O and must be applied to a file
@@ -1173,6 +1183,11 @@ or `the MSDN <http://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Window
:func:`fdopen`, or :data:`sys.stdout` or :data:`sys.stderr`, use its
:meth:`~file.write` method.
+ .. versionchanged:: 3.5
+ If the system call is interrupted and the signal handler does not raise an
+ exception, the function now retries the system call instead of raising an
+ :exc:`InterruptedError` exception (see :pep:`475` for the rationale).
+
.. function:: writev(fd, buffers)
@@ -1335,8 +1350,6 @@ features:
or not it is available using :data:`os.supports_effective_ids`. If it is
unavailable, using it will raise a :exc:`NotImplementedError`.
- Availability: Unix, Windows.
-
.. note::
Using :func:`access` to check if a user is authorized to e.g. open a file
@@ -1389,8 +1402,6 @@ features:
This function can support :ref:`specifying a file descriptor <path_fd>`. The
descriptor must refer to an opened directory, not an open file.
- Availability: Unix, Windows.
-
.. versionadded:: 3.3
Added support for specifying *path* as a file descriptor
on some platforms.
@@ -1452,8 +1463,6 @@ features:
:ref:`paths relative to directory descriptors <dir_fd>` and :ref:`not
following symlinks <follow_symlinks>`.
- Availability: Unix, Windows.
-
.. note::
Although Windows supports :func:`chmod`, you can only set the file's
@@ -1504,15 +1513,11 @@ features:
Return a string representing the current working directory.
- Availability: Unix, Windows.
-
.. function:: getcwdb()
Return a bytestring representing the current working directory.
- Availability: Unix, Windows.
-
.. function:: lchflags(path, flags)
@@ -1575,7 +1580,11 @@ features:
.. note::
To encode ``str`` filenames to ``bytes``, use :func:`~os.fsencode`.
- Availability: Unix, Windows.
+ .. seealso::
+
+ The :func:`scandir` function returns directory entries along with
+ file attribute information, giving better performance for many
+ common use cases.
.. versionchanged:: 3.2
The *path* parameter became optional.
@@ -1624,8 +1633,6 @@ features:
It is also possible to create temporary directories; see the
:mod:`tempfile` module's :func:`tempfile.mkdtemp` function.
- Availability: Unix, Windows.
-
.. versionadded:: 3.3
The *dir_fd* argument.
@@ -1784,8 +1791,6 @@ features:
This function is identical to :func:`unlink`.
- Availability: Unix, Windows.
-
.. versionadded:: 3.3
The *dir_fd* argument.
@@ -1819,8 +1824,6 @@ features:
If you want cross-platform overwriting of the destination, use :func:`replace`.
- Availability: Unix, Windows.
-
.. versionadded:: 3.3
The *src_dir_fd* and *dst_dir_fd* arguments.
@@ -1849,8 +1852,6 @@ features:
This function can support specifying *src_dir_fd* and/or *dst_dir_fd* to
supply :ref:`paths relative to directory descriptors <dir_fd>`.
- Availability: Unix, Windows.
-
.. versionadded:: 3.3
@@ -1863,12 +1864,180 @@ features:
This function can support :ref:`paths relative to directory descriptors
<dir_fd>`.
- Availability: Unix, Windows.
-
.. versionadded:: 3.3
The *dir_fd* parameter.
+.. function:: scandir(path='.')
+
+ Return an iterator of :class:`DirEntry` objects corresponding to the entries
+ in the directory given by *path*. The entries are yielded in arbitrary
+ order, and the special entries ``'.'`` and ``'..'`` are not included.
+
+ Using :func:`scandir` instead of :func:`listdir` can significantly
+ increase the performance of code that also needs file type or file
+ attribute information, because :class:`DirEntry` objects expose this
+ information if the operating system provides it when scanning a directory.
+ All :class:`DirEntry` methods may perform a system call, but
+ :func:`~DirEntry.is_dir` and :func:`~DirEntry.is_file` usually only
+ require a system call for symbolic links; :func:`DirEntry.stat`
+ always requires a system call on Unix but only requires one for
+ symbolic links on Windows.
+
+ On Unix, *path* can be of type :class:`str` or :class:`bytes` (use
+ :func:`~os.fsencode` and :func:`~os.fsdecode` to encode and decode
+ :class:`bytes` paths). On Windows, *path* must be of type :class:`str`.
+ On both sytems, the type of the :attr:`~DirEntry.name` and
+ :attr:`~DirEntry.path` attributes of each :class:`DirEntry` will be of
+ the same type as *path*.
+
+ The following example shows a simple use of :func:`scandir` to display all
+ the files (excluding directories) in the given *path* that don't start with
+ ``'.'``. The ``entry.is_file()`` call will generally not make an additional
+ system call::
+
+ for entry in os.scandir(path):
+ if not entry.name.startswith('.') and entry.is_file():
+ print(entry.name)
+
+ .. note::
+
+ On Unix-based systems, :func:`scandir` uses the system's
+ `opendir() <http://pubs.opengroup.org/onlinepubs/009695399/functions/opendir.html>`_
+ and
+ `readdir() <http://pubs.opengroup.org/onlinepubs/009695399/functions/readdir_r.html>`_
+ functions. On Windows, it uses the Win32
+ `FindFirstFileW <http://msdn.microsoft.com/en-us/library/windows/desktop/aa364418(v=vs.85).aspx>`_
+ and
+ `FindNextFileW <http://msdn.microsoft.com/en-us/library/windows/desktop/aa364428(v=vs.85).aspx>`_
+ functions.
+
+ .. versionadded:: 3.5
+
+
+.. class:: DirEntry
+
+ Object yielded by :func:`scandir` to expose the file path and other file
+ attributes of a directory entry.
+
+ :func:`scandir` will provide as much of this information as possible without
+ making additional system calls. When a ``stat()`` or ``lstat()`` system call
+ is made, the ``DirEntry`` object will cache the result.
+
+ ``DirEntry`` instances are not intended to be stored in long-lived data
+ structures; if you know the file metadata has changed or if a long time has
+ elapsed since calling :func:`scandir`, call ``os.stat(entry.path)`` to fetch
+ up-to-date information.
+
+ Because the ``DirEntry`` methods can make operating system calls, they may
+ also raise :exc:`OSError`. If you need very fine-grained
+ control over errors, you can catch :exc:`OSError` when calling one of the
+ ``DirEntry`` methods and handle as appropriate.
+
+ Attributes and methods on a ``DirEntry`` instance are as follows:
+
+ .. attribute:: name
+
+ The entry's base filename, relative to the :func:`scandir` *path*
+ argument.
+
+ The :attr:`name` attribute will be of the same type (``str`` or
+ ``bytes``) as the :func:`scandir` *path* argument. Use
+ :func:`~os.fsdecode` to decode byte filenames.
+
+ .. attribute:: path
+
+ The entry's full path name: equivalent to ``os.path.join(scandir_path,
+ entry.name)`` where *scandir_path* is the :func:`scandir` *path*
+ argument. The path is only absolute if the :func:`scandir` *path*
+ argument was absolute.
+
+ The :attr:`path` attribute will be of the same type (``str`` or
+ ``bytes``) as the :func:`scandir` *path* argument. Use
+ :func:`~os.fsdecode` to decode byte filenames.
+
+ .. method:: inode()
+
+ Return the inode number of the entry.
+
+ The result is cached on the ``DirEntry`` object, use ``os.stat(entry.path,
+ follow_symlinks=False).st_ino`` to fetch up-to-date information.
+
+ On Unix, no system call is required.
+
+ .. method:: is_dir(\*, follow_symlinks=True)
+
+ If *follow_symlinks* is ``True`` (the default), return ``True`` if the
+ entry is a directory or a symbolic link pointing to a directory;
+ return ``False`` if it is or points to any other kind of file, or if it
+ doesn't exist anymore.
+
+ If *follow_symlinks* is ``False``, return ``True`` only if this entry
+ is a directory; return ``False`` if it is any other kind of file
+ or if it doesn't exist anymore.
+
+ The result is cached on the ``DirEntry`` object. Call :func:`os.stat`
+ along with :func:`stat.S_ISDIR` to fetch up-to-date information.
+
+ This method can raise :exc:`OSError`, such as :exc:`PermissionError`,
+ but :exc:`FileNotFoundError` is caught and not raised.
+
+ In most cases, no system call is required.
+
+ .. method:: is_file(\*, follow_symlinks=True)
+
+ If *follow_symlinks* is ``True`` (the default), return ``True`` if the
+ entry is a file or a symbolic link pointing to a file; return ``False``
+ if it is or points to a directory or other non-file entry, or if it
+ doesn't exist anymore.
+
+ If *follow_symlinks* is ``False``, return ``True`` only if this entry
+ is a file; return ``False`` if it is a directory or other non-file entry,
+ or if it doesn't exist anymore.
+
+ The result is cached on the ``DirEntry`` object. Call :func:`os.stat`
+ along with :func:`stat.S_ISREG` to fetch up-to-date information.
+
+ This method can raise :exc:`OSError`, such as :exc:`PermissionError`,
+ but :exc:`FileNotFoundError` is caught and not raised.
+
+ In most cases, no system call is required.
+
+ .. method:: is_symlink()
+
+ Return ``True`` if this entry is a symbolic link (even if broken);
+ return ``False`` if it points to a directory or any kind of file,
+ or if it doesn't exist anymore.
+
+ The result is cached on the ``DirEntry`` object. Call
+ :func:`os.path.islink` to fetch up-to-date information.
+
+ The method can raise :exc:`OSError`, such as :exc:`PermissionError`,
+ but :exc:`FileNotFoundError` is caught and not raised.
+
+ In most cases, no system call is required.
+
+ .. method:: stat(\*, follow_symlinks=True)
+
+ Return a :class:`stat_result` object for this entry. This method
+ follows symbolic links by default; to stat a symbolic link add the
+ ``follow_symlinks=False`` argument.
+
+ On Unix, this method always requires a system call. On Windows,
+ ``DirEntry.stat()`` requires a system call only if the
+ entry is a symbolic link, and ``DirEntry.stat(follow_symlinks=False)``
+ never requires a system call.
+
+ On Windows, the ``st_ino``, ``st_dev`` and ``st_nlink`` attributes of the
+ :class:`stat_result` are always set to zero. Call :func:`os.stat` to
+ get these attributes.
+
+ The result is cached on the ``DirEntry`` object. Call :func:`os.stat`
+ to fetch up-to-date information.
+
+ .. versionadded:: 3.5
+
+
.. function:: stat(path, \*, dir_fd=None, follow_symlinks=True)
Get the status of a file or a file descriptor. Perform the equivalent of a
@@ -1895,8 +2064,6 @@ features:
>>> statinfo.st_size
264
- Availability: Unix, Windows.
-
.. seealso::
:func:`fstat` and :func:`lstat` functions.
@@ -2044,6 +2211,15 @@ features:
File type.
+ On Windows systems, the following attribute is also available:
+
+ .. attribute:: st_file_attributes
+
+ Windows file attributes: ``dwFileAttributes`` member of the
+ ``BY_HANDLE_FILE_INFORMATION`` structure returned by
+ :c:func:`GetFileInformationByHandle`. See the ``FILE_ATTRIBUTE_*``
+ constants in the :mod:`stat` module.
+
The standard module :mod:`stat` defines functions and constants that are
useful for extracting information from a :c:type:`stat` structure. (On
Windows, some items are filled with dummy values.)
@@ -2061,6 +2237,9 @@ features:
Added the :attr:`st_atime_ns`, :attr:`st_mtime_ns`, and
:attr:`st_ctime_ns` members.
+ .. versionadded:: 3.5
+ Added the :attr:`st_file_attributes` member on Windows.
+
.. function:: stat_float_times([newvalue])
@@ -2264,10 +2443,12 @@ features:
This function can support :ref:`specifying a file descriptor <path_fd>`.
- Availability: Unix.
+ Availability: Unix, Windows.
.. versionadded:: 3.3
+ .. versionchanged:: 3.5
+ Added support for Windows
.. function:: unlink(path, *, dir_fd=None)
@@ -2276,8 +2457,6 @@ features:
name. Please see the documentation for :func:`remove` for
further information.
- Availability: Unix, Windows.
-
.. versionadded:: 3.3
The *dir_fd* parameter.
@@ -2314,8 +2493,6 @@ features:
:ref:`paths relative to directory descriptors <dir_fd>` and :ref:`not
following symlinks <follow_symlinks>`.
- Availability: Unix, Windows.
-
.. versionadded:: 3.3
Added support for specifying an open file descriptor for *path*,
and the *dir_fd*, *follow_symlinks*, and *ns* parameters.
@@ -2406,6 +2583,10 @@ features:
for name in dirs:
os.rmdir(os.path.join(root, name))
+ .. versionchanged:: 3.5
+ This function now calls :func:`os.scandir` instead of :func:`os.listdir`,
+ making it faster by reducing the number of calls to :func:`os.stat`.
+
.. function:: fwalk(top='.', topdown=True, onerror=None, *, follow_symlinks=False, dir_fd=None)
@@ -2562,8 +2743,6 @@ to be ignored.
Python signal handler registered for :const:`SIGABRT` with
:func:`signal.signal`.
- Availability: Unix, Windows.
-
.. function:: execl(path, arg0, arg1, ...)
execle(path, arg0, arg1, ..., env)
@@ -2627,8 +2806,6 @@ to be ignored.
Exit the process with status *n*, without calling cleanup handlers, flushing
stdio buffers, etc.
- Availability: Unix, Windows.
-
.. note::
The standard way to exit is ``sys.exit(n)``. :func:`_exit` should
@@ -2990,6 +3167,10 @@ written in Python, such as a mail server's external command delivery program.
doesn't work if it is. Use the :func:`os.path.normpath` function to ensure that
the path is properly encoded for Win32.
+ To reduce interpreter startup overhead, the Win32 :c:func:`ShellExecute`
+ function is not resolved until this function is first called. If the function
+ cannot be resolved, :exc:`NotImplementedError` will be raised.
+
Availability: Windows.
@@ -3137,6 +3318,11 @@ written in Python, such as a mail server's external command delivery program.
id is known, not necessarily a child process. The :func:`spawn\* <spawnl>`
functions called with :const:`P_NOWAIT` return suitable process handles.
+ .. versionchanged:: 3.5
+ If the system call is interrupted and the signal handler does not raise an
+ exception, the function now retries the system call instead of raising an
+ :exc:`InterruptedError` exception (see :pep:`475` for the rationale).
+
.. function:: wait3(options)
@@ -3544,3 +3730,9 @@ Miscellaneous Functions
For an easy-to-use interface to the random number generator
provided by your platform, please see :class:`random.SystemRandom`.
+
+ .. versionchanged:: 3.5
+ On Linux 3.17 and newer, the ``getrandom()`` syscall is now used
+ when available. On OpenBSD 5.6 and newer, the C ``getentropy()``
+ function is now used. These functions avoid the usage of an internal file
+ descriptor.
diff --git a/Doc/library/ossaudiodev.rst b/Doc/library/ossaudiodev.rst
index bb5081a..c60d596 100644
--- a/Doc/library/ossaudiodev.rst
+++ b/Doc/library/ossaudiodev.rst
@@ -148,21 +148,30 @@ and (read-only) attributes:
.. method:: oss_audio_device.write(data)
- Write the Python string *data* to the audio device and return the number of
- bytes written. If the audio device is in blocking mode (the default), the
- entire string is always written (again, this is different from usual Unix device
- semantics). If the device is in non-blocking mode, some data may not be written
+ Write a :term:`bytes-like object` *data* to the audio device and return the
+ number of bytes written. If the audio device is in blocking mode (the
+ default), the entire data is always written (again, this is different from
+ usual Unix device semantics). If the device is in non-blocking mode, some
+ data may not be written
---see :meth:`writeall`.
+ .. versionchanged: 3.5
+ Writable :term:`bytes-like object` is now accepted.
+
.. method:: oss_audio_device.writeall(data)
- Write the entire Python string *data* to the audio device: waits until the audio
- device is able to accept data, writes as much data as it will accept, and
- repeats until *data* has been completely written. If the device is in blocking
- mode (the default), this has the same effect as :meth:`write`; :meth:`writeall`
- is only useful in non-blocking mode. Has no return value, since the amount of
- data written is always equal to the amount of data supplied.
+ Write a :term:`bytes-like object` *data* to the audio device: waits until
+ the audio device is able to accept data, writes as much data as it will
+ accept, and repeats until *data* has been completely written. If the device
+ is in blocking mode (the default), this has the same effect as
+ :meth:`write`; :meth:`writeall` is only useful in non-blocking mode. Has
+ no return value, since the amount of data written is always equal to the
+ amount of data supplied.
+
+ .. versionchanged: 3.5
+ Writable :term:`bytes-like object` is now accepted.
+
.. versionchanged:: 3.2
Audio device objects also support the context management protocol, i.e. they can
diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst
index 24e2a30..2f06544 100644
--- a/Doc/library/pathlib.rst
+++ b/Doc/library/pathlib.rst
@@ -628,6 +628,17 @@ call fails (for example because the path doesn't exist):
PosixPath('/home/antoine/pathlib')
+.. classmethod:: Path.home()
+
+ Return a new path object representing the user's home directory (as
+ returned by :func:`os.path.expanduser` with ``~`` construct)::
+
+ >>> Path.home()
+ PosixPath('/home/antoine')
+
+ .. versionadded:: 3.5
+
+
.. method:: Path.stat()
Return information about this path (similarly to :func:`os.stat`).
@@ -670,6 +681,18 @@ call fails (for example because the path doesn't exist):
symlink *points to* an existing file or directory.
+.. method:: Path.expanduser()
+
+ Return a new path with expanded ``~`` and ``~user`` constructs,
+ as returned by :meth:`os.path.expanduser`::
+
+ >>> p = PosixPath('~/films/Monty Python')
+ >>> p.expanduser()
+ PosixPath('/home/eric/films/Monty Python')
+
+ .. versionadded:: 3.5
+
+
.. method:: Path.glob(pattern)
Glob the given *pattern* in the directory represented by this path,
@@ -791,7 +814,7 @@ call fails (for example because the path doesn't exist):
the symbolic link's information rather than its target's.
-.. method:: Path.mkdir(mode=0o777, parents=False)
+.. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False)
Create a new directory at this given path. If *mode* is given, it is
combined with the process' ``umask`` value to determine the file mode
@@ -805,6 +828,16 @@ call fails (for example because the path doesn't exist):
If *parents* is false (the default), a missing parent raises
:exc:`FileNotFoundError`.
+ If *exist_ok* is false (the default), an :exc:`FileExistsError` is
+ raised if the target directory already exists.
+
+ If *exist_ok* is true, :exc:`FileExistsError` exceptions will be
+ ignored (same behavior as the POSIX ``mkdir -p`` command), but only if the
+ last path component is not an existing non-directory file.
+
+ .. versionchanged:: 3.5
+ The *exist_ok* parameter was added.
+
.. method:: Path.open(mode='r', buffering=-1, encoding=None, errors=None, newline=None)
@@ -824,6 +857,34 @@ call fails (for example because the path doesn't exist):
if the file's uid isn't found in the system database.
+.. method:: Path.read_bytes()
+
+ Return the binary contents of the pointed-to file as a bytes object::
+
+ >>> p = Path('my_binary_file')
+ >>> p.write_bytes(b'Binary file contents')
+ 20
+ >>> p.read_bytes()
+ b'Binary file contents'
+
+ .. versionadded:: 3.5
+
+
+.. method:: Path.read_text(encoding=None, errors=None)
+
+ Return the decoded contents of the pointed-to file as a string::
+
+ >>> p = Path('my_text_file')
+ >>> p.write_text('Text file contents')
+ 18
+ >>> p.read_text()
+ 'Text file contents'
+
+ The optional parameters have the same meaning as in :func:`open`.
+
+ .. versionadded:: 3.5
+
+
.. method:: Path.rename(target)
Rename this file or directory to the given *target*. *target* can be
@@ -884,6 +945,25 @@ call fails (for example because the path doesn't exist):
Remove this directory. The directory must be empty.
+.. method:: Path.samefile(other_path)
+
+ Return whether this path points to the same file as *other_path*, which
+ can be either a Path object, or a string. The semantics are similar
+ to :func:`os.path.samefile` and :func:`os.path.samestat`.
+
+ An :exc:`OSError` can be raised if either file cannot be accessed for some
+ reason.
+
+ >>> p = Path('spam')
+ >>> q = Path('eggs')
+ >>> p.samefile(q)
+ False
+ >>> p.samefile('spam')
+ True
+
+ .. versionadded:: 3.5
+
+
.. method:: Path.symlink_to(target, target_is_directory=False)
Make this path a symbolic link to *target*. Under Windows,
@@ -917,3 +997,33 @@ call fails (for example because the path doesn't exist):
Remove this file or symbolic link. If the path points to a directory,
use :func:`Path.rmdir` instead.
+
+
+.. method:: Path.write_bytes(data)
+
+ Open the file pointed to in bytes mode, write *data* to it, and close the
+ file::
+
+ >>> p = Path('my_binary_file')
+ >>> p.write_bytes(b'Binary file contents')
+ 20
+ >>> p.read_bytes()
+ b'Binary file contents'
+
+ An existing file of the same name is overwritten.
+
+ .. versionadded:: 3.5
+
+
+.. method:: Path.write_text(data, encoding=None, errors=None)
+
+ Open the file pointed to in text mode, write *data* to it, and close the
+ file::
+
+ >>> p = Path('my_text_file')
+ >>> p.write_text('Text file contents')
+ 18
+ >>> p.read_text()
+ 'Text file contents'
+
+ .. versionadded:: 3.5
diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst
index 0f48cb1..a92526f 100644
--- a/Doc/library/pickle.rst
+++ b/Doc/library/pickle.rst
@@ -425,7 +425,7 @@ The following types can be pickled:
Attempts to pickle unpicklable objects will raise the :exc:`PicklingError`
exception; when this happens, an unspecified number of bytes may have already
been written to the underlying file. Trying to pickle a highly recursive data
-structure may exceed the maximum recursion depth, a :exc:`RuntimeError` will be
+structure may exceed the maximum recursion depth, a :exc:`RecursionError` will be
raised in this case. You can carefully raise this limit with
:func:`sys.setrecursionlimit`.
@@ -859,7 +859,7 @@ For the simplest code, use the :func:`dump` and :func:`load` functions. ::
data = {
'a': [1, 2.0, 3, 4+6j],
'b': ("character string", b"byte string"),
- 'c': set([None, True, False])
+ 'c': {None, True, False}
}
with open('data.pickle', 'wb') as f:
diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst
index 13ea7b9..5d3295d 100644
--- a/Doc/library/pkgutil.rst
+++ b/Doc/library/pkgutil.rst
@@ -58,7 +58,7 @@ support.
.. deprecated:: 3.3
This emulation is no longer needed, as the standard import mechanism
- is now fully PEP 302 compliant and available in :mod:`importlib`
+ is now fully PEP 302 compliant and available in :mod:`importlib`.
.. class:: ImpLoader(fullname, file, filename, etc)
@@ -67,7 +67,7 @@ support.
.. deprecated:: 3.3
This emulation is no longer needed, as the standard import mechanism
- is now fully PEP 302 compliant and available in :mod:`importlib`
+ is now fully PEP 302 compliant and available in :mod:`importlib`.
.. function:: find_loader(fullname)
diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst
index e27f2ad..679cc6b 100644
--- a/Doc/library/platform.rst
+++ b/Doc/library/platform.rst
@@ -247,6 +247,8 @@ Unix Platforms
This is another name for :func:`linux_distribution`.
+ .. deprecated-removed:: 3.5 3.7
+
.. function:: linux_distribution(distname='', version='', id='', supported_dists=('SuSE','debian','redhat','mandrake',...), full_distribution_name=1)
Tries to determine the name of the Linux OS distribution name.
@@ -263,6 +265,8 @@ Unix Platforms
parameters. ``id`` is the item in parentheses after the version number. It
is usually the version codename.
+ .. deprecated-removed:: 3.5 3.7
+
.. function:: libc_ver(executable=sys.executable, lib='', version='', chunksize=2048)
Tries to determine the libc version against which the file executable (defaults
diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst
index 45baad9..8468f4c 100644
--- a/Doc/library/poplib.rst
+++ b/Doc/library/poplib.rst
@@ -194,6 +194,15 @@ An :class:`POP3` instance has the following methods:
the unique id for that message in the form ``'response mesgnum uid``, otherwise
result is list ``(response, ['mesgnum uid', ...], octets)``.
+
+.. method:: POP3.utf8()
+
+ Try to switch to UTF-8 mode. Returns the server response if successful,
+ raises :class:`error_proto` if not. Specified in :RFC:`6856`.
+
+ .. versionadded:: 3.5
+
+
.. method:: POP3.stls(context=None)
Start a TLS session on the active connection as specified in :rfc:`2595`.
diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst
index c0589a3..0b44dc8 100644
--- a/Doc/library/pprint.rst
+++ b/Doc/library/pprint.rst
@@ -235,10 +235,10 @@ In its basic form, :func:`pprint` shows the whole object::
'classifiers': ['Programming Language :: Python :: 2.6',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 2 :: Only'],
- 'description': 'An extensible framework for Python programming, '
- 'with special focus\r\n'
- 'on event-based network programming and '
- 'multiprotocol integration.',
+ 'description': 'An extensible framework for Python programming, with '
+ 'special focus\r\n'
+ 'on event-based network programming and multiprotocol '
+ 'integration.',
'docs_url': '',
'download_url': 'UNKNOWN',
'home_page': 'http://twistedmatrix.com/',
@@ -288,10 +288,10 @@ contents)::
'cheesecake_documentation_id': None,
'cheesecake_installability_id': None,
'classifiers': [...],
- 'description': 'An extensible framework for Python programming, '
- 'with special focus\r\n'
- 'on event-based network programming and '
- 'multiprotocol integration.',
+ 'description': 'An extensible framework for Python programming, with '
+ 'special focus\r\n'
+ 'on event-based network programming and multiprotocol '
+ 'integration.',
'docs_url': '',
'download_url': 'UNKNOWN',
'home_page': 'http://twistedmatrix.com/',
@@ -323,13 +323,12 @@ cannot be split, the specified width will be exceeded::
'cheesecake_installability_id': None,
'classifiers': [...],
'description': 'An extensible '
- 'framework for '
- 'Python programming, '
- 'with special '
- 'focus\r\n'
- 'on event-based '
- 'network programming '
- 'and multiprotocol '
+ 'framework for Python '
+ 'programming, with '
+ 'special focus\r\n'
+ 'on event-based network '
+ 'programming and '
+ 'multiprotocol '
'integration.',
'docs_url': '',
'download_url': 'UNKNOWN',
@@ -344,8 +343,8 @@ cannot be split, the specified width will be exceeded::
'release_url': 'http://pypi.python.org/pypi/Twisted/12.3.0',
'requires_python': None,
'stable_version': None,
- 'summary': 'An asynchronous '
- 'networking framework '
- 'written in Python',
+ 'summary': 'An asynchronous networking '
+ 'framework written in '
+ 'Python',
'version': '12.3.0'},
'urls': [{...}, {...}]}
diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst
index bae8450..97f2b20 100644
--- a/Doc/library/py_compile.rst
+++ b/Doc/library/py_compile.rst
@@ -29,9 +29,9 @@ byte-code cache files in the directory containing the source code.
.. function:: compile(file, cfile=None, dfile=None, doraise=False, optimize=-1)
Compile a source file to byte-code and write out the byte-code cache file.
- The source code is loaded from the file name *file*. The byte-code is
- written to *cfile*, which defaults to the :PEP:`3147` path, ending in
- ``.pyc`` (``.pyo`` if optimization is enabled in the current interpreter).
+ The source code is loaded from the file name *file*. The byte-code is
+ written to *cfile*, which defaults to the :pep:`3147`/:pep:`488` path, ending
+ in ``.pyc``.
For example, if *file* is ``/foo/bar/baz.py`` *cfile* will default to
``/foo/bar/__pycache__/baz.cpython-32.pyc`` for Python 3.2. If *dfile* is
specified, it is used as the name of the source file in error messages when
@@ -68,7 +68,7 @@ byte-code cache files in the directory containing the source code.
.. function:: main(args=None)
Compile several source files. The files named in *args* (or on the command
- line, if *args* is ``None``) are compiled and the resulting bytecode is
+ line, if *args* is ``None``) are compiled and the resulting byte-code is
cached in the normal manner. This function does not search a directory
structure to locate source files; it only compiles files named explicitly.
If ``'-'`` is the only parameter in args, the list of files is taken from
@@ -86,4 +86,3 @@ could not be compiled.
Module :mod:`compileall`
Utilities to compile all Python source files in a directory tree.
-
diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst
index 680d690..1cb0935 100644
--- a/Doc/library/queue.rst
+++ b/Doc/library/queue.rst
@@ -158,22 +158,32 @@ fully processed by daemon consumer threads.
Example of how to wait for enqueued tasks to be completed::
- def worker():
- while True:
- item = q.get()
- do_work(item)
- q.task_done()
-
- q = Queue()
- for i in range(num_worker_threads):
- t = Thread(target=worker)
- t.daemon = True
+ def worker():
+ while True:
+ item = q.get()
+ if item is None:
+ break
+ do_work(item)
+ q.task_done()
+
+ q = queue.Queue()
+ threads = []
+ for i in range(num_worker_threads):
+ t = threading.Thread(target=worker)
t.start()
+ threads.append(t)
- for item in source():
- q.put(item)
+ for item in source():
+ q.put(item)
- q.join() # block until all tasks are done
+ # block until all tasks are done
+ q.join()
+
+ # stop workers
+ for i in range(num_worker_threads):
+ q.put(None)
+ for t in threads:
+ t.join()
.. seealso::
diff --git a/Doc/library/random.rst b/Doc/library/random.rst
index 11dd367..f8b7727 100644
--- a/Doc/library/random.rst
+++ b/Doc/library/random.rst
@@ -46,8 +46,7 @@ from sources provided by the operating system.
.. warning::
The pseudo-random generators of this module should not be used for
- security purposes. Use :func:`os.urandom` or :class:`SystemRandom` if
- you require a cryptographically secure pseudo-random number generator.
+ security purposes.
Bookkeeping functions:
diff --git a/Doc/library/re.rst b/Doc/library/re.rst
index c3c8b65..8884584 100644
--- a/Doc/library/re.rst
+++ b/Doc/library/re.rst
@@ -281,9 +281,7 @@ The special characters are:
assertion`. ``(?<=abc)def`` will find a match in ``abcdef``, since the
lookbehind will back up 3 characters and check if the contained pattern matches.
The contained pattern must only match strings of some fixed length, meaning that
- ``abc`` or ``a|b`` are allowed, but ``a*`` and ``a{3,4}`` are not. Group
- references are not supported even if they match strings of some fixed length.
- Note that
+ ``abc`` or ``a|b`` are allowed, but ``a*`` and ``a{3,4}`` are not. Note that
patterns which start with positive lookbehind assertions will not match at the
beginning of the string being searched; you will most likely want to use the
:func:`search` function rather than the :func:`match` function:
@@ -299,12 +297,14 @@ The special characters are:
>>> m.group(0)
'egg'
+ .. versionchanged: 3.5
+ Added support for group references of fixed length.
+
``(?<!...)``
Matches if the current position in the string is not preceded by a match for
``...``. This is called a :dfn:`negative lookbehind assertion`. Similar to
positive lookbehind assertions, the contained pattern must only match strings of
- some fixed length and shouldn't contain group references.
- Patterns which start with negative lookbehind assertions may
+ some fixed length. Patterns which start with negative lookbehind assertions may
match at the beginning of the string being searched.
``(?(id/name)yes-pattern|no-pattern)``
@@ -438,6 +438,10 @@ three digits in length.
.. versionchanged:: 3.3
The ``'\u'`` and ``'\U'`` escape sequences have been added.
+.. deprecated-removed:: 3.5 3.6
+ Unknown escapes consist of ``'\'`` and ASCII letter now raise a
+ deprecation warning and will be forbidden in Python 3.6.
+
.. seealso::
@@ -524,7 +528,11 @@ form.
current locale. The use of this flag is discouraged as the locale mechanism
is very unreliable, and it only handles one "culture" at a time anyway;
you should use Unicode matching instead, which is the default in Python 3
- for Unicode (str) patterns.
+ for Unicode (str) patterns. This flag makes sense only with bytes patterns.
+
+ .. deprecated-removed:: 3.5 3.6
+ Deprecated the use of :const:`re.LOCALE` with string patterns or
+ :const:`re.ASCII`.
.. data:: M
@@ -625,17 +633,37 @@ form.
That way, separator components are always found at the same relative
indices within the result list.
- Note that *split* will never split a string on an empty pattern match.
- For example:
+ .. note::
+
+ :func:`split` doesn't currently split a string on an empty pattern match.
+ For example:
- >>> re.split('x*', 'foo')
- ['foo']
- >>> re.split("(?m)^$", "foo\n\nbar\n")
- ['foo\n\nbar\n']
+ >>> re.split('x*', 'axbc')
+ ['a', 'bc']
+
+ Even though ``'x*'`` also matches 0 'x' before 'a', between 'b' and 'c',
+ and after 'c', currently these matches are ignored. The correct behavior
+ (i.e. splitting on empty matches too and returning ``['', 'a', 'b', 'c',
+ '']``) will be implemented in future versions of Python, but since this
+ is a backward incompatible change, a :exc:`FutureWarning` will be raised
+ in the meanwhile.
+
+ Patterns that can only match empty strings currently never split the
+ string. Since this doesn't match the expected behavior, a
+ :exc:`ValueError` will be raised starting from Python 3.5::
+
+ >>> re.split("^$", "foo\n\nbar\n", flags=re.M)
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ ...
+ ValueError: split() requires a non-empty pattern match.
.. versionchanged:: 3.1
Added the optional flags argument.
+ .. versionchanged:: 3.5
+ Splitting on a pattern that could match an empty string now raises
+ a warning. Patterns that can only match empty strings are now rejected.
.. function:: findall(pattern, string, flags=0)
@@ -663,7 +691,7 @@ form.
*string* is returned unchanged. *repl* can be a string or a function; if it is
a string, any backslash escapes in it are processed. That is, ``\n`` is
converted to a single newline character, ``\r`` is converted to a carriage return, and
- so forth. Unknown escapes such as ``\j`` are left alone. Backreferences, such
+ so forth. Unknown escapes such as ``\&`` are left alone. Backreferences, such
as ``\6``, are replaced with the substring matched by group 6 in the pattern.
For example:
@@ -705,6 +733,13 @@ form.
.. versionchanged:: 3.1
Added the optional flags argument.
+ .. versionchanged:: 3.5
+ Unmatched groups are replaced with an empty string.
+
+ .. deprecated-removed:: 3.5 3.6
+ Unknown escapes consist of ``'\'`` and ASCII letter now raise a
+ deprecation warning and will be forbidden in Python 3.6.
+
.. function:: subn(pattern, repl, string, count=0, flags=0)
@@ -714,6 +749,9 @@ form.
.. versionchanged:: 3.1
Added the optional flags argument.
+ .. versionchanged:: 3.5
+ Unmatched groups are replaced with an empty string.
+
.. function:: escape(string)
@@ -730,13 +768,36 @@ form.
Clear the regular expression cache.
-.. exception:: error
+.. exception:: error(msg, pattern=None, pos=None)
Exception raised when a string passed to one of the functions here is not a
valid regular expression (for example, it might contain unmatched parentheses)
or when some other error occurs during compilation or matching. It is never an
- error if a string contains no match for a pattern.
+ error if a string contains no match for a pattern. The error instance has
+ the following additional attributes:
+
+ .. attribute:: msg
+
+ The unformatted error message.
+
+ .. attribute:: pattern
+
+ The regular expression pattern.
+
+ .. attribute:: pos
+
+ The index of *pattern* where compilation failed.
+
+ .. attribute:: lineno
+
+ The line corresponding to *pos*.
+
+ .. attribute:: colno
+
+ The column corresponding to *pos*.
+ .. versionchanged:: 3.5
+ Added additional attributes.
.. _re-objects:
@@ -889,6 +950,8 @@ Match objects support the following methods and attributes:
(``\g<1>``, ``\g<name>``) are replaced by the contents of the
corresponding group.
+ .. versionchanged:: 3.5
+ Unmatched groups are replaced with an empty string.
.. method:: match.group([group1, ...])
diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst
index 692310b..3864f0d 100644
--- a/Doc/library/readline.rst
+++ b/Doc/library/readline.rst
@@ -59,6 +59,14 @@ The :mod:`readline` module defines the following functions:
Save a readline history file. The default filename is :file:`~/.history`.
+.. function:: append_history_file(nelements[, filename])
+
+ Append the last *nelements* of history to a file. The default filename is
+ :file:`~/.history`. The file must already exist.
+
+ .. versionadded:: 3.5
+
+
.. function:: clear_history()
Clear the current history. (Note: this function is not available if the
@@ -209,6 +217,26 @@ from the user's :envvar:`PYTHONSTARTUP` file. ::
This code is actually automatically run when Python is run in
:ref:`interactive mode <tut-interactive>` (see :ref:`rlcompleter-config`).
+The following example achieves the same goal but supports concurrent interactive
+sessions, by only appending the new history. ::
+
+ import atexit
+ import os
+ import realine
+ histfile = os.path.join(os.path.expanduser("~"), ".python_history")
+
+ try:
+ readline.read_history_file(histfile)
+ h_len = readline.get_history_length()
+ except FileNotFoundError:
+ open(histfile, 'wb').close()
+ h_len = 0
+
+ def save(prev_h_len, histfile):
+ new_h_len = readline.get_history_length()
+ readline.append_history_file(new_h_len - prev_h_len, histfile)
+ atexit.register(save, h_len, histfile)
+
The following example extends the :class:`code.InteractiveConsole` class to
support history save/restore. ::
@@ -234,4 +262,3 @@ support history save/restore. ::
def save_history(self, histfile):
readline.write_history_file(histfile)
-
diff --git a/Doc/library/select.rst b/Doc/library/select.rst
index 5334af8..a62dc84 100644
--- a/Doc/library/select.rst
+++ b/Doc/library/select.rst
@@ -145,6 +145,13 @@ The module defines the following:
library, and does not handle file descriptors that don't originate from
WinSock.
+ .. versionchanged:: 3.5
+ The function is now retried with a recomputed timeout when interrupted by
+ a signal, except if the signal handler raises an exception (see
+ :pep:`475` for the rationale), instead of raising
+ :exc:`InterruptedError`.
+
+
.. attribute:: PIPE_BUF
The minimum number of bytes which can be written without blocking to a pipe
@@ -242,6 +249,12 @@ object.
returning. If *timeout* is omitted, -1, or :const:`None`, the call will
block until there is an event for this poll object.
+ .. versionchanged:: 3.5
+ The function is now retried with a recomputed timeout when interrupted by
+ a signal, except if the signal handler raises an exception (see
+ :pep:`475` for the rationale), instead of raising
+ :exc:`InterruptedError`.
+
.. _epoll-objects:
@@ -322,6 +335,12 @@ Edge and Level Trigger Polling (epoll) Objects
Wait for events. timeout in seconds (float)
+ .. versionchanged:: 3.5
+ The function is now retried with a recomputed timeout when interrupted by
+ a signal, except if the signal handler raises an exception (see
+ :pep:`475` for the rationale), instead of raising
+ :exc:`InterruptedError`.
+
.. _poll-objects:
@@ -401,6 +420,12 @@ linearly scanned again. :c:func:`select` is O(highest file descriptor), while
returning. If *timeout* is omitted, negative, or :const:`None`, the call will
block until there is an event for this poll object.
+ .. versionchanged:: 3.5
+ The function is now retried with a recomputed timeout when interrupted by
+ a signal, except if the signal handler raises an exception (see
+ :pep:`475` for the rationale), instead of raising
+ :exc:`InterruptedError`.
+
.. _kqueue-objects:
@@ -435,6 +460,12 @@ Kqueue Objects
- max_events must be 0 or a positive integer
- timeout in seconds (floats possible)
+ .. versionchanged:: 3.5
+ The function is now retried with a recomputed timeout when interrupted by
+ a signal, except if the signal handler raises an exception (see
+ :pep:`475` for the rationale), instead of raising
+ :exc:`InterruptedError`.
+
.. _kevent-objects:
diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst
index 98377c8..f6ef24b 100644
--- a/Doc/library/selectors.rst
+++ b/Doc/library/selectors.rst
@@ -45,6 +45,7 @@ Classes hierarchy::
+-- SelectSelector
+-- PollSelector
+-- EpollSelector
+ +-- DevpollSelector
+-- KqueueSelector
@@ -158,6 +159,12 @@ below:
timeout has elapsed if the current process receives a signal: in this
case, an empty list will be returned.
+ .. versionchanged:: 3.5
+ The selector is now retried with a recomputed timeout when interrupted
+ by a signal if the signal handler did not raise an exception (see
+ :pep:`475` for the rationale), instead of returning an empty list
+ of events before the timeout.
+
.. method:: close()
Close the selector.
@@ -207,6 +214,16 @@ below:
This returns the file descriptor used by the underlying
:func:`select.epoll` object.
+.. class:: DevpollSelector()
+
+ :func:`select.devpoll`-based selector.
+
+ .. method:: fileno()
+
+ This returns the file descriptor used by the underlying
+ :func:`select.devpoll` object.
+
+ .. versionadded:: 3.5
.. class:: KqueueSelector()
diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst
index 04afe92..3b467e0 100644
--- a/Doc/library/shutil.rst
+++ b/Doc/library/shutil.rst
@@ -191,7 +191,8 @@ Directory and files operations
match one of the glob-style *patterns* provided. See the example below.
-.. function:: copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, ignore_dangling_symlinks=False)
+.. function:: copytree(src, dst, symlinks=False, ignore=None, \
+ copy_function=copy2, ignore_dangling_symlinks=False)
Recursively copy an entire directory tree rooted at *src*, returning the
destination directory. The destination
@@ -282,7 +283,7 @@ Directory and files operations
.. versionadded:: 3.3
-.. function:: move(src, dst)
+.. function:: move(src, dst, copy_function=copy2)
Recursively move a file or directory (*src*) to another location (*dst*)
and return the destination.
@@ -292,15 +293,26 @@ Directory and files operations
be overwritten depending on :func:`os.rename` semantics.
If the destination is on the current filesystem, then :func:`os.rename` is
- used. Otherwise, *src* is copied (using :func:`shutil.copy2`) to *dst* and
- then removed. In case of symlinks, a new symlink pointing to the target of
- *src* will be created in or as *dst* and *src* will be removed.
+ used. Otherwise, *src* is copied to *dst* using *copy_function* and then
+ removed. In case of symlinks, a new symlink pointing to the target of *src*
+ will be created in or as *dst* and *src* will be removed.
+
+ If *copy_function* is given, it must be a callable that takes two arguments
+ *src* and *dst*, and will be used to copy *src* to *dest* if
+ :func:`os.rename` cannot be used. If the source is a directory,
+ :func:`copytree` is called, passing it the :func:`copy_function`. The
+ default *copy_function* is :func:`copy2`. Using :func:`copy` as the
+ *copy_function* allows the move to succeed when it is not possible to also
+ copy the metadata, at the expense of not copying any of the metadata.
.. versionchanged:: 3.3
Added explicit symlink handling for foreign filesystems, thus adapting
it to the behavior of GNU's :program:`mv`.
Now returns *dst*.
+ .. versionchanged:: 3.5
+ Added the *copy_function* keyword argument.
+
.. function:: disk_usage(path)
Return disk usage statistics about the given path as a :term:`named tuple`
@@ -418,6 +430,26 @@ Another example that uses the *ignore* argument to add a logging call::
copytree(source, destination, ignore=_logpath)
+.. _shutil-rmtree-example:
+
+rmtree example
+~~~~~~~~~~~~~~
+
+This example shows how to remove a directory tree on Windows where some
+of the files have their read-only bit set. It uses the onerror callback
+to clear the readonly bit and reattempt the remove. Any subsequent failure
+will propagate. ::
+
+ import os, stat
+ import shutil
+
+ def remove_readonly(func, path, _):
+ "Clear the readonly bit and reattempt the removal"
+ os.chmod(path, stat.S_IWRITE)
+ func(path)
+
+ shutil.rmtree(directory, onerror=remove_readonly)
+
.. _archiving-operations:
Archiving operations
@@ -434,7 +466,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules.
*base_name* is the name of the file to create, including the path, minus
any format-specific extension. *format* is the archive format: one of
- "zip", "tar", "bztar" (if the :mod:`bz2` module is available) or "gztar".
+ "zip", "tar", "bztar" (if the :mod:`bz2` module is available), "xztar"
+ (if the :mod:`lzma` module is available) or "gztar".
*root_dir* is a directory that will be the root directory of the
archive; for example, we typically chdir into *root_dir* before creating the
@@ -457,6 +490,9 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules.
The *verbose* argument is unused and deprecated.
+ .. versionchanged:: 3.5
+ Added support for the *xztar* format.
+
.. function:: get_archive_formats()
@@ -467,6 +503,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules.
- *gztar*: gzip'ed tar-file
- *bztar*: bzip2'ed tar-file (if the :mod:`bz2` module is available.)
+ - *xztar*: xz'ed tar-file (if the :mod:`lzma` module is available.)
- *tar*: uncompressed tar file
- *zip*: ZIP file
@@ -542,6 +579,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules.
- *gztar*: gzip'ed tar-file
- *bztar*: bzip2'ed tar-file (if the :mod:`bz2` module is available.)
+ - *xztar*: xz'ed tar-file (if the :mod:`lzma` module is available.)
- *tar*: uncompressed tar file
- *zip*: ZIP file
diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst
index 84e2836..9549838 100644
--- a/Doc/library/signal.rst
+++ b/Doc/library/signal.rst
@@ -65,6 +65,16 @@ Besides, only the main thread is allowed to set a new signal handler.
Module contents
---------------
+.. versionchanged:: 3.5
+ signal (SIG*), handler (:const:`SIG_DFL`, :const:`SIG_IGN`) and sigmask
+ (:const:`SIG_BLOCK`, :const:`SIG_UNBLOCK`, :const:`SIG_SETMASK`)
+ related constants listed below were turned into
+ :class:`enums <enum.IntEnum>`.
+ :func:`getsignal`, :func:`pthread_sigmask`, :func:`sigpending` and
+ :func:`sigwait` functions return human-readable
+ :class:`enums <enum.IntEnum>`.
+
+
The variables defined in the :mod:`signal` module are:
@@ -209,21 +219,21 @@ The :mod:`signal` module defines the following functions:
:func:`sigpending`.
-.. function:: pthread_kill(thread_id, signum)
+.. function:: pthread_kill(thread_id, signalnum)
- Send the signal *signum* to the thread *thread_id*, another thread in the
+ Send the signal *signalnum* to the thread *thread_id*, another thread in the
same process as the caller. The target thread can be executing any code
(Python or not). However, if the target thread is executing the Python
interpreter, the Python signal handlers will be :ref:`executed by the main
- thread <signals-and-threads>`. Therefore, the only point of sending a signal to a particular
- Python thread would be to force a running system call to fail with
- :exc:`InterruptedError`.
+ thread <signals-and-threads>`. Therefore, the only point of sending a
+ signal to a particular Python thread would be to force a running system call
+ to fail with :exc:`InterruptedError`.
Use :func:`threading.get_ident()` or the :attr:`~threading.Thread.ident`
attribute of :class:`threading.Thread` objects to get a suitable value
for *thread_id*.
- If *signum* is 0, then no signal is sent, but error checking is still
+ If *signalnum* is 0, then no signal is sent, but error checking is still
performed; this can be used to check if the target thread is still running.
Availability: Unix (see the man page :manpage:`pthread_kill(3)` for further
@@ -308,6 +318,9 @@ The :mod:`signal` module defines the following functions:
attempting to call it from other threads will cause a :exc:`ValueError`
exception to be raised.
+ .. versionchanged:: 3.5
+ On Windows, the function now also supports socket handles.
+
.. function:: siginterrupt(signalnum, flag)
@@ -395,6 +408,11 @@ The :mod:`signal` module defines the following functions:
.. versionadded:: 3.3
+ .. versionchanged:: 3.5
+ The function is now retried if interrupted by a signal not in *sigset*
+ and the signal handler does not raise an exception (see :pep:`475` for
+ the rationale).
+
.. function:: sigtimedwait(sigset, timeout)
@@ -409,6 +427,11 @@ The :mod:`signal` module defines the following functions:
.. versionadded:: 3.3
+ .. versionchanged:: 3.5
+ The function is now retried with the recomputed *timeout* if interrupted
+ by a signal not in *sigset* and the signal handler does not raise an
+ exception (see :pep:`475` for the rationale).
+
.. _signal-example:
diff --git a/Doc/library/site.rst b/Doc/library/site.rst
index 51e5da8..43daf79 100644
--- a/Doc/library/site.rst
+++ b/Doc/library/site.rst
@@ -26,24 +26,23 @@ additions, call the :func:`site.main` function.
:option:`-S`.
.. index::
- pair: site-python; directory
pair: site-packages; directory
It starts by constructing up to four directories from a head and a tail part.
For the head part, it uses ``sys.prefix`` and ``sys.exec_prefix``; empty heads
are skipped. For the tail part, it uses the empty string and then
:file:`lib/site-packages` (on Windows) or
-:file:`lib/python{X.Y}/site-packages` and then :file:`lib/site-python` (on
-Unix and Macintosh). For each of the distinct head-tail combinations, it sees
-if it refers to an existing directory, and if so, adds it to ``sys.path`` and
-also inspects the newly added path for configuration files.
+:file:`lib/python{X.Y}/site-packages` (on Unix and Macintosh). For each
+of the distinct head-tail combinations, it sees if it refers to an existing
+directory, and if so, adds it to ``sys.path`` and also inspects the newly
+added path for configuration files.
-.. deprecated:: 3.4
- Support for the "site-python" directory will be removed in 3.5.
+.. versionchanged:: 3.5
+ Support for the "site-python" directory has been removed.
If a file named "pyvenv.cfg" exists one directory above sys.executable,
sys.prefix and sys.exec_prefix are set to that directory and
-it is also checked for site-packages and site-python (sys.base_prefix and
+it is also checked for site-packages (sys.base_prefix and
sys.base_exec_prefix will always be the "real" prefixes of the Python
installation). If "pyvenv.cfg" (a bootstrap configuration file) contains
the key "include-system-site-packages" set to anything other than "false"
@@ -195,8 +194,7 @@ Module contents
.. function:: getsitepackages()
- Return a list containing all global site-packages directories (and possibly
- site-python).
+ Return a list containing all global site-packages directories.
.. versionadded:: 3.2
diff --git a/Doc/library/smtpd.rst b/Doc/library/smtpd.rst
index 3ebed06..977f9a8 100644
--- a/Doc/library/smtpd.rst
+++ b/Doc/library/smtpd.rst
@@ -20,7 +20,8 @@ specific mail-sending strategies.
Additionally the SMTPChannel may be extended to implement very specific
interaction behaviour with SMTP clients.
-The code supports :RFC:`5321`, plus the :rfc:`1870` SIZE extension.
+The code supports :RFC:`5321`, plus the :rfc:`1870` SIZE and :rfc:`6531`
+SMTPUTF8 extensions.
SMTPServer Objects
@@ -28,7 +29,7 @@ SMTPServer Objects
.. class:: SMTPServer(localaddr, remoteaddr, data_size_limit=33554432,\
- map=None)
+ map=None, enable_SMTPUTF8=False, decode_data=True)
Create a new :class:`SMTPServer` object, which binds to local address
*localaddr*. It will treat *remoteaddr* as an upstream SMTP relayer. It
@@ -39,25 +40,77 @@ SMTPServer Objects
accepted in a ``DATA`` command. A value of ``None`` or ``0`` means no
limit.
- A dictionary can be specified in *map* to avoid using a global socket map.
-
- .. method:: process_message(peer, mailfrom, rcpttos, data)
-
- Raise :exc:`NotImplementedError` exception. Override this in subclasses to
+ *map* is the socket map to use for connections (an initially empty
+ dictionary is a suitable value). If not specified the :mod:`asyncore`
+ global socket map is used.
+
+ *enable_SMTPUTF8* determins whether the ``SMTPUTF8`` extension (as defined
+ in :RFC:`6531`) should be enabled. The default is ``False``. If set to
+ ``True``, *decode_data* must be ``False`` (otherwise an error is raised).
+ When ``True``, ``SMTPUTF8`` is accepted as a parameter to the ``MAIL``
+ command and when present is passed to :meth:`process_message` in the
+ ``kwargs['mail_options']`` list.
+
+ *decode_data* specifies whether the data portion of the SMTP transaction
+ should be decoded using UTF-8. The default is ``True`` for backward
+ compatibility reasons, but will change to ``False`` in Python 3.6; specify
+ the keyword value explicitly to avoid the :exc:`DeprecationWarning`. When
+ *decode_data* is set to ``False`` the server advertises the ``8BITMIME``
+ extension (:rfc:`6152`), accepts the ``BODY=8BITMIME`` parameter to
+ the ``MAIL`` command, and when present passes it to :meth:`process_message`
+ in the ``kwargs['mail_options']`` list.
+
+ .. method:: process_message(peer, mailfrom, rcpttos, data, **kwargs)
+
+ Raise a :exc:`NotImplementedError` exception. Override this in subclasses to
do something useful with this message. Whatever was passed in the
constructor as *remoteaddr* will be available as the :attr:`_remoteaddr`
attribute. *peer* is the remote host's address, *mailfrom* is the envelope
originator, *rcpttos* are the envelope recipients and *data* is a string
- containing the contents of the e-mail (which should be in :rfc:`2822`
+ containing the contents of the e-mail (which should be in :rfc:`5321`
format).
+ If the *decode_data* constructor keyword is set to ``True``, the *data*
+ argument will be a unicode string. If it is set to ``False``, it
+ will be a bytes object.
+
+ *kwargs* is a dictionary containing additional information. It is empty
+ unless at least one of ``decode_data=False`` or ``enable_SMTPUTF8=True``
+ was given as an init parameter, in which case it contains the following
+ keys:
+
+ *mail_options*:
+ a list of all received parameters to the ``MAIL``
+ command (the elements are uppercase strings; example:
+ ``['BODY=8BITMIME', 'SMTPUTF8']``).
+
+ *rcpt_options*:
+ same as *mail_options* but for the ``RCPT`` command.
+ Currently no ``RCPT TO`` options are supported, so for now
+ this will always be an empty list.
+
+ Implementations of ``process_message`` should use the ``**kwargs``
+ signature to accept arbitrary keyword arguments, since future feature
+ enhancements may add keys to the kwargs dictionary.
+
+ Return ``None`` to request a normal ``250 Ok`` response; otherwise
+ return the desired response string in :RFC:`5321` format.
+
.. attribute:: channel_class
Override this in subclasses to use a custom :class:`SMTPChannel` for
managing SMTP clients.
- .. versionchanged:: 3.4
- The *map* argument was added.
+ .. versionadded:: 3.4
+ The *map* constructor argument.
+
+ .. versionchanged:: 3.5
+ *localaddr* and *remoteaddr* may now contain IPv6 addresses.
+
+ .. versionadded:: 3.5
+ the *decode_data* and *enable_SMTPUTF8* constructor arguments, and the
+ *kwargs* argument to :meth:`process_message` when one or more of these is
+ specified.
DebuggingServer Objects
@@ -97,7 +150,7 @@ SMTPChannel Objects
-------------------
.. class:: SMTPChannel(server, conn, addr, data_size_limit=33554432,\
- map=None))
+ map=None, enable_SMTPUTF8=False, decode_data=True)
Create a new :class:`SMTPChannel` object which manages the communication
between the server and a single SMTP client.
@@ -108,11 +161,24 @@ SMTPChannel Objects
accepted in a ``DATA`` command. A value of ``None`` or ``0`` means no
limit.
+ *enable_SMTPUTF8* determins whether the ``SMTPUTF8`` extension (as defined
+ in :RFC:`6531`) should be enabled. The default is ``False``. A
+ :exc:`ValueError` is raised if both *enable_SMTPUTF8* and *decode_data* are
+ set to ``True`` at the same time.
+
A dictionary can be specified in *map* to avoid using a global socket map.
+ *decode_data* specifies whether the data portion of the SMTP transaction
+ should be decoded using UTF-8. The default is ``True`` for backward
+ compatibility reasons, but will change to ``False`` in Python 3.6. Specify
+ the keyword value explicitly to avoid the :exc:`DeprecationWarning`.
+
To use a custom SMTPChannel implementation you need to override the
:attr:`SMTPServer.channel_class` of your :class:`SMTPServer`.
+ .. versionchanged:: 3.5
+ the *decode_data* and *enable_SMTPUTF8* arguments were added.
+
The :class:`SMTPChannel` has the following instance variables:
.. attribute:: smtp_server
diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst
index 8e1bfb5..a71ee58 100644
--- a/Doc/library/smtplib.rst
+++ b/Doc/library/smtplib.rst
@@ -61,6 +61,10 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions).
.. versionchanged:: 3.3
source_address argument was added.
+ .. versionadded:: 3.5
+ The SMTPUTF8 extension (:rfc:`6531`) is now supported.
+
+
.. class:: SMTP_SSL(host='', port=0, local_hostname=None, keyfile=None, \
certfile=None [, timeout], context=None, \
source_address=None)
@@ -161,6 +165,13 @@ A nice selection of exceptions is defined as well:
The server refused our ``HELO`` message.
+.. exception:: SMTPNotSupportedError
+
+ The command or option attempted is not supported by the server.
+
+ .. versionadded:: 3.5
+
+
.. exception:: SMTPAuthenticationError
SMTP authentication went wrong. Most probably the server didn't accept the
@@ -189,8 +200,12 @@ An :class:`SMTP` instance has the following methods:
.. method:: SMTP.set_debuglevel(level)
- Set the debug output level. A true value for *level* results in debug messages
- for connection and for all messages sent to and received from the server.
+ Set the debug output level. A value of 1 or ``True`` for *level* results in
+ debug messages for connection and for all messages sent to and received from
+ the server. A value of 2 for *level* results in these messages being
+ timestamped.
+
+ .. versionchanged:: 3.5 Added debuglevel 2.
.. method:: SMTP.docmd(cmd, args='')
@@ -240,8 +255,7 @@ An :class:`SMTP` instance has the following methods:
the server is stored as the :attr:`ehlo_resp` attribute, :attr:`does_esmtp`
is set to true or false depending on whether the server supports ESMTP, and
:attr:`esmtp_features` will be a dictionary containing the names of the
- SMTP service extensions this server supports, and their
- parameters (if any).
+ SMTP service extensions this server supports, and their parameters (if any).
Unless you wish to use :meth:`has_extn` before sending mail, it should not be
necessary to call this method explicitly. It will be implicitly called by
@@ -274,7 +288,7 @@ An :class:`SMTP` instance has the following methods:
Many sites disable SMTP ``VRFY`` in order to foil spammers.
-.. method:: SMTP.login(user, password)
+.. method:: SMTP.login(user, password, *, initial_response_ok=True)
Log in on an SMTP server that requires authentication. The arguments are the
username and the password to authenticate with. If there has been no previous
@@ -288,9 +302,68 @@ An :class:`SMTP` instance has the following methods:
:exc:`SMTPAuthenticationError`
The server didn't accept the username/password combination.
+ :exc:`SMTPNotSupportedError`
+ The ``AUTH`` command is not supported by the server.
+
:exc:`SMTPException`
No suitable authentication method was found.
+ Each of the authentication methods supported by :mod:`smtplib` are tried in
+ turn if they are advertised as supported by the server. See :meth:`auth`
+ for a list of supported authentication methods. *initial_response_ok* is
+ passed through to :meth:`auth`.
+
+ Optional keyword argument *initial_response_ok* specifies whether, for
+ authentication methods that support it, an "initial response" as specified
+ in :rfc:`4954` can be sent along with the ``AUTH`` command, rather than
+ requiring a challenge/response.
+
+ .. versionchanged:: 3.5
+ :exc:`SMTPNotSupportedError` may be raised, and the
+ *initial_response_ok* parameter was added.
+
+
+.. method:: SMTP.auth(mechanism, authobject, *, initial_response_ok=True)
+
+ Issue an ``SMTP`` ``AUTH`` command for the specified authentication
+ *mechanism*, and handle the challenge response via *authobject*.
+
+ *mechanism* specifies which authentication mechanism is to
+ be used as argument to the ``AUTH`` command; the valid values are
+ those listed in the ``auth`` element of :attr:`esmtp_features`.
+
+ *authobject* must be a callable object taking an optional single argument:
+
+ data = authobject(challenge=None)
+
+ If optional keyword argument *initial_response_ok* is true,
+ ``authobject()`` will be called first with no argument. It can return the
+ :rfc:`4954` "initial response" bytes which will be encoded and sent with
+ the ``AUTH`` command as below. If the ``authobject()`` does not support an
+ initial response (e.g. because it requires a challenge), it should return
+ None when called with ``challenge=None``. If *initial_response_ok* is
+ false, then ``authobject()`` will not be called first with None.
+
+ If the initial response check returns None, or if *initial_response_ok* is
+ false, ``authobject()`` will be called to process the server's challenge
+ response; the *challenge* argument it is passed will be a ``bytes``. It
+ should return ``bytes`` *data* that will be base64 encoded and sent to the
+ server.
+
+ The ``SMTP`` class provides ``authobjects`` for the ``CRAM-MD5``, ``PLAIN``,
+ and ``LOGIN`` mechanisms; they are named ``SMTP.auth_cram_md5``,
+ ``SMTP.auth_plain``, and ``SMTP.auth_login`` respectively. They all require
+ that the ``user`` and ``password`` properties of the ``SMTP`` instance are
+ set to appropriate values.
+
+ User code does not normally need to call ``auth`` directly, but can instead
+ call the :meth:`login` method, which will try each of the above mechanisms
+ in turn, in the order listed. ``auth`` is exposed to facilitate the
+ implementation of authentication methods not (or not yet) supported
+ directly by :mod:`smtplib`.
+
+ .. versionadded:: 3.5
+
.. method:: SMTP.starttls(keyfile=None, certfile=None, context=None)
@@ -310,7 +383,7 @@ An :class:`SMTP` instance has the following methods:
:exc:`SMTPHeloError`
The server didn't reply properly to the ``HELO`` greeting.
- :exc:`SMTPException`
+ :exc:`SMTPNotSupportedError`
The server does not support the STARTTLS extension.
:exc:`RuntimeError`
@@ -324,6 +397,11 @@ An :class:`SMTP` instance has the following methods:
:attr:`SSLContext.check_hostname` and *Server Name Indicator* (see
:data:`~ssl.HAS_SNI`).
+ .. versionchanged:: 3.5
+ The error raised for lack of STARTTLS support is now the
+ :exc:`SMTPNotSupportedError` subclass instead of the base
+ :exc:`SMTPException`.
+
.. method:: SMTP.sendmail(from_addr, to_addrs, msg, mail_options=[], rcpt_options=[])
@@ -360,6 +438,9 @@ An :class:`SMTP` instance has the following methods:
recipient that was refused. Each entry contains a tuple of the SMTP error code
and the accompanying error message sent by the server.
+ If ``SMTPUTF8`` is included in *mail_options*, and the server supports it,
+ *from_addr* and *to_addr* may contain non-ASCII characters.
+
This method may raise the following exceptions:
:exc:`SMTPRecipientsRefused`
@@ -378,12 +459,20 @@ An :class:`SMTP` instance has the following methods:
The server replied with an unexpected error code (other than a refusal of a
recipient).
+ :exc:`SMTPNotSupportedError`
+ ``SMTPUTF8`` was given in the *mail_options* but is not supported by the
+ server.
+
Unless otherwise noted, the connection will be open even after an exception is
raised.
.. versionchanged:: 3.2
*msg* may be a byte string.
+ .. versionchanged:: 3.5
+ ``SMTPUTF8`` support added, and :exc:`SMTPNotSupportedError` may be
+ raised if ``SMTPUTF8`` is specified but the server does not support it.
+
.. method:: SMTP.send_message(msg, from_addr=None, to_addrs=None, \
mail_options=[], rcpt_options=[])
@@ -395,7 +484,7 @@ An :class:`SMTP` instance has the following methods:
If *from_addr* is ``None`` or *to_addrs* is ``None``, ``send_message`` fills
those arguments with addresses extracted from the headers of *msg* as
- specified in :rfc:`2822`\: *from_addr* is set to the :mailheader:`Sender`
+ specified in :rfc:`5322`\: *from_addr* is set to the :mailheader:`Sender`
field if it is present, and otherwise to the :mailheader:`From` field.
*to_adresses* combines the values (if any) of the :mailheader:`To`,
:mailheader:`Cc`, and :mailheader:`Bcc` fields from *msg*. If exactly one
@@ -410,10 +499,18 @@ An :class:`SMTP` instance has the following methods:
calls :meth:`sendmail` to transmit the resulting message. Regardless of the
values of *from_addr* and *to_addrs*, ``send_message`` does not transmit any
:mailheader:`Bcc` or :mailheader:`Resent-Bcc` headers that may appear
- in *msg*.
+ in *msg*. If any of the addresses in *from_addr* and *to_addrs* contain
+ non-ASCII characters and the server does not advertise ``SMTPUTF8`` support,
+ an :exc:`SMTPNotSupported` error is raised. Otherwise the ``Message`` is
+ serialized with a clone of its :mod:`~email.policy` with the
+ :attr:`~email.policy.EmailPolicy.utf8` attribute set to ``True``, and
+ ``SMTPUTF8`` and ``BODY=8BITMIME`` are added to *mail_options*.
.. versionadded:: 3.2
+ .. versionadded:: 3.5
+ Support for internationalized addresses (``SMTPUTF8``).
+
.. method:: SMTP.quit()
diff --git a/Doc/library/sndhdr.rst b/Doc/library/sndhdr.rst
index f36df68..f8b5d8b 100644
--- a/Doc/library/sndhdr.rst
+++ b/Doc/library/sndhdr.rst
@@ -16,8 +16,9 @@
The :mod:`sndhdr` provides utility functions which attempt to determine the type
of sound data which is in a file. When these functions are able to determine
-what type of sound data is stored in a file, they return a tuple ``(type,
-sampling_rate, channels, frames, bits_per_sample)``. The value for *type*
+what type of sound data is stored in a file, they return a
+:func:`~collections.namedtuple`, containing five attributes: (``filetype``,
+``framerate``, ``nchannels``, ``nframes``, ``sampwidth``). The value for *type*
indicates the data type and will be one of the strings ``'aifc'``, ``'aiff'``,
``'au'``, ``'hcom'``, ``'sndr'``, ``'sndt'``, ``'voc'``, ``'wav'``, ``'8svx'``,
``'sb'``, ``'ub'``, or ``'ul'``. The *sampling_rate* will be either the actual
@@ -31,13 +32,19 @@ be the sample size in bits or ``'A'`` for A-LAW or ``'U'`` for u-LAW.
.. function:: what(filename)
Determines the type of sound data stored in the file *filename* using
- :func:`whathdr`. If it succeeds, returns a tuple as described above, otherwise
+ :func:`whathdr`. If it succeeds, returns a namedtuple as described above, otherwise
``None`` is returned.
+ .. versionchanged:: 3.5
+ Result changed from a tuple to a namedtuple.
+
.. function:: whathdr(filename)
Determines the type of sound data stored in a file based on the file header.
- The name of the file is given by *filename*. This function returns a tuple as
+ The name of the file is given by *filename*. This function returns a namedtuple as
described above on success, or ``None``.
+ .. versionchanged:: 3.5
+ Result changed from a tuple to a namedtuple.
+
diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst
index 827191e..1dcdb2f 100644
--- a/Doc/library/socket.rst
+++ b/Doc/library/socket.rst
@@ -46,17 +46,20 @@ created. Socket addresses are represented as follows:
- The address of an :const:`AF_UNIX` socket bound to a file system node
is represented as a string, using the file system encoding and the
``'surrogateescape'`` error handler (see :pep:`383`). An address in
- Linux's abstract namespace is returned as a :class:`bytes` object with
+ Linux's abstract namespace is returned as a :term:`bytes-like object` with
an initial null byte; note that sockets in this namespace can
communicate with normal file system sockets, so programs intended to
run on Linux may need to deal with both types of address. A string or
- :class:`bytes` object can be used for either type of address when
+ bytes-like object can be used for either type of address when
passing it as an argument.
.. versionchanged:: 3.3
Previously, :const:`AF_UNIX` socket paths were assumed to use UTF-8
encoding.
+ .. versionchanged: 3.5
+ Writable :term:`bytes-like object` is now accepted.
+
- A pair ``(host, port)`` is used for the :const:`AF_INET` address family,
where *host* is a string representing either a hostname in Internet domain
notation like ``'daring.cwi.nl'`` or an IPv4 address like ``'100.50.200.5'``,
@@ -276,6 +279,18 @@ Constants
.. versionadded:: 3.4
+.. data:: CAN_RAW_FD_FRAMES
+
+ Enables CAN FD support in a CAN_RAW socket. This is disabled by default.
+ This allows your application to send both CAN and CAN FD frames; however,
+ you one must accept both CAN and CAN FD frames when reading from the socket.
+
+ This constant is documented in the Linux documentation.
+
+ Availability: Linux >= 3.6.
+
+ .. versionadded:: 3.5
+
.. data:: AF_RDS
PF_RDS
SOL_RDS
@@ -352,7 +367,6 @@ The following functions all create :ref:`socket objects <socket-objects>`.
type, and protocol number. Address family, socket type, and protocol number are
as for the :func:`.socket` function above. The default family is :const:`AF_UNIX`
if defined on the platform; otherwise, the default is :const:`AF_INET`.
- Availability: Unix.
The newly created sockets are :ref:`non-inheritable <fd_inheritance>`.
@@ -363,6 +377,9 @@ The following functions all create :ref:`socket objects <socket-objects>`.
.. versionchanged:: 3.4
The returned sockets are now non-inheritable.
+ .. versionchanged:: 3.5
+ Windows support added.
+
.. function:: create_connection(address[, timeout[, source_address]])
@@ -609,8 +626,8 @@ The :mod:`socket` module also offers various network-related services:
.. function:: inet_ntoa(packed_ip)
- Convert a 32-bit packed IPv4 address (a bytes object four characters in
- length) to its standard dotted-quad string representation (for example,
+ Convert a 32-bit packed IPv4 address (a :term:`bytes-like object` four
+ bytes in length) to its standard dotted-quad string representation (for example,
'123.45.67.89'). This is useful when conversing with a program that uses the
standard C library and needs objects of type :c:type:`struct in_addr`, which
is the C type for the 32-bit packed binary data this function takes as an
@@ -621,6 +638,9 @@ The :mod:`socket` module also offers various network-related services:
support IPv6, and :func:`inet_ntop` should be used instead for IPv4/v6 dual
stack support.
+ .. versionchanged: 3.5
+ Writable :term:`bytes-like object` is now accepted.
+
.. function:: inet_pton(address_family, ip_string)
@@ -643,22 +663,26 @@ The :mod:`socket` module also offers various network-related services:
.. function:: inet_ntop(address_family, packed_ip)
- Convert a packed IP address (a bytes object of some number of characters) to its
- standard, family-specific string representation (for example, ``'7.10.0.5'`` or
- ``'5aef:2b::8'``). :func:`inet_ntop` is useful when a library or network protocol
- returns an object of type :c:type:`struct in_addr` (similar to :func:`inet_ntoa`)
- or :c:type:`struct in6_addr`.
+ Convert a packed IP address (a :term:`bytes-like object` of some number of
+ bytes) to its standard, family-specific string representation (for
+ example, ``'7.10.0.5'`` or ``'5aef:2b::8'``).
+ :func:`inet_ntop` is useful when a library or network protocol returns an
+ object of type :c:type:`struct in_addr` (similar to :func:`inet_ntoa`) or
+ :c:type:`struct in6_addr`.
Supported values for *address_family* are currently :const:`AF_INET` and
- :const:`AF_INET6`. If the string *packed_ip* is not the correct length for the
- specified address family, :exc:`ValueError` will be raised. A
- :exc:`OSError` is raised for errors from the call to :func:`inet_ntop`.
+ :const:`AF_INET6`. If the bytes object *packed_ip* is not the correct
+ length for the specified address family, :exc:`ValueError` will be raised.
+ A :exc:`OSError` is raised for errors from the call to :func:`inet_ntop`.
Availability: Unix (maybe not all platforms), Windows.
.. versionchanged:: 3.4
Windows support added
+ .. versionchanged: 3.5
+ Writable :term:`bytes-like object` is now accepted.
+
..
XXX: Are sendmsg(), recvmsg() and CMSG_*() available on any
@@ -783,6 +807,11 @@ to sockets.
.. versionchanged:: 3.4
The socket is now non-inheritable.
+ .. versionchanged:: 3.5
+ If the system call is interrupted and the signal handler does not raise
+ an exception, the method now retries the system call instead of raising
+ an :exc:`InterruptedError` exception (see :pep:`475` for the rationale).
+
.. method:: socket.bind(address)
@@ -815,6 +844,19 @@ to sockets.
Connect to a remote socket at *address*. (The format of *address* depends on the
address family --- see above.)
+ If the connection is interrupted by a signal, the method waits until the
+ connection completes, or raise a :exc:`socket.timeout` on timeout, if the
+ signal handler doesn't raise an exception and the socket is blocking or has
+ a timeout. For non-blocking sockets, the method raises an
+ :exc:`InterruptedError` exception if the connection is interrupted by a
+ signal (or the exception raised by the signal handler).
+
+ .. versionchanged:: 3.5
+ The method now waits until the connection completes instead of raising an
+ :exc:`InterruptedError` exception if the connection is interrupted by a
+ signal, the signal handler doesn't raise an exception and the socket is
+ blocking or has a timeout (see the :pep:`475` for the rationale).
+
.. method:: socket.connect_ex(address)
@@ -910,12 +952,15 @@ to sockets.
On other platforms, the generic :func:`fcntl.fcntl` and :func:`fcntl.ioctl`
functions may be used; they accept a socket object as their first argument.
-.. method:: socket.listen(backlog)
+.. method:: socket.listen([backlog])
- Listen for connections made to the socket. The *backlog* argument specifies the
- maximum number of queued connections and should be at least 0; the maximum value
- is system-dependent (usually 5), the minimum value is forced to 0.
+ Enable a server to accept connections. If *backlog* is specified, it must
+ be at least 0 (if it is lower, it is set to 0); it specifies the number of
+ unaccepted connections that the system will allow before refusing new
+ connections. If not specified, a default reasonable value is chosen.
+ .. versionchanged:: 3.5
+ The *backlog* parameter is now optional.
.. method:: socket.makefile(mode='r', buffering=None, *, encoding=None, \
errors=None, newline=None)
@@ -953,6 +998,11 @@ to sockets.
For best match with hardware and network realities, the value of *bufsize*
should be a relatively small power of 2, for example, 4096.
+ .. versionchanged:: 3.5
+ If the system call is interrupted and the signal handler does not raise
+ an exception, the method now retries the system call instead of raising
+ an :exc:`InterruptedError` exception (see :pep:`475` for the rationale).
+
.. method:: socket.recvfrom(bufsize[, flags])
@@ -962,6 +1012,11 @@ to sockets.
:manpage:`recv(2)` for the meaning of the optional argument *flags*; it defaults
to zero. (The format of *address* depends on the address family --- see above.)
+ .. versionchanged:: 3.5
+ If the system call is interrupted and the signal handler does not raise
+ an exception, the method now retries the system call instead of raising
+ an :exc:`InterruptedError` exception (see :pep:`475` for the rationale).
+
.. method:: socket.recvmsg(bufsize[, ancbufsize[, flags]])
@@ -1028,6 +1083,11 @@ to sockets.
.. versionadded:: 3.3
+ .. versionchanged:: 3.5
+ If the system call is interrupted and the signal handler does not raise
+ an exception, the method now retries the system call instead of raising
+ an :exc:`InterruptedError` exception (see :pep:`475` for the rationale).
+
.. method:: socket.recvmsg_into(buffers[, ancbufsize[, flags]])
@@ -1094,6 +1154,11 @@ to sockets.
application needs to attempt delivery of the remaining data. For further
information on this topic, consult the :ref:`socket-howto`.
+ .. versionchanged:: 3.5
+ If the system call is interrupted and the signal handler does not raise
+ an exception, the method now retries the system call instead of raising
+ an :exc:`InterruptedError` exception (see :pep:`475` for the rationale).
+
.. method:: socket.sendall(bytes[, flags])
@@ -1104,6 +1169,15 @@ to sockets.
success. On error, an exception is raised, and there is no way to determine how
much data, if any, was successfully sent.
+ .. versionchanged:: 3.5
+ The socket timeout is no more reset each time data is sent successfuly.
+ The socket timeout is now the maximum total duration to send all data.
+
+ .. versionchanged:: 3.5
+ If the system call is interrupted and the signal handler does not raise
+ an exception, the method now retries the system call instead of raising
+ an :exc:`InterruptedError` exception (see :pep:`475` for the rationale).
+
.. method:: socket.sendto(bytes, address)
socket.sendto(bytes, flags, address)
@@ -1114,6 +1188,11 @@ to sockets.
bytes sent. (The format of *address* depends on the address family --- see
above.)
+ .. versionchanged:: 3.5
+ If the system call is interrupted and the signal handler does not raise
+ an exception, the method now retries the system call instead of raising
+ an :exc:`InterruptedError` exception (see :pep:`475` for the rationale).
+
.. method:: socket.sendmsg(buffers[, ancdata[, flags[, address]]])
@@ -1150,6 +1229,26 @@ to sockets.
.. versionadded:: 3.3
+ .. versionchanged:: 3.5
+ If the system call is interrupted and the signal handler does not raise
+ an exception, the method now retries the system call instead of raising
+ an :exc:`InterruptedError` exception (see :pep:`475` for the rationale).
+
+.. method:: socket.sendfile(file, offset=0, count=None)
+
+ Send a file until EOF is reached by using high-performance
+ :mod:`os.sendfile` and return the total number of bytes which were sent.
+ *file* must be a regular file object opened in binary mode. If
+ :mod:`os.sendfile` is not available (e.g. Windows) or *file* is not a
+ regular file :meth:`send` will be used instead. *offset* tells from where to
+ start reading the file. If specified, *count* is the total number of bytes
+ to transmit as opposed to sending the file until EOF is reached. File
+ position is updated on return or also in case of error in which case
+ :meth:`file.tell() <io.IOBase.tell>` can be used to figure out the number of
+ bytes which were sent. The socket must be of :const:`SOCK_STREAM` type. Non-
+ blocking sockets are not supported.
+
+ .. versionadded:: 3.5
.. method:: socket.set_inheritable(inheritable)
@@ -1189,11 +1288,15 @@ to sockets.
Set the value of the given socket option (see the Unix manual page
:manpage:`setsockopt(2)`). The needed symbolic constants are defined in the
- :mod:`socket` module (:const:`SO_\*` etc.). The value can be an integer or a
- bytes object representing a buffer. In the latter case it is up to the caller to
+ :mod:`socket` module (:const:`SO_\*` etc.). The value can be an integer or
+ a :term:`bytes-like object` representing a buffer. In the latter case it is
+ up to the caller to
ensure that the bytestring contains the proper bits (see the optional built-in
module :mod:`struct` for a way to encode C structures as bytestrings).
+ .. versionchanged: 3.5
+ Writable :term:`bytes-like object` is now accepted.
+
.. method:: socket.shutdown(how)
diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst
index 3e49af6..18be936 100644
--- a/Doc/library/socketserver.rst
+++ b/Doc/library/socketserver.rst
@@ -114,7 +114,7 @@ the request handler class :meth:`handle` method.
Another approach to handling multiple simultaneous requests in an environment
that supports neither threads nor :func:`~os.fork` (or where these are too
expensive or inappropriate for the service) is to maintain an explicit table of
-partially finished requests and to use :func:`~select.select` to decide which
+partially finished requests and to use :mod:`selectors` to decide which
request to work on next (or whether to handle a new incoming request). This is
particularly important for stream services where each client can potentially be
connected for a long time (if threads or subprocesses cannot be used). See
@@ -137,7 +137,7 @@ Server Objects
.. method:: BaseServer.fileno()
Return an integer file descriptor for the socket on which the server is
- listening. This function is most commonly passed to :func:`select.select`, to
+ listening. This function is most commonly passed to :mod:`selectors`, to
allow monitoring multiple servers in the same process.
diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst
index 6097e7a..fc69a80 100644
--- a/Doc/library/sqlite3.rst
+++ b/Doc/library/sqlite3.rst
@@ -649,6 +649,9 @@ Row Objects
This method returns a list of column names. Immediately after a query,
it is the first member of each tuple in :attr:`Cursor.description`.
+ .. versionchanged:: 3.5
+ Added support of slicing.
+
Let's assume we initialize a table as in the example given above::
conn = sqlite3.connect(":memory:")
diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst
index 233de8d..4075134 100644
--- a/Doc/library/ssl.rst
+++ b/Doc/library/ssl.rst
@@ -315,6 +315,8 @@ Random generation
For almost all applications :func:`os.urandom` is preferable.
+ For almost all applications :func:`os.urandom` is preferable.
+
.. versionadded:: 3.3
.. function:: RAND_status()
@@ -335,6 +337,8 @@ Random generation
See http://egd.sourceforge.net/ or http://prngd.sourceforge.net/ for sources
of entropy-gathering daemons.
+ Availability: not available with LibreSSL.
+
.. function:: RAND_add(bytes, entropy)
Mix the given *bytes* into the SSL pseudo-random number generator. The
@@ -342,6 +346,9 @@ Random generation
string (so you can always use :const:`0.0`). See :rfc:`1750` for more
information on sources of entropy.
+ .. versionchanged: 3.5
+ Writable :term:`bytes-like object` is now accepted.
+
Certificate handling
^^^^^^^^^^^^^^^^^^^^
@@ -350,10 +357,9 @@ Certificate handling
Verify that *cert* (in decoded format as returned by
:meth:`SSLSocket.getpeercert`) matches the given *hostname*. The rules
applied are those for checking the identity of HTTPS servers as outlined
- in :rfc:`2818` and :rfc:`6125`, except that IP addresses are not currently
- supported. In addition to HTTPS, this function should be suitable for
- checking the identity of servers in various SSL-based protocols such as
- FTPS, IMAPS, POPS and others.
+ in :rfc:`2818` and :rfc:`6125`. In addition to HTTPS, this function
+ should be suitable for checking the identity of servers in various
+ SSL-based protocols such as FTPS, IMAPS, POPS and others.
:exc:`CertificateError` is raised on failure. On success, the function
returns nothing::
@@ -375,22 +381,38 @@ Certificate handling
IDN A-labels such as ``www*.xn--pthon-kva.org`` are still supported,
but ``x*.python.org`` no longer matches ``xn--tda.python.org``.
-.. function:: cert_time_to_seconds(timestring)
+ .. versionchanged:: 3.5
+ Matching of IP addresses, when present in the subjectAltName field
+ of the certificate, is now supported.
+
+.. function:: cert_time_to_seconds(cert_time)
- Returns a floating-point value containing a normal seconds-after-the-epoch
- time value, given the time-string representing the "notBefore" or "notAfter"
- date from a certificate.
+ Return the time in seconds since the Epoch, given the ``cert_time``
+ string representing the "notBefore" or "notAfter" date from a
+ certificate in ``"%b %d %H:%M:%S %Y %Z"`` strptime format (C
+ locale).
- Here's an example::
+ Here's an example:
- >>> import ssl
- >>> ssl.cert_time_to_seconds("May 9 00:00:00 2007 GMT")
- 1178694000.0
- >>> import time
- >>> time.ctime(ssl.cert_time_to_seconds("May 9 00:00:00 2007 GMT"))
- 'Wed May 9 00:00:00 2007'
+ .. doctest:: newcontext
-.. function:: get_server_certificate(addr, ssl_version=PROTOCOL_SSLv3, ca_certs=None)
+ >>> import ssl
+ >>> timestamp = ssl.cert_time_to_seconds("Jan 5 09:34:43 2018 GMT")
+ >>> timestamp
+ 1515144883
+ >>> from datetime import datetime
+ >>> print(datetime.utcfromtimestamp(timestamp))
+ 2018-01-05 09:34:43
+
+ "notBefore" or "notAfter" dates must use GMT (:rfc:`5280`).
+
+ .. versionchanged:: 3.5
+ Interpret the input time as a time in UTC as specified by 'GMT'
+ timezone in the input string. Local timezone was used
+ previously. Return an integer (no fractions of a second in the
+ input format)
+
+.. function:: get_server_certificate(addr, ssl_version=PROTOCOL_SSLv23, ca_certs=None)
Given the address ``addr`` of an SSL-protected server, as a (*hostname*,
*port-number*) pair, fetches the server's certificate, and returns it as a
@@ -404,6 +426,10 @@ Certificate handling
.. versionchanged:: 3.3
This function is now IPv6-compatible.
+ .. versionchanged:: 3.5
+ The default *ssl_version* is changed from :data:`PROTOCOL_SSLv3` to
+ :data:`PROTOCOL_SSLv23` for maximum compatibility with modern servers.
+
.. function:: DER_cert_to_PEM_cert(DER_cert_bytes)
Given a certificate as a DER-encoded blob of bytes, returns a PEM-encoded
@@ -671,6 +697,13 @@ Constants
.. versionadded:: 3.3
+.. data:: HAS_ALPN
+
+ Whether the OpenSSL library has built-in support for the *Application-Layer
+ Protocol Negotiation* TLS extension as described in :rfc:`7301`.
+
+ .. versionadded:: 3.5
+
.. data:: HAS_ECDH
Whether the OpenSSL library has built-in support for Elliptic Curve-based
@@ -788,6 +821,8 @@ SSL Sockets
(but passing a non-zero ``flags`` argument is not allowed)
- :meth:`~socket.socket.send()`, :meth:`~socket.socket.sendall()` (with
the same limitation)
+ - :meth:`~socket.socket.sendfile()` (but :mod:`os.sendfile` will be used
+ for plain-text sockets only, else :meth:`~socket.socket.send()` will be used)
- :meth:`~socket.socket.shutdown()`
However, since the SSL (and TLS) protocol has its own framing atop
@@ -798,6 +833,15 @@ SSL Sockets
Usually, :class:`SSLSocket` are not created directly, but using the
:func:`wrap_socket` function or the :meth:`SSLContext.wrap_socket` method.
+ .. versionchanged:: 3.5
+ The :meth:`sendfile` method was added.
+
+ .. versionchanged:: 3.5
+ The :meth:`shutdown` does not reset the socket timeout each time bytes
+ are received or sent. The socket timeout is now to maximum total duration
+ of the shutdown.
+
+
SSL sockets also have the following additional methods and attributes:
.. method:: SSLSocket.read(len=0, buffer=None)
@@ -812,6 +856,11 @@ SSL sockets also have the following additional methods and attributes:
As at any time a re-negotiation is possible, a call to :meth:`read` can also
cause write operations.
+ .. versionchanged:: 3.5
+ The socket timeout is no more reset each time bytes are received or sent.
+ The socket timeout is now to maximum total duration to read up to *len*
+ bytes.
+
.. method:: SSLSocket.write(buf)
Write *buf* to the SSL socket and return the number of bytes written. The
@@ -823,6 +872,10 @@ SSL sockets also have the following additional methods and attributes:
As at any time a re-negotiation is possible, a call to :meth:`write` can
also cause read operations.
+ .. versionchanged:: 3.5
+ The socket timeout is no more reset each time bytes are received or sent.
+ The socket timeout is now to maximum total duration to write *buf*.
+
.. note::
The :meth:`~SSLSocket.read` and :meth:`~SSLSocket.write` methods are the
@@ -844,6 +897,10 @@ SSL sockets also have the following additional methods and attributes:
:attr:`~SSLContext.check_hostname` attribute of the socket's
:attr:`~SSLSocket.context` is true.
+ .. versionchanged:: 3.5
+ The socket timeout is no more reset each time bytes are received or sent.
+ The socket timeout is now to maximum total duration of the handshake.
+
.. method:: SSLSocket.getpeercert(binary_form=False)
If there is no certificate for the peer on the other end of the connection,
@@ -917,6 +974,17 @@ SSL sockets also have the following additional methods and attributes:
version of the SSL protocol that defines its use, and the number of secret
bits being used. If no connection has been established, returns ``None``.
+.. method:: SSLSocket.shared_ciphers()
+
+ Return the list of ciphers shared by the client during the handshake. Each
+ entry of the returned list is a three-value tuple containing the name of the
+ cipher, the version of the SSL protocol that defines its use, and the number
+ of secret bits the cipher uses. :meth:`~SSLSocket.shared_ciphers` returns
+ ``None`` if no connection has been established or the socket is a client
+ socket.
+
+ .. versionadded:: 3.5
+
.. method:: SSLSocket.compression()
Return the compression algorithm being used as a string, or ``None``
@@ -940,12 +1008,22 @@ SSL sockets also have the following additional methods and attributes:
.. versionadded:: 3.3
+.. method:: SSLSocket.selected_alpn_protocol()
+
+ Return the protocol that was selected during the TLS handshake. If
+ :meth:`SSLContext.set_alpn_protocols` was not called, if the other party does
+ not support ALPN, if this socket does not support any of the client's
+ proposed protocols, or if the handshake has not happened yet, ``None`` is
+ returned.
+
+ .. versionadded:: 3.5
+
.. method:: SSLSocket.selected_npn_protocol()
- Returns the protocol that was selected during the TLS/SSL handshake. If
- :meth:`SSLContext.set_npn_protocols` was not called, or if the other party
- does not support NPN, or if the handshake has not yet happened, this will
- return ``None``.
+ Return the higher-level protocol that was selected during the TLS/SSL
+ handshake. If :meth:`SSLContext.set_npn_protocols` was not called, or
+ if the other party does not support NPN, or if the handshake has not yet
+ happened, this will return ``None``.
.. versionadded:: 3.3
@@ -957,6 +1035,16 @@ SSL sockets also have the following additional methods and attributes:
returned socket should always be used for further communication with the
other side of the connection, rather than the original socket.
+.. method:: SSLSocket.version()
+
+ Return the actual SSL protocol version negotiated by the connection
+ as a string, or ``None`` is no secure connection is established.
+ As of this writing, possible return values include ``"SSLv2"``,
+ ``"SSLv3"``, ``"TLSv1"``, ``"TLSv1.1"`` and ``"TLSv1.2"``.
+ Recent OpenSSL versions may define more return values.
+
+ .. versionadded:: 3.5
+
.. method:: SSLSocket.pending()
Returns the number of already decrypted bytes available for read, pending on
@@ -1135,6 +1223,20 @@ to speed up repeated connections from the same clients.
when connected, the :meth:`SSLSocket.cipher` method of SSL sockets will
give the currently selected cipher.
+.. method:: SSLContext.set_alpn_protocols(protocols)
+
+ Specify which protocols the socket should advertise during the SSL/TLS
+ handshake. It should be a list of ASCII strings, like ``['http/1.1',
+ 'spdy/2']``, ordered by preference. The selection of a protocol will happen
+ during the handshake, and will play out according to :rfc:`7301`. After a
+ successful handshake, the :meth:`SSLSocket.selected_alpn_protocol` method will
+ return the agreed-upon protocol.
+
+ This method will raise :exc:`NotImplementedError` if :data:`HAS_ALPN` is
+ False.
+
+ .. versionadded:: 3.5
+
.. method:: SSLContext.set_npn_protocols(protocols)
Specify which protocols the socket should advertise during the SSL/TLS
@@ -1175,7 +1277,7 @@ to speed up repeated connections from the same clients.
Due to the early negotiation phase of the TLS connection, only limited
methods and attributes are usable like
- :meth:`SSLSocket.selected_npn_protocol` and :attr:`SSLSocket.context`.
+ :meth:`SSLSocket.selected_alpn_protocol` and :attr:`SSLSocket.context`.
:meth:`SSLSocket.getpeercert`, :meth:`SSLSocket.getpeercert`,
:meth:`SSLSocket.cipher` and :meth:`SSLSocket.compress` methods require that
the TLS connection has progressed beyond the TLS Client Hello and therefore
@@ -1251,10 +1353,20 @@ to speed up repeated connections from the same clients.
quite similarly to HTTP virtual hosts. Specifying *server_hostname* will
raise a :exc:`ValueError` if *server_side* is true.
- .. versionchanged:: 3.4.3
+ .. versionchanged:: 3.5
Always allow a server_hostname to be passed, even if OpenSSL does not
have SNI.
+.. method:: SSLContext.wrap_bio(incoming, outgoing, server_side=False, \
+ server_hostname=None)
+
+ Create a new :class:`SSLObject` instance by wrapping the BIO objects
+ *incoming* and *outgoing*. The SSL routines will read input data from the
+ incoming BIO and write data to the outgoing BIO.
+
+ The *server_side* and *server_hostname* parameters have the same meaning as
+ in :meth:`SSLContext.wrap_socket`.
+
.. method:: SSLContext.session_stats()
Get statistics about the SSL sessions created or managed by this context.
@@ -1627,7 +1739,7 @@ are finished with the client (or the client is finished with you)::
And go back to listening for new client connections (of course, a real server
would probably handle each client connection in a separate thread, or put
-the sockets in non-blocking mode and use an event loop).
+the sockets in :ref:`non-blocking mode <ssl-nonblocking>` and use an event loop).
.. _ssl-nonblocking:
@@ -1649,6 +1761,12 @@ thus several things you need to be aware of:
socket first, and attempts to *read* from the SSL socket may require
a prior *write* to the underlying socket.
+ .. versionchanged:: 3.5
+
+ In earlier Python versions, the :meth:`!SSLSocket.send` method
+ returned zero instead of raising :exc:`SSLWantWriteError` or
+ :exc:`SSLWantReadError`.
+
- Calling :func:`~select.select` tells you that the OS-level socket can be
read from (or written to), but it does not imply that there is sufficient
data at the upper SSL layer. For example, only part of an SSL frame might
@@ -1681,13 +1799,143 @@ thus several things you need to be aware of:
.. seealso::
- The :mod:`asyncio` module supports non-blocking SSL sockets and provides a
+ The :mod:`asyncio` module supports :ref:`non-blocking SSL sockets
+ <ssl-nonblocking>` and provides a
higher level API. It polls for events using the :mod:`selectors` module and
handles :exc:`SSLWantWriteError`, :exc:`SSLWantReadError` and
:exc:`BlockingIOError` exceptions. It runs the SSL handshake asynchronously
as well.
+Memory BIO Support
+------------------
+
+.. versionadded:: 3.5
+
+Ever since the SSL module was introduced in Python 2.6, the :class:`SSLSocket`
+class has provided two related but distinct areas of functionality:
+
+- SSL protocol handling
+- Network IO
+
+The network IO API is identical to that provided by :class:`socket.socket`,
+from which :class:`SSLSocket` also inherits. This allows an SSL socket to be
+used as a drop-in replacement for a regular socket, making it very easy to add
+SSL support to an existing application.
+
+Combining SSL protocol handling and network IO usually works well, but there
+are some cases where it doesn't. An example is async IO frameworks that want to
+use a different IO multiplexing model than the "select/poll on a file
+descriptor" (readiness based) model that is assumed by :class:`socket.socket`
+and by the internal OpenSSL socket IO routines. This is mostly relevant for
+platforms like Windows where this model is not efficient. For this purpose, a
+reduced scope variant of :class:`SSLSocket` called :class:`SSLObject` is
+provided.
+
+.. class:: SSLObject
+
+ A reduced-scope variant of :class:`SSLSocket` representing an SSL protocol
+ instance that does not contain any network IO methods. This class is
+ typically used by framework authors that want to implement asynchronous IO
+ for SSL through memory buffers.
+
+ This class implements an interface on top of a low-level SSL object as
+ implemented by OpenSSL. This object captures the state of an SSL connection
+ but does not provide any network IO itself. IO needs to be performed through
+ separate "BIO" objects which are OpenSSL's IO abstraction layer.
+
+ An :class:`SSLObject` instance can be created using the
+ :meth:`~SSLContext.wrap_bio` method. This method will create the
+ :class:`SSLObject` instance and bind it to a pair of BIOs. The *incoming*
+ BIO is used to pass data from Python to the SSL protocol instance, while the
+ *outgoing* BIO is used to pass data the other way around.
+
+ The following methods are available:
+
+ - :attr:`~SSLSocket.context`
+ - :attr:`~SSLSocket.server_side`
+ - :attr:`~SSLSocket.server_hostname`
+ - :meth:`~SSLSocket.read`
+ - :meth:`~SSLSocket.write`
+ - :meth:`~SSLSocket.getpeercert`
+ - :meth:`~SSLSocket.selected_npn_protocol`
+ - :meth:`~SSLSocket.cipher`
+ - :meth:`~SSLSocket.shared_ciphers`
+ - :meth:`~SSLSocket.compression`
+ - :meth:`~SSLSocket.pending`
+ - :meth:`~SSLSocket.do_handshake`
+ - :meth:`~SSLSocket.unwrap`
+ - :meth:`~SSLSocket.get_channel_binding`
+
+ When compared to :class:`SSLSocket`, this object lacks the following
+ features:
+
+ - Any form of network IO incluging methods such as ``recv()`` and
+ ``send()``.
+
+ - There is no *do_handshake_on_connect* machinery. You must always manually
+ call :meth:`~SSLSocket.do_handshake` to start the handshake.
+
+ - There is no handling of *suppress_ragged_eofs*. All end-of-file conditions
+ that are in violation of the protocol are reported via the
+ :exc:`SSLEOFError` exception.
+
+ - The method :meth:`~SSLSocket.unwrap` call does not return anything,
+ unlike for an SSL socket where it returns the underlying socket.
+
+ - The *server_name_callback* callback passed to
+ :meth:`SSLContext.set_servername_callback` will get an :class:`SSLObject`
+ instance instead of a :class:`SSLSocket` instance as its first parameter.
+
+ Some notes related to the use of :class:`SSLObject`:
+
+ - All IO on an :class:`SSLObject` is :ref:`non-blocking <ssl-nonblocking>`.
+ This means that for example :meth:`~SSLSocket.read` will raise an
+ :exc:`SSLWantReadError` if it needs more data than the incoming BIO has
+ available.
+
+ - There is no module-level ``wrap_bio()`` call like there is for
+ :meth:`~SSLContext.wrap_socket`. An :class:`SSLObject` is always created
+ via an :class:`SSLContext`.
+
+An SSLObject communicates with the outside world using memory buffers. The
+class :class:`MemoryBIO` provides a memory buffer that can be used for this
+purpose. It wraps an OpenSSL memory BIO (Basic IO) object:
+
+.. class:: MemoryBIO
+
+ A memory buffer that can be used to pass data between Python and an SSL
+ protocol instance.
+
+ .. attribute:: MemoryBIO.pending
+
+ Return the number of bytes currently in the memory buffer.
+
+ .. attribute:: MemoryBIO.eof
+
+ A boolean indicating whether the memory BIO is current at the end-of-file
+ position.
+
+ .. method:: MemoryBIO.read(n=-1)
+
+ Read up to *n* bytes from the memory buffer. If *n* is not specified or
+ negative, all bytes are returned.
+
+ .. method:: MemoryBIO.write(buf)
+
+ Write the bytes from *buf* to the memory BIO. The *buf* argument must be an
+ object supporting the buffer protocol.
+
+ The return value is the number of bytes written, which is always equal to
+ the length of *buf*.
+
+ .. method:: MemoryBIO.write_eof()
+
+ Write an EOF marker to the memory BIO. After this method has been called, it
+ is illegal to call :meth:`~MemoryBIO.write`. The attribute :attr:`eof` will
+ become true after all data currently in the buffer has been read.
+
+
.. _ssl-security:
Security considerations
diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst
index 24769f6..845b2ef 100644
--- a/Doc/library/stat.rst
+++ b/Doc/library/stat.rst
@@ -126,7 +126,7 @@ Example::
if __name__ == '__main__':
walktree(sys.argv[1], visitfile)
-An additional utility function is provided to covert a file's mode in a human
+An additional utility function is provided to convert a file's mode in a human
readable string:
.. function:: filemode(mode)
@@ -399,3 +399,29 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`:
The file is a snapshot file.
See the \*BSD or Mac OS systems man page :manpage:`chflags(2)` for more information.
+
+On Windows, the following file attribute constants are available for use when
+testing bits in the ``st_file_attributes`` member returned by :func:`os.stat`.
+See the `Windows API documentation
+<http://msdn.microsoft.com/en-us/library/windows/desktop/gg258117.aspx>`_
+for more detail on the meaning of these constants.
+
+.. data:: FILE_ATTRIBUTE_ARCHIVE
+ FILE_ATTRIBUTE_COMPRESSED
+ FILE_ATTRIBUTE_DEVICE
+ FILE_ATTRIBUTE_DIRECTORY
+ FILE_ATTRIBUTE_ENCRYPTED
+ FILE_ATTRIBUTE_HIDDEN
+ FILE_ATTRIBUTE_INTEGRITY_STREAM
+ FILE_ATTRIBUTE_NORMAL
+ FILE_ATTRIBUTE_NOT_CONTENT_INDEXED
+ FILE_ATTRIBUTE_NO_SCRUB_DATA
+ FILE_ATTRIBUTE_OFFLINE
+ FILE_ATTRIBUTE_READONLY
+ FILE_ATTRIBUTE_REPARSE_POINT
+ FILE_ATTRIBUTE_SPARSE_FILE
+ FILE_ATTRIBUTE_SYSTEM
+ FILE_ATTRIBUTE_TEMPORARY
+ FILE_ATTRIBUTE_VIRTUAL
+
+ .. versionadded:: 3.5
diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index 05df97c..8573416 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -354,7 +354,7 @@ Notes:
The numeric literals accepted include the digits ``0`` to ``9`` or any
Unicode equivalent (code points with the ``Nd`` property).
- See http://www.unicode.org/Public/6.3.0/ucd/extracted/DerivedNumericType.txt
+ See http://www.unicode.org/Public/8.0.0/ucd/extracted/DerivedNumericType.txt
for a complete list of code points with the ``Nd`` property.
@@ -1937,6 +1937,16 @@ expression support in the :mod:`re` module).
>>> 'www.example.com'.strip('cmowz.')
'example'
+ The outermost leading and trailing *chars* argument values are stripped
+ from the string. Characters are removed from the leading end until
+ reaching a string character that is not contained in the set of
+ characters in *chars*. A similar action takes place on the trailing end.
+ For example::
+
+ >>> comment_string = '#....... Section 3.2.1 Issue #32 .......'
+ >>> comment_string.strip('.#! ')
+ 'Section 3.2.1 Issue #32'
+
.. method:: str.swapcase()
@@ -2290,6 +2300,19 @@ the bytes type has an additional class method to read data in that format:
>>> bytes.fromhex('2Ef0 F1f2 ')
b'.\xf0\xf1\xf2'
+A reverse conversion function exists to transform a bytes object into its
+hexadecimal representation.
+
+.. method:: bytes.hex()
+
+ Return a string object containing two hexadecimal digits for each
+ byte in the instance.
+
+ >>> b'\xf0\xf1\xf2'.hex()
+ 'f0f1f2'
+
+ .. versionadded:: 3.5
+
Since bytes objects are sequences of integers (akin to a tuple), for a bytes
object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytes
object of length 1. (This contrasts with text strings, where both indexing
@@ -2345,6 +2368,19 @@ the bytearray type has an additional class method to read data in that format:
>>> bytearray.fromhex('2Ef0 F1f2 ')
bytearray(b'.\xf0\xf1\xf2')
+A reverse conversion function exists to transform a bytearray object into its
+hexadecimal representation.
+
+.. method:: bytearray.hex()
+
+ Return a string object containing two hexadecimal digits for each
+ byte in the instance.
+
+ >>> bytearray(b'\xf0\xf1\xf2').hex()
+ 'f0f1f2'
+
+ .. versionadded:: 3.5
+
Since bytearray objects are sequences of integers (akin to a list), for a
bytearray object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be
a bytearray object of length 1. (This contrasts with text strings, where
@@ -3090,6 +3126,203 @@ place, and instead produce new objects.
always produces a new object, even if no changes were made.
+.. _bytes-formatting:
+
+``printf``-style Bytes Formatting
+----------------------------------
+
+.. index::
+ single: formatting, bytes (%)
+ single: formatting, bytearray (%)
+ single: interpolation, bytes (%)
+ single: interpolation, bytearray (%)
+ single: bytes; formatting
+ single: bytearray; formatting
+ single: bytes; interpolation
+ single: bytearray; interpolation
+ single: printf-style formatting
+ single: sprintf-style formatting
+ single: % formatting
+ single: % interpolation
+
+.. note::
+
+ The formatting operations described here exhibit a variety of quirks that
+ lead to a number of common errors (such as failing to display tuples and
+ dictionaries correctly). If the value being printed may be a tuple or
+ dictionary, wrap it in a tuple.
+
+Bytes objects (``bytes``/``bytearray``) have one unique built-in operation:
+the ``%`` operator (modulo).
+This is also known as the bytes *formatting* or *interpolation* operator.
+Given ``format % values`` (where *format* is a bytes object), ``%`` conversion
+specifications in *format* are replaced with zero or more elements of *values*.
+The effect is similar to using the :c:func:`sprintf` in the C language.
+
+If *format* requires a single argument, *values* may be a single non-tuple
+object. [5]_ Otherwise, *values* must be a tuple with exactly the number of
+items specified by the format bytes object, or a single mapping object (for
+example, a dictionary).
+
+A conversion specifier contains two or more characters and has the following
+components, which must occur in this order:
+
+#. The ``'%'`` character, which marks the start of the specifier.
+
+#. Mapping key (optional), consisting of a parenthesised sequence of characters
+ (for example, ``(somename)``).
+
+#. Conversion flags (optional), which affect the result of some conversion
+ types.
+
+#. Minimum field width (optional). If specified as an ``'*'`` (asterisk), the
+ actual width is read from the next element of the tuple in *values*, and the
+ object to convert comes after the minimum field width and optional precision.
+
+#. Precision (optional), given as a ``'.'`` (dot) followed by the precision. If
+ specified as ``'*'`` (an asterisk), the actual precision is read from the next
+ element of the tuple in *values*, and the value to convert comes after the
+ precision.
+
+#. Length modifier (optional).
+
+#. Conversion type.
+
+When the right argument is a dictionary (or other mapping type), then the
+formats in the bytes object *must* include a parenthesised mapping key into that
+dictionary inserted immediately after the ``'%'`` character. The mapping key
+selects the value to be formatted from the mapping. For example:
+
+ >>> print(b'%(language)s has %(number)03d quote types.' %
+ ... {b'language': b"Python", b"number": 2})
+ b'Python has 002 quote types.'
+
+In this case no ``*`` specifiers may occur in a format (since they require a
+sequential parameter list).
+
+The conversion flag characters are:
+
++---------+---------------------------------------------------------------------+
+| Flag | Meaning |
++=========+=====================================================================+
+| ``'#'`` | The value conversion will use the "alternate form" (where defined |
+| | below). |
++---------+---------------------------------------------------------------------+
+| ``'0'`` | The conversion will be zero padded for numeric values. |
++---------+---------------------------------------------------------------------+
+| ``'-'`` | The converted value is left adjusted (overrides the ``'0'`` |
+| | conversion if both are given). |
++---------+---------------------------------------------------------------------+
+| ``' '`` | (a space) A blank should be left before a positive number (or empty |
+| | string) produced by a signed conversion. |
++---------+---------------------------------------------------------------------+
+| ``'+'`` | A sign character (``'+'`` or ``'-'``) will precede the conversion |
+| | (overrides a "space" flag). |
++---------+---------------------------------------------------------------------+
+
+A length modifier (``h``, ``l``, or ``L``) may be present, but is ignored as it
+is not necessary for Python -- so e.g. ``%ld`` is identical to ``%d``.
+
+The conversion types are:
+
++------------+-----------------------------------------------------+-------+
+| Conversion | Meaning | Notes |
++============+=====================================================+=======+
+| ``'d'`` | Signed integer decimal. | |
++------------+-----------------------------------------------------+-------+
+| ``'i'`` | Signed integer decimal. | |
++------------+-----------------------------------------------------+-------+
+| ``'o'`` | Signed octal value. | \(1) |
++------------+-----------------------------------------------------+-------+
+| ``'u'`` | Obsolete type -- it is identical to ``'d'``. | \(8) |
++------------+-----------------------------------------------------+-------+
+| ``'x'`` | Signed hexadecimal (lowercase). | \(2) |
++------------+-----------------------------------------------------+-------+
+| ``'X'`` | Signed hexadecimal (uppercase). | \(2) |
++------------+-----------------------------------------------------+-------+
+| ``'e'`` | Floating point exponential format (lowercase). | \(3) |
++------------+-----------------------------------------------------+-------+
+| ``'E'`` | Floating point exponential format (uppercase). | \(3) |
++------------+-----------------------------------------------------+-------+
+| ``'f'`` | Floating point decimal format. | \(3) |
++------------+-----------------------------------------------------+-------+
+| ``'F'`` | Floating point decimal format. | \(3) |
++------------+-----------------------------------------------------+-------+
+| ``'g'`` | Floating point format. Uses lowercase exponential | \(4) |
+| | format if exponent is less than -4 or not less than | |
+| | precision, decimal format otherwise. | |
++------------+-----------------------------------------------------+-------+
+| ``'G'`` | Floating point format. Uses uppercase exponential | \(4) |
+| | format if exponent is less than -4 or not less than | |
+| | precision, decimal format otherwise. | |
++------------+-----------------------------------------------------+-------+
+| ``'c'`` | Single byte (accepts integer or single | |
+| | byte objects). | |
++------------+-----------------------------------------------------+-------+
+| ``'b'`` | Bytes (any object that follows the | \(5) |
+| | :ref:`buffer protocol <bufferobjects>` or has | |
+| | :meth:`__bytes__`). | |
++------------+-----------------------------------------------------+-------+
+| ``'s'`` | ``'s'`` is an alias for ``'b'`` and should only | \(6) |
+| | be used for Python2/3 code bases. | |
++------------+-----------------------------------------------------+-------+
+| ``'a'`` | Bytes (converts any Python object using | \(5) |
+| | ``repr(obj).encode('ascii','backslashreplace)``). | |
++------------+-----------------------------------------------------+-------+
+| ``'r'`` | ``'r'`` is an alias for ``'a'`` and should only | \(7) |
+| | be used for Python2/3 code bases. | |
++------------+-----------------------------------------------------+-------+
+| ``'%'`` | No argument is converted, results in a ``'%'`` | |
+| | character in the result. | |
++------------+-----------------------------------------------------+-------+
+
+Notes:
+
+(1)
+ The alternate form causes a leading zero (``'0'``) to be inserted between
+ left-hand padding and the formatting of the number if the leading character
+ of the result is not already a zero.
+
+(2)
+ The alternate form causes a leading ``'0x'`` or ``'0X'`` (depending on whether
+ the ``'x'`` or ``'X'`` format was used) to be inserted between left-hand padding
+ and the formatting of the number if the leading character of the result is not
+ already a zero.
+
+(3)
+ The alternate form causes the result to always contain a decimal point, even if
+ no digits follow it.
+
+ The precision determines the number of digits after the decimal point and
+ defaults to 6.
+
+(4)
+ The alternate form causes the result to always contain a decimal point, and
+ trailing zeroes are not removed as they would otherwise be.
+
+ The precision determines the number of significant digits before and after the
+ decimal point and defaults to 6.
+
+(5)
+ If precision is ``N``, the output is truncated to ``N`` characters.
+
+(6)
+ ``b'%s'`` is deprecated, but will not be removed during the 3.x series.
+
+(7)
+ ``b'%r'`` is deprecated, but will not be removed during the 3.x series.
+
+(8)
+ See :pep:`237`.
+
+.. note::
+
+ The bytearray version of this method does *not* operate in place - it
+ always produces a new object, even if no changes were made.
+
+.. seealso:: :pep:`461`.
+.. versionadded:: 3.5
+
.. _typememoryview:
Memory Views
@@ -3118,10 +3351,8 @@ copying.
the view. The :class:`~memoryview.itemsize` attribute will give you the
number of bytes in a single element.
- A :class:`memoryview` supports slicing to expose its data. If
- :class:`~memoryview.format` is one of the native format specifiers
- from the :mod:`struct` module, indexing will return a single element
- with the correct type. Full slicing will result in a subview::
+ A :class:`memoryview` supports slicing and indexing to expose its data.
+ One-dimensional slicing will result in a subview::
>>> v = memoryview(b'abcefg')
>>> v[1]
@@ -3133,25 +3364,29 @@ copying.
>>> bytes(v[1:4])
b'bce'
- Other native formats::
+ If :class:`~memoryview.format` is one of the native format specifiers
+ from the :mod:`struct` module, indexing with an integer or a tuple of
+ integers is also supported and returns a single *element* with
+ the correct type. One-dimensional memoryviews can be indexed
+ with an integer or a one-integer tuple. Multi-dimensional memoryviews
+ can be indexed with tuples of exactly *ndim* integers where *ndim* is
+ the number of dimensions. Zero-dimensional memoryviews can be indexed
+ with the empty tuple.
+
+ Here is an example with a non-byte format::
>>> import array
>>> a = array.array('l', [-11111111, 22222222, -33333333, 44444444])
- >>> a[0]
+ >>> m = memoryview(a)
+ >>> m[0]
-11111111
- >>> a[-1]
+ >>> m[-1]
44444444
- >>> a[2:3].tolist()
- [-33333333]
- >>> a[::2].tolist()
+ >>> m[::2].tolist()
[-11111111, -33333333]
- >>> a[::-1].tolist()
- [44444444, -33333333, 22222222, -11111111]
-
- .. versionadded:: 3.3
- If the underlying object is writable, the memoryview supports slice
- assignment. Resizing is not allowed::
+ If the underlying object is writable, the memoryview supports
+ one-dimensional slice assignment. Resizing is not allowed::
>>> data = bytearray(b'abcefg')
>>> v = memoryview(data)
@@ -3184,12 +3419,16 @@ copying.
True
.. versionchanged:: 3.3
+ One-dimensional memoryviews can now be sliced.
One-dimensional memoryviews with formats 'B', 'b' or 'c' are now hashable.
.. versionchanged:: 3.4
memoryview is now registered automatically with
:class:`collections.abc.Sequence`
+ .. versionchanged:: 3.5
+ memoryviews can now be indexed with tuple of integers.
+
:class:`memoryview` has several methods:
.. method:: __eq__(exporter)
@@ -3256,6 +3495,17 @@ copying.
supports all format strings, including those that are not in
:mod:`struct` module syntax.
+ .. method:: hex()
+
+ Return a string object containing two hexadecimal digits for each
+ byte in the buffer. ::
+
+ >>> m = memoryview(b"abc")
+ >>> m.hex()
+ '616263'
+
+ .. versionadded:: 3.5
+
.. method:: tolist()
Return the data in the buffer as a list of elements. ::
@@ -3311,10 +3561,10 @@ copying.
Cast a memoryview to a new format or shape. *shape* defaults to
``[byte_length//new_itemsize]``, which means that the result view
will be one-dimensional. The return value is a new memoryview, but
- the buffer itself is not copied. Supported casts are 1D -> C-contiguous
+ the buffer itself is not copied. Supported casts are 1D -> C-:term:`contiguous`
and C-contiguous -> 1D.
- Both formats are restricted to single element native formats in
+ The destination format is restricted to a single element native format in
:mod:`struct` syntax. One of the formats must be a byte format
('B', 'b' or 'c'). The byte length of the result must be the same
as the original length.
@@ -3395,6 +3645,9 @@ copying.
.. versionadded:: 3.3
+ .. versionchanged:: 3.5
+ The source format is no longer restricted when casting to a byte view.
+
There are also several readonly attributes available:
.. attribute:: obj
@@ -3499,19 +3752,19 @@ copying.
.. attribute:: c_contiguous
- A bool indicating whether the memory is C-contiguous.
+ A bool indicating whether the memory is C-:term:`contiguous`.
.. versionadded:: 3.3
.. attribute:: f_contiguous
- A bool indicating whether the memory is Fortran contiguous.
+ A bool indicating whether the memory is Fortran :term:`contiguous`.
.. versionadded:: 3.3
.. attribute:: contiguous
- A bool indicating whether the memory is contiguous.
+ A bool indicating whether the memory is :term:`contiguous`.
.. versionadded:: 3.3
diff --git a/Doc/library/string.rst b/Doc/library/string.rst
index 19bdb21..2bd8dfd 100644
--- a/Doc/library/string.rst
+++ b/Doc/library/string.rst
@@ -95,6 +95,10 @@ implementation as the built-in :meth:`format` method.
an arbitrary set of positional and keyword arguments.
:meth:`format` is just a wrapper that calls :meth:`vformat`.
+ .. deprecated:: 3.5
+ Passing a format string as keyword argument *format_string* has been
+ deprecated.
+
.. method:: vformat(format_string, args, kwargs)
This function does the actual work of formatting. It is exposed as a
diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst
index 9ac8882..adf99ec 100644
--- a/Doc/library/subprocess.rst
+++ b/Doc/library/subprocess.rst
@@ -25,160 +25,99 @@ modules and functions can be found in the following sections.
Using the :mod:`subprocess` Module
----------------------------------
-The recommended approach to invoking subprocesses is to use the following
-convenience functions for all use cases they can handle. For more advanced
-use cases, the underlying :class:`Popen` interface can be used directly.
+The recommended approach to invoking subprocesses is to use the :func:`run`
+function for all use cases it can handle. For more advanced use cases, the
+underlying :class:`Popen` interface can be used directly.
+The :func:`run` function was added in Python 3.5; if you need to retain
+compatibility with older versions, see the :ref:`call-function-trio` section.
-.. function:: call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None)
+
+.. function:: run(args, *, stdin=None, input=None, stdout=None, stderr=None,\
+ shell=False, timeout=None, check=False)
Run the command described by *args*. Wait for command to complete, then
- return the :attr:`returncode` attribute.
+ return a :class:`CompletedProcess` instance.
The arguments shown above are merely the most common ones, described below
in :ref:`frequently-used-arguments` (hence the use of keyword-only notation
in the abbreviated signature). The full function signature is largely the
- same as that of the :class:`Popen` constructor - this function passes all
- supplied arguments other than *timeout* directly through to that interface.
+ same as that of the :class:`Popen` constructor - apart from *timeout*,
+ *input* and *check*, all the arguments to this function are passed through to
+ that interface.
+
+ This does not capture stdout or stderr by default. To do so, pass
+ :data:`PIPE` for the *stdout* and/or *stderr* arguments.
- The *timeout* argument is passed to :meth:`Popen.wait`. If the timeout
- expires, the child process will be killed and then waited for again. The
+ The *timeout* argument is passed to :meth:`Popen.communicate`. If the timeout
+ expires, the child process will be killed and waited for. The
:exc:`TimeoutExpired` exception will be re-raised after the child process
has terminated.
- Examples::
-
- >>> subprocess.call(["ls", "-l"])
- 0
-
- >>> subprocess.call("exit 1", shell=True)
- 1
-
- .. note::
-
- Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this
- function. The child process will block if it generates enough
- output to a pipe to fill up the OS pipe buffer as the pipes are
- not being read from.
-
- .. versionchanged:: 3.3
- *timeout* was added.
-
-
-.. function:: check_call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None)
-
- Run command with arguments. Wait for command to complete. If the return
- code was zero then return, otherwise raise :exc:`CalledProcessError`. The
- :exc:`CalledProcessError` object will have the return code in the
- :attr:`~CalledProcessError.returncode` attribute.
-
- The arguments shown above are merely the most common ones, described below
- in :ref:`frequently-used-arguments` (hence the use of keyword-only notation
- in the abbreviated signature). The full function signature is largely the
- same as that of the :class:`Popen` constructor - this function passes all
- supplied arguments other than *timeout* directly through to that interface.
+ The *input* argument is passed to :meth:`Popen.communicate` and thus to the
+ subprocess's stdin. If used it must be a byte sequence, or a string if
+ ``universal_newlines=True``. When used, the internal :class:`Popen` object
+ is automatically created with ``stdin=PIPE``, and the *stdin* argument may
+ not be used as well.
- The *timeout* argument is passed to :meth:`Popen.wait`. If the timeout
- expires, the child process will be killed and then waited for again. The
- :exc:`TimeoutExpired` exception will be re-raised after the child process
- has terminated.
+ If *check* is True, and the process exits with a non-zero exit code, a
+ :exc:`CalledProcessError` exception will be raised. Attributes of that
+ exception hold the arguments, the exit code, and stdout and stderr if they
+ were captured.
Examples::
- >>> subprocess.check_call(["ls", "-l"])
- 0
+ >>> subprocess.run(["ls", "-l"]) # doesn't capture output
+ CompletedProcess(args=['ls', '-l'], returncode=0)
- >>> subprocess.check_call("exit 1", shell=True)
+ >>> subprocess.run("exit 1", shell=True, check=True)
Traceback (most recent call last):
- ...
+ ...
subprocess.CalledProcessError: Command 'exit 1' returned non-zero exit status 1
- .. note::
+ >>> subprocess.run(["ls", "-l", "/dev/null"], stdout=subprocess.PIPE)
+ CompletedProcess(args=['ls', '-l', '/dev/null'], returncode=0,
+ stdout=b'crw-rw-rw- 1 root root 1, 3 Jan 23 16:23 /dev/null\n')
- Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this
- function. The child process will block if it generates enough
- output to a pipe to fill up the OS pipe buffer as the pipes are
- not being read from.
+ .. versionadded:: 3.5
- .. versionchanged:: 3.3
- *timeout* was added.
-
-
-.. function:: check_output(args, *, input=None, stdin=None, stderr=None, shell=False, universal_newlines=False, timeout=None)
-
- Run command with arguments and return its output.
-
- If the return code was non-zero it raises a :exc:`CalledProcessError`. The
- :exc:`CalledProcessError` object will have the return code in the
- :attr:`~CalledProcessError.returncode` attribute and any output in the
- :attr:`~CalledProcessError.output` attribute.
-
- The arguments shown above are merely the most common ones, described below
- in :ref:`frequently-used-arguments` (hence the use of keyword-only notation
- in the abbreviated signature). The full function signature is largely the
- same as that of the :class:`Popen` constructor - this functions passes all
- supplied arguments other than *input* and *timeout* directly through to
- that interface. In addition, *stdout* is not permitted as an argument, as
- it is used internally to collect the output from the subprocess.
-
- The *timeout* argument is passed to :meth:`Popen.wait`. If the timeout
- expires, the child process will be killed and then waited for again. The
- :exc:`TimeoutExpired` exception will be re-raised after the child process
- has terminated.
-
- The *input* argument is passed to :meth:`Popen.communicate` and thus to the
- subprocess's stdin. If used it must be a byte sequence, or a string if
- ``universal_newlines=True``. When used, the internal :class:`Popen` object
- is automatically created with ``stdin=PIPE``, and the *stdin* argument may
- not be used as well.
+.. class:: CompletedProcess
- Examples::
+ The return value from :func:`run`, representing a process that has finished.
- >>> subprocess.check_output(["echo", "Hello World!"])
- b'Hello World!\n'
+ .. attribute:: args
- >>> subprocess.check_output(["echo", "Hello World!"], universal_newlines=True)
- 'Hello World!\n'
+ The arguments used to launch the process. This may be a list or a string.
- >>> subprocess.check_output(["sed", "-e", "s/foo/bar/"],
- ... input=b"when in the course of fooman events\n")
- b'when in the course of barman events\n'
+ .. attribute:: returncode
- >>> subprocess.check_output("exit 1", shell=True)
- Traceback (most recent call last):
- ...
- subprocess.CalledProcessError: Command 'exit 1' returned non-zero exit status 1
+ Exit status of the child process. Typically, an exit status of 0 indicates
+ that it ran successfully.
- By default, this function will return the data as encoded bytes. The actual
- encoding of the output data may depend on the command being invoked, so the
- decoding to text will often need to be handled at the application level.
+ A negative value ``-N`` indicates that the child was terminated by signal
+ ``N`` (POSIX only).
- This behaviour may be overridden by setting *universal_newlines* to
- ``True`` as described below in :ref:`frequently-used-arguments`.
+ .. attribute:: stdout
- To also capture standard error in the result, use
- ``stderr=subprocess.STDOUT``::
+ Captured stdout from the child process. A bytes sequence, or a string if
+ :func:`run` was called with ``universal_newlines=True``. None if stdout
+ was not captured.
- >>> subprocess.check_output(
- ... "ls non_existent_file; exit 0",
- ... stderr=subprocess.STDOUT,
- ... shell=True)
- 'ls: non_existent_file: No such file or directory\n'
+ If you ran the process with ``stderr=subprocess.STDOUT``, stdout and
+ stderr will be combined in this attribute, and :attr:`stderr` will be
+ None.
- .. note::
+ .. attribute:: stderr
- Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this
- function. The child process will block if it generates enough
- output to a pipe to fill up the OS pipe buffer as the pipes are
- not being read from.
+ Captured stderr from the child process. A bytes sequence, or a string if
+ :func:`run` was called with ``universal_newlines=True``. None if stderr
+ was not captured.
- .. versionadded:: 3.1
+ .. method:: check_returncode()
- .. versionchanged:: 3.3
- *timeout* was added.
+ If :attr:`returncode` is non-zero, raise a :exc:`CalledProcessError`.
- .. versionchanged:: 3.4
- *input* was added.
+ .. versionadded:: 3.5
.. data:: DEVNULL
@@ -225,11 +164,22 @@ use cases, the underlying :class:`Popen` interface can be used directly.
.. attribute:: output
- Output of the child process if this exception is raised by
+ Output of the child process if it was captured by :func:`run` or
:func:`check_output`. Otherwise, ``None``.
+ .. attribute:: stdout
+
+ Alias for output, for symmetry with :attr:`stderr`.
+
+ .. attribute:: stderr
+
+ Stderr output of the child process if it was captured by :func:`run`.
+ Otherwise, ``None``.
+
.. versionadded:: 3.3
+ .. versionchanged:: 3.5
+ *stdout* and *stderr* attributes added
.. exception:: CalledProcessError
@@ -246,9 +196,20 @@ use cases, the underlying :class:`Popen` interface can be used directly.
.. attribute:: output
- Output of the child process if this exception is raised by
+ Output of the child process if it was captured by :func:`run` or
:func:`check_output`. Otherwise, ``None``.
+ .. attribute:: stdout
+
+ Alias for output, for symmetry with :attr:`stderr`.
+
+ .. attribute:: stderr
+
+ Stderr output of the child process if it was captured by :func:`run`.
+ Otherwise, ``None``.
+
+ .. versionchanged:: 3.5
+ *stdout* and *stderr* attributes added
.. _frequently-used-arguments:
@@ -635,6 +596,7 @@ Instances of the :class:`Popen` class have the following methods:
must be bytes or, if *universal_newlines* was ``True``, a string.
:meth:`communicate` returns a tuple ``(stdout_data, stderr_data)``.
+ The data will be bytes or, if *universal_newlines* was ``True``, strings.
Note that if you want to send data to the process's stdin, you need to create
the Popen object with ``stdin=PIPE``. Similarly, to get anything other than
@@ -851,6 +813,112 @@ The :mod:`subprocess` module exposes the following constants.
This flag is ignored if :data:`CREATE_NEW_CONSOLE` is specified.
+.. _call-function-trio:
+
+Older high-level API
+--------------------
+
+Prior to Python 3.5, these three functions comprised the high level API to
+subprocess. You can now use :func:`run` in many cases, but lots of existing code
+calls these functions.
+
+.. function:: call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None)
+
+ Run the command described by *args*. Wait for command to complete, then
+ return the :attr:`~Popen.returncode` attribute.
+
+ This is equivalent to::
+
+ run(...).returncode
+
+ (except that the *input* and *check* parameters are not supported)
+
+ The arguments shown above are merely the most
+ common ones. The full function signature is largely the
+ same as that of the :class:`Popen` constructor - this function passes all
+ supplied arguments other than *timeout* directly through to that interface.
+
+ .. note::
+
+ Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this
+ function. The child process will block if it generates enough
+ output to a pipe to fill up the OS pipe buffer as the pipes are
+ not being read from.
+
+ .. versionchanged:: 3.3
+ *timeout* was added.
+
+.. function:: check_call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None)
+
+ Run command with arguments. Wait for command to complete. If the return
+ code was zero then return, otherwise raise :exc:`CalledProcessError`. The
+ :exc:`CalledProcessError` object will have the return code in the
+ :attr:`~CalledProcessError.returncode` attribute.
+
+ This is equivalent to::
+
+ run(..., check=True)
+
+ (except that the *input* parameter is not supported)
+
+ The arguments shown above are merely the most
+ common ones. The full function signature is largely the
+ same as that of the :class:`Popen` constructor - this function passes all
+ supplied arguments other than *timeout* directly through to that interface.
+
+ .. note::
+
+ Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this
+ function. The child process will block if it generates enough
+ output to a pipe to fill up the OS pipe buffer as the pipes are
+ not being read from.
+
+ .. versionchanged:: 3.3
+ *timeout* was added.
+
+
+.. function:: check_output(args, *, stdin=None, stderr=None, shell=False, universal_newlines=False, timeout=None)
+
+ Run command with arguments and return its output.
+
+ If the return code was non-zero it raises a :exc:`CalledProcessError`. The
+ :exc:`CalledProcessError` object will have the return code in the
+ :attr:`~CalledProcessError.returncode` attribute and any output in the
+ :attr:`~CalledProcessError.output` attribute.
+
+ This is equivalent to::
+
+ run(..., check=True, stdout=PIPE).stdout
+
+ The arguments shown above are merely the most common ones.
+ The full function signature is largely the same as that of :func:`run` -
+ most arguments are passed directly through to that interface.
+ However, explicitly passing ``input=None`` to inherit the parent's
+ standard input file handle is not supported.
+
+ By default, this function will return the data as encoded bytes. The actual
+ encoding of the output data may depend on the command being invoked, so the
+ decoding to text will often need to be handled at the application level.
+
+ This behaviour may be overridden by setting *universal_newlines* to
+ ``True`` as described above in :ref:`frequently-used-arguments`.
+
+ To also capture standard error in the result, use
+ ``stderr=subprocess.STDOUT``::
+
+ >>> subprocess.check_output(
+ ... "ls non_existent_file; exit 0",
+ ... stderr=subprocess.STDOUT,
+ ... shell=True)
+ 'ls: non_existent_file: No such file or directory\n'
+
+ .. versionadded:: 3.1
+
+ .. versionchanged:: 3.3
+ *timeout* was added.
+
+ .. versionchanged:: 3.4
+ Support for the *input* keyword argument was added.
.. _subprocess-replacements:
diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst
index 2503d33..ba2caff 100644
--- a/Doc/library/symtable.rst
+++ b/Doc/library/symtable.rst
@@ -71,10 +71,6 @@ Examining Symbol Tables
Return ``True`` if the block uses ``exec``.
- .. method:: has_import_star()
-
- Return ``True`` if the block uses a starred from-import.
-
.. method:: get_identifiers()
Return a list of names of symbols in this table.
diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
index 3024086..bb9bdc8 100644
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -167,7 +167,7 @@ always available.
.. data:: dont_write_bytecode
- If this is true, Python won't try to write ``.pyc`` or ``.pyo`` files on the
+ If this is true, Python won't try to write ``.pyc`` files on the
import of source modules. This value is initially set to ``True`` or
``False`` depending on the :option:`-B` command line option and the
:envvar:`PYTHONDONTWRITEBYTECODE` environment variable, but you can set it
@@ -576,6 +576,18 @@ always available.
*service_pack_major*, *suite_mask*, and *product_type*.
+.. function:: get_coroutine_wrapper()
+
+ Returns ``None``, or a wrapper set by :func:`set_coroutine_wrapper`.
+
+ .. versionadded:: 3.5
+ See :pep:`492` for more details.
+
+ .. note::
+ This function has been added on a provisional basis (see :pep:`411`
+ for details.) Use it only for debugging purposes.
+
+
.. data:: hash_info
A :term:`struct sequence` giving parameters of the numeric hash
@@ -718,6 +730,14 @@ always available.
value of :func:`intern` around to benefit from it.
+.. function:: is_finalizing()
+
+ Return :const:`True` if the Python interpreter is
+ :term:`shutting down <interpreter shutdown>`, :const:`False` otherwise.
+
+ .. versionadded:: 3.5
+
+
.. data:: last_type
last_value
last_traceback
@@ -1053,6 +1073,46 @@ always available.
thus not likely to be implemented elsewhere.
+.. function:: set_coroutine_wrapper(wrapper)
+
+ Allows intercepting creation of :term:`coroutine` objects (only ones that
+ are created by an :keyword:`async def` function; generators decorated with
+ :func:`types.coroutine` or :func:`asyncio.coroutine` will not be
+ intercepted).
+
+ The *wrapper* argument must be either:
+
+ * a callable that accepts one argument (a coroutine object);
+ * ``None``, to reset the wrapper.
+
+ If called twice, the new wrapper replaces the previous one. The function
+ is thread-specific.
+
+ The *wrapper* callable cannot define new coroutines directly or indirectly::
+
+ def wrapper(coro):
+ async def wrap(coro):
+ return await coro
+ return wrap(coro)
+ sys.set_coroutine_wrapper(wrapper)
+
+ async def foo():
+ pass
+
+ # The following line will fail with a RuntimeError, because
+ # ``wrapper`` creates a ``wrap(coro)`` coroutine:
+ foo()
+
+ See also :func:`get_coroutine_wrapper`.
+
+ .. versionadded:: 3.5
+ See :pep:`492` for more details.
+
+ .. note::
+ This function has been added on a provisional basis (see :pep:`411`
+ for details.) Use it only for debugging purposes.
+
+
.. data:: stdin
stdout
stderr
@@ -1223,4 +1283,3 @@ always available.
.. rubric:: Citations
.. [C99] ISO/IEC 9899:1999. "Programming languages -- C." A public draft of this standard is available at http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1256.pdf\ .
-
diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst
index 05f29ad..adacb0a 100644
--- a/Doc/library/tarfile.rst
+++ b/Doc/library/tarfile.rst
@@ -62,6 +62,23 @@ Some facts and figures:
+------------------+---------------------------------------------+
| ``'r:xz'`` | Open for reading with lzma compression. |
+------------------+---------------------------------------------+
+ | ``'x'`` or | Create a tarfile exclusively without |
+ | ``'x:'`` | compression. |
+ | | Raise an :exc:`FileExistsError` exception |
+ | | if it is already exists. |
+ +------------------+---------------------------------------------+
+ | ``'x:gz'`` | Create a tarfile with gzip compression. |
+ | | Raise an :exc:`FileExistsError` exception |
+ | | if it is already exists. |
+ +------------------+---------------------------------------------+
+ | ``'x:bz2'`` | Create a tarfile with bzip2 compression. |
+ | | Raise an :exc:`FileExistsError` exception |
+ | | if it is already exists. |
+ +------------------+---------------------------------------------+
+ | ``'x:xz'`` | Create a tarfile with lzma compression. |
+ | | Raise an :exc:`FileExistsError` exception |
+ | | if it is already exists. |
+ +------------------+---------------------------------------------+
| ``'a' or 'a:'`` | Open for appending with no compression. The |
| | file is created if it does not exist. |
+------------------+---------------------------------------------+
@@ -82,9 +99,9 @@ Some facts and figures:
If *fileobj* is specified, it is used as an alternative to a :term:`file object`
opened in binary mode for *name*. It is supposed to be at position 0.
- For modes ``'w:gz'``, ``'r:gz'``, ``'w:bz2'``, ``'r:bz2'``, :func:`tarfile.open`
- accepts the keyword argument *compresslevel* to specify the compression level of
- the file.
+ For modes ``'w:gz'``, ``'r:gz'``, ``'w:bz2'``, ``'r:bz2'``, ``'x:gz'``,
+ ``'x:bz2'``, :func:`tarfile.open` accepts the keyword argument
+ *compresslevel* to specify the compression level of the file.
For special purposes, there is a second format for *mode*:
``'filemode|[compression]'``. :func:`tarfile.open` will return a :class:`TarFile`
@@ -127,6 +144,8 @@ Some facts and figures:
| | writing. |
+-------------+--------------------------------------------+
+ .. versionchanged:: 3.5
+ The ``'x'`` (exclusive creation) mode was added.
.. class:: TarFile
@@ -252,8 +271,8 @@ be finalized; only the internally used file object will be closed. See the
In this case, the file object's :attr:`name` attribute is used if it exists.
*mode* is either ``'r'`` to read from an existing archive, ``'a'`` to append
- data to an existing file or ``'w'`` to create a new file overwriting an existing
- one.
+ data to an existing file, ``'w'`` to create a new file overwriting an existing
+ one or ``'x'`` to create a new file only if it's not exists.
If *fileobj* is given, it is used for reading or writing data. If it can be
determined, *mode* is overridden by *fileobj*'s mode. *fileobj* will be used
@@ -292,12 +311,14 @@ be finalized; only the internally used file object will be closed. See the
to be handled. The default settings will work for most users.
See section :ref:`tar-unicode` for in-depth information.
- .. versionchanged:: 3.2
- Use ``'surrogateescape'`` as the default for the *errors* argument.
-
The *pax_headers* argument is an optional dictionary of strings which
will be added as a pax global header if *format* is :const:`PAX_FORMAT`.
+ .. versionchanged:: 3.2
+ Use ``'surrogateescape'`` as the default for the *errors* argument.
+
+ .. versionchanged:: 3.5
+ The ``'x'`` (exclusive creation) mode was added.
.. classmethod:: TarFile.open(...)
@@ -328,11 +349,15 @@ be finalized; only the internally used file object will be closed. See the
returned by :meth:`getmembers`.
-.. method:: TarFile.list(verbose=True)
+.. method:: TarFile.list(verbose=True, *, members=None)
Print a table of contents to ``sys.stdout``. If *verbose* is :const:`False`,
only the names of the members are printed. If it is :const:`True`, output
- similar to that of :program:`ls -l` is produced.
+ similar to that of :program:`ls -l` is produced. If optional *members* is
+ given, it must be a subset of the list returned by :meth:`getmembers`.
+
+ .. versionchanged:: 3.5
+ Added the *members* parameter.
.. method:: TarFile.next()
@@ -342,7 +367,7 @@ be finalized; only the internally used file object will be closed. See the
available.
-.. method:: TarFile.extractall(path=".", members=None)
+.. method:: TarFile.extractall(path=".", members=None, *, numeric_owner=False)
Extract all members from the archive to the current working directory or
directory *path*. If optional *members* is given, it must be a subset of the
@@ -352,6 +377,10 @@ be finalized; only the internally used file object will be closed. See the
reset each time a file is created in it. And, if a directory's permissions do
not allow writing, extracting files to it will fail.
+ If *numeric_owner* is :const:`True`, the uid and gid numbers from the tarfile
+ are used to set the owner/group for the extracted files. Otherwise, the named
+ values from the tarfile are used.
+
.. warning::
Never extract archives from untrusted sources without prior inspection.
@@ -359,8 +388,11 @@ be finalized; only the internally used file object will be closed. See the
that have absolute filenames starting with ``"/"`` or filenames with two
dots ``".."``.
+ .. versionchanged:: 3.5
+ Added the *numeric_only* parameter.
-.. method:: TarFile.extract(member, path="", set_attrs=True)
+
+.. method:: TarFile.extract(member, path="", set_attrs=True, *, numeric_owner=False)
Extract a member from the archive to the current working directory, using its
full name. Its file information is extracted as accurately as possible. *member*
@@ -368,6 +400,10 @@ be finalized; only the internally used file object will be closed. See the
directory using *path*. File attributes (owner, mtime, mode) are set unless
*set_attrs* is false.
+ If *numeric_owner* is :const:`True`, the uid and gid numbers from the tarfile
+ are used to set the owner/group for the extracted files. Otherwise, the named
+ values from the tarfile are used.
+
.. note::
The :meth:`extract` method does not take care of several extraction issues.
@@ -380,6 +416,9 @@ be finalized; only the internally used file object will be closed. See the
.. versionchanged:: 3.2
Added the *set_attrs* parameter.
+ .. versionchanged:: 3.5
+ Added the *numeric_only* parameter.
+
.. method:: TarFile.extractfile(member)
Extract a member from the archive as a file object. *member* may be a filename
@@ -802,4 +841,3 @@ In case of :const:`PAX_FORMAT` archives, *encoding* is generally not needed
because all the metadata is stored using *UTF-8*. *encoding* is only used in
the rare cases when binary pax headers are decoded or when strings with
surrogate characters are stored.
-
diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst
index 44d025d..83f9941 100644
--- a/Doc/library/tempfile.rst
+++ b/Doc/library/tempfile.rst
@@ -16,16 +16,18 @@
--------------
-This module generates temporary files and directories. It works on all
-supported platforms. It provides three new functions,
-:func:`NamedTemporaryFile`, :func:`mkstemp`, and :func:`mkdtemp`, which should
-eliminate all remaining need to use the insecure :func:`mktemp` function.
-Temporary file names created by this module no longer contain the process ID;
-instead a string of six random characters is used.
-
-Also, all the user-callable functions now take additional arguments which
-allow direct control over the location and name of temporary files. It is
-no longer necessary to use the global *tempdir* variable.
+This module creates temporary files and directories. It works on all
+supported platforms. :class:`TemporaryFile`, :class:`NamedTemporaryFile`,
+:class:`TemporaryDirectory`, and :class:`SpooledTemporaryFile` are high-level
+interfaces which provide automatic cleanup and can be used as
+context managers. :func:`mkstemp` and
+:func:`mkdtemp` are lower-level functions which require manual cleanup.
+
+All the user-callable functions and constructors take additional arguments which
+allow direct control over the location and name of temporary files and
+directories. Files names used by this module include a string of
+random characters which allows those files to be securely created in
+shared temporary directories.
To maintain backward compatibility, the argument order is somewhat odd; it
is recommended to use keyword arguments for clarity.
@@ -34,25 +36,37 @@ The module defines the following user-callable items:
.. function:: TemporaryFile(mode='w+b', buffering=None, encoding=None, newline=None, suffix='', prefix='tmp', dir=None)
Return a :term:`file-like object` that can be used as a temporary storage area.
- The file is created using :func:`mkstemp`. It will be destroyed as soon
+ The file is created securely, using the same rules as :func:`mkstemp`. It will be destroyed as soon
as it is closed (including an implicit close when the object is garbage
- collected). Under Unix, the directory entry for the file is removed
+ collected). Under Unix, the directory entry for the file is either not created at all or is removed
immediately after the file is created. Other platforms do not support
this; your code should not rely on a temporary file created using this
function having or not having a visible name in the file system.
+ The resulting object can be used as a context manager (see
+ :ref:`tempfile-examples`). On completion of the context or
+ destruction of the file object the temporary file will be removed
+ from the filesystem.
+
The *mode* parameter defaults to ``'w+b'`` so that the file created can
be read and written without being closed. Binary mode is used so that it
behaves consistently on all platforms without regard for the data that is
stored. *buffering*, *encoding* and *newline* are interpreted as for
:func:`open`.
- The *dir*, *prefix* and *suffix* parameters are passed to :func:`mkstemp`.
+ The *dir*, *prefix* and *suffix* parameters have the same meaning
+ as with :func:`mkstemp`.
The returned object is a true file object on POSIX platforms. On other
platforms, it is a file-like object whose :attr:`!file` attribute is the
- underlying true file object. This file-like object can be used in a
- :keyword:`with` statement, just like a normal file.
+ underlying true file object.
+
+ The :py:data:`os.O_TMPFILE` flag is used if it is available and works
+ (Linux-specific, requires Linux kernel 3.11 or later).
+
+ .. versionchanged:: 3.5
+
+ The :py:data:`os.O_TMPFILE` flag is now used if available.
.. function:: NamedTemporaryFile(mode='w+b', buffering=None, encoding=None, newline=None, suffix='', prefix='tmp', dir=None, delete=True)
@@ -94,10 +108,9 @@ The module defines the following user-callable items:
.. function:: TemporaryDirectory(suffix='', prefix='tmp', dir=None)
- This function creates a temporary directory using :func:`mkdtemp`
- (the supplied arguments are passed directly to the underlying function).
+ This function securely creates a temporary directory using the same rules as :func:`mkdtemp`.
The resulting object can be used as a context manager (see
- :ref:`context-managers`). On completion of the context or destruction
+ :ref:`tempfile-examples`). On completion of the context or destruction
of the temporary directory object the newly created temporary directory
and all its contents are removed from the filesystem.
@@ -112,7 +125,7 @@ The module defines the following user-callable items:
.. versionadded:: 3.2
-.. function:: mkstemp(suffix='', prefix='tmp', dir=None, text=False)
+.. function:: mkstemp(suffix=None, prefix=None, dir=None, text=False)
Creates a temporary file in the most secure manner possible. There are
no race conditions in the file's creation, assuming that the platform
@@ -141,6 +154,16 @@ The module defines the following user-callable items:
filename will have any nice properties, such as not requiring quoting
when passed to external commands via ``os.popen()``.
+ *suffix*, *prefix*, and *dir* must all contain the same type, if specified.
+ If they are bytes, the returned name will be bytes instead of str.
+ If you want to force a bytes return value with otherwise default behavior,
+ pass ``suffix=b''``.
+
+ A *prefix* value of ``None`` means use the return value of
+ :func:`gettempprefix` or :func:`gettempprefixb` as appropriate.
+
+ A *suffix* value of ``None`` means use an appropriate empty value.
+
If *text* is specified, it indicates whether to open the file in binary
mode (the default) or text mode. On some platforms, this makes no
difference.
@@ -149,8 +172,14 @@ The module defines the following user-callable items:
file (as would be returned by :func:`os.open`) and the absolute pathname
of that file, in that order.
+ .. versionchanged:: 3.5
+ *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to
+ obtain a bytes return value. Prior to this, only str was allowed.
+ *suffix* and *prefix* now accept and default to ``None`` to cause
+ an appropriate default value to be used.
+
-.. function:: mkdtemp(suffix='', prefix='tmp', dir=None)
+.. function:: mkdtemp(suffix=None, prefix=None, dir=None)
Creates a temporary directory in the most secure manner possible. There
are no race conditions in the directory's creation. The directory is
@@ -164,50 +193,21 @@ The module defines the following user-callable items:
:func:`mkdtemp` returns the absolute pathname of the new directory.
+ .. versionchanged:: 3.5
+ *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to
+ obtain a bytes return value. Prior to this, only str was allowed.
+ *suffix* and *prefix* now accept and default to ``None`` to cause
+ an appropriate default value to be used.
-.. function:: mktemp(suffix='', prefix='tmp', dir=None)
-
- .. deprecated:: 2.3
- Use :func:`mkstemp` instead.
-
- Return an absolute pathname of a file that did not exist at the time the
- call is made. The *prefix*, *suffix*, and *dir* arguments are the same
- as for :func:`mkstemp`.
- .. warning::
-
- Use of this function may introduce a security hole in your program. By
- the time you get around to doing anything with the file name it returns,
- someone else may have beaten you to the punch. :func:`mktemp` usage can
- be replaced easily with :func:`NamedTemporaryFile`, passing it the
- ``delete=False`` parameter::
-
- >>> f = NamedTemporaryFile(delete=False)
- >>> f.name
- '/tmp/tmptjujjt'
- >>> f.write(b"Hello World!\n")
- 13
- >>> f.close()
- >>> os.unlink(f.name)
- >>> os.path.exists(f.name)
- False
-
-The module uses a global variable that tell it how to construct a
-temporary name. They are initialized at the first call to any of the
-functions above. The caller may change them, but this is discouraged; use
-the appropriate function arguments, instead.
-
-
-.. data:: tempdir
+.. function:: gettempdir()
- When set to a value other than ``None``, this variable defines the
- default value for the *dir* argument to all the functions defined in this
- module.
+ Return the name of the directory used for temporary files. This
+ defines the default value for the *dir* argument to all functions
+ in this module.
- If ``tempdir`` is unset or ``None`` at any call to any of the above
- functions, Python searches a standard list of directories and sets
- *tempdir* to the first one which the calling user can create files in.
- The list is:
+ Python searches a standard list of directories to find one which
+ the calling user can create files in. The list is:
#. The directory named by the :envvar:`TMPDIR` environment variable.
@@ -225,19 +225,43 @@ the appropriate function arguments, instead.
#. As a last resort, the current working directory.
+ The result of this search is cached, see the description of
+ :data:`tempdir` below.
-.. function:: gettempdir()
+.. function:: gettempdirb()
- Return the directory currently selected to create temporary files in. If
- :data:`tempdir` is not ``None``, this simply returns its contents; otherwise,
- the search described above is performed, and the result returned.
+ Same as :func:`gettempdir` but the return value is in bytes.
+ .. versionadded:: 3.5
.. function:: gettempprefix()
Return the filename prefix used to create temporary files. This does not
contain the directory component.
+.. function:: gettempprefixb()
+
+ Same as :func:`gettempprefixb` but the return value is in bytes.
+
+ .. versionadded:: 3.5
+
+The module uses a global variable to store the name of the directory
+used for temporary files returned by :func:`gettempdir`. It can be
+set directly to override the selection process, but this is discouraged.
+All functions in this module take a *dir* argument which can be used
+to specify the directory and this is the recommend approach.
+
+.. data:: tempdir
+
+ When set to a value other than ``None``, this variable defines the
+ default value for the *dir* argument to all the functions defined in this
+ module.
+
+ If ``tempdir`` is unset or ``None`` at any call to any of the above
+ functions except :func:`gettempprefix` it is initalized following the
+ algorithm described in :func:`gettempdir`.
+
+.. _tempfile-examples:
Examples
--------
@@ -271,3 +295,42 @@ Here are some examples of typical usage of the :mod:`tempfile` module::
>>>
# directory and contents have been removed
+
+Deprecated functions and variables
+----------------------------------
+
+A historical way to create temporary files was to first generate a
+file name with the :func:`mktemp` function and then create a file
+using this name. Unfortunately this is not secure, because a different
+process may create a file with this name in the time between the call
+to :func:`mktemp` and the subsequent attempt to create the file by the
+first process. The solution is to combine the two steps and create the
+file immediately. This approach is used by :func:`mkstemp` and the
+other functions described above.
+
+.. function:: mktemp(suffix='', prefix='tmp', dir=None)
+
+ .. deprecated:: 2.3
+ Use :func:`mkstemp` instead.
+
+ Return an absolute pathname of a file that did not exist at the time the
+ call is made. The *prefix*, *suffix*, and *dir* arguments are the same
+ as for :func:`mkstemp`.
+
+ .. warning::
+
+ Use of this function may introduce a security hole in your program. By
+ the time you get around to doing anything with the file name it returns,
+ someone else may have beaten you to the punch. :func:`mktemp` usage can
+ be replaced easily with :func:`NamedTemporaryFile`, passing it the
+ ``delete=False`` parameter::
+
+ >>> f = NamedTemporaryFile(delete=False)
+ >>> f.name
+ '/tmp/tmptjujjt'
+ >>> f.write(b"Hello World!\n")
+ 13
+ >>> f.close()
+ >>> os.unlink(f.name)
+ >>> os.path.exists(f.name)
+ False
diff --git a/Doc/library/test.rst b/Doc/library/test.rst
index 974909e..f7ad475 100644
--- a/Doc/library/test.rst
+++ b/Doc/library/test.rst
@@ -568,6 +568,17 @@ The :mod:`test.support` module defines the following functions:
def load_tests(*args):
return load_package_tests(os.path.dirname(__file__), *args)
+.. function:: detect_api_mismatch(ref_api, other_api, *, ignore=()):
+
+ Returns the set of attributes, functions or methods of *ref_api* not
+ found on *other_api*, except for a defined list of items to be
+ ignored in this check specified in *ignore*.
+
+ By default this skips private attributes beginning with '_' but
+ includes all magic methods, i.e. those starting and ending in '__'.
+
+ .. versionadded:: 3.5
+
The :mod:`test.support` module defines the following classes:
diff --git a/Doc/library/time.rst b/Doc/library/time.rst
index 8d8b7d4..3d335c8 100644
--- a/Doc/library/time.rst
+++ b/Doc/library/time.rst
@@ -314,9 +314,9 @@ The module defines the following functions and data items:
processes running for more than 49 days. On more recent versions of Windows
and on other operating systems, :func:`monotonic` is system-wide.
- Availability: Windows, Mac OS X, Linux, FreeBSD, OpenBSD, Solaris.
-
.. versionadded:: 3.3
+ .. versionchanged:: 3.5
+ The function is now always available.
.. function:: perf_counter()
@@ -350,6 +350,11 @@ The module defines the following functions and data items:
requested by an arbitrary amount because of the scheduling of other activity
in the system.
+ .. versionchanged:: 3.5
+ The function now sleeps at least *secs* even if the sleep is interrupted
+ by a signal, except if the signal handler raises an exception (see
+ :pep:`475` for the rationale).
+
.. function:: strftime(format[, t])
diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst
index 70df409..d1051f6 100644
--- a/Doc/library/timeit.rst
+++ b/Doc/library/timeit.rst
@@ -59,18 +59,26 @@ Python Interface
The module defines three convenience functions and a public class:
-.. function:: timeit(stmt='pass', setup='pass', timer=<default timer>, number=1000000)
+.. function:: timeit(stmt='pass', setup='pass', timer=<default timer>, number=1000000, globals=None)
Create a :class:`Timer` instance with the given statement, *setup* code and
*timer* function and run its :meth:`.timeit` method with *number* executions.
+ The optional *globals* argument specifies a namespace in which to execute the
+ code.
+ .. versionchanged:: 3.5
+ The optional *globals* parameter was added.
-.. function:: repeat(stmt='pass', setup='pass', timer=<default timer>, repeat=3, number=1000000)
+
+.. function:: repeat(stmt='pass', setup='pass', timer=<default timer>, repeat=3, number=1000000, globals=None)
Create a :class:`Timer` instance with the given statement, *setup* code and
*timer* function and run its :meth:`.repeat` method with the given *repeat*
- count and *number* executions.
+ count and *number* executions. The optional *globals* argument specifies a
+ namespace in which to execute the code.
+ .. versionchanged:: 3.5
+ The optional *globals* parameter was added.
.. function:: default_timer()
@@ -80,7 +88,7 @@ The module defines three convenience functions and a public class:
:func:`time.perf_counter` is now the default timer.
-.. class:: Timer(stmt='pass', setup='pass', timer=<timer function>)
+.. class:: Timer(stmt='pass', setup='pass', timer=<timer function>, globals=None)
Class for timing execution speed of small code snippets.
@@ -88,7 +96,9 @@ The module defines three convenience functions and a public class:
for setup, and a timer function. Both statements default to ``'pass'``;
the timer function is platform-dependent (see the module doc string).
*stmt* and *setup* may also contain multiple statements separated by ``;``
- or newlines, as long as they don't contain multi-line string literals.
+ or newlines, as long as they don't contain multi-line string literals. The
+ statement will by default be executed within timeit's namespace; this behavior
+ can be controlled by passing a namespace to *globals*.
To measure the execution time of the first statement, use the :meth:`.timeit`
method. The :meth:`.repeat` method is a convenience to call :meth:`.timeit`
@@ -101,6 +111,8 @@ The module defines three convenience functions and a public class:
will then be executed by :meth:`.timeit`. Note that the timing overhead is a
little larger in this case because of the extra function calls.
+ .. versionchanged:: 3.5
+ The optional *globals* parameter was added.
.. method:: Timer.timeit(number=1000000)
@@ -169,7 +181,7 @@ Command-Line Interface
When called as a program from the command line, the following form is used::
- python -m timeit [-n N] [-r N] [-s S] [-t] [-c] [-h] [statement ...]
+ python -m timeit [-n N] [-r N] [-u U] [-s S] [-t] [-c] [-h] [statement ...]
Where the following options are understood:
@@ -198,6 +210,12 @@ Where the following options are understood:
use :func:`time.time` (deprecated)
+.. cmdoption:: -u, --unit=U
+
+ specify a time unit for timer output; can select usec, msec, or sec
+
+ .. versionadded:: 3.5
+
.. cmdoption:: -c, --clock
use :func:`time.clock` (deprecated)
@@ -320,3 +338,17 @@ To give the :mod:`timeit` module access to functions you define, you can pass a
if __name__ == '__main__':
import timeit
print(timeit.timeit("test()", setup="from __main__ import test"))
+
+Another option is to pass :func:`globals` to the *globals* parameter, which will cause the code
+to be executed within your current global namespace. This can be more convenient
+than individually specifying imports::
+
+ def f(x):
+ return x**2
+ def g(x):
+ return x**4
+ def h(x):
+ return x**8
+
+ import timeit
+ print(timeit.timeit('[func(42) for func in (f,g,h)]', globals=globals()))
diff --git a/Doc/library/token.rst b/Doc/library/token.rst
index 4cd7098..88fb38b 100644
--- a/Doc/library/token.rst
+++ b/Doc/library/token.rst
@@ -93,6 +93,7 @@ The token constants are:
DOUBLESLASH
DOUBLESLASHEQUAL
AT
+ ATEQUAL
RARROW
ELLIPSIS
OP
diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst
index 15fbedc..8d216d0 100644
--- a/Doc/library/traceback.rst
+++ b/Doc/library/traceback.rst
@@ -22,15 +22,20 @@ The module defines the following functions:
.. function:: print_tb(traceback, limit=None, file=None)
- Print up to *limit* stack trace entries from *traceback*. If *limit* is omitted
- or ``None``, all entries are printed. If *file* is omitted or ``None``, the
- output goes to ``sys.stderr``; otherwise it should be an open file or file-like
- object to receive the output.
+ Print up to *limit* stack trace entries from *traceback* (starting from
+ the caller's frame) if *limit* is positive. Otherwise, print the last
+ ``abs(limit)`` entries. If *limit* is omitted or ``None``, all entries
+ are printed. If *file* is omitted or ``None``, the output goes to
+ ``sys.stderr``; otherwise it should be an open file or file-like object
+ to receive the output.
+
+ .. versionchanged:: 3.5
+ Added negative *limit* support.
.. function:: print_exception(type, value, traceback, limit=None, file=None, chain=True)
- Print exception information and up to *limit* stack trace entries from
+ Print exception information and stack trace entries from
*traceback* to *file*. This differs from :func:`print_tb` in the following
ways:
@@ -41,6 +46,7 @@ The module defines the following functions:
prints the line where the syntax error occurred with a caret indicating the
approximate position of the error.
+ The optional *limit* argument has the same meaning as for :func:`print_tb`.
If *chain* is true (the default), then chained exceptions (the
:attr:`__cause__` or :attr:`__context__` attributes of the exception) will be
printed as well, like the interpreter itself does when printing an unhandled
@@ -49,33 +55,41 @@ The module defines the following functions:
.. function:: print_exc(limit=None, file=None, chain=True)
- This is a shorthand for ``print_exception(*sys.exc_info())``.
+ This is a shorthand for ``print_exception(*sys.exc_info(), limit, file,
+ chain)``.
.. function:: print_last(limit=None, file=None, chain=True)
This is a shorthand for ``print_exception(sys.last_type, sys.last_value,
- sys.last_traceback, limit, file)``. In general it will work only after
- an exception has reached an interactive prompt (see :data:`sys.last_type`).
+ sys.last_traceback, limit, file, chain)``. In general it will work only
+ after an exception has reached an interactive prompt (see
+ :data:`sys.last_type`).
.. function:: print_stack(f=None, limit=None, file=None)
- This function prints a stack trace from its invocation point. The optional *f*
- argument can be used to specify an alternate stack frame to start. The optional
- *limit* and *file* arguments have the same meaning as for
- :func:`print_exception`.
+ Print up to *limit* stack trace entries (starting from the invocation
+ point) if *limit* is positive. Otherwise, print the last ``abs(limit)``
+ entries. If *limit* is omitted or ``None``, all entries are printed.
+ The optional *f* argument can be used to specify an alternate stack frame
+ to start. The optional *file* argument has the same meaning as for
+ :func:`print_tb`.
+
+ .. versionchanged:: 3.5
+ Added negative *limit* support.
.. function:: extract_tb(traceback, limit=None)
- Return a list of up to *limit* "pre-processed" stack trace entries extracted
- from the traceback object *traceback*. It is useful for alternate formatting of
- stack traces. If *limit* is omitted or ``None``, all entries are extracted. A
- "pre-processed" stack trace entry is a 4-tuple (*filename*, *line number*,
- *function name*, *text*) representing the information that is usually printed
- for a stack trace. The *text* is a string with leading and trailing whitespace
- stripped; if the source is not available it is ``None``.
+ Return a list of "pre-processed" stack trace entries extracted from the
+ traceback object *traceback*. It is useful for alternate formatting of
+ stack traces. The optional *limit* argument has the same meaning as for
+ :func:`print_tb`. A "pre-processed" stack trace entry is a 4-tuple
+ (*filename*, *line number*, *function name*, *text*) representing the
+ information that is usually printed for a stack trace. The *text* is a
+ string with leading and trailing whitespace stripped; if the source is
+ not available it is ``None``.
.. function:: extract_stack(f=None, limit=None)
@@ -136,6 +150,162 @@ The module defines the following functions:
.. versionadded:: 3.4
+.. function:: walk_stack(f)
+
+ Walk a stack following ``f.f_back`` from the given frame, yielding the frame
+ and line number for each frame. If *f* is ``None``, the current stack is
+ used. This helper is used with :meth:`StackSummary.extract`.
+
+ .. versionadded:: 3.5
+
+.. function:: walk_tb(tb)
+
+ Walk a traceback following ``tb_next`` yielding the frame and line number
+ for each frame. This helper is used with :meth:`StackSummary.extract`.
+
+ .. versionadded:: 3.5
+
+The module also defines the following classes:
+
+:class:`TracebackException` Objects
+-----------------------------------
+
+.. versionadded:: 3.5
+
+:class:`TracebackException` objects are created from actual exceptions to
+capture data for later printing in a lightweight fashion.
+
+.. class:: TracebackException(exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False)
+
+ Capture an exception for later rendering. *limit*, *lookup_lines* and
+ *capture_locals* are as for the :class:`StackSummary` class.
+
+ Note that when locals are captured, they are also shown in the traceback.
+
+ .. attribute:: __cause__
+
+ A :class:`TracebackException` of the original ``__cause__``.
+
+ .. attribute:: __context__
+
+ A :class:`TracebackException` of the original ``__context__``.
+
+ .. attribute:: __suppress_context__
+
+ The ``__suppress_context__`` value from the original exception.
+
+ .. attribute:: stack
+
+ A :class:`StackSummary` representing the traceback.
+
+ .. attribute:: exc_type
+
+ The class of the original traceback.
+
+ .. attribute:: filename
+
+ For syntax errors - the file name where the error occurred.
+
+ .. attribute:: lineno
+
+ For syntax errors - the line number where the error occurred.
+
+ .. attribute:: text
+
+ For syntax errors - the text where the error occurred.
+
+ .. attribute:: offset
+
+ For syntax errors - the offset into the text where the error occurred.
+
+ .. attribute:: msg
+
+ For syntax errors - the compiler error message.
+
+ .. classmethod:: from_exception(exc, *, limit=None, lookup_lines=True, capture_locals=False)
+
+ Capture an exception for later rendering. *limit*, *lookup_lines* and
+ *capture_locals* are as for the :class:`StackSummary` class.
+
+ Note that when locals are captured, they are also shown in the traceback.
+
+ .. method:: format(*, chain=True)
+
+ Format the exception.
+
+ If *chain* is not ``True``, ``__cause__`` and ``__context__`` will not
+ be formatted.
+
+ The return value is a generator of strings, each ending in a newline and
+ some containing internal newlines. :func:`~traceback.print_exception`
+ is a wrapper around this method which just prints the lines to a file.
+
+ The message indicating which exception occurred is always the last
+ string in the output.
+
+ .. method:: format_exception_only()
+
+ Format the exception part of the traceback.
+
+ The return value is a generator of strings, each ending in a newline.
+
+ Normally, the generator emits a single string; however, for
+ :exc:`SyntaxError` exceptions, it emits several lines that (when
+ printed) display detailed information about where the syntax
+ error occurred.
+
+ The message indicating which exception occurred is always the last
+ string in the output.
+
+
+:class:`StackSummary` Objects
+-----------------------------
+
+.. versionadded:: 3.5
+
+:class:`StackSummary` objects represent a call stack ready for formatting.
+
+.. class:: StackSummary
+
+ .. classmethod:: extract(frame_gen, *, limit=None, lookup_lines=True, capture_locals=False)
+
+ Construct a :class:`StackSummary` object from a frame generator (such as
+ is returned by :func:`~traceback.walk_stack` or
+ :func:`~traceback.walk_tb`).
+
+ If *limit* is supplied, only this many frames are taken from *frame_gen*.
+ If *lookup_lines* is ``False``, the returned :class:`FrameSummary`
+ objects will not have read their lines in yet, making the cost of
+ creating the :class:`StackSummary` cheaper (which may be valuable if it
+ may not actually get formatted). If *capture_locals* is ``True`` the
+ local variables in each :class:`FrameSummary` are captured as object
+ representations.
+
+ .. classmethod:: from_list(a_list)
+
+ Construct a :class:`StackSummary` object from a supplied old-style list
+ of tuples. Each tuple should be a 4-tuple with filename, lineno, name,
+ line as the elements.
+
+
+:class:`FrameSummary` Objects
+-----------------------------
+
+.. versionadded:: 3.5
+
+:class:`FrameSummary` objects represent a single frame in a traceback.
+
+.. class:: FrameSummary(filename, lineno, name, lookup_line=True, locals=None, line=None)
+
+ Represent a single frame in the traceback or stack that is being formatted
+ or printed. It may optionally have a stringified version of the frames
+ locals included in it. If *lookup_line* is ``False``, the source code is not
+ looked up until the :class:`FrameSummary` has the :attr:`~FrameSummary.line`
+ attribute accessed (which also happens when casting it to a tuple).
+ :attr:`~FrameSummary.line` may be directly provided, and will prevent line
+ lookups happening at all. *locals* is an optional local variable
+ dictionary, and if supplied the variable representations are stored in the
+ summary for later display.
.. _traceback-example:
diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst
index a04a432..13c81a7 100644
--- a/Doc/library/tracemalloc.rst
+++ b/Doc/library/tracemalloc.rst
@@ -363,7 +363,7 @@ Filter
Filter on traces of memory blocks.
See the :func:`fnmatch.fnmatch` function for the syntax of
- *filename_pattern*. The ``'.pyc'`` and ``'.pyo'`` file extensions are
+ *filename_pattern*. The ``'.pyc'`` file extension is
replaced with ``'.py'``.
Examples:
@@ -374,6 +374,10 @@ Filter
:mod:`tracemalloc` module
* ``Filter(False, "<unknown>")`` excludes empty tracebacks
+
+ .. versionchanged:: 3.5
+ The ``'.pyo'`` file extension is no longer replaced with ``'.py'``.
+
.. attribute:: inclusive
If *inclusive* is ``True`` (include), only trace memory blocks allocated
@@ -631,4 +635,3 @@ Traceback
obj = Object()
File "test.py", line 12
tb = tracemalloc.get_object_traceback(f())
-
diff --git a/Doc/library/tulip_coro.dia b/Doc/library/tulip_coro.dia
index eccf4fa..70a33e3 100644
--- a/Doc/library/tulip_coro.dia
+++ b/Doc/library/tulip_coro.dia
Binary files differ
diff --git a/Doc/library/tulip_coro.png b/Doc/library/tulip_coro.png
index 65b6951..36ced8d 100644
--- a/Doc/library/tulip_coro.png
+++ b/Doc/library/tulip_coro.png
Binary files differ
diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst
index efe5c54..30dd6ef 100644
--- a/Doc/library/turtle.rst
+++ b/Doc/library/turtle.rst
@@ -1879,7 +1879,7 @@ Settings and special methods
>>> cv = screen.getcanvas()
>>> cv
- <turtle.ScrolledCanvas object at ...>
+ <turtle.ScrolledCanvas object ...>
.. function:: getshapes()
@@ -2351,6 +2351,9 @@ The demo scripts are:
| | pairwise in opposite | shapesize, tilt, |
| | direction | get_shapepoly, update |
+----------------+------------------------------+-----------------------+
+| sorting_animate| visual demonstration of | simple alignment, |
+| | different sorting methods | randomization |
++----------------+------------------------------+-----------------------+
| tree | a (graphical) breadth | :func:`clone` |
| | first tree (using generators)| |
+----------------+------------------------------+-----------------------+
diff --git a/Doc/library/types.rst b/Doc/library/types.rst
index abdb939..ff75de1 100644
--- a/Doc/library/types.rst
+++ b/Doc/library/types.rst
@@ -90,6 +90,14 @@ Standard names are defined for the following types:
generator function.
+.. data:: CoroutineType
+
+ The type of :term:`coroutine` objects, produced by calling a
+ function defined with an :keyword:`async def` statement.
+
+ .. versionadded:: 3.5
+
+
.. data:: CodeType
.. index:: builtin: compile
@@ -115,6 +123,10 @@ Standard names are defined for the following types:
The type of :term:`modules <module>`. Constructor takes the name of the
module to be created and optionally its :term:`docstring`.
+ .. note::
+ Use :func:`importlib.util.module_from_spec` to create a new module if you
+ wish to set the various import-controlled attributes.
+
.. attribute:: __doc__
The :term:`docstring` of the module. Defaults to ``None``.
@@ -267,3 +279,25 @@ Additional Utility Classes and Functions
attributes on the class with the same name (see Enum for an example).
.. versionadded:: 3.4
+
+
+Coroutine Utility Functions
+---------------------------
+
+.. function:: coroutine(gen_func)
+
+ This function transforms a :term:`generator` function into a
+ :term:`coroutine function` which returns a generator-based coroutine.
+ The generator-based coroutine is still a :term:`generator iterator`,
+ but is also considered to be a :term:`coroutine` object and is
+ :term:`awaitable`. However, it may not necessarily implement
+ the :meth:`__await__` method.
+
+ If *gen_func* is a generator function, it will be modified in-place.
+
+ If *gen_func* is not a generator function, it will be wrapped. If it
+ returns an instance of :class:`collections.abc.Generator`, the instance
+ will be wrapped in an *awaitable* proxy object. All other types
+ of objects will be returned as is.
+
+ .. versionadded:: 3.5
diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst
new file mode 100644
index 0000000..e09b647
--- /dev/null
+++ b/Doc/library/typing.rst
@@ -0,0 +1,433 @@
+:mod:`typing` --- Support for type hints
+========================================
+
+.. module:: typing
+ :synopsis: Support for type hints (see PEP 484).
+
+**Source code:** :source:`Lib/typing.py`
+
+--------------
+
+This module supports type hints as specified by :pep:`484`. The most
+fundamental support consists of the type :class:`Any`, :class:`Union`,
+:class:`Tuple`, :class:`Callable`, :class:`TypeVar`, and
+:class:`Generic`. For full specification please see :pep:`484`. For
+a simplified introduction to type hints see :pep:`483`.
+
+
+The function below takes and returns a string and is annotated as follows::
+
+ def greeting(name: str) -> str:
+ return 'Hello ' + name
+
+In the function ``greeting``, the argument ``name`` is expected to by of type
+:class:`str` and the return type :class:`str`. Subtypes are accepted as
+arguments.
+
+Type aliases
+------------
+
+A type alias is defined by assigning the type to the alias::
+
+ Vector = List[float]
+
+Callable
+--------
+
+Frameworks expecting callback functions of specific signatures might be
+type hinted using `Callable[[Arg1Type, Arg2Type], ReturnType]`.
+
+For example::
+
+ from typing import Callable
+
+ def feeder(get_next_item: Callable[[], str]) -> None:
+ # Body
+
+ def async_query(on_success: Callable[[int], None],
+ on_error: Callable[[int, Exception], None]) -> None:
+ # Body
+
+It is possible to declare the return type of a callable without specifying
+the call signature by substituting a literal ellipsis
+for the list of arguments in the type hint: ``Callable[..., ReturnType]``.
+``None`` as a type hint is a special case and is replaced by ``type(None)``.
+
+Generics
+--------
+
+Since type information about objects kept in containers cannot be statically
+inferred in a generic way, abstract base classes have been extended to support
+subscription to denote expected types for container elements.
+
+.. code-block:: python
+
+ from typing import Mapping, Sequence
+
+ def notify_by_email(employees: Sequence[Employee],
+ overrides: Mapping[str, str]) -> None: ...
+
+Generics can be parametrized by using a new factory available in typing
+called TypeVar.
+
+.. code-block:: python
+
+ from typing import Sequence, TypeVar
+
+ T = TypeVar('T') # Declare type variable
+
+ def first(l: Sequence[T]) -> T: # Generic function
+ return l[0]
+
+
+User-defined generic types
+--------------------------
+
+A user-defined class can be defined as a generic class.
+
+.. code-block:: python
+
+ from typing import TypeVar, Generic
+ from logging import Logger
+
+ T = TypeVar('T')
+
+ class LoggedVar(Generic[T]):
+ def __init__(self, value: T, name: str, logger: Logger) -> None:
+ self.name = name
+ self.logger = logger
+ self.value = value
+
+ def set(self, new: T) -> None:
+ self.log('Set ' + repr(self.value))
+ self.value = new
+
+ def get(self) -> T:
+ self.log('Get ' + repr(self.value))
+ return self.value
+
+ def log(self, message: str) -> None:
+ self.logger.info('{}: {}'.format(self.name, message))
+
+``Generic[T]`` as a base class defines that the class ``LoggedVar`` takes a
+single type parameter ``T`` . This also makes ``T`` valid as a type within the
+class body.
+
+The :class:`Generic` base class uses a metaclass that defines
+:meth:`__getitem__` so that ``LoggedVar[t]`` is valid as a type::
+
+ from typing import Iterable
+
+ def zero_all_vars(vars: Iterable[LoggedVar[int]]) -> None:
+ for var in vars:
+ var.set(0)
+
+A generic type can have any number of type variables, and type variables may
+be constrained::
+
+ from typing import TypeVar, Generic
+ ...
+
+ T = TypeVar('T')
+ S = TypeVar('S', int, str)
+
+ class StrangePair(Generic[T, S]):
+ ...
+
+Each type variable argument to :class:`Generic` must be distinct.
+This is thus invalid::
+
+ from typing import TypeVar, Generic
+ ...
+
+ T = TypeVar('T')
+
+ class Pair(Generic[T, T]): # INVALID
+ ...
+
+You can use multiple inheritance with `Generic`::
+
+ from typing import TypeVar, Generic, Sized
+
+ T = TypeVar('T')
+
+ class LinkedList(Sized, Generic[T]):
+ ...
+
+Subclassing a generic class without specifying type parameters assumes
+:class:`Any` for each position. In the following example, ``MyIterable`` is
+not generic but implicitly inherits from ``Iterable[Any]``::
+
+ from typing import Iterable
+
+ class MyIterable(Iterable): # Same as Iterable[Any]
+
+Generic metaclasses are not supported.
+
+The :class:`Any` type
+---------------------
+
+A special kind of type is :class:`Any`. Every type is a subtype of
+:class:`Any`. This is also true for the builtin type object. However, to the
+static type checker these are completely different.
+
+When the type of a value is :class:`object`, the type checker will reject
+almost all operations on it, and assigning it to a variable (or using it as a
+return value) of a more specialized type is a type error. On the other hand,
+when a value has type :class:`Any`, the type checker will allow all operations
+on it, and a value of type :class:`Any` can be assigned to a variable (or used
+as a return value) of a more constrained type.
+
+Default argument values
+-----------------------
+
+Use a literal ellipsis ``...`` to declare an argument as having a default value::
+
+ from typing import AnyStr
+
+ def foo(x: AnyStr, y: AnyStr = ...) -> AnyStr: ...
+
+
+Classes, functions, and decorators
+----------------------------------
+
+The module defines the following classes, functions and decorators:
+
+.. class:: Any
+
+ Special type indicating an unconstrained type.
+
+ * Any object is an instance of :class:`Any`.
+ * Any class is a subclass of :class:`Any`.
+ * As a special case, :class:`Any` and :class:`object` are subclasses of
+ each other.
+
+.. class:: TypeVar
+
+ Type variable.
+
+ Usage::
+
+ T = TypeVar('T') # Can be anything
+ A = TypeVar('A', str, bytes) # Must be str or bytes
+
+ Type variables exist primarily for the benefit of static type
+ checkers. They serve as the parameters for generic types as well
+ as for generic function definitions. See class Generic for more
+ information on generic types. Generic functions work as follows:
+
+ .. code-block:: python
+
+ def repeat(x: T, n: int) -> Sequence[T]:
+ """Return a list containing n references to x."""
+ return [x]*n
+
+ def longest(x: A, y: A) -> A:
+ """Return the longest of two strings."""
+ return x if len(x) >= len(y) else y
+
+ The latter example's signature is essentially the overloading
+ of ``(str, str) -> str`` and ``(bytes, bytes) -> bytes``. Also note
+ that if the arguments are instances of some subclass of :class:`str`,
+ the return type is still plain :class:`str`.
+
+ At runtime, ``isinstance(x, T)`` will raise :exc:`TypeError`. In general,
+ :func:`isinstance` and :func:`issublass` should not be used with types.
+
+ Type variables may be marked covariant or contravariant by passing
+ ``covariant=True`` or ``contravariant=True``. See :pep:`484` for more
+ details. By default type variables are invariant.
+
+.. class:: Union
+
+ Union type; ``Union[X, Y]`` means either X or Y.
+
+ To define a union, use e.g. ``Union[int, str]``. Details:
+
+ * The arguments must be types and there must be at least one.
+
+ * Unions of unions are flattened, e.g.::
+
+ Union[Union[int, str], float] == Union[int, str, float]
+
+ * Unions of a single argument vanish, e.g.::
+
+ Union[int] == int # The constructor actually returns int
+
+ * Redundant arguments are skipped, e.g.::
+
+ Union[int, str, int] == Union[int, str]
+
+ * When comparing unions, the argument order is ignored, e.g.::
+
+ Union[int, str] == Union[str, int]
+
+ * If :class:`Any` is present it is the sole survivor, e.g.::
+
+ Union[int, Any] == Any
+
+ * You cannot subclass or instantiate a union.
+
+ * You cannot write ``Union[X][Y]``
+
+ * You can use ``Optional[X]`` as a shorthand for ``Union[X, None]``.
+
+.. class:: Optional
+
+ Optional type.
+
+ ``Optional[X]`` is equivalent to ``Union[X, type(None)]``.
+
+.. class:: Tuple
+
+ Tuple type; ``Tuple[X, Y]`` is the is the type of a tuple of two items
+ with the first item of type X and the second of type Y.
+
+ Example: ``Tuple[T1, T2]`` is a tuple of two elements corresponding
+ to type variables T1 and T2. ``Tuple[int, float, str]`` is a tuple
+ of an int, a float and a string.
+
+ To specify a variable-length tuple of homogeneous type,
+ use literal ellipsis, e.g. ``Tuple[int, ...]``.
+
+.. class:: Callable
+
+ Callable type; ``Callable[[int], str]`` is a function of (int) -> str.
+
+ The subscription syntax must always be used with exactly two
+ values: the argument list and the return type. The argument list
+ must be a list of types; the return type must be a single type.
+
+ There is no syntax to indicate optional or keyword arguments,
+ such function types are rarely used as callback types.
+ ``Callable[..., ReturnType]`` could be used to type hint a callable
+ taking any number of arguments and returning ``ReturnType``.
+ A plain :class:`Callable` is equivalent to ``Callable[..., Any]``.
+
+.. class:: Generic
+
+ Abstract base class for generic types.
+
+ A generic type is typically declared by inheriting from an
+ instantiation of this class with one or more type variables.
+ For example, a generic mapping type might be defined as::
+
+ class Mapping(Generic[KT, VT]):
+ def __getitem__(self, key: KT) -> VT:
+ ...
+ # Etc.
+
+ This class can then be used as follows::
+
+ X = TypeVar('X')
+ Y = TypeVar('Y')
+ def lookup_name(mapping: Mapping[X, Y], key: X, default: Y) -> Y:
+ try:
+ return mapping[key]
+ except KeyError:
+ return default
+
+.. class:: Iterable(Generic[T_co])
+
+.. class:: Iterator(Iterable[T_co])
+
+.. class:: SupportsInt
+
+.. class:: SupportsFloat
+
+.. class:: SupportsAbs
+
+.. class:: SupportsRound
+
+.. class:: Reversible
+
+.. class:: Container(Generic[T_co])
+
+.. class:: AbstractSet(Sized, Iterable[T_co], Container[T_co])
+
+.. class:: MutableSet(AbstractSet[T])
+
+.. class:: Mapping(Sized, Iterable[KT_co], Container[KT_co], Generic[KT_co, VT_co])
+
+.. class:: MutableMapping(Mapping[KT, VT])
+
+.. class:: Sequence(Sized, Iterable[T_co], Container[T_co])
+
+.. class:: MutableSequence(Sequence[T])
+
+.. class:: ByteString(Sequence[int])
+
+.. class:: List(list, MutableSequence[T])
+
+.. class:: Set(set, MutableSet[T])
+
+.. class:: MappingView(Sized, Iterable[T_co])
+
+.. class:: KeysView(MappingView[KT_co], AbstractSet[KT_co])
+
+.. class:: ItemsView(MappingView, Generic[KT_co, VT_co])
+
+.. class:: ValuesView(MappingView[VT_co])
+
+.. class:: Dict(dict, MutableMapping[KT, VT])
+
+.. class:: Generator(Iterator[T_co], Generic[T_co, T_contra, V_co])
+
+.. class:: io
+
+ Wrapper namespace for IO generic classes.
+
+.. class:: re
+
+ Wrapper namespace for re type classes.
+
+.. function:: NamedTuple(typename, fields)
+
+ Typed version of namedtuple.
+
+ Usage::
+
+ Employee = typing.NamedTuple('Employee', [('name', str), 'id', int)])
+
+ This is equivalent to::
+
+ Employee = collections.namedtuple('Employee', ['name', 'id'])
+
+ The resulting class has one extra attribute: _field_types,
+ giving a dict mapping field names to types. (The field names
+ are in the _fields attribute, which is part of the namedtuple
+ API.)
+
+.. function:: cast(typ, val)
+
+ Cast a value to a type.
+
+ This returns the value unchanged. To the type checker this
+ signals that the return value has the designated type, but at
+ runtime we intentionally don't check anything (we want this
+ to be as fast as possible).
+
+.. function:: get_type_hints(obj)
+
+ Return type hints for a function or method object.
+
+ This is often the same as obj.__annotations__, but it handles
+ forward references encoded as string literals, and if necessary
+ adds Optional[t] if a default value equal to None is set.
+
+.. decorator:: no_type_check(arg)
+
+ Decorator to indicate that annotations are not type hints.
+
+ The argument must be a class or function; if it is a class, it
+ applies recursively to all methods defined in that class (but not
+ to methods defined in its superclasses or subclasses).
+
+ This mutates the function(s) in place.
+
+.. decorator:: no_type_check_decorator(decorator)
+
+ Decorator to give another decorator the @no_type_check effect.
+
+ This wraps the decorator with something that wraps the decorated
+ function in @no_type_check.
diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst
index 3b3d3a0..1430d9b 100644
--- a/Doc/library/unicodedata.rst
+++ b/Doc/library/unicodedata.rst
@@ -15,8 +15,8 @@
This module provides access to the Unicode Character Database (UCD) which
defines character properties for all Unicode characters. The data contained in
-this database is compiled from the `UCD version 6.3.0
-<http://www.unicode.org/Public/6.3.0/ucd>`_.
+this database is compiled from the `UCD version 8.0.0
+<http://www.unicode.org/Public/8.0.0/ucd>`_.
The module uses the same names and symbols as defined by Unicode
Standard Annex #44, `"Unicode Character Database"
@@ -166,6 +166,6 @@ Examples:
.. rubric:: Footnotes
-.. [#] http://www.unicode.org/Public/6.3.0/ucd/NameAliases.txt
+.. [#] http://www.unicode.org/Public/8.0.0/ucd/NameAliases.txt
-.. [#] http://www.unicode.org/Public/6.3.0/ucd/NamedSequences.txt
+.. [#] http://www.unicode.org/Public/8.0.0/ucd/NamedSequences.txt
diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst
index 13b3133..0f621c8 100644
--- a/Doc/library/unittest.mock.rst
+++ b/Doc/library/unittest.mock.rst
@@ -198,7 +198,7 @@ a :class:`MagicMock` for you. You can specify an alternative class of :class:`Mo
the *new_callable* argument to :func:`patch`.
-.. class:: Mock(spec=None, side_effect=None, return_value=DEFAULT, wraps=None, name=None, spec_set=None, **kwargs)
+.. class:: Mock(spec=None, side_effect=None, return_value=DEFAULT, wraps=None, name=None, spec_set=None, unsafe=False, **kwargs)
Create a new :class:`Mock` object. :class:`Mock` takes several optional arguments
that specify the behaviour of the Mock object:
@@ -235,6 +235,12 @@ the *new_callable* argument to :func:`patch`.
this is a new Mock (created on first access). See the
:attr:`return_value` attribute.
+ * *unsafe*: By default if any attribute starts with *assert* or
+ *assret* will raise an :exc:`AttributeError`. Passing ``unsafe=True``
+ will allow access to these attributes.
+
+ .. versionadded:: 3.5
+
* *wraps*: Item for the mock object to wrap. If *wraps* is not None then
calling the Mock will pass the call through to the wrapped object
(returning the real result). Attribute access on the mock will return a
@@ -315,6 +321,20 @@ the *new_callable* argument to :func:`patch`.
>>> calls = [call(4), call(2), call(3)]
>>> mock.assert_has_calls(calls, any_order=True)
+ .. method:: assert_not_called(*args, **kwargs)
+
+ Assert the mock was never called.
+
+ >>> m = Mock()
+ >>> m.hello.assert_not_called()
+ >>> obj = m.hello()
+ >>> m.hello.assert_not_called()
+ Traceback (most recent call last):
+ ...
+ AssertionError: Expected 'hello' to not have been called. Called 1 times.
+
+ .. versionadded:: 3.5
+
.. method:: reset_mock()
@@ -1032,6 +1052,12 @@ patch
default because it can be dangerous. With it switched on you can write
passing tests against APIs that don't actually exist!
+ .. note::
+
+ .. versionchanged:: 3.5
+ If you are patching builtins in a module then you don't
+ need to pass ``create=True``, it will be added by default.
+
Patch can be used as a :class:`TestCase` class decorator. It works by
decorating each test method in the class. This reduces the boilerplate
code when your test methods share a common patchings set. :func:`patch` finds
@@ -1403,6 +1429,22 @@ It is also possible to stop all patches which have been started by using
Stop all active patches. Only stops patches started with ``start``.
+.. _patch-builtins:
+
+patch builtins
+~~~~~~~~~~~~~~
+You can patch any builtins within a module. The following example patches
+builtin :func:`ord`:
+
+ >>> @patch('__main__.ord')
+ ... def test(mock_ord):
+ ... mock_ord.return_value = 101
+ ... print(ord('c'))
+ ...
+ >>> test()
+ 101
+
+
TEST_PREFIX
~~~~~~~~~~~
@@ -1587,7 +1629,7 @@ The full list of supported magic methods is:
* Context manager: ``__enter__`` and ``__exit__``
* Unary numeric methods: ``__neg__``, ``__pos__`` and ``__invert__``
* The numeric methods (including right hand and in-place variants):
- ``__add__``, ``__sub__``, ``__mul__``, ``__div__``, ``__truediv__``,
+ ``__add__``, ``__sub__``, ``__mul__``, ``__matmul__``, ``__div__``, ``__truediv__``,
``__floordiv__``, ``__mod__``, ``__divmod__``, ``__lshift__``,
``__rshift__``, ``__and__``, ``__xor__``, ``__or__``, and ``__pow__``
* Numeric conversion methods: ``__complex__``, ``__int__``, ``__float__``
@@ -2006,7 +2048,7 @@ mock_open
The mock of :meth:`~io.IOBase.read` changed to consume *read_data* rather
than returning it on each call.
- .. versionchanged:: 3.4.4
+ .. versionchanged:: 3.5
*read_data* is now reset on each call to the *mock*.
Using :func:`open` as a context manager is a great way to ensure your file handles
@@ -2023,7 +2065,7 @@ Mocking context managers with a :class:`MagicMock` is common enough and fiddly
enough that a helper function is useful.
>>> m = mock_open()
- >>> with patch('__main__.open', m, create=True):
+ >>> with patch('__main__.open', m):
... with open('foo', 'w') as h:
... h.write('some stuff')
...
@@ -2038,7 +2080,7 @@ enough that a helper function is useful.
And for reading files:
- >>> with patch('__main__.open', mock_open(read_data='bibble'), create=True) as m:
+ >>> with patch('__main__.open', mock_open(read_data='bibble')) as m:
... with open('foo') as h:
... result = h.read()
...
diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst
index a8a684c..1153f83 100644
--- a/Doc/library/unittest.rst
+++ b/Doc/library/unittest.rst
@@ -214,9 +214,16 @@ Command-line options
Stop the test run on the first error or failure.
+.. cmdoption:: --locals
+
+ Show local variables in tracebacks.
+
.. versionadded:: 3.2
The command-line options ``-b``, ``-c`` and ``-f`` were added.
+.. versionadded:: 3.5
+ The command-line option ``--locals``.
+
The command line can also be used for test discovery, for running all of the
tests in a project or just a subset.
@@ -1541,6 +1548,20 @@ Loading and running tests
:data:`unittest.defaultTestLoader`. Using a subclass or instance, however,
allows customization of some configurable properties.
+ :class:`TestLoader` objects have the following attributes:
+
+
+ .. attribute:: errors
+
+ A list of the non-fatal errors encountered while loading tests. Not reset
+ by the loader at any point. Fatal errors are signalled by the relevant
+ a method raising an exception to the caller. Non-fatal errors are also
+ indicated by a synthetic test that will raise the original error when
+ run.
+
+ .. versionadded:: 3.5
+
+
:class:`TestLoader` objects have the following methods:
@@ -1556,7 +1577,7 @@ Loading and running tests
case is created for that method instead.
- .. method:: loadTestsFromModule(module)
+ .. method:: loadTestsFromModule(module, pattern=None)
Return a suite of all tests cases contained in the given module. This
method searches *module* for classes derived from :class:`TestCase` and
@@ -1573,11 +1594,18 @@ Loading and running tests
If a module provides a ``load_tests`` function it will be called to
load the tests. This allows modules to customize test loading.
- This is the `load_tests protocol`_.
+ This is the `load_tests protocol`_. The *pattern* argument is passed as
+ the third argument to ``load_tests``.
.. versionchanged:: 3.2
Support for ``load_tests`` added.
+ .. versionchanged:: 3.5
+ The undocumented and unofficial *use_load_tests* default argument is
+ deprecated and ignored, although it is still accepted for backward
+ compatibility. The method also now accepts a keyword-only argument
+ *pattern* which is passed to ``load_tests`` as the third argument.
+
.. method:: loadTestsFromName(name, module=None)
@@ -1603,6 +1631,12 @@ Loading and running tests
The method optionally resolves *name* relative to the given *module*.
+ .. versionchanged:: 3.5
+ If an :exc:`ImportError` or :exc:`AttributeError` occurs while traversing
+ *name* then a synthetic test that raises that error when run will be
+ returned. These errors are included in the errors accumulated by
+ self.errors.
+
.. method:: loadTestsFromNames(names, module=None)
@@ -1629,18 +1663,22 @@ Loading and running tests
the start directory is not the top level directory then the top level
directory must be specified separately.
- If importing a module fails, for example due to a syntax error, then this
- will be recorded as a single error and discovery will continue. If the
- import failure is due to :exc:`SkipTest` being raised, it will be recorded
- as a skip instead of an error.
+ If importing a module fails, for example due to a syntax error, then
+ this will be recorded as a single error and discovery will continue. If
+ the import failure is due to :exc:`SkipTest` being raised, it will be
+ recorded as a skip instead of an error.
- If a test package name (directory with :file:`__init__.py`) matches the
- pattern then the package will be checked for a ``load_tests``
- function. If this exists then it will be called with *loader*, *tests*,
- *pattern*.
+ If a package (a directory containing a file named :file:`__init__.py`) is
+ found, the package will be checked for a ``load_tests`` function. If this
+ exists then it will be called
+ ``package.load_tests(loader, tests, pattern)``. Test discovery takes care
+ to ensure that a package is only checked for tests once during an
+ invocation, even if the load_tests function itself calls
+ ``loader.discover``.
- If ``load_tests`` exists then discovery does *not* recurse into the package,
- ``load_tests`` is responsible for loading all tests in the package.
+ If ``load_tests`` exists then discovery does *not* recurse into the
+ package, ``load_tests`` is responsible for loading all tests in the
+ package.
The pattern is deliberately not stored as a loader attribute so that
packages can continue discovery themselves. *top_level_dir* is stored so
@@ -1659,6 +1697,11 @@ Loading and running tests
the same even if the underlying file system's ordering is not
dependent on file name.
+ .. versionchanged:: 3.5
+ Found packages are now checked for ``load_tests`` regardless of
+ whether their path matches *pattern*, because it is impossible for
+ a package name to match the default pattern.
+
The following attributes of a :class:`TestLoader` can be configured either by
subclassing or assignment on an instance:
@@ -1741,12 +1784,10 @@ Loading and running tests
Set to ``True`` when the execution of tests should stop by :meth:`stop`.
-
.. attribute:: testsRun
The total number of tests run so far.
-
.. attribute:: buffer
If set to true, ``sys.stdout`` and ``sys.stderr`` will be buffered in between
@@ -1756,7 +1797,6 @@ Loading and running tests
.. versionadded:: 3.2
-
.. attribute:: failfast
If set to true :meth:`stop` will be called on the first failure or error,
@@ -1764,6 +1804,11 @@ Loading and running tests
.. versionadded:: 3.2
+ .. attribute:: tb_locals
+
+ If set to true then local variables will be shown in tracebacks.
+
+ .. versionadded:: 3.5
.. method:: wasSuccessful()
@@ -1774,7 +1819,6 @@ Loading and running tests
Returns ``False`` if there were any :attr:`unexpectedSuccesses`
from tests marked with the :func:`expectedFailure` decorator.
-
.. method:: stop()
This method can be called to signal that the set of tests being run should
@@ -1906,12 +1950,14 @@ Loading and running tests
.. class:: TextTestRunner(stream=None, descriptions=True, verbosity=1, failfast=False, \
- buffer=False, resultclass=None, warnings=None)
+ buffer=False, resultclass=None, warnings=None, *, tb_locals=False)
A basic test runner implementation that outputs results to a stream. If *stream*
is ``None``, the default, :data:`sys.stderr` is used as the output stream. This class
has a few configurable parameters, but is essentially very simple. Graphical
- applications which run test suites should provide alternate implementations.
+ applications which run test suites should provide alternate implementations. Such
+ implementations should accept ``**kwargs`` as the interface to construct runners
+ changes when features are added to unittest.
By default this runner shows :exc:`DeprecationWarning`,
:exc:`PendingDeprecationWarning`, :exc:`ResourceWarning` and
@@ -1930,6 +1976,9 @@ Loading and running tests
The default stream is set to :data:`sys.stderr` at instantiation time rather
than import time.
+ .. versionchanged:: 3.5
+ Added the tb_locals parameter.
+
.. method:: _makeResult()
This method returns the instance of ``TestResult`` used by :meth:`run`.
@@ -2027,7 +2076,10 @@ test runs or test discovery by implementing a function called ``load_tests``.
If a test module defines ``load_tests`` it will be called by
:meth:`TestLoader.loadTestsFromModule` with the following arguments::
- load_tests(loader, standard_tests, None)
+ load_tests(loader, standard_tests, pattern)
+
+where *pattern* is passed straight through from ``loadTestsFromModule``. It
+defaults to ``None``.
It should return a :class:`TestSuite`.
@@ -2049,21 +2101,12 @@ A typical ``load_tests`` function that loads tests from a specific set of
suite.addTests(tests)
return suite
-If discovery is started, either from the command line or by calling
-:meth:`TestLoader.discover`, with a pattern that matches a package
-name then the package :file:`__init__.py` will be checked for ``load_tests``.
-
-.. note::
-
- The default pattern is ``'test*.py'``. This matches all Python files
- that start with ``'test'`` but *won't* match any test directories.
-
- A pattern like ``'test*'`` will match test packages as well as
- modules.
-
-If the package :file:`__init__.py` defines ``load_tests`` then it will be
-called and discovery not continued into the package. ``load_tests``
-is called with the following arguments::
+If discovery is started in a directory containing a package, either from the
+command line or by calling :meth:`TestLoader.discover`, then the package
+:file:`__init__.py` will be checked for ``load_tests``. If that function does
+not exist, discovery will recurse into the package as though it were just
+another directory. Otherwise, discovery of the package's tests will be left up
+to ``load_tests`` which is called with the following arguments::
load_tests(loader, standard_tests, pattern)
@@ -2082,6 +2125,11 @@ continue (and potentially modify) test discovery. A 'do nothing'
standard_tests.addTests(package_tests)
return standard_tests
+.. versionchanged:: 3.5
+ Discovery no longer checks package names for matching *pattern* due to the
+ impossibility of package names matching the default pattern.
+
+
Class and Module Fixtures
-------------------------
diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst
index fbbabca..40098d0 100644
--- a/Doc/library/urllib.parse.rst
+++ b/Doc/library/urllib.parse.rst
@@ -270,6 +270,11 @@ or on combining URL components into a URL string.
:func:`urlunsplit`, removing possible *scheme* and *netloc* parts.
+ .. versionchanged:: 3.5
+
+ Behaviour updated to match the semantics defined in :rfc:`3986`.
+
+
.. function:: urldefrag(url)
If *url* contains a fragment identifier, return a modified version of *url*
@@ -516,7 +521,8 @@ task isn't already covered by the URL parsing functions above.
Example: ``unquote_to_bytes('a%26%EF')`` yields ``b'a&\xef'``.
-.. function:: urlencode(query, doseq=False, safe='', encoding=None, errors=None)
+.. function:: urlencode(query, doseq=False, safe='', encoding=None, \
+ errors=None, quote_via=quote_plus)
Convert a mapping object or a sequence of two-element tuples, which may
contain :class:`str` or :class:`bytes` objects, to a "percent-encoded"
@@ -525,8 +531,16 @@ task isn't already covered by the URL parsing functions above.
properly encoded to bytes, otherwise it would result in a :exc:`TypeError`.
The resulting string is a series of ``key=value`` pairs separated by ``'&'``
- characters, where both *key* and *value* are quoted using :func:`quote_plus`
- above. When a sequence of two-element tuples is used as the *query*
+ characters, where both *key* and *value* are quoted using the *quote_via*
+ function. By default, :func:`quote_plus` is used to quote the values, which
+ means spaces are quoted as a ``'+'`` character and '/' characters are
+ encoded as ``%2F``, which follows the standard for GET requests
+ (``application/x-www-form-urlencoded``). An alternate function that can be
+ passed as *quote_via* is :func:`quote`, which will encode spaces as ``%20``
+ and not encode '/' characters. For maximum control of what is quoted, use
+ ``quote`` and specify a value for *safe*.
+
+ When a sequence of two-element tuples is used as the *query*
argument, the first element of each tuple is a key and the second is a
value. The value element in itself can be a sequence and in that case, if
the optional parameter *doseq* is evaluates to *True*, individual
@@ -535,7 +549,7 @@ task isn't already covered by the URL parsing functions above.
string will match the order of parameter tuples in the sequence.
The *safe*, *encoding*, and *errors* parameters are passed down to
- :func:`quote_plus` (the *encoding* and *errors* parameters are only passed
+ *quote_via* (the *encoding* and *errors* parameters are only passed
when a query element is a :class:`str`).
To reverse this encoding process, :func:`parse_qs` and :func:`parse_qsl` are
@@ -547,6 +561,9 @@ task isn't already covered by the URL parsing functions above.
.. versionchanged:: 3.2
Query parameter supports bytes and string objects.
+ .. versionadded:: 3.5
+ *quote_via* parameter.
+
.. seealso::
diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst
index 75b95d9..349ba70 100644
--- a/Doc/library/urllib.request.rst
+++ b/Doc/library/urllib.request.rst
@@ -115,6 +115,7 @@ The :mod:`urllib.request` module defines the following functions:
.. versionchanged:: 3.4.3
*context* was added.
+
.. function:: install_opener(opener)
Install an :class:`OpenerDirector` instance as the default global opener.
@@ -287,13 +288,37 @@ The following classes are provided:
fits.
+.. class:: HTTPPasswordMgrWithPriorAuth()
+
+ A variant of :class:`HTTPPasswordMgrWithDefaultRealm` that also has a
+ database of ``uri -> is_authenticated`` mappings. Can be used by a
+ BasicAuth handler to determine when to send authentication credentials
+ immediately instead of waiting for a ``401`` response first.
+
+ .. versionadded:: 3.5
+
+
.. class:: AbstractBasicAuthHandler(password_mgr=None)
This is a mixin class that helps with HTTP authentication, both to the remote
host and to a proxy. *password_mgr*, if given, should be something that is
compatible with :class:`HTTPPasswordMgr`; refer to section
:ref:`http-password-mgr` for information on the interface that must be
- supported.
+ supported. If *passwd_mgr* also provides ``is_authenticated`` and
+ ``update_authenticated`` methods (see
+ :ref:`http-password-mgr-with-prior-auth`), then the handler will use the
+ ``is_authenticated`` result for a given URI to determine whether or not to
+ send authentication credentials with the request. If ``is_authenticated``
+ returns ``True`` for the URI, credentials are sent. If ``is_authenticated``
+ is ``False``, credentials are not sent, and then if a ``401`` response is
+ received the request is re-sent with the authentication credentials. If
+ authentication succeeds, ``update_authenticated`` is called to set
+ ``is_authenticated`` ``True`` for the URI, so that subsequent requests to
+ the URI or any of its super-URIs will automatically include the
+ authentication credentials.
+
+ .. versionadded:: 3.5
+ Added ``is_authenticated`` support.
.. class:: HTTPBasicAuthHandler(password_mgr=None)
@@ -845,6 +870,42 @@ These methods are available on :class:`HTTPPasswordMgr` and
searched if the given *realm* has no matching user/password.
+.. _http-password-mgr-with-prior-auth:
+
+HTTPPasswordMgrWithPriorAuth Objects
+------------------------------------
+
+This password manager extends :class:`HTTPPasswordMgrWithDefaultRealm` to support
+tracking URIs for which authentication credentials should always be sent.
+
+
+.. method:: HTTPPasswordMgrWithPriorAuth.add_password(realm, uri, user, \
+ passwd, is_authenticated=False)
+
+ *realm*, *uri*, *user*, *passwd* are as for
+ :meth:`HTTPPasswordMgr.add_password`. *is_authenticated* sets the initial
+ value of the ``is_authenticated`` flag for the given URI or list of URIs.
+ If *is_authenticated* is specified as ``True``, *realm* is ignored.
+
+
+.. method:: HTTPPasswordMgr.find_user_password(realm, authuri)
+
+ Same as for :class:`HTTPPasswordMgrWithDefaultRealm` objects
+
+
+.. method:: HTTPPasswordMgrWithPriorAuth.update_authenticated(self, uri, \
+ is_authenticated=False)
+
+ Update the ``is_authenticated`` flag for the given *uri* or list
+ of URIs.
+
+
+.. method:: HTTPPasswordMgrWithPriorAuth.is_authenticated(self, authuri)
+
+ Returns the current state of the ``is_authenticated`` flag for
+ the given URI.
+
+
.. _abstract-basic-auth-handler:
AbstractBasicAuthHandler Objects
diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst
index cc883b1..2e16077 100644
--- a/Doc/library/weakref.rst
+++ b/Doc/library/weakref.rst
@@ -258,7 +258,7 @@ These method have the same issues as the and :meth:`keyrefs` method of
are called in reverse order of creation.
A finalizer will never invoke its callback during the later part of
- the interpreter shutdown when module globals are liable to have
+ the :term:`interpreter shutdown` when module globals are liable to have
been replaced by :const:`None`.
.. method:: __call__()
@@ -527,8 +527,8 @@ follows::
Starting with Python 3.4, :meth:`__del__` methods no longer prevent
reference cycles from being garbage collected, and module globals are
-no longer forced to :const:`None` during interpreter shutdown. So this
-code should work without any issues on CPython.
+no longer forced to :const:`None` during :term:`interpreter shutdown`.
+So this code should work without any issues on CPython.
However, handling of :meth:`__del__` methods is notoriously implementation
specific, since it depends on internal details of the interpreter's garbage
diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst
index a9e19da..de74c30 100644
--- a/Doc/library/wsgiref.rst
+++ b/Doc/library/wsgiref.rst
@@ -184,10 +184,11 @@ This module provides a single class, :class:`Headers`, for convenient
manipulation of WSGI response headers using a mapping-like interface.
-.. class:: Headers(headers)
+.. class:: Headers([headers])
Create a mapping-like object wrapping *headers*, which must be a list of header
- name/value tuples as described in :pep:`3333`.
+ name/value tuples as described in :pep:`3333`. The default value of *headers* is
+ an empty list.
:class:`Headers` objects support typical mapping operations including
:meth:`__getitem__`, :meth:`get`, :meth:`__setitem__`, :meth:`setdefault`,
@@ -251,6 +252,10 @@ manipulation of WSGI response headers using a mapping-like interface.
Content-Disposition: attachment; filename="bud.gif"
+ .. versionchanged:: 3.5
+ *headers* parameter is optional.
+
+
:mod:`wsgiref.simple_server` -- a simple WSGI HTTP server
---------------------------------------------------------
diff --git a/Doc/library/xml.sax.reader.rst b/Doc/library/xml.sax.reader.rst
index 3ab6063..31ca260 100644
--- a/Doc/library/xml.sax.reader.rst
+++ b/Doc/library/xml.sax.reader.rst
@@ -100,8 +100,10 @@ The :class:`XMLReader` interface supports the following methods:
system identifier (a string identifying the input source -- typically a file
name or an URL), a file-like object, or an :class:`InputSource` object. When
:meth:`parse` returns, the input is completely processed, and the parser object
- can be discarded or reset. As a limitation, the current implementation only
- accepts byte streams; processing of character streams is for further study.
+ can be discarded or reset.
+
+ .. versionchanged:: 3.5
+ Added support of character streams.
.. method:: XMLReader.getContentHandler()
@@ -288,8 +290,7 @@ InputSource Objects
.. method:: InputSource.setByteStream(bytefile)
- Set the byte stream (a Python file-like object which does not perform
- byte-to-character conversion) for this input source.
+ Set the byte stream (a :term:`binary file`) for this input source.
The SAX parser will ignore this if there is also a character stream specified,
but it will use a byte stream in preference to opening a URI connection itself.
@@ -308,8 +309,7 @@ InputSource Objects
.. method:: InputSource.setCharacterStream(charfile)
- Set the character stream for this input source. (The stream must be a Python 1.6
- Unicode-wrapped file-like that performs conversion to strings.)
+ Set the character stream (a :term:`text file`) for this input source.
If there is a character stream specified, the SAX parser will ignore any byte
stream and will not attempt to open a URI connection to the system identifier.
diff --git a/Doc/library/xml.sax.rst b/Doc/library/xml.sax.rst
index e95d6b0..55f9799 100644
--- a/Doc/library/xml.sax.rst
+++ b/Doc/library/xml.sax.rst
@@ -47,7 +47,11 @@ The convenience functions are:
.. function:: parseString(string, handler, error_handler=handler.ErrorHandler())
Similar to :func:`parse`, but parses from a buffer *string* received as a
- parameter.
+ parameter. *string* must be a :class:`str` instance or a
+ :term:`bytes-like object`.
+
+ .. versionchanged:: 3.5
+ Added support of :class:`str` instances.
A typical SAX application uses three kinds of objects: readers, handlers and
input sources. "Reader" in this context is another term for parser, i.e. some
diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst
index cc5e83a..e199931 100644
--- a/Doc/library/xmlrpc.client.rst
+++ b/Doc/library/xmlrpc.client.rst
@@ -27,7 +27,7 @@ between conformable Python objects and XML on the wire.
constructed data. If you need to parse untrusted or unauthenticated data see
:ref:`xml-vulnerabilities`.
-.. versionchanged:: 3.4.3
+.. versionchanged:: 3.5
For https URIs, :mod:`xmlrpc.client` now performs all the necessary
certificate and hostname checks by default
@@ -129,7 +129,7 @@ between conformable Python objects and XML on the wire.
:class:`Server` is retained as an alias for :class:`ServerProxy` for backwards
compatibility. New code should use :class:`ServerProxy`.
- .. versionchanged:: 3.4.3
+ .. versionchanged:: 3.5
Added the *context* argument.
@@ -200,6 +200,11 @@ grouped under the reserved :attr:`system` attribute:
no such string is available, an empty string is returned. The documentation
string may contain HTML markup.
+.. versionchanged:: 3.5
+
+ Instances of :class:`ServerProxy` support the :term:`context manager` protocol
+ for closing the underlying transport.
+
A working example follows. The server code::
@@ -217,9 +222,9 @@ The client code for the preceding server::
import xmlrpc.client
- proxy = xmlrpc.client.ServerProxy("http://localhost:8000/")
- print("3 is even: %s" % str(proxy.is_even(3)))
- print("100 is even: %s" % str(proxy.is_even(100)))
+ with xmlrpc.client.ServerProxy("http://localhost:8000/") as proxy:
+ print("3 is even: %s" % str(proxy.is_even(3)))
+ print("100 is even: %s" % str(proxy.is_even(100)))
.. _datetime-objects:
@@ -527,14 +532,14 @@ Example of Client Usage
from xmlrpc.client import ServerProxy, Error
# server = ServerProxy("http://localhost:8000") # local server
- server = ServerProxy("http://betty.userland.com")
+ with ServerProxy("http://betty.userland.com") as proxy:
- print(server)
+ print(proxy)
- try:
- print(server.examples.getStateName(41))
- except Error as v:
- print("ERROR", v)
+ try:
+ print(proxy.examples.getStateName(41))
+ except Error as v:
+ print("ERROR", v)
To access an XML-RPC server through a proxy, you need to define a custom
transport. The following example shows how:
diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst
new file mode 100644
index 0000000..b4484c8
--- /dev/null
+++ b/Doc/library/zipapp.rst
@@ -0,0 +1,259 @@
+:mod:`zipapp` --- Manage executable python zip archives
+=======================================================
+
+.. module:: zipapp
+ :synopsis: Manage executable python zip archives
+
+
+.. index::
+ single: Executable Zip Files
+
+.. versionadded:: 3.5
+
+**Source code:** :source:`Lib/zipapp.py`
+
+--------------
+
+This module provides tools to manage the creation of zip files containing
+Python code, which can be :ref:`executed directly by the Python interpreter
+<using-on-interface-options>`. The module provides both a
+:ref:`zipapp-command-line-interface` and a :ref:`zipapp-python-api`.
+
+
+Basic Example
+-------------
+
+The following example shows how the :ref:`command-line-interface`
+can be used to create an executable archive from a directory containing
+Python code. When run, the archive will execute the ``main`` function from
+the module ``myapp`` in the archive.
+
+.. code-block:: sh
+
+ $ python -m zipapp myapp -m "myapp:main"
+ $ python myapp.pyz
+ <output from myapp>
+
+
+.. _zipapp-command-line-interface:
+
+Command-Line Interface
+----------------------
+
+When called as a program from the command line, the following form is used:
+
+.. code-block:: sh
+
+ $ python -m zipapp source [options]
+
+If *source* is a directory, this will create an archive from the contents of
+*source*. If *source* is a file, it should be an archive, and it will be
+copied to the target archive (or the contents of its shebang line will be
+displayed if the --info option is specified).
+
+The following options are understood:
+
+.. program:: zipapp
+
+.. cmdoption:: -o <output>, --output=<output>
+
+ Write the output to a file named *output*. If this option is not specified,
+ the output filename will be the same as the input *source*, with the
+ extension ``.pyz`` added. If an explicit filename is given, it is used as
+ is (so a ``.pyz`` extension should be included if required).
+
+ An output filename must be specified if the *source* is an archive (and in
+ that case, *output* must not be the same as *source*).
+
+.. cmdoption:: -p <interpreter>, --python=<interpreter>
+
+ Add a ``#!`` line to the archive specifying *interpreter* as the command
+ to run. Also, on POSIX, make the archive executable. The default is to
+ write no ``#!`` line, and not make the file executable.
+
+.. cmdoption:: -m <mainfn>, --main=<mainfn>
+
+ Write a ``__main__.py`` file to the archive that executes *mainfn*. The
+ *mainfn* argument should have the form "pkg.mod:fn", where "pkg.mod" is a
+ package/module in the archive, and "fn" is a callable in the given module.
+ The ``__main__.py`` file will execute that callable.
+
+ :option:`--main` cannot be specified when copying an archive.
+
+.. cmdoption:: --info
+
+ Display the interpreter embedded in the archive, for diagnostic purposes. In
+ this case, any other options are ignored and SOURCE must be an archive, not a
+ directory.
+
+.. cmdoption:: -h, --help
+
+ Print a short usage message and exit.
+
+
+.. _zipapp-python-api:
+
+Python API
+----------
+
+The module defines two convenience functions:
+
+
+.. function:: create_archive(source, target=None, interpreter=None, main=None)
+
+ Create an application archive from *source*. The source can be any
+ of the following:
+
+ * The name of a directory, or a :class:`pathlib.Path` object referring
+ to a directory, in which case a new application archive will be
+ created from the content of that directory.
+ * The name of an existing application archive file, or a :class:`pathlib.Path`
+ object referring to such a file, in which case the file is copied to
+ the target (modifying it to reflect the value given for the *interpreter*
+ argument). The file name should include the ``.pyz`` extension, if required.
+ * A file object open for reading in bytes mode. The content of the
+ file should be an application archive, and the file object is
+ assumed to be positioned at the start of the archive.
+
+ The *target* argument determines where the resulting archive will be
+ written:
+
+ * If it is the name of a file, or a :class:`pathlb.Path` object,
+ the archive will be written to that file.
+ * If it is an open file object, the archive will be written to that
+ file object, which must be open for writing in bytes mode.
+ * If the target is omitted (or None), the source must be a directory
+ and the target will be a file with the same name as the source, with
+ a ``.pyz`` extension added.
+
+ The *interpreter* argument specifies the name of the Python
+ interpreter with which the archive will be executed. It is written as
+ a "shebang" line at the start of the archive. On POSIX, this will be
+ interpreted by the OS, and on Windows it will be handled by the Python
+ launcher. Omitting the *interpreter* results in no shebang line being
+ written. If an interpreter is specified, and the target is a
+ filename, the executable bit of the target file will be set.
+
+ The *main* argument specifies the name of a callable which will be
+ used as the main program for the archive. It can only be specified if
+ the source is a directory, and the source does not already contain a
+ ``__main__.py`` file. The *main* argument should take the form
+ "pkg.module:callable" and the archive will be run by importing
+ "pkg.module" and executing the given callable with no arguments. It
+ is an error to omit *main* if the source is a directory and does not
+ contain a ``__main__.py`` file, as otherwise the resulting archive
+ would not be executable.
+
+ If a file object is specified for *source* or *target*, it is the
+ caller's responsibility to close it after calling create_archive.
+
+ When copying an existing archive, file objects supplied only need
+ ``read`` and ``readline``, or ``write`` methods. When creating an
+ archive from a directory, if the target is a file object it will be
+ passed to the ``zipfile.ZipFile`` class, and must supply the methods
+ needed by that class.
+
+.. function:: get_interpreter(archive)
+
+ Return the interpreter specified in the ``#!`` line at the start of the
+ archive. If there is no ``#!`` line, return :const:`None`.
+ The *archive* argument can be a filename or a file-like object open
+ for reading in bytes mode. It is assumed to be at the start of the archive.
+
+
+.. _zipapp-examples:
+
+Examples
+--------
+
+Pack up a directory into an archive, and run it.
+
+.. code-block:: sh
+
+ $ python -m zipapp myapp
+ $ python myapp.pyz
+ <output from myapp>
+
+The same can be done using the :func:`create_archive` functon::
+
+ >>> import zipapp
+ >>> zipapp.create_archive('myapp.pyz', 'myapp')
+
+To make the application directly executable on POSIX, specify an interpreter
+to use.
+
+.. code-block:: sh
+
+ $ python -m zipapp myapp -p "/usr/bin/env python"
+ $ ./myapp.pyz
+ <output from myapp>
+
+To replace the shebang line on an existing archive, create a modified archive
+using the :func:`create_archive` function::
+
+ >>> import zipapp
+ >>> zipapp.create_archive('old_archive.pyz', 'new_archive.pyz', '/usr/bin/python3')
+
+To update the file in place, do the replacement in memory using a :class:`BytesIO`
+object, and then overwrite the source afterwards. Note that there is a risk
+when overwriting a file in place that an error will result in the loss of
+the original file. This code does not protect against such errors, but
+production code should do so. Also, this method will only work if the archive
+fits in memory::
+
+ >>> import zipapp
+ >>> import io
+ >>> temp = io.BytesIO()
+ >>> zipapp.create_archive('myapp.pyz', temp, '/usr/bin/python2')
+ >>> with open('myapp.pyz', 'wb') as f:
+ >>> f.write(temp.getvalue())
+
+Note that if you specify an interpreter and then distribute your application
+archive, you need to ensure that the interpreter used is portable. The Python
+launcher for Windows supports most common forms of POSIX ``#!`` line, but there
+are other issues to consider:
+
+* If you use "/usr/bin/env python" (or other forms of the "python" command,
+ such as "/usr/bin/python"), you need to consider that your users may have
+ either Python 2 or Python 3 as their default, and write your code to work
+ under both versions.
+* If you use an explicit version, for example "/usr/bin/env python3" your
+ application will not work for users who do not have that version. (This
+ may be what you want if you have not made your code Python 2 compatible).
+* There is no way to say "python X.Y or later", so be careful of using an
+ exact version like "/usr/bin/env python3.4" as you will need to change your
+ shebang line for users of Python 3.5, for example.
+
+The Python Zip Application Archive Format
+-----------------------------------------
+
+Python has been able to execute zip files which contain a ``__main__.py`` file
+since version 2.6. In order to be executed by Python, an application archive
+simply has to be a standard zip file containing a ``__main__.py`` file which
+will be run as the entry point for the application. As usual for any Python
+script, the parent of the script (in this case the zip file) will be placed on
+:data:`sys.path` and thus further modules can be imported from the zip file.
+
+The zip file format allows arbitrary data to be prepended to a zip file. The
+zip application format uses this ability to prepend a standard POSIX "shebang"
+line to the file (``#!/path/to/interpreter``).
+
+Formally, the Python zip application format is therefore:
+
+1. An optional shebang line, containing the characters ``b'#!'`` followed by an
+ interpreter name, and then a newline (``b'\n'``) character. The interpreter
+ name can be anything acceptable to the OS "shebang" processing, or the Python
+ launcher on Windows. The interpreter should be encoded in UTF-8 on Windows,
+ and in :func:`sys.getfilesystemencoding()` on POSIX.
+2. Standard zipfile data, as generated by the :mod:`zipfile` module. The
+ zipfile content *must* include a file called ``__main__.py`` (which must be
+ in the "root" of the zipfile - i.e., it cannot be in a subdirectory). The
+ zipfile data can be compressed or uncompressed.
+
+If an application archive has a shebang line, it may have the executable bit set
+on POSIX systems, to allow it to be executed directly.
+
+There is no requirement that the tools in this module are used to create
+application archives - the module is a convenience, but archives in the above
+format created by any means are acceptable to Python.
+
diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst
index 10a094f..d40315e 100644
--- a/Doc/library/zipfile.rst
+++ b/Doc/library/zipfile.rst
@@ -134,12 +134,16 @@ ZipFile Objects
Open a ZIP file, where *file* can be either a path to a file (a string) or a
file-like object. The *mode* parameter should be ``'r'`` to read an existing
- file, ``'w'`` to truncate and write a new file, or ``'a'`` to append to an
- existing file. If *mode* is ``'a'`` and *file* refers to an existing ZIP
+ file, ``'w'`` to truncate and write a new file, ``'x'`` to exclusive create
+ and write a new file, or ``'a'`` to append to an existing file.
+ If *mode* is ``'x'`` and *file* refers to an existing file,
+ a :exc:`FileExistsError` will be raised.
+ If *mode* is ``'a'`` and *file* refers to an existing ZIP
file, then additional files are added to it. If *file* does not refer to a
ZIP file, then a new ZIP archive is appended to the file. This is meant for
adding a ZIP archive to another file (such as :file:`python.exe`). If
*mode* is ``a`` and the file does not exist at all, it is created.
+ If *mode* is ``r`` or ``a``, the file should be seekable.
*compression* is the ZIP compression method to use when writing the archive,
and should be :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`,
:const:`ZIP_BZIP2` or :const:`ZIP_LZMA`; unrecognized
@@ -151,7 +155,7 @@ ZipFile Objects
extensions when the zipfile is larger than 2 GiB. If it is false :mod:`zipfile`
will raise an exception when the ZIP file would require ZIP64 extensions.
- If the file is created with mode ``'a'`` or ``'w'`` and then
+ If the file is created with mode ``'w'``, ``'x'`` or ``'a'`` and then
:meth:`closed <close>` without adding any files to the archive, the appropriate
ZIP structures for an empty archive will be written to the file.
@@ -171,6 +175,10 @@ ZipFile Objects
.. versionchanged:: 3.4
ZIP64 extensions are enabled by default.
+ .. versionchanged:: 3.5
+ Added support for writing to unseekable streams.
+ Added support for the ``'x'`` mode.
+
.. method:: ZipFile.close()
@@ -226,14 +234,8 @@ ZipFile Objects
.. note::
- If the ZipFile was created by passing in a file-like object as the first
- argument to the constructor, then the object returned by :meth:`.open` shares the
- ZipFile's file pointer. Under these circumstances, the object returned by
- :meth:`.open` should not be used after any additional operations are performed
- on the ZipFile object. If the ZipFile was created by passing in a string (the
- filename) as the first argument to the constructor, then :meth:`.open` will
- create a new file object that will be held by the ZipExtFile, allowing it to
- operate independently of the ZipFile.
+ Objects returned by :meth:`.open` can operate independently of the
+ ZipFile.
.. note::
@@ -318,7 +320,8 @@ ZipFile Objects
*arcname* (by default, this will be the same as *filename*, but without a drive
letter and with leading path separators removed). If given, *compress_type*
overrides the value given for the *compression* parameter to the constructor for
- the new entry. The archive must be open with mode ``'w'`` or ``'a'`` -- calling
+ the new entry.
+ The archive must be open with mode ``'w'``, ``'x'`` or ``'a'`` -- calling
:meth:`write` on a ZipFile created with mode ``'r'`` will raise a
:exc:`RuntimeError`. Calling :meth:`write` on a closed ZipFile will raise a
:exc:`RuntimeError`.
@@ -340,16 +343,16 @@ ZipFile Objects
If ``arcname`` (or ``filename``, if ``arcname`` is not given) contains a null
byte, the name of the file in the archive will be truncated at the null byte.
-
.. method:: ZipFile.writestr(zinfo_or_arcname, bytes[, compress_type])
Write the string *bytes* to the archive; *zinfo_or_arcname* is either the file
name it will be given in the archive, or a :class:`ZipInfo` instance. If it's
an instance, at least the filename, date, and time must be given. If it's a
- name, the date and time is set to the current date and time. The archive must be
- opened with mode ``'w'`` or ``'a'`` -- calling :meth:`writestr` on a ZipFile
- created with mode ``'r'`` will raise a :exc:`RuntimeError`. Calling
- :meth:`writestr` on a closed ZipFile will raise a :exc:`RuntimeError`.
+ name, the date and time is set to the current date and time.
+ The archive must be opened with mode ``'w'``, ``'x'`` or ``'a'`` -- calling
+ :meth:`writestr` on a ZipFile created with mode ``'r'`` will raise a
+ :exc:`RuntimeError`. Calling :meth:`writestr` on a closed ZipFile will
+ raise a :exc:`RuntimeError`.
If given, *compress_type* overrides the value given for the *compression*
parameter to the constructor for the new entry, or in the *zinfo_or_arcname*
@@ -377,7 +380,8 @@ The following data attributes are also available:
.. attribute:: ZipFile.comment
The comment text associated with the ZIP file. If assigning a comment to a
- :class:`ZipFile` instance created with mode 'a' or 'w', this should be a
+ :class:`ZipFile` instance created with mode ``'w'``, ``'x'`` or ``'a'``,
+ this should be a
string no longer than 65535 bytes. Comments longer than this will be
truncated in the written archive when :meth:`close` is called.
@@ -407,8 +411,7 @@ The :class:`PyZipFile` constructor takes the same parameters as the
archive.
If the *optimize* parameter to :class:`PyZipFile` was not given or ``-1``,
- the corresponding file is a :file:`\*.pyo` file if available, else a
- :file:`\*.pyc` file, compiling if necessary.
+ the corresponding file is a :file:`\*.pyc` file, compiling if necessary.
If the *optimize* parameter to :class:`PyZipFile` was ``0``, ``1`` or
``2``, only files with that optimization level (see :func:`compile`) are
@@ -571,4 +574,3 @@ Instances have the following attributes:
.. attribute:: ZipInfo.file_size
Size of the uncompressed file.
-
diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst
index 2cf508b..8a5d5d1 100644
--- a/Doc/library/zipimport.rst
+++ b/Doc/library/zipimport.rst
@@ -20,10 +20,10 @@ subdirectory. For example, the path :file:`example.zip/lib/` would only
import from the :file:`lib/` subdirectory within the archive.
Any files may be present in the ZIP archive, but only files :file:`.py` and
-:file:`.py[co]` are available for import. ZIP import of dynamic modules
+:file:`.pyc` are available for import. ZIP import of dynamic modules
(:file:`.pyd`, :file:`.so`) is disallowed. Note that if an archive only contains
:file:`.py` files, Python will not attempt to modify the archive by adding the
-corresponding :file:`.pyc` or :file:`.pyo` file, meaning that if a ZIP archive
+corresponding :file:`.pyc` file, meaning that if a ZIP archive
doesn't contain :file:`.pyc` files, importing may be rather slow.
ZIP archives with an archive comment are currently not supported.
@@ -161,4 +161,3 @@ Here is an example that imports a module from a ZIP archive - note that the
>>> import jwzthreading
>>> jwzthreading.__file__
'example.zip/jwzthreading.py'
-
diff --git a/Doc/make.bat b/Doc/make.bat
index 251f822..5ab8085 100644
--- a/Doc/make.bat
+++ b/Doc/make.bat
@@ -8,9 +8,23 @@ set this=%~n0
if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build
if "%PYTHON%" EQU "" set PYTHON=py
-if DEFINED ProgramFiles(x86) set _PRGMFLS=%ProgramFiles(x86)%
-if NOT DEFINED ProgramFiles(x86) set _PRGMFLS=%ProgramFiles%
-if "%HTMLHELP%" EQU "" set HTMLHELP=%_PRGMFLS%\HTML Help Workshop\hhc.exe
+if "%1" NEQ "htmlhelp" goto :skiphhcsearch
+if exist "%HTMLHELP%" goto :skiphhcsearch
+
+rem Search for HHC in likely places
+set HTMLHELP=
+where hhc /q && set HTMLHELP=hhc && goto :skiphhcsearch
+where /R ..\externals hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc"
+if not exist "%HTMLHELP%" where /R "%ProgramFiles(x86)%" hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc"
+if not exist "%HTMLHELP%" where /R "%ProgramFiles%" hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc"
+if not exist "%HTMLHELP%" (
+ echo.
+ echo.The HTML Help Workshop was not found. Set the HTMLHELP variable
+ echo.to the path to hhc.exe or download and install it from
+ echo.http://msdn.microsoft.com/en-us/library/ms669985
+ exit /B 1
+)
+:skiphhcsearch
if "%DISTVERSION%" EQU "" for /f "usebackq" %%v in (`%PYTHON% tools/extensions/patchlevel.py`) do set DISTVERSION=%%v
@@ -36,7 +50,8 @@ if errorlevel 9009 (
echo.
echo.If you don't have Sphinx installed, grab it from
echo.http://sphinx-doc.org/
- goto end
+ popd
+ exit /B 1
)
rem Targets that do require sphinx-build and have their own label
@@ -76,15 +91,6 @@ if NOT "%PAPER%" == "" (
cmd /C %SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%*
if "%1" EQU "htmlhelp" (
- if not exist "%HTMLHELP%" (
- echo.
- echo.The HTML Help Workshop was not found. Set the HTMLHELP variable
- echo.to the path to hhc.exe or download and install it from
- echo.http://msdn.microsoft.com/en-us/library/ms669985
- rem Set errorlevel to 1 and exit
- cmd /C exit /b 1
- goto end
- )
cmd /C "%HTMLHELP%" build\htmlhelp\python%DISTVERSION:.=%.hhp
rem hhc.exe seems to always exit with code 1, reset to 0 for less than 2
if not errorlevel 2 cmd /C exit /b 0
diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst
index 5dd17eb..01cfd6d 100644
--- a/Doc/reference/compound_stmts.rst
+++ b/Doc/reference/compound_stmts.rst
@@ -51,6 +51,9 @@ Summarizing:
: | `with_stmt`
: | `funcdef`
: | `classdef`
+ : | `async_with_stmt`
+ : | `async_for_stmt`
+ : | `async_funcdef`
suite: `stmt_list` NEWLINE | NEWLINE INDENT `statement`+ DEDENT
statement: `stmt_list` NEWLINE | `compound_stmt`
stmt_list: `simple_stmt` (";" `simple_stmt`)* [";"]
@@ -660,6 +663,130 @@ can be used to create instance variables with different implementation details.
:pep:`3129` - Class Decorators
+Coroutines
+==========
+
+.. versionadded:: 3.5
+
+.. index:: statement: async def
+.. _`async def`:
+
+Coroutine function definition
+-----------------------------
+
+.. productionlist::
+ async_funcdef: [`decorators`] "async" "def" `funcname` "(" [`parameter_list`] ")" ["->" `expression`] ":" `suite`
+
+.. index::
+ keyword: async
+ keyword: await
+
+Execution of Python coroutines can be suspended and resumed at many points
+(see :term:`coroutine`). In the body of a coroutine, any ``await`` and
+``async`` identifiers become reserved keywords; :keyword:`await` expressions,
+:keyword:`async for` and :keyword:`async with` can only be used in
+coroutine bodies.
+
+Functions defined with ``async def`` syntax are always coroutine functions,
+even if they do not contain ``await`` or ``async`` keywords.
+
+It is a :exc:`SyntaxError` to use :keyword:`yield` expressions in
+``async def`` coroutines.
+
+An example of a coroutine function::
+
+ async def func(param1, param2):
+ do_stuff()
+ await some_coroutine()
+
+
+.. index:: statement: async for
+.. _`async for`:
+
+The :keyword:`async for` statement
+----------------------------------
+
+.. productionlist::
+ async_for_stmt: "async" `for_stmt`
+
+An :term:`asynchronous iterable` is able to call asynchronous code in its
+*iter* implementation, and :term:`asynchronous iterator` can call asynchronous
+code in its *next* method.
+
+The ``async for`` statement allows convenient iteration over asynchronous
+iterators.
+
+The following code::
+
+ async for TARGET in ITER:
+ BLOCK
+ else:
+ BLOCK2
+
+Is semantically equivalent to::
+
+ iter = (ITER)
+ iter = await type(iter).__aiter__(iter)
+ running = True
+ while running:
+ try:
+ TARGET = await type(iter).__anext__(iter)
+ except StopAsyncIteration:
+ running = False
+ else:
+ BLOCK
+ else:
+ BLOCK2
+
+See also :meth:`__aiter__` and :meth:`__anext__` for details.
+
+It is a :exc:`SyntaxError` to use ``async for`` statement outside of an
+:keyword:`async def` function.
+
+
+.. index:: statement: async with
+.. _`async with`:
+
+The :keyword:`async with` statement
+-----------------------------------
+
+.. productionlist::
+ async_with_stmt: "async" `with_stmt`
+
+An :term:`asynchronous context manager` is a :term:`context manager` that is
+able to suspend execution in its *enter* and *exit* methods.
+
+The following code::
+
+ async with EXPR as VAR:
+ BLOCK
+
+Is semantically equivalent to::
+
+ mgr = (EXPR)
+ aexit = type(mgr).__aexit__
+ aenter = type(mgr).__aenter__(mgr)
+ exc = True
+
+ VAR = await aenter
+ try:
+ BLOCK
+ except:
+ if not await aexit(mgr, *sys.exc_info()):
+ raise
+ else:
+ await aexit(mgr, None, None, None)
+
+See also :meth:`__aenter__` and :meth:`__aexit__` for details.
+
+It is a :exc:`SyntaxError` to use ``async with`` statement outside of an
+:keyword:`async def` function.
+
+.. seealso::
+
+ :pep:`492` - Coroutines with async and await syntax
+
+
.. rubric:: Footnotes
.. [#] The exception is propagated to the invocation stack unless
diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst
index 4acf13e..e1d71b8 100644
--- a/Doc/reference/datamodel.rst
+++ b/Doc/reference/datamodel.rst
@@ -616,6 +616,16 @@ Callable types
exception is raised and the iterator will have reached the end of the set of
values to be returned.
+ Coroutine functions
+ .. index::
+ single: coroutine; function
+
+ A function or method which is defined using :keyword:`async def` is called
+ a :dfn:`coroutine function`. Such a function, when called, returns a
+ :term:`coroutine` object. It may contain :keyword:`await` expressions,
+ as well as :keyword:`async with` and :keyword:`async for` statements. See
+ also the :ref:`coroutine-objects` section.
+
Built-in functions
.. index::
object: built-in function
@@ -1990,6 +2000,7 @@ left undefined.
.. method:: object.__add__(self, other)
object.__sub__(self, other)
object.__mul__(self, other)
+ object.__matmul__(self, other)
object.__truediv__(self, other)
object.__floordiv__(self, other)
object.__mod__(self, other)
@@ -2006,15 +2017,16 @@ left undefined.
builtin: pow
builtin: pow
- These methods are called to implement the binary arithmetic operations (``+``,
- ``-``, ``*``, ``/``, ``//``, ``%``, :func:`divmod`, :func:`pow`, ``**``, ``<<``,
- ``>>``, ``&``, ``^``, ``|``). For instance, to evaluate the expression
- ``x + y``, where *x* is an instance of a class that has an :meth:`__add__`
- method, ``x.__add__(y)`` is called. The :meth:`__divmod__` method should be the
- equivalent to using :meth:`__floordiv__` and :meth:`__mod__`; it should not be
- related to :meth:`__truediv__`. Note that :meth:`__pow__` should be defined
- to accept an optional third argument if the ternary version of the built-in
- :func:`pow` function is to be supported.
+ These methods are called to implement the binary arithmetic operations
+ (``+``, ``-``, ``*``, ``@``, ``/``, ``//``, ``%``, :func:`divmod`,
+ :func:`pow`, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``). For instance, to
+ evaluate the expression ``x + y``, where *x* is an instance of a class that
+ has an :meth:`__add__` method, ``x.__add__(y)`` is called. The
+ :meth:`__divmod__` method should be the equivalent to using
+ :meth:`__floordiv__` and :meth:`__mod__`; it should not be related to
+ :meth:`__truediv__`. Note that :meth:`__pow__` should be defined to accept
+ an optional third argument if the ternary version of the built-in :func:`pow`
+ function is to be supported.
If one of those methods does not support the operation with the supplied
arguments, it should return ``NotImplemented``.
@@ -2023,6 +2035,7 @@ left undefined.
.. method:: object.__radd__(self, other)
object.__rsub__(self, other)
object.__rmul__(self, other)
+ object.__rmatmul__(self, other)
object.__rtruediv__(self, other)
object.__rfloordiv__(self, other)
object.__rmod__(self, other)
@@ -2038,14 +2051,14 @@ left undefined.
builtin: divmod
builtin: pow
- These methods are called to implement the binary arithmetic operations (``+``,
- ``-``, ``*``, ``/``, ``//``, ``%``, :func:`divmod`, :func:`pow`, ``**``,
- ``<<``, ``>>``, ``&``, ``^``, ``|``) with reflected (swapped) operands.
- These functions are only called if the left operand does not support the
- corresponding operation and the operands are of different types. [#]_ For
- instance, to evaluate the expression ``x - y``, where *y* is an instance of
- a class that has an :meth:`__rsub__` method, ``y.__rsub__(x)`` is called if
- ``x.__sub__(y)`` returns *NotImplemented*.
+ These methods are called to implement the binary arithmetic operations
+ (``+``, ``-``, ``*``, ``@``, ``/``, ``//``, ``%``, :func:`divmod`,
+ :func:`pow`, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with reflected
+ (swapped) operands. These functions are only called if the left operand does
+ not support the corresponding operation and the operands are of different
+ types. [#]_ For instance, to evaluate the expression ``x - y``, where *y* is
+ an instance of a class that has an :meth:`__rsub__` method, ``y.__rsub__(x)``
+ is called if ``x.__sub__(y)`` returns *NotImplemented*.
.. index:: builtin: pow
@@ -2063,6 +2076,7 @@ left undefined.
.. method:: object.__iadd__(self, other)
object.__isub__(self, other)
object.__imul__(self, other)
+ object.__imatmul__(self, other)
object.__itruediv__(self, other)
object.__ifloordiv__(self, other)
object.__imod__(self, other)
@@ -2074,17 +2088,17 @@ left undefined.
object.__ior__(self, other)
These methods are called to implement the augmented arithmetic assignments
- (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``, ``**=``, ``<<=``, ``>>=``,
- ``&=``, ``^=``, ``|=``). These methods should attempt to do the operation
- in-place (modifying *self*) and return the result (which could be, but does
- not have to be, *self*). If a specific method is not defined, the augmented
- assignment falls back to the normal methods. For instance, if *x* is an
- instance of a class with an :meth:`__iadd__` method, ``x += y`` is equivalent
- to ``x = x.__iadd__(y)`` . Otherwise, ``x.__add__(y)`` and ``y.__radd__(x)``
- are considered, as with the evaluation of ``x + y``. In certain situations,
- augmented assignment can result in unexpected errors (see
- :ref:`faq-augmented-assignment-tuple-error`), but this behavior is in
- fact part of the data model.
+ (``+=``, ``-=``, ``*=``, ``@=``, ``/=``, ``//=``, ``%=``, ``**=``, ``<<=``,
+ ``>>=``, ``&=``, ``^=``, ``|=``). These methods should attempt to do the
+ operation in-place (modifying *self*) and return the result (which could be,
+ but does not have to be, *self*). If a specific method is not defined, the
+ augmented assignment falls back to the normal methods. For instance, if *x*
+ is an instance of a class with an :meth:`__iadd__` method, ``x += y`` is
+ equivalent to ``x = x.__iadd__(y)`` . Otherwise, ``x.__add__(y)`` and
+ ``y.__radd__(x)`` are considered, as with the evaluation of ``x + y``. In
+ certain situations, augmented assignment can result in unexpected errors (see
+ :ref:`faq-augmented-assignment-tuple-error`), but this behavior is in fact
+ part of the data model.
.. method:: object.__neg__(self)
@@ -2254,6 +2268,155 @@ special methods (the special method *must* be set on the class
object itself in order to be consistently invoked by the interpreter).
+.. index::
+ single: coroutine
+
+Coroutines
+==========
+
+
+Awaitable Objects
+-----------------
+
+An :term:`awaitable` object generally implements an :meth:`__await__` method.
+:term:`Coroutine` objects returned from :keyword:`async def` functions
+are awaitable.
+
+.. note::
+
+ The :term:`generator iterator` objects returned from generators
+ decorated with :func:`types.coroutine` or :func:`asyncio.coroutine`
+ are also awaitable, but they do not implement :meth:`__await__`.
+
+.. method:: object.__await__(self)
+
+ Must return an :term:`iterator`. Should be used to implement
+ :term:`awaitable` objects. For instance, :class:`asyncio.Future` implements
+ this method to be compatible with the :keyword:`await` expression.
+
+.. versionadded:: 3.5
+
+.. seealso:: :pep:`492` for additional information about awaitable objects.
+
+
+.. _coroutine-objects:
+
+Coroutine Objects
+-----------------
+
+:term:`Coroutine` objects are :term:`awaitable` objects.
+A coroutine's execution can be controlled by calling :meth:`__await__` and
+iterating over the result. When the coroutine has finished executing and
+returns, the iterator raises :exc:`StopIteration`, and the exception's
+:attr:`~StopIteration.value` attribute holds the return value. If the
+coroutine raises an exception, it is propagated by the iterator. Coroutines
+should not directly raise unhandled :exc:`StopIteration` exceptions.
+
+Coroutines also have the methods listed below, which are analogous to
+those of generators (see :ref:`generator-methods`). However, unlike
+generators, coroutines do not directly support iteration.
+
+.. method:: coroutine.send(value)
+
+ Starts or resumes execution of the coroutine. If *value* is ``None``,
+ this is equivalent to advancing the iterator returned by
+ :meth:`__await__`. If *value* is not ``None``, this method delegates
+ to the :meth:`~generator.send` method of the iterator that caused
+ the coroutine to suspend. The result (return value,
+ :exc:`StopIteration`, or other exception) is the same as when
+ iterating over the :meth:`__await__` return value, described above.
+
+.. method:: coroutine.throw(type[, value[, traceback]])
+
+ Raises the specified exception in the coroutine. This method delegates
+ to the :meth:`~generator.throw` method of the iterator that caused
+ the coroutine to suspend, if it has such a method. Otherwise,
+ the exception is raised at the suspension point. The result
+ (return value, :exc:`StopIteration`, or other exception) is the same as
+ when iterating over the :meth:`__await__` return value, described
+ above. If the exception is not caught in the coroutine, it propagates
+ back to the caller.
+
+.. method:: coroutine.close()
+
+ Causes the coroutine to clean itself up and exit. If the coroutine
+ is suspended, this method first delegates to the :meth:`~generator.close`
+ method of the iterator that caused the coroutine to suspend, if it
+ has such a method. Then it raises :exc:`GeneratorExit` at the
+ suspension point, causing the coroutine to immediately clean itself up.
+ Finally, the coroutine is marked as having finished executing, even if
+ it was never started.
+
+ Coroutine objects are automatically closed using the above process when
+ they are about to be destroyed.
+
+
+Asynchronous Iterators
+----------------------
+
+An *asynchronous iterable* is able to call asynchronous code in its
+``__aiter__`` implementation, and an *asynchronous iterator* can call
+asynchronous code in its ``__anext__`` method.
+
+Asynchronous iterators can be used in a :keyword:`async for` statement.
+
+.. method:: object.__aiter__(self)
+
+ Must return an *awaitable* resulting in an *asynchronous iterator* object.
+
+.. method:: object.__anext__(self)
+
+ Must return an *awaitable* resulting in a next value of the iterator. Should
+ raise a :exc:`StopAsyncIteration` error when the iteration is over.
+
+An example of an asynchronous iterable object::
+
+ class Reader:
+ async def readline(self):
+ ...
+
+ async def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ val = await self.readline()
+ if val == b'':
+ raise StopAsyncIteration
+ return val
+
+.. versionadded:: 3.5
+
+
+Asynchronous Context Managers
+-----------------------------
+
+An *asynchronous context manager* is a *context manager* that is able to
+suspend execution in its ``__aenter__`` and ``__aexit__`` methods.
+
+Asynchronous context managers can be used in a :keyword:`async with` statement.
+
+.. method:: object.__aenter__(self)
+
+ This method is semantically similar to the :meth:`__enter__`, with only
+ difference that it must return an *awaitable*.
+
+.. method:: object.__aexit__(self, exc_type, exc_value, traceback)
+
+ This method is semantically similar to the :meth:`__exit__`, with only
+ difference that it must return an *awaitable*.
+
+An example of an asynchronous context manager class::
+
+ class AsyncContextManager:
+ async def __aenter__(self):
+ await log('entering context')
+
+ async def __aexit__(self, exc_type, exc, tb):
+ await log('exiting context')
+
+.. versionadded:: 3.5
+
+
.. rubric:: Footnotes
.. [#] It *is* possible in some cases to change an object's type, under certain
diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst
index 71684b7..3a51f97 100644
--- a/Doc/reference/expressions.rst
+++ b/Doc/reference/expressions.rst
@@ -390,6 +390,7 @@ on the right hand side of an assignment statement.
to sub-generators easy.
.. index:: object: generator
+.. _generator-methods:
Generator-iterator methods
^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -443,12 +444,12 @@ is already executing raises a :exc:`ValueError` exception.
.. method:: generator.close()
Raises a :exc:`GeneratorExit` at the point where the generator function was
- paused. If the generator function then raises :exc:`StopIteration` (by
- exiting normally, or due to already being closed) or :exc:`GeneratorExit` (by
- not catching the exception), close returns to its caller. If the generator
- yields a value, a :exc:`RuntimeError` is raised. If the generator raises any
- other exception, it is propagated to the caller. :meth:`close` does nothing
- if the generator has already exited due to an exception or normal exit.
+ paused. If the generator function then exits gracefully, is already closed,
+ or raises :exc:`GeneratorExit` (by not catching the exception), close
+ returns to its caller. If the generator yields a value, a
+ :exc:`RuntimeError` is raised. If the generator raises any other exception,
+ it is propagated to the caller. :meth:`close` does nothing if the generator
+ has already exited due to an exception or normal exit.
.. index:: single: yield; examples
@@ -811,6 +812,20 @@ a class instance:
if that method was called.
+.. _await:
+
+Await expression
+================
+
+Suspend the execution of :term:`coroutine` on an :term:`awaitable` object.
+Can only be used inside a :term:`coroutine function`.
+
+.. productionlist::
+ await: ["await"] `primary`
+
+.. versionadded:: 3.5
+
+
.. _power:
The power operator
@@ -820,7 +835,7 @@ The power operator binds more tightly than unary operators on its left; it binds
less tightly than unary operators on its right. The syntax is:
.. productionlist::
- power: `primary` ["**" `u_expr`]
+ power: `await` ["**" `u_expr`]
Thus, in an unparenthesized sequence of power and unary operators, the operators
are evaluated from right to left (this does not constrain the evaluation order
@@ -891,8 +906,9 @@ from the power operator, there are only two levels, one for multiplicative
operators and one for additive operators:
.. productionlist::
- m_expr: `u_expr` | `m_expr` "*" `u_expr` | `m_expr` "//" `u_expr` | `m_expr` "/" `u_expr`
- : | `m_expr` "%" `u_expr`
+ m_expr: `u_expr` | `m_expr` "*" `u_expr` | `m_expr` "@" `m_expr` |
+ : `m_expr` "//" `u_expr`| `m_expr` "/" `u_expr` |
+ : `m_expr` "%" `u_expr`
a_expr: `m_expr` | `a_expr` "+" `m_expr` | `a_expr` "-" `m_expr`
.. index:: single: multiplication
@@ -903,6 +919,13 @@ the other must be a sequence. In the former case, the numbers are converted to a
common type and then multiplied together. In the latter case, sequence
repetition is performed; a negative repetition factor yields an empty sequence.
+.. index:: single: matrix multiplication
+
+The ``@`` (at) operator is intended to be used for matrix multiplication. No
+builtin Python types implement this operator.
+
+.. versionadded:: 3.5
+
.. index::
exception: ZeroDivisionError
single: division
@@ -1346,13 +1369,16 @@ precedence and have a left-to-right chaining feature as described in the
+-----------------------------------------------+-------------------------------------+
| ``+``, ``-`` | Addition and subtraction |
+-----------------------------------------------+-------------------------------------+
-| ``*``, ``/``, ``//``, ``%`` | Multiplication, division, remainder |
-| | [#]_ |
+| ``*``, ``@``, ``/``, ``//``, ``%`` | Multiplication, matrix |
+| | multiplication division, |
+| | remainder [#]_ |
+-----------------------------------------------+-------------------------------------+
| ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT |
+-----------------------------------------------+-------------------------------------+
| ``**`` | Exponentiation [#]_ |
+-----------------------------------------------+-------------------------------------+
+| ``await`` ``x`` | Await expression |
++-----------------------------------------------+-------------------------------------+
| ``x[index]``, ``x[index:index]``, | Subscription, slicing, |
| ``x(arguments...)``, ``x.attribute`` | call, attribute reference |
+-----------------------------------------------+-------------------------------------+
diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst
index 16d28d8..d549b26 100644
--- a/Doc/reference/import.rst
+++ b/Doc/reference/import.rst
@@ -339,6 +339,7 @@ of what happens during the loading portion of import::
module = None
if spec.loader is not None and hasattr(spec.loader, 'create_module'):
+ # It is assumed 'exec_module' will also be defined on the loader.
module = spec.loader.create_module(spec)
if module is None:
module = ModuleType(spec.name)
@@ -427,7 +428,7 @@ Module loaders may opt in to creating the module object during loading
by implementing a :meth:`~importlib.abc.Loader.create_module` method.
It takes one argument, the module spec, and returns the new module object
to use during loading. ``create_module()`` does not need to set any attributes
-on the module object. If the loader does not define ``create_module()``, the
+on the module object. If the method returns ``None``, the
import machinery will create the new module itself.
.. versionadded:: 3.4
@@ -459,7 +460,13 @@ import machinery will create the new module itself.
* If loading fails, the loader must remove any modules it has inserted
into :data:`sys.modules`, but it must remove **only** the failing
- module, and only if the loader itself has loaded it explicitly.
+ module(s), and only if the loader itself has loaded the module(s)
+ explicitly.
+
+.. versionchanged:: 3.5
+ A :exc:`DeprecationWarning` is raised when ``exec_module()`` is defined but
+ ``create_module()`` is not. Starting in Python 3.6 it will be an error to not
+ define ``create_module()`` on a loader attached to a ModuleSpec.
Submodules
----------
@@ -674,7 +681,7 @@ path entry finder that knows how to handle that particular kind of path.
The default set of path entry finders implement all the semantics for finding
modules on the file system, handling special file types such as Python source
-code (``.py`` files), Python byte code (``.pyc`` and ``.pyo`` files) and
+code (``.py`` files), Python byte code (``.pyc`` files) and
shared libraries (e.g. ``.so`` files). When supported by the :mod:`zipimport`
module in the standard library, the default path entry finders also handle
loading all of these file types (other than shared libraries) from zipfiles.
@@ -788,6 +795,15 @@ hook` callables on :data:`sys.path_hooks`, then the following protocol is used
to ask the finder for a module spec, which is then used when loading the
module.
+The current working directory -- denoted by an empty string -- is handled
+slightly differently from other entries on :data:`sys.path`. First, if the
+current working directory is found to not exist, no value is stored in
+:data:`sys.path_importer_cache`. Second, the value for the current working
+directory is looked up fresh for each module lookup. Third, the path used for
+:data:`sys.path_importer_cache` and returned by
+:meth:`importlib.machinery.PathFinder.find_spec` will be the actual current
+working directory and not the empty string.
+
Path entry finder protocol
--------------------------
diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst
index a1265cf..8ad975f 100644
--- a/Doc/reference/lexical_analysis.rst
+++ b/Doc/reference/lexical_analysis.rst
@@ -313,7 +313,7 @@ The Unicode category codes mentioned above stand for:
* *Nd* - decimal numbers
* *Pc* - connector punctuations
* *Other_ID_Start* - explicit list of characters in `PropList.txt
- <http://www.unicode.org/Public/6.3.0/ucd/PropList.txt>`_ to support backwards
+ <http://www.unicode.org/Public/8.0.0/ucd/PropList.txt>`_ to support backwards
compatibility
* *Other_ID_Continue* - likewise
@@ -692,7 +692,7 @@ Operators
The following tokens are operators::
- + - * ** / // %
+ + - * ** / // % @
<< >> & | ^ ~
< > <= >= == !=
@@ -708,7 +708,7 @@ The following tokens serve as delimiters in the grammar::
( ) [ ] { }
, : . ; @ = ->
- += -= *= /= //= %=
+ += -= *= /= //= %= @=
&= |= ^= >>= <<= **=
The period can also occur in floating-point and imaginary literals. A sequence
@@ -729,4 +729,4 @@ occurrence outside string literals and comments is an unconditional error::
.. rubric:: Footnotes
-.. [#] http://www.unicode.org/Public/6.3.0/ucd/NameAliases.txt
+.. [#] http://www.unicode.org/Public/8.0.0/ucd/NameAliases.txt
diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst
index 8946b4f..5f60540 100644
--- a/Doc/reference/simple_stmts.rst
+++ b/Doc/reference/simple_stmts.rst
@@ -281,7 +281,7 @@ operation and an assignment statement:
.. productionlist::
augmented_assignment_stmt: `augtarget` `augop` (`expression_list` | `yield_expression`)
augtarget: `identifier` | `attributeref` | `subscription` | `slicing`
- augop: "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="
+ augop: "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**="
: | ">>=" | "<<=" | "&=" | "^=" | "|="
(See section :ref:`primaries` for the syntax definitions of the last three
diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py
index 64a5665..d44b052 100644
--- a/Doc/tools/extensions/pyspecific.py
+++ b/Doc/tools/extensions/pyspecific.py
@@ -34,7 +34,7 @@ import suspicious
ISSUE_URI = 'https://bugs.python.org/issue%s'
-SOURCE_URI = 'https://hg.python.org/cpython/file/3.4/%s'
+SOURCE_URI = 'https://hg.python.org/cpython/file/3.5/%s'
# monkey-patch reST parser to disable alphabetic and roman enumerated lists
from docutils.parsers.rst.states import Body
diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv
index fe7130c..7664f92 100644
--- a/Doc/tools/susp-ignored.csv
+++ b/Doc/tools/susp-ignored.csv
@@ -122,6 +122,8 @@ library/ipaddress,,:db8,>>> ipaddress.IPv6Address('2001:db8::1000')
library/ipaddress,,::,>>> ipaddress.IPv6Address('2001:db8::1000')
library/ipaddress,,:db8,IPv6Address('2001:db8::1000')
library/ipaddress,,::,IPv6Address('2001:db8::1000')
+library/ipaddress,,:db8,">>> ipaddress.ip_address(""2001:db8::1"").reverse_pointer"
+library/ipaddress,,::,">>> ipaddress.ip_address(""2001:db8::1"").reverse_pointer"
library/ipaddress,,::,"""::abc:7:def"""
library/ipaddress,,:def,"""::abc:7:def"""
library/ipaddress,,::,::FFFF/96
@@ -139,8 +141,6 @@ library/itertools,,:stop,elements from seq[start:stop:step]
library/logging.handlers,,:port,host:port
library/mmap,,:i2,obj[i1:i2]
library/multiprocessing,,`,# Add more tasks using `put()`
-library/multiprocessing,,`,">>> l._callmethod('__getitem__', (20,)) # equiv to `l[20]`"
-library/multiprocessing,,`,">>> l._callmethod('__getitem__', (slice(2, 7),)) # equiv to `l[2:7]`"
library/multiprocessing,,:queue,">>> QueueManager.register('get_queue', callable=lambda:queue)"
library/multiprocessing,,`,# register the Foo class; make `f()` and `g()` accessible via proxy
library/multiprocessing,,`,# register the Foo class; make `g()` and `_h()` accessible via proxy
@@ -177,8 +177,6 @@ library/ssl,,:MyState,State or Province Name (full name) [Some-State]:MyState
library/ssl,,:ops,Email Address []:ops@myserver.mygroup.myorganization.com
library/ssl,,:Some,"Locality Name (eg, city) []:Some City"
library/ssl,,:US,Country Name (2 letter code) [AU]:US
-library/stdtypes,,::,>>> a[::-1].tolist()
-library/stdtypes,,::,>>> a[::2].tolist()
library/stdtypes,,:end,s[start:end]
library/stdtypes,,::,>>> hash(v[::-2]) == hash(b'abcefg'[::-2])
library/stdtypes,,:len,s[len(s):len(s)]
@@ -208,16 +206,6 @@ library/venv,,:param,":param nodist: If True, setuptools and pip are not install
library/venv,,:param,":param progress: If setuptools or pip are installed, the progress of the"
library/venv,,:param,":param nopip: If True, pip is not installed into the created"
library/venv,,:param,:param context: The information for the environment creation request
-library/xml.etree.elementtree,,:sometag,prefix:sometag
-library/xml.etree.elementtree,,:fictional,"<actors xmlns:fictional=""http://characters.example.com"""
-library/xml.etree.elementtree,,:character,<fictional:character>Lancelot</fictional:character>
-library/xml.etree.elementtree,,:character,<fictional:character>Archie Leach</fictional:character>
-library/xml.etree.elementtree,,:character,<fictional:character>Sir Robin</fictional:character>
-library/xml.etree.elementtree,,:character,<fictional:character>Gunther</fictional:character>
-library/xml.etree.elementtree,,:character,<fictional:character>Commander Clement</fictional:character>
-library/xml.etree.elementtree,,:actor,"for actor in root.findall('real_person:actor', ns):"
-library/xml.etree.elementtree,,:name,"name = actor.find('real_person:name', ns)"
-library/xml.etree.elementtree,,:character,"for char in actor.findall('role:character', ns):"
library/xmlrpc.client,,:pass,http://user:pass@host:port/path
library/xmlrpc.client,,:pass,user:pass
library/xmlrpc.client,,:port,http://user:pass@host:port/path
@@ -244,7 +232,6 @@ tutorial/stdlib2,,:start,extra = data[start:start+extra_size]
tutorial/stdlib2,,:start,"fields = struct.unpack('<IIIHH', data[start:start+16])"
tutorial/stdlib2,,:start,filename = data[start:start+filenamesize]
tutorial/stdlib2,,:Warning,WARNING:root:Warning:config file server.conf not found
-tutorial/venv,,:c7b9645a6f35,"Python 3.4.3+ (3.4:c7b9645a6f35+, May 22 2015, 09:31:25)"
using/cmdline,,:category,action:message:category:module:line
using/cmdline,,:errorhandler,:errorhandler
using/cmdline,,:line,action:message:category:module:line
@@ -265,6 +252,11 @@ whatsnew/2.4,,:System,
whatsnew/2.5,,:memory,:memory:
whatsnew/2.5,,:step,[start:stop:step]
whatsnew/2.5,,:stop,[start:stop:step]
+whatsnew/2.7,,::,"ParseResult(scheme='http', netloc='[1080::8:800:200C:417A]',"
+whatsnew/2.7,,::,>>> urlparse.urlparse('http://[1080::8:800:200C:417A]/foo')
+whatsnew/2.7,,:Sunday,'2009:4:Sunday'
+whatsnew/2.7,,:Cookie,"export PYTHONWARNINGS=all,error:::Cookie:0"
+whatsnew/2.7,,::,"export PYTHONWARNINGS=all,error:::Cookie:0"
whatsnew/3.2,,:affe,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]',"
whatsnew/3.2,,:affe,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/')
whatsnew/3.2,,:beef,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]',"
@@ -280,14 +272,22 @@ whatsnew/3.2,,:feed,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:
whatsnew/3.2,,:gz,">>> with tarfile.open(name='myarchive.tar.gz', mode='w:gz') as tf:"
whatsnew/3.2,,:location,zope9-location = ${zope9:location}
whatsnew/3.2,,:prefix,zope-conf = ${custom:prefix}/etc/zope.conf
-whatsnew/changelog,,:platform,:platform:
whatsnew/changelog,,:gz,": TarFile opened with external fileobj and ""w:gz"" mode didn't"
-whatsnew/changelog,,:PythonCmd,"With Tk < 8.5 _tkinter.c:PythonCmd() raised UnicodeDecodeError, caused"
-whatsnew/changelog,,::,": Fix FTP tests for IPv6, bind to ""::1"" instead of ""localhost""."
whatsnew/changelog,,::,": Use ""127.0.0.1"" or ""::1"" instead of ""localhost"" as much as"
-whatsnew/changelog,,:password,user:password
-whatsnew/2.7,780,:Sunday,'2009:4:Sunday'
-whatsnew/2.7,907,::,"export PYTHONWARNINGS=all,error:::Cookie:0"
-whatsnew/2.7,907,:Cookie,"export PYTHONWARNINGS=all,error:::Cookie:0"
-whatsnew/2.7,1657,::,>>> urlparse.urlparse('http://[1080::8:800:200C:417A]/foo')
-whatsnew/2.7,1657,::,"ParseResult(scheme='http', netloc='[1080::8:800:200C:417A]',"
+library/tarfile,149,:xz,'x:xz'
+library/xml.etree.elementtree,290,:sometag,prefix:sometag
+library/xml.etree.elementtree,301,:fictional,"<actors xmlns:fictional=""http://characters.example.com"""
+library/xml.etree.elementtree,301,:character,<fictional:character>Lancelot</fictional:character>
+library/xml.etree.elementtree,301,:character,<fictional:character>Archie Leach</fictional:character>
+library/xml.etree.elementtree,301,:character,<fictional:character>Sir Robin</fictional:character>
+library/xml.etree.elementtree,301,:character,<fictional:character>Gunther</fictional:character>
+library/xml.etree.elementtree,301,:character,<fictional:character>Commander Clement</fictional:character>
+library/xml.etree.elementtree,332,:actor,"for actor in root.findall('real_person:actor', ns):"
+library/xml.etree.elementtree,332,:name,"name = actor.find('real_person:name', ns)"
+library/xml.etree.elementtree,332,:character,"for char in actor.findall('role:character', ns):"
+library/zipapp,31,:main,"$ python -m zipapp myapp -m ""myapp:main"""
+library/zipapp,82,:fn,"argument should have the form ""pkg.mod:fn"", where ""pkg.mod"" is a"
+library/zipapp,155,:callable,"""pkg.module:callable"" and the archive will be run by importing"
+library/stdtypes,3767,::,>>> m[::2].tolist()
+library/sys,1115,`,# ``wrapper`` creates a ``wrap(coro)`` coroutine:
+tutorial/venv,77,:c7b9645a6f35,"Python 3.4.3+ (3.4:c7b9645a6f35+, May 22 2015, 09:31:25)"
diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html
index abdf070..78e9c4f 100644
--- a/Doc/tools/templates/indexsidebar.html
+++ b/Doc/tools/templates/indexsidebar.html
@@ -3,8 +3,7 @@
<h3>Docs for other versions</h3>
<ul>
<li><a href="https://docs.python.org/2.7/">Python 2.7 (stable)</a></li>
- <li><a href="https://docs.python.org/3.3/">Python 3.3 (stable)</a></li>
- <li><a href="https://docs.python.org/3.5/">Python 3.5 (in development)</a></li>
+ <li><a href="https://docs.python.org/3.4/">Python 3.4 (stable)</a></li>
<li><a href="https://www.python.org/doc/versions/">Old versions</a></li>
</ul>
diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst
index 8670efc..67262a1 100644
--- a/Doc/tutorial/appendix.rst
+++ b/Doc/tutorial/appendix.rst
@@ -40,7 +40,7 @@ Executable Python Scripts
On BSD'ish Unix systems, Python scripts can be made directly executable, like
shell scripts, by putting the line ::
- #!/usr/bin/env python3.4
+ #!/usr/bin/env python3.5
(assuming that the interpreter is on the user's :envvar:`PATH`) at the beginning
of the script and giving the file an executable mode. The ``#!`` must be the
@@ -107,7 +107,7 @@ of your user site-packages directory. Start Python and run this code::
>>> import site
>>> site.getusersitepackages()
- '/home/user/.local/lib/python3.4/site-packages'
+ '/home/user/.local/lib/python3.5/site-packages'
Now you can create a file named :file:`usercustomize.py` in that directory and
put anything you want in it. It will affect every invocation of Python, unless
diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst
index 1ea299f..0d51480 100644
--- a/Doc/tutorial/datastructures.rst
+++ b/Doc/tutorial/datastructures.rst
@@ -73,10 +73,11 @@ objects:
Return the number of times *x* appears in the list.
-.. method:: list.sort()
+.. method:: list.sort(key=None, reverse=False)
:noindex:
- Sort the items of the list in place.
+ Sort the items of the list in place (the arguments can be used for sort
+ customization, see :func:`sorted` for their explanation).
.. method:: list.reverse()
@@ -611,18 +612,18 @@ returns a new sorted list while leaving the source unaltered. ::
orange
pear
-To change a sequence you are iterating over while inside the loop (for
-example to duplicate certain items), it is recommended that you first make
-a copy. Looping over a sequence does not implicitly make a copy. The slice
-notation makes this especially convenient::
+It is sometimes tempting to change a list while you are looping over it;
+however, it is often simpler and safer to create a new list instead. ::
- >>> words = ['cat', 'window', 'defenestrate']
- >>> for w in words[:]: # Loop over a slice copy of the entire list.
- ... if len(w) > 6:
- ... words.insert(0, w)
+ >>> import math
+ >>> raw_data = [56.2, float('NaN'), 51.7, 55.3, 52.5, float('NaN'), 47.8]
+ >>> filtered_data = []
+ >>> for value in raw_data:
+ ... if not math.isnan(value):
+ ... filtered_data.append(value)
...
- >>> words
- ['defenestrate', 'cat', 'window', 'defenestrate']
+ >>> filtered_data
+ [56.2, 51.7, 55.3, 52.5, 47.8]
.. _tut-conditions:
diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst
index 8051634..d5789a6 100644
--- a/Doc/tutorial/interpreter.rst
+++ b/Doc/tutorial/interpreter.rst
@@ -10,13 +10,13 @@ Using the Python Interpreter
Invoking the Interpreter
========================
-The Python interpreter is usually installed as :file:`/usr/local/bin/python3.4`
+The Python interpreter is usually installed as :file:`/usr/local/bin/python3.5`
on those machines where it is available; putting :file:`/usr/local/bin` in your
Unix shell's search path makes it possible to start it by typing the command:
.. code-block:: text
- python3.4
+ python3.5
to the shell. [#]_ Since the choice of the directory where the interpreter lives
is an installation option, other places are possible; check with your local
@@ -24,11 +24,11 @@ Python guru or system administrator. (E.g., :file:`/usr/local/python` is a
popular alternative location.)
On Windows machines, the Python installation is usually placed in
-:file:`C:\\Python34`, though you can change this when you're running the
+:file:`C:\\Python35`, though you can change this when you're running the
installer. To add this directory to your path, you can type the following
command into the command prompt in a DOS box::
- set path=%path%;C:\python34
+ set path=%path%;C:\python35
Typing an end-of-file character (:kbd:`Control-D` on Unix, :kbd:`Control-Z` on
Windows) at the primary prompt causes the interpreter to exit with a zero exit
@@ -96,8 +96,8 @@ with the *secondary prompt*, by default three dots (``...``). The interpreter
prints a welcome message stating its version number and a copyright notice
before printing the first prompt::
- $ python3.4
- Python 3.4 (default, Mar 16 2014, 09:25:04)
+ $ python3.5
+ Python 3.5 (default, Sep 16 2015, 09:25:04)
[GCC 4.8.2] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>>
diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst
index fd361ae..9ae64b0 100644
--- a/Doc/tutorial/modules.rst
+++ b/Doc/tutorial/modules.rst
@@ -216,15 +216,15 @@ Some tips for experts:
statements, the ``-OO`` switch removes both assert statements and __doc__
strings. Since some programs may rely on having these available, you should
only use this option if you know what you're doing. "Optimized" modules have
- a .pyo rather than a .pyc suffix and are usually smaller. Future releases may
+ an ``opt-`` tag and are usually smaller. Future releases may
change the effects of optimization.
-* A program doesn't run any faster when it is read from a ``.pyc`` or ``.pyo``
+* A program doesn't run any faster when it is read from a ``.pyc``
file than when it is read from a ``.py`` file; the only thing that's faster
- about ``.pyc`` or ``.pyo`` files is the speed with which they are loaded.
+ about ``.pyc`` files is the speed with which they are loaded.
-* The module :mod:`compileall` can create .pyc files (or .pyo files when
- :option:`-O` is used) for all modules in a directory.
+* The module :mod:`compileall` can create .pyc files for all modules in a
+ directory.
* There is more detail on this process, including a flow chart of the
decisions, in PEP 3147.
@@ -548,4 +548,3 @@ modules found in a package.
.. [#] In fact function definitions are also 'statements' that are 'executed'; the
execution of a module-level function definition enters the function name in
the module's global symbol table.
-
diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst
index 72d51de..0954eba 100644
--- a/Doc/tutorial/stdlib.rst
+++ b/Doc/tutorial/stdlib.rst
@@ -15,7 +15,7 @@ operating system::
>>> import os
>>> os.getcwd() # Return the current working directory
- 'C:\\Python34'
+ 'C:\\Python35'
>>> os.chdir('/server/accesslogs') # Change current working directory
>>> os.system('mkdir today') # Run the command mkdir in the system shell
0
diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst
index c0197ea..f7d2a0a 100644
--- a/Doc/tutorial/stdlib2.rst
+++ b/Doc/tutorial/stdlib2.rst
@@ -18,7 +18,7 @@ abbreviated displays of large or deeply nested containers::
>>> import reprlib
>>> reprlib.repr(set('supercalifragilisticexpialidocious'))
- "set(['a', 'c', 'd', 'e', 'f', 'g', ...])"
+ "{'a', 'c', 'd', 'e', 'f', 'g', ...}"
The :mod:`pprint` module offers more sophisticated control over printing both
built-in and user defined objects in a way that is readable by the interpreter.
@@ -277,7 +277,7 @@ applications include caching objects that are expensive to create::
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
d['primary'] # entry was automatically removed
- File "C:/python34/lib/weakref.py", line 46, in __getitem__
+ File "C:/python35/lib/weakref.py", line 46, in __getitem__
o = self.data[key]()
KeyError: 'primary'
diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst
index 4017ce8..701c624 100644
--- a/Doc/using/cmdline.rst
+++ b/Doc/using/cmdline.rst
@@ -190,13 +190,16 @@ Miscellaneous options
.. cmdoption:: -b
- Issue a warning when comparing str and bytes. Issue an error when the
+ Issue a warning when comparing :class:`bytes` or :class:`bytearray` with
+ :class:`str` or :class:`bytes` with :class:`int`. Issue an error when the
option is given twice (:option:`-bb`).
+ .. versionchanged: 3.5
+ Affects comparisons of :class:`bytes` with :class:`int`.
.. cmdoption:: -B
- If given, Python won't try to write ``.pyc`` or ``.pyo`` files on the
+ If given, Python won't try to write ``.pyc`` files on the
import of source modules. See also :envvar:`PYTHONDONTWRITEBYTECODE`.
diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc
index 45bdd5a..02bcbee 100644
--- a/Doc/using/venv-create.inc
+++ b/Doc/using/venv-create.inc
@@ -21,11 +21,11 @@ subdirectory (on Windows, this is ``Lib\site-packages``).
On Windows, you may have to invoke the ``pyvenv`` script as follows, if you
don't have the relevant PATH and PATHEXT settings::
- c:\Temp>c:\Python34\python c:\Python34\Tools\Scripts\pyvenv.py myenv
+ c:\Temp>c:\Python35\python c:\Python35\Tools\Scripts\pyvenv.py myenv
or equivalently::
- c:\Temp>c:\Python34\python -m venv myenv
+ c:\Temp>c:\Python35\python -m venv myenv
The command, if run with ``-h``, will show the available options::
diff --git a/Doc/using/win_installer.png b/Doc/using/win_installer.png
new file mode 100644
index 0000000..00c88a8
--- /dev/null
+++ b/Doc/using/win_installer.png
Binary files differ
diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst
index c05f72a..5c19b81 100644
--- a/Doc/using/windows.rst
+++ b/Doc/using/windows.rst
@@ -7,22 +7,213 @@
*************************
.. sectionauthor:: Robert Lehmann <lehmannro@gmail.com>
+.. sectionauthor:: Steve Dower <steve.dower@microsoft.com>
This document aims to give an overview of Windows-specific behaviour you should
know about when using Python on Microsoft Windows.
-.. XXX (ncoghlan)
-
- This looks rather stale to me...
-
-
Installing Python
=================
-Unlike most Unix systems and services, Windows does not require Python natively
-and thus does not pre-install a version of Python. However, the CPython team
+Unlike most Unix systems and services, Windows does not include a system
+supported installation of Python. To make Python available, the CPython team
has compiled Windows installers (MSI packages) with every `release
-<https://www.python.org/download/releases/>`_ for many years.
+<https://www.python.org/download/releases/>`_ for many years. These installers
+are primarily intended to add a per-user installation of Python, with the
+core interpreter and library being used by a single user. The installer is also
+able to install for all users of a single machine, and a separate ZIP file is
+available for application-local distributions.
+
+Installation Steps
+------------------
+
+Four Python 3.5 installers are available for download - two each for the 32-bit
+and 64-bit versions of the interpreter. The *web installer* is a small initial
+download, and it will automatically download the required components as
+necessary. The *offline installer* includes the components necessary for a
+default installation and only requires an internet connection for optional
+features. See :ref:`install-layout-option` for other ways to avoid downloading
+during installation.
+
+After starting the installer, one of two options may be selected:
+
+.. image:: win_installer.png
+
+If you select "Install Now":
+
+* You will *not* need to be an administrator (unless a system update for the
+ C Runtime Library is required or you install the :ref:`launcher` for all
+ users)
+* Python will be installed into your user directory
+* The :ref:`launcher` will be installed according to the option at the bottom
+ of the first pace
+* The standard library, test suite, launcher and pip will be installed
+* If selected, the install directory will be added to your :envvar:`PATH`
+* Shortcuts will only be visible for the current user
+
+Selecting "Customize installation" will allow you to select the features to
+install, the installation location and other options or post-install actions.
+To install debugging symbols or binaries, you will need to use this option.
+
+To perform an all-users installation, you should select "Customize
+installation". In this case:
+
+* You may be required to provide administrative credentials or approval
+* Python will be installed into the Program Files directory
+* The :ref:`launcher` will be installed into the Windows directory
+* Optional features may be selected during installation
+* The standard library can be pre-compiled to bytecode
+* If selected, the install directory will be added to the system :envvar:`PATH`
+* Shortcuts are available for all users
+
+.. _install-quiet-option:
+
+Installing Without UI
+---------------------
+
+All of the options available in the installer UI can also be specified from the
+command line, allowing scripted installers to replicate an installation on many
+machines without user interaction. These options may also be set without
+suppressing the UI in order to change some of the defaults.
+
+To completely hide the installer UI and install Python silently, pass the
+``/quiet`` (``/q``) option. To skip past the user interaction but still display
+progress and errors, pass the ``/passive`` (``/p``) option. The ``/uninstall``
+option may be passed to immediately begin removing Python - no prompt will be
+displayed.
+
+All other options are passed as ``name=value``, where the value is usually
+``0`` to disable a feature, ``1`` to enable a feature, or a path. The full list
+of available options is shown below.
+
++---------------------------+--------------------------------------+--------------------------+
+| Name | Description | Default |
++===========================+======================================+==========================+
+| InstallAllUsers | Perform a system-wide installation. | 1 |
++---------------------------+--------------------------------------+--------------------------+
+| TargetDir | The installation directory | Selected based on |
+| | | InstallAllUsers |
++---------------------------+--------------------------------------+--------------------------+
+| DefaultAllUsersTargetDir | The default installation directory | :file:`%ProgramFiles%\\\ |
+| | for all-user installs | Python X.Y` or :file:`\ |
+| | | %ProgramFiles(x86)%\\\ |
+| | | Python X.Y` |
++---------------------------+--------------------------------------+--------------------------+
+| DefaultJustForMeTargetDir | The default install directory for | :file:`%LocalAppData%\\\ |
+| | just-for-me installs | Programs\\PythonXY` or |
+| | | :file:`%LocalAppData%\\\ |
+| | | Programs\\PythonXY-32` |
++---------------------------+--------------------------------------+--------------------------+
+| DefaultCustomTargetDir | The default custom install directory | (empty) |
+| | displayed in the UI | |
++---------------------------+--------------------------------------+--------------------------+
+| AssociateFiles | Create file associations if the | 1 |
+| | launcher is also installed. | |
++---------------------------+--------------------------------------+--------------------------+
+| CompileAll | Compile all ``.py`` files to | 0 |
+| | ``.pyc``. | |
++---------------------------+--------------------------------------+--------------------------+
+| PrependPath | Add install and Scripts directories | 0 |
+| | tho :envvar:`PATH` and ``.PY`` to | |
+| | :envvar:`PATHEXT` | |
++---------------------------+--------------------------------------+--------------------------+
+| Shortcuts | Create shortcuts for the interpreter,| 1 |
+| | documentation and IDLE if installed. | |
++---------------------------+--------------------------------------+--------------------------+
+| Include_doc | Install Python manual | 1 |
++---------------------------+--------------------------------------+--------------------------+
+| Include_debug | Install debug binaries | 0 |
++---------------------------+--------------------------------------+--------------------------+
+| Include_dev | Install developer headers and | 1 |
+| | libraries | |
++---------------------------+--------------------------------------+--------------------------+
+| Include_exe | Install :file:`python.exe` and | 1 |
+| | related files | |
++---------------------------+--------------------------------------+--------------------------+
+| Include_launcher | Install :ref:`launcher`. | 1 |
++---------------------------+--------------------------------------+--------------------------+
+| InstallLauncherAllUsers | Installs :ref:`launcher` for all | 1 |
+| | users. | |
++---------------------------+--------------------------------------+--------------------------+
+| Include_lib | Install standard library and | 1 |
+| | extension modules | |
++---------------------------+--------------------------------------+--------------------------+
+| Include_pip | Install bundled pip and setuptools | 1 |
++---------------------------+--------------------------------------+--------------------------+
+| Include_symbols | Install debugging symbols (`*`.pdb) | 0 |
++---------------------------+--------------------------------------+--------------------------+
+| Include_tcltk | Install Tcl/Tk support and IDLE | 1 |
++---------------------------+--------------------------------------+--------------------------+
+| Include_test | Install standard library test suite | 1 |
++---------------------------+--------------------------------------+--------------------------+
+| Include_tools | Install utility scripts | 1 |
++---------------------------+--------------------------------------+--------------------------+
+| LauncherOnly | Only installs the launcher. This | 0 |
+| | will override most other options. | |
++---------------------------+--------------------------------------+--------------------------+
+| SimpleInstall | Disable most install UI | 0 |
++---------------------------+--------------------------------------+--------------------------+
+| SimpleInstallDescription | A custom message to display when the | (empty) |
+| | simplified install UI is used. | |
++---------------------------+--------------------------------------+--------------------------+
+
+For example, to silently install a default, system-wide Python installation,
+you could use the following command (from an elevated command prompt)::
+
+ python-3.5.0.exe /quiet InstallAllUsers=1 PrependPath=1 Include_test=0
+
+To allow users to easily install a personal copy of Python without the test
+suite, you could provide a shortcut with the following command. This will
+display a simplified initial page and disallow customization::
+
+ python-3.5.0.exe InstallAllUsers=0 Include_launcher=0 Include_test=0
+ SimpleInstall=1 SimpleInstallDescription="Just for me, no test suite."
+
+(Note that omitting the launcher also omits file associations, and is only
+recommended for per-user installs when there is also a system-wide installation
+that included the launcher.)
+
+The options listed above can also be provided in a file named ``unattend.xml``
+alongside the executable. This file specifies a list of options and values.
+When a value is provided as an attribute, it will be converted to a number if
+possible. Values provided as element text are always left as strings. This
+example file sets the same options and the previous example::
+
+ <Options>
+ <Option Name="InstallAllUsers" Value="no" />
+ <Option Name="Include_launcher" Value="0" />
+ <Option Name="Include_test" Value="no" />
+ <Option Name="SimpleInstall" Value="yes" />
+ <Option Name="SimpleInstallDescription">Just for me, no test suite</Option>
+ </Options>
+
+.. _install-layout-option:
+
+Installing Without Downloading
+------------------------------
+
+As some features of Python are not included in the initial installer download,
+selecting those features may require an internet connection. To avoid this
+need, all possible components may be downloaded on-demand to create a complete
+*layout* that will no longer require an internet connection regardless of the
+selected features. Note that this download may be bigger than required, but
+where a large number of installations are going to be performed it is very
+useful to have a locally cached copy.
+
+Execute the following command from Command Prompt to download all possible
+required files. Remember to substitute ``python-3.5.0.exe`` for the actual
+name of your installer, and to create layouts in their own directories to
+avoid collisions between files with the same name.
+
+::
+
+ python-3.5.0.exe /layout [optional target directory]
+
+You may also specify the ``/quiet`` option to hide the progress display.
+
+
+Other Platforms
+---------------
With ongoing development of Python, some platforms that used to be supported
earlier are no longer supported (due to the lack of users or developers).
@@ -66,19 +257,31 @@ key features:
`ActivePython <http://www.activestate.com/activepython/>`_
Installer with multi-platform compatibility, documentation, PyWin32
-`Enthought Python Distribution <https://www.enthought.com/products/epd/>`_
- Popular modules (such as PyWin32) with their respective documentation, tool
- suite for building extensible Python applications
+`Anaconda <http://www.continuum.io/downloads/>`_
+ Popular scientific modules (such as numpy, scipy and pandas) and the
+ ``conda`` package manager.
+
+`Canopy <https://www.enthought.com/products/canopy/>`_
+ A "comprehensive Python analysis environment" with editors and other
+ development tools.
-Notice that these packages are likely to install *older* versions of Python.
+`WinPython <https://winpython.github.io/>`_
+ Windows-specific distribution with prebuilt scientific packages and
+ tools for building packages.
+
+Note that these packages may not include the latest versions of Python or
+other libraries, and are not maintained or supported by the core Python team.
Configuring Python
==================
-In order to run Python flawlessly, you might have to change certain environment
-settings in Windows.
+To run Python conveniently from a command prompt, you might consider changing
+some default environment variables in Windows. While the installer provides an
+option to configure the PATH and PATHEXT variables for you, this is only
+reliable for a single, system-wide installation. If you regularly use multiple
+versions of Python, consider using the :ref:`launcher`.
.. _setting-envvars:
@@ -86,163 +289,86 @@ settings in Windows.
Excursus: Setting environment variables
---------------------------------------
-Windows has a built-in dialog for changing environment variables (following
-guide applies to XP classical view): Right-click the icon for your machine
-(usually located on your Desktop and called "My Computer") and choose
-:menuselection:`Properties` there. Then, open the :guilabel:`Advanced` tab
-and click the :guilabel:`Environment Variables` button.
+Windows allows environment variables to be configured permanently at both the
+User level and the System level, or temporarily in a command prompt.
+
+To temporarily set environment variables, open Command Prompt and use the
+:command:`set` command::
+
+ C:\>set PATH=C:\Program Files\Python 3.5;%PATH%
+ C:\>set PYTHONPATH=%PYTHONPATH%;C:\My_python_lib
+ C:\>python
-In short, your path is:
+These changes will apply to any further commands executed in that console, and
+will be inherited by any applications started from the console.
- :menuselection:`My Computer
- --> Properties
- --> Advanced
- --> Environment Variables`
+Including the variable name within percent signs will expand to the existing
+value, allowing you to add your new value at either the start or the end.
+Modifying :envvar:`PATH` by adding the directory containing
+:program:`python.exe` to the start is a common way to ensure the correct version
+of Python is launched.
+To permanently modify the default environment variables, click Start and search
+for 'edit environment variables', or open System properties, :guilabel:`Advanced
+system settings` and click the :guilabel:`Environment Variables` button.
In this dialog, you can add or modify User and System variables. To change
System variables, you need non-restricted access to your machine
(i.e. Administrator rights).
-Another way of adding variables to your environment is using the :command:`set`
-command::
-
- set PYTHONPATH=%PYTHONPATH%;C:\My_python_lib
-
-To make this setting permanent, you could add the corresponding command line to
-your :file:`autoexec.bat`. :program:`msconfig` is a graphical interface to this
-file.
+.. note::
-Viewing environment variables can also be done more straight-forward: The
-command prompt will expand strings wrapped into percent signs automatically::
+ Windows will concatenate User variables *after* System variables, which may
+ cause unexpected results when modifying :envvar:`PATH`.
- echo %PATH%
-
-Consult :command:`set /?` for details on this behaviour.
+ The :envvar:`PYTHONPATH` variable is used by all versions of Python 2 and
+ Python 3, so you should not permanently configure this variable unless it
+ only includes code that is compatible with all of your installed Python
+ versions.
.. seealso::
- http://support.microsoft.com/kb/100843
+ http://support.microsoft.com/kb/100843
Environment variables in Windows NT
- http://support.microsoft.com/kb/310519
+ http://technet.microsoft.com/en-us/library/cc754250.aspx
+ The SET command, for temporarily modifying environment variables
+
+ http://technet.microsoft.com/en-us/library/cc755104.aspx
+ The SETX command, for permanently modifying environment variables
+
+ http://support.microsoft.com/kb/310519
How To Manage Environment Variables in Windows XP
- http://www.chem.gla.ac.uk/~louis/software/faq/q1.html
+ http://www.chem.gla.ac.uk/~louis/software/faq/q1.html
Setting Environment variables, Louis J. Farrugia
-
.. _windows-path-mod:
Finding the Python executable
-----------------------------
-.. versionchanged:: 3.3
+.. versionchanged:: 3.5
Besides using the automatically created start menu entry for the Python
-interpreter, you might want to start Python in the command prompt. As of
-Python 3.3, the installer has an option to set that up for you.
-
-At the "Customize Python 3.3" screen, an option called
-"Add python.exe to search path" can be enabled to have the installer place
-your installation into the :envvar:`%PATH%`. This allows you to type
-:command:`python` to run the interpreter. Thus, you can also execute your
+interpreter, you might want to start Python in the command prompt. The
+installer for Python 3.5 and later has an option to set that up for you.
+
+On the first page of the installer, an option labelled "Add Python 3.5 to
+PATH" can be selected to have the installer add the install location into the
+:envvar:`PATH`. The location of the :file:`Scripts\\` folder is also added.
+This allows you to type :command:`python` to run the interpreter, and
+:command:`pip` or . Thus, you can also execute your
scripts with command line options, see :ref:`using-on-cmdline` documentation.
If you don't enable this option at install time, you can always re-run the
-installer to choose it.
-
-The alternative is manually modifying the :envvar:`%PATH%` using the
-directions in :ref:`setting-envvars`. You need to set your :envvar:`%PATH%`
-environment variable to include the directory of your Python distribution,
-delimited by a semicolon from other entries. An example variable could look
-like this (assuming the first two entries are Windows' default)::
-
- C:\WINDOWS\system32;C:\WINDOWS;C:\Python33
-
-
-Finding modules
----------------
-
-Python usually stores its library (and thereby your site-packages folder) in the
-installation directory. So, if you had installed Python to
-:file:`C:\\Python\\`, the default library would reside in
-:file:`C:\\Python\\Lib\\` and third-party modules should be stored in
-:file:`C:\\Python\\Lib\\site-packages\\`.
-
-This is how :data:`sys.path` is populated on Windows:
-
-* An empty entry is added at the start, which corresponds to the current
- directory.
-
-* If the environment variable :envvar:`PYTHONPATH` exists, as described in
- :ref:`using-on-envvars`, its entries are added next. Note that on Windows,
- paths in this variable must be separated by semicolons, to distinguish them
- from the colon used in drive identifiers (``C:\`` etc.).
-
-* Additional "application paths" can be added in the registry as subkeys of
- :samp:`\\SOFTWARE\\Python\\PythonCore\\{version}\\PythonPath` under both the
- ``HKEY_CURRENT_USER`` and ``HKEY_LOCAL_MACHINE`` hives. Subkeys which have
- semicolon-delimited path strings as their default value will cause each path
- to be added to :data:`sys.path`. (Note that all known installers only use
- HKLM, so HKCU is typically empty.)
-
-* If the environment variable :envvar:`PYTHONHOME` is set, it is assumed as
- "Python Home". Otherwise, the path of the main Python executable is used to
- locate a "landmark file" (``Lib\os.py``) to deduce the "Python Home". If a
- Python home is found, the relevant sub-directories added to :data:`sys.path`
- (``Lib``, ``plat-win``, etc) are based on that folder. Otherwise, the core
- Python path is constructed from the PythonPath stored in the registry.
-
-* If the Python Home cannot be located, no :envvar:`PYTHONPATH` is specified in
- the environment, and no registry entries can be found, a default path with
- relative entries is used (e.g. ``.\Lib;.\plat-win``, etc).
-
-The end result of all this is:
-
-* When running :file:`python.exe`, or any other .exe in the main Python
- directory (either an installed version, or directly from the PCbuild
- directory), the core path is deduced, and the core paths in the registry are
- ignored. Other "application paths" in the registry are always read.
-
-* When Python is hosted in another .exe (different directory, embedded via COM,
- etc), the "Python Home" will not be deduced, so the core path from the
- registry is used. Other "application paths" in the registry are always read.
-
-* If Python can't find its home and there is no registry (eg, frozen .exe, some
- very strange installation setup) you get a path with some default, but
- relative, paths.
-
-
-Executing scripts
------------------
-
-As of Python 3.3, Python includes a launcher which facilitates running Python
-scripts. See :ref:`launcher` for more information.
-
-Executing scripts without the Python launcher
----------------------------------------------
-
-Without the Python launcher installed, Python scripts (files with the extension
-``.py``) will be executed by :program:`python.exe` by default. This executable
-opens a terminal, which stays open even if the program uses a GUI. If you do
-not want this to happen, use the extension ``.pyw`` which will cause the script
-to be executed by :program:`pythonw.exe` by default (both executables are
-located in the top-level of your Python installation directory). This
-suppresses the terminal window on startup.
-
-You can also make all ``.py`` scripts execute with :program:`pythonw.exe`,
-setting this through the usual facilities, for example (might require
-administrative rights):
-
-#. Launch a command prompt.
-#. Associate the correct file group with ``.py`` scripts::
-
- assoc .py=Python.File
-
-#. Redirect all Python files to the new executable::
-
- ftype Python.File=C:\Path\to\pythonw.exe "%1" %*
+installer, select Modify, and enable it. Alternatively, you can manually
+modify the :envvar:`PATH` using the directions in :ref:`setting-envvars`. You
+need to set your :envvar:`PATH` environment variable to include the directory
+of your Python installation, delimited by a semicolon from other entries. An
+example variable could look like this (assuming the first two entries already
+existed)::
+ C:\WINDOWS\system32;C:\WINDOWS;C:\Program Files\Python 3.5
.. _launcher:
@@ -251,21 +377,26 @@ Python Launcher for Windows
.. versionadded:: 3.3
-The Python launcher for Windows is a utility which aids in the location and
-execution of different Python versions. It allows scripts (or the
+The Python launcher for Windows is a utility which aids in locating and
+executing of different Python versions. It allows scripts (or the
command-line) to indicate a preference for a specific Python version, and
will locate and execute that version.
+Unlike the :envvar:`PATH` variable, the launcher will correctly select the most
+appropriate version of Python. It will prefer per-user installations over
+system-wide ones, and orders by language version rather than using the most
+recently installed version.
+
Getting started
---------------
From the command-line
^^^^^^^^^^^^^^^^^^^^^
-You should ensure the launcher is on your PATH - depending on how it was
-installed it may already be there, but check just in case it is not.
-
-From a command-prompt, execute the following command:
+System-wide installations of Python 3.3 and later will put the launcher on your
+:envvar:`PATH`. The launcher is compatible with all available versions of
+Python, so it does not matter which version is installed. To check that the
+launcher is available, execute the following command in Command Prompt:
::
@@ -291,6 +422,28 @@ If you have a Python 3.x installed, try the command:
You should find the latest version of Python 3.x starts.
+If you see the following error, you do not have the launcher installed:
+
+::
+
+ 'py' is not recognized as an internal or external command,
+ operable program or batch file.
+
+Per-user installations of Python do not add the launcher to :envvar:`PATH`
+unless the option was selected on installation.
+
+Virtual environments
+^^^^^^^^^^^^^^^^^^^^
+
+.. versionadded:: 3.5
+
+If the launcher is run with no explicit Python version specification, and a
+virtual environment (created with the standard library :mod:`venv` module or
+the external ``virtualenv`` tool) active, the launcher will run the virtual
+environment's interpreter rather than the global one. To run the global
+interpreter, either deactivate the virtual environment, or explicitly specify
+the global Python version.
+
From a script
^^^^^^^^^^^^^
@@ -326,7 +479,7 @@ From file associations
^^^^^^^^^^^^^^^^^^^^^^
The launcher should have been associated with Python files (i.e. ``.py``,
-``.pyw``, ``.pyc``, ``.pyo`` files) when it was installed. This means that
+``.pyw``, ``.pyc`` files) when it was installed. This means that
when you double-click on one of these files from Windows explorer the launcher
will be used, and therefore you can use the same facilities described above to
have the script specify the version which should be used.
@@ -365,6 +518,16 @@ be used by the launcher without modification. If you are writing a new script
on Windows which you hope will be useful on Unix, you should use one of the
shebang lines starting with ``/usr``.
+Any of the above virtual commands can be suffixed with an explicit version
+(either just the major version, or the major and minor version) - for example
+``/usr/bin/python2.7`` - which will cause that specific version to be located
+and used.
+
+The ``/usr/bin/env`` form of shebang line has one further special property.
+Before looking for installed Python interpreters, this form will search the
+executable :envvar:`PATH` for a Python executable. This corresponds to the
+behaviour of the Unix ``env`` program, which performs a :envvar:`PATH` search.
+
Arguments in shebang lines
--------------------------
@@ -383,17 +546,16 @@ Customization
Customization via INI files
^^^^^^^^^^^^^^^^^^^^^^^^^^^
- Two .ini files will be searched by the launcher - ``py.ini`` in the
- current user's "application data" directory (i.e. the directory returned
- by calling the Windows function SHGetFolderPath with CSIDL_LOCAL_APPDATA)
- and ``py.ini`` in the same directory as the launcher. The same .ini
- files are used for both the 'console' version of the launcher (i.e.
- py.exe) and for the 'windows' version (i.e. pyw.exe)
+Two .ini files will be searched by the launcher - ``py.ini`` in the current
+user's "application data" directory (i.e. the directory returned by calling the
+Windows function SHGetFolderPath with CSIDL_LOCAL_APPDATA) and ``py.ini`` in the
+same directory as the launcher. The same .ini files are used for both the
+'console' version of the launcher (i.e. py.exe) and for the 'windows' version
+(i.e. pyw.exe)
- Customization specified in the "application directory" will have
- precedence over the one next to the executable, so a user, who may not
- have write access to the .ini file next to the launcher, can override
- commands in that global .ini file)
+Customization specified in the "application directory" will have precedence over
+the one next to the executable, so a user, who may not have write access to the
+.ini file next to the launcher, can override commands in that global .ini file)
Customizing default Python versions
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -488,6 +650,99 @@ particular version was chosen and the exact command-line used to execute the
target Python.
+
+.. finding_modules:
+
+Finding modules
+===============
+
+Python usually stores its library (and thereby your site-packages folder) in the
+installation directory. So, if you had installed Python to
+:file:`C:\\Python\\`, the default library would reside in
+:file:`C:\\Python\\Lib\\` and third-party modules should be stored in
+:file:`C:\\Python\\Lib\\site-packages\\`.
+
+This is how :data:`sys.path` is populated on Windows:
+
+* An empty entry is added at the start, which corresponds to the current
+ directory.
+
+* If the environment variable :envvar:`PYTHONPATH` exists, as described in
+ :ref:`using-on-envvars`, its entries are added next. Note that on Windows,
+ paths in this variable must be separated by semicolons, to distinguish them
+ from the colon used in drive identifiers (``C:\`` etc.).
+
+* Additional "application paths" can be added in the registry as subkeys of
+ :samp:`\\SOFTWARE\\Python\\PythonCore\\{version}\\PythonPath` under both the
+ ``HKEY_CURRENT_USER`` and ``HKEY_LOCAL_MACHINE`` hives. Subkeys which have
+ semicolon-delimited path strings as their default value will cause each path
+ to be added to :data:`sys.path`. (Note that all known installers only use
+ HKLM, so HKCU is typically empty.)
+
+* If the environment variable :envvar:`PYTHONHOME` is set, it is assumed as
+ "Python Home". Otherwise, the path of the main Python executable is used to
+ locate a "landmark file" (``Lib\os.py``) to deduce the "Python Home". If a
+ Python home is found, the relevant sub-directories added to :data:`sys.path`
+ (``Lib``, ``plat-win``, etc) are based on that folder. Otherwise, the core
+ Python path is constructed from the PythonPath stored in the registry.
+
+* If the Python Home cannot be located, no :envvar:`PYTHONPATH` is specified in
+ the environment, and no registry entries can be found, a default path with
+ relative entries is used (e.g. ``.\Lib;.\plat-win``, etc).
+
+If a ``pyvenv.cfg`` file is found alongside the main executable or in the
+directory one level above the executable, the following variations apply:
+
+* If ``home`` is an absolute path and :envvar:`PYTHONHOME` is not set, this
+ path is used instead of the path to the main executable when deducing the
+ home location.
+
+* If ``applocal`` is set to true, the ``home`` property or the main executable
+ is always used as the home path, and all environment variables or registry
+ values affecting the path are ignored. The landmark file is not checked.
+
+The end result of all this is:
+
+* When running :file:`python.exe`, or any other .exe in the main Python
+ directory (either an installed version, or directly from the PCbuild
+ directory), the core path is deduced, and the core paths in the registry are
+ ignored. Other "application paths" in the registry are always read.
+
+* When Python is hosted in another .exe (different directory, embedded via COM,
+ etc), the "Python Home" will not be deduced, so the core path from the
+ registry is used. Other "application paths" in the registry are always read.
+
+* If Python can't find its home and there are no registry value (frozen .exe,
+ some very strange installation setup) you get a path with some default, but
+ relative, paths.
+
+For those who want to bundle Python into their application or distribution, the
+following advice will prevent conflicts with other installations:
+
+* Include a ``pyvenv.cfg`` file alongside your executable containing
+ ``applocal = true``. This will ensure that your own directory will be used to
+ resolve paths even if you have included the standard library in a ZIP file.
+ It will also ignore user site-packages and other paths listed in the
+ registry.
+
+* If you are loading :file:`python3.dll` or :file:`python35.dll` in your own
+ executable, explicitly call :c:func:`Py_SetPath` or (at least)
+ :c:func:`Py_SetProgramName` before :c:func:`Py_Initialize`.
+
+* Clear and/or overwrite :envvar:`PYTHONPATH` and set :envvar:`PYTHONHOME`
+ before launching :file:`python.exe` from your application.
+
+* If you cannot use the previous suggestions (for example, you are a
+ distribution that allows people to run :file:`python.exe` directly), ensure
+ that the landmark file (:file:`Lib\\os.py`) exists in your install directory.
+ (Note that it will not be detected inside a ZIP file.)
+
+These will ensure that the files in a system-wide installation will not take
+precedence over the copy of the standard library bundled with your application.
+Otherwise, your users may experience problems using your application. Note that
+the first suggestion is the best, as the other may still be susceptible to
+non-standard paths in the registry and user site-packages.
+
Additional modules
==================
@@ -498,7 +753,6 @@ and external, and snippets exist to use these features.
The Windows-specific standard modules are documented in
:ref:`mswin-specific-services`.
-
PyWin32
-------
@@ -557,20 +811,8 @@ latest release's source or just grab a fresh `checkout
<https://docs.python.org/devguide/setup.html#getting-the-source-code>`_.
The source tree contains a build solution and project files for Microsoft
-Visual C++, which is the compiler used to build the official Python releases.
-View the :file:`readme.txt` in their respective directories:
-
-+--------------------+--------------+-----------------------+
-| Directory | MSVC version | Visual Studio version |
-+====================+==============+=======================+
-| :file:`PC/VS9.0/` | 9.0 | 2008 |
-+--------------------+--------------+-----------------------+
-| :file:`PCbuild/` | 10.0 | 2010 |
-+--------------------+--------------+-----------------------+
-
-Note that any build directories within the :file:`PC` directory are not
-necessarily fully supported. The :file:`PCbuild` directory contains the files
-for the compiler used to build the official release.
+Visual Studio 2015, which is the compiler used to build the official Python
+releases. These files are in the :file:`PCbuild` directory.
Check :file:`PCbuild/readme.txt` for general information on the build process.
@@ -588,6 +830,83 @@ For extension modules, consult :ref:`building-on-windows`.
by Trent Apted et al, 2007
+Embedded Distribution
+=====================
+
+.. versionadded:: 3.5
+
+The embedded distribution is a ZIP file containing a minimal Python environment.
+It is intended for acting as part of another application, rather than being
+directly accessed by end-users.
+
+When extracted, the embedded distribution is (almost) fully isolated from the
+user's system, including environment variables, system registry settings, and
+installed packages. The standard library is included as pre-compiled and
+optimized ``.pyc`` files in a ZIP, and ``python3.dll``, ``python35.dll``,
+``python.exe`` and ``pythonw.exe`` are all provided. Tcl/tk (including all
+dependants, such as Idle), pip and the Python documentation are not included.
+
+.. note::
+
+ The embedded distribution does not include the `Microsoft C Runtime
+ <http://www.microsoft.com/en-us/download/details.aspx?id=48145>`_ and it is
+ the responsibility of the application installer to provide this. The
+ runtime may have already been installed on a user's system previously or
+ automatically via Windows Update, and can be detected by finding
+ ``ucrtbase.dll`` in the system directory.
+
+Third-party packages should be installed by the application installer alongside
+the embedded distribution. Using pip to manage dependencies as for a regular
+Python installation is not supported with this distribution, though with some
+care it may be possible to include and use pip for automatic updates. In
+general, third-party packages should be treated as part of the application
+("vendoring") so that the developer can ensure compatibility with newer
+versions before providing updates to users.
+
+The two recommended use cases for this distribution are described below.
+
+Python Application
+------------------
+
+An application written in Python does not necessarily require users to be aware
+of that fact. The embedded distribution may be used in this case to include a
+private version of Python in an install package. Depending on how transparent it
+should be (or conversely, how professional it should appear), there are two
+options.
+
+Using a specialized executable as a launcher requires some coding, but provides
+the most transparent experience for users. With a customized launcher, there are
+no obvious indications that the program is running on Python: icons can be
+customized, company and version information can be specified, and file
+associations behave properly. In most cases, a custom launcher should simply be
+able to call ``Py_Main`` with a hard-coded command line.
+
+The simpler approach is to provide a batch file or generated shortcut that
+directly calls the ``python.exe`` or ``pythonw.exe`` with the required
+command-line arguments. In this case, the application will appear to be Python
+and not its actual name, and users may have trouble distinguishing it from other
+running Python processes or file associations.
+
+With the latter approach, packages should be installed as directories alongside
+the Python executable to ensure they are available on the path. With the
+specialized launcher, packages can be located in other locations as there is an
+opportunity to specify the search path before launching the application.
+
+Embedding Python
+----------------
+
+Applications written in native code often require some form of scripting
+language, and the embedded Python distribution can be used for this purpose. In
+general, the majority of the application is in native code, and some part will
+either invoke ``python.exe`` or directly use ``python3.dll``. For either case,
+extracting the embedded distribution to a subdirectory of the application
+installation is sufficient to provide a loadable Python interpreter.
+
+As with the application use, packages can be installed to any location as there
+is an opportunity to specify search paths before initializing the interpreter.
+Otherwise, there is no fundamental differences between using the embedded
+distribution and a regular installation.
+
Other resources
===============
@@ -603,5 +922,3 @@ Other resources
:pep:`397` - Python launcher for Windows
The proposal for the launcher to be included in the Python distribution.
-
-
diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst
new file mode 100644
index 0000000..11f871b
--- /dev/null
+++ b/Doc/whatsnew/3.5.rst
@@ -0,0 +1,1249 @@
+****************************
+ What's New In Python 3.5
+****************************
+
+:Release: |release|
+:Date: |today|
+
+.. Rules for maintenance:
+
+ * Anyone can add text to this document. Do not spend very much time
+ on the wording of your changes, because your text will probably
+ get rewritten to some degree.
+
+ * The maintainer will go through Misc/NEWS periodically and add
+ changes; it's therefore more important to add your changes to
+ Misc/NEWS than to this file.
+
+ * This is not a complete list of every single change; completeness
+ is the purpose of Misc/NEWS. Some changes I consider too small
+ or esoteric to include. If such a change is added to the text,
+ I'll just remove it. (This is another reason you shouldn't spend
+ too much time on writing your addition.)
+
+ * If you want to draw your new text to the attention of the
+ maintainer, add 'XXX' to the beginning of the paragraph or
+ section.
+
+ * It's OK to just add a fragmentary note about a change. For
+ example: "XXX Describe the transmogrify() function added to the
+ socket module." The maintainer will research the change and
+ write the necessary text.
+
+ * You can comment out your additions if you like, but it's not
+ necessary (especially when a final release is some months away).
+
+ * Credit the author of a patch or bugfix. Just the name is
+ sufficient; the e-mail address isn't necessary.
+
+ * It's helpful to add the bug/patch number as a comment:
+
+ XXX Describe the transmogrify() function added to the socket
+ module.
+ (Contributed by P.Y. Developer in :issue:`12345`.)
+
+ This saves the maintainer the effort of going through the Mercurial log
+ when researching a change.
+
+This article explains the new features in Python 3.5, compared to 3.4.
+
+For full details, see the :source:`Misc/NEWS` file.
+
+.. note::
+
+ Prerelease users should be aware that this document is currently in draft
+ form. It will be updated substantially as Python 3.5 moves towards release,
+ so it's worth checking back even after reading earlier versions.
+
+
+.. seealso::
+
+ :pep:`478` - Python 3.5 Release Schedule
+
+
+Summary -- Release highlights
+=============================
+
+.. This section singles out the most important changes in Python 3.5.
+ Brevity is key.
+
+New syntax features:
+
+* :pep:`465`, a new matrix multiplication operator: ``a @ b``.
+* :pep:`492`, coroutines with async and await syntax.
+* :pep:`448`, additional unpacking generalizations.
+
+New library modules:
+
+* :mod:`zipapp`: :ref:`Improving Python ZIP Application Support
+ <whatsnew-zipapp>` (:pep:`441`).
+
+New built-in features:
+
+* ``bytes % args``, ``bytearray % args``: :pep:`461` - Adding ``%`` formatting
+ to bytes and bytearray
+* ``b'\xf0\x9f\x90\x8d'.hex()``, ``bytearray(b'\xf0\x9f\x90\x8d').hex()``,
+ ``memoryview(b'\xf0\x9f\x90\x8d').hex()``: :issue:`9951` - A ``hex`` method
+ has been added to bytes, bytearray, and memoryview.
+* Generators have new ``gi_yieldfrom`` attribute, which returns the
+ object being iterated by ``yield from`` expressions. (Contributed
+ by Benno Leslie and Yury Selivanov in :issue:`24450`.)
+* New :exc:`RecursionError` exception. (Contributed by Georg Brandl
+ in :issue:`19235`.)
+
+Implementation improvements:
+
+* When the ``LC_TYPE`` locale is the POSIX locale (``C`` locale),
+ :py:data:`sys.stdin` and :py:data:`sys.stdout` are now using the
+ ``surrogateescape`` error handler, instead of the ``strict`` error handler
+ (:issue:`19977`).
+
+* :pep:`488`, the elimination of ``.pyo`` files.
+* :pep:`489`, multi-phase initialization of extension modules.
+
+Significantly Improved Library Modules:
+
+* :class:`collections.OrderedDict` is now implemented in C, which improves
+ its performance between 4x to 100x times. Contributed by Eric Snow in
+ :issue:`16991`.
+
+* You may now pass bytes to the :mod:`tempfile` module's APIs and it will
+ return the temporary pathname as bytes instead of str. It also accepts
+ a value of ``None`` on parameters where only str was accepted in the past to
+ do the right thing based on the types of the other inputs. Two functions,
+ :func:`gettempdirb` and :func:`gettempprefixb`, have been added to go along
+ with this. This behavior matches that of the :mod:`os` APIs.
+
+* :mod:`ssl` module gained support for Memory BIO, which decouples SSL
+ protocol handling from network IO. (Contributed by Geert Jansen in
+ :issue:`21965`.)
+
+Security improvements:
+
+* None yet.
+
+Windows improvements:
+
+* A new installer for Windows has replaced the old MSI. See :ref:`using-on-windows`
+ for more information.
+* Windows builds now use Microsoft Visual C++ 14.0, and extension modules
+ should use the same.
+
+
+Please read on for a comprehensive list of user-facing changes.
+
+
+.. PEP-sized items next.
+
+.. _pep-4XX:
+
+.. PEP 4XX: Virtual Environments
+.. =============================
+
+
+.. (Implemented by Foo Bar.)
+
+.. .. seealso::
+
+ :pep:`4XX` - Python Virtual Environments
+ PEP written by Carl Meyer
+
+
+PEP 492 - Coroutines with async and await syntax
+------------------------------------------------
+
+The PEP added dedicated syntax for declaring :term:`coroutines <coroutine>`,
+:keyword:`await` expressions, new asynchronous :keyword:`async for`
+and :keyword:`async with` statements.
+
+Example::
+
+ async def read_data(db):
+ async with db.transaction():
+ data = await db.fetch('SELECT ...')
+
+PEP written and implemented by Yury Selivanov.
+
+.. seealso::
+
+ :pep:`492` -- Coroutines with async and await syntax
+
+
+PEP 461 - Formatting support for bytes and bytearray
+----------------------------------------------------
+
+This PEP proposes adding % formatting operations similar to Python 2's ``str``
+type to :class:`bytes` and :class:`bytearray`.
+
+Examples::
+
+ >>> b'Hello %s!' % b'World'
+ b'Hello World!'
+ >>> b'x=%i y=%f' % (1, 2.5)
+ b'x=1 y=2.500000'
+
+Unicode is not allowed for ``%s``, but it is accepted by ``%a`` (equivalent of
+``repr(obj).encode('ascii', 'backslashreplace')``)::
+
+ >>> b'Hello %s!' % 'World'
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ TypeError: %b requires bytes, or an object that implements __bytes__, not 'str'
+ >>> b'price: %a' % '10€'
+ b"price: '10\\u20ac'"
+
+.. seealso::
+
+ :pep:`461` -- Adding % formatting to bytes and bytearray
+
+
+PEP 465 - A dedicated infix operator for matrix multiplication
+--------------------------------------------------------------
+
+This PEP proposes a new binary operator to be used for matrix multiplication,
+called ``@``. (Mnemonic: ``@`` is ``*`` for mATrices.)
+
+.. seealso::
+
+ :pep:`465` -- A dedicated infix operator for matrix multiplication
+
+
+PEP 448 - Additional Unpacking Generalizations
+----------------------------------------------
+
+This PEP proposes extended usages of the ``*`` iterable unpacking
+operator and ``**`` dictionary unpacking operators
+to allow unpacking in more positions, an arbitrary number of
+times, and in additional circumstances. Specifically,
+in function calls, in comprehensions and generator expressions, and
+in displays.
+
+Function calls are proposed to support an arbitrary number of
+unpackings rather than just one::
+
+ >>> print(*[1], *[2], 3)
+ 1 2 3
+ >>> dict(**{'x': 1}, y=2, **{'z': 3})
+ {'x': 1, 'y': 2, 'z': 3}
+
+Unpacking is proposed to be allowed inside tuple, list, set,
+and dictionary displays::
+
+ >>> *range(4), 4
+ (0, 1, 2, 3, 4)
+ >>> [*range(4), 4]
+ [0, 1, 2, 3, 4]
+ >>> {*range(4), 4}
+ {0, 1, 2, 3, 4}
+ >>> {'x': 1, **{'y': 2}}
+ {'x': 1, 'y': 2}
+
+In dictionaries, later values will always override earlier ones::
+
+ >>> {'x': 1, **{'x': 2}}
+ {'x': 2}
+
+ >>> {**{'x': 2}, 'x': 1}
+ {'x': 1}
+
+.. seealso::
+
+ :pep:`448` -- Additional Unpacking Generalizations
+
+
+PEP 484 - Type Hints
+--------------------
+
+This PEP introduces a provisional module to provide these standard
+definitions and tools, along with some conventions for situations
+where annotations are not available.
+
+For example, here is a simple function whose argument and return type
+are declared in the annotations::
+
+ def greeting(name: str) -> str:
+ return 'Hello ' + name
+
+The type system supports unions, generic types, and a special type
+named ``Any`` which is consistent with (i.e. assignable to and from) all
+types.
+
+.. seealso::
+
+ * :pep:`484` -- Type Hints
+ * :mod:`typing` module documentation
+
+
+PEP 471 - os.scandir() function -- a better and faster directory iterator
+-------------------------------------------------------------------------
+
+:pep:`471` adds a new directory iteration function, :func:`os.scandir`,
+to the standard library. Additionally, :func:`os.walk` is now
+implemented using :func:`os.scandir`, which speeds it up by 3-5 times
+on POSIX systems and by 7-20 times on Windows systems.
+
+PEP and implementation written by Ben Hoyt with the help of Victor Stinner.
+
+.. seealso::
+
+ :pep:`471` -- os.scandir() function -- a better and faster directory
+ iterator
+
+
+PEP 475: Retry system calls failing with EINTR
+----------------------------------------------
+
+:pep:`475` adds support for automatic retry of system calls failing with
+:py:data:`~errno.EINTR`: this means that user code doesn't have to deal with
+EINTR or :exc:`InterruptedError` manually, and should make it more robust
+against asynchronous signal reception.
+
+Examples of functions which are now retried when interrupted by a signal
+instead of raising :exc:`InterruptedError` if the Python signal handler does
+not raise an exception:
+
+* :func:`open`, :func:`os.open`, :func:`io.open`
+* functions of the :mod:`faulthandler` module
+* :mod:`os` functions:
+
+ - :func:`os.fchdir`
+ - :func:`os.fchmod`
+ - :func:`os.fchown`
+ - :func:`os.fdatasync`
+ - :func:`os.fstat`
+ - :func:`os.fstatvfs`
+ - :func:`os.fsync`
+ - :func:`os.ftruncate`
+ - :func:`os.mkfifo`
+ - :func:`os.mknod`
+ - :func:`os.posix_fadvise`
+ - :func:`os.posix_fallocate`
+ - :func:`os.pread`
+ - :func:`os.pwrite`
+ - :func:`os.read`
+ - :func:`os.readv`
+ - :func:`os.sendfile`
+ - :func:`os.wait3`
+ - :func:`os.wait4`
+ - :func:`os.wait`
+ - :func:`os.waitid`
+ - :func:`os.waitpid`
+ - :func:`os.write`
+ - :func:`os.writev`
+ - special cases: :func:`os.close` and :func:`os.dup2` now ignore
+ :py:data:`~errno.EINTR` error, the syscall is not retried (see the PEP
+ for the rationale)
+
+* :mod:`select` functions:
+
+ - :func:`select.devpoll.poll`
+ - :func:`select.epoll.poll`
+ - :func:`select.kqueue.control`
+ - :func:`select.poll.poll`
+ - :func:`select.select`
+
+* :func:`socket.socket` methods:
+
+ - :meth:`~socket.socket.accept`
+ - :meth:`~socket.socket.connect` (except for non-blocking sockets)
+ - :meth:`~socket.socket.recv`
+ - :meth:`~socket.socket.recvfrom`
+ - :meth:`~socket.socket.recvmsg`
+ - :meth:`~socket.socket.send`
+ - :meth:`~socket.socket.sendall`
+ - :meth:`~socket.socket.sendmsg`
+ - :meth:`~socket.socket.sendto`
+
+* :func:`signal.sigtimedwait`, :func:`signal.sigwaitinfo`
+* :func:`time.sleep`
+
+PEP and implementation written by Charles-François Natali and Victor Stinner,
+with the help of Antoine Pitrou (the french connection).
+
+.. seealso::
+
+ :pep:`475` -- Retry system calls failing with EINTR
+
+
+PEP 479: Change StopIteration handling inside generators
+--------------------------------------------------------
+
+:pep:`479` changes the behavior of generators: when a :exc:`StopIteration`
+exception is raised inside a generator, it is replaced with a
+:exc:`RuntimeError`. To enable the feature a ``__future__`` import should
+be used::
+
+ from __future__ import generator_stop
+
+Without a ``__future__`` import, a :exc:`PendingDeprecationWarning` will be
+raised.
+
+PEP written by Chris Angelico and Guido van Rossum. Implemented by
+Chris Angelico, Yury Selivanov and Nick Coghlan.
+
+.. seealso::
+
+ :pep:`479` -- Change StopIteration handling inside generators
+
+
+PEP 486: Make the Python Launcher aware of virtual environments
+---------------------------------------------------------------
+
+:pep:`486` makes the Windows launcher (see :pep:`397`) aware of an active
+virtual environment. When the default interpreter would be used and the
+``VIRTUAL_ENV`` environment variable is set, the interpreter in the virtual
+environment will be used.
+
+.. seealso::
+
+ :pep:`486` -- Make the Python Launcher aware of virtual environments
+
+
+PEP 488: Elimination of PYO files
+---------------------------------
+
+:pep:`488` does away with the concept of ``.pyo`` files. This means that
+``.pyc`` files represent both unoptimized and optimized bytecode. To prevent the
+need to constantly regenerate bytecode files, ``.pyc`` files now have an
+optional ``opt-`` tag in their name when the bytecode is optimized. This has the
+side-effect of no more bytecode file name clashes when running under either
+``-O`` or ``-OO``. Consequently, bytecode files generated from ``-O``, and
+``-OO`` may now exist simultaneously. :func:`importlib.util.cache_from_source`
+has an updated API to help with this change.
+
+.. seealso::
+
+ :pep:`488` -- Elimination of PYO files
+
+
+PEP 489: Multi-phase extension module initialization
+----------------------------------------------------
+
+:pep:`489` updates extension module initialization to take advantage of the
+two step module loading mechanism introduced by :pep:`451` in Python 3.4.
+
+This change brings the import semantics of extension modules that opt-in to
+using the new mechanism much closer to those of Python source and bytecode
+modules, including the ability to use any valid identifier as a module name,
+rather than being restricted to ASCII.
+
+.. seealso::
+
+ :pep:`488` -- Multi-phase extension module initialization
+
+PEP 485: A function for testing approximate equality
+----------------------------------------------------
+
+:pep:`485` adds the :func:`math.isclose` and :func:`cmath.isclose`
+functions which tell whether two values are approximately equal or
+"close" to each other. Whether or not two values are considered
+close is determined according to given absolute and relative tolerances.
+
+.. seealso::
+
+ :pep:`485` -- A function for testing approximate equality
+
+Other Language Changes
+======================
+
+Some smaller changes made to the core Python language are:
+
+* Added the ``'namereplace'`` error handlers. The ``'backslashreplace'``
+ error handlers now works with decoding and translating.
+ (Contributed by Serhiy Storchaka in :issue:`19676` and :issue:`22286`.)
+
+* The :option:`-b` option now affects comparisons of :class:`bytes` with
+ :class:`int`. (Contributed by Serhiy Storchaka in :issue:`23681`)
+
+* New Kazakh :ref:`codec <standard-encodings>` ``kz1048``. (Contributed by
+ Serhiy Storchaka in :issue:`22682`.)
+
+* Property docstrings are now writable. This is especially useful for
+ :func:`collections.namedtuple` docstrings.
+ (Contributed by Berker Peksag in :issue:`24064`.)
+
+* New Tajik :ref:`codec <standard-encodings>` ``koi8_t``. (Contributed by
+ Serhiy Storchaka in :issue:`22681`.)
+
+
+New Modules
+===========
+
+.. _whatsnew-zipapp:
+
+zipapp
+------
+
+The new :mod:`zipapp` module (specified in :pep:`441`) provides an API and
+command line tool for creating executable Python Zip Applications, which
+were introduced in Python 2.6 in :issue:`1739468` but which were not well
+publicised, either at the time or since.
+
+With the new module, bundling your application is as simple as putting all
+the files, including a ``__main__.py`` file, into a directory ``myapp``
+and running::
+
+ $ python -m zipapp myapp
+ $ python myapp.pyz
+
+
+Improved Modules
+================
+
+argparse
+--------
+
+* :class:`~argparse.ArgumentParser` now allows to disable
+ :ref:`abbreviated usage <prefix-matching>` of long options by setting
+ :ref:`allow_abbrev` to ``False``.
+ (Contributed by Jonathan Paugh, Steven Bethard, paul j3 and Daniel Eriksson.)
+
+cgi
+---
+
+* :class:`~cgi.FieldStorage` now supports the context management protocol.
+ (Contributed by Berker Peksag in :issue:`20289`.)
+
+cmath
+-----
+
+* :func:`cmath.isclose` function added.
+ (Contributed by Chris Barker and Tal Einat in :issue:`24270`.)
+
+
+code
+----
+
+* The :func:`code.InteractiveInterpreter.showtraceback` method now prints
+ the full chained traceback, just like the interactive interpreter.
+ (Contributed by Claudiu Popa in :issue:`17442`.)
+
+collections
+-----------
+
+* You can now update docstrings produced by :func:`collections.namedtuple`::
+
+ Point = namedtuple('Point', ['x', 'y'])
+ Point.__doc__ = 'ordered pair'
+ Point.x.__doc__ = 'abscissa'
+ Point.y.__doc__ = 'ordinate'
+
+ (Contributed by Berker Peksag in :issue:`24064`.)
+
+compileall
+----------
+
+* :func:`compileall.compile_dir` and :mod:`compileall`'s command-line interface
+ can now do parallel bytecode compilation.
+ (Contributed by Claudiu Popa in :issue:`16104`.)
+
+contextlib
+----------
+
+* The new :func:`contextlib.redirect_stderr` context manager(similar to
+ :func:`contextlib.redirect_stdout`) makes it easier for utility scripts to
+ handle inflexible APIs that write their output to :data:`sys.stderr` and
+ don't provide any options to redirect it.
+ (Contributed by Berker Peksag in :issue:`22389`.)
+
+curses
+------
+* The new :func:`curses.update_lines_cols` function updates the variables
+ :envvar:`curses.LINES` and :envvar:`curses.COLS`.
+
+difflib
+-------
+
+* The charset of the HTML document generated by :meth:`difflib.HtmlDiff.make_file`
+ can now be customized by using *charset* keyword-only parameter. The default
+ charset of HTML document changed from ``'ISO-8859-1'`` to ``'utf-8'``.
+ (Contributed by Berker Peksag in :issue:`2052`.)
+
+* It's now possible to compare lists of byte strings with
+ :func:`difflib.diff_bytes` (fixes a regression from Python 2).
+
+distutils
+---------
+
+* The ``build`` and ``build_ext`` commands now accept a ``-j``
+ option to enable parallel building of extension modules.
+ (Contributed by Antoine Pitrou in :issue:`5309`.)
+
+* Added support for the LZMA compression.
+ (Contributed by Serhiy Storchaka in :issue:`16314`.)
+
+doctest
+-------
+
+* :func:`doctest.DocTestSuite` returns an empty :class:`unittest.TestSuite` if
+ *module* contains no docstrings instead of raising :exc:`ValueError`.
+ (Contributed by Glenn Jones in :issue:`15916`.)
+
+email
+-----
+
+* A new policy option :attr:`~email.policy.Policy.mangle_from_` controls
+ whether or not lines that start with "From " in email bodies are prefixed with
+ a '>' character by generators. The default is ``True`` for
+ :attr:`~email.policy.compat32` and ``False`` for all other policies.
+ (Contributed by Milan Oberkirch in :issue:`20098`.)
+
+* A new method :meth:`~email.message.Message.get_content_disposition` provides
+ easy access to a canonical value for the :mailheader:`Content-Disposition`
+ header (``None`` if there is no such header). (Contributed by Abhilash Raj
+ in :issue:`21083`.)
+
+* A new policy option :attr:`~email.policy.EmailPolicy.utf8` can be set
+ ``True`` to encode email headers using the utf8 charset instead of using
+ encoded words. This allows ``Messages`` to be formatted according to
+ :rfc:`6532` and used with an SMTP server that supports the :rfc:`6531`
+ ``SMTPUTF8`` extension. (Contributed by R. David Murray in :issue:`24211`.)
+
+glob
+----
+
+* :func:`~glob.iglob` and :func:`~glob.glob` now support recursive search in
+ subdirectories using the "``**``" pattern.
+ (Contributed by Serhiy Storchaka in :issue:`13968`.)
+
+idlelib and IDLE
+----------------
+
+Since idlelib implements the IDLE shell and editor and is not intended for
+import by other programs, it gets improvements with every release. See
+:file:`Lib/idlelib/NEWS.txt` for a cumulative list of changes since 3.4.0,
+as well as changes made in future 3.5.x releases. This file is also available
+from the IDLE Help -> About Idle dialog.
+
+imaplib
+-------
+
+* :class:`IMAP4` now supports the context management protocol. When used in a
+ :keyword:`with` statement, the IMAP4 ``LOGOUT`` command will be called
+ automatically at the end of the block. (Contributed by Tarek Ziadé and
+ Serhiy Storchaka in :issue:`4972`.)
+
+* :mod:`imaplib` now supports :rfc:`5161`: the :meth:`~imaplib.IMAP4.enable`
+ extension), and :rfc:`6855`: utf-8 support (internationalized email, via the
+ ``UTF8=ACCEPT`` argument to :meth:`~imaplib.IMAP4.enable`). A new attribute,
+ :attr:`~imaplib.IMAP4.utf8_enabled`, tracks whether or not :rfc:`6855`
+ support is enabled. Milan Oberkirch, R. David Murray, and Maciej Szulik in
+ :issue:`21800`.)
+
+* :mod:`imaplib` now automatically encodes non-ASCII string usernames and
+ passwords using ``UTF8``, as recommended by the RFCs. (Contributed by Milan
+ Oberkirch in :issue:`21800`.)
+
+imghdr
+------
+
+* :func:`~imghdr.what` now recognizes the `OpenEXR <http://www.openexr.com>`_
+ format. (Contributed by Martin Vignali and Claudiu Popa in :issue:`20295`.)
+
+importlib
+---------
+
+* :class:`importlib.util.LazyLoader` allows for the lazy loading of modules in
+ applications where startup time is paramount.
+ (Contributed by Brett Cannon in :issue:`17621`.)
+
+* :func:`importlib.abc.InspectLoader.source_to_code` is now a
+ static method to make it easier to work with source code in a string.
+ With a module object that you want to initialize you can then use
+ ``exec(code, module.__dict__)`` to execute the code in the module.
+
+* :func:`importlib.util.module_from_spec` is now the preferred way to create a
+ new module. Compared to :class:`types.ModuleType`, this new function will set
+ the various import-controlled attributes based on the passed-in spec object.
+
+inspect
+-------
+
+* :class:`inspect.Signature` and :class:`inspect.Parameter` are now
+ picklable and hashable. (Contributed by Yury Selivanov in :issue:`20726`
+ and :issue:`20334`.)
+
+* New method :meth:`inspect.BoundArguments.apply_defaults`. (Contributed
+ by Yury Selivanov in :issue:`24190`.)
+
+* New class method :meth:`inspect.Signature.from_callable`, which makes
+ subclassing of :class:`~inspect.Signature` easier. (Contributed
+ by Yury Selivanov and Eric Snow in :issue:`17373`.)
+
+* New argument ``follow_wrapped`` for :func:`inspect.signature`.
+ (Contributed by Yury Selivanov in :issue:`20691`.)
+
+* New :func:`~inspect.iscoroutine`, :func:`~inspect.iscoroutinefunction`
+ and :func:`~inspect.isawaitable` functions. (Contributed by
+ Yury Selivanov in :issue:`24017`.)
+
+* New :func:`~inspect.getcoroutinelocals` and :func:`~inspect.getcoroutinestate`
+ functions. (Contributed by Yury Selivanov in :issue:`24400`.)
+
+ipaddress
+---------
+
+* :class:`ipaddress.IPv4Network` and :class:`ipaddress.IPv6Network` now
+ accept an ``(address, netmask)`` tuple argument, so as to easily construct
+ network objects from existing addresses. (Contributed by Peter Moody
+ and Antoine Pitrou in :issue:`16531`.)
+
+json
+----
+
+* The output of :mod:`json.tool` command line interface is now in the same
+ order as the input. Use the :option:`--sort-keys` option to sort the output
+ of dictionaries alphabetically by key. (Contributed by Berker Peksag in
+ :issue:`21650`.)
+
+* JSON decoder now raises :exc:`json.JSONDecodeError` instead of
+ :exc:`ValueError`. (Contributed by Serhiy Storchaka in :issue:`19361`.)
+
+math
+----
+
+* :data:`math.inf` and :data:`math.nan` constants added. (Contributed by Mark
+ Dickinson in :issue:`23185`.)
+* :func:`math.isclose` function added.
+ (Contributed by Chris Barker and Tal Einat in :issue:`24270`.)
+
+os
+--
+
+* New :func:`os.scandir` function that exposes file information from
+ the operating system when listing a directory. :func:`os.scandir`
+ returns an iterator of :class:`os.DirEntry` objects corresponding to
+ the entries in the directory given by *path*. (Contributed by Ben
+ Hoyt with the help of Victor Stinner in :issue:`22524`.)
+
+* :class:`os.stat_result` now has a :attr:`~os.stat_result.st_file_attributes`
+ attribute on Windows. (Contributed by Ben Hoyt in :issue:`21719`.)
+
+* :func:`os.urandom`: On Linux 3.17 and newer, the ``getrandom()`` syscall is
+ now used when available. On OpenBSD 5.6 and newer, the C ``getentropy()``
+ function is now used. These functions avoid the usage of an internal file
+ descriptor.
+
+os.path
+-------
+
+* New :func:`~os.path.commonpath` function that extracts common path prefix.
+ Unlike the :func:`~os.path.commonprefix` function, it always returns a valid
+ path. (Contributed by Rafik Draoui and Serhiy Storchaka in :issue:`10395`.)
+
+pickle
+------
+
+* Serializing more "lookupable" objects (such as unbound methods or nested
+ classes) now are supported with pickle protocols < 4.
+ (Contributed by Serhiy Storchaka in :issue:`23611`.)
+
+poplib
+------
+
+* A new command :meth:`~poplib.POP3.utf8` enables :rfc:`6856`
+ (internationalized email) support if the POP server supports it. (Contributed
+ by Milan OberKirch in :issue:`21804`.)
+
+re
+--
+
+* Number of capturing groups in regular expression is no longer limited by 100.
+ (Contributed by Serhiy Storchaka in :issue:`22437`.)
+
+* Now unmatched groups are replaced with empty strings in :func:`re.sub`
+ and :func:`re.subn`. (Contributed by Serhiy Storchaka in :issue:`1519638`.)
+
+shutil
+------
+
+* :func:`~shutil.move` now accepts a *copy_function* argument, allowing,
+ for example, :func:`~shutil.copy` to be used instead of the default
+ :func:`~shutil.copy2` if there is a need to ignore metadata. (Contributed by
+ Claudiu Popa in :issue:`19840`.)
+
+signal
+------
+
+* On Windows, :func:`signal.set_wakeup_fd` now also supports socket handles.
+ (Contributed by Victor Stinner in :issue:`22018`.)
+
+* Different constants of :mod:`signal` module are now enumeration values using
+ the :mod:`enum` module. This allows meaningful names to be printed during
+ debugging, instead of integer “magic numbers”. (Contributed by Giampaolo
+ Rodola' in :issue:`21076`.)
+
+smtpd
+-----
+
+* Both :class:`~smtpd.SMTPServer` and :class:`smtpd.SMTPChannel` now accept a
+ *decode_data* keyword to determine if the DATA portion of the SMTP
+ transaction is decoded using the ``utf-8`` codec or is instead provided to
+ :meth:`~smtpd.SMTPServer.process_message` as a byte string. The default
+ is ``True`` for backward compatibility reasons, but will change to ``False``
+ in Python 3.6. If *decode_data* is set to ``False``, the
+ :meth:`~smtpd.SMTPServer.process_message` method must be prepared to accept
+ keyword arguments. (Contributed by Maciej Szulik in :issue:`19662`.)
+
+* :class:`~smtpd.SMTPServer` now advertises the ``8BITMIME`` extension
+ (:rfc:`6152`) if if *decode_data* has been set ``True``. If the client
+ specifies ``BODY=8BITMIME`` on the ``MAIL`` command, it is passed to
+ :meth:`~smtpd.SMTPServer.process_message` via the ``mail_options`` keyword.
+ (Contributed by Milan Oberkirch and R. David Murray in :issue:`21795`.)
+
+* :class:`~smtpd.SMTPServer` now supports the ``SMTPUTF8`` extension
+ (:rfc:`6531`: Internationalized Email). If the client specified ``SMTPUTF8
+ BODY=8BITMIME`` on the ``MAIL`` command, they are passed to
+ :meth:`~smtpd.SMTPServer.process_message` via the ``mail_options`` keyword.
+ It is the responsibility of the :meth:`~smtpd.SMTPServer.process_message`
+ method to correctly handle the ``SMTPUTF8`` data. (Contributed by Milan
+ Oberkirch in :issue:`21725`.)
+
+* It is now possible to provide, directly or via name resolution, IPv6
+ addresses in the :class:`~smtpd.SMTPServer` constructor, and have it
+ successfully connect. (Contributed by Milan Oberkirch in :issue:`14758`.)
+
+smtplib
+-------
+
+* A new :meth:`~smtplib.SMTP.auth` method provides a convenient way to
+ implement custom authentication mechanisms.
+ (Contributed by Milan Oberkirch in :issue:`15014`.)
+
+* Additional debuglevel (2) shows timestamps for debug messages in
+ :class:`smtplib.SMTP`. (Contributed by Gavin Chappell and Maciej Szulik in
+ :issue:`16914`.)
+
+* :mod:`smtplib` now supports :rfc:`6531` (SMTPUTF8) in both the
+ :meth:`~smtplib.SMTP.sendmail` and :meth:`~smtplib.SMTP.send_message`
+ commands. (Contributed by Milan Oberkirch and R. David Murray in
+ :issue:`22027`.)
+
+sndhdr
+------
+
+* :func:`~sndhdr.what` and :func:`~sndhdr.whathdr` now return
+ :func:`~collections.namedtuple`.
+ (Contributed by Claudiu Popa in :issue:`18615`.)
+
+ssl
+---
+
+* The :meth:`~ssl.SSLSocket.do_handshake`, :meth:`~ssl.SSLSocket.read`,
+ :meth:`~ssl.SSLSocket.shutdown`, and :meth:`~ssl.SSLSocket.write` methods of
+ :class:`ssl.SSLSocket` don't reset the socket timeout anymore each time bytes
+ are received or sent. The socket timeout is now the maximum total duration of
+ the method.
+
+* Memory BIO Support: new classes :class:`~ssl.SSLObject`,
+ :class:`~ssl.MemoryBIO`, and new
+ :meth:`SSLContext.wrap_bio <ssl.SSLContext.wrap_bio>` method.
+ (Contributed by Geert Jansen in :issue:`21965`.)
+
+socket
+------
+
+* New :meth:`socket.socket.sendfile` method allows to send a file over a socket
+ by using high-performance :func:`os.sendfile` function on UNIX resulting in
+ uploads being from 2x to 3x faster than when using plain
+ :meth:`socket.socket.send`.
+ (Contributed by Giampaolo Rodola' in :issue:`17552`.)
+
+* The :meth:`socket.socket.sendall` method don't reset the socket timeout
+ anymore each time bytes are received or sent. The socket timeout is now the
+ maximum total duration to send all data.
+
+subprocess
+----------
+
+* The new :func:`subprocess.run` function runs subprocesses and returns a
+ :class:`subprocess.CompletedProcess` object. It Provides a more consistent
+ API than :func:`~subprocess.call`, :func:`~subprocess.check_call` and
+ :func:`~subprocess.check_output`.
+
+sys
+---
+
+* New :func:`~sys.set_coroutine_wrapper` and :func:`~sys.get_coroutine_wrapper`
+ functions. (Contributed by Yury Selivanov in :issue:`24017`.)
+
+sysconfig
+---------
+
+* The user scripts directory on Windows is now versioned.
+ (Contributed by Paul Moore in :issue:`23437`.)
+
+tarfile
+-------
+
+* The :func:`tarfile.open` function now supports ``'x'`` (exclusive creation)
+ mode. (Contributed by Berker Peksag in :issue:`21717`.)
+
+* The :meth:`~tarfile.TarFile.extractall` and :meth:`~tarfile.TarFile.extract`
+ methods now take a keyword parameter *numeric_only*. If set to ``True``,
+ the extracted files and directories will be owned by the numeric uid and gid
+ from the tarfile. If set to ``False`` (the default, and the behavior in
+ versions prior to 3.5), they will be owned bythe named user and group in the
+ tarfile. (Contributed by Michael Vogt and Eric Smith in :issue:`23193`.)
+
+time
+----
+
+* The :func:`time.monotonic` function is now always available. (Contributed by
+ Victor Stinner in :issue:`22043`.)
+
+tkinter
+-------
+
+* The :mod:`tkinter._fix` module used for setting up the Tcl/Tk environment
+ on Windows has been replaced by a private function in the :mod:`_tkinter`
+ module which makes no permanent changes to environment variables.
+ (Contributed by Zachary Ware in :issue:`20035`.)
+
+types
+-----
+
+* New :func:`~types.coroutine` function. (Contributed by Yury Selivanov
+ in :issue:`24017`.)
+
+* New :class:`~types.CoroutineType`. (Contributed by Yury Selivanov
+ in :issue:`24400`.)
+
+urllib
+------
+
+* A new :class:`~urllib.request.HTTPPasswordMgrWithPriorAuth` allows HTTP Basic
+ Authentication credentials to be managed so as to eliminate unnecessary
+ ``401`` response handling, or to unconditionally send credentials
+ on the first request in order to communicate with servers that return a
+ ``404`` response instead of a ``401`` if the ``Authorization`` header is not
+ sent. (Contributed by Matej Cepl in :issue:`19494` and Akshit Khurana in
+ :issue:`7159`.)
+
+* A new :func:`~urllib.parse.urlencode` parameter *quote_via* provides a way to
+ control the encoding of query parts if needed. (Contributed by Samwyse and
+ Arnon Yaari in :issue:`13866`.)
+
+unicodedata
+-----------
+
+* The :mod:`unicodedata` module now uses data from `Unicode 8.0.0
+ <http://unicode.org/versions/Unicode8.0.0/>`_.
+
+
+wsgiref
+-------
+
+* *headers* parameter of :class:`wsgiref.headers.Headers` is now optional.
+ (Contributed by Pablo Torres Navarrete and SilentGhost in :issue:`5800`.)
+
+xmlrpc
+------
+
+* :class:`xmlrpc.client.ServerProxy` is now a :term:`context manager`.
+ (Contributed by Claudiu Popa in :issue:`20627`.)
+
+xml.sax
+-------
+
+* SAX parsers now support a character stream of
+ :class:`~xml.sax.xmlreader.InputSource` object.
+ (Contributed by Serhiy Storchaka in :issue:`2175`.)
+
+faulthandler
+------------
+
+* :func:`~faulthandler.enable`, :func:`~faulthandler.register`,
+ :func:`~faulthandler.dump_traceback` and
+ :func:`~faulthandler.dump_traceback_later` functions now accept file
+ descriptors. (Contributed by Wei Wu in :issue:`23566`.)
+
+zipfile
+-------
+
+* Added support for writing ZIP files to unseekable streams.
+ (Contributed by Serhiy Storchaka in :issue:`23252`.)
+
+* The :func:`zipfile.ZipFile.open` function now supports ``'x'`` (exclusive
+ creation) mode. (Contributed by Serhiy Storchaka in :issue:`21717`.)
+
+
+Optimizations
+=============
+
+The following performance enhancements have been added:
+
+* :func:`os.walk` has been sped up by 3-5x on POSIX systems and 7-20x
+ on Windows. This was done using the new :func:`os.scandir` function,
+ which exposes file information from the underlying ``readdir`` and
+ ``FindFirstFile``/``FindNextFile`` system calls. (Contributed by
+ Ben Hoyt with help from Victor Stinner in :issue:`23605`.)
+
+* Construction of ``bytes(int)`` (filled by zero bytes) is faster and uses less
+ memory for large objects. ``calloc()`` is used instead of ``malloc()`` to
+ allocate memory for these objects.
+
+* Some operations on :class:`~ipaddress.IPv4Network` and
+ :class:`~ipaddress.IPv6Network` have been massively sped up, such as
+ :meth:`~ipaddress.IPv4Network.subnets`, :meth:`~ipaddress.IPv4Network.supernet`,
+ :func:`~ipaddress.summarize_address_range`, :func:`~ipaddress.collapse_addresses`.
+ The speed up can range from 3x to 15x.
+ (:issue:`21486`, :issue:`21487`, :issue:`20826`)
+
+* Many operations on :class:`io.BytesIO` are now 50% to 100% faster.
+ (Contributed by Serhiy Storchaka in :issue:`15381` and David Wilson in
+ :issue:`22003`.)
+
+* :func:`marshal.dumps` is now faster (65%-85% with versions 3--4, 20-25% with
+ versions 0--2 on typical data, and up to 5x in best cases).
+ (Contributed by Serhiy Storchaka in :issue:`20416` and :issue:`23344`.)
+
+* The UTF-32 encoder is now 3x to 7x faster. (Contributed by Serhiy Storchaka
+ in :issue:`15027`.)
+
+
+Build and C API Changes
+=======================
+
+Changes to Python's build process and to the C API include:
+
+* New ``calloc`` functions:
+
+ * :c:func:`PyMem_RawCalloc`
+ * :c:func:`PyMem_Calloc`
+ * :c:func:`PyObject_Calloc`
+ * :c:func:`_PyObject_GC_Calloc`
+
+* Windows builds now require Microsoft Visual C++ 14.0, which
+ is available as part of `Visual Studio 2015 <http://www.visualstudio.com>`_.
+
+Deprecated
+==========
+
+New Keywords
+------------
+
+``async`` and ``await`` are not recommended to be used as variable, class,
+function or module names. Introduced by :pep:`492` in Python 3.5, they will
+become proper keywords in Python 3.7.
+
+
+Unsupported Operating Systems
+-----------------------------
+
+* Windows XP - Per :PEP:`11`, Microsoft support of Windows XP has ended.
+
+
+Deprecated Python modules, functions and methods
+------------------------------------------------
+
+* The :mod:`formatter` module has now graduated to full deprecation and is still
+ slated for removal in Python 3.6.
+
+* :mod:`smtpd` has in the past always decoded the DATA portion of email
+ messages using the ``utf-8`` codec. This can now be controlled by the new
+ *decode_data* keyword to :class:`~smtpd.SMTPServer`. The default value is
+ ``True``, but this default is deprecated. Specify the *decode_data* keyword
+ with an appropriate value to avoid the deprecation warning.
+
+* Directly assigning values to the :attr:`~http.cookies.Morsel.key`,
+ :attr:`~http.cookies.Morsel.value` and
+ :attr:`~http.cookies.Morsel.coded_value` of :class:`~http.cookies.Morsel`
+ objects is deprecated. Use the :func:`~http.cookies.Morsel.set` method
+ instead. In addition, the undocumented *LegalChars* parameter of
+ :func:`~http.cookies.Morsel.set` is deprecated, and is now ignored.
+
+* Passing a format string as keyword argument *format_string* to the
+ :meth:`~string.Formatter.format` method of the :class:`string.Formatter`
+ class has been deprecated.
+
+* :func:`platform.dist` and :func:`platform.linux_distribution` functions are
+ now deprecated and will be removed in Python 3.7. Linux distributions use
+ too many different ways of describing themselves, so the functionality is
+ left to a package.
+ (Contributed by Vajrasky Kok and Berker Peksag in :issue:`1322`.)
+
+* The previously undocumented ``from_function`` and ``from_builtin`` methods of
+ :class:`inspect.Signature` are deprecated. Use new
+ :meth:`inspect.Signature.from_callable` instead. (Contributed by Yury
+ Selivanov in :issue:`24248`.)
+
+* :func:`inspect.getargspec` is deprecated and scheduled to be removed in
+ Python 3.6. (See :issue:`20438` for details.)
+
+* :func:`~inspect.getfullargspec`, :func:`~inspect.getargvalues`,
+ :func:`~inspect.getcallargs`, :func:`~inspect.getargvalues`,
+ :func:`~inspect.formatargspec`, and :func:`~inspect.formatargvalues` are
+ deprecated in favor of :func:`inspect.signature` API. (See :issue:`20438`
+ for details.)
+
+
+Deprecated functions and types of the C API
+-------------------------------------------
+
+* None yet.
+
+
+Deprecated features
+-------------------
+
+* None yet.
+
+
+Removed
+=======
+
+API and Feature Removals
+------------------------
+
+The following obsolete and previously deprecated APIs and features have been
+removed:
+
+* The ``__version__`` attribute has been dropped from the email package. The
+ email code hasn't been shipped separately from the stdlib for a long time,
+ and the ``__version__`` string was not updated in the last few releases.
+
+* The internal ``Netrc`` class in the :mod:`ftplib` module was deprecated in
+ 3.4, and has now been removed.
+ (Contributed by Matt Chaput in :issue:`6623`.)
+
+* The concept of ``.pyo`` files has been removed.
+
+* The JoinableQueue class in the provisional asyncio module was deprecated
+ in 3.4.4 and is now removed (:issue:`23464`).
+
+
+Porting to Python 3.5
+=====================
+
+This section lists previously described changes and other bugfixes
+that may require changes to your code.
+
+Changes in the Python API
+-------------------------
+
+* :pep:`475`: System calls are now retried when interrupted by a signal instead
+ of raising :exc:`InterruptedError` if the Python signal handler does not
+ raise an exception.
+
+* Before Python 3.5, a :class:`datetime.time` object was considered to be false
+ if it represented midnight in UTC. This behavior was considered obscure and
+ error-prone and has been removed in Python 3.5. See :issue:`13936` for full
+ details.
+
+* :meth:`ssl.SSLSocket.send()` now raises either :exc:`ssl.SSLWantReadError`
+ or :exc:`ssl.SSLWantWriteError` on a non-blocking socket if the operation
+ would block. Previously, it would return 0. See :issue:`20951`.
+
+* The ``__name__`` attribute of generator is now set from the function name,
+ instead of being set from the code name. Use ``gen.gi_code.co_name`` to
+ retrieve the code name. Generators also have a new ``__qualname__``
+ attribute, the qualified name, which is now used for the representation
+ of a generator (``repr(gen)``). See :issue:`21205`.
+
+* The deprecated "strict" mode and argument of :class:`~html.parser.HTMLParser`,
+ :meth:`HTMLParser.error`, and the :exc:`HTMLParserError` exception have been
+ removed. (Contributed by Ezio Melotti in :issue:`15114`.)
+ The *convert_charrefs* argument of :class:`~html.parser.HTMLParser` is
+ now ``True`` by default. (Contributed by Berker Peksag in :issue:`21047`.)
+
+* Although it is not formally part of the API, it is worth noting for porting
+ purposes (ie: fixing tests) that error messages that were previously of the
+ form "'sometype' does not support the buffer protocol" are now of the form "a
+ bytes-like object is required, not 'sometype'". (Contributed by Ezio Melotti
+ in :issue:`16518`.)
+
+* If the current directory is set to a directory that no longer exists then
+ :exc:`FileNotFoundError` will no longer be raised and instead
+ :meth:`~importlib.machinery.FileFinder.find_spec` will return ``None``
+ **without** caching ``None`` in :data:`sys.path_importer_cache` which is
+ different than the typical case (:issue:`22834`).
+
+* HTTP status code and messages from :mod:`http.client` and :mod:`http.server`
+ were refactored into a common :class:`~http.HTTPStatus` enum. The values in
+ :mod:`http.client` and :mod:`http.server` remain available for backwards
+ compatibility. (Contributed by Demian Brecht in :issue:`21793`.)
+
+* When an import loader defines :meth:`~importlib.machinery.Loader.exec_module`
+ it is now expected to also define
+ :meth:`~importlib.machinery.Loader.create_module` (raises a
+ :exc:`DeprecationWarning` now, will be an error in Python 3.6). If the loader
+ inherits from :class:`importlib.abc.Loader` then there is nothing to do, else
+ simply define :meth:`~importlib.machinery.Loader.create_module` to return
+ ``None`` (:issue:`23014`).
+
+* :func:`re.split` always ignored empty pattern matches, so the ``'x*'``
+ pattern worked the same as ``'x+'``, and the ``'\b'`` pattern never worked.
+ Now :func:`re.split` raises a warning if the pattern could match
+ an empty string. For compatibility use patterns that never match an empty
+ string (e.g. ``'x+'`` instead of ``'x*'``). Patterns that could only match
+ an empty string (such as ``'\b'``) now raise an error.
+
+* The :class:`~http.cookies.Morsel` dict-like interface has been made self
+ consistent: morsel comparison now takes the :attr:`~http.cookies.Morsel.key`
+ and :attr:`~http.cookies.Morsel.value` into account,
+ :meth:`~http.cookies.Morsel.copy` now results in a
+ :class:`~http.cookies.Morsel` instance rather than a :class:`dict`, and
+ :meth:`~http.cookies.Morsel.update` will now raise an exception if any of the
+ keys in the update dictionary are invalid. In addition, the undocumented
+ *LegalChars* parameter of :func:`~http.cookies.Morsel.set` is deprecated and
+ is now ignored. (:issue:`2211`)
+
+* :pep:`488` has removed ``.pyo`` files from Python and introduced the optional
+ ``opt-`` tag in ``.pyc`` file names. The
+ :func:`importlib.util.cache_from_source` has gained an *optimization*
+ parameter to help control the ``opt-`` tag. Because of this, the
+ *debug_override* parameter of the function is now deprecated. `.pyo` files
+ are also no longer supported as a file argument to the Python interpreter and
+ thus serve no purpose when distributed on their own (i.e. sourcless code
+ distribution). Due to the fact that the magic number for bytecode has changed
+ in Python 3.5, all old `.pyo` files from previous versions of Python are
+ invalid regardless of this PEP.
+
+* The :mod:`socket` module now exports the CAN_RAW_FD_FRAMES constant on linux
+ 3.6 and greater.
+
+* The ``pygettext.py`` Tool now uses the standard +NNNN format for timezones in
+ the POT-Creation-Date header.
+
+* The :mod:`smtplib` module now uses :data:`sys.stderr` instead of previous
+ module level :data:`stderr` variable for debug output. If your (test)
+ program depends on patching the module level variable to capture the debug
+ output, you will need to update it to capture sys.stderr instead.
+
+* The :meth:`str.startswith` and :meth:`str.endswith` methods no longer return
+ ``True`` when finding the empty string and the indexes are completely out of
+ range. See :issue:`24284`.
+
+* The :func:`inspect.getdoc` function now returns documentation strings
+ inherited from base classes. Documentation strings no longer need to be
+ duplicated if the inherited documentation is appropriate. To suppress an
+ inherited string, an empty string must be specified (or the documentation
+ may be filled in). This change affects the output of the :mod:`pydoc`
+ module and the :func:`help` function. See :issue:`15582`.
+
+Changes in the C API
+--------------------
+
+* The undocumented :c:member:`~PyMemoryViewObject.format` member of the
+ (non-public) :c:type:`PyMemoryViewObject` structure has been removed.
+
+ All extensions relying on the relevant parts in ``memoryobject.h``
+ must be rebuilt.
+
+* The :c:type:`PyMemAllocator` structure was renamed to
+ :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added.
+
+* Removed non-documented macro :c:macro:`PyObject_REPR` which leaked references.
+ Use format character ``%R`` in :c:func:`PyUnicode_FromFormat`-like functions
+ to format the :func:`repr` of the object.
+
+* Because the lack of the :attr:`__module__` attribute breaks pickling and
+ introspection, a deprecation warning now is raised for builtin type without
+ the :attr:`__module__` attribute. Would be an AttributeError in future.
+ (:issue:`20204`)
+
+* As part of :pep:`492` implementation, ``tp_reserved`` slot of
+ :c:type:`PyTypeObject` was replaced with a
+ :c:member:`tp_as_async` slot. Refer to :ref:`coro-objects` for
+ new types, structures and functions.
diff --git a/Doc/whatsnew/index.rst b/Doc/whatsnew/index.rst
index 29902e4..edb5502 100644
--- a/Doc/whatsnew/index.rst
+++ b/Doc/whatsnew/index.rst
@@ -11,6 +11,7 @@ anyone wishing to stay up-to-date after a new release.
.. toctree::
:maxdepth: 2
+ 3.5.rst
3.4.rst
3.3.rst
3.2.rst
diff --git a/Grammar/Grammar b/Grammar/Grammar
index 05c3181..99fcea0 100644
--- a/Grammar/Grammar
+++ b/Grammar/Grammar
@@ -21,8 +21,11 @@ eval_input: testlist NEWLINE* ENDMARKER
decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
decorators: decorator+
-decorated: decorators (classdef | funcdef)
+decorated: decorators (classdef | funcdef | async_funcdef)
+
+async_funcdef: ASYNC funcdef
funcdef: 'def' NAME parameters ['->' test] ':' suite
+
parameters: '(' [typedargslist] ')'
typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [','
['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]]
@@ -40,7 +43,7 @@ small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) |
('=' (yield_expr|testlist_star_expr))*)
testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
-augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
+augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
'<<=' | '>>=' | '**=' | '//=')
# For normal assignments, additional restrictions enforced by the interpreter
del_stmt: 'del' exprlist
@@ -65,7 +68,8 @@ global_stmt: 'global' NAME (',' NAME)*
nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
assert_stmt: 'assert' test [',' test]
-compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
+compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
+async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
while_stmt: 'while' test ':' suite ['else' ':' suite]
for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
@@ -89,7 +93,7 @@ and_test: not_test ('and' not_test)*
not_test: 'not' not_test | comparison
comparison: expr (comp_op expr)*
# <> isn't actually a valid comparison operator in Python. It's here for the
-# sake of a __future__ import described in PEP 401
+# sake of a __future__ import described in PEP 401 (which really works :-)
comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
star_expr: '*' expr
expr: xor_expr ('|' xor_expr)*
@@ -97,9 +101,10 @@ xor_expr: and_expr ('^' and_expr)*
and_expr: shift_expr ('&' shift_expr)*
shift_expr: arith_expr (('<<'|'>>') arith_expr)*
arith_expr: term (('+'|'-') term)*
-term: factor (('*'|'/'|'%'|'//') factor)*
+term: factor (('*'|'@'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power
-power: atom trailer* ['**' factor]
+power: atom_expr ['**' factor]
+atom_expr: [AWAIT] atom trailer*
atom: ('(' [yield_expr|testlist_comp] ')' |
'[' [testlist_comp] ']' |
'{' [dictorsetmaker] '}' |
@@ -111,17 +116,29 @@ subscript: test | [test] ':' [test] [sliceop]
sliceop: ':' [test]
exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
testlist: test (',' test)* [',']
-dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
- (test (comp_for | (',' test)* [','])) )
+dictorsetmaker: ( ((test ':' test | '**' expr)
+ (comp_for | (',' (test ':' test | '**' expr))* [','])) |
+ ((test | star_expr)
+ (comp_for | (',' (test | star_expr))* [','])) )
classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
-arglist: (argument ',')* (argument [',']
- |'*' test (',' argument)* [',' '**' test]
- |'**' test)
+arglist: argument (',' argument)* [',']
+
# The reason that keywords are test nodes instead of NAME is that using NAME
# results in an ambiguity. ast.c makes sure it's a NAME.
-argument: test [comp_for] | test '=' test # Really [keyword '='] test
+# "test '=' test" is really "keyword '=' test", but we have no such token.
+# These need to be in a single rule to avoid grammar that is ambiguous
+# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr,
+# we explicitly match '*' here, too, to give it proper precedence.
+# Illegal combinations and orderings are blocked in ast.c:
+# multiple (test comp_for) arguements are blocked; keyword unpackings
+# that precede iterable unpackings are blocked; etc.
+argument: ( test [comp_for] |
+ test '=' test |
+ '**' test |
+ '*' test )
+
comp_iter: comp_for | comp_if
comp_for: 'for' exprlist 'in' or_test [comp_iter]
comp_if: 'if' test_nocond [comp_iter]
diff --git a/Include/Python-ast.h b/Include/Python-ast.h
index 67d677b..3bc015f 100644
--- a/Include/Python-ast.h
+++ b/Include/Python-ast.h
@@ -15,9 +15,9 @@ typedef struct _slice *slice_ty;
typedef enum _boolop { And=1, Or=2 } boolop_ty;
-typedef enum _operator { Add=1, Sub=2, Mult=3, Div=4, Mod=5, Pow=6, LShift=7,
- RShift=8, BitOr=9, BitXor=10, BitAnd=11, FloorDiv=12 }
- operator_ty;
+typedef enum _operator { Add=1, Sub=2, Mult=3, MatMult=4, Div=5, Mod=6, Pow=7,
+ LShift=8, RShift=9, BitOr=10, BitXor=11, BitAnd=12,
+ FloorDiv=13 } operator_ty;
typedef enum _unaryop { Invert=1, Not=2, UAdd=3, USub=4 } unaryop_ty;
@@ -63,12 +63,13 @@ struct _mod {
} v;
};
-enum _stmt_kind {FunctionDef_kind=1, ClassDef_kind=2, Return_kind=3,
- Delete_kind=4, Assign_kind=5, AugAssign_kind=6, For_kind=7,
- While_kind=8, If_kind=9, With_kind=10, Raise_kind=11,
- Try_kind=12, Assert_kind=13, Import_kind=14,
- ImportFrom_kind=15, Global_kind=16, Nonlocal_kind=17,
- Expr_kind=18, Pass_kind=19, Break_kind=20, Continue_kind=21};
+enum _stmt_kind {FunctionDef_kind=1, AsyncFunctionDef_kind=2, ClassDef_kind=3,
+ Return_kind=4, Delete_kind=5, Assign_kind=6,
+ AugAssign_kind=7, For_kind=8, AsyncFor_kind=9, While_kind=10,
+ If_kind=11, With_kind=12, AsyncWith_kind=13, Raise_kind=14,
+ Try_kind=15, Assert_kind=16, Import_kind=17,
+ ImportFrom_kind=18, Global_kind=19, Nonlocal_kind=20,
+ Expr_kind=21, Pass_kind=22, Break_kind=23, Continue_kind=24};
struct _stmt {
enum _stmt_kind kind;
union {
@@ -82,10 +83,16 @@ struct _stmt {
struct {
identifier name;
+ arguments_ty args;
+ asdl_seq *body;
+ asdl_seq *decorator_list;
+ expr_ty returns;
+ } AsyncFunctionDef;
+
+ struct {
+ identifier name;
asdl_seq *bases;
asdl_seq *keywords;
- expr_ty starargs;
- expr_ty kwargs;
asdl_seq *body;
asdl_seq *decorator_list;
} ClassDef;
@@ -117,6 +124,13 @@ struct _stmt {
} For;
struct {
+ expr_ty target;
+ expr_ty iter;
+ asdl_seq *body;
+ asdl_seq *orelse;
+ } AsyncFor;
+
+ struct {
expr_ty test;
asdl_seq *body;
asdl_seq *orelse;
@@ -134,6 +148,11 @@ struct _stmt {
} With;
struct {
+ asdl_seq *items;
+ asdl_seq *body;
+ } AsyncWith;
+
+ struct {
expr_ty exc;
expr_ty cause;
} Raise;
@@ -180,11 +199,11 @@ struct _stmt {
enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
IfExp_kind=5, Dict_kind=6, Set_kind=7, ListComp_kind=8,
SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11,
- Yield_kind=12, YieldFrom_kind=13, Compare_kind=14,
- Call_kind=15, Num_kind=16, Str_kind=17, Bytes_kind=18,
- NameConstant_kind=19, Ellipsis_kind=20, Attribute_kind=21,
- Subscript_kind=22, Starred_kind=23, Name_kind=24,
- List_kind=25, Tuple_kind=26};
+ Await_kind=12, Yield_kind=13, YieldFrom_kind=14,
+ Compare_kind=15, Call_kind=16, Num_kind=17, Str_kind=18,
+ Bytes_kind=19, NameConstant_kind=20, Ellipsis_kind=21,
+ Attribute_kind=22, Subscript_kind=23, Starred_kind=24,
+ Name_kind=25, List_kind=26, Tuple_kind=27};
struct _expr {
enum _expr_kind kind;
union {
@@ -247,6 +266,10 @@ struct _expr {
struct {
expr_ty value;
+ } Await;
+
+ struct {
+ expr_ty value;
} Yield;
struct {
@@ -263,8 +286,6 @@ struct _expr {
expr_ty func;
asdl_seq *args;
asdl_seq *keywords;
- expr_ty starargs;
- expr_ty kwargs;
} Call;
struct {
@@ -406,11 +427,14 @@ mod_ty _Py_Suite(asdl_seq * body, PyArena *arena);
stmt_ty _Py_FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
asdl_seq * decorator_list, expr_ty returns, int lineno,
int col_offset, PyArena *arena);
-#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9)
+#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
+stmt_ty _Py_AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq *
+ body, asdl_seq * decorator_list, expr_ty returns,
+ int lineno, int col_offset, PyArena *arena);
+#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7)
stmt_ty _Py_ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords,
- expr_ty starargs, expr_ty kwargs, asdl_seq * body,
- asdl_seq * decorator_list, int lineno, int col_offset,
- PyArena *arena);
+ asdl_seq * body, asdl_seq * decorator_list, int lineno,
+ int col_offset, PyArena *arena);
#define Return(a0, a1, a2, a3) _Py_Return(a0, a1, a2, a3)
stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, PyArena *arena);
#define Delete(a0, a1, a2, a3) _Py_Delete(a0, a1, a2, a3)
@@ -425,6 +449,9 @@ stmt_ty _Py_AugAssign(expr_ty target, operator_ty op, expr_ty value, int
#define For(a0, a1, a2, a3, a4, a5, a6) _Py_For(a0, a1, a2, a3, a4, a5, a6)
stmt_ty _Py_For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq *
orelse, int lineno, int col_offset, PyArena *arena);
+#define AsyncFor(a0, a1, a2, a3, a4, a5, a6) _Py_AsyncFor(a0, a1, a2, a3, a4, a5, a6)
+stmt_ty _Py_AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq *
+ orelse, int lineno, int col_offset, PyArena *arena);
#define While(a0, a1, a2, a3, a4, a5) _Py_While(a0, a1, a2, a3, a4, a5)
stmt_ty _Py_While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
int col_offset, PyArena *arena);
@@ -434,6 +461,9 @@ stmt_ty _Py_If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
#define With(a0, a1, a2, a3, a4) _Py_With(a0, a1, a2, a3, a4)
stmt_ty _Py_With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset,
PyArena *arena);
+#define AsyncWith(a0, a1, a2, a3, a4) _Py_AsyncWith(a0, a1, a2, a3, a4)
+stmt_ty _Py_AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int
+ col_offset, PyArena *arena);
#define Raise(a0, a1, a2, a3, a4) _Py_Raise(a0, a1, a2, a3, a4)
stmt_ty _Py_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset,
PyArena *arena);
@@ -496,6 +526,8 @@ expr_ty _Py_DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int
#define GeneratorExp(a0, a1, a2, a3, a4) _Py_GeneratorExp(a0, a1, a2, a3, a4)
expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int
col_offset, PyArena *arena);
+#define Await(a0, a1, a2, a3) _Py_Await(a0, a1, a2, a3)
+expr_ty _Py_Await(expr_ty value, int lineno, int col_offset, PyArena *arena);
#define Yield(a0, a1, a2, a3) _Py_Yield(a0, a1, a2, a3)
expr_ty _Py_Yield(expr_ty value, int lineno, int col_offset, PyArena *arena);
#define YieldFrom(a0, a1, a2, a3) _Py_YieldFrom(a0, a1, a2, a3)
@@ -504,10 +536,9 @@ expr_ty _Py_YieldFrom(expr_ty value, int lineno, int col_offset, PyArena
#define Compare(a0, a1, a2, a3, a4, a5) _Py_Compare(a0, a1, a2, a3, a4, a5)
expr_ty _Py_Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators,
int lineno, int col_offset, PyArena *arena);
-#define Call(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Call(a0, a1, a2, a3, a4, a5, a6, a7)
-expr_ty _Py_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty
- starargs, expr_ty kwargs, int lineno, int col_offset, PyArena
- *arena);
+#define Call(a0, a1, a2, a3, a4, a5) _Py_Call(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int
+ lineno, int col_offset, PyArena *arena);
#define Num(a0, a1, a2, a3) _Py_Num(a0, a1, a2, a3)
expr_ty _Py_Num(object n, int lineno, int col_offset, PyArena *arena);
#define Str(a0, a1, a2, a3) _Py_Str(a0, a1, a2, a3)
diff --git a/Include/Python.h b/Include/Python.h
index 2dd8290..858dbd1 100644
--- a/Include/Python.h
+++ b/Include/Python.h
@@ -85,6 +85,7 @@
#include "tupleobject.h"
#include "listobject.h"
#include "dictobject.h"
+#include "odictobject.h"
#include "enumobject.h"
#include "setobject.h"
#include "methodobject.h"
@@ -112,6 +113,7 @@
#include "pyarena.h"
#include "modsupport.h"
#include "pythonrun.h"
+#include "pylifecycle.h"
#include "ceval.h"
#include "sysmodule.h"
#include "intrcheck.h"
diff --git a/Include/abstract.h b/Include/abstract.h
index 6e850b8..83dbf94 100644
--- a/Include/abstract.h
+++ b/Include/abstract.h
@@ -266,6 +266,12 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
PyAPI_FUNC(PyObject *) PyObject_Call(PyObject *callable_object,
PyObject *args, PyObject *kw);
+#ifndef Py_LIMITED_API
+ PyAPI_FUNC(PyObject *) _Py_CheckFunctionResult(PyObject *func,
+ PyObject *result,
+ const char *where);
+#endif
+
/*
Call a callable Python object, callable_object, with
arguments and keywords arguments. The 'args' argument can not be
@@ -658,6 +664,12 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
o1*o2.
*/
+ PyAPI_FUNC(PyObject *) PyNumber_MatrixMultiply(PyObject *o1, PyObject *o2);
+
+ /*
+ This is the equivalent of the Python expression: o1 @ o2.
+ */
+
PyAPI_FUNC(PyObject *) PyNumber_FloorDivide(PyObject *o1, PyObject *o2);
/*
@@ -832,6 +844,12 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
o1 *= o2.
*/
+ PyAPI_FUNC(PyObject *) PyNumber_InPlaceMatrixMultiply(PyObject *o1, PyObject *o2);
+
+ /*
+ This is the equivalent of the Python expression: o1 @= o2.
+ */
+
PyAPI_FUNC(PyObject *) PyNumber_InPlaceFloorDivide(PyObject *o1,
PyObject *o2);
diff --git a/Include/bytes_methods.h b/Include/bytes_methods.h
index 1498b8f..11d5f42 100644
--- a/Include/bytes_methods.h
+++ b/Include/bytes_methods.h
@@ -21,8 +21,8 @@ extern void _Py_bytes_title(char *result, char *s, Py_ssize_t len);
extern void _Py_bytes_capitalize(char *result, char *s, Py_ssize_t len);
extern void _Py_bytes_swapcase(char *result, char *s, Py_ssize_t len);
-/* This one gets the raw argument list. */
-extern PyObject* _Py_bytes_maketrans(PyObject *args);
+/* The maketrans() static method. */
+extern PyObject* _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to);
/* Shared __doc__ strings. */
extern const char _Py_isspace__doc__[];
diff --git a/Include/bytesobject.h b/Include/bytesobject.h
index 0ee8d36..e379bac 100644
--- a/Include/bytesobject.h
+++ b/Include/bytesobject.h
@@ -62,6 +62,7 @@ PyAPI_FUNC(void) PyBytes_Concat(PyObject **, PyObject *);
PyAPI_FUNC(void) PyBytes_ConcatAndDel(PyObject **, PyObject *);
#ifndef Py_LIMITED_API
PyAPI_FUNC(int) _PyBytes_Resize(PyObject **, Py_ssize_t);
+PyAPI_FUNC(PyObject *) _PyBytes_Format(PyObject *, PyObject *);
#endif
PyAPI_FUNC(PyObject *) PyBytes_DecodeEscape(const char *, Py_ssize_t,
const char *, Py_ssize_t,
diff --git a/Include/ceval.h b/Include/ceval.h
index 4937f2c..eb1ee43 100644
--- a/Include/ceval.h
+++ b/Include/ceval.h
@@ -23,6 +23,8 @@ PyAPI_FUNC(PyObject *) PyEval_CallMethod(PyObject *obj,
#ifndef Py_LIMITED_API
PyAPI_FUNC(void) PyEval_SetProfile(Py_tracefunc, PyObject *);
PyAPI_FUNC(void) PyEval_SetTrace(Py_tracefunc, PyObject *);
+PyAPI_FUNC(void) _PyEval_SetCoroutineWrapper(PyObject *);
+PyAPI_FUNC(PyObject *) _PyEval_GetCoroutineWrapper(void);
#endif
struct _frame; /* Avoid including frameobject.h */
@@ -46,16 +48,16 @@ PyAPI_FUNC(int) Py_MakePendingCalls(void);
In Python 3.0, this protection has two levels:
* normal anti-recursion protection is triggered when the recursion level
- exceeds the current recursion limit. It raises a RuntimeError, and sets
+ exceeds the current recursion limit. It raises a RecursionError, and sets
the "overflowed" flag in the thread state structure. This flag
temporarily *disables* the normal protection; this allows cleanup code
to potentially outgrow the recursion limit while processing the
- RuntimeError.
+ RecursionError.
* "last chance" anti-recursion protection is triggered when the recursion
level exceeds "current recursion limit + 50". By construction, this
protection can only be triggered when the "overflowed" flag is set. It
means the cleanup code has itself gone into an infinite loop, or the
- RuntimeError has been mistakingly ignored. When this protection is
+ RecursionError has been mistakingly ignored. When this protection is
triggered, the interpreter aborts with a Fatal Error.
In addition, the "overflowed" flag is automatically reset when the
diff --git a/Include/code.h b/Include/code.h
index 7c7e5bf..56e6ec1 100644
--- a/Include/code.h
+++ b/Include/code.h
@@ -21,7 +21,12 @@ typedef struct {
PyObject *co_varnames; /* tuple of strings (local variable names) */
PyObject *co_freevars; /* tuple of strings (free variable names) */
PyObject *co_cellvars; /* tuple of strings (cell variable names) */
- /* The rest doesn't count for hash or comparisons */
+ /* The rest aren't used in either hash or comparisons, except for
+ co_name (used in both) and co_firstlineno (used only in
+ comparisons). This is done to preserve the name and line number
+ for tracebacks and debuggers; otherwise, constant de-duplication
+ would collapse identical functions/lambdas defined on different lines.
+ */
unsigned char *co_cell2arg; /* Maps cell vars which are arguments. */
PyObject *co_filename; /* unicode (where it was loaded from) */
PyObject *co_name; /* unicode (name, for reference) */
@@ -46,6 +51,11 @@ typedef struct {
*/
#define CO_NOFREE 0x0040
+/* The CO_COROUTINE flag is set for coroutine functions (defined with
+ ``async def`` keywords) */
+#define CO_COROUTINE 0x0080
+#define CO_ITERABLE_COROUTINE 0x0100
+
/* These are no longer used. */
#if 0
#define CO_GENERATOR_ALLOWED 0x1000
@@ -57,6 +67,7 @@ typedef struct {
#define CO_FUTURE_UNICODE_LITERALS 0x20000
#define CO_FUTURE_BARRY_AS_BDFL 0x40000
+#define CO_FUTURE_GENERATOR_STOP 0x80000
/* This value is found in the co_cell2arg array when the associated cell
variable does not correspond to an argument. The maximum number of
diff --git a/Include/codecs.h b/Include/codecs.h
index b3088e4..9e4f305 100644
--- a/Include/codecs.h
+++ b/Include/codecs.h
@@ -71,7 +71,7 @@ PyAPI_FUNC(int) PyCodec_KnownEncoding(
object is passed through the encoder function found for the given
encoding using the error handling method defined by errors. errors
may be NULL to use the default method defined for the codec.
-
+
Raises a LookupError in case no encoder can be found.
*/
@@ -87,7 +87,7 @@ PyAPI_FUNC(PyObject *) PyCodec_Encode(
object is passed through the decoder function found for the given
encoding using the error handling method defined by errors. errors
may be NULL to use the default method defined for the codec.
-
+
Raises a LookupError in case no encoder can be found.
*/
@@ -145,7 +145,7 @@ PyAPI_FUNC(PyObject *) _PyCodecInfo_GetIncrementalEncoder(
-/* --- Codec Lookup APIs --------------------------------------------------
+/* --- Codec Lookup APIs --------------------------------------------------
All APIs return a codec object with incremented refcount and are
based on _PyCodec_Lookup(). The same comments w/r to the encoding
@@ -225,6 +225,9 @@ PyAPI_FUNC(PyObject *) PyCodec_XMLCharRefReplaceErrors(PyObject *exc);
/* replace the unicode encode error with backslash escapes (\x, \u and \U) */
PyAPI_FUNC(PyObject *) PyCodec_BackslashReplaceErrors(PyObject *exc);
+/* replace the unicode encode error with backslash escapes (\N, \x, \u and \U) */
+PyAPI_FUNC(PyObject *) PyCodec_NameReplaceErrors(PyObject *exc);
+
PyAPI_DATA(const char *) Py_hexdigits;
#ifdef __cplusplus
diff --git a/Include/compile.h b/Include/compile.h
index c6650d7f..ecd8dc1 100644
--- a/Include/compile.h
+++ b/Include/compile.h
@@ -27,6 +27,7 @@ typedef struct {
#define FUTURE_PRINT_FUNCTION "print_function"
#define FUTURE_UNICODE_LITERALS "unicode_literals"
#define FUTURE_BARRY_AS_BDFL "barry_as_FLUFL"
+#define FUTURE_GENERATOR_STOP "generator_stop"
struct _mod; /* Declare the existence of this type */
#define PyAST_Compile(mod, s, f, ar) PyAST_CompileEx(mod, s, f, -1, ar)
diff --git a/Include/complexobject.h b/Include/complexobject.h
index 1934f3b..cb8c52c 100644
--- a/Include/complexobject.h
+++ b/Include/complexobject.h
@@ -14,21 +14,13 @@ typedef struct {
/* Operations on complex numbers from complexmodule.c */
-#define c_sum _Py_c_sum
-#define c_diff _Py_c_diff
-#define c_neg _Py_c_neg
-#define c_prod _Py_c_prod
-#define c_quot _Py_c_quot
-#define c_pow _Py_c_pow
-#define c_abs _Py_c_abs
-
-PyAPI_FUNC(Py_complex) c_sum(Py_complex, Py_complex);
-PyAPI_FUNC(Py_complex) c_diff(Py_complex, Py_complex);
-PyAPI_FUNC(Py_complex) c_neg(Py_complex);
-PyAPI_FUNC(Py_complex) c_prod(Py_complex, Py_complex);
-PyAPI_FUNC(Py_complex) c_quot(Py_complex, Py_complex);
-PyAPI_FUNC(Py_complex) c_pow(Py_complex, Py_complex);
-PyAPI_FUNC(double) c_abs(Py_complex);
+PyAPI_FUNC(Py_complex) _Py_c_sum(Py_complex, Py_complex);
+PyAPI_FUNC(Py_complex) _Py_c_diff(Py_complex, Py_complex);
+PyAPI_FUNC(Py_complex) _Py_c_neg(Py_complex);
+PyAPI_FUNC(Py_complex) _Py_c_prod(Py_complex, Py_complex);
+PyAPI_FUNC(Py_complex) _Py_c_quot(Py_complex, Py_complex);
+PyAPI_FUNC(Py_complex) _Py_c_pow(Py_complex, Py_complex);
+PyAPI_FUNC(double) _Py_c_abs(Py_complex);
#endif
/* Complex object interface */
diff --git a/Include/dictobject.h b/Include/dictobject.h
index ef122bd..320f9ec 100644
--- a/Include/dictobject.h
+++ b/Include/dictobject.h
@@ -27,6 +27,11 @@ typedef struct {
PyObject **ma_values;
} PyDictObject;
+typedef struct {
+ PyObject_HEAD
+ PyDictObject *dv_dict;
+} _PyDictViewObject;
+
#endif /* Py_LIMITED_API */
PyAPI_DATA(PyTypeObject) PyDict_Type;
@@ -40,9 +45,9 @@ PyAPI_DATA(PyTypeObject) PyDictValues_Type;
#define PyDict_Check(op) \
PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_DICT_SUBCLASS)
#define PyDict_CheckExact(op) (Py_TYPE(op) == &PyDict_Type)
-#define PyDictKeys_Check(op) (Py_TYPE(op) == &PyDictKeys_Type)
-#define PyDictItems_Check(op) (Py_TYPE(op) == &PyDictItems_Type)
-#define PyDictValues_Check(op) (Py_TYPE(op) == &PyDictValues_Type)
+#define PyDictKeys_Check(op) PyObject_TypeCheck(op, &PyDictKeys_Type)
+#define PyDictItems_Check(op) PyObject_TypeCheck(op, &PyDictItems_Type)
+#define PyDictValues_Check(op) PyObject_TypeCheck(op, &PyDictValues_Type)
/* This excludes Values, since they are not sets. */
# define PyDictViewSet_Check(op) \
(PyDictKeys_Check(op) || PyDictItems_Check(op))
@@ -50,6 +55,10 @@ PyAPI_DATA(PyTypeObject) PyDictValues_Type;
PyAPI_FUNC(PyObject *) PyDict_New(void);
PyAPI_FUNC(PyObject *) PyDict_GetItem(PyObject *mp, PyObject *key);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key,
+ Py_hash_t hash);
+#endif
PyAPI_FUNC(PyObject *) PyDict_GetItemWithError(PyObject *mp, PyObject *key);
PyAPI_FUNC(PyObject *) _PyDict_GetItemIdWithError(PyObject *dp,
struct _Py_Identifier *key);
@@ -58,6 +67,10 @@ PyAPI_FUNC(PyObject *) PyDict_SetDefault(
PyObject *mp, PyObject *key, PyObject *defaultobj);
#endif
PyAPI_FUNC(int) PyDict_SetItem(PyObject *mp, PyObject *key, PyObject *item);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key,
+ PyObject *item, Py_hash_t hash);
+#endif
PyAPI_FUNC(int) PyDict_DelItem(PyObject *mp, PyObject *key);
PyAPI_FUNC(void) PyDict_Clear(PyObject *mp);
PyAPI_FUNC(int) PyDict_Next(
@@ -67,6 +80,7 @@ PyDictKeysObject *_PyDict_NewKeysForClass(void);
PyAPI_FUNC(PyObject *) PyObject_GenericGetDict(PyObject *, void *);
PyAPI_FUNC(int) _PyDict_Next(
PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, Py_hash_t *hash);
+PyObject *_PyDictView_New(PyObject *, PyTypeObject *);
#endif
PyAPI_FUNC(PyObject *) PyDict_Keys(PyObject *mp);
PyAPI_FUNC(PyObject *) PyDict_Values(PyObject *mp);
@@ -80,6 +94,9 @@ PyAPI_FUNC(PyObject *) _PyDict_NewPresized(Py_ssize_t minused);
PyAPI_FUNC(void) _PyDict_MaybeUntrack(PyObject *mp);
PyAPI_FUNC(int) _PyDict_HasOnlyStringKeys(PyObject *mp);
Py_ssize_t _PyDict_KeysSize(PyDictKeysObject *keys);
+PyObject *_PyDict_SizeOf(PyDictObject *);
+PyObject *_PyDict_Pop(PyDictObject *, PyObject *, PyObject *);
+PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *);
#define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL)
PyAPI_FUNC(int) PyDict_ClearFreeList(void);
@@ -97,6 +114,10 @@ PyAPI_FUNC(int) PyDict_Merge(PyObject *mp,
PyObject *other,
int override);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject *) _PyDictView_Intersect(PyObject* self, PyObject *other);
+#endif
+
/* PyDict_MergeFromSeq2 updates/merges from an iterable object producing
iterable objects of length 2. If override is true, the last occurrence
of a key wins, else the first. The Python dict constructor dict(seq2)
diff --git a/Include/fileobject.h b/Include/fileobject.h
index 0939744..03155d3 100644
--- a/Include/fileobject.h
+++ b/Include/fileobject.h
@@ -32,17 +32,6 @@ PyAPI_DATA(int) Py_HasFileSystemDefaultEncoding;
#ifndef Py_LIMITED_API
PyAPI_FUNC(PyObject *) PyFile_NewStdPrinter(int);
PyAPI_DATA(PyTypeObject) PyStdPrinter_Type;
-
-#if defined _MSC_VER && _MSC_VER >= 1400
-/* A routine to check if a file descriptor is valid on Windows. Returns 0
- * and sets errno to EBADF if it isn't. This is to avoid Assertions
- * from various functions in the Windows CRT beginning with
- * Visual Studio 2005
- */
-int _PyVerify_fd(int fd);
-#else
-#define _PyVerify_fd(A) (1) /* dummy */
-#endif
#endif /* Py_LIMITED_API */
/* A routine to check if a file descriptor can be select()-ed. */
diff --git a/Include/fileutils.h b/Include/fileutils.h
index e9bad80..b4a683c 100644
--- a/Include/fileutils.h
+++ b/Include/fileutils.h
@@ -7,30 +7,59 @@ extern "C" {
PyAPI_FUNC(PyObject *) _Py_device_encoding(int);
-PyAPI_FUNC(wchar_t *) _Py_char2wchar(
+PyAPI_FUNC(wchar_t *) Py_DecodeLocale(
const char *arg,
size_t *size);
-PyAPI_FUNC(char*) _Py_wchar2char(
+PyAPI_FUNC(char*) Py_EncodeLocale(
const wchar_t *text,
size_t *error_pos);
-#if defined(HAVE_STAT) && !defined(MS_WINDOWS)
-PyAPI_FUNC(int) _Py_wstat(
- const wchar_t* path,
- struct stat *buf);
+#ifndef Py_LIMITED_API
+
+#ifdef MS_WINDOWS
+struct _Py_stat_struct {
+ unsigned long st_dev;
+ __int64 st_ino;
+ unsigned short st_mode;
+ int st_nlink;
+ int st_uid;
+ int st_gid;
+ unsigned long st_rdev;
+ __int64 st_size;
+ time_t st_atime;
+ int st_atime_nsec;
+ time_t st_mtime;
+ int st_mtime_nsec;
+ time_t st_ctime;
+ int st_ctime_nsec;
+ unsigned long st_file_attributes;
+};
+#else
+# define _Py_stat_struct stat
#endif
-#ifdef HAVE_STAT
+PyAPI_FUNC(int) _Py_fstat(
+ int fd,
+ struct _Py_stat_struct *status);
+
+PyAPI_FUNC(int) _Py_fstat_noraise(
+ int fd,
+ struct _Py_stat_struct *status);
+#endif /* Py_LIMITED_API */
+
PyAPI_FUNC(int) _Py_stat(
PyObject *path,
- struct stat *statbuf);
-#endif
+ struct stat *status);
#ifndef Py_LIMITED_API
PyAPI_FUNC(int) _Py_open(
const char *pathname,
int flags);
+
+PyAPI_FUNC(int) _Py_open_noraise(
+ const char *pathname,
+ int flags);
#endif
PyAPI_FUNC(FILE *) _Py_wfopen(
@@ -45,6 +74,21 @@ PyAPI_FUNC(FILE*) _Py_fopen_obj(
PyObject *path,
const char *mode);
+PyAPI_FUNC(Py_ssize_t) _Py_read(
+ int fd,
+ void *buf,
+ size_t count);
+
+PyAPI_FUNC(Py_ssize_t) _Py_write(
+ int fd,
+ const void *buf,
+ size_t count);
+
+PyAPI_FUNC(Py_ssize_t) _Py_write_noraise(
+ int fd,
+ const void *buf,
+ size_t count);
+
#ifdef HAVE_READLINK
PyAPI_FUNC(int) _Py_wreadlink(
const wchar_t *path,
@@ -70,8 +114,27 @@ PyAPI_FUNC(int) _Py_set_inheritable(int fd, int inheritable,
int *atomic_flag_works);
PyAPI_FUNC(int) _Py_dup(int fd);
+
+#ifndef MS_WINDOWS
+PyAPI_FUNC(int) _Py_get_blocking(int fd);
+
+PyAPI_FUNC(int) _Py_set_blocking(int fd, int blocking);
+#endif /* !MS_WINDOWS */
+
+#if defined _MSC_VER && _MSC_VER >= 1400 && _MSC_VER < 1900
+/* A routine to check if a file descriptor is valid on Windows. Returns 0
+ * and sets errno to EBADF if it isn't. This is to avoid Assertions
+ * from various functions in the Windows CRT beginning with
+ * Visual Studio 2005
+ */
+int _PyVerify_fd(int fd);
+
+#else
+#define _PyVerify_fd(A) (1) /* dummy */
#endif
+#endif /* Py_LIMITED_API */
+
#ifdef __cplusplus
}
#endif
diff --git a/Include/genobject.h b/Include/genobject.h
index 65f1ecf..4c71861 100644
--- a/Include/genobject.h
+++ b/Include/genobject.h
@@ -10,21 +10,26 @@ extern "C" {
struct _frame; /* Avoid including frameobject.h */
+/* _PyGenObject_HEAD defines the initial segment of generator
+ and coroutine objects. */
+#define _PyGenObject_HEAD(prefix) \
+ PyObject_HEAD \
+ /* Note: gi_frame can be NULL if the generator is "finished" */ \
+ struct _frame *prefix##_frame; \
+ /* True if generator is being executed. */ \
+ char prefix##_running; \
+ /* The code object backing the generator */ \
+ PyObject *prefix##_code; \
+ /* List of weak reference. */ \
+ PyObject *prefix##_weakreflist; \
+ /* Name of the generator. */ \
+ PyObject *prefix##_name; \
+ /* Qualified name of the generator. */ \
+ PyObject *prefix##_qualname;
+
typedef struct {
- PyObject_HEAD
/* The gi_ prefix is intended to remind of generator-iterator. */
-
- /* Note: gi_frame can be NULL if the generator is "finished" */
- struct _frame *gi_frame;
-
- /* True if generator is being executed. */
- char gi_running;
-
- /* The code object backing the generator */
- PyObject *gi_code;
-
- /* List of weak reference. */
- PyObject *gi_weakreflist;
+ _PyGenObject_HEAD(gi)
} PyGenObject;
PyAPI_DATA(PyTypeObject) PyGen_Type;
@@ -33,11 +38,28 @@ PyAPI_DATA(PyTypeObject) PyGen_Type;
#define PyGen_CheckExact(op) (Py_TYPE(op) == &PyGen_Type)
PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *);
+PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(struct _frame *,
+ PyObject *name, PyObject *qualname);
PyAPI_FUNC(int) PyGen_NeedsFinalizing(PyGenObject *);
PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **);
PyObject *_PyGen_Send(PyGenObject *, PyObject *);
PyAPI_FUNC(void) _PyGen_Finalize(PyObject *self);
+#ifndef Py_LIMITED_API
+typedef struct {
+ _PyGenObject_HEAD(cr)
+} PyCoroObject;
+
+PyAPI_DATA(PyTypeObject) PyCoro_Type;
+PyAPI_DATA(PyTypeObject) _PyCoroWrapper_Type;
+
+#define PyCoro_CheckExact(op) (Py_TYPE(op) == &PyCoro_Type)
+PyObject *_PyCoro_GetAwaitableIter(PyObject *o);
+PyAPI_FUNC(PyObject *) PyCoro_New(struct _frame *,
+ PyObject *name, PyObject *qualname);
+#endif
+
+#undef _PyGenObject_HEAD
#ifdef __cplusplus
}
diff --git a/Include/graminit.h b/Include/graminit.h
index 3ec949a..d030bc3 100644
--- a/Include/graminit.h
+++ b/Include/graminit.h
@@ -6,79 +6,82 @@
#define decorator 259
#define decorators 260
#define decorated 261
-#define funcdef 262
-#define parameters 263
-#define typedargslist 264
-#define tfpdef 265
-#define varargslist 266
-#define vfpdef 267
-#define stmt 268
-#define simple_stmt 269
-#define small_stmt 270
-#define expr_stmt 271
-#define testlist_star_expr 272
-#define augassign 273
-#define del_stmt 274
-#define pass_stmt 275
-#define flow_stmt 276
-#define break_stmt 277
-#define continue_stmt 278
-#define return_stmt 279
-#define yield_stmt 280
-#define raise_stmt 281
-#define import_stmt 282
-#define import_name 283
-#define import_from 284
-#define import_as_name 285
-#define dotted_as_name 286
-#define import_as_names 287
-#define dotted_as_names 288
-#define dotted_name 289
-#define global_stmt 290
-#define nonlocal_stmt 291
-#define assert_stmt 292
-#define compound_stmt 293
-#define if_stmt 294
-#define while_stmt 295
-#define for_stmt 296
-#define try_stmt 297
-#define with_stmt 298
-#define with_item 299
-#define except_clause 300
-#define suite 301
-#define test 302
-#define test_nocond 303
-#define lambdef 304
-#define lambdef_nocond 305
-#define or_test 306
-#define and_test 307
-#define not_test 308
-#define comparison 309
-#define comp_op 310
-#define star_expr 311
-#define expr 312
-#define xor_expr 313
-#define and_expr 314
-#define shift_expr 315
-#define arith_expr 316
-#define term 317
-#define factor 318
-#define power 319
-#define atom 320
-#define testlist_comp 321
-#define trailer 322
-#define subscriptlist 323
-#define subscript 324
-#define sliceop 325
-#define exprlist 326
-#define testlist 327
-#define dictorsetmaker 328
-#define classdef 329
-#define arglist 330
-#define argument 331
-#define comp_iter 332
-#define comp_for 333
-#define comp_if 334
-#define encoding_decl 335
-#define yield_expr 336
-#define yield_arg 337
+#define async_funcdef 262
+#define funcdef 263
+#define parameters 264
+#define typedargslist 265
+#define tfpdef 266
+#define varargslist 267
+#define vfpdef 268
+#define stmt 269
+#define simple_stmt 270
+#define small_stmt 271
+#define expr_stmt 272
+#define testlist_star_expr 273
+#define augassign 274
+#define del_stmt 275
+#define pass_stmt 276
+#define flow_stmt 277
+#define break_stmt 278
+#define continue_stmt 279
+#define return_stmt 280
+#define yield_stmt 281
+#define raise_stmt 282
+#define import_stmt 283
+#define import_name 284
+#define import_from 285
+#define import_as_name 286
+#define dotted_as_name 287
+#define import_as_names 288
+#define dotted_as_names 289
+#define dotted_name 290
+#define global_stmt 291
+#define nonlocal_stmt 292
+#define assert_stmt 293
+#define compound_stmt 294
+#define async_stmt 295
+#define if_stmt 296
+#define while_stmt 297
+#define for_stmt 298
+#define try_stmt 299
+#define with_stmt 300
+#define with_item 301
+#define except_clause 302
+#define suite 303
+#define test 304
+#define test_nocond 305
+#define lambdef 306
+#define lambdef_nocond 307
+#define or_test 308
+#define and_test 309
+#define not_test 310
+#define comparison 311
+#define comp_op 312
+#define star_expr 313
+#define expr 314
+#define xor_expr 315
+#define and_expr 316
+#define shift_expr 317
+#define arith_expr 318
+#define term 319
+#define factor 320
+#define power 321
+#define atom_expr 322
+#define atom 323
+#define testlist_comp 324
+#define trailer 325
+#define subscriptlist 326
+#define subscript 327
+#define sliceop 328
+#define exprlist 329
+#define testlist 330
+#define dictorsetmaker 331
+#define classdef 332
+#define arglist 333
+#define argument 334
+#define comp_iter 335
+#define comp_for 336
+#define comp_if 337
+#define encoding_decl 338
+#define yield_expr 339
+#define yield_arg 340
diff --git a/Include/grammar.h b/Include/grammar.h
index ba7d19d..85120b9 100644
--- a/Include/grammar.h
+++ b/Include/grammar.h
@@ -37,7 +37,7 @@ typedef struct {
typedef struct {
int s_narcs;
arc *s_arc; /* Array of arcs */
-
+
/* Optional accelerators */
int s_lower; /* Lowest label index */
int s_upper; /* Highest label index */
diff --git a/Include/listobject.h b/Include/listobject.h
index 74cf46f..daa513f 100644
--- a/Include/listobject.h
+++ b/Include/listobject.h
@@ -72,6 +72,7 @@ PyAPI_FUNC(void) _PyList_DebugMallocStats(FILE *out);
#define PyList_GET_ITEM(op, i) (((PyListObject *)(op))->ob_item[i])
#define PyList_SET_ITEM(op, i, v) (((PyListObject *)(op))->ob_item[i] = (v))
#define PyList_GET_SIZE(op) Py_SIZE(op)
+#define _PyList_ITEMS(op) (((PyListObject *)(op))->ob_item)
#endif
#ifdef __cplusplus
diff --git a/Include/longobject.h b/Include/longobject.h
index ff43309..aed59ce 100644
--- a/Include/longobject.h
+++ b/Include/longobject.h
@@ -198,6 +198,9 @@ PyAPI_FUNC(int) _PyLong_FormatAdvancedWriter(
PyAPI_FUNC(unsigned long) PyOS_strtoul(const char *, char **, int);
PyAPI_FUNC(long) PyOS_strtol(const char *, char **, int);
+/* For use by the gcd function in mathmodule.c */
+PyAPI_FUNC(PyObject *) _PyLong_GCD(PyObject *, PyObject *);
+
#ifdef __cplusplus
}
#endif
diff --git a/Include/memoryobject.h b/Include/memoryobject.h
index 382ca92..ab5ee09 100644
--- a/Include/memoryobject.h
+++ b/Include/memoryobject.h
@@ -45,9 +45,6 @@ typedef struct {
} _PyManagedBufferObject;
-/* deprecated, removed in 3.5 */
-#define _Py_MEMORYVIEW_MAX_FORMAT 3 /* must be >= 3 */
-
/* memoryview state flags */
#define _Py_MEMORYVIEW_RELEASED 0x001 /* access to master buffer blocked */
#define _Py_MEMORYVIEW_C 0x002 /* C-contiguous layout */
@@ -62,7 +59,6 @@ typedef struct {
int flags; /* state flags */
Py_ssize_t exports; /* number of buffer re-exports */
Py_buffer view; /* private copy of the exporter's view */
- char format[_Py_MEMORYVIEW_MAX_FORMAT]; /* deprecated, removed in 3.5 */
PyObject *weakreflist;
Py_ssize_t ob_array[1]; /* shape, strides, suboffsets */
} PyMemoryViewObject;
diff --git a/Include/methodobject.h b/Include/methodobject.h
index 3cc2ea9..e2ad804 100644
--- a/Include/methodobject.h
+++ b/Include/methodobject.h
@@ -47,7 +47,7 @@ struct PyMethodDef {
typedef struct PyMethodDef PyMethodDef;
#define PyCFunction_New(ML, SELF) PyCFunction_NewEx((ML), (SELF), NULL)
-PyAPI_FUNC(PyObject *) PyCFunction_NewEx(PyMethodDef *, PyObject *,
+PyAPI_FUNC(PyObject *) PyCFunction_NewEx(PyMethodDef *, PyObject *,
PyObject *);
/* Flag passed to newmethodobject */
@@ -66,7 +66,7 @@ PyAPI_FUNC(PyObject *) PyCFunction_NewEx(PyMethodDef *, PyObject *,
/* METH_COEXIST allows a method to be entered even though a slot has
already filled the entry. When defined, the flag allows a separate
- method, "__contains__" for example, to coexist with a defined
+ method, "__contains__" for example, to coexist with a defined
slot like sq_contains. */
#define METH_COEXIST 0x0040
@@ -77,6 +77,7 @@ typedef struct {
PyMethodDef *m_ml; /* Description of the C function to call */
PyObject *m_self; /* Passed as 'self' arg to the C func, can be NULL */
PyObject *m_module; /* The __module__ attribute, can be anything */
+ PyObject *m_weakreflist; /* List of weak references */
} PyCFunctionObject;
#endif
diff --git a/Include/modsupport.h b/Include/modsupport.h
index 5de0458..829aaf8 100644
--- a/Include/modsupport.h
+++ b/Include/modsupport.h
@@ -12,13 +12,13 @@ extern "C" {
/* If PY_SSIZE_T_CLEAN is defined, each functions treats #-specifier
to mean Py_ssize_t */
#ifdef PY_SSIZE_T_CLEAN
-#define PyArg_Parse _PyArg_Parse_SizeT
-#define PyArg_ParseTuple _PyArg_ParseTuple_SizeT
-#define PyArg_ParseTupleAndKeywords _PyArg_ParseTupleAndKeywords_SizeT
-#define PyArg_VaParse _PyArg_VaParse_SizeT
-#define PyArg_VaParseTupleAndKeywords _PyArg_VaParseTupleAndKeywords_SizeT
-#define Py_BuildValue _Py_BuildValue_SizeT
-#define Py_VaBuildValue _Py_VaBuildValue_SizeT
+#define PyArg_Parse _PyArg_Parse_SizeT
+#define PyArg_ParseTuple _PyArg_ParseTuple_SizeT
+#define PyArg_ParseTupleAndKeywords _PyArg_ParseTupleAndKeywords_SizeT
+#define PyArg_VaParse _PyArg_VaParse_SizeT
+#define PyArg_VaParseTupleAndKeywords _PyArg_VaParseTupleAndKeywords_SizeT
+#define Py_BuildValue _Py_BuildValue_SizeT
+#define Py_VaBuildValue _Py_VaBuildValue_SizeT
#else
PyAPI_FUNC(PyObject *) _Py_VaBuildValue_SizeT(const char *, va_list);
#endif
@@ -50,6 +50,13 @@ PyAPI_FUNC(int) PyModule_AddStringConstant(PyObject *, const char *, const char
#define PyModule_AddIntMacro(m, c) PyModule_AddIntConstant(m, #c, c)
#define PyModule_AddStringMacro(m, c) PyModule_AddStringConstant(m, #c, c)
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000
+/* New in 3.5 */
+PyAPI_FUNC(int) PyModule_SetDocString(PyObject *, const char *);
+PyAPI_FUNC(int) PyModule_AddFunctions(PyObject *, PyMethodDef *);
+PyAPI_FUNC(int) PyModule_ExecDef(PyObject *module, PyModuleDef *def);
+#endif
+
#define Py_CLEANUP_SUPPORTED 0x20000
#define PYTHON_API_VERSION 1013
@@ -67,35 +74,35 @@ PyAPI_FUNC(int) PyModule_AddStringConstant(PyObject *, const char *, const char
Please add a line or two to the top of this log for each API
version change:
- 22-Feb-2006 MvL 1013 PEP 353 - long indices for sequence lengths
+ 22-Feb-2006 MvL 1013 PEP 353 - long indices for sequence lengths
- 19-Aug-2002 GvR 1012 Changes to string object struct for
- interning changes, saving 3 bytes.
+ 19-Aug-2002 GvR 1012 Changes to string object struct for
+ interning changes, saving 3 bytes.
- 17-Jul-2001 GvR 1011 Descr-branch, just to be on the safe side
+ 17-Jul-2001 GvR 1011 Descr-branch, just to be on the safe side
25-Jan-2001 FLD 1010 Parameters added to PyCode_New() and
PyFrame_New(); Python 2.1a2
14-Mar-2000 GvR 1009 Unicode API added
- 3-Jan-1999 GvR 1007 Decided to change back! (Don't reuse 1008!)
+ 3-Jan-1999 GvR 1007 Decided to change back! (Don't reuse 1008!)
- 3-Dec-1998 GvR 1008 Python 1.5.2b1
+ 3-Dec-1998 GvR 1008 Python 1.5.2b1
- 18-Jan-1997 GvR 1007 string interning and other speedups
+ 18-Jan-1997 GvR 1007 string interning and other speedups
- 11-Oct-1996 GvR renamed Py_Ellipses to Py_Ellipsis :-(
+ 11-Oct-1996 GvR renamed Py_Ellipses to Py_Ellipsis :-(
- 30-Jul-1996 GvR Slice and ellipses syntax added
+ 30-Jul-1996 GvR Slice and ellipses syntax added
- 23-Jul-1996 GvR For 1.4 -- better safe than sorry this time :-)
+ 23-Jul-1996 GvR For 1.4 -- better safe than sorry this time :-)
- 7-Nov-1995 GvR Keyword arguments (should've been done at 1.3 :-( )
+ 7-Nov-1995 GvR Keyword arguments (should've been done at 1.3 :-( )
- 10-Jan-1995 GvR Renamed globals to new naming scheme
+ 10-Jan-1995 GvR Renamed globals to new naming scheme
- 9-Jan-1995 GvR Initial version (incompatible with older API)
+ 9-Jan-1995 GvR Initial version (incompatible with older API)
*/
/* The PYTHON_ABI_VERSION is introduced in PEP 384. For the lifetime of
@@ -105,10 +112,11 @@ PyAPI_FUNC(int) PyModule_AddStringConstant(PyObject *, const char *, const char
#define PYTHON_ABI_STRING "3"
#ifdef Py_TRACE_REFS
- /* When we are tracing reference counts, rename PyModule_Create2 so
+ /* When we are tracing reference counts, rename module creation functions so
modules compiled with incompatible settings will generate a
link-time error. */
#define PyModule_Create2 PyModule_Create2TraceRefs
+ #define PyModule_FromDefAndSpec2 PyModule_FromDefAndSpec2TraceRefs
#endif
PyAPI_FUNC(PyObject *) PyModule_Create2(struct PyModuleDef*,
@@ -116,12 +124,27 @@ PyAPI_FUNC(PyObject *) PyModule_Create2(struct PyModuleDef*,
#ifdef Py_LIMITED_API
#define PyModule_Create(module) \
- PyModule_Create2(module, PYTHON_ABI_VERSION)
+ PyModule_Create2(module, PYTHON_ABI_VERSION)
#else
#define PyModule_Create(module) \
- PyModule_Create2(module, PYTHON_API_VERSION)
+ PyModule_Create2(module, PYTHON_API_VERSION)
#endif
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000
+/* New in 3.5 */
+PyAPI_FUNC(PyObject *) PyModule_FromDefAndSpec2(PyModuleDef *def,
+ PyObject *spec,
+ int module_api_version);
+
+#ifdef Py_LIMITED_API
+#define PyModule_FromDefAndSpec(module, spec) \
+ PyModule_FromDefAndSpec2(module, spec, PYTHON_ABI_VERSION)
+#else
+#define PyModule_FromDefAndSpec(module, spec) \
+ PyModule_FromDefAndSpec2(module, spec, PYTHON_API_VERSION)
+#endif /* Py_LIMITED_API */
+#endif /* New in 3.5 */
+
#ifndef Py_LIMITED_API
PyAPI_DATA(char *) _Py_PackageContext;
#endif
diff --git a/Include/moduleobject.h b/Include/moduleobject.h
index f119364..229d7fa 100644
--- a/Include/moduleobject.h
+++ b/Include/moduleobject.h
@@ -30,6 +30,12 @@ PyAPI_FUNC(void) _PyModule_ClearDict(PyObject *);
PyAPI_FUNC(struct PyModuleDef*) PyModule_GetDef(PyObject*);
PyAPI_FUNC(void*) PyModule_GetState(PyObject*);
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000
+/* New in 3.5 */
+PyAPI_FUNC(PyObject *) PyModuleDef_Init(struct PyModuleDef*);
+PyAPI_DATA(PyTypeObject) PyModuleDef_Type;
+#endif
+
typedef struct PyModuleDef_Base {
PyObject_HEAD
PyObject* (*m_init)(void);
@@ -44,19 +50,35 @@ typedef struct PyModuleDef_Base {
NULL, /* m_copy */ \
}
+struct PyModuleDef_Slot;
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000
+/* New in 3.5 */
+typedef struct PyModuleDef_Slot{
+ int slot;
+ void *value;
+} PyModuleDef_Slot;
+
+#define Py_mod_create 1
+#define Py_mod_exec 2
+
+#ifndef Py_LIMITED_API
+#define _Py_mod_LAST_SLOT 2
+#endif
+
+#endif /* New in 3.5 */
+
typedef struct PyModuleDef{
PyModuleDef_Base m_base;
const char* m_name;
const char* m_doc;
Py_ssize_t m_size;
PyMethodDef *m_methods;
- inquiry m_reload;
+ struct PyModuleDef_Slot* m_slots;
traverseproc m_traverse;
inquiry m_clear;
freefunc m_free;
}PyModuleDef;
-
#ifdef __cplusplus
}
#endif
diff --git a/Include/node.h b/Include/node.h
index 2e4e2ba..654ad85 100644
--- a/Include/node.h
+++ b/Include/node.h
@@ -26,7 +26,7 @@ PyAPI_FUNC(Py_ssize_t) _PyNode_SizeOf(node *n);
/* Node access functions */
#define NCH(n) ((n)->n_nchildren)
-
+
#define CHILD(n, i) (&(n)->n_child[i])
#define RCHILD(n, i) (CHILD(n, NCH(n) + i))
#define TYPE(n) ((n)->n_type)
diff --git a/Include/object.h b/Include/object.h
index 5f862ab..4d286ef 100644
--- a/Include/object.h
+++ b/Include/object.h
@@ -65,6 +65,7 @@ whose size is determined when the object is allocated.
#error Py_LIMITED_API is incompatible with Py_DEBUG, Py_TRACE_REFS, and Py_REF_DEBUG
#endif
+
#ifdef Py_TRACE_REFS
/* Define pointers to support a doubly-linked list of all live heap objects. */
#define _PyObject_HEAD_EXTRA \
@@ -275,6 +276,9 @@ typedef struct {
binaryfunc nb_inplace_true_divide;
unaryfunc nb_index;
+
+ binaryfunc nb_matrix_multiply;
+ binaryfunc nb_inplace_matrix_multiply;
} PyNumberMethods;
typedef struct {
@@ -297,6 +301,11 @@ typedef struct {
objobjargproc mp_ass_subscript;
} PyMappingMethods;
+typedef struct {
+ unaryfunc am_await;
+ unaryfunc am_aiter;
+ unaryfunc am_anext;
+} PyAsyncMethods;
typedef struct {
getbufferproc bf_getbuffer;
@@ -342,7 +351,8 @@ typedef struct _typeobject {
printfunc tp_print;
getattrfunc tp_getattr;
setattrfunc tp_setattr;
- void *tp_reserved; /* formerly known as tp_compare */
+ PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2)
+ or tp_reserved (Python 3) */
reprfunc tp_repr;
/* Method suites for standard classes */
@@ -449,6 +459,7 @@ typedef struct _heaptypeobject {
/* Note: there's a dependency on the order of these members
in slotptr() in typeobject.c . */
PyTypeObject ht_type;
+ PyAsyncMethods as_async;
PyNumberMethods as_number;
PyMappingMethods as_mapping;
PySequenceMethods as_sequence; /* as_sequence comes after as_mapping,
@@ -572,13 +583,6 @@ PyAPI_FUNC(PyObject *) PyObject_Dir(PyObject *);
PyAPI_FUNC(int) Py_ReprEnter(PyObject *);
PyAPI_FUNC(void) Py_ReprLeave(PyObject *);
-#ifndef Py_LIMITED_API
-/* Helper for passing objects to printf and the like.
- Leaks refcounts. Don't use it!
-*/
-#define PyObject_REPR(obj) PyUnicode_AsUTF8(PyObject_Repr(obj))
-#endif
-
/* Flag bits for printing: */
#define Py_PRINT_RAW 1 /* No string quotes etc. */
@@ -714,11 +718,17 @@ PyAPI_FUNC(Py_ssize_t) _Py_GetRefTotal(void);
_Py_NegativeRefcount(__FILE__, __LINE__, \
(PyObject *)(OP)); \
}
+/* Py_REF_DEBUG also controls the display of refcounts and memory block
+ * allocations at the interactive prompt and at interpreter shutdown
+ */
+PyAPI_FUNC(void) _PyDebug_PrintTotalRefs(void);
+#define _PY_DEBUG_PRINT_TOTAL_REFS() _PyDebug_PrintTotalRefs()
#else
#define _Py_INC_REFTOTAL
#define _Py_DEC_REFTOTAL
#define _Py_REF_DEBUG_COMMA
#define _Py_CHECK_REFCNT(OP) /* a semicolon */;
+#define _PY_DEBUG_PRINT_TOTAL_REFS()
#endif /* Py_REF_DEBUG */
#ifdef COUNT_ALLOCS
diff --git a/Include/objimpl.h b/Include/objimpl.h
index 3f21b70..65b6d91 100644
--- a/Include/objimpl.h
+++ b/Include/objimpl.h
@@ -95,6 +95,7 @@ PyObject_{New, NewVar, Del}.
the raw memory.
*/
PyAPI_FUNC(void *) PyObject_Malloc(size_t size);
+PyAPI_FUNC(void *) PyObject_Calloc(size_t nelem, size_t elsize);
PyAPI_FUNC(void *) PyObject_Realloc(void *ptr, size_t new_size);
PyAPI_FUNC(void) PyObject_Free(void *ptr);
@@ -321,7 +322,8 @@ extern PyGC_Head *_PyGC_generation0;
(!PyTuple_CheckExact(obj) || _PyObject_GC_IS_TRACKED(obj)))
#endif /* Py_LIMITED_API */
-PyAPI_FUNC(PyObject *) _PyObject_GC_Malloc(size_t);
+PyAPI_FUNC(PyObject *) _PyObject_GC_Malloc(size_t size);
+PyAPI_FUNC(PyObject *) _PyObject_GC_Calloc(size_t size);
PyAPI_FUNC(PyObject *) _PyObject_GC_New(PyTypeObject *);
PyAPI_FUNC(PyVarObject *) _PyObject_GC_NewVar(PyTypeObject *, Py_ssize_t);
PyAPI_FUNC(void) PyObject_GC_Track(void *);
diff --git a/Include/odictobject.h b/Include/odictobject.h
new file mode 100644
index 0000000..c1d9592
--- /dev/null
+++ b/Include/odictobject.h
@@ -0,0 +1,43 @@
+#ifndef Py_ODICTOBJECT_H
+#define Py_ODICTOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/* OrderedDict */
+
+#ifndef Py_LIMITED_API
+
+typedef struct _odictobject PyODictObject;
+
+PyAPI_DATA(PyTypeObject) PyODict_Type;
+PyAPI_DATA(PyTypeObject) PyODictIter_Type;
+PyAPI_DATA(PyTypeObject) PyODictKeys_Type;
+PyAPI_DATA(PyTypeObject) PyODictItems_Type;
+PyAPI_DATA(PyTypeObject) PyODictValues_Type;
+
+#endif /* Py_LIMITED_API */
+
+#define PyODict_Check(op) PyObject_TypeCheck(op, &PyODict_Type)
+#define PyODict_CheckExact(op) (Py_TYPE(op) == &PyODict_Type)
+#define PyODict_SIZE(op) ((PyDictObject *)op)->ma_used
+#define PyODict_HasKey(od, key) (PyMapping_HasKey(PyObject *)od, key)
+
+PyAPI_FUNC(PyObject *) PyODict_New(void);
+PyAPI_FUNC(int) PyODict_SetItem(PyObject *od, PyObject *key, PyObject *item);
+PyAPI_FUNC(int) PyODict_DelItem(PyObject *od, PyObject *key);
+
+/* wrappers around PyDict* functions */
+#define PyODict_GetItem(od, key) PyDict_GetItem((PyObject *)od, key)
+#define PyODict_GetItemWithError(od, key) \
+ PyDict_GetItemWithError((PyObject *)od, key)
+#define PyODict_Contains(od, key) PyDict_Contains((PyObject *)od, key)
+#define PyODict_Size(od) PyDict_Size((PyObject *)od)
+#define PyODict_GetItemString(od, key) \
+ PyDict_GetItemString((PyObject *)od, key)
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_ODICTOBJECT_H */
diff --git a/Include/opcode.h b/Include/opcode.h
index 0936f2d..3f917fb 100644
--- a/Include/opcode.h
+++ b/Include/opcode.h
@@ -1,3 +1,4 @@
+/* Auto-generated by Tools/scripts/generate_opcode_h.py */
#ifndef Py_OPCODE_H
#define Py_OPCODE_H
#ifdef __cplusplus
@@ -5,141 +6,122 @@ extern "C" {
#endif
-/* Instruction opcodes for compiled code */
-
-#define POP_TOP 1
-#define ROT_TWO 2
-#define ROT_THREE 3
-#define DUP_TOP 4
-#define DUP_TOP_TWO 5
-#define NOP 9
-
-#define UNARY_POSITIVE 10
-#define UNARY_NEGATIVE 11
-#define UNARY_NOT 12
-
-#define UNARY_INVERT 15
-
-#define BINARY_POWER 19
-
-#define BINARY_MULTIPLY 20
-
-#define BINARY_MODULO 22
-#define BINARY_ADD 23
-#define BINARY_SUBTRACT 24
-#define BINARY_SUBSCR 25
-#define BINARY_FLOOR_DIVIDE 26
-#define BINARY_TRUE_DIVIDE 27
-#define INPLACE_FLOOR_DIVIDE 28
-#define INPLACE_TRUE_DIVIDE 29
-
-#define STORE_MAP 54
-#define INPLACE_ADD 55
-#define INPLACE_SUBTRACT 56
-#define INPLACE_MULTIPLY 57
-
-#define INPLACE_MODULO 59
-#define STORE_SUBSCR 60
-#define DELETE_SUBSCR 61
-
-#define BINARY_LSHIFT 62
-#define BINARY_RSHIFT 63
-#define BINARY_AND 64
-#define BINARY_XOR 65
-#define BINARY_OR 66
-#define INPLACE_POWER 67
-#define GET_ITER 68
-#define PRINT_EXPR 70
-#define LOAD_BUILD_CLASS 71
-#define YIELD_FROM 72
-
-#define INPLACE_LSHIFT 75
-#define INPLACE_RSHIFT 76
-#define INPLACE_AND 77
-#define INPLACE_XOR 78
-#define INPLACE_OR 79
-#define BREAK_LOOP 80
-#define WITH_CLEANUP 81
-
-#define RETURN_VALUE 83
-#define IMPORT_STAR 84
-
-#define YIELD_VALUE 86
-#define POP_BLOCK 87
-#define END_FINALLY 88
-#define POP_EXCEPT 89
-
-#define HAVE_ARGUMENT 90 /* Opcodes from here have an argument: */
-
-#define STORE_NAME 90 /* Index in name list */
-#define DELETE_NAME 91 /* "" */
-#define UNPACK_SEQUENCE 92 /* Number of sequence items */
-#define FOR_ITER 93
-#define UNPACK_EX 94 /* Num items before variable part +
- (Num items after variable part << 8) */
-
-#define STORE_ATTR 95 /* Index in name list */
-#define DELETE_ATTR 96 /* "" */
-#define STORE_GLOBAL 97 /* "" */
-#define DELETE_GLOBAL 98 /* "" */
-
-#define LOAD_CONST 100 /* Index in const list */
-#define LOAD_NAME 101 /* Index in name list */
-#define BUILD_TUPLE 102 /* Number of tuple items */
-#define BUILD_LIST 103 /* Number of list items */
-#define BUILD_SET 104 /* Number of set items */
-#define BUILD_MAP 105 /* Always zero for now */
-#define LOAD_ATTR 106 /* Index in name list */
-#define COMPARE_OP 107 /* Comparison operator */
-#define IMPORT_NAME 108 /* Index in name list */
-#define IMPORT_FROM 109 /* Index in name list */
-
-#define JUMP_FORWARD 110 /* Number of bytes to skip */
-#define JUMP_IF_FALSE_OR_POP 111 /* Target byte offset from beginning of code */
-#define JUMP_IF_TRUE_OR_POP 112 /* "" */
-#define JUMP_ABSOLUTE 113 /* "" */
-#define POP_JUMP_IF_FALSE 114 /* "" */
-#define POP_JUMP_IF_TRUE 115 /* "" */
-
-#define LOAD_GLOBAL 116 /* Index in name list */
-
-#define CONTINUE_LOOP 119 /* Start of loop (absolute) */
-#define SETUP_LOOP 120 /* Target address (relative) */
-#define SETUP_EXCEPT 121 /* "" */
-#define SETUP_FINALLY 122 /* "" */
-
-#define LOAD_FAST 124 /* Local variable number */
-#define STORE_FAST 125 /* Local variable number */
-#define DELETE_FAST 126 /* Local variable number */
-
-#define RAISE_VARARGS 130 /* Number of raise arguments (1, 2 or 3) */
-/* CALL_FUNCTION_XXX opcodes defined below depend on this definition */
-#define CALL_FUNCTION 131 /* #args + (#kwargs<<8) */
-#define MAKE_FUNCTION 132 /* #defaults + #kwdefaults<<8 + #annotations<<16 */
-#define BUILD_SLICE 133 /* Number of items */
-
-#define MAKE_CLOSURE 134 /* same as MAKE_FUNCTION */
-#define LOAD_CLOSURE 135 /* Load free variable from closure */
-#define LOAD_DEREF 136 /* Load and dereference from closure cell */
-#define STORE_DEREF 137 /* Store into cell */
-#define DELETE_DEREF 138 /* Delete closure cell */
-
-/* The next 3 opcodes must be contiguous and satisfy
- (CALL_FUNCTION_VAR - CALL_FUNCTION) & 3 == 1 */
-#define CALL_FUNCTION_VAR 140 /* #args + (#kwargs<<8) */
-#define CALL_FUNCTION_KW 141 /* #args + (#kwargs<<8) */
-#define CALL_FUNCTION_VAR_KW 142 /* #args + (#kwargs<<8) */
-
-#define SETUP_WITH 143
-
-/* Support for opargs more than 16 bits long */
-#define EXTENDED_ARG 144
-
-#define LIST_APPEND 145
-#define SET_ADD 146
-#define MAP_ADD 147
-
-#define LOAD_CLASSDEREF 148
+ /* Instruction opcodes for compiled code */
+#define POP_TOP 1
+#define ROT_TWO 2
+#define ROT_THREE 3
+#define DUP_TOP 4
+#define DUP_TOP_TWO 5
+#define NOP 9
+#define UNARY_POSITIVE 10
+#define UNARY_NEGATIVE 11
+#define UNARY_NOT 12
+#define UNARY_INVERT 15
+#define BINARY_MATRIX_MULTIPLY 16
+#define INPLACE_MATRIX_MULTIPLY 17
+#define BINARY_POWER 19
+#define BINARY_MULTIPLY 20
+#define BINARY_MODULO 22
+#define BINARY_ADD 23
+#define BINARY_SUBTRACT 24
+#define BINARY_SUBSCR 25
+#define BINARY_FLOOR_DIVIDE 26
+#define BINARY_TRUE_DIVIDE 27
+#define INPLACE_FLOOR_DIVIDE 28
+#define INPLACE_TRUE_DIVIDE 29
+#define GET_AITER 50
+#define GET_ANEXT 51
+#define BEFORE_ASYNC_WITH 52
+#define INPLACE_ADD 55
+#define INPLACE_SUBTRACT 56
+#define INPLACE_MULTIPLY 57
+#define INPLACE_MODULO 59
+#define STORE_SUBSCR 60
+#define DELETE_SUBSCR 61
+#define BINARY_LSHIFT 62
+#define BINARY_RSHIFT 63
+#define BINARY_AND 64
+#define BINARY_XOR 65
+#define BINARY_OR 66
+#define INPLACE_POWER 67
+#define GET_ITER 68
+#define GET_YIELD_FROM_ITER 69
+#define PRINT_EXPR 70
+#define LOAD_BUILD_CLASS 71
+#define YIELD_FROM 72
+#define GET_AWAITABLE 73
+#define INPLACE_LSHIFT 75
+#define INPLACE_RSHIFT 76
+#define INPLACE_AND 77
+#define INPLACE_XOR 78
+#define INPLACE_OR 79
+#define BREAK_LOOP 80
+#define WITH_CLEANUP_START 81
+#define WITH_CLEANUP_FINISH 82
+#define RETURN_VALUE 83
+#define IMPORT_STAR 84
+#define YIELD_VALUE 86
+#define POP_BLOCK 87
+#define END_FINALLY 88
+#define POP_EXCEPT 89
+#define HAVE_ARGUMENT 90
+#define STORE_NAME 90
+#define DELETE_NAME 91
+#define UNPACK_SEQUENCE 92
+#define FOR_ITER 93
+#define UNPACK_EX 94
+#define STORE_ATTR 95
+#define DELETE_ATTR 96
+#define STORE_GLOBAL 97
+#define DELETE_GLOBAL 98
+#define LOAD_CONST 100
+#define LOAD_NAME 101
+#define BUILD_TUPLE 102
+#define BUILD_LIST 103
+#define BUILD_SET 104
+#define BUILD_MAP 105
+#define LOAD_ATTR 106
+#define COMPARE_OP 107
+#define IMPORT_NAME 108
+#define IMPORT_FROM 109
+#define JUMP_FORWARD 110
+#define JUMP_IF_FALSE_OR_POP 111
+#define JUMP_IF_TRUE_OR_POP 112
+#define JUMP_ABSOLUTE 113
+#define POP_JUMP_IF_FALSE 114
+#define POP_JUMP_IF_TRUE 115
+#define LOAD_GLOBAL 116
+#define CONTINUE_LOOP 119
+#define SETUP_LOOP 120
+#define SETUP_EXCEPT 121
+#define SETUP_FINALLY 122
+#define LOAD_FAST 124
+#define STORE_FAST 125
+#define DELETE_FAST 126
+#define RAISE_VARARGS 130
+#define CALL_FUNCTION 131
+#define MAKE_FUNCTION 132
+#define BUILD_SLICE 133
+#define MAKE_CLOSURE 134
+#define LOAD_CLOSURE 135
+#define LOAD_DEREF 136
+#define STORE_DEREF 137
+#define DELETE_DEREF 138
+#define CALL_FUNCTION_VAR 140
+#define CALL_FUNCTION_KW 141
+#define CALL_FUNCTION_VAR_KW 142
+#define SETUP_WITH 143
+#define EXTENDED_ARG 144
+#define LIST_APPEND 145
+#define SET_ADD 146
+#define MAP_ADD 147
+#define LOAD_CLASSDEREF 148
+#define BUILD_LIST_UNPACK 149
+#define BUILD_MAP_UNPACK 150
+#define BUILD_MAP_UNPACK_WITH_CALL 151
+#define BUILD_TUPLE_UNPACK 152
+#define BUILD_SET_UNPACK 153
+#define SETUP_ASYNC_WITH 154
/* EXCEPT_HANDLER is a special, implicit block type which is created when
entering an except handler. It is not an opcode but we define it here
@@ -148,8 +130,9 @@ extern "C" {
#define EXCEPT_HANDLER 257
-enum cmp_op {PyCmp_LT=Py_LT, PyCmp_LE=Py_LE, PyCmp_EQ=Py_EQ, PyCmp_NE=Py_NE, PyCmp_GT=Py_GT, PyCmp_GE=Py_GE,
- PyCmp_IN, PyCmp_NOT_IN, PyCmp_IS, PyCmp_IS_NOT, PyCmp_EXC_MATCH, PyCmp_BAD};
+enum cmp_op {PyCmp_LT=Py_LT, PyCmp_LE=Py_LE, PyCmp_EQ=Py_EQ, PyCmp_NE=Py_NE,
+ PyCmp_GT=Py_GT, PyCmp_GE=Py_GE, PyCmp_IN, PyCmp_NOT_IN,
+ PyCmp_IS, PyCmp_IS_NOT, PyCmp_EXC_MATCH, PyCmp_BAD};
#define HAS_ARG(op) ((op) >= HAVE_ARGUMENT)
diff --git a/Include/osdefs.h b/Include/osdefs.h
index 0c2e34b..bd84c1c 100644
--- a/Include/osdefs.h
+++ b/Include/osdefs.h
@@ -7,15 +7,12 @@ extern "C" {
/* Operating system dependencies */
-/* Mod by chrish: QNX has WATCOM, but isn't DOS */
-#if !defined(__QNX__)
-#if defined(MS_WINDOWS) || defined(__BORLANDC__) || defined(__WATCOMC__) || defined(__DJGPP__)
+#ifdef MS_WINDOWS
#define SEP L'\\'
#define ALTSEP L'/'
#define MAXPATHLEN 256
#define DELIM L';'
#endif
-#endif
/* Filename separator */
#ifndef SEP
diff --git a/Include/patchlevel.h b/Include/patchlevel.h
index f884cd1..9e81990 100644
--- a/Include/patchlevel.h
+++ b/Include/patchlevel.h
@@ -17,13 +17,13 @@
/* Version parsed out into numeric values */
/*--start constants--*/
#define PY_MAJOR_VERSION 3
-#define PY_MINOR_VERSION 4
-#define PY_MICRO_VERSION 3
-#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL
-#define PY_RELEASE_SERIAL 0
+#define PY_MINOR_VERSION 5
+#define PY_MICRO_VERSION 0
+#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_GAMMA
+#define PY_RELEASE_SERIAL 2
/* Version as a string */
-#define PY_VERSION "3.4.3+"
+#define PY_VERSION "3.5.0rc2+"
/*--end constants--*/
/* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2.
diff --git a/Include/pyatomic.h b/Include/pyatomic.h
index d4e19e0..99816a5 100644
--- a/Include/pyatomic.h
+++ b/Include/pyatomic.h
@@ -1,14 +1,15 @@
-#ifndef Py_LIMITED_API
+/* Issue #23644: <stdatomic.h> is incompatible with C++, see:
+ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=60932 */
+#if !defined(Py_LIMITED_API) && !defined(__cplusplus)
#ifndef Py_ATOMIC_H
#define Py_ATOMIC_H
-/* XXX: When compilers start offering a stdatomic.h with lock-free
- atomic_int and atomic_address types, include that here and rewrite
- the atomic operations in terms of it. */
#include "dynamic_annotations.h"
-#ifdef __cplusplus
-extern "C" {
+#include "pyconfig.h"
+
+#if defined(HAVE_STD_ATOMIC)
+#include <stdatomic.h>
#endif
/* This is modeled after the atomics interface from C1x, according to
@@ -20,6 +21,76 @@ extern "C" {
* Beware, the implementations here are deep magic.
*/
+#if defined(HAVE_STD_ATOMIC)
+
+typedef enum _Py_memory_order {
+ _Py_memory_order_relaxed = memory_order_relaxed,
+ _Py_memory_order_acquire = memory_order_acquire,
+ _Py_memory_order_release = memory_order_release,
+ _Py_memory_order_acq_rel = memory_order_acq_rel,
+ _Py_memory_order_seq_cst = memory_order_seq_cst
+} _Py_memory_order;
+
+typedef struct _Py_atomic_address {
+ _Atomic void *_value;
+} _Py_atomic_address;
+
+typedef struct _Py_atomic_int {
+ atomic_int _value;
+} _Py_atomic_int;
+
+#define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \
+ atomic_signal_fence(ORDER)
+
+#define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \
+ atomic_thread_fence(ORDER)
+
+#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
+ atomic_store_explicit(&(ATOMIC_VAL)->_value, NEW_VAL, ORDER)
+
+#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
+ atomic_load_explicit(&(ATOMIC_VAL)->_value, ORDER)
+
+/* Use builtin atomic operations in GCC >= 4.7 */
+#elif defined(HAVE_BUILTIN_ATOMIC)
+
+typedef enum _Py_memory_order {
+ _Py_memory_order_relaxed = __ATOMIC_RELAXED,
+ _Py_memory_order_acquire = __ATOMIC_ACQUIRE,
+ _Py_memory_order_release = __ATOMIC_RELEASE,
+ _Py_memory_order_acq_rel = __ATOMIC_ACQ_REL,
+ _Py_memory_order_seq_cst = __ATOMIC_SEQ_CST
+} _Py_memory_order;
+
+typedef struct _Py_atomic_address {
+ void *_value;
+} _Py_atomic_address;
+
+typedef struct _Py_atomic_int {
+ int _value;
+} _Py_atomic_int;
+
+#define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \
+ __atomic_signal_fence(ORDER)
+
+#define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \
+ __atomic_thread_fence(ORDER)
+
+#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
+ (assert((ORDER) == __ATOMIC_RELAXED \
+ || (ORDER) == __ATOMIC_SEQ_CST \
+ || (ORDER) == __ATOMIC_RELEASE), \
+ __atomic_store_n(&(ATOMIC_VAL)->_value, NEW_VAL, ORDER))
+
+#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
+ (assert((ORDER) == __ATOMIC_RELAXED \
+ || (ORDER) == __ATOMIC_SEQ_CST \
+ || (ORDER) == __ATOMIC_ACQUIRE \
+ || (ORDER) == __ATOMIC_CONSUME), \
+ __atomic_load_n(&(ATOMIC_VAL)->_value, ORDER))
+
+#else
+
typedef enum _Py_memory_order {
_Py_memory_order_relaxed,
_Py_memory_order_acquire,
@@ -162,6 +233,7 @@ _Py_ANNOTATE_MEMORY_ORDER(const volatile void *address, _Py_memory_order order)
((ATOMIC_VAL)->_value)
#endif /* !gcc x86 */
+#endif
/* Standardized shortcuts. */
#define _Py_atomic_store(ATOMIC_VAL, NEW_VAL) \
@@ -176,9 +248,5 @@ _Py_ANNOTATE_MEMORY_ORDER(const volatile void *address, _Py_memory_order order)
#define _Py_atomic_load_relaxed(ATOMIC_VAL) \
_Py_atomic_load_explicit(ATOMIC_VAL, _Py_memory_order_relaxed)
-#ifdef __cplusplus
-}
-#endif
-
#endif /* Py_ATOMIC_H */
#endif /* Py_LIMITED_API */
diff --git a/Include/pydebug.h b/Include/pydebug.h
index 8fe9818..19bec2b 100644
--- a/Include/pydebug.h
+++ b/Include/pydebug.h
@@ -5,6 +5,8 @@
extern "C" {
#endif
+/* These global variable are defined in pylifecycle.c */
+/* XXX (ncoghlan): move these declarations to pylifecycle.h? */
PyAPI_DATA(int) Py_DebugFlag;
PyAPI_DATA(int) Py_VerboseFlag;
PyAPI_DATA(int) Py_QuietFlag;
diff --git a/Include/pyerrors.h b/Include/pyerrors.h
index 02f65d6..35aedb7 100644
--- a/Include/pyerrors.h
+++ b/Include/pyerrors.h
@@ -99,6 +99,7 @@ PyAPI_FUNC(void) PyErr_SetExcInfo(PyObject *, PyObject *, PyObject *);
#define _Py_NO_RETURN
#endif
+/* Defined in Python/pylifecycle.c */
PyAPI_FUNC(void) Py_FatalError(const char *message) _Py_NO_RETURN;
#if defined(Py_DEBUG) || defined(Py_LIMITED_API)
@@ -146,6 +147,7 @@ PyAPI_FUNC(void) _PyErr_ChainExceptions(PyObject *, PyObject *, PyObject *);
PyAPI_DATA(PyObject *) PyExc_BaseException;
PyAPI_DATA(PyObject *) PyExc_Exception;
+PyAPI_DATA(PyObject *) PyExc_StopAsyncIteration;
PyAPI_DATA(PyObject *) PyExc_StopIteration;
PyAPI_DATA(PyObject *) PyExc_GeneratorExit;
PyAPI_DATA(PyObject *) PyExc_ArithmeticError;
@@ -165,6 +167,7 @@ PyAPI_DATA(PyObject *) PyExc_MemoryError;
PyAPI_DATA(PyObject *) PyExc_NameError;
PyAPI_DATA(PyObject *) PyExc_OverflowError;
PyAPI_DATA(PyObject *) PyExc_RuntimeError;
+PyAPI_DATA(PyObject *) PyExc_RecursionError;
PyAPI_DATA(PyObject *) PyExc_NotImplementedError;
PyAPI_DATA(PyObject *) PyExc_SyntaxError;
PyAPI_DATA(PyObject *) PyExc_IndentationError;
@@ -244,6 +247,12 @@ PyAPI_FUNC(PyObject *) PyErr_Format(
const char *format, /* ASCII-encoded string */
...
);
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000
+PyAPI_FUNC(PyObject *) PyErr_FormatV(
+ PyObject *exception,
+ const char *format,
+ va_list vargs);
+#endif
#ifdef MS_WINDOWS
PyAPI_FUNC(PyObject *) PyErr_SetFromWindowsErrWithFilename(
diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h
new file mode 100644
index 0000000..ccdebe2
--- /dev/null
+++ b/Include/pylifecycle.h
@@ -0,0 +1,124 @@
+
+/* Interfaces to configure, query, create & destroy the Python runtime */
+
+#ifndef Py_PYLIFECYCLE_H
+#define Py_PYLIFECYCLE_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+PyAPI_FUNC(void) Py_SetProgramName(wchar_t *);
+PyAPI_FUNC(wchar_t *) Py_GetProgramName(void);
+
+PyAPI_FUNC(void) Py_SetPythonHome(wchar_t *);
+PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void);
+
+#ifndef Py_LIMITED_API
+/* Only used by applications that embed the interpreter and need to
+ * override the standard encoding determination mechanism
+ */
+PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding,
+ const char *errors);
+#endif
+
+PyAPI_FUNC(void) Py_Initialize(void);
+PyAPI_FUNC(void) Py_InitializeEx(int);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void) _Py_InitializeEx_Private(int, int);
+#endif
+PyAPI_FUNC(void) Py_Finalize(void);
+PyAPI_FUNC(int) Py_IsInitialized(void);
+PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void);
+PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *);
+
+
+/* Py_PyAtExit is for the atexit module, Py_AtExit is for low-level
+ * exit functions.
+ */
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void) _Py_PyAtExit(void (*func)(void));
+#endif
+PyAPI_FUNC(int) Py_AtExit(void (*func)(void));
+
+PyAPI_FUNC(void) Py_Exit(int);
+
+/* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. */
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void) _Py_RestoreSignals(void);
+
+PyAPI_FUNC(int) Py_FdIsInteractive(FILE *, const char *);
+#endif
+
+/* Bootstrap __main__ (defined in Modules/main.c) */
+PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv);
+
+/* In getpath.c */
+PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void);
+PyAPI_FUNC(wchar_t *) Py_GetPrefix(void);
+PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void);
+PyAPI_FUNC(wchar_t *) Py_GetPath(void);
+PyAPI_FUNC(void) Py_SetPath(const wchar_t *);
+#ifdef MS_WINDOWS
+int _Py_CheckPython3();
+#endif
+
+/* In their own files */
+PyAPI_FUNC(const char *) Py_GetVersion(void);
+PyAPI_FUNC(const char *) Py_GetPlatform(void);
+PyAPI_FUNC(const char *) Py_GetCopyright(void);
+PyAPI_FUNC(const char *) Py_GetCompiler(void);
+PyAPI_FUNC(const char *) Py_GetBuildInfo(void);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(const char *) _Py_hgidentifier(void);
+PyAPI_FUNC(const char *) _Py_hgversion(void);
+#endif
+
+/* Internal -- various one-time initializations */
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject *) _PyBuiltin_Init(void);
+PyAPI_FUNC(PyObject *) _PySys_Init(void);
+PyAPI_FUNC(void) _PyImport_Init(void);
+PyAPI_FUNC(void) _PyExc_Init(PyObject * bltinmod);
+PyAPI_FUNC(void) _PyImportHooks_Init(void);
+PyAPI_FUNC(int) _PyFrame_Init(void);
+PyAPI_FUNC(int) _PyFloat_Init(void);
+PyAPI_FUNC(int) PyByteArray_Init(void);
+PyAPI_FUNC(void) _PyRandom_Init(void);
+#endif
+
+/* Various internal finalizers */
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void) _PyExc_Fini(void);
+PyAPI_FUNC(void) _PyImport_Fini(void);
+PyAPI_FUNC(void) PyMethod_Fini(void);
+PyAPI_FUNC(void) PyFrame_Fini(void);
+PyAPI_FUNC(void) PyCFunction_Fini(void);
+PyAPI_FUNC(void) PyDict_Fini(void);
+PyAPI_FUNC(void) PyTuple_Fini(void);
+PyAPI_FUNC(void) PyList_Fini(void);
+PyAPI_FUNC(void) PySet_Fini(void);
+PyAPI_FUNC(void) PyBytes_Fini(void);
+PyAPI_FUNC(void) PyByteArray_Fini(void);
+PyAPI_FUNC(void) PyFloat_Fini(void);
+PyAPI_FUNC(void) PyOS_FiniInterrupts(void);
+PyAPI_FUNC(void) _PyGC_DumpShutdownStats(void);
+PyAPI_FUNC(void) _PyGC_Fini(void);
+PyAPI_FUNC(void) PySlice_Fini(void);
+PyAPI_FUNC(void) _PyType_Fini(void);
+PyAPI_FUNC(void) _PyRandom_Fini(void);
+
+PyAPI_DATA(PyThreadState *) _Py_Finalizing;
+#endif
+
+/* Signals */
+typedef void (*PyOS_sighandler_t)(int);
+PyAPI_FUNC(PyOS_sighandler_t) PyOS_getsig(int);
+PyAPI_FUNC(PyOS_sighandler_t) PyOS_setsig(int, PyOS_sighandler_t);
+
+/* Random */
+PyAPI_FUNC(int) _PyOS_URandom (void *buffer, Py_ssize_t size);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_PYLIFECYCLE_H */
diff --git a/Include/pymacro.h b/Include/pymacro.h
index 7997c55..3f6f5dc 100644
--- a/Include/pymacro.h
+++ b/Include/pymacro.h
@@ -1,13 +1,26 @@
#ifndef Py_PYMACRO_H
#define Py_PYMACRO_H
+/* Minimum value between x and y */
#define Py_MIN(x, y) (((x) > (y)) ? (y) : (x))
+
+/* Maximum value between x and y */
#define Py_MAX(x, y) (((x) > (y)) ? (x) : (y))
+/* Absolute value of the number x */
+#define Py_ABS(x) ((x) < 0 ? -(x) : (x))
+
+#define _Py_XSTRINGIFY(x) #x
+
+/* Convert the argument to a string. For example, Py_STRINGIFY(123) is replaced
+ with "123" by the preprocessor. Defines are also replaced by their value.
+ For example Py_STRINGIFY(__LINE__) is replaced by the line number, not
+ by "__LINE__". */
+#define Py_STRINGIFY(x) _Py_XSTRINGIFY(x)
+
/* Argument must be a char or an int in [-128, 127] or [0, 255]. */
#define Py_CHARMASK(c) ((unsigned char)((c) & 0xff))
-
/* Assert a build-time dependency, as an expression.
Your compile will fail if the condition isn't true, or can't be evaluated
diff --git a/Include/pymem.h b/Include/pymem.h
index 2372b86..043db64 100644
--- a/Include/pymem.h
+++ b/Include/pymem.h
@@ -13,6 +13,7 @@ extern "C" {
#ifndef Py_LIMITED_API
PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size);
+PyAPI_FUNC(void *) PyMem_RawCalloc(size_t nelem, size_t elsize);
PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size);
PyAPI_FUNC(void) PyMem_RawFree(void *ptr);
#endif
@@ -57,6 +58,7 @@ PyAPI_FUNC(void) PyMem_RawFree(void *ptr);
*/
PyAPI_FUNC(void *) PyMem_Malloc(size_t size);
+PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize);
PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size);
PyAPI_FUNC(void) PyMem_Free(void *ptr);
@@ -126,22 +128,25 @@ typedef enum {
} PyMemAllocatorDomain;
typedef struct {
- /* user context passed as the first argument to the 3 functions */
+ /* user context passed as the first argument to the 4 functions */
void *ctx;
/* allocate a memory block */
void* (*malloc) (void *ctx, size_t size);
+ /* allocate a memory block initialized by zeros */
+ void* (*calloc) (void *ctx, size_t nelem, size_t elsize);
+
/* allocate or resize a memory block */
void* (*realloc) (void *ctx, void *ptr, size_t new_size);
/* release a memory block */
void (*free) (void *ctx, void *ptr);
-} PyMemAllocator;
+} PyMemAllocatorEx;
/* Get the memory block allocator of the specified domain. */
PyAPI_FUNC(void) PyMem_GetAllocator(PyMemAllocatorDomain domain,
- PyMemAllocator *allocator);
+ PyMemAllocatorEx *allocator);
/* Set the memory block allocator of the specified domain.
@@ -155,7 +160,7 @@ PyAPI_FUNC(void) PyMem_GetAllocator(PyMemAllocatorDomain domain,
PyMem_SetupDebugHooks() function must be called to reinstall the debug hooks
on top on the new allocator. */
PyAPI_FUNC(void) PyMem_SetAllocator(PyMemAllocatorDomain domain,
- PyMemAllocator *allocator);
+ PyMemAllocatorEx *allocator);
/* Setup hooks to detect bugs in the following Python memory allocator
functions:
diff --git a/Include/pyport.h b/Include/pyport.h
index b29f9bd..66e00d4 100644
--- a/Include/pyport.h
+++ b/Include/pyport.h
@@ -357,28 +357,6 @@ typedef int Py_ssize_clean_t;
* stat() and fstat() fiddling *
*******************************/
-/* We expect that stat and fstat exist on most systems.
- * It's confirmed on Unix, Mac and Windows.
- * If you don't have them, add
- * #define DONT_HAVE_STAT
- * and/or
- * #define DONT_HAVE_FSTAT
- * to your pyconfig.h. Python code beyond this should check HAVE_STAT and
- * HAVE_FSTAT instead.
- * Also
- * #define HAVE_SYS_STAT_H
- * if <sys/stat.h> exists on your platform, and
- * #define HAVE_STAT_H
- * if <stat.h> does.
- */
-#ifndef DONT_HAVE_STAT
-#define HAVE_STAT
-#endif
-
-#ifndef DONT_HAVE_FSTAT
-#define HAVE_FSTAT
-#endif
-
#ifdef HAVE_SYS_STAT_H
#include <sys/stat.h>
#elif defined(HAVE_STAT_H)
@@ -588,6 +566,25 @@ extern "C" {
} while (0)
#endif
+#ifdef HAVE_GCC_ASM_FOR_MC68881
+#define HAVE_PY_SET_53BIT_PRECISION 1
+#define _Py_SET_53BIT_PRECISION_HEADER \
+ unsigned int old_fpcr, new_fpcr
+#define _Py_SET_53BIT_PRECISION_START \
+ do { \
+ __asm__ ("fmove.l %%fpcr,%0" : "=g" (old_fpcr)); \
+ /* Set double precision / round to nearest. */ \
+ new_fpcr = (old_fpcr & ~0xf0) | 0x80; \
+ if (new_fpcr != old_fpcr) \
+ __asm__ volatile ("fmove.l %0,%%fpcr" : : "g" (new_fpcr)); \
+ } while (0)
+#define _Py_SET_53BIT_PRECISION_END \
+ do { \
+ if (new_fpcr != old_fpcr) \
+ __asm__ volatile ("fmove.l %0,%%fpcr" : : "g" (old_fpcr)); \
+ } while (0)
+#endif
+
/* default definitions are empty */
#ifndef HAVE_PY_SET_53BIT_PRECISION
#define _Py_SET_53BIT_PRECISION_HEADER
@@ -880,4 +877,24 @@ extern pid_t forkpty(int *, char *, struct termios *, struct winsize *);
#define PY_LITTLE_ENDIAN 1
#endif
+#ifdef Py_BUILD_CORE
+/*
+ * Macros to protect CRT calls against instant termination when passed an
+ * invalid parameter (issue23524).
+ */
+#if defined _MSC_VER && _MSC_VER >= 1900
+
+extern _invalid_parameter_handler _Py_silent_invalid_parameter_handler;
+#define _Py_BEGIN_SUPPRESS_IPH { _invalid_parameter_handler _Py_old_handler = \
+ _set_thread_local_invalid_parameter_handler(_Py_silent_invalid_parameter_handler);
+#define _Py_END_SUPPRESS_IPH _set_thread_local_invalid_parameter_handler(_Py_old_handler); }
+
+#else
+
+#define _Py_BEGIN_SUPPRESS_IPH
+#define _Py_END_SUPPRESS_IPH
+
+#endif /* _MSC_VER >= 1900 */
+#endif /* Py_BUILD_CORE */
+
#endif /* Py_PYPORT_H */
diff --git a/Include/pystate.h b/Include/pystate.h
index 4992c22..a2fd803 100644
--- a/Include/pystate.h
+++ b/Include/pystate.h
@@ -134,6 +134,9 @@ typedef struct _ts {
void (*on_delete)(void *);
void *on_delete_data;
+ PyObject *coroutine_wrapper;
+ int in_coroutine_wrapper;
+
/* XXX signal handlers should also be here */
} PyThreadState;
@@ -174,12 +177,16 @@ PyAPI_FUNC(int) PyThreadState_SetAsyncExc(long, PyObject *);
/* Variable and macro for in-line access to current thread state */
/* Assuming the current thread holds the GIL, this is the
- PyThreadState for the current thread. */
-#ifndef Py_LIMITED_API
+ PyThreadState for the current thread.
+
+ Issue #23644: pyatomic.h is incompatible with C++ (yet). Disable
+ PyThreadState_GET() optimization: declare it as an alias to
+ PyThreadState_Get(), as done for limited API. */
+#if !defined(Py_LIMITED_API) && !defined(__cplusplus)
PyAPI_DATA(_Py_atomic_address) _PyThreadState_Current;
#endif
-#if defined(Py_DEBUG) || defined(Py_LIMITED_API)
+#if defined(Py_DEBUG) || defined(Py_LIMITED_API) || defined(__cplusplus)
#define PyThreadState_GET() PyThreadState_Get()
#else
#define PyThreadState_GET() \
diff --git a/Include/pystrhex.h b/Include/pystrhex.h
new file mode 100644
index 0000000..1dc1255
--- /dev/null
+++ b/Include/pystrhex.h
@@ -0,0 +1,17 @@
+#ifndef Py_STRHEX_H
+#define Py_STRHEX_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Returns a str() containing the hex representation of argbuf. */
+PyAPI_FUNC(PyObject*) _Py_strhex(const char* argbuf, const Py_ssize_t arglen);
+/* Returns a bytes() containing the ASCII hex representation of argbuf. */
+PyAPI_FUNC(PyObject*) _Py_strhex_bytes(const char* argbuf, const Py_ssize_t arglen);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !Py_STRHEX_H */
diff --git a/Include/pythonrun.h b/Include/pythonrun.h
index 2fc5578..f92148d 100644
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -9,7 +9,8 @@ extern "C" {
#define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | \
CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | \
- CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL)
+ CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \
+ CO_FUTURE_GENERATOR_STOP)
#define PyCF_MASK_OBSOLETE (CO_NESTED)
#define PyCF_SOURCE_IS_UTF8 0x0100
#define PyCF_DONT_IMPLY_DEDENT 0x0200
@@ -22,30 +23,6 @@ typedef struct {
} PyCompilerFlags;
#endif
-PyAPI_FUNC(void) Py_SetProgramName(wchar_t *);
-PyAPI_FUNC(wchar_t *) Py_GetProgramName(void);
-
-PyAPI_FUNC(void) Py_SetPythonHome(wchar_t *);
-PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void);
-
-#ifndef Py_LIMITED_API
-/* Only used by applications that embed the interpreter and need to
- * override the standard encoding determination mechanism
- */
-PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding,
- const char *errors);
-#endif
-
-PyAPI_FUNC(void) Py_Initialize(void);
-PyAPI_FUNC(void) Py_InitializeEx(int);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(void) _Py_InitializeEx_Private(int, int);
-#endif
-PyAPI_FUNC(void) Py_Finalize(void);
-PyAPI_FUNC(int) Py_IsInitialized(void);
-PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void);
-PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *);
-
#ifndef Py_LIMITED_API
PyAPI_FUNC(int) PyRun_SimpleStringFlags(const char *, PyCompilerFlags *);
PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, const char *, PyCompilerFlags *);
@@ -166,26 +143,6 @@ PyAPI_FUNC(void) PyErr_Print(void);
PyAPI_FUNC(void) PyErr_PrintEx(int);
PyAPI_FUNC(void) PyErr_Display(PyObject *, PyObject *, PyObject *);
-/* Py_PyAtExit is for the atexit module, Py_AtExit is for low-level
- * exit functions.
- */
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(void) _Py_PyAtExit(void (*func)(void));
-#endif
-PyAPI_FUNC(int) Py_AtExit(void (*func)(void));
-
-PyAPI_FUNC(void) Py_Exit(int);
-
-/* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. */
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(void) _Py_RestoreSignals(void);
-
-PyAPI_FUNC(int) Py_FdIsInteractive(FILE *, const char *);
-#endif
-
-/* Bootstrap */
-PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv);
-
#ifndef Py_LIMITED_API
/* Use macros for a bunch of old variants */
#define PyRun_String(str, s, g, l) PyRun_StringFlags(str, s, g, l, NULL)
@@ -207,64 +164,6 @@ PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv);
PyRun_FileExFlags(fp, p, s, g, l, 0, flags)
#endif
-/* In getpath.c */
-PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void);
-PyAPI_FUNC(wchar_t *) Py_GetPrefix(void);
-PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void);
-PyAPI_FUNC(wchar_t *) Py_GetPath(void);
-PyAPI_FUNC(void) Py_SetPath(const wchar_t *);
-#ifdef MS_WINDOWS
-int _Py_CheckPython3();
-#endif
-
-/* In their own files */
-PyAPI_FUNC(const char *) Py_GetVersion(void);
-PyAPI_FUNC(const char *) Py_GetPlatform(void);
-PyAPI_FUNC(const char *) Py_GetCopyright(void);
-PyAPI_FUNC(const char *) Py_GetCompiler(void);
-PyAPI_FUNC(const char *) Py_GetBuildInfo(void);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(const char *) _Py_hgidentifier(void);
-PyAPI_FUNC(const char *) _Py_hgversion(void);
-#endif
-
-/* Internal -- various one-time initializations */
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject *) _PyBuiltin_Init(void);
-PyAPI_FUNC(PyObject *) _PySys_Init(void);
-PyAPI_FUNC(void) _PyImport_Init(void);
-PyAPI_FUNC(void) _PyExc_Init(PyObject * bltinmod);
-PyAPI_FUNC(void) _PyImportHooks_Init(void);
-PyAPI_FUNC(int) _PyFrame_Init(void);
-PyAPI_FUNC(int) _PyFloat_Init(void);
-PyAPI_FUNC(int) PyByteArray_Init(void);
-PyAPI_FUNC(void) _PyRandom_Init(void);
-#endif
-
-/* Various internal finalizers */
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(void) _PyExc_Fini(void);
-PyAPI_FUNC(void) _PyImport_Fini(void);
-PyAPI_FUNC(void) PyMethod_Fini(void);
-PyAPI_FUNC(void) PyFrame_Fini(void);
-PyAPI_FUNC(void) PyCFunction_Fini(void);
-PyAPI_FUNC(void) PyDict_Fini(void);
-PyAPI_FUNC(void) PyTuple_Fini(void);
-PyAPI_FUNC(void) PyList_Fini(void);
-PyAPI_FUNC(void) PySet_Fini(void);
-PyAPI_FUNC(void) PyBytes_Fini(void);
-PyAPI_FUNC(void) PyByteArray_Fini(void);
-PyAPI_FUNC(void) PyFloat_Fini(void);
-PyAPI_FUNC(void) PyOS_FiniInterrupts(void);
-PyAPI_FUNC(void) _PyGC_DumpShutdownStats(void);
-PyAPI_FUNC(void) _PyGC_Fini(void);
-PyAPI_FUNC(void) PySlice_Fini(void);
-PyAPI_FUNC(void) _PyType_Fini(void);
-PyAPI_FUNC(void) _PyRandom_Fini(void);
-
-PyAPI_DATA(PyThreadState *) _Py_Finalizing;
-#endif
-
/* Stuff with no proper home (yet) */
#ifndef Py_LIMITED_API
PyAPI_FUNC(char *) PyOS_Readline(FILE *, FILE *, const char *);
@@ -290,14 +189,6 @@ PyAPI_DATA(PyThreadState*) _PyOS_ReadlineTState;
PyAPI_FUNC(int) PyOS_CheckStack(void);
#endif
-/* Signals */
-typedef void (*PyOS_sighandler_t)(int);
-PyAPI_FUNC(PyOS_sighandler_t) PyOS_getsig(int);
-PyAPI_FUNC(PyOS_sighandler_t) PyOS_setsig(int, PyOS_sighandler_t);
-
-/* Random */
-PyAPI_FUNC(int) _PyOS_URandom (void *buffer, Py_ssize_t size);
-
#ifdef __cplusplus
}
#endif
diff --git a/Include/pytime.h b/Include/pytime.h
index b0fc6d0..027c3d8 100644
--- a/Include/pytime.h
+++ b/Include/pytime.h
@@ -13,60 +13,26 @@ functions and constants
extern "C" {
#endif
-#ifdef HAVE_GETTIMEOFDAY
-typedef struct timeval _PyTime_timeval;
+#ifdef PY_INT64_T
+/* _PyTime_t: Python timestamp with subsecond precision. It can be used to
+ store a duration, and so indirectly a date (related to another date, like
+ UNIX epoch). */
+typedef PY_INT64_T _PyTime_t;
+#define _PyTime_MIN PY_LLONG_MIN
+#define _PyTime_MAX PY_LLONG_MAX
#else
-typedef struct {
- time_t tv_sec; /* seconds since Jan. 1, 1970 */
- long tv_usec; /* and microseconds */
-} _PyTime_timeval;
+# error "_PyTime_t need signed 64-bit integer type"
#endif
-/* Structure used by time.get_clock_info() */
-typedef struct {
- const char *implementation;
- int monotonic;
- int adjustable;
- double resolution;
-} _Py_clock_info_t;
-
-/* Similar to POSIX gettimeofday but cannot fail. If system gettimeofday
- * fails or is not available, fall back to lower resolution clocks.
- */
-PyAPI_FUNC(void) _PyTime_gettimeofday(_PyTime_timeval *tp);
-
-/* Similar to _PyTime_gettimeofday() but retrieve also information on the
- * clock used to get the current time. */
-PyAPI_FUNC(void) _PyTime_gettimeofday_info(
- _PyTime_timeval *tp,
- _Py_clock_info_t *info);
-
-#define _PyTime_ADD_SECONDS(tv, interval) \
-do { \
- tv.tv_usec += (long) (((long) interval - interval) * 1000000); \
- tv.tv_sec += (time_t) interval + (time_t) (tv.tv_usec / 1000000); \
- tv.tv_usec %= 1000000; \
-} while (0)
-
-#define _PyTime_INTERVAL(tv_start, tv_end) \
- ((tv_end.tv_sec - tv_start.tv_sec) + \
- (tv_end.tv_usec - tv_start.tv_usec) * 0.000001)
-
-#ifndef Py_LIMITED_API
-
typedef enum {
- /* Round towards zero. */
- _PyTime_ROUND_DOWN=0,
- /* Round away from zero. */
- _PyTime_ROUND_UP
+ /* Round towards minus infinity (-inf).
+ For example, used to read a clock. */
+ _PyTime_ROUND_FLOOR=0,
+ /* Round towards infinity (+inf).
+ For example, used for timeout to wait "at least" N seconds. */
+ _PyTime_ROUND_CEILING
} _PyTime_round_t;
-/* Convert a number of seconds, int or float, to time_t. */
-PyAPI_FUNC(int) _PyTime_ObjectToTime_t(
- PyObject *obj,
- time_t *sec,
- _PyTime_round_t);
-
/* Convert a time_t to a PyLong. */
PyAPI_FUNC(PyObject *) _PyLong_FromTime_t(
time_t sec);
@@ -75,6 +41,12 @@ PyAPI_FUNC(PyObject *) _PyLong_FromTime_t(
PyAPI_FUNC(time_t) _PyLong_AsTime_t(
PyObject *obj);
+/* Convert a number of seconds, int or float, to time_t. */
+PyAPI_FUNC(int) _PyTime_ObjectToTime_t(
+ PyObject *obj,
+ time_t *sec,
+ _PyTime_round_t);
+
/* Convert a number of seconds, int or float, to a timeval structure.
usec is in the range [0; 999999] and rounded towards zero.
For example, -1.2 is converted to (-2, 800000). */
@@ -92,10 +64,114 @@ PyAPI_FUNC(int) _PyTime_ObjectToTimespec(
time_t *sec,
long *nsec,
_PyTime_round_t);
+
+
+/* Create a timestamp from a number of seconds. */
+PyAPI_FUNC(_PyTime_t) _PyTime_FromSeconds(int seconds);
+
+/* Macro to create a timestamp from a number of seconds, no integer overflow.
+ Only use the macro for small values, prefer _PyTime_FromSeconds(). */
+#define _PYTIME_FROMSECONDS(seconds) \
+ ((_PyTime_t)(seconds) * (1000 * 1000 * 1000))
+
+/* Create a timestamp from a number of nanoseconds. */
+PyAPI_FUNC(_PyTime_t) _PyTime_FromNanoseconds(PY_LONG_LONG ns);
+
+/* Convert a number of seconds (Python float or int) to a timetamp.
+ Raise an exception and return -1 on error, return 0 on success. */
+PyAPI_FUNC(int) _PyTime_FromSecondsObject(_PyTime_t *t,
+ PyObject *obj,
+ _PyTime_round_t round);
+
+/* Convert a number of milliseconds (Python float or int, 10^-3) to a timetamp.
+ Raise an exception and return -1 on error, return 0 on success. */
+PyAPI_FUNC(int) _PyTime_FromMillisecondsObject(_PyTime_t *t,
+ PyObject *obj,
+ _PyTime_round_t round);
+
+/* Convert a timestamp to a number of seconds as a C double. */
+PyAPI_FUNC(double) _PyTime_AsSecondsDouble(_PyTime_t t);
+
+/* Convert timestamp to a number of milliseconds (10^-3 seconds). */
+PyAPI_FUNC(_PyTime_t) _PyTime_AsMilliseconds(_PyTime_t t,
+ _PyTime_round_t round);
+
+/* Convert timestamp to a number of microseconds (10^-6 seconds). */
+PyAPI_FUNC(_PyTime_t) _PyTime_AsMicroseconds(_PyTime_t t,
+ _PyTime_round_t round);
+
+/* Convert timestamp to a number of nanoseconds (10^-9 seconds) as a Python int
+ object. */
+PyAPI_FUNC(PyObject *) _PyTime_AsNanosecondsObject(_PyTime_t t);
+
+/* Convert a timestamp to a timeval structure (microsecond resolution).
+ tv_usec is always positive.
+ Raise an exception and return -1 if the conversion overflowed,
+ return 0 on success. */
+PyAPI_FUNC(int) _PyTime_AsTimeval(_PyTime_t t,
+ struct timeval *tv,
+ _PyTime_round_t round);
+
+/* Similar to _PyTime_AsTimeval(), but don't raise an exception on error. */
+PyAPI_FUNC(int) _PyTime_AsTimeval_noraise(_PyTime_t t,
+ struct timeval *tv,
+ _PyTime_round_t round);
+
+#if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE)
+/* Convert a timestamp to a timespec structure (nanosecond resolution).
+ tv_nsec is always positive.
+ Raise an exception and return -1 on error, return 0 on success. */
+PyAPI_FUNC(int) _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts);
#endif
-/* Dummy to force linking. */
-PyAPI_FUNC(void) _PyTime_Init(void);
+/* Get the current time from the system clock.
+
+ The function cannot fail. _PyTime_Init() ensures that the system clock
+ works. */
+PyAPI_FUNC(_PyTime_t) _PyTime_GetSystemClock(void);
+
+/* Get the time of a monotonic clock, i.e. a clock that cannot go backwards.
+ The clock is not affected by system clock updates. The reference point of
+ the returned value is undefined, so that only the difference between the
+ results of consecutive calls is valid.
+
+ The function cannot fail. _PyTime_Init() ensures that a monotonic clock
+ is available and works. */
+PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void);
+
+
+/* Structure used by time.get_clock_info() */
+typedef struct {
+ const char *implementation;
+ int monotonic;
+ int adjustable;
+ double resolution;
+} _Py_clock_info_t;
+
+/* Get the current time from the system clock.
+ * Fill clock information if info is not NULL.
+ * Raise an exception and return -1 on error, return 0 on success.
+ */
+PyAPI_FUNC(int) _PyTime_GetSystemClockWithInfo(
+ _PyTime_t *t,
+ _Py_clock_info_t *info);
+
+/* Get the time of a monotonic clock, i.e. a clock that cannot go backwards.
+ The clock is not affected by system clock updates. The reference point of
+ the returned value is undefined, so that only the difference between the
+ results of consecutive calls is valid.
+
+ Fill info (if set) with information of the function used to get the time.
+
+ Return 0 on success, raise an exception and return -1 on error. */
+PyAPI_FUNC(int) _PyTime_GetMonotonicClockWithInfo(
+ _PyTime_t *t,
+ _Py_clock_info_t *info);
+
+
+/* Initialize time.
+ Return 0 on success, raise an exception and return -1 on error. */
+PyAPI_FUNC(int) _PyTime_Init(void);
#ifdef __cplusplus
}
diff --git a/Include/setobject.h b/Include/setobject.h
index ae3f556..f17bc1b 100644
--- a/Include/setobject.h
+++ b/Include/setobject.h
@@ -6,38 +6,43 @@
extern "C" {
#endif
+#ifndef Py_LIMITED_API
-/*
-There are three kinds of slots in the table:
+/* There are three kinds of entries in the table:
1. Unused: key == NULL
2. Active: key != NULL and key != dummy
3. Dummy: key == dummy
-Note: .pop() abuses the hash field of an Unused or Dummy slot to
-hold a search finger. The hash field of Unused or Dummy slots has
-no meaning otherwise.
+The hash field of Unused slots have no meaning.
+The hash field of Dummny slots are set to -1
+meaning that dummy entries can be detected by
+either entry->key==dummy or by entry->hash==-1.
*/
-#ifndef Py_LIMITED_API
+
#define PySet_MINSIZE 8
typedef struct {
- /* Cached hash code of the key. */
PyObject *key;
- Py_hash_t hash;
+ Py_hash_t hash; /* Cached hash code of the key */
} setentry;
+/* The SetObject data structure is shared by set and frozenset objects.
+
+Invariant for sets:
+ - hash is -1
+
+Invariants for frozensets:
+ - data is immutable.
+ - hash is the hash of the frozenset or -1 if not computed yet.
-/*
-This data structure is shared by set and frozenset objects.
*/
-typedef struct _setobject PySetObject;
-struct _setobject {
+typedef struct {
PyObject_HEAD
- Py_ssize_t fill; /* # Active + # Dummy */
- Py_ssize_t used; /* # Active */
+ Py_ssize_t fill; /* Number active and dummy entries*/
+ Py_ssize_t used; /* Number active entries */
/* The table contains mask + 1 slots, and that's a power of 2.
* We store the mask instead of the size because the mask is more
@@ -45,33 +50,42 @@ struct _setobject {
*/
Py_ssize_t mask;
- /* table points to smalltable for small tables, else to
- * additional malloc'ed memory. table is never NULL! This rule
- * saves repeated runtime null-tests.
+ /* The table points to a fixed-size smalltable for small tables
+ * or to additional malloc'ed memory for bigger tables.
+ * The table pointer is never NULL which saves us from repeated
+ * runtime null-tests.
*/
setentry *table;
- setentry *(*lookup)(PySetObject *so, PyObject *key, Py_hash_t hash);
- Py_hash_t hash; /* only used by frozenset objects */
- setentry smalltable[PySet_MINSIZE];
+ Py_hash_t hash; /* Only used by frozenset objects */
+ Py_ssize_t finger; /* Search finger for pop() */
+ setentry smalltable[PySet_MINSIZE];
PyObject *weakreflist; /* List of weak references */
-};
-#endif /* Py_LIMITED_API */
+} PySetObject;
+
+#define PySet_GET_SIZE(so) (((PySetObject *)(so))->used)
+
+PyAPI_DATA(PyObject *) _PySet_Dummy;
+
+PyAPI_FUNC(int) _PySet_NextEntry(PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash);
+PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable);
+PyAPI_FUNC(int) PySet_ClearFreeList(void);
+
+#endif /* Section excluded by Py_LIMITED_API */
PyAPI_DATA(PyTypeObject) PySet_Type;
PyAPI_DATA(PyTypeObject) PyFrozenSet_Type;
PyAPI_DATA(PyTypeObject) PySetIter_Type;
-#ifndef Py_LIMITED_API
-PyAPI_DATA(PyObject *) _PySet_Dummy;
-#endif
+PyAPI_FUNC(PyObject *) PySet_New(PyObject *);
+PyAPI_FUNC(PyObject *) PyFrozenSet_New(PyObject *);
-/* Invariants for frozensets:
- * data is immutable.
- * hash is the hash of the frozenset or -1 if not computed yet.
- * Invariants for sets:
- * hash is -1
- */
+PyAPI_FUNC(int) PySet_Add(PyObject *set, PyObject *key);
+PyAPI_FUNC(int) PySet_Clear(PyObject *set);
+PyAPI_FUNC(int) PySet_Contains(PyObject *anyset, PyObject *key);
+PyAPI_FUNC(int) PySet_Discard(PyObject *set, PyObject *key);
+PyAPI_FUNC(PyObject *) PySet_Pop(PyObject *set);
+PyAPI_FUNC(Py_ssize_t) PySet_Size(PyObject *anyset);
#define PyFrozenSet_CheckExact(ob) (Py_TYPE(ob) == &PyFrozenSet_Type)
#define PyAnySet_CheckExact(ob) \
@@ -87,26 +101,6 @@ PyAPI_DATA(PyObject *) _PySet_Dummy;
(Py_TYPE(ob) == &PyFrozenSet_Type || \
PyType_IsSubtype(Py_TYPE(ob), &PyFrozenSet_Type))
-PyAPI_FUNC(PyObject *) PySet_New(PyObject *);
-PyAPI_FUNC(PyObject *) PyFrozenSet_New(PyObject *);
-PyAPI_FUNC(Py_ssize_t) PySet_Size(PyObject *anyset);
-#ifndef Py_LIMITED_API
-#define PySet_GET_SIZE(so) (((PySetObject *)(so))->used)
-#endif
-PyAPI_FUNC(int) PySet_Clear(PyObject *set);
-PyAPI_FUNC(int) PySet_Contains(PyObject *anyset, PyObject *key);
-PyAPI_FUNC(int) PySet_Discard(PyObject *set, PyObject *key);
-PyAPI_FUNC(int) PySet_Add(PyObject *set, PyObject *key);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(int) _PySet_NextEntry(PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash);
-#endif
-PyAPI_FUNC(PyObject *) PySet_Pop(PyObject *set);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable);
-
-PyAPI_FUNC(int) PySet_ClearFreeList(void);
-#endif
-
#ifdef __cplusplus
}
#endif
diff --git a/Include/sliceobject.h b/Include/sliceobject.h
index f7ee90c..26370e0 100644
--- a/Include/sliceobject.h
+++ b/Include/sliceobject.h
@@ -41,7 +41,7 @@ PyAPI_FUNC(int) _PySlice_GetLongIndices(PySliceObject *self, PyObject *length,
PyAPI_FUNC(int) PySlice_GetIndices(PyObject *r, Py_ssize_t length,
Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step);
PyAPI_FUNC(int) PySlice_GetIndicesEx(PyObject *r, Py_ssize_t length,
- Py_ssize_t *start, Py_ssize_t *stop,
+ Py_ssize_t *start, Py_ssize_t *stop,
Py_ssize_t *step, Py_ssize_t *slicelength);
#ifdef __cplusplus
diff --git a/Include/symtable.h b/Include/symtable.h
index 1cfd884..1409cd9 100644
--- a/Include/symtable.h
+++ b/Include/symtable.h
@@ -43,7 +43,6 @@ typedef struct _symtable_entry {
PyObject *ste_children; /* list of child blocks */
PyObject *ste_directives;/* locations of global and nonlocal statements */
_Py_block_ty ste_type; /* module, class, or function */
- int ste_unoptimized; /* false if namespace is optimized */
int ste_nested; /* true if block is nested */
unsigned ste_free : 1; /* true if block has free variables */
unsigned ste_child_free : 1; /* true if a child block has free vars,
@@ -108,10 +107,6 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
#define FREE 4
#define CELL 5
-/* The following two names are used for the ste_unoptimized bit field */
-#define OPT_IMPORT_STAR 1
-#define OPT_TOPLEVEL 2 /* top-level names, including eval and exec */
-
#define GENERATOR 1
#define GENERATOR_EXPRESSION 2
diff --git a/Include/token.h b/Include/token.h
index 905022b..595afa0 100644
--- a/Include/token.h
+++ b/Include/token.h
@@ -58,13 +58,16 @@ extern "C" {
#define DOUBLESTAREQUAL 46
#define DOUBLESLASH 47
#define DOUBLESLASHEQUAL 48
-#define AT 49
-#define RARROW 50
-#define ELLIPSIS 51
+#define AT 49
+#define ATEQUAL 50
+#define RARROW 51
+#define ELLIPSIS 52
/* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */
-#define OP 52
-#define ERRORTOKEN 53
-#define N_TOKENS 54
+#define OP 53
+#define AWAIT 54
+#define ASYNC 55
+#define ERRORTOKEN 56
+#define N_TOKENS 57
/* Special definitions for cooperation with parser */
diff --git a/Include/typeslots.h b/Include/typeslots.h
index ad3cdfb..0ce6a37 100644
--- a/Include/typeslots.h
+++ b/Include/typeslots.h
@@ -74,3 +74,12 @@
#define Py_tp_members 72
#define Py_tp_getset 73
#define Py_tp_free 74
+#define Py_nb_matrix_multiply 75
+#define Py_nb_inplace_matrix_multiply 76
+#define Py_am_await 77
+#define Py_am_aiter 78
+#define Py_am_anext 79
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000
+/* New in 3.5 */
+#define Py_tp_finalize 80
+#endif
diff --git a/Include/ucnhash.h b/Include/ucnhash.h
index 8de9ba0..45362e9 100644
--- a/Include/ucnhash.h
+++ b/Include/ucnhash.h
@@ -16,7 +16,7 @@ typedef struct {
int size;
/* Get name for a given character code. Returns non-zero if
- success, zero if not. Does not set Python exceptions.
+ success, zero if not. Does not set Python exceptions.
If self is NULL, data come from the default version of the database.
If it is not NULL, it should be a unicodedata.ucd_X_Y_Z object */
int (*getname)(PyObject *self, Py_UCS4 code, char* buffer, int buflen,
diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h
index d7b2ace..4ba6328 100644
--- a/Include/unicodeobject.h
+++ b/Include/unicodeobject.h
@@ -2060,12 +2060,6 @@ PyAPI_FUNC(int) PyUnicode_Contains(
PyObject *element /* Element string */
);
-/* Checks whether the string contains any NUL characters. */
-
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(int) _PyUnicode_HasNULChars(PyObject *);
-#endif
-
/* Checks whether argument is a valid identifier. */
PyAPI_FUNC(int) PyUnicode_IsIdentifier(PyObject *s);
@@ -2245,6 +2239,8 @@ PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strrchr(
Py_UNICODE c
);
+PyAPI_FUNC(PyObject*) _PyUnicode_FormatLong(PyObject *, int, int, int);
+
/* Create a copy of a unicode string ending with a nul character. Return NULL
and raise a MemoryError exception on memory allocation failure, otherwise
return a new allocated buffer (use PyMem_Free() to free the buffer). */
diff --git a/Lib/__future__.py b/Lib/__future__.py
index 3b2d5ec..63b2be3 100644
--- a/Lib/__future__.py
+++ b/Lib/__future__.py
@@ -56,6 +56,7 @@ all_feature_names = [
"print_function",
"unicode_literals",
"barry_as_FLUFL",
+ "generator_stop",
]
__all__ = ["all_feature_names"] + all_feature_names
@@ -72,6 +73,7 @@ CO_FUTURE_WITH_STATEMENT = 0x8000 # with statement
CO_FUTURE_PRINT_FUNCTION = 0x10000 # print function
CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals
CO_FUTURE_BARRY_AS_BDFL = 0x40000
+CO_FUTURE_GENERATOR_STOP = 0x80000 # StopIteration becomes RuntimeError in generators
class _Feature:
def __init__(self, optionalRelease, mandatoryRelease, compiler_flag):
@@ -132,3 +134,7 @@ unicode_literals = _Feature((2, 6, 0, "alpha", 2),
barry_as_FLUFL = _Feature((3, 1, 0, "alpha", 2),
(3, 9, 0, "alpha", 0),
CO_FUTURE_BARRY_AS_BDFL)
+
+generator_stop = _Feature((3, 5, 0, "beta", 1),
+ (3, 7, 0, "alpha", 0),
+ CO_FUTURE_GENERATOR_STOP)
diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py
index 33b59ab..f89bb6f 100644
--- a/Lib/_collections_abc.py
+++ b/Lib/_collections_abc.py
@@ -9,7 +9,8 @@ Unit tests are in test_collections.
from abc import ABCMeta, abstractmethod
import sys
-__all__ = ["Hashable", "Iterable", "Iterator",
+__all__ = ["Awaitable", "Coroutine", "AsyncIterable", "AsyncIterator",
+ "Hashable", "Iterable", "Iterator", "Generator",
"Sized", "Container", "Callable",
"Set", "MutableSet",
"Mapping", "MutableMapping",
@@ -50,6 +51,13 @@ dict_values = type({}.values())
dict_items = type({}.items())
## misc ##
mappingproxy = type(type.__dict__)
+generator = type((lambda: (yield))())
+## coroutine ##
+async def _coro(): pass
+_coro = _coro()
+coroutine = type(_coro)
+_coro.close() # Prevent ResourceWarning
+del _coro
### ONE-TRICK PONIES ###
@@ -73,6 +81,113 @@ class Hashable(metaclass=ABCMeta):
return NotImplemented
+class Awaitable(metaclass=ABCMeta):
+
+ __slots__ = ()
+
+ @abstractmethod
+ def __await__(self):
+ yield
+
+ @classmethod
+ def __subclasshook__(cls, C):
+ if cls is Awaitable:
+ for B in C.__mro__:
+ if "__await__" in B.__dict__:
+ if B.__dict__["__await__"]:
+ return True
+ break
+ return NotImplemented
+
+
+class Coroutine(Awaitable):
+
+ __slots__ = ()
+
+ @abstractmethod
+ def send(self, value):
+ """Send a value into the coroutine.
+ Return next yielded value or raise StopIteration.
+ """
+ raise StopIteration
+
+ @abstractmethod
+ def throw(self, typ, val=None, tb=None):
+ """Raise an exception in the coroutine.
+ Return next yielded value or raise StopIteration.
+ """
+ if val is None:
+ if tb is None:
+ raise typ
+ val = typ()
+ if tb is not None:
+ val = val.with_traceback(tb)
+ raise val
+
+ def close(self):
+ """Raise GeneratorExit inside coroutine.
+ """
+ try:
+ self.throw(GeneratorExit)
+ except (GeneratorExit, StopIteration):
+ pass
+ else:
+ raise RuntimeError("coroutine ignored GeneratorExit")
+
+ @classmethod
+ def __subclasshook__(cls, C):
+ if cls is Coroutine:
+ mro = C.__mro__
+ for method in ('__await__', 'send', 'throw', 'close'):
+ for base in mro:
+ if method in base.__dict__:
+ break
+ else:
+ return NotImplemented
+ return True
+ return NotImplemented
+
+
+Coroutine.register(coroutine)
+
+
+class AsyncIterable(metaclass=ABCMeta):
+
+ __slots__ = ()
+
+ @abstractmethod
+ async def __aiter__(self):
+ return AsyncIterator()
+
+ @classmethod
+ def __subclasshook__(cls, C):
+ if cls is AsyncIterable:
+ if any("__aiter__" in B.__dict__ for B in C.__mro__):
+ return True
+ return NotImplemented
+
+
+class AsyncIterator(AsyncIterable):
+
+ __slots__ = ()
+
+ @abstractmethod
+ async def __anext__(self):
+ """Return the next item or raise StopAsyncIteration when exhausted."""
+ raise StopAsyncIteration
+
+ async def __aiter__(self):
+ return self
+
+ @classmethod
+ def __subclasshook__(cls, C):
+ if cls is AsyncIterator:
+ if (any("__anext__" in B.__dict__ for B in C.__mro__) and
+ any("__aiter__" in B.__dict__ for B in C.__mro__)):
+ return True
+ return NotImplemented
+
+
class Iterable(metaclass=ABCMeta):
__slots__ = ()
@@ -124,6 +239,64 @@ Iterator.register(str_iterator)
Iterator.register(tuple_iterator)
Iterator.register(zip_iterator)
+
+class Generator(Iterator):
+
+ __slots__ = ()
+
+ def __next__(self):
+ """Return the next item from the generator.
+ When exhausted, raise StopIteration.
+ """
+ return self.send(None)
+
+ @abstractmethod
+ def send(self, value):
+ """Send a value into the generator.
+ Return next yielded value or raise StopIteration.
+ """
+ raise StopIteration
+
+ @abstractmethod
+ def throw(self, typ, val=None, tb=None):
+ """Raise an exception in the generator.
+ Return next yielded value or raise StopIteration.
+ """
+ if val is None:
+ if tb is None:
+ raise typ
+ val = typ()
+ if tb is not None:
+ val = val.with_traceback(tb)
+ raise val
+
+ def close(self):
+ """Raise GeneratorExit inside generator.
+ """
+ try:
+ self.throw(GeneratorExit)
+ except (GeneratorExit, StopIteration):
+ pass
+ else:
+ raise RuntimeError("generator ignored GeneratorExit")
+
+ @classmethod
+ def __subclasshook__(cls, C):
+ if cls is Generator:
+ mro = C.__mro__
+ for method in ('__iter__', '__next__', 'send', 'throw', 'close'):
+ for base in mro:
+ if method in base.__dict__:
+ break
+ else:
+ return NotImplemented
+ return True
+ return NotImplemented
+
+
+Generator.register(generator)
+
+
class Sized(metaclass=ABCMeta):
__slots__ = ()
@@ -453,6 +626,8 @@ Mapping.register(mappingproxy)
class MappingView(Sized):
+ __slots__ = '_mapping',
+
def __init__(self, mapping):
self._mapping = mapping
@@ -465,6 +640,8 @@ class MappingView(Sized):
class KeysView(MappingView, Set):
+ __slots__ = ()
+
@classmethod
def _from_iterable(self, it):
return set(it)
@@ -480,6 +657,8 @@ KeysView.register(dict_keys)
class ItemsView(MappingView, Set):
+ __slots__ = ()
+
@classmethod
def _from_iterable(self, it):
return set(it)
@@ -502,6 +681,8 @@ ItemsView.register(dict_items)
class ValuesView(MappingView):
+ __slots__ = ()
+
def __contains__(self, value):
for key in self._mapping:
if value == self._mapping[key]:
@@ -647,13 +828,23 @@ class Sequence(Sized, Iterable, Container):
for i in reversed(range(len(self))):
yield self[i]
- def index(self, value):
- '''S.index(value) -> integer -- return first index of value.
+ def index(self, value, start=0, stop=None):
+ '''S.index(value, [start, [stop]]) -> integer -- return first index of value.
Raises ValueError if the value is not present.
'''
- for i, v in enumerate(self):
- if v == value:
- return i
+ if start is not None and start < 0:
+ start = max(len(self) + start, 0)
+ if stop is not None and stop < 0:
+ stop += len(self)
+
+ i = start
+ while stop is None or i < stop:
+ try:
+ if self[i] == value:
+ return i
+ except IndexError:
+ break
+ i += 1
raise ValueError
def count(self, value):
diff --git a/Lib/_compression.py b/Lib/_compression.py
new file mode 100644
index 0000000..b00f31b
--- /dev/null
+++ b/Lib/_compression.py
@@ -0,0 +1,152 @@
+"""Internal classes used by the gzip, lzma and bz2 modules"""
+
+import io
+
+
+BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE # Compressed data read chunk size
+
+
+class BaseStream(io.BufferedIOBase):
+ """Mode-checking helper functions."""
+
+ def _check_not_closed(self):
+ if self.closed:
+ raise ValueError("I/O operation on closed file")
+
+ def _check_can_read(self):
+ if not self.readable():
+ raise io.UnsupportedOperation("File not open for reading")
+
+ def _check_can_write(self):
+ if not self.writable():
+ raise io.UnsupportedOperation("File not open for writing")
+
+ def _check_can_seek(self):
+ if not self.readable():
+ raise io.UnsupportedOperation("Seeking is only supported "
+ "on files open for reading")
+ if not self.seekable():
+ raise io.UnsupportedOperation("The underlying file object "
+ "does not support seeking")
+
+
+class DecompressReader(io.RawIOBase):
+ """Adapts the decompressor API to a RawIOBase reader API"""
+
+ def readable(self):
+ return True
+
+ def __init__(self, fp, decomp_factory, trailing_error=(), **decomp_args):
+ self._fp = fp
+ self._eof = False
+ self._pos = 0 # Current offset in decompressed stream
+
+ # Set to size of decompressed stream once it is known, for SEEK_END
+ self._size = -1
+
+ # Save the decompressor factory and arguments.
+ # If the file contains multiple compressed streams, each
+ # stream will need a separate decompressor object. A new decompressor
+ # object is also needed when implementing a backwards seek().
+ self._decomp_factory = decomp_factory
+ self._decomp_args = decomp_args
+ self._decompressor = self._decomp_factory(**self._decomp_args)
+
+ # Exception class to catch from decompressor signifying invalid
+ # trailing data to ignore
+ self._trailing_error = trailing_error
+
+ def close(self):
+ self._decompressor = None
+ return super().close()
+
+ def seekable(self):
+ return self._fp.seekable()
+
+ def readinto(self, b):
+ with memoryview(b) as view, view.cast("B") as byte_view:
+ data = self.read(len(byte_view))
+ byte_view[:len(data)] = data
+ return len(data)
+
+ def read(self, size=-1):
+ if size < 0:
+ return self.readall()
+
+ if not size or self._eof:
+ return b""
+ data = None # Default if EOF is encountered
+ # Depending on the input data, our call to the decompressor may not
+ # return any data. In this case, try again after reading another block.
+ while True:
+ if self._decompressor.eof:
+ rawblock = (self._decompressor.unused_data or
+ self._fp.read(BUFFER_SIZE))
+ if not rawblock:
+ break
+ # Continue to next stream.
+ self._decompressor = self._decomp_factory(
+ **self._decomp_args)
+ try:
+ data = self._decompressor.decompress(rawblock, size)
+ except self._trailing_error:
+ # Trailing data isn't a valid compressed stream; ignore it.
+ break
+ else:
+ if self._decompressor.needs_input:
+ rawblock = self._fp.read(BUFFER_SIZE)
+ if not rawblock:
+ raise EOFError("Compressed file ended before the "
+ "end-of-stream marker was reached")
+ else:
+ rawblock = b""
+ data = self._decompressor.decompress(rawblock, size)
+ if data:
+ break
+ if not data:
+ self._eof = True
+ self._size = self._pos
+ return b""
+ self._pos += len(data)
+ return data
+
+ # Rewind the file to the beginning of the data stream.
+ def _rewind(self):
+ self._fp.seek(0)
+ self._eof = False
+ self._pos = 0
+ self._decompressor = self._decomp_factory(**self._decomp_args)
+
+ def seek(self, offset, whence=io.SEEK_SET):
+ # Recalculate offset as an absolute file position.
+ if whence == io.SEEK_SET:
+ pass
+ elif whence == io.SEEK_CUR:
+ offset = self._pos + offset
+ elif whence == io.SEEK_END:
+ # Seeking relative to EOF - we need to know the file's size.
+ if self._size < 0:
+ while self.read(io.DEFAULT_BUFFER_SIZE):
+ pass
+ offset = self._size + offset
+ else:
+ raise ValueError("Invalid value for whence: {}".format(whence))
+
+ # Make it so that offset is the number of bytes to skip forward.
+ if offset < self._pos:
+ self._rewind()
+ else:
+ offset -= self._pos
+
+ # Read and discard data until we reach the desired position.
+ while offset > 0:
+ data = self.read(min(io.DEFAULT_BUFFER_SIZE, offset))
+ if not data:
+ break
+ offset -= len(data)
+
+ return self._pos
+
+ def tell(self):
+ """Return the current file position."""
+ return self._pos
diff --git a/Lib/_dummy_thread.py b/Lib/_dummy_thread.py
index b67cfb9..36e5f38 100644
--- a/Lib/_dummy_thread.py
+++ b/Lib/_dummy_thread.py
@@ -140,6 +140,14 @@ class LockType(object):
def locked(self):
return self.locked_status
+ def __repr__(self):
+ return "<%s %s.%s object at %s>" % (
+ "locked" if self.locked_status else "unlocked",
+ self.__class__.__module__,
+ self.__class__.__qualname__,
+ hex(id(self))
+ )
+
# Used to signal that interrupt_main was called in a "thread"
_interrupt = False
# True when not executing in a "thread"
diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py
new file mode 100644
index 0000000..05ba4ee
--- /dev/null
+++ b/Lib/_pydecimal.py
@@ -0,0 +1,6380 @@
+# Copyright (c) 2004 Python Software Foundation.
+# All rights reserved.
+
+# Written by Eric Price <eprice at tjhsst.edu>
+# and Facundo Batista <facundo at taniquetil.com.ar>
+# and Raymond Hettinger <python at rcn.com>
+# and Aahz <aahz at pobox.com>
+# and Tim Peters
+
+# This module should be kept in sync with the latest updates of the
+# IBM specification as it evolves. Those updates will be treated
+# as bug fixes (deviation from the spec is a compatibility, usability
+# bug) and will be backported. At this point the spec is stabilizing
+# and the updates are becoming fewer, smaller, and less significant.
+
+"""
+This is an implementation of decimal floating point arithmetic based on
+the General Decimal Arithmetic Specification:
+
+ http://speleotrove.com/decimal/decarith.html
+
+and IEEE standard 854-1987:
+
+ http://en.wikipedia.org/wiki/IEEE_854-1987
+
+Decimal floating point has finite precision with arbitrarily large bounds.
+
+The purpose of this module is to support arithmetic using familiar
+"schoolhouse" rules and to avoid some of the tricky representation
+issues associated with binary floating point. The package is especially
+useful for financial applications or for contexts where users have
+expectations that are at odds with binary floating point (for instance,
+in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead
+of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected
+Decimal('0.00')).
+
+Here are some examples of using the decimal module:
+
+>>> from decimal import *
+>>> setcontext(ExtendedContext)
+>>> Decimal(0)
+Decimal('0')
+>>> Decimal('1')
+Decimal('1')
+>>> Decimal('-.0123')
+Decimal('-0.0123')
+>>> Decimal(123456)
+Decimal('123456')
+>>> Decimal('123.45e12345678')
+Decimal('1.2345E+12345680')
+>>> Decimal('1.33') + Decimal('1.27')
+Decimal('2.60')
+>>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41')
+Decimal('-2.20')
+>>> dig = Decimal(1)
+>>> print(dig / Decimal(3))
+0.333333333
+>>> getcontext().prec = 18
+>>> print(dig / Decimal(3))
+0.333333333333333333
+>>> print(dig.sqrt())
+1
+>>> print(Decimal(3).sqrt())
+1.73205080756887729
+>>> print(Decimal(3) ** 123)
+4.85192780976896427E+58
+>>> inf = Decimal(1) / Decimal(0)
+>>> print(inf)
+Infinity
+>>> neginf = Decimal(-1) / Decimal(0)
+>>> print(neginf)
+-Infinity
+>>> print(neginf + inf)
+NaN
+>>> print(neginf * inf)
+-Infinity
+>>> print(dig / 0)
+Infinity
+>>> getcontext().traps[DivisionByZero] = 1
+>>> print(dig / 0)
+Traceback (most recent call last):
+ ...
+ ...
+ ...
+decimal.DivisionByZero: x / 0
+>>> c = Context()
+>>> c.traps[InvalidOperation] = 0
+>>> print(c.flags[InvalidOperation])
+0
+>>> c.divide(Decimal(0), Decimal(0))
+Decimal('NaN')
+>>> c.traps[InvalidOperation] = 1
+>>> print(c.flags[InvalidOperation])
+1
+>>> c.flags[InvalidOperation] = 0
+>>> print(c.flags[InvalidOperation])
+0
+>>> print(c.divide(Decimal(0), Decimal(0)))
+Traceback (most recent call last):
+ ...
+ ...
+ ...
+decimal.InvalidOperation: 0 / 0
+>>> print(c.flags[InvalidOperation])
+1
+>>> c.flags[InvalidOperation] = 0
+>>> c.traps[InvalidOperation] = 0
+>>> print(c.divide(Decimal(0), Decimal(0)))
+NaN
+>>> print(c.flags[InvalidOperation])
+1
+>>>
+"""
+
+__all__ = [
+ # Two major classes
+ 'Decimal', 'Context',
+
+ # Named tuple representation
+ 'DecimalTuple',
+
+ # Contexts
+ 'DefaultContext', 'BasicContext', 'ExtendedContext',
+
+ # Exceptions
+ 'DecimalException', 'Clamped', 'InvalidOperation', 'DivisionByZero',
+ 'Inexact', 'Rounded', 'Subnormal', 'Overflow', 'Underflow',
+ 'FloatOperation',
+
+ # Exceptional conditions that trigger InvalidOperation
+ 'DivisionImpossible', 'InvalidContext', 'ConversionSyntax', 'DivisionUndefined',
+
+ # Constants for use in setting up contexts
+ 'ROUND_DOWN', 'ROUND_HALF_UP', 'ROUND_HALF_EVEN', 'ROUND_CEILING',
+ 'ROUND_FLOOR', 'ROUND_UP', 'ROUND_HALF_DOWN', 'ROUND_05UP',
+
+ # Functions for manipulating contexts
+ 'setcontext', 'getcontext', 'localcontext',
+
+ # Limits for the C version for compatibility
+ 'MAX_PREC', 'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY',
+
+ # C version: compile time choice that enables the thread local context
+ 'HAVE_THREADS'
+]
+
+__xname__ = __name__ # sys.modules lookup (--without-threads)
+__name__ = 'decimal' # For pickling
+__version__ = '1.70' # Highest version of the spec this complies with
+ # See http://speleotrove.com/decimal/
+__libmpdec_version__ = "2.4.1" # compatible libmpdec version
+
+import math as _math
+import numbers as _numbers
+import sys
+
+try:
+ from collections import namedtuple as _namedtuple
+ DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent')
+except ImportError:
+ DecimalTuple = lambda *args: args
+
+# Rounding
+ROUND_DOWN = 'ROUND_DOWN'
+ROUND_HALF_UP = 'ROUND_HALF_UP'
+ROUND_HALF_EVEN = 'ROUND_HALF_EVEN'
+ROUND_CEILING = 'ROUND_CEILING'
+ROUND_FLOOR = 'ROUND_FLOOR'
+ROUND_UP = 'ROUND_UP'
+ROUND_HALF_DOWN = 'ROUND_HALF_DOWN'
+ROUND_05UP = 'ROUND_05UP'
+
+# Compatibility with the C version
+HAVE_THREADS = True
+if sys.maxsize == 2**63-1:
+ MAX_PREC = 999999999999999999
+ MAX_EMAX = 999999999999999999
+ MIN_EMIN = -999999999999999999
+else:
+ MAX_PREC = 425000000
+ MAX_EMAX = 425000000
+ MIN_EMIN = -425000000
+
+MIN_ETINY = MIN_EMIN - (MAX_PREC-1)
+
+# Errors
+
+class DecimalException(ArithmeticError):
+ """Base exception class.
+
+ Used exceptions derive from this.
+ If an exception derives from another exception besides this (such as
+ Underflow (Inexact, Rounded, Subnormal) that indicates that it is only
+ called if the others are present. This isn't actually used for
+ anything, though.
+
+ handle -- Called when context._raise_error is called and the
+ trap_enabler is not set. First argument is self, second is the
+ context. More arguments can be given, those being after
+ the explanation in _raise_error (For example,
+ context._raise_error(NewError, '(-x)!', self._sign) would
+ call NewError().handle(context, self._sign).)
+
+ To define a new exception, it should be sufficient to have it derive
+ from DecimalException.
+ """
+ def handle(self, context, *args):
+ pass
+
+
+class Clamped(DecimalException):
+ """Exponent of a 0 changed to fit bounds.
+
+ This occurs and signals clamped if the exponent of a result has been
+ altered in order to fit the constraints of a specific concrete
+ representation. This may occur when the exponent of a zero result would
+ be outside the bounds of a representation, or when a large normal
+ number would have an encoded exponent that cannot be represented. In
+ this latter case, the exponent is reduced to fit and the corresponding
+ number of zero digits are appended to the coefficient ("fold-down").
+ """
+
+class InvalidOperation(DecimalException):
+ """An invalid operation was performed.
+
+ Various bad things cause this:
+
+ Something creates a signaling NaN
+ -INF + INF
+ 0 * (+-)INF
+ (+-)INF / (+-)INF
+ x % 0
+ (+-)INF % x
+ x._rescale( non-integer )
+ sqrt(-x) , x > 0
+ 0 ** 0
+ x ** (non-integer)
+ x ** (+-)INF
+ An operand is invalid
+
+ The result of the operation after these is a quiet positive NaN,
+ except when the cause is a signaling NaN, in which case the result is
+ also a quiet NaN, but with the original sign, and an optional
+ diagnostic information.
+ """
+ def handle(self, context, *args):
+ if args:
+ ans = _dec_from_triple(args[0]._sign, args[0]._int, 'n', True)
+ return ans._fix_nan(context)
+ return _NaN
+
+class ConversionSyntax(InvalidOperation):
+ """Trying to convert badly formed string.
+
+ This occurs and signals invalid-operation if an string is being
+ converted to a number and it does not conform to the numeric string
+ syntax. The result is [0,qNaN].
+ """
+ def handle(self, context, *args):
+ return _NaN
+
+class DivisionByZero(DecimalException, ZeroDivisionError):
+ """Division by 0.
+
+ This occurs and signals division-by-zero if division of a finite number
+ by zero was attempted (during a divide-integer or divide operation, or a
+ power operation with negative right-hand operand), and the dividend was
+ not zero.
+
+ The result of the operation is [sign,inf], where sign is the exclusive
+ or of the signs of the operands for divide, or is 1 for an odd power of
+ -0, for power.
+ """
+
+ def handle(self, context, sign, *args):
+ return _SignedInfinity[sign]
+
+class DivisionImpossible(InvalidOperation):
+ """Cannot perform the division adequately.
+
+ This occurs and signals invalid-operation if the integer result of a
+ divide-integer or remainder operation had too many digits (would be
+ longer than precision). The result is [0,qNaN].
+ """
+
+ def handle(self, context, *args):
+ return _NaN
+
+class DivisionUndefined(InvalidOperation, ZeroDivisionError):
+ """Undefined result of division.
+
+ This occurs and signals invalid-operation if division by zero was
+ attempted (during a divide-integer, divide, or remainder operation), and
+ the dividend is also zero. The result is [0,qNaN].
+ """
+
+ def handle(self, context, *args):
+ return _NaN
+
+class Inexact(DecimalException):
+ """Had to round, losing information.
+
+ This occurs and signals inexact whenever the result of an operation is
+ not exact (that is, it needed to be rounded and any discarded digits
+ were non-zero), or if an overflow or underflow condition occurs. The
+ result in all cases is unchanged.
+
+ The inexact signal may be tested (or trapped) to determine if a given
+ operation (or sequence of operations) was inexact.
+ """
+
+class InvalidContext(InvalidOperation):
+ """Invalid context. Unknown rounding, for example.
+
+ This occurs and signals invalid-operation if an invalid context was
+ detected during an operation. This can occur if contexts are not checked
+ on creation and either the precision exceeds the capability of the
+ underlying concrete representation or an unknown or unsupported rounding
+ was specified. These aspects of the context need only be checked when
+ the values are required to be used. The result is [0,qNaN].
+ """
+
+ def handle(self, context, *args):
+ return _NaN
+
+class Rounded(DecimalException):
+ """Number got rounded (not necessarily changed during rounding).
+
+ This occurs and signals rounded whenever the result of an operation is
+ rounded (that is, some zero or non-zero digits were discarded from the
+ coefficient), or if an overflow or underflow condition occurs. The
+ result in all cases is unchanged.
+
+ The rounded signal may be tested (or trapped) to determine if a given
+ operation (or sequence of operations) caused a loss of precision.
+ """
+
+class Subnormal(DecimalException):
+ """Exponent < Emin before rounding.
+
+ This occurs and signals subnormal whenever the result of a conversion or
+ operation is subnormal (that is, its adjusted exponent is less than
+ Emin, before any rounding). The result in all cases is unchanged.
+
+ The subnormal signal may be tested (or trapped) to determine if a given
+ or operation (or sequence of operations) yielded a subnormal result.
+ """
+
+class Overflow(Inexact, Rounded):
+ """Numerical overflow.
+
+ This occurs and signals overflow if the adjusted exponent of a result
+ (from a conversion or from an operation that is not an attempt to divide
+ by zero), after rounding, would be greater than the largest value that
+ can be handled by the implementation (the value Emax).
+
+ The result depends on the rounding mode:
+
+ For round-half-up and round-half-even (and for round-half-down and
+ round-up, if implemented), the result of the operation is [sign,inf],
+ where sign is the sign of the intermediate result. For round-down, the
+ result is the largest finite number that can be represented in the
+ current precision, with the sign of the intermediate result. For
+ round-ceiling, the result is the same as for round-down if the sign of
+ the intermediate result is 1, or is [0,inf] otherwise. For round-floor,
+ the result is the same as for round-down if the sign of the intermediate
+ result is 0, or is [1,inf] otherwise. In all cases, Inexact and Rounded
+ will also be raised.
+ """
+
+ def handle(self, context, sign, *args):
+ if context.rounding in (ROUND_HALF_UP, ROUND_HALF_EVEN,
+ ROUND_HALF_DOWN, ROUND_UP):
+ return _SignedInfinity[sign]
+ if sign == 0:
+ if context.rounding == ROUND_CEILING:
+ return _SignedInfinity[sign]
+ return _dec_from_triple(sign, '9'*context.prec,
+ context.Emax-context.prec+1)
+ if sign == 1:
+ if context.rounding == ROUND_FLOOR:
+ return _SignedInfinity[sign]
+ return _dec_from_triple(sign, '9'*context.prec,
+ context.Emax-context.prec+1)
+
+
+class Underflow(Inexact, Rounded, Subnormal):
+ """Numerical underflow with result rounded to 0.
+
+ This occurs and signals underflow if a result is inexact and the
+ adjusted exponent of the result would be smaller (more negative) than
+ the smallest value that can be handled by the implementation (the value
+ Emin). That is, the result is both inexact and subnormal.
+
+ The result after an underflow will be a subnormal number rounded, if
+ necessary, so that its exponent is not less than Etiny. This may result
+ in 0 with the sign of the intermediate result and an exponent of Etiny.
+
+ In all cases, Inexact, Rounded, and Subnormal will also be raised.
+ """
+
+class FloatOperation(DecimalException, TypeError):
+ """Enable stricter semantics for mixing floats and Decimals.
+
+ If the signal is not trapped (default), mixing floats and Decimals is
+ permitted in the Decimal() constructor, context.create_decimal() and
+ all comparison operators. Both conversion and comparisons are exact.
+ Any occurrence of a mixed operation is silently recorded by setting
+ FloatOperation in the context flags. Explicit conversions with
+ Decimal.from_float() or context.create_decimal_from_float() do not
+ set the flag.
+
+ Otherwise (the signal is trapped), only equality comparisons and explicit
+ conversions are silent. All other mixed operations raise FloatOperation.
+ """
+
+# List of public traps and flags
+_signals = [Clamped, DivisionByZero, Inexact, Overflow, Rounded,
+ Underflow, InvalidOperation, Subnormal, FloatOperation]
+
+# Map conditions (per the spec) to signals
+_condition_map = {ConversionSyntax:InvalidOperation,
+ DivisionImpossible:InvalidOperation,
+ DivisionUndefined:InvalidOperation,
+ InvalidContext:InvalidOperation}
+
+# Valid rounding modes
+_rounding_modes = (ROUND_DOWN, ROUND_HALF_UP, ROUND_HALF_EVEN, ROUND_CEILING,
+ ROUND_FLOOR, ROUND_UP, ROUND_HALF_DOWN, ROUND_05UP)
+
+##### Context Functions ##################################################
+
+# The getcontext() and setcontext() function manage access to a thread-local
+# current context. Py2.4 offers direct support for thread locals. If that
+# is not available, use threading.current_thread() which is slower but will
+# work for older Pythons. If threads are not part of the build, create a
+# mock threading object with threading.local() returning the module namespace.
+
+try:
+ import threading
+except ImportError:
+ # Python was compiled without threads; create a mock object instead
+ class MockThreading(object):
+ def local(self, sys=sys):
+ return sys.modules[__xname__]
+ threading = MockThreading()
+ del MockThreading
+
+try:
+ threading.local
+
+except AttributeError:
+
+ # To fix reloading, force it to create a new context
+ # Old contexts have different exceptions in their dicts, making problems.
+ if hasattr(threading.current_thread(), '__decimal_context__'):
+ del threading.current_thread().__decimal_context__
+
+ def setcontext(context):
+ """Set this thread's context to context."""
+ if context in (DefaultContext, BasicContext, ExtendedContext):
+ context = context.copy()
+ context.clear_flags()
+ threading.current_thread().__decimal_context__ = context
+
+ def getcontext():
+ """Returns this thread's context.
+
+ If this thread does not yet have a context, returns
+ a new context and sets this thread's context.
+ New contexts are copies of DefaultContext.
+ """
+ try:
+ return threading.current_thread().__decimal_context__
+ except AttributeError:
+ context = Context()
+ threading.current_thread().__decimal_context__ = context
+ return context
+
+else:
+
+ local = threading.local()
+ if hasattr(local, '__decimal_context__'):
+ del local.__decimal_context__
+
+ def getcontext(_local=local):
+ """Returns this thread's context.
+
+ If this thread does not yet have a context, returns
+ a new context and sets this thread's context.
+ New contexts are copies of DefaultContext.
+ """
+ try:
+ return _local.__decimal_context__
+ except AttributeError:
+ context = Context()
+ _local.__decimal_context__ = context
+ return context
+
+ def setcontext(context, _local=local):
+ """Set this thread's context to context."""
+ if context in (DefaultContext, BasicContext, ExtendedContext):
+ context = context.copy()
+ context.clear_flags()
+ _local.__decimal_context__ = context
+
+ del threading, local # Don't contaminate the namespace
+
+def localcontext(ctx=None):
+ """Return a context manager for a copy of the supplied context
+
+ Uses a copy of the current context if no context is specified
+ The returned context manager creates a local decimal context
+ in a with statement:
+ def sin(x):
+ with localcontext() as ctx:
+ ctx.prec += 2
+ # Rest of sin calculation algorithm
+ # uses a precision 2 greater than normal
+ return +s # Convert result to normal precision
+
+ def sin(x):
+ with localcontext(ExtendedContext):
+ # Rest of sin calculation algorithm
+ # uses the Extended Context from the
+ # General Decimal Arithmetic Specification
+ return +s # Convert result to normal context
+
+ >>> setcontext(DefaultContext)
+ >>> print(getcontext().prec)
+ 28
+ >>> with localcontext():
+ ... ctx = getcontext()
+ ... ctx.prec += 2
+ ... print(ctx.prec)
+ ...
+ 30
+ >>> with localcontext(ExtendedContext):
+ ... print(getcontext().prec)
+ ...
+ 9
+ >>> print(getcontext().prec)
+ 28
+ """
+ if ctx is None: ctx = getcontext()
+ return _ContextManager(ctx)
+
+
+##### Decimal class #######################################################
+
+# Do not subclass Decimal from numbers.Real and do not register it as such
+# (because Decimals are not interoperable with floats). See the notes in
+# numbers.py for more detail.
+
+class Decimal(object):
+ """Floating point class for decimal arithmetic."""
+
+ __slots__ = ('_exp','_int','_sign', '_is_special')
+ # Generally, the value of the Decimal instance is given by
+ # (-1)**_sign * _int * 10**_exp
+ # Special values are signified by _is_special == True
+
+ # We're immutable, so use __new__ not __init__
+ def __new__(cls, value="0", context=None):
+ """Create a decimal point instance.
+
+ >>> Decimal('3.14') # string input
+ Decimal('3.14')
+ >>> Decimal((0, (3, 1, 4), -2)) # tuple (sign, digit_tuple, exponent)
+ Decimal('3.14')
+ >>> Decimal(314) # int
+ Decimal('314')
+ >>> Decimal(Decimal(314)) # another decimal instance
+ Decimal('314')
+ >>> Decimal(' 3.14 \\n') # leading and trailing whitespace okay
+ Decimal('3.14')
+ """
+
+ # Note that the coefficient, self._int, is actually stored as
+ # a string rather than as a tuple of digits. This speeds up
+ # the "digits to integer" and "integer to digits" conversions
+ # that are used in almost every arithmetic operation on
+ # Decimals. This is an internal detail: the as_tuple function
+ # and the Decimal constructor still deal with tuples of
+ # digits.
+
+ self = object.__new__(cls)
+
+ # From a string
+ # REs insist on real strings, so we can too.
+ if isinstance(value, str):
+ m = _parser(value.strip())
+ if m is None:
+ if context is None:
+ context = getcontext()
+ return context._raise_error(ConversionSyntax,
+ "Invalid literal for Decimal: %r" % value)
+
+ if m.group('sign') == "-":
+ self._sign = 1
+ else:
+ self._sign = 0
+ intpart = m.group('int')
+ if intpart is not None:
+ # finite number
+ fracpart = m.group('frac') or ''
+ exp = int(m.group('exp') or '0')
+ self._int = str(int(intpart+fracpart))
+ self._exp = exp - len(fracpart)
+ self._is_special = False
+ else:
+ diag = m.group('diag')
+ if diag is not None:
+ # NaN
+ self._int = str(int(diag or '0')).lstrip('0')
+ if m.group('signal'):
+ self._exp = 'N'
+ else:
+ self._exp = 'n'
+ else:
+ # infinity
+ self._int = '0'
+ self._exp = 'F'
+ self._is_special = True
+ return self
+
+ # From an integer
+ if isinstance(value, int):
+ if value >= 0:
+ self._sign = 0
+ else:
+ self._sign = 1
+ self._exp = 0
+ self._int = str(abs(value))
+ self._is_special = False
+ return self
+
+ # From another decimal
+ if isinstance(value, Decimal):
+ self._exp = value._exp
+ self._sign = value._sign
+ self._int = value._int
+ self._is_special = value._is_special
+ return self
+
+ # From an internal working value
+ if isinstance(value, _WorkRep):
+ self._sign = value.sign
+ self._int = str(value.int)
+ self._exp = int(value.exp)
+ self._is_special = False
+ return self
+
+ # tuple/list conversion (possibly from as_tuple())
+ if isinstance(value, (list,tuple)):
+ if len(value) != 3:
+ raise ValueError('Invalid tuple size in creation of Decimal '
+ 'from list or tuple. The list or tuple '
+ 'should have exactly three elements.')
+ # process sign. The isinstance test rejects floats
+ if not (isinstance(value[0], int) and value[0] in (0,1)):
+ raise ValueError("Invalid sign. The first value in the tuple "
+ "should be an integer; either 0 for a "
+ "positive number or 1 for a negative number.")
+ self._sign = value[0]
+ if value[2] == 'F':
+ # infinity: value[1] is ignored
+ self._int = '0'
+ self._exp = value[2]
+ self._is_special = True
+ else:
+ # process and validate the digits in value[1]
+ digits = []
+ for digit in value[1]:
+ if isinstance(digit, int) and 0 <= digit <= 9:
+ # skip leading zeros
+ if digits or digit != 0:
+ digits.append(digit)
+ else:
+ raise ValueError("The second value in the tuple must "
+ "be composed of integers in the range "
+ "0 through 9.")
+ if value[2] in ('n', 'N'):
+ # NaN: digits form the diagnostic
+ self._int = ''.join(map(str, digits))
+ self._exp = value[2]
+ self._is_special = True
+ elif isinstance(value[2], int):
+ # finite number: digits give the coefficient
+ self._int = ''.join(map(str, digits or [0]))
+ self._exp = value[2]
+ self._is_special = False
+ else:
+ raise ValueError("The third value in the tuple must "
+ "be an integer, or one of the "
+ "strings 'F', 'n', 'N'.")
+ return self
+
+ if isinstance(value, float):
+ if context is None:
+ context = getcontext()
+ context._raise_error(FloatOperation,
+ "strict semantics for mixing floats and Decimals are "
+ "enabled")
+ value = Decimal.from_float(value)
+ self._exp = value._exp
+ self._sign = value._sign
+ self._int = value._int
+ self._is_special = value._is_special
+ return self
+
+ raise TypeError("Cannot convert %r to Decimal" % value)
+
+ @classmethod
+ def from_float(cls, f):
+ """Converts a float to a decimal number, exactly.
+
+ Note that Decimal.from_float(0.1) is not the same as Decimal('0.1').
+ Since 0.1 is not exactly representable in binary floating point, the
+ value is stored as the nearest representable value which is
+ 0x1.999999999999ap-4. The exact equivalent of the value in decimal
+ is 0.1000000000000000055511151231257827021181583404541015625.
+
+ >>> Decimal.from_float(0.1)
+ Decimal('0.1000000000000000055511151231257827021181583404541015625')
+ >>> Decimal.from_float(float('nan'))
+ Decimal('NaN')
+ >>> Decimal.from_float(float('inf'))
+ Decimal('Infinity')
+ >>> Decimal.from_float(-float('inf'))
+ Decimal('-Infinity')
+ >>> Decimal.from_float(-0.0)
+ Decimal('-0')
+
+ """
+ if isinstance(f, int): # handle integer inputs
+ return cls(f)
+ if not isinstance(f, float):
+ raise TypeError("argument must be int or float.")
+ if _math.isinf(f) or _math.isnan(f):
+ return cls(repr(f))
+ if _math.copysign(1.0, f) == 1.0:
+ sign = 0
+ else:
+ sign = 1
+ n, d = abs(f).as_integer_ratio()
+ k = d.bit_length() - 1
+ result = _dec_from_triple(sign, str(n*5**k), -k)
+ if cls is Decimal:
+ return result
+ else:
+ return cls(result)
+
+ def _isnan(self):
+ """Returns whether the number is not actually one.
+
+ 0 if a number
+ 1 if NaN
+ 2 if sNaN
+ """
+ if self._is_special:
+ exp = self._exp
+ if exp == 'n':
+ return 1
+ elif exp == 'N':
+ return 2
+ return 0
+
+ def _isinfinity(self):
+ """Returns whether the number is infinite
+
+ 0 if finite or not a number
+ 1 if +INF
+ -1 if -INF
+ """
+ if self._exp == 'F':
+ if self._sign:
+ return -1
+ return 1
+ return 0
+
+ def _check_nans(self, other=None, context=None):
+ """Returns whether the number is not actually one.
+
+ if self, other are sNaN, signal
+ if self, other are NaN return nan
+ return 0
+
+ Done before operations.
+ """
+
+ self_is_nan = self._isnan()
+ if other is None:
+ other_is_nan = False
+ else:
+ other_is_nan = other._isnan()
+
+ if self_is_nan or other_is_nan:
+ if context is None:
+ context = getcontext()
+
+ if self_is_nan == 2:
+ return context._raise_error(InvalidOperation, 'sNaN',
+ self)
+ if other_is_nan == 2:
+ return context._raise_error(InvalidOperation, 'sNaN',
+ other)
+ if self_is_nan:
+ return self._fix_nan(context)
+
+ return other._fix_nan(context)
+ return 0
+
+ def _compare_check_nans(self, other, context):
+ """Version of _check_nans used for the signaling comparisons
+ compare_signal, __le__, __lt__, __ge__, __gt__.
+
+ Signal InvalidOperation if either self or other is a (quiet
+ or signaling) NaN. Signaling NaNs take precedence over quiet
+ NaNs.
+
+ Return 0 if neither operand is a NaN.
+
+ """
+ if context is None:
+ context = getcontext()
+
+ if self._is_special or other._is_special:
+ if self.is_snan():
+ return context._raise_error(InvalidOperation,
+ 'comparison involving sNaN',
+ self)
+ elif other.is_snan():
+ return context._raise_error(InvalidOperation,
+ 'comparison involving sNaN',
+ other)
+ elif self.is_qnan():
+ return context._raise_error(InvalidOperation,
+ 'comparison involving NaN',
+ self)
+ elif other.is_qnan():
+ return context._raise_error(InvalidOperation,
+ 'comparison involving NaN',
+ other)
+ return 0
+
+ def __bool__(self):
+ """Return True if self is nonzero; otherwise return False.
+
+ NaNs and infinities are considered nonzero.
+ """
+ return self._is_special or self._int != '0'
+
+ def _cmp(self, other):
+ """Compare the two non-NaN decimal instances self and other.
+
+ Returns -1 if self < other, 0 if self == other and 1
+ if self > other. This routine is for internal use only."""
+
+ if self._is_special or other._is_special:
+ self_inf = self._isinfinity()
+ other_inf = other._isinfinity()
+ if self_inf == other_inf:
+ return 0
+ elif self_inf < other_inf:
+ return -1
+ else:
+ return 1
+
+ # check for zeros; Decimal('0') == Decimal('-0')
+ if not self:
+ if not other:
+ return 0
+ else:
+ return -((-1)**other._sign)
+ if not other:
+ return (-1)**self._sign
+
+ # If different signs, neg one is less
+ if other._sign < self._sign:
+ return -1
+ if self._sign < other._sign:
+ return 1
+
+ self_adjusted = self.adjusted()
+ other_adjusted = other.adjusted()
+ if self_adjusted == other_adjusted:
+ self_padded = self._int + '0'*(self._exp - other._exp)
+ other_padded = other._int + '0'*(other._exp - self._exp)
+ if self_padded == other_padded:
+ return 0
+ elif self_padded < other_padded:
+ return -(-1)**self._sign
+ else:
+ return (-1)**self._sign
+ elif self_adjusted > other_adjusted:
+ return (-1)**self._sign
+ else: # self_adjusted < other_adjusted
+ return -((-1)**self._sign)
+
+ # Note: The Decimal standard doesn't cover rich comparisons for
+ # Decimals. In particular, the specification is silent on the
+ # subject of what should happen for a comparison involving a NaN.
+ # We take the following approach:
+ #
+ # == comparisons involving a quiet NaN always return False
+ # != comparisons involving a quiet NaN always return True
+ # == or != comparisons involving a signaling NaN signal
+ # InvalidOperation, and return False or True as above if the
+ # InvalidOperation is not trapped.
+ # <, >, <= and >= comparisons involving a (quiet or signaling)
+ # NaN signal InvalidOperation, and return False if the
+ # InvalidOperation is not trapped.
+ #
+ # This behavior is designed to conform as closely as possible to
+ # that specified by IEEE 754.
+
+ def __eq__(self, other, context=None):
+ self, other = _convert_for_comparison(self, other, equality_op=True)
+ if other is NotImplemented:
+ return other
+ if self._check_nans(other, context):
+ return False
+ return self._cmp(other) == 0
+
+ def __lt__(self, other, context=None):
+ self, other = _convert_for_comparison(self, other)
+ if other is NotImplemented:
+ return other
+ ans = self._compare_check_nans(other, context)
+ if ans:
+ return False
+ return self._cmp(other) < 0
+
+ def __le__(self, other, context=None):
+ self, other = _convert_for_comparison(self, other)
+ if other is NotImplemented:
+ return other
+ ans = self._compare_check_nans(other, context)
+ if ans:
+ return False
+ return self._cmp(other) <= 0
+
+ def __gt__(self, other, context=None):
+ self, other = _convert_for_comparison(self, other)
+ if other is NotImplemented:
+ return other
+ ans = self._compare_check_nans(other, context)
+ if ans:
+ return False
+ return self._cmp(other) > 0
+
+ def __ge__(self, other, context=None):
+ self, other = _convert_for_comparison(self, other)
+ if other is NotImplemented:
+ return other
+ ans = self._compare_check_nans(other, context)
+ if ans:
+ return False
+ return self._cmp(other) >= 0
+
+ def compare(self, other, context=None):
+ """Compare self to other. Return a decimal value:
+
+ a or b is a NaN ==> Decimal('NaN')
+ a < b ==> Decimal('-1')
+ a == b ==> Decimal('0')
+ a > b ==> Decimal('1')
+ """
+ other = _convert_other(other, raiseit=True)
+
+ # Compare(NaN, NaN) = NaN
+ if (self._is_special or other and other._is_special):
+ ans = self._check_nans(other, context)
+ if ans:
+ return ans
+
+ return Decimal(self._cmp(other))
+
+ def __hash__(self):
+ """x.__hash__() <==> hash(x)"""
+
+ # In order to make sure that the hash of a Decimal instance
+ # agrees with the hash of a numerically equal integer, float
+ # or Fraction, we follow the rules for numeric hashes outlined
+ # in the documentation. (See library docs, 'Built-in Types').
+ if self._is_special:
+ if self.is_snan():
+ raise TypeError('Cannot hash a signaling NaN value.')
+ elif self.is_nan():
+ return _PyHASH_NAN
+ else:
+ if self._sign:
+ return -_PyHASH_INF
+ else:
+ return _PyHASH_INF
+
+ if self._exp >= 0:
+ exp_hash = pow(10, self._exp, _PyHASH_MODULUS)
+ else:
+ exp_hash = pow(_PyHASH_10INV, -self._exp, _PyHASH_MODULUS)
+ hash_ = int(self._int) * exp_hash % _PyHASH_MODULUS
+ ans = hash_ if self >= 0 else -hash_
+ return -2 if ans == -1 else ans
+
+ def as_tuple(self):
+ """Represents the number as a triple tuple.
+
+ To show the internals exactly as they are.
+ """
+ return DecimalTuple(self._sign, tuple(map(int, self._int)), self._exp)
+
+ def __repr__(self):
+ """Represents the number as an instance of Decimal."""
+ # Invariant: eval(repr(d)) == d
+ return "Decimal('%s')" % str(self)
+
+ def __str__(self, eng=False, context=None):
+ """Return string representation of the number in scientific notation.
+
+ Captures all of the information in the underlying representation.
+ """
+
+ sign = ['', '-'][self._sign]
+ if self._is_special:
+ if self._exp == 'F':
+ return sign + 'Infinity'
+ elif self._exp == 'n':
+ return sign + 'NaN' + self._int
+ else: # self._exp == 'N'
+ return sign + 'sNaN' + self._int
+
+ # number of digits of self._int to left of decimal point
+ leftdigits = self._exp + len(self._int)
+
+ # dotplace is number of digits of self._int to the left of the
+ # decimal point in the mantissa of the output string (that is,
+ # after adjusting the exponent)
+ if self._exp <= 0 and leftdigits > -6:
+ # no exponent required
+ dotplace = leftdigits
+ elif not eng:
+ # usual scientific notation: 1 digit on left of the point
+ dotplace = 1
+ elif self._int == '0':
+ # engineering notation, zero
+ dotplace = (leftdigits + 1) % 3 - 1
+ else:
+ # engineering notation, nonzero
+ dotplace = (leftdigits - 1) % 3 + 1
+
+ if dotplace <= 0:
+ intpart = '0'
+ fracpart = '.' + '0'*(-dotplace) + self._int
+ elif dotplace >= len(self._int):
+ intpart = self._int+'0'*(dotplace-len(self._int))
+ fracpart = ''
+ else:
+ intpart = self._int[:dotplace]
+ fracpart = '.' + self._int[dotplace:]
+ if leftdigits == dotplace:
+ exp = ''
+ else:
+ if context is None:
+ context = getcontext()
+ exp = ['e', 'E'][context.capitals] + "%+d" % (leftdigits-dotplace)
+
+ return sign + intpart + fracpart + exp
+
+ def to_eng_string(self, context=None):
+ """Convert to engineering-type string.
+
+ Engineering notation has an exponent which is a multiple of 3, so there
+ are up to 3 digits left of the decimal place.
+
+ Same rules for when in exponential and when as a value as in __str__.
+ """
+ return self.__str__(eng=True, context=context)
+
+ def __neg__(self, context=None):
+ """Returns a copy with the sign switched.
+
+ Rounds, if it has reason.
+ """
+ if self._is_special:
+ ans = self._check_nans(context=context)
+ if ans:
+ return ans
+
+ if context is None:
+ context = getcontext()
+
+ if not self and context.rounding != ROUND_FLOOR:
+ # -Decimal('0') is Decimal('0'), not Decimal('-0'), except
+ # in ROUND_FLOOR rounding mode.
+ ans = self.copy_abs()
+ else:
+ ans = self.copy_negate()
+
+ return ans._fix(context)
+
+ def __pos__(self, context=None):
+ """Returns a copy, unless it is a sNaN.
+
+ Rounds the number (if more then precision digits)
+ """
+ if self._is_special:
+ ans = self._check_nans(context=context)
+ if ans:
+ return ans
+
+ if context is None:
+ context = getcontext()
+
+ if not self and context.rounding != ROUND_FLOOR:
+ # + (-0) = 0, except in ROUND_FLOOR rounding mode.
+ ans = self.copy_abs()
+ else:
+ ans = Decimal(self)
+
+ return ans._fix(context)
+
+ def __abs__(self, round=True, context=None):
+ """Returns the absolute value of self.
+
+ If the keyword argument 'round' is false, do not round. The
+ expression self.__abs__(round=False) is equivalent to
+ self.copy_abs().
+ """
+ if not round:
+ return self.copy_abs()
+
+ if self._is_special:
+ ans = self._check_nans(context=context)
+ if ans:
+ return ans
+
+ if self._sign:
+ ans = self.__neg__(context=context)
+ else:
+ ans = self.__pos__(context=context)
+
+ return ans
+
+ def __add__(self, other, context=None):
+ """Returns self + other.
+
+ -INF + INF (or the reverse) cause InvalidOperation errors.
+ """
+ other = _convert_other(other)
+ if other is NotImplemented:
+ return other
+
+ if context is None:
+ context = getcontext()
+
+ if self._is_special or other._is_special:
+ ans = self._check_nans(other, context)
+ if ans:
+ return ans
+
+ if self._isinfinity():
+ # If both INF, same sign => same as both, opposite => error.
+ if self._sign != other._sign and other._isinfinity():
+ return context._raise_error(InvalidOperation, '-INF + INF')
+ return Decimal(self)
+ if other._isinfinity():
+ return Decimal(other) # Can't both be infinity here
+
+ exp = min(self._exp, other._exp)
+ negativezero = 0
+ if context.rounding == ROUND_FLOOR and self._sign != other._sign:
+ # If the answer is 0, the sign should be negative, in this case.
+ negativezero = 1
+
+ if not self and not other:
+ sign = min(self._sign, other._sign)
+ if negativezero:
+ sign = 1
+ ans = _dec_from_triple(sign, '0', exp)
+ ans = ans._fix(context)
+ return ans
+ if not self:
+ exp = max(exp, other._exp - context.prec-1)
+ ans = other._rescale(exp, context.rounding)
+ ans = ans._fix(context)
+ return ans
+ if not other:
+ exp = max(exp, self._exp - context.prec-1)
+ ans = self._rescale(exp, context.rounding)
+ ans = ans._fix(context)
+ return ans
+
+ op1 = _WorkRep(self)
+ op2 = _WorkRep(other)
+ op1, op2 = _normalize(op1, op2, context.prec)
+
+ result = _WorkRep()
+ if op1.sign != op2.sign:
+ # Equal and opposite
+ if op1.int == op2.int:
+ ans = _dec_from_triple(negativezero, '0', exp)
+ ans = ans._fix(context)
+ return ans
+ if op1.int < op2.int:
+ op1, op2 = op2, op1
+ # OK, now abs(op1) > abs(op2)
+ if op1.sign == 1:
+ result.sign = 1
+ op1.sign, op2.sign = op2.sign, op1.sign
+ else:
+ result.sign = 0
+ # So we know the sign, and op1 > 0.
+ elif op1.sign == 1:
+ result.sign = 1
+ op1.sign, op2.sign = (0, 0)
+ else:
+ result.sign = 0
+ # Now, op1 > abs(op2) > 0
+
+ if op2.sign == 0:
+ result.int = op1.int + op2.int
+ else:
+ result.int = op1.int - op2.int
+
+ result.exp = op1.exp
+ ans = Decimal(result)
+ ans = ans._fix(context)
+ return ans
+
+ __radd__ = __add__
+
+ def __sub__(self, other, context=None):
+ """Return self - other"""
+ other = _convert_other(other)
+ if other is NotImplemented:
+ return other
+
+ if self._is_special or other._is_special:
+ ans = self._check_nans(other, context=context)
+ if ans:
+ return ans
+
+ # self - other is computed as self + other.copy_negate()
+ return self.__add__(other.copy_negate(), context=context)
+
+ def __rsub__(self, other, context=None):
+ """Return other - self"""
+ other = _convert_other(other)
+ if other is NotImplemented:
+ return other
+
+ return other.__sub__(self, context=context)
+
+ def __mul__(self, other, context=None):
+ """Return self * other.
+
+ (+-) INF * 0 (or its reverse) raise InvalidOperation.
+ """
+ other = _convert_other(other)
+ if other is NotImplemented:
+ return other
+
+ if context is None:
+ context = getcontext()
+
+ resultsign = self._sign ^ other._sign
+
+ if self._is_special or other._is_special:
+ ans = self._check_nans(other, context)
+ if ans:
+ return ans
+
+ if self._isinfinity():
+ if not other:
+ return context._raise_error(InvalidOperation, '(+-)INF * 0')
+ return _SignedInfinity[resultsign]
+
+ if other._isinfinity():
+ if not self:
+ return context._raise_error(InvalidOperation, '0 * (+-)INF')
+ return _SignedInfinity[resultsign]
+
+ resultexp = self._exp + other._exp
+
+ # Special case for multiplying by zero
+ if not self or not other:
+ ans = _dec_from_triple(resultsign, '0', resultexp)
+ # Fixing in case the exponent is out of bounds
+ ans = ans._fix(context)
+ return ans
+
+ # Special case for multiplying by power of 10
+ if self._int == '1':
+ ans = _dec_from_triple(resultsign, other._int, resultexp)
+ ans = ans._fix(context)
+ return ans
+ if other._int == '1':
+ ans = _dec_from_triple(resultsign, self._int, resultexp)
+ ans = ans._fix(context)
+ return ans
+
+ op1 = _WorkRep(self)
+ op2 = _WorkRep(other)
+
+ ans = _dec_from_triple(resultsign, str(op1.int * op2.int), resultexp)
+ ans = ans._fix(context)
+
+ return ans
+ __rmul__ = __mul__
+
+ def __truediv__(self, other, context=None):
+ """Return self / other."""
+ other = _convert_other(other)
+ if other is NotImplemented:
+ return NotImplemented
+
+ if context is None:
+ context = getcontext()
+
+ sign = self._sign ^ other._sign
+
+ if self._is_special or other._is_special:
+ ans = self._check_nans(other, context)
+ if ans:
+ return ans
+
+ if self._isinfinity() and other._isinfinity():
+ return context._raise_error(InvalidOperation, '(+-)INF/(+-)INF')
+
+ if self._isinfinity():
+ return _SignedInfinity[sign]
+
+ if other._isinfinity():
+ context._raise_error(Clamped, 'Division by infinity')
+ return _dec_from_triple(sign, '0', context.Etiny())
+
+ # Special cases for zeroes
+ if not other:
+ if not self:
+ return context._raise_error(DivisionUndefined, '0 / 0')
+ return context._raise_error(DivisionByZero, 'x / 0', sign)
+
+ if not self:
+ exp = self._exp - other._exp
+ coeff = 0
+ else:
+ # OK, so neither = 0, INF or NaN
+ shift = len(other._int) - len(self._int) + context.prec + 1
+ exp = self._exp - other._exp - shift
+ op1 = _WorkRep(self)
+ op2 = _WorkRep(other)
+ if shift >= 0:
+ coeff, remainder = divmod(op1.int * 10**shift, op2.int)
+ else:
+ coeff, remainder = divmod(op1.int, op2.int * 10**-shift)
+ if remainder:
+ # result is not exact; adjust to ensure correct rounding
+ if coeff % 5 == 0:
+ coeff += 1
+ else:
+ # result is exact; get as close to ideal exponent as possible
+ ideal_exp = self._exp - other._exp
+ while exp < ideal_exp and coeff % 10 == 0:
+ coeff //= 10
+ exp += 1
+
+ ans = _dec_from_triple(sign, str(coeff), exp)
+ return ans._fix(context)
+
+ def _divide(self, other, context):
+ """Return (self // other, self % other), to context.prec precision.
+
+ Assumes that neither self nor other is a NaN, that self is not
+ infinite and that other is nonzero.
+ """
+ sign = self._sign ^ other._sign
+ if other._isinfinity():
+ ideal_exp = self._exp
+ else:
+ ideal_exp = min(self._exp, other._exp)
+
+ expdiff = self.adjusted() - other.adjusted()
+ if not self or other._isinfinity() or expdiff <= -2:
+ return (_dec_from_triple(sign, '0', 0),
+ self._rescale(ideal_exp, context.rounding))
+ if expdiff <= context.prec:
+ op1 = _WorkRep(self)
+ op2 = _WorkRep(other)
+ if op1.exp >= op2.exp:
+ op1.int *= 10**(op1.exp - op2.exp)
+ else:
+ op2.int *= 10**(op2.exp - op1.exp)
+ q, r = divmod(op1.int, op2.int)
+ if q < 10**context.prec:
+ return (_dec_from_triple(sign, str(q), 0),
+ _dec_from_triple(self._sign, str(r), ideal_exp))
+
+ # Here the quotient is too large to be representable
+ ans = context._raise_error(DivisionImpossible,
+ 'quotient too large in //, % or divmod')
+ return ans, ans
+
+ def __rtruediv__(self, other, context=None):
+ """Swaps self/other and returns __truediv__."""
+ other = _convert_other(other)
+ if other is NotImplemented:
+ return other
+ return other.__truediv__(self, context=context)
+
+ def __divmod__(self, other, context=None):
+ """
+ Return (self // other, self % other)
+ """
+ other = _convert_other(other)
+ if other is NotImplemented:
+ return other
+
+ if context is None:
+ context = getcontext()
+
+ ans = self._check_nans(other, context)
+ if ans:
+ return (ans, ans)
+
+ sign = self._sign ^ other._sign
+ if self._isinfinity():
+ if other._isinfinity():
+ ans = context._raise_error(InvalidOperation, 'divmod(INF, INF)')
+ return ans, ans
+ else:
+ return (_SignedInfinity[sign],
+ context._raise_error(InvalidOperation, 'INF % x'))
+
+ if not other:
+ if not self:
+ ans = context._raise_error(DivisionUndefined, 'divmod(0, 0)')
+ return ans, ans
+ else:
+ return (context._raise_error(DivisionByZero, 'x // 0', sign),
+ context._raise_error(InvalidOperation, 'x % 0'))
+
+ quotient, remainder = self._divide(other, context)
+ remainder = remainder._fix(context)
+ return quotient, remainder
+
+ def __rdivmod__(self, other, context=None):
+ """Swaps self/other and returns __divmod__."""
+ other = _convert_other(other)
+ if other is NotImplemented:
+ return other
+ return other.__divmod__(self, context=context)
+
+ def __mod__(self, other, context=None):
+ """
+ self % other
+ """
+ other = _convert_other(other)
+ if other is NotImplemented:
+ return other
+
+ if context is None:
+ context = getcontext()
+
+ ans = self._check_nans(other, context)
+ if ans:
+ return ans
+
+ if self._isinfinity():
+ return context._raise_error(InvalidOperation, 'INF % x')
+ elif not other:
+ if self:
+ return context._raise_error(InvalidOperation, 'x % 0')
+ else:
+ return context._raise_error(DivisionUndefined, '0 % 0')
+
+ remainder = self._divide(other, context)[1]
+ remainder = remainder._fix(context)
+ return remainder
+
+ def __rmod__(self, other, context=None):
+ """Swaps self/other and returns __mod__."""
+ other = _convert_other(other)
+ if other is NotImplemented:
+ return other
+ return other.__mod__(self, context=context)
+
+ def remainder_near(self, other, context=None):
+ """
+ Remainder nearest to 0- abs(remainder-near) <= other/2
+ """
+ if context is None:
+ context = getcontext()
+
+ other = _convert_other(other, raiseit=True)
+
+ ans = self._check_nans(other, context)
+ if ans:
+ return ans
+
+ # self == +/-infinity -> InvalidOperation
+ if self._isinfinity():
+ return context._raise_error(InvalidOperation,
+ 'remainder_near(infinity, x)')
+
+ # other == 0 -> either InvalidOperation or DivisionUndefined
+ if not other:
+ if self:
+ return context._raise_error(InvalidOperation,
+ 'remainder_near(x, 0)')
+ else:
+ return context._raise_error(DivisionUndefined,
+ 'remainder_near(0, 0)')
+
+ # other = +/-infinity -> remainder = self
+ if other._isinfinity():
+ ans = Decimal(self)
+ return ans._fix(context)
+
+ # self = 0 -> remainder = self, with ideal exponent
+ ideal_exponent = min(self._exp, other._exp)
+ if not self:
+ ans = _dec_from_triple(self._sign, '0', ideal_exponent)
+ return ans._fix(context)
+
+ # catch most cases of large or small quotient
+ expdiff = self.adjusted() - other.adjusted()
+ if expdiff >= context.prec + 1:
+ # expdiff >= prec+1 => abs(self/other) > 10**prec
+ return context._raise_error(DivisionImpossible)
+ if expdiff <= -2:
+ # expdiff <= -2 => abs(self/other) < 0.1
+ ans = self._rescale(ideal_exponent, context.rounding)
+ return ans._fix(context)
+
+ # adjust both arguments to have the same exponent, then divide
+ op1 = _WorkRep(self)
+ op2 = _WorkRep(other)
+ if op1.exp >= op2.exp:
+ op1.int *= 10**(op1.exp - op2.exp)
+ else:
+ op2.int *= 10**(op2.exp - op1.exp)
+ q, r = divmod(op1.int, op2.int)
+ # remainder is r*10**ideal_exponent; other is +/-op2.int *
+ # 10**ideal_exponent. Apply correction to ensure that
+ # abs(remainder) <= abs(other)/2
+ if 2*r + (q&1) > op2.int:
+ r -= op2.int
+ q += 1
+
+ if q >= 10**context.prec:
+ return context._raise_error(DivisionImpossible)
+
+ # result has same sign as self unless r is negative
+ sign = self._sign
+ if r < 0:
+ sign = 1-sign
+ r = -r
+
+ ans = _dec_from_triple(sign, str(r), ideal_exponent)
+ return ans._fix(context)
+
+ def __floordiv__(self, other, context=None):
+ """self // other"""
+ other = _convert_other(other)
+ if other is NotImplemented:
+ return other
+
+ if context is None:
+ context = getcontext()
+
+ ans = self._check_nans(other, context)
+ if ans:
+ return ans
+
+ if self._isinfinity():
+ if other._isinfinity():
+ return context._raise_error(InvalidOperation, 'INF // INF')
+ else:
+ return _SignedInfinity[self._sign ^ other._sign]
+
+ if not other:
+ if self:
+ return context._raise_error(DivisionByZero, 'x // 0',
+ self._sign ^ other._sign)
+ else:
+ return context._raise_error(DivisionUndefined, '0 // 0')
+
+ return self._divide(other, context)[0]
+
+ def __rfloordiv__(self, other, context=None):
+ """Swaps self/other and returns __floordiv__."""
+ other = _convert_other(other)
+ if other is NotImplemented:
+ return other
+ return other.__floordiv__(self, context=context)
+
+ def __float__(self):
+ """Float representation."""
+ if self._isnan():
+ if self.is_snan():
+ raise ValueError("Cannot convert signaling NaN to float")
+ s = "-nan" if self._sign else "nan"
+ else:
+ s = str(self)
+ return float(s)
+
+ def __int__(self):
+ """Converts self to an int, truncating if necessary."""
+ if self._is_special:
+ if self._isnan():
+ raise ValueError("Cannot convert NaN to integer")
+ elif self._isinfinity():
+ raise OverflowError("Cannot convert infinity to integer")
+ s = (-1)**self._sign
+ if self._exp >= 0:
+ return s*int(self._int)*10**self._exp
+ else:
+ return s*int(self._int[:self._exp] or '0')
+
+ __trunc__ = __int__
+
+ def real(self):
+ return self
+ real = property(real)
+
+ def imag(self):
+ return Decimal(0)
+ imag = property(imag)
+
+ def conjugate(self):
+ return self
+
+ def __complex__(self):
+ return complex(float(self))
+
+ def _fix_nan(self, context):
+ """Decapitate the payload of a NaN to fit the context"""
+ payload = self._int
+
+ # maximum length of payload is precision if clamp=0,
+ # precision-1 if clamp=1.
+ max_payload_len = context.prec - context.clamp
+ if len(payload) > max_payload_len:
+ payload = payload[len(payload)-max_payload_len:].lstrip('0')
+ return _dec_from_triple(self._sign, payload, self._exp, True)
+ return Decimal(self)
+
+ def _fix(self, context):
+ """Round if it is necessary to keep self within prec precision.
+
+ Rounds and fixes the exponent. Does not raise on a sNaN.
+
+ Arguments:
+ self - Decimal instance
+ context - context used.
+ """
+
+ if self._is_special:
+ if self._isnan():
+ # decapitate payload if necessary
+ return self._fix_nan(context)
+ else:
+ # self is +/-Infinity; return unaltered
+ return Decimal(self)
+
+ # if self is zero then exponent should be between Etiny and
+ # Emax if clamp==0, and between Etiny and Etop if clamp==1.
+ Etiny = context.Etiny()
+ Etop = context.Etop()
+ if not self:
+ exp_max = [context.Emax, Etop][context.clamp]
+ new_exp = min(max(self._exp, Etiny), exp_max)
+ if new_exp != self._exp:
+ context._raise_error(Clamped)
+ return _dec_from_triple(self._sign, '0', new_exp)
+ else:
+ return Decimal(self)
+
+ # exp_min is the smallest allowable exponent of the result,
+ # equal to max(self.adjusted()-context.prec+1, Etiny)
+ exp_min = len(self._int) + self._exp - context.prec
+ if exp_min > Etop:
+ # overflow: exp_min > Etop iff self.adjusted() > Emax
+ ans = context._raise_error(Overflow, 'above Emax', self._sign)
+ context._raise_error(Inexact)
+ context._raise_error(Rounded)
+ return ans
+
+ self_is_subnormal = exp_min < Etiny
+ if self_is_subnormal:
+ exp_min = Etiny
+
+ # round if self has too many digits
+ if self._exp < exp_min:
+ digits = len(self._int) + self._exp - exp_min
+ if digits < 0:
+ self = _dec_from_triple(self._sign, '1', exp_min-1)
+ digits = 0
+ rounding_method = self._pick_rounding_function[context.rounding]
+ changed = rounding_method(self, digits)
+ coeff = self._int[:digits] or '0'
+ if changed > 0:
+ coeff = str(int(coeff)+1)
+ if len(coeff) > context.prec:
+ coeff = coeff[:-1]
+ exp_min += 1
+
+ # check whether the rounding pushed the exponent out of range
+ if exp_min > Etop:
+ ans = context._raise_error(Overflow, 'above Emax', self._sign)
+ else:
+ ans = _dec_from_triple(self._sign, coeff, exp_min)
+
+ # raise the appropriate signals, taking care to respect
+ # the precedence described in the specification
+ if changed and self_is_subnormal:
+ context._raise_error(Underflow)
+ if self_is_subnormal:
+ context._raise_error(Subnormal)
+ if changed:
+ context._raise_error(Inexact)
+ context._raise_error(Rounded)
+ if not ans:
+ # raise Clamped on underflow to 0
+ context._raise_error(Clamped)
+ return ans
+
+ if self_is_subnormal:
+ context._raise_error(Subnormal)
+
+ # fold down if clamp == 1 and self has too few digits
+ if context.clamp == 1 and self._exp > Etop:
+ context._raise_error(Clamped)
+ self_padded = self._int + '0'*(self._exp - Etop)
+ return _dec_from_triple(self._sign, self_padded, Etop)
+
+ # here self was representable to begin with; return unchanged
+ return Decimal(self)
+
+ # for each of the rounding functions below:
+ # self is a finite, nonzero Decimal
+ # prec is an integer satisfying 0 <= prec < len(self._int)
+ #
+ # each function returns either -1, 0, or 1, as follows:
+ # 1 indicates that self should be rounded up (away from zero)
+ # 0 indicates that self should be truncated, and that all the
+ # digits to be truncated are zeros (so the value is unchanged)
+ # -1 indicates that there are nonzero digits to be truncated
+
+ def _round_down(self, prec):
+ """Also known as round-towards-0, truncate."""
+ if _all_zeros(self._int, prec):
+ return 0
+ else:
+ return -1
+
+ def _round_up(self, prec):
+ """Rounds away from 0."""
+ return -self._round_down(prec)
+
+ def _round_half_up(self, prec):
+ """Rounds 5 up (away from 0)"""
+ if self._int[prec] in '56789':
+ return 1
+ elif _all_zeros(self._int, prec):
+ return 0
+ else:
+ return -1
+
+ def _round_half_down(self, prec):
+ """Round 5 down"""
+ if _exact_half(self._int, prec):
+ return -1
+ else:
+ return self._round_half_up(prec)
+
+ def _round_half_even(self, prec):
+ """Round 5 to even, rest to nearest."""
+ if _exact_half(self._int, prec) and \
+ (prec == 0 or self._int[prec-1] in '02468'):
+ return -1
+ else:
+ return self._round_half_up(prec)
+
+ def _round_ceiling(self, prec):
+ """Rounds up (not away from 0 if negative.)"""
+ if self._sign:
+ return self._round_down(prec)
+ else:
+ return -self._round_down(prec)
+
+ def _round_floor(self, prec):
+ """Rounds down (not towards 0 if negative)"""
+ if not self._sign:
+ return self._round_down(prec)
+ else:
+ return -self._round_down(prec)
+
+ def _round_05up(self, prec):
+ """Round down unless digit prec-1 is 0 or 5."""
+ if prec and self._int[prec-1] not in '05':
+ return self._round_down(prec)
+ else:
+ return -self._round_down(prec)
+
+ _pick_rounding_function = dict(
+ ROUND_DOWN = _round_down,
+ ROUND_UP = _round_up,
+ ROUND_HALF_UP = _round_half_up,
+ ROUND_HALF_DOWN = _round_half_down,
+ ROUND_HALF_EVEN = _round_half_even,
+ ROUND_CEILING = _round_ceiling,
+ ROUND_FLOOR = _round_floor,
+ ROUND_05UP = _round_05up,
+ )
+
+ def __round__(self, n=None):
+ """Round self to the nearest integer, or to a given precision.
+
+ If only one argument is supplied, round a finite Decimal
+ instance self to the nearest integer. If self is infinite or
+ a NaN then a Python exception is raised. If self is finite
+ and lies exactly halfway between two integers then it is
+ rounded to the integer with even last digit.
+
+ >>> round(Decimal('123.456'))
+ 123
+ >>> round(Decimal('-456.789'))
+ -457
+ >>> round(Decimal('-3.0'))
+ -3
+ >>> round(Decimal('2.5'))
+ 2
+ >>> round(Decimal('3.5'))
+ 4
+ >>> round(Decimal('Inf'))
+ Traceback (most recent call last):
+ ...
+ OverflowError: cannot round an infinity
+ >>> round(Decimal('NaN'))
+ Traceback (most recent call last):
+ ...
+ ValueError: cannot round a NaN
+
+ If a second argument n is supplied, self is rounded to n
+ decimal places using the rounding mode for the current
+ context.
+
+ For an integer n, round(self, -n) is exactly equivalent to
+ self.quantize(Decimal('1En')).
+
+ >>> round(Decimal('123.456'), 0)
+ Decimal('123')
+ >>> round(Decimal('123.456'), 2)
+ Decimal('123.46')
+ >>> round(Decimal('123.456'), -2)
+ Decimal('1E+2')
+ >>> round(Decimal('-Infinity'), 37)
+ Decimal('NaN')
+ >>> round(Decimal('sNaN123'), 0)
+ Decimal('NaN123')
+
+ """
+ if n is not None:
+ # two-argument form: use the equivalent quantize call
+ if not isinstance(n, int):
+ raise TypeError('Second argument to round should be integral')
+ exp = _dec_from_triple(0, '1', -n)
+ return self.quantize(exp)
+
+ # one-argument form
+ if self._is_special:
+ if self.is_nan():
+ raise ValueError("cannot round a NaN")
+ else:
+ raise OverflowError("cannot round an infinity")
+ return int(self._rescale(0, ROUND_HALF_EVEN))
+
+ def __floor__(self):
+ """Return the floor of self, as an integer.
+
+ For a finite Decimal instance self, return the greatest
+ integer n such that n <= self. If self is infinite or a NaN
+ then a Python exception is raised.
+
+ """
+ if self._is_special:
+ if self.is_nan():
+ raise ValueError("cannot round a NaN")
+ else:
+ raise OverflowError("cannot round an infinity")
+ return int(self._rescale(0, ROUND_FLOOR))
+
+ def __ceil__(self):
+ """Return the ceiling of self, as an integer.
+
+ For a finite Decimal instance self, return the least integer n
+ such that n >= self. If self is infinite or a NaN then a
+ Python exception is raised.
+
+ """
+ if self._is_special:
+ if self.is_nan():
+ raise ValueError("cannot round a NaN")
+ else:
+ raise OverflowError("cannot round an infinity")
+ return int(self._rescale(0, ROUND_CEILING))
+
+ def fma(self, other, third, context=None):
+ """Fused multiply-add.
+
+ Returns self*other+third with no rounding of the intermediate
+ product self*other.
+
+ self and other are multiplied together, with no rounding of
+ the result. The third operand is then added to the result,
+ and a single final rounding is performed.
+ """
+
+ other = _convert_other(other, raiseit=True)
+ third = _convert_other(third, raiseit=True)
+
+ # compute product; raise InvalidOperation if either operand is
+ # a signaling NaN or if the product is zero times infinity.
+ if self._is_special or other._is_special:
+ if context is None:
+ context = getcontext()
+ if self._exp == 'N':
+ return context._raise_error(InvalidOperation, 'sNaN', self)
+ if other._exp == 'N':
+ return context._raise_error(InvalidOperation, 'sNaN', other)
+ if self._exp == 'n':
+ product = self
+ elif other._exp == 'n':
+ product = other
+ elif self._exp == 'F':
+ if not other:
+ return context._raise_error(InvalidOperation,
+ 'INF * 0 in fma')
+ product = _SignedInfinity[self._sign ^ other._sign]
+ elif other._exp == 'F':
+ if not self:
+ return context._raise_error(InvalidOperation,
+ '0 * INF in fma')
+ product = _SignedInfinity[self._sign ^ other._sign]
+ else:
+ product = _dec_from_triple(self._sign ^ other._sign,
+ str(int(self._int) * int(other._int)),
+ self._exp + other._exp)
+
+ return product.__add__(third, context)
+
+ def _power_modulo(self, other, modulo, context=None):
+ """Three argument version of __pow__"""
+
+ other = _convert_other(other)
+ if other is NotImplemented:
+ return other
+ modulo = _convert_other(modulo)
+ if modulo is NotImplemented:
+ return modulo
+
+ if context is None:
+ context = getcontext()
+
+ # deal with NaNs: if there are any sNaNs then first one wins,
+ # (i.e. behaviour for NaNs is identical to that of fma)
+ self_is_nan = self._isnan()
+ other_is_nan = other._isnan()
+ modulo_is_nan = modulo._isnan()
+ if self_is_nan or other_is_nan or modulo_is_nan:
+ if self_is_nan == 2:
+ return context._raise_error(InvalidOperation, 'sNaN',
+ self)
+ if other_is_nan == 2:
+ return context._raise_error(InvalidOperation, 'sNaN',
+ other)
+ if modulo_is_nan == 2:
+ return context._raise_error(InvalidOperation, 'sNaN',
+ modulo)
+ if self_is_nan:
+ return self._fix_nan(context)
+ if other_is_nan:
+ return other._fix_nan(context)
+ return modulo._fix_nan(context)
+
+ # check inputs: we apply same restrictions as Python's pow()
+ if not (self._isinteger() and
+ other._isinteger() and
+ modulo._isinteger()):
+ return context._raise_error(InvalidOperation,
+ 'pow() 3rd argument not allowed '
+ 'unless all arguments are integers')
+ if other < 0:
+ return context._raise_error(InvalidOperation,
+ 'pow() 2nd argument cannot be '
+ 'negative when 3rd argument specified')
+ if not modulo:
+ return context._raise_error(InvalidOperation,
+ 'pow() 3rd argument cannot be 0')
+
+ # additional restriction for decimal: the modulus must be less
+ # than 10**prec in absolute value
+ if modulo.adjusted() >= context.prec:
+ return context._raise_error(InvalidOperation,
+ 'insufficient precision: pow() 3rd '
+ 'argument must not have more than '
+ 'precision digits')
+
+ # define 0**0 == NaN, for consistency with two-argument pow
+ # (even though it hurts!)
+ if not other and not self:
+ return context._raise_error(InvalidOperation,
+ 'at least one of pow() 1st argument '
+ 'and 2nd argument must be nonzero ;'
+ '0**0 is not defined')
+
+ # compute sign of result
+ if other._iseven():
+ sign = 0
+ else:
+ sign = self._sign
+
+ # convert modulo to a Python integer, and self and other to
+ # Decimal integers (i.e. force their exponents to be >= 0)
+ modulo = abs(int(modulo))
+ base = _WorkRep(self.to_integral_value())
+ exponent = _WorkRep(other.to_integral_value())
+
+ # compute result using integer pow()
+ base = (base.int % modulo * pow(10, base.exp, modulo)) % modulo
+ for i in range(exponent.exp):
+ base = pow(base, 10, modulo)
+ base = pow(base, exponent.int, modulo)
+
+ return _dec_from_triple(sign, str(base), 0)
+
+ def _power_exact(self, other, p):
+ """Attempt to compute self**other exactly.
+
+ Given Decimals self and other and an integer p, attempt to
+ compute an exact result for the power self**other, with p
+ digits of precision. Return None if self**other is not
+ exactly representable in p digits.
+
+ Assumes that elimination of special cases has already been
+ performed: self and other must both be nonspecial; self must
+ be positive and not numerically equal to 1; other must be
+ nonzero. For efficiency, other._exp should not be too large,
+ so that 10**abs(other._exp) is a feasible calculation."""
+
+ # In the comments below, we write x for the value of self and y for the
+ # value of other. Write x = xc*10**xe and abs(y) = yc*10**ye, with xc
+ # and yc positive integers not divisible by 10.
+
+ # The main purpose of this method is to identify the *failure*
+ # of x**y to be exactly representable with as little effort as
+ # possible. So we look for cheap and easy tests that
+ # eliminate the possibility of x**y being exact. Only if all
+ # these tests are passed do we go on to actually compute x**y.
+
+ # Here's the main idea. Express y as a rational number m/n, with m and
+ # n relatively prime and n>0. Then for x**y to be exactly
+ # representable (at *any* precision), xc must be the nth power of a
+ # positive integer and xe must be divisible by n. If y is negative
+ # then additionally xc must be a power of either 2 or 5, hence a power
+ # of 2**n or 5**n.
+ #
+ # There's a limit to how small |y| can be: if y=m/n as above
+ # then:
+ #
+ # (1) if xc != 1 then for the result to be representable we
+ # need xc**(1/n) >= 2, and hence also xc**|y| >= 2. So
+ # if |y| <= 1/nbits(xc) then xc < 2**nbits(xc) <=
+ # 2**(1/|y|), hence xc**|y| < 2 and the result is not
+ # representable.
+ #
+ # (2) if xe != 0, |xe|*(1/n) >= 1, so |xe|*|y| >= 1. Hence if
+ # |y| < 1/|xe| then the result is not representable.
+ #
+ # Note that since x is not equal to 1, at least one of (1) and
+ # (2) must apply. Now |y| < 1/nbits(xc) iff |yc|*nbits(xc) <
+ # 10**-ye iff len(str(|yc|*nbits(xc)) <= -ye.
+ #
+ # There's also a limit to how large y can be, at least if it's
+ # positive: the normalized result will have coefficient xc**y,
+ # so if it's representable then xc**y < 10**p, and y <
+ # p/log10(xc). Hence if y*log10(xc) >= p then the result is
+ # not exactly representable.
+
+ # if len(str(abs(yc*xe)) <= -ye then abs(yc*xe) < 10**-ye,
+ # so |y| < 1/xe and the result is not representable.
+ # Similarly, len(str(abs(yc)*xc_bits)) <= -ye implies |y|
+ # < 1/nbits(xc).
+
+ x = _WorkRep(self)
+ xc, xe = x.int, x.exp
+ while xc % 10 == 0:
+ xc //= 10
+ xe += 1
+
+ y = _WorkRep(other)
+ yc, ye = y.int, y.exp
+ while yc % 10 == 0:
+ yc //= 10
+ ye += 1
+
+ # case where xc == 1: result is 10**(xe*y), with xe*y
+ # required to be an integer
+ if xc == 1:
+ xe *= yc
+ # result is now 10**(xe * 10**ye); xe * 10**ye must be integral
+ while xe % 10 == 0:
+ xe //= 10
+ ye += 1
+ if ye < 0:
+ return None
+ exponent = xe * 10**ye
+ if y.sign == 1:
+ exponent = -exponent
+ # if other is a nonnegative integer, use ideal exponent
+ if other._isinteger() and other._sign == 0:
+ ideal_exponent = self._exp*int(other)
+ zeros = min(exponent-ideal_exponent, p-1)
+ else:
+ zeros = 0
+ return _dec_from_triple(0, '1' + '0'*zeros, exponent-zeros)
+
+ # case where y is negative: xc must be either a power
+ # of 2 or a power of 5.
+ if y.sign == 1:
+ last_digit = xc % 10
+ if last_digit in (2,4,6,8):
+ # quick test for power of 2
+ if xc & -xc != xc:
+ return None
+ # now xc is a power of 2; e is its exponent
+ e = _nbits(xc)-1
+
+ # We now have:
+ #
+ # x = 2**e * 10**xe, e > 0, and y < 0.
+ #
+ # The exact result is:
+ #
+ # x**y = 5**(-e*y) * 10**(e*y + xe*y)
+ #
+ # provided that both e*y and xe*y are integers. Note that if
+ # 5**(-e*y) >= 10**p, then the result can't be expressed
+ # exactly with p digits of precision.
+ #
+ # Using the above, we can guard against large values of ye.
+ # 93/65 is an upper bound for log(10)/log(5), so if
+ #
+ # ye >= len(str(93*p//65))
+ #
+ # then
+ #
+ # -e*y >= -y >= 10**ye > 93*p/65 > p*log(10)/log(5),
+ #
+ # so 5**(-e*y) >= 10**p, and the coefficient of the result
+ # can't be expressed in p digits.
+
+ # emax >= largest e such that 5**e < 10**p.
+ emax = p*93//65
+ if ye >= len(str(emax)):
+ return None
+
+ # Find -e*y and -xe*y; both must be integers
+ e = _decimal_lshift_exact(e * yc, ye)
+ xe = _decimal_lshift_exact(xe * yc, ye)
+ if e is None or xe is None:
+ return None
+
+ if e > emax:
+ return None
+ xc = 5**e
+
+ elif last_digit == 5:
+ # e >= log_5(xc) if xc is a power of 5; we have
+ # equality all the way up to xc=5**2658
+ e = _nbits(xc)*28//65
+ xc, remainder = divmod(5**e, xc)
+ if remainder:
+ return None
+ while xc % 5 == 0:
+ xc //= 5
+ e -= 1
+
+ # Guard against large values of ye, using the same logic as in
+ # the 'xc is a power of 2' branch. 10/3 is an upper bound for
+ # log(10)/log(2).
+ emax = p*10//3
+ if ye >= len(str(emax)):
+ return None
+
+ e = _decimal_lshift_exact(e * yc, ye)
+ xe = _decimal_lshift_exact(xe * yc, ye)
+ if e is None or xe is None:
+ return None
+
+ if e > emax:
+ return None
+ xc = 2**e
+ else:
+ return None
+
+ if xc >= 10**p:
+ return None
+ xe = -e-xe
+ return _dec_from_triple(0, str(xc), xe)
+
+ # now y is positive; find m and n such that y = m/n
+ if ye >= 0:
+ m, n = yc*10**ye, 1
+ else:
+ if xe != 0 and len(str(abs(yc*xe))) <= -ye:
+ return None
+ xc_bits = _nbits(xc)
+ if xc != 1 and len(str(abs(yc)*xc_bits)) <= -ye:
+ return None
+ m, n = yc, 10**(-ye)
+ while m % 2 == n % 2 == 0:
+ m //= 2
+ n //= 2
+ while m % 5 == n % 5 == 0:
+ m //= 5
+ n //= 5
+
+ # compute nth root of xc*10**xe
+ if n > 1:
+ # if 1 < xc < 2**n then xc isn't an nth power
+ if xc != 1 and xc_bits <= n:
+ return None
+
+ xe, rem = divmod(xe, n)
+ if rem != 0:
+ return None
+
+ # compute nth root of xc using Newton's method
+ a = 1 << -(-_nbits(xc)//n) # initial estimate
+ while True:
+ q, r = divmod(xc, a**(n-1))
+ if a <= q:
+ break
+ else:
+ a = (a*(n-1) + q)//n
+ if not (a == q and r == 0):
+ return None
+ xc = a
+
+ # now xc*10**xe is the nth root of the original xc*10**xe
+ # compute mth power of xc*10**xe
+
+ # if m > p*100//_log10_lb(xc) then m > p/log10(xc), hence xc**m >
+ # 10**p and the result is not representable.
+ if xc > 1 and m > p*100//_log10_lb(xc):
+ return None
+ xc = xc**m
+ xe *= m
+ if xc > 10**p:
+ return None
+
+ # by this point the result *is* exactly representable
+ # adjust the exponent to get as close as possible to the ideal
+ # exponent, if necessary
+ str_xc = str(xc)
+ if other._isinteger() and other._sign == 0:
+ ideal_exponent = self._exp*int(other)
+ zeros = min(xe-ideal_exponent, p-len(str_xc))
+ else:
+ zeros = 0
+ return _dec_from_triple(0, str_xc+'0'*zeros, xe-zeros)
+
+ def __pow__(self, other, modulo=None, context=None):
+ """Return self ** other [ % modulo].
+
+ With two arguments, compute self**other.
+
+ With three arguments, compute (self**other) % modulo. For the
+ three argument form, the following restrictions on the
+ arguments hold:
+
+ - all three arguments must be integral
+ - other must be nonnegative
+ - either self or other (or both) must be nonzero
+ - modulo must be nonzero and must have at most p digits,
+ where p is the context precision.
+
+ If any of these restrictions is violated the InvalidOperation
+ flag is raised.
+
+ The result of pow(self, other, modulo) is identical to the
+ result that would be obtained by computing (self**other) %
+ modulo with unbounded precision, but is computed more
+ efficiently. It is always exact.
+ """
+
+ if modulo is not None:
+ return self._power_modulo(other, modulo, context)
+
+ other = _convert_other(other)
+ if other is NotImplemented:
+ return other
+
+ if context is None:
+ context = getcontext()
+
+ # either argument is a NaN => result is NaN
+ ans = self._check_nans(other, context)
+ if ans:
+ return ans
+
+ # 0**0 = NaN (!), x**0 = 1 for nonzero x (including +/-Infinity)
+ if not other:
+ if not self:
+ return context._raise_error(InvalidOperation, '0 ** 0')
+ else:
+ return _One
+
+ # result has sign 1 iff self._sign is 1 and other is an odd integer
+ result_sign = 0
+ if self._sign == 1:
+ if other._isinteger():
+ if not other._iseven():
+ result_sign = 1
+ else:
+ # -ve**noninteger = NaN
+ # (-0)**noninteger = 0**noninteger
+ if self:
+ return context._raise_error(InvalidOperation,
+ 'x ** y with x negative and y not an integer')
+ # negate self, without doing any unwanted rounding
+ self = self.copy_negate()
+
+ # 0**(+ve or Inf)= 0; 0**(-ve or -Inf) = Infinity
+ if not self:
+ if other._sign == 0:
+ return _dec_from_triple(result_sign, '0', 0)
+ else:
+ return _SignedInfinity[result_sign]
+
+ # Inf**(+ve or Inf) = Inf; Inf**(-ve or -Inf) = 0
+ if self._isinfinity():
+ if other._sign == 0:
+ return _SignedInfinity[result_sign]
+ else:
+ return _dec_from_triple(result_sign, '0', 0)
+
+ # 1**other = 1, but the choice of exponent and the flags
+ # depend on the exponent of self, and on whether other is a
+ # positive integer, a negative integer, or neither
+ if self == _One:
+ if other._isinteger():
+ # exp = max(self._exp*max(int(other), 0),
+ # 1-context.prec) but evaluating int(other) directly
+ # is dangerous until we know other is small (other
+ # could be 1e999999999)
+ if other._sign == 1:
+ multiplier = 0
+ elif other > context.prec:
+ multiplier = context.prec
+ else:
+ multiplier = int(other)
+
+ exp = self._exp * multiplier
+ if exp < 1-context.prec:
+ exp = 1-context.prec
+ context._raise_error(Rounded)
+ else:
+ context._raise_error(Inexact)
+ context._raise_error(Rounded)
+ exp = 1-context.prec
+
+ return _dec_from_triple(result_sign, '1'+'0'*-exp, exp)
+
+ # compute adjusted exponent of self
+ self_adj = self.adjusted()
+
+ # self ** infinity is infinity if self > 1, 0 if self < 1
+ # self ** -infinity is infinity if self < 1, 0 if self > 1
+ if other._isinfinity():
+ if (other._sign == 0) == (self_adj < 0):
+ return _dec_from_triple(result_sign, '0', 0)
+ else:
+ return _SignedInfinity[result_sign]
+
+ # from here on, the result always goes through the call
+ # to _fix at the end of this function.
+ ans = None
+ exact = False
+
+ # crude test to catch cases of extreme overflow/underflow. If
+ # log10(self)*other >= 10**bound and bound >= len(str(Emax))
+ # then 10**bound >= 10**len(str(Emax)) >= Emax+1 and hence
+ # self**other >= 10**(Emax+1), so overflow occurs. The test
+ # for underflow is similar.
+ bound = self._log10_exp_bound() + other.adjusted()
+ if (self_adj >= 0) == (other._sign == 0):
+ # self > 1 and other +ve, or self < 1 and other -ve
+ # possibility of overflow
+ if bound >= len(str(context.Emax)):
+ ans = _dec_from_triple(result_sign, '1', context.Emax+1)
+ else:
+ # self > 1 and other -ve, or self < 1 and other +ve
+ # possibility of underflow to 0
+ Etiny = context.Etiny()
+ if bound >= len(str(-Etiny)):
+ ans = _dec_from_triple(result_sign, '1', Etiny-1)
+
+ # try for an exact result with precision +1
+ if ans is None:
+ ans = self._power_exact(other, context.prec + 1)
+ if ans is not None:
+ if result_sign == 1:
+ ans = _dec_from_triple(1, ans._int, ans._exp)
+ exact = True
+
+ # usual case: inexact result, x**y computed directly as exp(y*log(x))
+ if ans is None:
+ p = context.prec
+ x = _WorkRep(self)
+ xc, xe = x.int, x.exp
+ y = _WorkRep(other)
+ yc, ye = y.int, y.exp
+ if y.sign == 1:
+ yc = -yc
+
+ # compute correctly rounded result: start with precision +3,
+ # then increase precision until result is unambiguously roundable
+ extra = 3
+ while True:
+ coeff, exp = _dpower(xc, xe, yc, ye, p+extra)
+ if coeff % (5*10**(len(str(coeff))-p-1)):
+ break
+ extra += 3
+
+ ans = _dec_from_triple(result_sign, str(coeff), exp)
+
+ # unlike exp, ln and log10, the power function respects the
+ # rounding mode; no need to switch to ROUND_HALF_EVEN here
+
+ # There's a difficulty here when 'other' is not an integer and
+ # the result is exact. In this case, the specification
+ # requires that the Inexact flag be raised (in spite of
+ # exactness), but since the result is exact _fix won't do this
+ # for us. (Correspondingly, the Underflow signal should also
+ # be raised for subnormal results.) We can't directly raise
+ # these signals either before or after calling _fix, since
+ # that would violate the precedence for signals. So we wrap
+ # the ._fix call in a temporary context, and reraise
+ # afterwards.
+ if exact and not other._isinteger():
+ # pad with zeros up to length context.prec+1 if necessary; this
+ # ensures that the Rounded signal will be raised.
+ if len(ans._int) <= context.prec:
+ expdiff = context.prec + 1 - len(ans._int)
+ ans = _dec_from_triple(ans._sign, ans._int+'0'*expdiff,
+ ans._exp-expdiff)
+
+ # create a copy of the current context, with cleared flags/traps
+ newcontext = context.copy()
+ newcontext.clear_flags()
+ for exception in _signals:
+ newcontext.traps[exception] = 0
+
+ # round in the new context
+ ans = ans._fix(newcontext)
+
+ # raise Inexact, and if necessary, Underflow
+ newcontext._raise_error(Inexact)
+ if newcontext.flags[Subnormal]:
+ newcontext._raise_error(Underflow)
+
+ # propagate signals to the original context; _fix could
+ # have raised any of Overflow, Underflow, Subnormal,
+ # Inexact, Rounded, Clamped. Overflow needs the correct
+ # arguments. Note that the order of the exceptions is
+ # important here.
+ if newcontext.flags[Overflow]:
+ context._raise_error(Overflow, 'above Emax', ans._sign)
+ for exception in Underflow, Subnormal, Inexact, Rounded, Clamped:
+ if newcontext.flags[exception]:
+ context._raise_error(exception)
+
+ else:
+ ans = ans._fix(context)
+
+ return ans
+
+ def __rpow__(self, other, context=None):
+ """Swaps self/other and returns __pow__."""
+ other = _convert_other(other)
+ if other is NotImplemented:
+ return other
+ return other.__pow__(self, context=context)
+
+ def normalize(self, context=None):
+ """Normalize- strip trailing 0s, change anything equal to 0 to 0e0"""
+
+ if context is None:
+ context = getcontext()
+
+ if self._is_special:
+ ans = self._check_nans(context=context)
+ if ans:
+ return ans
+
+ dup = self._fix(context)
+ if dup._isinfinity():
+ return dup
+
+ if not dup:
+ return _dec_from_triple(dup._sign, '0', 0)
+ exp_max = [context.Emax, context.Etop()][context.clamp]
+ end = len(dup._int)
+ exp = dup._exp
+ while dup._int[end-1] == '0' and exp < exp_max:
+ exp += 1
+ end -= 1
+ return _dec_from_triple(dup._sign, dup._int[:end], exp)
+
+ def quantize(self, exp, rounding=None, context=None):
+ """Quantize self so its exponent is the same as that of exp.
+
+ Similar to self._rescale(exp._exp) but with error checking.
+ """
+ exp = _convert_other(exp, raiseit=True)
+
+ if context is None:
+ context = getcontext()
+ if rounding is None:
+ rounding = context.rounding
+
+ if self._is_special or exp._is_special:
+ ans = self._check_nans(exp, context)
+ if ans:
+ return ans
+
+ if exp._isinfinity() or self._isinfinity():
+ if exp._isinfinity() and self._isinfinity():
+ return Decimal(self) # if both are inf, it is OK
+ return context._raise_error(InvalidOperation,
+ 'quantize with one INF')
+
+ # exp._exp should be between Etiny and Emax
+ if not (context.Etiny() <= exp._exp <= context.Emax):
+ return context._raise_error(InvalidOperation,
+ 'target exponent out of bounds in quantize')
+
+ if not self:
+ ans = _dec_from_triple(self._sign, '0', exp._exp)
+ return ans._fix(context)
+
+ self_adjusted = self.adjusted()
+ if self_adjusted > context.Emax:
+ return context._raise_error(InvalidOperation,
+ 'exponent of quantize result too large for current context')
+ if self_adjusted - exp._exp + 1 > context.prec:
+ return context._raise_error(InvalidOperation,
+ 'quantize result has too many digits for current context')
+
+ ans = self._rescale(exp._exp, rounding)
+ if ans.adjusted() > context.Emax:
+ return context._raise_error(InvalidOperation,
+ 'exponent of quantize result too large for current context')
+ if len(ans._int) > context.prec:
+ return context._raise_error(InvalidOperation,
+ 'quantize result has too many digits for current context')
+
+ # raise appropriate flags
+ if ans and ans.adjusted() < context.Emin:
+ context._raise_error(Subnormal)
+ if ans._exp > self._exp:
+ if ans != self:
+ context._raise_error(Inexact)
+ context._raise_error(Rounded)
+
+ # call to fix takes care of any necessary folddown, and
+ # signals Clamped if necessary
+ ans = ans._fix(context)
+ return ans
+
+ def same_quantum(self, other, context=None):
+ """Return True if self and other have the same exponent; otherwise
+ return False.
+
+ If either operand is a special value, the following rules are used:
+ * return True if both operands are infinities
+ * return True if both operands are NaNs
+ * otherwise, return False.
+ """
+ other = _convert_other(other, raiseit=True)
+ if self._is_special or other._is_special:
+ return (self.is_nan() and other.is_nan() or
+ self.is_infinite() and other.is_infinite())
+ return self._exp == other._exp
+
+ def _rescale(self, exp, rounding):
+ """Rescale self so that the exponent is exp, either by padding with zeros
+ or by truncating digits, using the given rounding mode.
+
+ Specials are returned without change. This operation is
+ quiet: it raises no flags, and uses no information from the
+ context.
+
+ exp = exp to scale to (an integer)
+ rounding = rounding mode
+ """
+ if self._is_special:
+ return Decimal(self)
+ if not self:
+ return _dec_from_triple(self._sign, '0', exp)
+
+ if self._exp >= exp:
+ # pad answer with zeros if necessary
+ return _dec_from_triple(self._sign,
+ self._int + '0'*(self._exp - exp), exp)
+
+ # too many digits; round and lose data. If self.adjusted() <
+ # exp-1, replace self by 10**(exp-1) before rounding
+ digits = len(self._int) + self._exp - exp
+ if digits < 0:
+ self = _dec_from_triple(self._sign, '1', exp-1)
+ digits = 0
+ this_function = self._pick_rounding_function[rounding]
+ changed = this_function(self, digits)
+ coeff = self._int[:digits] or '0'
+ if changed == 1:
+ coeff = str(int(coeff)+1)
+ return _dec_from_triple(self._sign, coeff, exp)
+
+ def _round(self, places, rounding):
+ """Round a nonzero, nonspecial Decimal to a fixed number of
+ significant figures, using the given rounding mode.
+
+ Infinities, NaNs and zeros are returned unaltered.
+
+ This operation is quiet: it raises no flags, and uses no
+ information from the context.
+
+ """
+ if places <= 0:
+ raise ValueError("argument should be at least 1 in _round")
+ if self._is_special or not self:
+ return Decimal(self)
+ ans = self._rescale(self.adjusted()+1-places, rounding)
+ # it can happen that the rescale alters the adjusted exponent;
+ # for example when rounding 99.97 to 3 significant figures.
+ # When this happens we end up with an extra 0 at the end of
+ # the number; a second rescale fixes this.
+ if ans.adjusted() != self.adjusted():
+ ans = ans._rescale(ans.adjusted()+1-places, rounding)
+ return ans
+
+ def to_integral_exact(self, rounding=None, context=None):
+ """Rounds to a nearby integer.
+
+ If no rounding mode is specified, take the rounding mode from
+ the context. This method raises the Rounded and Inexact flags
+ when appropriate.
+
+ See also: to_integral_value, which does exactly the same as
+ this method except that it doesn't raise Inexact or Rounded.
+ """
+ if self._is_special:
+ ans = self._check_nans(context=context)
+ if ans:
+ return ans
+ return Decimal(self)
+ if self._exp >= 0:
+ return Decimal(self)
+ if not self:
+ return _dec_from_triple(self._sign, '0', 0)
+ if context is None:
+ context = getcontext()
+ if rounding is None:
+ rounding = context.rounding
+ ans = self._rescale(0, rounding)
+ if ans != self:
+ context._raise_error(Inexact)
+ context._raise_error(Rounded)
+ return ans
+
+ def to_integral_value(self, rounding=None, context=None):
+ """Rounds to the nearest integer, without raising inexact, rounded."""
+ if context is None:
+ context = getcontext()
+ if rounding is None:
+ rounding = context.rounding
+ if self._is_special:
+ ans = self._check_nans(context=context)
+ if ans:
+ return ans
+ return Decimal(self)
+ if self._exp >= 0:
+ return Decimal(self)
+ else:
+ return self._rescale(0, rounding)
+
+ # the method name changed, but we provide also the old one, for compatibility
+ to_integral = to_integral_value
+
+ def sqrt(self, context=None):
+ """Return the square root of self."""
+ if context is None:
+ context = getcontext()
+
+ if self._is_special:
+ ans = self._check_nans(context=context)
+ if ans:
+ return ans
+
+ if self._isinfinity() and self._sign == 0:
+ return Decimal(self)
+
+ if not self:
+ # exponent = self._exp // 2. sqrt(-0) = -0
+ ans = _dec_from_triple(self._sign, '0', self._exp // 2)
+ return ans._fix(context)
+
+ if self._sign == 1:
+ return context._raise_error(InvalidOperation, 'sqrt(-x), x > 0')
+
+ # At this point self represents a positive number. Let p be
+ # the desired precision and express self in the form c*100**e
+ # with c a positive real number and e an integer, c and e
+ # being chosen so that 100**(p-1) <= c < 100**p. Then the
+ # (exact) square root of self is sqrt(c)*10**e, and 10**(p-1)
+ # <= sqrt(c) < 10**p, so the closest representable Decimal at
+ # precision p is n*10**e where n = round_half_even(sqrt(c)),
+ # the closest integer to sqrt(c) with the even integer chosen
+ # in the case of a tie.
+ #
+ # To ensure correct rounding in all cases, we use the
+ # following trick: we compute the square root to an extra
+ # place (precision p+1 instead of precision p), rounding down.
+ # Then, if the result is inexact and its last digit is 0 or 5,
+ # we increase the last digit to 1 or 6 respectively; if it's
+ # exact we leave the last digit alone. Now the final round to
+ # p places (or fewer in the case of underflow) will round
+ # correctly and raise the appropriate flags.
+
+ # use an extra digit of precision
+ prec = context.prec+1
+
+ # write argument in the form c*100**e where e = self._exp//2
+ # is the 'ideal' exponent, to be used if the square root is
+ # exactly representable. l is the number of 'digits' of c in
+ # base 100, so that 100**(l-1) <= c < 100**l.
+ op = _WorkRep(self)
+ e = op.exp >> 1
+ if op.exp & 1:
+ c = op.int * 10
+ l = (len(self._int) >> 1) + 1
+ else:
+ c = op.int
+ l = len(self._int)+1 >> 1
+
+ # rescale so that c has exactly prec base 100 'digits'
+ shift = prec-l
+ if shift >= 0:
+ c *= 100**shift
+ exact = True
+ else:
+ c, remainder = divmod(c, 100**-shift)
+ exact = not remainder
+ e -= shift
+
+ # find n = floor(sqrt(c)) using Newton's method
+ n = 10**prec
+ while True:
+ q = c//n
+ if n <= q:
+ break
+ else:
+ n = n + q >> 1
+ exact = exact and n*n == c
+
+ if exact:
+ # result is exact; rescale to use ideal exponent e
+ if shift >= 0:
+ # assert n % 10**shift == 0
+ n //= 10**shift
+ else:
+ n *= 10**-shift
+ e += shift
+ else:
+ # result is not exact; fix last digit as described above
+ if n % 5 == 0:
+ n += 1
+
+ ans = _dec_from_triple(0, str(n), e)
+
+ # round, and fit to current context
+ context = context._shallow_copy()
+ rounding = context._set_rounding(ROUND_HALF_EVEN)
+ ans = ans._fix(context)
+ context.rounding = rounding
+
+ return ans
+
+ def max(self, other, context=None):
+ """Returns the larger value.
+
+ Like max(self, other) except if one is not a number, returns
+ NaN (and signals if one is sNaN). Also rounds.
+ """
+ other = _convert_other(other, raiseit=True)
+
+ if context is None:
+ context = getcontext()
+
+ if self._is_special or other._is_special:
+ # If one operand is a quiet NaN and the other is number, then the
+ # number is always returned
+ sn = self._isnan()
+ on = other._isnan()
+ if sn or on:
+ if on == 1 and sn == 0:
+ return self._fix(context)
+ if sn == 1 and on == 0:
+ return other._fix(context)
+ return self._check_nans(other, context)
+
+ c = self._cmp(other)
+ if c == 0:
+ # If both operands are finite and equal in numerical value
+ # then an ordering is applied:
+ #
+ # If the signs differ then max returns the operand with the
+ # positive sign and min returns the operand with the negative sign
+ #
+ # If the signs are the same then the exponent is used to select
+ # the result. This is exactly the ordering used in compare_total.
+ c = self.compare_total(other)
+
+ if c == -1:
+ ans = other
+ else:
+ ans = self
+
+ return ans._fix(context)
+
+ def min(self, other, context=None):
+ """Returns the smaller value.
+
+ Like min(self, other) except if one is not a number, returns
+ NaN (and signals if one is sNaN). Also rounds.
+ """
+ other = _convert_other(other, raiseit=True)
+
+ if context is None:
+ context = getcontext()
+
+ if self._is_special or other._is_special:
+ # If one operand is a quiet NaN and the other is number, then the
+ # number is always returned
+ sn = self._isnan()
+ on = other._isnan()
+ if sn or on:
+ if on == 1 and sn == 0:
+ return self._fix(context)
+ if sn == 1 and on == 0:
+ return other._fix(context)
+ return self._check_nans(other, context)
+
+ c = self._cmp(other)
+ if c == 0:
+ c = self.compare_total(other)
+
+ if c == -1:
+ ans = self
+ else:
+ ans = other
+
+ return ans._fix(context)
+
+ def _isinteger(self):
+ """Returns whether self is an integer"""
+ if self._is_special:
+ return False
+ if self._exp >= 0:
+ return True
+ rest = self._int[self._exp:]
+ return rest == '0'*len(rest)
+
+ def _iseven(self):
+ """Returns True if self is even. Assumes self is an integer."""
+ if not self or self._exp > 0:
+ return True
+ return self._int[-1+self._exp] in '02468'
+
+ def adjusted(self):
+ """Return the adjusted exponent of self"""
+ try:
+ return self._exp + len(self._int) - 1
+ # If NaN or Infinity, self._exp is string
+ except TypeError:
+ return 0
+
+ def canonical(self):
+ """Returns the same Decimal object.
+
+ As we do not have different encodings for the same number, the
+ received object already is in its canonical form.
+ """
+ return self
+
+ def compare_signal(self, other, context=None):
+ """Compares self to the other operand numerically.
+
+ It's pretty much like compare(), but all NaNs signal, with signaling
+ NaNs taking precedence over quiet NaNs.
+ """
+ other = _convert_other(other, raiseit = True)
+ ans = self._compare_check_nans(other, context)
+ if ans:
+ return ans
+ return self.compare(other, context=context)
+
+ def compare_total(self, other, context=None):
+ """Compares self to other using the abstract representations.
+
+ This is not like the standard compare, which use their numerical
+ value. Note that a total ordering is defined for all possible abstract
+ representations.
+ """
+ other = _convert_other(other, raiseit=True)
+
+ # if one is negative and the other is positive, it's easy
+ if self._sign and not other._sign:
+ return _NegativeOne
+ if not self._sign and other._sign:
+ return _One
+ sign = self._sign
+
+ # let's handle both NaN types
+ self_nan = self._isnan()
+ other_nan = other._isnan()
+ if self_nan or other_nan:
+ if self_nan == other_nan:
+ # compare payloads as though they're integers
+ self_key = len(self._int), self._int
+ other_key = len(other._int), other._int
+ if self_key < other_key:
+ if sign:
+ return _One
+ else:
+ return _NegativeOne
+ if self_key > other_key:
+ if sign:
+ return _NegativeOne
+ else:
+ return _One
+ return _Zero
+
+ if sign:
+ if self_nan == 1:
+ return _NegativeOne
+ if other_nan == 1:
+ return _One
+ if self_nan == 2:
+ return _NegativeOne
+ if other_nan == 2:
+ return _One
+ else:
+ if self_nan == 1:
+ return _One
+ if other_nan == 1:
+ return _NegativeOne
+ if self_nan == 2:
+ return _One
+ if other_nan == 2:
+ return _NegativeOne
+
+ if self < other:
+ return _NegativeOne
+ if self > other:
+ return _One
+
+ if self._exp < other._exp:
+ if sign:
+ return _One
+ else:
+ return _NegativeOne
+ if self._exp > other._exp:
+ if sign:
+ return _NegativeOne
+ else:
+ return _One
+ return _Zero
+
+
+ def compare_total_mag(self, other, context=None):
+ """Compares self to other using abstract repr., ignoring sign.
+
+ Like compare_total, but with operand's sign ignored and assumed to be 0.
+ """
+ other = _convert_other(other, raiseit=True)
+
+ s = self.copy_abs()
+ o = other.copy_abs()
+ return s.compare_total(o)
+
+ def copy_abs(self):
+ """Returns a copy with the sign set to 0. """
+ return _dec_from_triple(0, self._int, self._exp, self._is_special)
+
+ def copy_negate(self):
+ """Returns a copy with the sign inverted."""
+ if self._sign:
+ return _dec_from_triple(0, self._int, self._exp, self._is_special)
+ else:
+ return _dec_from_triple(1, self._int, self._exp, self._is_special)
+
+ def copy_sign(self, other, context=None):
+ """Returns self with the sign of other."""
+ other = _convert_other(other, raiseit=True)
+ return _dec_from_triple(other._sign, self._int,
+ self._exp, self._is_special)
+
+ def exp(self, context=None):
+ """Returns e ** self."""
+
+ if context is None:
+ context = getcontext()
+
+ # exp(NaN) = NaN
+ ans = self._check_nans(context=context)
+ if ans:
+ return ans
+
+ # exp(-Infinity) = 0
+ if self._isinfinity() == -1:
+ return _Zero
+
+ # exp(0) = 1
+ if not self:
+ return _One
+
+ # exp(Infinity) = Infinity
+ if self._isinfinity() == 1:
+ return Decimal(self)
+
+ # the result is now guaranteed to be inexact (the true
+ # mathematical result is transcendental). There's no need to
+ # raise Rounded and Inexact here---they'll always be raised as
+ # a result of the call to _fix.
+ p = context.prec
+ adj = self.adjusted()
+
+ # we only need to do any computation for quite a small range
+ # of adjusted exponents---for example, -29 <= adj <= 10 for
+ # the default context. For smaller exponent the result is
+ # indistinguishable from 1 at the given precision, while for
+ # larger exponent the result either overflows or underflows.
+ if self._sign == 0 and adj > len(str((context.Emax+1)*3)):
+ # overflow
+ ans = _dec_from_triple(0, '1', context.Emax+1)
+ elif self._sign == 1 and adj > len(str((-context.Etiny()+1)*3)):
+ # underflow to 0
+ ans = _dec_from_triple(0, '1', context.Etiny()-1)
+ elif self._sign == 0 and adj < -p:
+ # p+1 digits; final round will raise correct flags
+ ans = _dec_from_triple(0, '1' + '0'*(p-1) + '1', -p)
+ elif self._sign == 1 and adj < -p-1:
+ # p+1 digits; final round will raise correct flags
+ ans = _dec_from_triple(0, '9'*(p+1), -p-1)
+ # general case
+ else:
+ op = _WorkRep(self)
+ c, e = op.int, op.exp
+ if op.sign == 1:
+ c = -c
+
+ # compute correctly rounded result: increase precision by
+ # 3 digits at a time until we get an unambiguously
+ # roundable result
+ extra = 3
+ while True:
+ coeff, exp = _dexp(c, e, p+extra)
+ if coeff % (5*10**(len(str(coeff))-p-1)):
+ break
+ extra += 3
+
+ ans = _dec_from_triple(0, str(coeff), exp)
+
+ # at this stage, ans should round correctly with *any*
+ # rounding mode, not just with ROUND_HALF_EVEN
+ context = context._shallow_copy()
+ rounding = context._set_rounding(ROUND_HALF_EVEN)
+ ans = ans._fix(context)
+ context.rounding = rounding
+
+ return ans
+
+ def is_canonical(self):
+ """Return True if self is canonical; otherwise return False.
+
+ Currently, the encoding of a Decimal instance is always
+ canonical, so this method returns True for any Decimal.
+ """
+ return True
+
+ def is_finite(self):
+ """Return True if self is finite; otherwise return False.
+
+ A Decimal instance is considered finite if it is neither
+ infinite nor a NaN.
+ """
+ return not self._is_special
+
+ def is_infinite(self):
+ """Return True if self is infinite; otherwise return False."""
+ return self._exp == 'F'
+
+ def is_nan(self):
+ """Return True if self is a qNaN or sNaN; otherwise return False."""
+ return self._exp in ('n', 'N')
+
+ def is_normal(self, context=None):
+ """Return True if self is a normal number; otherwise return False."""
+ if self._is_special or not self:
+ return False
+ if context is None:
+ context = getcontext()
+ return context.Emin <= self.adjusted()
+
+ def is_qnan(self):
+ """Return True if self is a quiet NaN; otherwise return False."""
+ return self._exp == 'n'
+
+ def is_signed(self):
+ """Return True if self is negative; otherwise return False."""
+ return self._sign == 1
+
+ def is_snan(self):
+ """Return True if self is a signaling NaN; otherwise return False."""
+ return self._exp == 'N'
+
+ def is_subnormal(self, context=None):
+ """Return True if self is subnormal; otherwise return False."""
+ if self._is_special or not self:
+ return False
+ if context is None:
+ context = getcontext()
+ return self.adjusted() < context.Emin
+
+ def is_zero(self):
+ """Return True if self is a zero; otherwise return False."""
+ return not self._is_special and self._int == '0'
+
+ def _ln_exp_bound(self):
+ """Compute a lower bound for the adjusted exponent of self.ln().
+ In other words, compute r such that self.ln() >= 10**r. Assumes
+ that self is finite and positive and that self != 1.
+ """
+
+ # for 0.1 <= x <= 10 we use the inequalities 1-1/x <= ln(x) <= x-1
+ adj = self._exp + len(self._int) - 1
+ if adj >= 1:
+ # argument >= 10; we use 23/10 = 2.3 as a lower bound for ln(10)
+ return len(str(adj*23//10)) - 1
+ if adj <= -2:
+ # argument <= 0.1
+ return len(str((-1-adj)*23//10)) - 1
+ op = _WorkRep(self)
+ c, e = op.int, op.exp
+ if adj == 0:
+ # 1 < self < 10
+ num = str(c-10**-e)
+ den = str(c)
+ return len(num) - len(den) - (num < den)
+ # adj == -1, 0.1 <= self < 1
+ return e + len(str(10**-e - c)) - 1
+
+
+ def ln(self, context=None):
+ """Returns the natural (base e) logarithm of self."""
+
+ if context is None:
+ context = getcontext()
+
+ # ln(NaN) = NaN
+ ans = self._check_nans(context=context)
+ if ans:
+ return ans
+
+ # ln(0.0) == -Infinity
+ if not self:
+ return _NegativeInfinity
+
+ # ln(Infinity) = Infinity
+ if self._isinfinity() == 1:
+ return _Infinity
+
+ # ln(1.0) == 0.0
+ if self == _One:
+ return _Zero
+
+ # ln(negative) raises InvalidOperation
+ if self._sign == 1:
+ return context._raise_error(InvalidOperation,
+ 'ln of a negative value')
+
+ # result is irrational, so necessarily inexact
+ op = _WorkRep(self)
+ c, e = op.int, op.exp
+ p = context.prec
+
+ # correctly rounded result: repeatedly increase precision by 3
+ # until we get an unambiguously roundable result
+ places = p - self._ln_exp_bound() + 2 # at least p+3 places
+ while True:
+ coeff = _dlog(c, e, places)
+ # assert len(str(abs(coeff)))-p >= 1
+ if coeff % (5*10**(len(str(abs(coeff)))-p-1)):
+ break
+ places += 3
+ ans = _dec_from_triple(int(coeff<0), str(abs(coeff)), -places)
+
+ context = context._shallow_copy()
+ rounding = context._set_rounding(ROUND_HALF_EVEN)
+ ans = ans._fix(context)
+ context.rounding = rounding
+ return ans
+
+ def _log10_exp_bound(self):
+ """Compute a lower bound for the adjusted exponent of self.log10().
+ In other words, find r such that self.log10() >= 10**r.
+ Assumes that self is finite and positive and that self != 1.
+ """
+
+ # For x >= 10 or x < 0.1 we only need a bound on the integer
+ # part of log10(self), and this comes directly from the
+ # exponent of x. For 0.1 <= x <= 10 we use the inequalities
+ # 1-1/x <= log(x) <= x-1. If x > 1 we have |log10(x)| >
+ # (1-1/x)/2.31 > 0. If x < 1 then |log10(x)| > (1-x)/2.31 > 0
+
+ adj = self._exp + len(self._int) - 1
+ if adj >= 1:
+ # self >= 10
+ return len(str(adj))-1
+ if adj <= -2:
+ # self < 0.1
+ return len(str(-1-adj))-1
+ op = _WorkRep(self)
+ c, e = op.int, op.exp
+ if adj == 0:
+ # 1 < self < 10
+ num = str(c-10**-e)
+ den = str(231*c)
+ return len(num) - len(den) - (num < den) + 2
+ # adj == -1, 0.1 <= self < 1
+ num = str(10**-e-c)
+ return len(num) + e - (num < "231") - 1
+
+ def log10(self, context=None):
+ """Returns the base 10 logarithm of self."""
+
+ if context is None:
+ context = getcontext()
+
+ # log10(NaN) = NaN
+ ans = self._check_nans(context=context)
+ if ans:
+ return ans
+
+ # log10(0.0) == -Infinity
+ if not self:
+ return _NegativeInfinity
+
+ # log10(Infinity) = Infinity
+ if self._isinfinity() == 1:
+ return _Infinity
+
+ # log10(negative or -Infinity) raises InvalidOperation
+ if self._sign == 1:
+ return context._raise_error(InvalidOperation,
+ 'log10 of a negative value')
+
+ # log10(10**n) = n
+ if self._int[0] == '1' and self._int[1:] == '0'*(len(self._int) - 1):
+ # answer may need rounding
+ ans = Decimal(self._exp + len(self._int) - 1)
+ else:
+ # result is irrational, so necessarily inexact
+ op = _WorkRep(self)
+ c, e = op.int, op.exp
+ p = context.prec
+
+ # correctly rounded result: repeatedly increase precision
+ # until result is unambiguously roundable
+ places = p-self._log10_exp_bound()+2
+ while True:
+ coeff = _dlog10(c, e, places)
+ # assert len(str(abs(coeff)))-p >= 1
+ if coeff % (5*10**(len(str(abs(coeff)))-p-1)):
+ break
+ places += 3
+ ans = _dec_from_triple(int(coeff<0), str(abs(coeff)), -places)
+
+ context = context._shallow_copy()
+ rounding = context._set_rounding(ROUND_HALF_EVEN)
+ ans = ans._fix(context)
+ context.rounding = rounding
+ return ans
+
+ def logb(self, context=None):
+ """ Returns the exponent of the magnitude of self's MSD.
+
+ The result is the integer which is the exponent of the magnitude
+ of the most significant digit of self (as though it were truncated
+ to a single digit while maintaining the value of that digit and
+ without limiting the resulting exponent).
+ """
+ # logb(NaN) = NaN
+ ans = self._check_nans(context=context)
+ if ans:
+ return ans
+
+ if context is None:
+ context = getcontext()
+
+ # logb(+/-Inf) = +Inf
+ if self._isinfinity():
+ return _Infinity
+
+ # logb(0) = -Inf, DivisionByZero
+ if not self:
+ return context._raise_error(DivisionByZero, 'logb(0)', 1)
+
+ # otherwise, simply return the adjusted exponent of self, as a
+ # Decimal. Note that no attempt is made to fit the result
+ # into the current context.
+ ans = Decimal(self.adjusted())
+ return ans._fix(context)
+
+ def _islogical(self):
+ """Return True if self is a logical operand.
+
+ For being logical, it must be a finite number with a sign of 0,
+ an exponent of 0, and a coefficient whose digits must all be
+ either 0 or 1.
+ """
+ if self._sign != 0 or self._exp != 0:
+ return False
+ for dig in self._int:
+ if dig not in '01':
+ return False
+ return True
+
+ def _fill_logical(self, context, opa, opb):
+ dif = context.prec - len(opa)
+ if dif > 0:
+ opa = '0'*dif + opa
+ elif dif < 0:
+ opa = opa[-context.prec:]
+ dif = context.prec - len(opb)
+ if dif > 0:
+ opb = '0'*dif + opb
+ elif dif < 0:
+ opb = opb[-context.prec:]
+ return opa, opb
+
+ def logical_and(self, other, context=None):
+ """Applies an 'and' operation between self and other's digits."""
+ if context is None:
+ context = getcontext()
+
+ other = _convert_other(other, raiseit=True)
+
+ if not self._islogical() or not other._islogical():
+ return context._raise_error(InvalidOperation)
+
+ # fill to context.prec
+ (opa, opb) = self._fill_logical(context, self._int, other._int)
+
+ # make the operation, and clean starting zeroes
+ result = "".join([str(int(a)&int(b)) for a,b in zip(opa,opb)])
+ return _dec_from_triple(0, result.lstrip('0') or '0', 0)
+
+ def logical_invert(self, context=None):
+ """Invert all its digits."""
+ if context is None:
+ context = getcontext()
+ return self.logical_xor(_dec_from_triple(0,'1'*context.prec,0),
+ context)
+
+ def logical_or(self, other, context=None):
+ """Applies an 'or' operation between self and other's digits."""
+ if context is None:
+ context = getcontext()
+
+ other = _convert_other(other, raiseit=True)
+
+ if not self._islogical() or not other._islogical():
+ return context._raise_error(InvalidOperation)
+
+ # fill to context.prec
+ (opa, opb) = self._fill_logical(context, self._int, other._int)
+
+ # make the operation, and clean starting zeroes
+ result = "".join([str(int(a)|int(b)) for a,b in zip(opa,opb)])
+ return _dec_from_triple(0, result.lstrip('0') or '0', 0)
+
+ def logical_xor(self, other, context=None):
+ """Applies an 'xor' operation between self and other's digits."""
+ if context is None:
+ context = getcontext()
+
+ other = _convert_other(other, raiseit=True)
+
+ if not self._islogical() or not other._islogical():
+ return context._raise_error(InvalidOperation)
+
+ # fill to context.prec
+ (opa, opb) = self._fill_logical(context, self._int, other._int)
+
+ # make the operation, and clean starting zeroes
+ result = "".join([str(int(a)^int(b)) for a,b in zip(opa,opb)])
+ return _dec_from_triple(0, result.lstrip('0') or '0', 0)
+
+ def max_mag(self, other, context=None):
+ """Compares the values numerically with their sign ignored."""
+ other = _convert_other(other, raiseit=True)
+
+ if context is None:
+ context = getcontext()
+
+ if self._is_special or other._is_special:
+ # If one operand is a quiet NaN and the other is number, then the
+ # number is always returned
+ sn = self._isnan()
+ on = other._isnan()
+ if sn or on:
+ if on == 1 and sn == 0:
+ return self._fix(context)
+ if sn == 1 and on == 0:
+ return other._fix(context)
+ return self._check_nans(other, context)
+
+ c = self.copy_abs()._cmp(other.copy_abs())
+ if c == 0:
+ c = self.compare_total(other)
+
+ if c == -1:
+ ans = other
+ else:
+ ans = self
+
+ return ans._fix(context)
+
+ def min_mag(self, other, context=None):
+ """Compares the values numerically with their sign ignored."""
+ other = _convert_other(other, raiseit=True)
+
+ if context is None:
+ context = getcontext()
+
+ if self._is_special or other._is_special:
+ # If one operand is a quiet NaN and the other is number, then the
+ # number is always returned
+ sn = self._isnan()
+ on = other._isnan()
+ if sn or on:
+ if on == 1 and sn == 0:
+ return self._fix(context)
+ if sn == 1 and on == 0:
+ return other._fix(context)
+ return self._check_nans(other, context)
+
+ c = self.copy_abs()._cmp(other.copy_abs())
+ if c == 0:
+ c = self.compare_total(other)
+
+ if c == -1:
+ ans = self
+ else:
+ ans = other
+
+ return ans._fix(context)
+
+ def next_minus(self, context=None):
+ """Returns the largest representable number smaller than itself."""
+ if context is None:
+ context = getcontext()
+
+ ans = self._check_nans(context=context)
+ if ans:
+ return ans
+
+ if self._isinfinity() == -1:
+ return _NegativeInfinity
+ if self._isinfinity() == 1:
+ return _dec_from_triple(0, '9'*context.prec, context.Etop())
+
+ context = context.copy()
+ context._set_rounding(ROUND_FLOOR)
+ context._ignore_all_flags()
+ new_self = self._fix(context)
+ if new_self != self:
+ return new_self
+ return self.__sub__(_dec_from_triple(0, '1', context.Etiny()-1),
+ context)
+
+ def next_plus(self, context=None):
+ """Returns the smallest representable number larger than itself."""
+ if context is None:
+ context = getcontext()
+
+ ans = self._check_nans(context=context)
+ if ans:
+ return ans
+
+ if self._isinfinity() == 1:
+ return _Infinity
+ if self._isinfinity() == -1:
+ return _dec_from_triple(1, '9'*context.prec, context.Etop())
+
+ context = context.copy()
+ context._set_rounding(ROUND_CEILING)
+ context._ignore_all_flags()
+ new_self = self._fix(context)
+ if new_self != self:
+ return new_self
+ return self.__add__(_dec_from_triple(0, '1', context.Etiny()-1),
+ context)
+
+ def next_toward(self, other, context=None):
+ """Returns the number closest to self, in the direction towards other.
+
+ The result is the closest representable number to self
+ (excluding self) that is in the direction towards other,
+ unless both have the same value. If the two operands are
+ numerically equal, then the result is a copy of self with the
+ sign set to be the same as the sign of other.
+ """
+ other = _convert_other(other, raiseit=True)
+
+ if context is None:
+ context = getcontext()
+
+ ans = self._check_nans(other, context)
+ if ans:
+ return ans
+
+ comparison = self._cmp(other)
+ if comparison == 0:
+ return self.copy_sign(other)
+
+ if comparison == -1:
+ ans = self.next_plus(context)
+ else: # comparison == 1
+ ans = self.next_minus(context)
+
+ # decide which flags to raise using value of ans
+ if ans._isinfinity():
+ context._raise_error(Overflow,
+ 'Infinite result from next_toward',
+ ans._sign)
+ context._raise_error(Inexact)
+ context._raise_error(Rounded)
+ elif ans.adjusted() < context.Emin:
+ context._raise_error(Underflow)
+ context._raise_error(Subnormal)
+ context._raise_error(Inexact)
+ context._raise_error(Rounded)
+ # if precision == 1 then we don't raise Clamped for a
+ # result 0E-Etiny.
+ if not ans:
+ context._raise_error(Clamped)
+
+ return ans
+
+ def number_class(self, context=None):
+ """Returns an indication of the class of self.
+
+ The class is one of the following strings:
+ sNaN
+ NaN
+ -Infinity
+ -Normal
+ -Subnormal
+ -Zero
+ +Zero
+ +Subnormal
+ +Normal
+ +Infinity
+ """
+ if self.is_snan():
+ return "sNaN"
+ if self.is_qnan():
+ return "NaN"
+ inf = self._isinfinity()
+ if inf == 1:
+ return "+Infinity"
+ if inf == -1:
+ return "-Infinity"
+ if self.is_zero():
+ if self._sign:
+ return "-Zero"
+ else:
+ return "+Zero"
+ if context is None:
+ context = getcontext()
+ if self.is_subnormal(context=context):
+ if self._sign:
+ return "-Subnormal"
+ else:
+ return "+Subnormal"
+ # just a normal, regular, boring number, :)
+ if self._sign:
+ return "-Normal"
+ else:
+ return "+Normal"
+
+ def radix(self):
+ """Just returns 10, as this is Decimal, :)"""
+ return Decimal(10)
+
+ def rotate(self, other, context=None):
+ """Returns a rotated copy of self, value-of-other times."""
+ if context is None:
+ context = getcontext()
+
+ other = _convert_other(other, raiseit=True)
+
+ ans = self._check_nans(other, context)
+ if ans:
+ return ans
+
+ if other._exp != 0:
+ return context._raise_error(InvalidOperation)
+ if not (-context.prec <= int(other) <= context.prec):
+ return context._raise_error(InvalidOperation)
+
+ if self._isinfinity():
+ return Decimal(self)
+
+ # get values, pad if necessary
+ torot = int(other)
+ rotdig = self._int
+ topad = context.prec - len(rotdig)
+ if topad > 0:
+ rotdig = '0'*topad + rotdig
+ elif topad < 0:
+ rotdig = rotdig[-topad:]
+
+ # let's rotate!
+ rotated = rotdig[torot:] + rotdig[:torot]
+ return _dec_from_triple(self._sign,
+ rotated.lstrip('0') or '0', self._exp)
+
+ def scaleb(self, other, context=None):
+ """Returns self operand after adding the second value to its exp."""
+ if context is None:
+ context = getcontext()
+
+ other = _convert_other(other, raiseit=True)
+
+ ans = self._check_nans(other, context)
+ if ans:
+ return ans
+
+ if other._exp != 0:
+ return context._raise_error(InvalidOperation)
+ liminf = -2 * (context.Emax + context.prec)
+ limsup = 2 * (context.Emax + context.prec)
+ if not (liminf <= int(other) <= limsup):
+ return context._raise_error(InvalidOperation)
+
+ if self._isinfinity():
+ return Decimal(self)
+
+ d = _dec_from_triple(self._sign, self._int, self._exp + int(other))
+ d = d._fix(context)
+ return d
+
+ def shift(self, other, context=None):
+ """Returns a shifted copy of self, value-of-other times."""
+ if context is None:
+ context = getcontext()
+
+ other = _convert_other(other, raiseit=True)
+
+ ans = self._check_nans(other, context)
+ if ans:
+ return ans
+
+ if other._exp != 0:
+ return context._raise_error(InvalidOperation)
+ if not (-context.prec <= int(other) <= context.prec):
+ return context._raise_error(InvalidOperation)
+
+ if self._isinfinity():
+ return Decimal(self)
+
+ # get values, pad if necessary
+ torot = int(other)
+ rotdig = self._int
+ topad = context.prec - len(rotdig)
+ if topad > 0:
+ rotdig = '0'*topad + rotdig
+ elif topad < 0:
+ rotdig = rotdig[-topad:]
+
+ # let's shift!
+ if torot < 0:
+ shifted = rotdig[:torot]
+ else:
+ shifted = rotdig + '0'*torot
+ shifted = shifted[-context.prec:]
+
+ return _dec_from_triple(self._sign,
+ shifted.lstrip('0') or '0', self._exp)
+
+ # Support for pickling, copy, and deepcopy
+ def __reduce__(self):
+ return (self.__class__, (str(self),))
+
+ def __copy__(self):
+ if type(self) is Decimal:
+ return self # I'm immutable; therefore I am my own clone
+ return self.__class__(str(self))
+
+ def __deepcopy__(self, memo):
+ if type(self) is Decimal:
+ return self # My components are also immutable
+ return self.__class__(str(self))
+
+ # PEP 3101 support. the _localeconv keyword argument should be
+ # considered private: it's provided for ease of testing only.
+ def __format__(self, specifier, context=None, _localeconv=None):
+ """Format a Decimal instance according to the given specifier.
+
+ The specifier should be a standard format specifier, with the
+ form described in PEP 3101. Formatting types 'e', 'E', 'f',
+ 'F', 'g', 'G', 'n' and '%' are supported. If the formatting
+ type is omitted it defaults to 'g' or 'G', depending on the
+ value of context.capitals.
+ """
+
+ # Note: PEP 3101 says that if the type is not present then
+ # there should be at least one digit after the decimal point.
+ # We take the liberty of ignoring this requirement for
+ # Decimal---it's presumably there to make sure that
+ # format(float, '') behaves similarly to str(float).
+ if context is None:
+ context = getcontext()
+
+ spec = _parse_format_specifier(specifier, _localeconv=_localeconv)
+
+ # special values don't care about the type or precision
+ if self._is_special:
+ sign = _format_sign(self._sign, spec)
+ body = str(self.copy_abs())
+ if spec['type'] == '%':
+ body += '%'
+ return _format_align(sign, body, spec)
+
+ # a type of None defaults to 'g' or 'G', depending on context
+ if spec['type'] is None:
+ spec['type'] = ['g', 'G'][context.capitals]
+
+ # if type is '%', adjust exponent of self accordingly
+ if spec['type'] == '%':
+ self = _dec_from_triple(self._sign, self._int, self._exp+2)
+
+ # round if necessary, taking rounding mode from the context
+ rounding = context.rounding
+ precision = spec['precision']
+ if precision is not None:
+ if spec['type'] in 'eE':
+ self = self._round(precision+1, rounding)
+ elif spec['type'] in 'fF%':
+ self = self._rescale(-precision, rounding)
+ elif spec['type'] in 'gG' and len(self._int) > precision:
+ self = self._round(precision, rounding)
+ # special case: zeros with a positive exponent can't be
+ # represented in fixed point; rescale them to 0e0.
+ if not self and self._exp > 0 and spec['type'] in 'fF%':
+ self = self._rescale(0, rounding)
+
+ # figure out placement of the decimal point
+ leftdigits = self._exp + len(self._int)
+ if spec['type'] in 'eE':
+ if not self and precision is not None:
+ dotplace = 1 - precision
+ else:
+ dotplace = 1
+ elif spec['type'] in 'fF%':
+ dotplace = leftdigits
+ elif spec['type'] in 'gG':
+ if self._exp <= 0 and leftdigits > -6:
+ dotplace = leftdigits
+ else:
+ dotplace = 1
+
+ # find digits before and after decimal point, and get exponent
+ if dotplace < 0:
+ intpart = '0'
+ fracpart = '0'*(-dotplace) + self._int
+ elif dotplace > len(self._int):
+ intpart = self._int + '0'*(dotplace-len(self._int))
+ fracpart = ''
+ else:
+ intpart = self._int[:dotplace] or '0'
+ fracpart = self._int[dotplace:]
+ exp = leftdigits-dotplace
+
+ # done with the decimal-specific stuff; hand over the rest
+ # of the formatting to the _format_number function
+ return _format_number(self._sign, intpart, fracpart, exp, spec)
+
+def _dec_from_triple(sign, coefficient, exponent, special=False):
+ """Create a decimal instance directly, without any validation,
+ normalization (e.g. removal of leading zeros) or argument
+ conversion.
+
+ This function is for *internal use only*.
+ """
+
+ self = object.__new__(Decimal)
+ self._sign = sign
+ self._int = coefficient
+ self._exp = exponent
+ self._is_special = special
+
+ return self
+
+# Register Decimal as a kind of Number (an abstract base class).
+# However, do not register it as Real (because Decimals are not
+# interoperable with floats).
+_numbers.Number.register(Decimal)
+
+
+##### Context class #######################################################
+
+class _ContextManager(object):
+ """Context manager class to support localcontext().
+
+ Sets a copy of the supplied context in __enter__() and restores
+ the previous decimal context in __exit__()
+ """
+ def __init__(self, new_context):
+ self.new_context = new_context.copy()
+ def __enter__(self):
+ self.saved_context = getcontext()
+ setcontext(self.new_context)
+ return self.new_context
+ def __exit__(self, t, v, tb):
+ setcontext(self.saved_context)
+
+class Context(object):
+ """Contains the context for a Decimal instance.
+
+ Contains:
+ prec - precision (for use in rounding, division, square roots..)
+ rounding - rounding type (how you round)
+ traps - If traps[exception] = 1, then the exception is
+ raised when it is caused. Otherwise, a value is
+ substituted in.
+ flags - When an exception is caused, flags[exception] is set.
+ (Whether or not the trap_enabler is set)
+ Should be reset by user of Decimal instance.
+ Emin - Minimum exponent
+ Emax - Maximum exponent
+ capitals - If 1, 1*10^1 is printed as 1E+1.
+ If 0, printed as 1e1
+ clamp - If 1, change exponents if too high (Default 0)
+ """
+
+ def __init__(self, prec=None, rounding=None, Emin=None, Emax=None,
+ capitals=None, clamp=None, flags=None, traps=None,
+ _ignored_flags=None):
+ # Set defaults; for everything except flags and _ignored_flags,
+ # inherit from DefaultContext.
+ try:
+ dc = DefaultContext
+ except NameError:
+ pass
+
+ self.prec = prec if prec is not None else dc.prec
+ self.rounding = rounding if rounding is not None else dc.rounding
+ self.Emin = Emin if Emin is not None else dc.Emin
+ self.Emax = Emax if Emax is not None else dc.Emax
+ self.capitals = capitals if capitals is not None else dc.capitals
+ self.clamp = clamp if clamp is not None else dc.clamp
+
+ if _ignored_flags is None:
+ self._ignored_flags = []
+ else:
+ self._ignored_flags = _ignored_flags
+
+ if traps is None:
+ self.traps = dc.traps.copy()
+ elif not isinstance(traps, dict):
+ self.traps = dict((s, int(s in traps)) for s in _signals + traps)
+ else:
+ self.traps = traps
+
+ if flags is None:
+ self.flags = dict.fromkeys(_signals, 0)
+ elif not isinstance(flags, dict):
+ self.flags = dict((s, int(s in flags)) for s in _signals + flags)
+ else:
+ self.flags = flags
+
+ def _set_integer_check(self, name, value, vmin, vmax):
+ if not isinstance(value, int):
+ raise TypeError("%s must be an integer" % name)
+ if vmin == '-inf':
+ if value > vmax:
+ raise ValueError("%s must be in [%s, %d]. got: %s" % (name, vmin, vmax, value))
+ elif vmax == 'inf':
+ if value < vmin:
+ raise ValueError("%s must be in [%d, %s]. got: %s" % (name, vmin, vmax, value))
+ else:
+ if value < vmin or value > vmax:
+ raise ValueError("%s must be in [%d, %d]. got %s" % (name, vmin, vmax, value))
+ return object.__setattr__(self, name, value)
+
+ def _set_signal_dict(self, name, d):
+ if not isinstance(d, dict):
+ raise TypeError("%s must be a signal dict" % d)
+ for key in d:
+ if not key in _signals:
+ raise KeyError("%s is not a valid signal dict" % d)
+ for key in _signals:
+ if not key in d:
+ raise KeyError("%s is not a valid signal dict" % d)
+ return object.__setattr__(self, name, d)
+
+ def __setattr__(self, name, value):
+ if name == 'prec':
+ return self._set_integer_check(name, value, 1, 'inf')
+ elif name == 'Emin':
+ return self._set_integer_check(name, value, '-inf', 0)
+ elif name == 'Emax':
+ return self._set_integer_check(name, value, 0, 'inf')
+ elif name == 'capitals':
+ return self._set_integer_check(name, value, 0, 1)
+ elif name == 'clamp':
+ return self._set_integer_check(name, value, 0, 1)
+ elif name == 'rounding':
+ if not value in _rounding_modes:
+ # raise TypeError even for strings to have consistency
+ # among various implementations.
+ raise TypeError("%s: invalid rounding mode" % value)
+ return object.__setattr__(self, name, value)
+ elif name == 'flags' or name == 'traps':
+ return self._set_signal_dict(name, value)
+ elif name == '_ignored_flags':
+ return object.__setattr__(self, name, value)
+ else:
+ raise AttributeError(
+ "'decimal.Context' object has no attribute '%s'" % name)
+
+ def __delattr__(self, name):
+ raise AttributeError("%s cannot be deleted" % name)
+
+ # Support for pickling, copy, and deepcopy
+ def __reduce__(self):
+ flags = [sig for sig, v in self.flags.items() if v]
+ traps = [sig for sig, v in self.traps.items() if v]
+ return (self.__class__,
+ (self.prec, self.rounding, self.Emin, self.Emax,
+ self.capitals, self.clamp, flags, traps))
+
+ def __repr__(self):
+ """Show the current context."""
+ s = []
+ s.append('Context(prec=%(prec)d, rounding=%(rounding)s, '
+ 'Emin=%(Emin)d, Emax=%(Emax)d, capitals=%(capitals)d, '
+ 'clamp=%(clamp)d'
+ % vars(self))
+ names = [f.__name__ for f, v in self.flags.items() if v]
+ s.append('flags=[' + ', '.join(names) + ']')
+ names = [t.__name__ for t, v in self.traps.items() if v]
+ s.append('traps=[' + ', '.join(names) + ']')
+ return ', '.join(s) + ')'
+
+ def clear_flags(self):
+ """Reset all flags to zero"""
+ for flag in self.flags:
+ self.flags[flag] = 0
+
+ def clear_traps(self):
+ """Reset all traps to zero"""
+ for flag in self.traps:
+ self.traps[flag] = 0
+
+ def _shallow_copy(self):
+ """Returns a shallow copy from self."""
+ nc = Context(self.prec, self.rounding, self.Emin, self.Emax,
+ self.capitals, self.clamp, self.flags, self.traps,
+ self._ignored_flags)
+ return nc
+
+ def copy(self):
+ """Returns a deep copy from self."""
+ nc = Context(self.prec, self.rounding, self.Emin, self.Emax,
+ self.capitals, self.clamp,
+ self.flags.copy(), self.traps.copy(),
+ self._ignored_flags)
+ return nc
+ __copy__ = copy
+
+ def _raise_error(self, condition, explanation = None, *args):
+ """Handles an error
+
+ If the flag is in _ignored_flags, returns the default response.
+ Otherwise, it sets the flag, then, if the corresponding
+ trap_enabler is set, it reraises the exception. Otherwise, it returns
+ the default value after setting the flag.
+ """
+ error = _condition_map.get(condition, condition)
+ if error in self._ignored_flags:
+ # Don't touch the flag
+ return error().handle(self, *args)
+
+ self.flags[error] = 1
+ if not self.traps[error]:
+ # The errors define how to handle themselves.
+ return condition().handle(self, *args)
+
+ # Errors should only be risked on copies of the context
+ # self._ignored_flags = []
+ raise error(explanation)
+
+ def _ignore_all_flags(self):
+ """Ignore all flags, if they are raised"""
+ return self._ignore_flags(*_signals)
+
+ def _ignore_flags(self, *flags):
+ """Ignore the flags, if they are raised"""
+ # Do not mutate-- This way, copies of a context leave the original
+ # alone.
+ self._ignored_flags = (self._ignored_flags + list(flags))
+ return list(flags)
+
+ def _regard_flags(self, *flags):
+ """Stop ignoring the flags, if they are raised"""
+ if flags and isinstance(flags[0], (tuple,list)):
+ flags = flags[0]
+ for flag in flags:
+ self._ignored_flags.remove(flag)
+
+ # We inherit object.__hash__, so we must deny this explicitly
+ __hash__ = None
+
+ def Etiny(self):
+ """Returns Etiny (= Emin - prec + 1)"""
+ return int(self.Emin - self.prec + 1)
+
+ def Etop(self):
+ """Returns maximum exponent (= Emax - prec + 1)"""
+ return int(self.Emax - self.prec + 1)
+
+ def _set_rounding(self, type):
+ """Sets the rounding type.
+
+ Sets the rounding type, and returns the current (previous)
+ rounding type. Often used like:
+
+ context = context.copy()
+ # so you don't change the calling context
+ # if an error occurs in the middle.
+ rounding = context._set_rounding(ROUND_UP)
+ val = self.__sub__(other, context=context)
+ context._set_rounding(rounding)
+
+ This will make it round up for that operation.
+ """
+ rounding = self.rounding
+ self.rounding= type
+ return rounding
+
+ def create_decimal(self, num='0'):
+ """Creates a new Decimal instance but using self as context.
+
+ This method implements the to-number operation of the
+ IBM Decimal specification."""
+
+ if isinstance(num, str) and num != num.strip():
+ return self._raise_error(ConversionSyntax,
+ "no trailing or leading whitespace is "
+ "permitted.")
+
+ d = Decimal(num, context=self)
+ if d._isnan() and len(d._int) > self.prec - self.clamp:
+ return self._raise_error(ConversionSyntax,
+ "diagnostic info too long in NaN")
+ return d._fix(self)
+
+ def create_decimal_from_float(self, f):
+ """Creates a new Decimal instance from a float but rounding using self
+ as the context.
+
+ >>> context = Context(prec=5, rounding=ROUND_DOWN)
+ >>> context.create_decimal_from_float(3.1415926535897932)
+ Decimal('3.1415')
+ >>> context = Context(prec=5, traps=[Inexact])
+ >>> context.create_decimal_from_float(3.1415926535897932)
+ Traceback (most recent call last):
+ ...
+ decimal.Inexact
+
+ """
+ d = Decimal.from_float(f) # An exact conversion
+ return d._fix(self) # Apply the context rounding
+
+ # Methods
+ def abs(self, a):
+ """Returns the absolute value of the operand.
+
+ If the operand is negative, the result is the same as using the minus
+ operation on the operand. Otherwise, the result is the same as using
+ the plus operation on the operand.
+
+ >>> ExtendedContext.abs(Decimal('2.1'))
+ Decimal('2.1')
+ >>> ExtendedContext.abs(Decimal('-100'))
+ Decimal('100')
+ >>> ExtendedContext.abs(Decimal('101.5'))
+ Decimal('101.5')
+ >>> ExtendedContext.abs(Decimal('-101.5'))
+ Decimal('101.5')
+ >>> ExtendedContext.abs(-1)
+ Decimal('1')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.__abs__(context=self)
+
+ def add(self, a, b):
+ """Return the sum of the two operands.
+
+ >>> ExtendedContext.add(Decimal('12'), Decimal('7.00'))
+ Decimal('19.00')
+ >>> ExtendedContext.add(Decimal('1E+2'), Decimal('1.01E+4'))
+ Decimal('1.02E+4')
+ >>> ExtendedContext.add(1, Decimal(2))
+ Decimal('3')
+ >>> ExtendedContext.add(Decimal(8), 5)
+ Decimal('13')
+ >>> ExtendedContext.add(5, 5)
+ Decimal('10')
+ """
+ a = _convert_other(a, raiseit=True)
+ r = a.__add__(b, context=self)
+ if r is NotImplemented:
+ raise TypeError("Unable to convert %s to Decimal" % b)
+ else:
+ return r
+
+ def _apply(self, a):
+ return str(a._fix(self))
+
+ def canonical(self, a):
+ """Returns the same Decimal object.
+
+ As we do not have different encodings for the same number, the
+ received object already is in its canonical form.
+
+ >>> ExtendedContext.canonical(Decimal('2.50'))
+ Decimal('2.50')
+ """
+ if not isinstance(a, Decimal):
+ raise TypeError("canonical requires a Decimal as an argument.")
+ return a.canonical()
+
+ def compare(self, a, b):
+ """Compares values numerically.
+
+ If the signs of the operands differ, a value representing each operand
+ ('-1' if the operand is less than zero, '0' if the operand is zero or
+ negative zero, or '1' if the operand is greater than zero) is used in
+ place of that operand for the comparison instead of the actual
+ operand.
+
+ The comparison is then effected by subtracting the second operand from
+ the first and then returning a value according to the result of the
+ subtraction: '-1' if the result is less than zero, '0' if the result is
+ zero or negative zero, or '1' if the result is greater than zero.
+
+ >>> ExtendedContext.compare(Decimal('2.1'), Decimal('3'))
+ Decimal('-1')
+ >>> ExtendedContext.compare(Decimal('2.1'), Decimal('2.1'))
+ Decimal('0')
+ >>> ExtendedContext.compare(Decimal('2.1'), Decimal('2.10'))
+ Decimal('0')
+ >>> ExtendedContext.compare(Decimal('3'), Decimal('2.1'))
+ Decimal('1')
+ >>> ExtendedContext.compare(Decimal('2.1'), Decimal('-3'))
+ Decimal('1')
+ >>> ExtendedContext.compare(Decimal('-3'), Decimal('2.1'))
+ Decimal('-1')
+ >>> ExtendedContext.compare(1, 2)
+ Decimal('-1')
+ >>> ExtendedContext.compare(Decimal(1), 2)
+ Decimal('-1')
+ >>> ExtendedContext.compare(1, Decimal(2))
+ Decimal('-1')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.compare(b, context=self)
+
+ def compare_signal(self, a, b):
+ """Compares the values of the two operands numerically.
+
+ It's pretty much like compare(), but all NaNs signal, with signaling
+ NaNs taking precedence over quiet NaNs.
+
+ >>> c = ExtendedContext
+ >>> c.compare_signal(Decimal('2.1'), Decimal('3'))
+ Decimal('-1')
+ >>> c.compare_signal(Decimal('2.1'), Decimal('2.1'))
+ Decimal('0')
+ >>> c.flags[InvalidOperation] = 0
+ >>> print(c.flags[InvalidOperation])
+ 0
+ >>> c.compare_signal(Decimal('NaN'), Decimal('2.1'))
+ Decimal('NaN')
+ >>> print(c.flags[InvalidOperation])
+ 1
+ >>> c.flags[InvalidOperation] = 0
+ >>> print(c.flags[InvalidOperation])
+ 0
+ >>> c.compare_signal(Decimal('sNaN'), Decimal('2.1'))
+ Decimal('NaN')
+ >>> print(c.flags[InvalidOperation])
+ 1
+ >>> c.compare_signal(-1, 2)
+ Decimal('-1')
+ >>> c.compare_signal(Decimal(-1), 2)
+ Decimal('-1')
+ >>> c.compare_signal(-1, Decimal(2))
+ Decimal('-1')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.compare_signal(b, context=self)
+
+ def compare_total(self, a, b):
+ """Compares two operands using their abstract representation.
+
+ This is not like the standard compare, which use their numerical
+ value. Note that a total ordering is defined for all possible abstract
+ representations.
+
+ >>> ExtendedContext.compare_total(Decimal('12.73'), Decimal('127.9'))
+ Decimal('-1')
+ >>> ExtendedContext.compare_total(Decimal('-127'), Decimal('12'))
+ Decimal('-1')
+ >>> ExtendedContext.compare_total(Decimal('12.30'), Decimal('12.3'))
+ Decimal('-1')
+ >>> ExtendedContext.compare_total(Decimal('12.30'), Decimal('12.30'))
+ Decimal('0')
+ >>> ExtendedContext.compare_total(Decimal('12.3'), Decimal('12.300'))
+ Decimal('1')
+ >>> ExtendedContext.compare_total(Decimal('12.3'), Decimal('NaN'))
+ Decimal('-1')
+ >>> ExtendedContext.compare_total(1, 2)
+ Decimal('-1')
+ >>> ExtendedContext.compare_total(Decimal(1), 2)
+ Decimal('-1')
+ >>> ExtendedContext.compare_total(1, Decimal(2))
+ Decimal('-1')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.compare_total(b)
+
+ def compare_total_mag(self, a, b):
+ """Compares two operands using their abstract representation ignoring sign.
+
+ Like compare_total, but with operand's sign ignored and assumed to be 0.
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.compare_total_mag(b)
+
+ def copy_abs(self, a):
+ """Returns a copy of the operand with the sign set to 0.
+
+ >>> ExtendedContext.copy_abs(Decimal('2.1'))
+ Decimal('2.1')
+ >>> ExtendedContext.copy_abs(Decimal('-100'))
+ Decimal('100')
+ >>> ExtendedContext.copy_abs(-1)
+ Decimal('1')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.copy_abs()
+
+ def copy_decimal(self, a):
+ """Returns a copy of the decimal object.
+
+ >>> ExtendedContext.copy_decimal(Decimal('2.1'))
+ Decimal('2.1')
+ >>> ExtendedContext.copy_decimal(Decimal('-1.00'))
+ Decimal('-1.00')
+ >>> ExtendedContext.copy_decimal(1)
+ Decimal('1')
+ """
+ a = _convert_other(a, raiseit=True)
+ return Decimal(a)
+
+ def copy_negate(self, a):
+ """Returns a copy of the operand with the sign inverted.
+
+ >>> ExtendedContext.copy_negate(Decimal('101.5'))
+ Decimal('-101.5')
+ >>> ExtendedContext.copy_negate(Decimal('-101.5'))
+ Decimal('101.5')
+ >>> ExtendedContext.copy_negate(1)
+ Decimal('-1')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.copy_negate()
+
+ def copy_sign(self, a, b):
+ """Copies the second operand's sign to the first one.
+
+ In detail, it returns a copy of the first operand with the sign
+ equal to the sign of the second operand.
+
+ >>> ExtendedContext.copy_sign(Decimal( '1.50'), Decimal('7.33'))
+ Decimal('1.50')
+ >>> ExtendedContext.copy_sign(Decimal('-1.50'), Decimal('7.33'))
+ Decimal('1.50')
+ >>> ExtendedContext.copy_sign(Decimal( '1.50'), Decimal('-7.33'))
+ Decimal('-1.50')
+ >>> ExtendedContext.copy_sign(Decimal('-1.50'), Decimal('-7.33'))
+ Decimal('-1.50')
+ >>> ExtendedContext.copy_sign(1, -2)
+ Decimal('-1')
+ >>> ExtendedContext.copy_sign(Decimal(1), -2)
+ Decimal('-1')
+ >>> ExtendedContext.copy_sign(1, Decimal(-2))
+ Decimal('-1')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.copy_sign(b)
+
+ def divide(self, a, b):
+ """Decimal division in a specified context.
+
+ >>> ExtendedContext.divide(Decimal('1'), Decimal('3'))
+ Decimal('0.333333333')
+ >>> ExtendedContext.divide(Decimal('2'), Decimal('3'))
+ Decimal('0.666666667')
+ >>> ExtendedContext.divide(Decimal('5'), Decimal('2'))
+ Decimal('2.5')
+ >>> ExtendedContext.divide(Decimal('1'), Decimal('10'))
+ Decimal('0.1')
+ >>> ExtendedContext.divide(Decimal('12'), Decimal('12'))
+ Decimal('1')
+ >>> ExtendedContext.divide(Decimal('8.00'), Decimal('2'))
+ Decimal('4.00')
+ >>> ExtendedContext.divide(Decimal('2.400'), Decimal('2.0'))
+ Decimal('1.20')
+ >>> ExtendedContext.divide(Decimal('1000'), Decimal('100'))
+ Decimal('10')
+ >>> ExtendedContext.divide(Decimal('1000'), Decimal('1'))
+ Decimal('1000')
+ >>> ExtendedContext.divide(Decimal('2.40E+6'), Decimal('2'))
+ Decimal('1.20E+6')
+ >>> ExtendedContext.divide(5, 5)
+ Decimal('1')
+ >>> ExtendedContext.divide(Decimal(5), 5)
+ Decimal('1')
+ >>> ExtendedContext.divide(5, Decimal(5))
+ Decimal('1')
+ """
+ a = _convert_other(a, raiseit=True)
+ r = a.__truediv__(b, context=self)
+ if r is NotImplemented:
+ raise TypeError("Unable to convert %s to Decimal" % b)
+ else:
+ return r
+
+ def divide_int(self, a, b):
+ """Divides two numbers and returns the integer part of the result.
+
+ >>> ExtendedContext.divide_int(Decimal('2'), Decimal('3'))
+ Decimal('0')
+ >>> ExtendedContext.divide_int(Decimal('10'), Decimal('3'))
+ Decimal('3')
+ >>> ExtendedContext.divide_int(Decimal('1'), Decimal('0.3'))
+ Decimal('3')
+ >>> ExtendedContext.divide_int(10, 3)
+ Decimal('3')
+ >>> ExtendedContext.divide_int(Decimal(10), 3)
+ Decimal('3')
+ >>> ExtendedContext.divide_int(10, Decimal(3))
+ Decimal('3')
+ """
+ a = _convert_other(a, raiseit=True)
+ r = a.__floordiv__(b, context=self)
+ if r is NotImplemented:
+ raise TypeError("Unable to convert %s to Decimal" % b)
+ else:
+ return r
+
+ def divmod(self, a, b):
+ """Return (a // b, a % b).
+
+ >>> ExtendedContext.divmod(Decimal(8), Decimal(3))
+ (Decimal('2'), Decimal('2'))
+ >>> ExtendedContext.divmod(Decimal(8), Decimal(4))
+ (Decimal('2'), Decimal('0'))
+ >>> ExtendedContext.divmod(8, 4)
+ (Decimal('2'), Decimal('0'))
+ >>> ExtendedContext.divmod(Decimal(8), 4)
+ (Decimal('2'), Decimal('0'))
+ >>> ExtendedContext.divmod(8, Decimal(4))
+ (Decimal('2'), Decimal('0'))
+ """
+ a = _convert_other(a, raiseit=True)
+ r = a.__divmod__(b, context=self)
+ if r is NotImplemented:
+ raise TypeError("Unable to convert %s to Decimal" % b)
+ else:
+ return r
+
+ def exp(self, a):
+ """Returns e ** a.
+
+ >>> c = ExtendedContext.copy()
+ >>> c.Emin = -999
+ >>> c.Emax = 999
+ >>> c.exp(Decimal('-Infinity'))
+ Decimal('0')
+ >>> c.exp(Decimal('-1'))
+ Decimal('0.367879441')
+ >>> c.exp(Decimal('0'))
+ Decimal('1')
+ >>> c.exp(Decimal('1'))
+ Decimal('2.71828183')
+ >>> c.exp(Decimal('0.693147181'))
+ Decimal('2.00000000')
+ >>> c.exp(Decimal('+Infinity'))
+ Decimal('Infinity')
+ >>> c.exp(10)
+ Decimal('22026.4658')
+ """
+ a =_convert_other(a, raiseit=True)
+ return a.exp(context=self)
+
+ def fma(self, a, b, c):
+ """Returns a multiplied by b, plus c.
+
+ The first two operands are multiplied together, using multiply,
+ the third operand is then added to the result of that
+ multiplication, using add, all with only one final rounding.
+
+ >>> ExtendedContext.fma(Decimal('3'), Decimal('5'), Decimal('7'))
+ Decimal('22')
+ >>> ExtendedContext.fma(Decimal('3'), Decimal('-5'), Decimal('7'))
+ Decimal('-8')
+ >>> ExtendedContext.fma(Decimal('888565290'), Decimal('1557.96930'), Decimal('-86087.7578'))
+ Decimal('1.38435736E+12')
+ >>> ExtendedContext.fma(1, 3, 4)
+ Decimal('7')
+ >>> ExtendedContext.fma(1, Decimal(3), 4)
+ Decimal('7')
+ >>> ExtendedContext.fma(1, 3, Decimal(4))
+ Decimal('7')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.fma(b, c, context=self)
+
+ def is_canonical(self, a):
+ """Return True if the operand is canonical; otherwise return False.
+
+ Currently, the encoding of a Decimal instance is always
+ canonical, so this method returns True for any Decimal.
+
+ >>> ExtendedContext.is_canonical(Decimal('2.50'))
+ True
+ """
+ if not isinstance(a, Decimal):
+ raise TypeError("is_canonical requires a Decimal as an argument.")
+ return a.is_canonical()
+
+ def is_finite(self, a):
+ """Return True if the operand is finite; otherwise return False.
+
+ A Decimal instance is considered finite if it is neither
+ infinite nor a NaN.
+
+ >>> ExtendedContext.is_finite(Decimal('2.50'))
+ True
+ >>> ExtendedContext.is_finite(Decimal('-0.3'))
+ True
+ >>> ExtendedContext.is_finite(Decimal('0'))
+ True
+ >>> ExtendedContext.is_finite(Decimal('Inf'))
+ False
+ >>> ExtendedContext.is_finite(Decimal('NaN'))
+ False
+ >>> ExtendedContext.is_finite(1)
+ True
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.is_finite()
+
+ def is_infinite(self, a):
+ """Return True if the operand is infinite; otherwise return False.
+
+ >>> ExtendedContext.is_infinite(Decimal('2.50'))
+ False
+ >>> ExtendedContext.is_infinite(Decimal('-Inf'))
+ True
+ >>> ExtendedContext.is_infinite(Decimal('NaN'))
+ False
+ >>> ExtendedContext.is_infinite(1)
+ False
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.is_infinite()
+
+ def is_nan(self, a):
+ """Return True if the operand is a qNaN or sNaN;
+ otherwise return False.
+
+ >>> ExtendedContext.is_nan(Decimal('2.50'))
+ False
+ >>> ExtendedContext.is_nan(Decimal('NaN'))
+ True
+ >>> ExtendedContext.is_nan(Decimal('-sNaN'))
+ True
+ >>> ExtendedContext.is_nan(1)
+ False
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.is_nan()
+
+ def is_normal(self, a):
+ """Return True if the operand is a normal number;
+ otherwise return False.
+
+ >>> c = ExtendedContext.copy()
+ >>> c.Emin = -999
+ >>> c.Emax = 999
+ >>> c.is_normal(Decimal('2.50'))
+ True
+ >>> c.is_normal(Decimal('0.1E-999'))
+ False
+ >>> c.is_normal(Decimal('0.00'))
+ False
+ >>> c.is_normal(Decimal('-Inf'))
+ False
+ >>> c.is_normal(Decimal('NaN'))
+ False
+ >>> c.is_normal(1)
+ True
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.is_normal(context=self)
+
+ def is_qnan(self, a):
+ """Return True if the operand is a quiet NaN; otherwise return False.
+
+ >>> ExtendedContext.is_qnan(Decimal('2.50'))
+ False
+ >>> ExtendedContext.is_qnan(Decimal('NaN'))
+ True
+ >>> ExtendedContext.is_qnan(Decimal('sNaN'))
+ False
+ >>> ExtendedContext.is_qnan(1)
+ False
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.is_qnan()
+
+ def is_signed(self, a):
+ """Return True if the operand is negative; otherwise return False.
+
+ >>> ExtendedContext.is_signed(Decimal('2.50'))
+ False
+ >>> ExtendedContext.is_signed(Decimal('-12'))
+ True
+ >>> ExtendedContext.is_signed(Decimal('-0'))
+ True
+ >>> ExtendedContext.is_signed(8)
+ False
+ >>> ExtendedContext.is_signed(-8)
+ True
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.is_signed()
+
+ def is_snan(self, a):
+ """Return True if the operand is a signaling NaN;
+ otherwise return False.
+
+ >>> ExtendedContext.is_snan(Decimal('2.50'))
+ False
+ >>> ExtendedContext.is_snan(Decimal('NaN'))
+ False
+ >>> ExtendedContext.is_snan(Decimal('sNaN'))
+ True
+ >>> ExtendedContext.is_snan(1)
+ False
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.is_snan()
+
+ def is_subnormal(self, a):
+ """Return True if the operand is subnormal; otherwise return False.
+
+ >>> c = ExtendedContext.copy()
+ >>> c.Emin = -999
+ >>> c.Emax = 999
+ >>> c.is_subnormal(Decimal('2.50'))
+ False
+ >>> c.is_subnormal(Decimal('0.1E-999'))
+ True
+ >>> c.is_subnormal(Decimal('0.00'))
+ False
+ >>> c.is_subnormal(Decimal('-Inf'))
+ False
+ >>> c.is_subnormal(Decimal('NaN'))
+ False
+ >>> c.is_subnormal(1)
+ False
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.is_subnormal(context=self)
+
+ def is_zero(self, a):
+ """Return True if the operand is a zero; otherwise return False.
+
+ >>> ExtendedContext.is_zero(Decimal('0'))
+ True
+ >>> ExtendedContext.is_zero(Decimal('2.50'))
+ False
+ >>> ExtendedContext.is_zero(Decimal('-0E+2'))
+ True
+ >>> ExtendedContext.is_zero(1)
+ False
+ >>> ExtendedContext.is_zero(0)
+ True
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.is_zero()
+
+ def ln(self, a):
+ """Returns the natural (base e) logarithm of the operand.
+
+ >>> c = ExtendedContext.copy()
+ >>> c.Emin = -999
+ >>> c.Emax = 999
+ >>> c.ln(Decimal('0'))
+ Decimal('-Infinity')
+ >>> c.ln(Decimal('1.000'))
+ Decimal('0')
+ >>> c.ln(Decimal('2.71828183'))
+ Decimal('1.00000000')
+ >>> c.ln(Decimal('10'))
+ Decimal('2.30258509')
+ >>> c.ln(Decimal('+Infinity'))
+ Decimal('Infinity')
+ >>> c.ln(1)
+ Decimal('0')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.ln(context=self)
+
+ def log10(self, a):
+ """Returns the base 10 logarithm of the operand.
+
+ >>> c = ExtendedContext.copy()
+ >>> c.Emin = -999
+ >>> c.Emax = 999
+ >>> c.log10(Decimal('0'))
+ Decimal('-Infinity')
+ >>> c.log10(Decimal('0.001'))
+ Decimal('-3')
+ >>> c.log10(Decimal('1.000'))
+ Decimal('0')
+ >>> c.log10(Decimal('2'))
+ Decimal('0.301029996')
+ >>> c.log10(Decimal('10'))
+ Decimal('1')
+ >>> c.log10(Decimal('70'))
+ Decimal('1.84509804')
+ >>> c.log10(Decimal('+Infinity'))
+ Decimal('Infinity')
+ >>> c.log10(0)
+ Decimal('-Infinity')
+ >>> c.log10(1)
+ Decimal('0')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.log10(context=self)
+
+ def logb(self, a):
+ """ Returns the exponent of the magnitude of the operand's MSD.
+
+ The result is the integer which is the exponent of the magnitude
+ of the most significant digit of the operand (as though the
+ operand were truncated to a single digit while maintaining the
+ value of that digit and without limiting the resulting exponent).
+
+ >>> ExtendedContext.logb(Decimal('250'))
+ Decimal('2')
+ >>> ExtendedContext.logb(Decimal('2.50'))
+ Decimal('0')
+ >>> ExtendedContext.logb(Decimal('0.03'))
+ Decimal('-2')
+ >>> ExtendedContext.logb(Decimal('0'))
+ Decimal('-Infinity')
+ >>> ExtendedContext.logb(1)
+ Decimal('0')
+ >>> ExtendedContext.logb(10)
+ Decimal('1')
+ >>> ExtendedContext.logb(100)
+ Decimal('2')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.logb(context=self)
+
+ def logical_and(self, a, b):
+ """Applies the logical operation 'and' between each operand's digits.
+
+ The operands must be both logical numbers.
+
+ >>> ExtendedContext.logical_and(Decimal('0'), Decimal('0'))
+ Decimal('0')
+ >>> ExtendedContext.logical_and(Decimal('0'), Decimal('1'))
+ Decimal('0')
+ >>> ExtendedContext.logical_and(Decimal('1'), Decimal('0'))
+ Decimal('0')
+ >>> ExtendedContext.logical_and(Decimal('1'), Decimal('1'))
+ Decimal('1')
+ >>> ExtendedContext.logical_and(Decimal('1100'), Decimal('1010'))
+ Decimal('1000')
+ >>> ExtendedContext.logical_and(Decimal('1111'), Decimal('10'))
+ Decimal('10')
+ >>> ExtendedContext.logical_and(110, 1101)
+ Decimal('100')
+ >>> ExtendedContext.logical_and(Decimal(110), 1101)
+ Decimal('100')
+ >>> ExtendedContext.logical_and(110, Decimal(1101))
+ Decimal('100')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.logical_and(b, context=self)
+
+ def logical_invert(self, a):
+ """Invert all the digits in the operand.
+
+ The operand must be a logical number.
+
+ >>> ExtendedContext.logical_invert(Decimal('0'))
+ Decimal('111111111')
+ >>> ExtendedContext.logical_invert(Decimal('1'))
+ Decimal('111111110')
+ >>> ExtendedContext.logical_invert(Decimal('111111111'))
+ Decimal('0')
+ >>> ExtendedContext.logical_invert(Decimal('101010101'))
+ Decimal('10101010')
+ >>> ExtendedContext.logical_invert(1101)
+ Decimal('111110010')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.logical_invert(context=self)
+
+ def logical_or(self, a, b):
+ """Applies the logical operation 'or' between each operand's digits.
+
+ The operands must be both logical numbers.
+
+ >>> ExtendedContext.logical_or(Decimal('0'), Decimal('0'))
+ Decimal('0')
+ >>> ExtendedContext.logical_or(Decimal('0'), Decimal('1'))
+ Decimal('1')
+ >>> ExtendedContext.logical_or(Decimal('1'), Decimal('0'))
+ Decimal('1')
+ >>> ExtendedContext.logical_or(Decimal('1'), Decimal('1'))
+ Decimal('1')
+ >>> ExtendedContext.logical_or(Decimal('1100'), Decimal('1010'))
+ Decimal('1110')
+ >>> ExtendedContext.logical_or(Decimal('1110'), Decimal('10'))
+ Decimal('1110')
+ >>> ExtendedContext.logical_or(110, 1101)
+ Decimal('1111')
+ >>> ExtendedContext.logical_or(Decimal(110), 1101)
+ Decimal('1111')
+ >>> ExtendedContext.logical_or(110, Decimal(1101))
+ Decimal('1111')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.logical_or(b, context=self)
+
+ def logical_xor(self, a, b):
+ """Applies the logical operation 'xor' between each operand's digits.
+
+ The operands must be both logical numbers.
+
+ >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('0'))
+ Decimal('0')
+ >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('1'))
+ Decimal('1')
+ >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('0'))
+ Decimal('1')
+ >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('1'))
+ Decimal('0')
+ >>> ExtendedContext.logical_xor(Decimal('1100'), Decimal('1010'))
+ Decimal('110')
+ >>> ExtendedContext.logical_xor(Decimal('1111'), Decimal('10'))
+ Decimal('1101')
+ >>> ExtendedContext.logical_xor(110, 1101)
+ Decimal('1011')
+ >>> ExtendedContext.logical_xor(Decimal(110), 1101)
+ Decimal('1011')
+ >>> ExtendedContext.logical_xor(110, Decimal(1101))
+ Decimal('1011')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.logical_xor(b, context=self)
+
+ def max(self, a, b):
+ """max compares two values numerically and returns the maximum.
+
+ If either operand is a NaN then the general rules apply.
+ Otherwise, the operands are compared as though by the compare
+ operation. If they are numerically equal then the left-hand operand
+ is chosen as the result. Otherwise the maximum (closer to positive
+ infinity) of the two operands is chosen as the result.
+
+ >>> ExtendedContext.max(Decimal('3'), Decimal('2'))
+ Decimal('3')
+ >>> ExtendedContext.max(Decimal('-10'), Decimal('3'))
+ Decimal('3')
+ >>> ExtendedContext.max(Decimal('1.0'), Decimal('1'))
+ Decimal('1')
+ >>> ExtendedContext.max(Decimal('7'), Decimal('NaN'))
+ Decimal('7')
+ >>> ExtendedContext.max(1, 2)
+ Decimal('2')
+ >>> ExtendedContext.max(Decimal(1), 2)
+ Decimal('2')
+ >>> ExtendedContext.max(1, Decimal(2))
+ Decimal('2')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.max(b, context=self)
+
+ def max_mag(self, a, b):
+ """Compares the values numerically with their sign ignored.
+
+ >>> ExtendedContext.max_mag(Decimal('7'), Decimal('NaN'))
+ Decimal('7')
+ >>> ExtendedContext.max_mag(Decimal('7'), Decimal('-10'))
+ Decimal('-10')
+ >>> ExtendedContext.max_mag(1, -2)
+ Decimal('-2')
+ >>> ExtendedContext.max_mag(Decimal(1), -2)
+ Decimal('-2')
+ >>> ExtendedContext.max_mag(1, Decimal(-2))
+ Decimal('-2')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.max_mag(b, context=self)
+
+ def min(self, a, b):
+ """min compares two values numerically and returns the minimum.
+
+ If either operand is a NaN then the general rules apply.
+ Otherwise, the operands are compared as though by the compare
+ operation. If they are numerically equal then the left-hand operand
+ is chosen as the result. Otherwise the minimum (closer to negative
+ infinity) of the two operands is chosen as the result.
+
+ >>> ExtendedContext.min(Decimal('3'), Decimal('2'))
+ Decimal('2')
+ >>> ExtendedContext.min(Decimal('-10'), Decimal('3'))
+ Decimal('-10')
+ >>> ExtendedContext.min(Decimal('1.0'), Decimal('1'))
+ Decimal('1.0')
+ >>> ExtendedContext.min(Decimal('7'), Decimal('NaN'))
+ Decimal('7')
+ >>> ExtendedContext.min(1, 2)
+ Decimal('1')
+ >>> ExtendedContext.min(Decimal(1), 2)
+ Decimal('1')
+ >>> ExtendedContext.min(1, Decimal(29))
+ Decimal('1')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.min(b, context=self)
+
+ def min_mag(self, a, b):
+ """Compares the values numerically with their sign ignored.
+
+ >>> ExtendedContext.min_mag(Decimal('3'), Decimal('-2'))
+ Decimal('-2')
+ >>> ExtendedContext.min_mag(Decimal('-3'), Decimal('NaN'))
+ Decimal('-3')
+ >>> ExtendedContext.min_mag(1, -2)
+ Decimal('1')
+ >>> ExtendedContext.min_mag(Decimal(1), -2)
+ Decimal('1')
+ >>> ExtendedContext.min_mag(1, Decimal(-2))
+ Decimal('1')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.min_mag(b, context=self)
+
+ def minus(self, a):
+ """Minus corresponds to unary prefix minus in Python.
+
+ The operation is evaluated using the same rules as subtract; the
+ operation minus(a) is calculated as subtract('0', a) where the '0'
+ has the same exponent as the operand.
+
+ >>> ExtendedContext.minus(Decimal('1.3'))
+ Decimal('-1.3')
+ >>> ExtendedContext.minus(Decimal('-1.3'))
+ Decimal('1.3')
+ >>> ExtendedContext.minus(1)
+ Decimal('-1')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.__neg__(context=self)
+
+ def multiply(self, a, b):
+ """multiply multiplies two operands.
+
+ If either operand is a special value then the general rules apply.
+ Otherwise, the operands are multiplied together
+ ('long multiplication'), resulting in a number which may be as long as
+ the sum of the lengths of the two operands.
+
+ >>> ExtendedContext.multiply(Decimal('1.20'), Decimal('3'))
+ Decimal('3.60')
+ >>> ExtendedContext.multiply(Decimal('7'), Decimal('3'))
+ Decimal('21')
+ >>> ExtendedContext.multiply(Decimal('0.9'), Decimal('0.8'))
+ Decimal('0.72')
+ >>> ExtendedContext.multiply(Decimal('0.9'), Decimal('-0'))
+ Decimal('-0.0')
+ >>> ExtendedContext.multiply(Decimal('654321'), Decimal('654321'))
+ Decimal('4.28135971E+11')
+ >>> ExtendedContext.multiply(7, 7)
+ Decimal('49')
+ >>> ExtendedContext.multiply(Decimal(7), 7)
+ Decimal('49')
+ >>> ExtendedContext.multiply(7, Decimal(7))
+ Decimal('49')
+ """
+ a = _convert_other(a, raiseit=True)
+ r = a.__mul__(b, context=self)
+ if r is NotImplemented:
+ raise TypeError("Unable to convert %s to Decimal" % b)
+ else:
+ return r
+
+ def next_minus(self, a):
+ """Returns the largest representable number smaller than a.
+
+ >>> c = ExtendedContext.copy()
+ >>> c.Emin = -999
+ >>> c.Emax = 999
+ >>> ExtendedContext.next_minus(Decimal('1'))
+ Decimal('0.999999999')
+ >>> c.next_minus(Decimal('1E-1007'))
+ Decimal('0E-1007')
+ >>> ExtendedContext.next_minus(Decimal('-1.00000003'))
+ Decimal('-1.00000004')
+ >>> c.next_minus(Decimal('Infinity'))
+ Decimal('9.99999999E+999')
+ >>> c.next_minus(1)
+ Decimal('0.999999999')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.next_minus(context=self)
+
+ def next_plus(self, a):
+ """Returns the smallest representable number larger than a.
+
+ >>> c = ExtendedContext.copy()
+ >>> c.Emin = -999
+ >>> c.Emax = 999
+ >>> ExtendedContext.next_plus(Decimal('1'))
+ Decimal('1.00000001')
+ >>> c.next_plus(Decimal('-1E-1007'))
+ Decimal('-0E-1007')
+ >>> ExtendedContext.next_plus(Decimal('-1.00000003'))
+ Decimal('-1.00000002')
+ >>> c.next_plus(Decimal('-Infinity'))
+ Decimal('-9.99999999E+999')
+ >>> c.next_plus(1)
+ Decimal('1.00000001')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.next_plus(context=self)
+
+ def next_toward(self, a, b):
+ """Returns the number closest to a, in direction towards b.
+
+ The result is the closest representable number from the first
+ operand (but not the first operand) that is in the direction
+ towards the second operand, unless the operands have the same
+ value.
+
+ >>> c = ExtendedContext.copy()
+ >>> c.Emin = -999
+ >>> c.Emax = 999
+ >>> c.next_toward(Decimal('1'), Decimal('2'))
+ Decimal('1.00000001')
+ >>> c.next_toward(Decimal('-1E-1007'), Decimal('1'))
+ Decimal('-0E-1007')
+ >>> c.next_toward(Decimal('-1.00000003'), Decimal('0'))
+ Decimal('-1.00000002')
+ >>> c.next_toward(Decimal('1'), Decimal('0'))
+ Decimal('0.999999999')
+ >>> c.next_toward(Decimal('1E-1007'), Decimal('-100'))
+ Decimal('0E-1007')
+ >>> c.next_toward(Decimal('-1.00000003'), Decimal('-10'))
+ Decimal('-1.00000004')
+ >>> c.next_toward(Decimal('0.00'), Decimal('-0.0000'))
+ Decimal('-0.00')
+ >>> c.next_toward(0, 1)
+ Decimal('1E-1007')
+ >>> c.next_toward(Decimal(0), 1)
+ Decimal('1E-1007')
+ >>> c.next_toward(0, Decimal(1))
+ Decimal('1E-1007')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.next_toward(b, context=self)
+
+ def normalize(self, a):
+ """normalize reduces an operand to its simplest form.
+
+ Essentially a plus operation with all trailing zeros removed from the
+ result.
+
+ >>> ExtendedContext.normalize(Decimal('2.1'))
+ Decimal('2.1')
+ >>> ExtendedContext.normalize(Decimal('-2.0'))
+ Decimal('-2')
+ >>> ExtendedContext.normalize(Decimal('1.200'))
+ Decimal('1.2')
+ >>> ExtendedContext.normalize(Decimal('-120'))
+ Decimal('-1.2E+2')
+ >>> ExtendedContext.normalize(Decimal('120.00'))
+ Decimal('1.2E+2')
+ >>> ExtendedContext.normalize(Decimal('0.00'))
+ Decimal('0')
+ >>> ExtendedContext.normalize(6)
+ Decimal('6')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.normalize(context=self)
+
+ def number_class(self, a):
+ """Returns an indication of the class of the operand.
+
+ The class is one of the following strings:
+ -sNaN
+ -NaN
+ -Infinity
+ -Normal
+ -Subnormal
+ -Zero
+ +Zero
+ +Subnormal
+ +Normal
+ +Infinity
+
+ >>> c = ExtendedContext.copy()
+ >>> c.Emin = -999
+ >>> c.Emax = 999
+ >>> c.number_class(Decimal('Infinity'))
+ '+Infinity'
+ >>> c.number_class(Decimal('1E-10'))
+ '+Normal'
+ >>> c.number_class(Decimal('2.50'))
+ '+Normal'
+ >>> c.number_class(Decimal('0.1E-999'))
+ '+Subnormal'
+ >>> c.number_class(Decimal('0'))
+ '+Zero'
+ >>> c.number_class(Decimal('-0'))
+ '-Zero'
+ >>> c.number_class(Decimal('-0.1E-999'))
+ '-Subnormal'
+ >>> c.number_class(Decimal('-1E-10'))
+ '-Normal'
+ >>> c.number_class(Decimal('-2.50'))
+ '-Normal'
+ >>> c.number_class(Decimal('-Infinity'))
+ '-Infinity'
+ >>> c.number_class(Decimal('NaN'))
+ 'NaN'
+ >>> c.number_class(Decimal('-NaN'))
+ 'NaN'
+ >>> c.number_class(Decimal('sNaN'))
+ 'sNaN'
+ >>> c.number_class(123)
+ '+Normal'
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.number_class(context=self)
+
+ def plus(self, a):
+ """Plus corresponds to unary prefix plus in Python.
+
+ The operation is evaluated using the same rules as add; the
+ operation plus(a) is calculated as add('0', a) where the '0'
+ has the same exponent as the operand.
+
+ >>> ExtendedContext.plus(Decimal('1.3'))
+ Decimal('1.3')
+ >>> ExtendedContext.plus(Decimal('-1.3'))
+ Decimal('-1.3')
+ >>> ExtendedContext.plus(-1)
+ Decimal('-1')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.__pos__(context=self)
+
+ def power(self, a, b, modulo=None):
+ """Raises a to the power of b, to modulo if given.
+
+ With two arguments, compute a**b. If a is negative then b
+ must be integral. The result will be inexact unless b is
+ integral and the result is finite and can be expressed exactly
+ in 'precision' digits.
+
+ With three arguments, compute (a**b) % modulo. For the
+ three argument form, the following restrictions on the
+ arguments hold:
+
+ - all three arguments must be integral
+ - b must be nonnegative
+ - at least one of a or b must be nonzero
+ - modulo must be nonzero and have at most 'precision' digits
+
+ The result of pow(a, b, modulo) is identical to the result
+ that would be obtained by computing (a**b) % modulo with
+ unbounded precision, but is computed more efficiently. It is
+ always exact.
+
+ >>> c = ExtendedContext.copy()
+ >>> c.Emin = -999
+ >>> c.Emax = 999
+ >>> c.power(Decimal('2'), Decimal('3'))
+ Decimal('8')
+ >>> c.power(Decimal('-2'), Decimal('3'))
+ Decimal('-8')
+ >>> c.power(Decimal('2'), Decimal('-3'))
+ Decimal('0.125')
+ >>> c.power(Decimal('1.7'), Decimal('8'))
+ Decimal('69.7575744')
+ >>> c.power(Decimal('10'), Decimal('0.301029996'))
+ Decimal('2.00000000')
+ >>> c.power(Decimal('Infinity'), Decimal('-1'))
+ Decimal('0')
+ >>> c.power(Decimal('Infinity'), Decimal('0'))
+ Decimal('1')
+ >>> c.power(Decimal('Infinity'), Decimal('1'))
+ Decimal('Infinity')
+ >>> c.power(Decimal('-Infinity'), Decimal('-1'))
+ Decimal('-0')
+ >>> c.power(Decimal('-Infinity'), Decimal('0'))
+ Decimal('1')
+ >>> c.power(Decimal('-Infinity'), Decimal('1'))
+ Decimal('-Infinity')
+ >>> c.power(Decimal('-Infinity'), Decimal('2'))
+ Decimal('Infinity')
+ >>> c.power(Decimal('0'), Decimal('0'))
+ Decimal('NaN')
+
+ >>> c.power(Decimal('3'), Decimal('7'), Decimal('16'))
+ Decimal('11')
+ >>> c.power(Decimal('-3'), Decimal('7'), Decimal('16'))
+ Decimal('-11')
+ >>> c.power(Decimal('-3'), Decimal('8'), Decimal('16'))
+ Decimal('1')
+ >>> c.power(Decimal('3'), Decimal('7'), Decimal('-16'))
+ Decimal('11')
+ >>> c.power(Decimal('23E12345'), Decimal('67E189'), Decimal('123456789'))
+ Decimal('11729830')
+ >>> c.power(Decimal('-0'), Decimal('17'), Decimal('1729'))
+ Decimal('-0')
+ >>> c.power(Decimal('-23'), Decimal('0'), Decimal('65537'))
+ Decimal('1')
+ >>> ExtendedContext.power(7, 7)
+ Decimal('823543')
+ >>> ExtendedContext.power(Decimal(7), 7)
+ Decimal('823543')
+ >>> ExtendedContext.power(7, Decimal(7), 2)
+ Decimal('1')
+ """
+ a = _convert_other(a, raiseit=True)
+ r = a.__pow__(b, modulo, context=self)
+ if r is NotImplemented:
+ raise TypeError("Unable to convert %s to Decimal" % b)
+ else:
+ return r
+
+ def quantize(self, a, b):
+ """Returns a value equal to 'a' (rounded), having the exponent of 'b'.
+
+ The coefficient of the result is derived from that of the left-hand
+ operand. It may be rounded using the current rounding setting (if the
+ exponent is being increased), multiplied by a positive power of ten (if
+ the exponent is being decreased), or is unchanged (if the exponent is
+ already equal to that of the right-hand operand).
+
+ Unlike other operations, if the length of the coefficient after the
+ quantize operation would be greater than precision then an Invalid
+ operation condition is raised. This guarantees that, unless there is
+ an error condition, the exponent of the result of a quantize is always
+ equal to that of the right-hand operand.
+
+ Also unlike other operations, quantize will never raise Underflow, even
+ if the result is subnormal and inexact.
+
+ >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.001'))
+ Decimal('2.170')
+ >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.01'))
+ Decimal('2.17')
+ >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.1'))
+ Decimal('2.2')
+ >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('1e+0'))
+ Decimal('2')
+ >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('1e+1'))
+ Decimal('0E+1')
+ >>> ExtendedContext.quantize(Decimal('-Inf'), Decimal('Infinity'))
+ Decimal('-Infinity')
+ >>> ExtendedContext.quantize(Decimal('2'), Decimal('Infinity'))
+ Decimal('NaN')
+ >>> ExtendedContext.quantize(Decimal('-0.1'), Decimal('1'))
+ Decimal('-0')
+ >>> ExtendedContext.quantize(Decimal('-0'), Decimal('1e+5'))
+ Decimal('-0E+5')
+ >>> ExtendedContext.quantize(Decimal('+35236450.6'), Decimal('1e-2'))
+ Decimal('NaN')
+ >>> ExtendedContext.quantize(Decimal('-35236450.6'), Decimal('1e-2'))
+ Decimal('NaN')
+ >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e-1'))
+ Decimal('217.0')
+ >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e-0'))
+ Decimal('217')
+ >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e+1'))
+ Decimal('2.2E+2')
+ >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e+2'))
+ Decimal('2E+2')
+ >>> ExtendedContext.quantize(1, 2)
+ Decimal('1')
+ >>> ExtendedContext.quantize(Decimal(1), 2)
+ Decimal('1')
+ >>> ExtendedContext.quantize(1, Decimal(2))
+ Decimal('1')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.quantize(b, context=self)
+
+ def radix(self):
+ """Just returns 10, as this is Decimal, :)
+
+ >>> ExtendedContext.radix()
+ Decimal('10')
+ """
+ return Decimal(10)
+
+ def remainder(self, a, b):
+ """Returns the remainder from integer division.
+
+ The result is the residue of the dividend after the operation of
+ calculating integer division as described for divide-integer, rounded
+ to precision digits if necessary. The sign of the result, if
+ non-zero, is the same as that of the original dividend.
+
+ This operation will fail under the same conditions as integer division
+ (that is, if integer division on the same two operands would fail, the
+ remainder cannot be calculated).
+
+ >>> ExtendedContext.remainder(Decimal('2.1'), Decimal('3'))
+ Decimal('2.1')
+ >>> ExtendedContext.remainder(Decimal('10'), Decimal('3'))
+ Decimal('1')
+ >>> ExtendedContext.remainder(Decimal('-10'), Decimal('3'))
+ Decimal('-1')
+ >>> ExtendedContext.remainder(Decimal('10.2'), Decimal('1'))
+ Decimal('0.2')
+ >>> ExtendedContext.remainder(Decimal('10'), Decimal('0.3'))
+ Decimal('0.1')
+ >>> ExtendedContext.remainder(Decimal('3.6'), Decimal('1.3'))
+ Decimal('1.0')
+ >>> ExtendedContext.remainder(22, 6)
+ Decimal('4')
+ >>> ExtendedContext.remainder(Decimal(22), 6)
+ Decimal('4')
+ >>> ExtendedContext.remainder(22, Decimal(6))
+ Decimal('4')
+ """
+ a = _convert_other(a, raiseit=True)
+ r = a.__mod__(b, context=self)
+ if r is NotImplemented:
+ raise TypeError("Unable to convert %s to Decimal" % b)
+ else:
+ return r
+
+ def remainder_near(self, a, b):
+ """Returns to be "a - b * n", where n is the integer nearest the exact
+ value of "x / b" (if two integers are equally near then the even one
+ is chosen). If the result is equal to 0 then its sign will be the
+ sign of a.
+
+ This operation will fail under the same conditions as integer division
+ (that is, if integer division on the same two operands would fail, the
+ remainder cannot be calculated).
+
+ >>> ExtendedContext.remainder_near(Decimal('2.1'), Decimal('3'))
+ Decimal('-0.9')
+ >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('6'))
+ Decimal('-2')
+ >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('3'))
+ Decimal('1')
+ >>> ExtendedContext.remainder_near(Decimal('-10'), Decimal('3'))
+ Decimal('-1')
+ >>> ExtendedContext.remainder_near(Decimal('10.2'), Decimal('1'))
+ Decimal('0.2')
+ >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('0.3'))
+ Decimal('0.1')
+ >>> ExtendedContext.remainder_near(Decimal('3.6'), Decimal('1.3'))
+ Decimal('-0.3')
+ >>> ExtendedContext.remainder_near(3, 11)
+ Decimal('3')
+ >>> ExtendedContext.remainder_near(Decimal(3), 11)
+ Decimal('3')
+ >>> ExtendedContext.remainder_near(3, Decimal(11))
+ Decimal('3')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.remainder_near(b, context=self)
+
+ def rotate(self, a, b):
+ """Returns a rotated copy of a, b times.
+
+ The coefficient of the result is a rotated copy of the digits in
+ the coefficient of the first operand. The number of places of
+ rotation is taken from the absolute value of the second operand,
+ with the rotation being to the left if the second operand is
+ positive or to the right otherwise.
+
+ >>> ExtendedContext.rotate(Decimal('34'), Decimal('8'))
+ Decimal('400000003')
+ >>> ExtendedContext.rotate(Decimal('12'), Decimal('9'))
+ Decimal('12')
+ >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('-2'))
+ Decimal('891234567')
+ >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('0'))
+ Decimal('123456789')
+ >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('+2'))
+ Decimal('345678912')
+ >>> ExtendedContext.rotate(1333333, 1)
+ Decimal('13333330')
+ >>> ExtendedContext.rotate(Decimal(1333333), 1)
+ Decimal('13333330')
+ >>> ExtendedContext.rotate(1333333, Decimal(1))
+ Decimal('13333330')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.rotate(b, context=self)
+
+ def same_quantum(self, a, b):
+ """Returns True if the two operands have the same exponent.
+
+ The result is never affected by either the sign or the coefficient of
+ either operand.
+
+ >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.001'))
+ False
+ >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.01'))
+ True
+ >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('1'))
+ False
+ >>> ExtendedContext.same_quantum(Decimal('Inf'), Decimal('-Inf'))
+ True
+ >>> ExtendedContext.same_quantum(10000, -1)
+ True
+ >>> ExtendedContext.same_quantum(Decimal(10000), -1)
+ True
+ >>> ExtendedContext.same_quantum(10000, Decimal(-1))
+ True
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.same_quantum(b)
+
+ def scaleb (self, a, b):
+ """Returns the first operand after adding the second value its exp.
+
+ >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('-2'))
+ Decimal('0.0750')
+ >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('0'))
+ Decimal('7.50')
+ >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('3'))
+ Decimal('7.50E+3')
+ >>> ExtendedContext.scaleb(1, 4)
+ Decimal('1E+4')
+ >>> ExtendedContext.scaleb(Decimal(1), 4)
+ Decimal('1E+4')
+ >>> ExtendedContext.scaleb(1, Decimal(4))
+ Decimal('1E+4')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.scaleb(b, context=self)
+
+ def shift(self, a, b):
+ """Returns a shifted copy of a, b times.
+
+ The coefficient of the result is a shifted copy of the digits
+ in the coefficient of the first operand. The number of places
+ to shift is taken from the absolute value of the second operand,
+ with the shift being to the left if the second operand is
+ positive or to the right otherwise. Digits shifted into the
+ coefficient are zeros.
+
+ >>> ExtendedContext.shift(Decimal('34'), Decimal('8'))
+ Decimal('400000000')
+ >>> ExtendedContext.shift(Decimal('12'), Decimal('9'))
+ Decimal('0')
+ >>> ExtendedContext.shift(Decimal('123456789'), Decimal('-2'))
+ Decimal('1234567')
+ >>> ExtendedContext.shift(Decimal('123456789'), Decimal('0'))
+ Decimal('123456789')
+ >>> ExtendedContext.shift(Decimal('123456789'), Decimal('+2'))
+ Decimal('345678900')
+ >>> ExtendedContext.shift(88888888, 2)
+ Decimal('888888800')
+ >>> ExtendedContext.shift(Decimal(88888888), 2)
+ Decimal('888888800')
+ >>> ExtendedContext.shift(88888888, Decimal(2))
+ Decimal('888888800')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.shift(b, context=self)
+
+ def sqrt(self, a):
+ """Square root of a non-negative number to context precision.
+
+ If the result must be inexact, it is rounded using the round-half-even
+ algorithm.
+
+ >>> ExtendedContext.sqrt(Decimal('0'))
+ Decimal('0')
+ >>> ExtendedContext.sqrt(Decimal('-0'))
+ Decimal('-0')
+ >>> ExtendedContext.sqrt(Decimal('0.39'))
+ Decimal('0.624499800')
+ >>> ExtendedContext.sqrt(Decimal('100'))
+ Decimal('10')
+ >>> ExtendedContext.sqrt(Decimal('1'))
+ Decimal('1')
+ >>> ExtendedContext.sqrt(Decimal('1.0'))
+ Decimal('1.0')
+ >>> ExtendedContext.sqrt(Decimal('1.00'))
+ Decimal('1.0')
+ >>> ExtendedContext.sqrt(Decimal('7'))
+ Decimal('2.64575131')
+ >>> ExtendedContext.sqrt(Decimal('10'))
+ Decimal('3.16227766')
+ >>> ExtendedContext.sqrt(2)
+ Decimal('1.41421356')
+ >>> ExtendedContext.prec
+ 9
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.sqrt(context=self)
+
+ def subtract(self, a, b):
+ """Return the difference between the two operands.
+
+ >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('1.07'))
+ Decimal('0.23')
+ >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('1.30'))
+ Decimal('0.00')
+ >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('2.07'))
+ Decimal('-0.77')
+ >>> ExtendedContext.subtract(8, 5)
+ Decimal('3')
+ >>> ExtendedContext.subtract(Decimal(8), 5)
+ Decimal('3')
+ >>> ExtendedContext.subtract(8, Decimal(5))
+ Decimal('3')
+ """
+ a = _convert_other(a, raiseit=True)
+ r = a.__sub__(b, context=self)
+ if r is NotImplemented:
+ raise TypeError("Unable to convert %s to Decimal" % b)
+ else:
+ return r
+
+ def to_eng_string(self, a):
+ """Converts a number to a string, using scientific notation.
+
+ The operation is not affected by the context.
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.to_eng_string(context=self)
+
+ def to_sci_string(self, a):
+ """Converts a number to a string, using scientific notation.
+
+ The operation is not affected by the context.
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.__str__(context=self)
+
+ def to_integral_exact(self, a):
+ """Rounds to an integer.
+
+ When the operand has a negative exponent, the result is the same
+ as using the quantize() operation using the given operand as the
+ left-hand-operand, 1E+0 as the right-hand-operand, and the precision
+ of the operand as the precision setting; Inexact and Rounded flags
+ are allowed in this operation. The rounding mode is taken from the
+ context.
+
+ >>> ExtendedContext.to_integral_exact(Decimal('2.1'))
+ Decimal('2')
+ >>> ExtendedContext.to_integral_exact(Decimal('100'))
+ Decimal('100')
+ >>> ExtendedContext.to_integral_exact(Decimal('100.0'))
+ Decimal('100')
+ >>> ExtendedContext.to_integral_exact(Decimal('101.5'))
+ Decimal('102')
+ >>> ExtendedContext.to_integral_exact(Decimal('-101.5'))
+ Decimal('-102')
+ >>> ExtendedContext.to_integral_exact(Decimal('10E+5'))
+ Decimal('1.0E+6')
+ >>> ExtendedContext.to_integral_exact(Decimal('7.89E+77'))
+ Decimal('7.89E+77')
+ >>> ExtendedContext.to_integral_exact(Decimal('-Inf'))
+ Decimal('-Infinity')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.to_integral_exact(context=self)
+
+ def to_integral_value(self, a):
+ """Rounds to an integer.
+
+ When the operand has a negative exponent, the result is the same
+ as using the quantize() operation using the given operand as the
+ left-hand-operand, 1E+0 as the right-hand-operand, and the precision
+ of the operand as the precision setting, except that no flags will
+ be set. The rounding mode is taken from the context.
+
+ >>> ExtendedContext.to_integral_value(Decimal('2.1'))
+ Decimal('2')
+ >>> ExtendedContext.to_integral_value(Decimal('100'))
+ Decimal('100')
+ >>> ExtendedContext.to_integral_value(Decimal('100.0'))
+ Decimal('100')
+ >>> ExtendedContext.to_integral_value(Decimal('101.5'))
+ Decimal('102')
+ >>> ExtendedContext.to_integral_value(Decimal('-101.5'))
+ Decimal('-102')
+ >>> ExtendedContext.to_integral_value(Decimal('10E+5'))
+ Decimal('1.0E+6')
+ >>> ExtendedContext.to_integral_value(Decimal('7.89E+77'))
+ Decimal('7.89E+77')
+ >>> ExtendedContext.to_integral_value(Decimal('-Inf'))
+ Decimal('-Infinity')
+ """
+ a = _convert_other(a, raiseit=True)
+ return a.to_integral_value(context=self)
+
+ # the method name changed, but we provide also the old one, for compatibility
+ to_integral = to_integral_value
+
+class _WorkRep(object):
+ __slots__ = ('sign','int','exp')
+ # sign: 0 or 1
+ # int: int
+ # exp: None, int, or string
+
+ def __init__(self, value=None):
+ if value is None:
+ self.sign = None
+ self.int = 0
+ self.exp = None
+ elif isinstance(value, Decimal):
+ self.sign = value._sign
+ self.int = int(value._int)
+ self.exp = value._exp
+ else:
+ # assert isinstance(value, tuple)
+ self.sign = value[0]
+ self.int = value[1]
+ self.exp = value[2]
+
+ def __repr__(self):
+ return "(%r, %r, %r)" % (self.sign, self.int, self.exp)
+
+ __str__ = __repr__
+
+
+
+def _normalize(op1, op2, prec = 0):
+ """Normalizes op1, op2 to have the same exp and length of coefficient.
+
+ Done during addition.
+ """
+ if op1.exp < op2.exp:
+ tmp = op2
+ other = op1
+ else:
+ tmp = op1
+ other = op2
+
+ # Let exp = min(tmp.exp - 1, tmp.adjusted() - precision - 1).
+ # Then adding 10**exp to tmp has the same effect (after rounding)
+ # as adding any positive quantity smaller than 10**exp; similarly
+ # for subtraction. So if other is smaller than 10**exp we replace
+ # it with 10**exp. This avoids tmp.exp - other.exp getting too large.
+ tmp_len = len(str(tmp.int))
+ other_len = len(str(other.int))
+ exp = tmp.exp + min(-1, tmp_len - prec - 2)
+ if other_len + other.exp - 1 < exp:
+ other.int = 1
+ other.exp = exp
+
+ tmp.int *= 10 ** (tmp.exp - other.exp)
+ tmp.exp = other.exp
+ return op1, op2
+
+##### Integer arithmetic functions used by ln, log10, exp and __pow__ #####
+
+_nbits = int.bit_length
+
+def _decimal_lshift_exact(n, e):
+ """ Given integers n and e, return n * 10**e if it's an integer, else None.
+
+ The computation is designed to avoid computing large powers of 10
+ unnecessarily.
+
+ >>> _decimal_lshift_exact(3, 4)
+ 30000
+ >>> _decimal_lshift_exact(300, -999999999) # returns None
+
+ """
+ if n == 0:
+ return 0
+ elif e >= 0:
+ return n * 10**e
+ else:
+ # val_n = largest power of 10 dividing n.
+ str_n = str(abs(n))
+ val_n = len(str_n) - len(str_n.rstrip('0'))
+ return None if val_n < -e else n // 10**-e
+
+def _sqrt_nearest(n, a):
+ """Closest integer to the square root of the positive integer n. a is
+ an initial approximation to the square root. Any positive integer
+ will do for a, but the closer a is to the square root of n the
+ faster convergence will be.
+
+ """
+ if n <= 0 or a <= 0:
+ raise ValueError("Both arguments to _sqrt_nearest should be positive.")
+
+ b=0
+ while a != b:
+ b, a = a, a--n//a>>1
+ return a
+
+def _rshift_nearest(x, shift):
+ """Given an integer x and a nonnegative integer shift, return closest
+ integer to x / 2**shift; use round-to-even in case of a tie.
+
+ """
+ b, q = 1 << shift, x >> shift
+ return q + (2*(x & (b-1)) + (q&1) > b)
+
+def _div_nearest(a, b):
+ """Closest integer to a/b, a and b positive integers; rounds to even
+ in the case of a tie.
+
+ """
+ q, r = divmod(a, b)
+ return q + (2*r + (q&1) > b)
+
+def _ilog(x, M, L = 8):
+ """Integer approximation to M*log(x/M), with absolute error boundable
+ in terms only of x/M.
+
+ Given positive integers x and M, return an integer approximation to
+ M * log(x/M). For L = 8 and 0.1 <= x/M <= 10 the difference
+ between the approximation and the exact result is at most 22. For
+ L = 8 and 1.0 <= x/M <= 10.0 the difference is at most 15. In
+ both cases these are upper bounds on the error; it will usually be
+ much smaller."""
+
+ # The basic algorithm is the following: let log1p be the function
+ # log1p(x) = log(1+x). Then log(x/M) = log1p((x-M)/M). We use
+ # the reduction
+ #
+ # log1p(y) = 2*log1p(y/(1+sqrt(1+y)))
+ #
+ # repeatedly until the argument to log1p is small (< 2**-L in
+ # absolute value). For small y we can use the Taylor series
+ # expansion
+ #
+ # log1p(y) ~ y - y**2/2 + y**3/3 - ... - (-y)**T/T
+ #
+ # truncating at T such that y**T is small enough. The whole
+ # computation is carried out in a form of fixed-point arithmetic,
+ # with a real number z being represented by an integer
+ # approximation to z*M. To avoid loss of precision, the y below
+ # is actually an integer approximation to 2**R*y*M, where R is the
+ # number of reductions performed so far.
+
+ y = x-M
+ # argument reduction; R = number of reductions performed
+ R = 0
+ while (R <= L and abs(y) << L-R >= M or
+ R > L and abs(y) >> R-L >= M):
+ y = _div_nearest((M*y) << 1,
+ M + _sqrt_nearest(M*(M+_rshift_nearest(y, R)), M))
+ R += 1
+
+ # Taylor series with T terms
+ T = -int(-10*len(str(M))//(3*L))
+ yshift = _rshift_nearest(y, R)
+ w = _div_nearest(M, T)
+ for k in range(T-1, 0, -1):
+ w = _div_nearest(M, k) - _div_nearest(yshift*w, M)
+
+ return _div_nearest(w*y, M)
+
+def _dlog10(c, e, p):
+ """Given integers c, e and p with c > 0, p >= 0, compute an integer
+ approximation to 10**p * log10(c*10**e), with an absolute error of
+ at most 1. Assumes that c*10**e is not exactly 1."""
+
+ # increase precision by 2; compensate for this by dividing
+ # final result by 100
+ p += 2
+
+ # write c*10**e as d*10**f with either:
+ # f >= 0 and 1 <= d <= 10, or
+ # f <= 0 and 0.1 <= d <= 1.
+ # Thus for c*10**e close to 1, f = 0
+ l = len(str(c))
+ f = e+l - (e+l >= 1)
+
+ if p > 0:
+ M = 10**p
+ k = e+p-f
+ if k >= 0:
+ c *= 10**k
+ else:
+ c = _div_nearest(c, 10**-k)
+
+ log_d = _ilog(c, M) # error < 5 + 22 = 27
+ log_10 = _log10_digits(p) # error < 1
+ log_d = _div_nearest(log_d*M, log_10)
+ log_tenpower = f*M # exact
+ else:
+ log_d = 0 # error < 2.31
+ log_tenpower = _div_nearest(f, 10**-p) # error < 0.5
+
+ return _div_nearest(log_tenpower+log_d, 100)
+
+def _dlog(c, e, p):
+ """Given integers c, e and p with c > 0, compute an integer
+ approximation to 10**p * log(c*10**e), with an absolute error of
+ at most 1. Assumes that c*10**e is not exactly 1."""
+
+ # Increase precision by 2. The precision increase is compensated
+ # for at the end with a division by 100.
+ p += 2
+
+ # rewrite c*10**e as d*10**f with either f >= 0 and 1 <= d <= 10,
+ # or f <= 0 and 0.1 <= d <= 1. Then we can compute 10**p * log(c*10**e)
+ # as 10**p * log(d) + 10**p*f * log(10).
+ l = len(str(c))
+ f = e+l - (e+l >= 1)
+
+ # compute approximation to 10**p*log(d), with error < 27
+ if p > 0:
+ k = e+p-f
+ if k >= 0:
+ c *= 10**k
+ else:
+ c = _div_nearest(c, 10**-k) # error of <= 0.5 in c
+
+ # _ilog magnifies existing error in c by a factor of at most 10
+ log_d = _ilog(c, 10**p) # error < 5 + 22 = 27
+ else:
+ # p <= 0: just approximate the whole thing by 0; error < 2.31
+ log_d = 0
+
+ # compute approximation to f*10**p*log(10), with error < 11.
+ if f:
+ extra = len(str(abs(f)))-1
+ if p + extra >= 0:
+ # error in f * _log10_digits(p+extra) < |f| * 1 = |f|
+ # after division, error < |f|/10**extra + 0.5 < 10 + 0.5 < 11
+ f_log_ten = _div_nearest(f*_log10_digits(p+extra), 10**extra)
+ else:
+ f_log_ten = 0
+ else:
+ f_log_ten = 0
+
+ # error in sum < 11+27 = 38; error after division < 0.38 + 0.5 < 1
+ return _div_nearest(f_log_ten + log_d, 100)
+
+class _Log10Memoize(object):
+ """Class to compute, store, and allow retrieval of, digits of the
+ constant log(10) = 2.302585.... This constant is needed by
+ Decimal.ln, Decimal.log10, Decimal.exp and Decimal.__pow__."""
+ def __init__(self):
+ self.digits = "23025850929940456840179914546843642076011014886"
+
+ def getdigits(self, p):
+ """Given an integer p >= 0, return floor(10**p)*log(10).
+
+ For example, self.getdigits(3) returns 2302.
+ """
+ # digits are stored as a string, for quick conversion to
+ # integer in the case that we've already computed enough
+ # digits; the stored digits should always be correct
+ # (truncated, not rounded to nearest).
+ if p < 0:
+ raise ValueError("p should be nonnegative")
+
+ if p >= len(self.digits):
+ # compute p+3, p+6, p+9, ... digits; continue until at
+ # least one of the extra digits is nonzero
+ extra = 3
+ while True:
+ # compute p+extra digits, correct to within 1ulp
+ M = 10**(p+extra+2)
+ digits = str(_div_nearest(_ilog(10*M, M), 100))
+ if digits[-extra:] != '0'*extra:
+ break
+ extra += 3
+ # keep all reliable digits so far; remove trailing zeros
+ # and next nonzero digit
+ self.digits = digits.rstrip('0')[:-1]
+ return int(self.digits[:p+1])
+
+_log10_digits = _Log10Memoize().getdigits
+
+def _iexp(x, M, L=8):
+ """Given integers x and M, M > 0, such that x/M is small in absolute
+ value, compute an integer approximation to M*exp(x/M). For 0 <=
+ x/M <= 2.4, the absolute error in the result is bounded by 60 (and
+ is usually much smaller)."""
+
+ # Algorithm: to compute exp(z) for a real number z, first divide z
+ # by a suitable power R of 2 so that |z/2**R| < 2**-L. Then
+ # compute expm1(z/2**R) = exp(z/2**R) - 1 using the usual Taylor
+ # series
+ #
+ # expm1(x) = x + x**2/2! + x**3/3! + ...
+ #
+ # Now use the identity
+ #
+ # expm1(2x) = expm1(x)*(expm1(x)+2)
+ #
+ # R times to compute the sequence expm1(z/2**R),
+ # expm1(z/2**(R-1)), ... , exp(z/2), exp(z).
+
+ # Find R such that x/2**R/M <= 2**-L
+ R = _nbits((x<<L)//M)
+
+ # Taylor series. (2**L)**T > M
+ T = -int(-10*len(str(M))//(3*L))
+ y = _div_nearest(x, T)
+ Mshift = M<<R
+ for i in range(T-1, 0, -1):
+ y = _div_nearest(x*(Mshift + y), Mshift * i)
+
+ # Expansion
+ for k in range(R-1, -1, -1):
+ Mshift = M<<(k+2)
+ y = _div_nearest(y*(y+Mshift), Mshift)
+
+ return M+y
+
+def _dexp(c, e, p):
+ """Compute an approximation to exp(c*10**e), with p decimal places of
+ precision.
+
+ Returns integers d, f such that:
+
+ 10**(p-1) <= d <= 10**p, and
+ (d-1)*10**f < exp(c*10**e) < (d+1)*10**f
+
+ In other words, d*10**f is an approximation to exp(c*10**e) with p
+ digits of precision, and with an error in d of at most 1. This is
+ almost, but not quite, the same as the error being < 1ulp: when d
+ = 10**(p-1) the error could be up to 10 ulp."""
+
+ # we'll call iexp with M = 10**(p+2), giving p+3 digits of precision
+ p += 2
+
+ # compute log(10) with extra precision = adjusted exponent of c*10**e
+ extra = max(0, e + len(str(c)) - 1)
+ q = p + extra
+
+ # compute quotient c*10**e/(log(10)) = c*10**(e+q)/(log(10)*10**q),
+ # rounding down
+ shift = e+q
+ if shift >= 0:
+ cshift = c*10**shift
+ else:
+ cshift = c//10**-shift
+ quot, rem = divmod(cshift, _log10_digits(q))
+
+ # reduce remainder back to original precision
+ rem = _div_nearest(rem, 10**extra)
+
+ # error in result of _iexp < 120; error after division < 0.62
+ return _div_nearest(_iexp(rem, 10**p), 1000), quot - p + 3
+
+def _dpower(xc, xe, yc, ye, p):
+ """Given integers xc, xe, yc and ye representing Decimals x = xc*10**xe and
+ y = yc*10**ye, compute x**y. Returns a pair of integers (c, e) such that:
+
+ 10**(p-1) <= c <= 10**p, and
+ (c-1)*10**e < x**y < (c+1)*10**e
+
+ in other words, c*10**e is an approximation to x**y with p digits
+ of precision, and with an error in c of at most 1. (This is
+ almost, but not quite, the same as the error being < 1ulp: when c
+ == 10**(p-1) we can only guarantee error < 10ulp.)
+
+ We assume that: x is positive and not equal to 1, and y is nonzero.
+ """
+
+ # Find b such that 10**(b-1) <= |y| <= 10**b
+ b = len(str(abs(yc))) + ye
+
+ # log(x) = lxc*10**(-p-b-1), to p+b+1 places after the decimal point
+ lxc = _dlog(xc, xe, p+b+1)
+
+ # compute product y*log(x) = yc*lxc*10**(-p-b-1+ye) = pc*10**(-p-1)
+ shift = ye-b
+ if shift >= 0:
+ pc = lxc*yc*10**shift
+ else:
+ pc = _div_nearest(lxc*yc, 10**-shift)
+
+ if pc == 0:
+ # we prefer a result that isn't exactly 1; this makes it
+ # easier to compute a correctly rounded result in __pow__
+ if ((len(str(xc)) + xe >= 1) == (yc > 0)): # if x**y > 1:
+ coeff, exp = 10**(p-1)+1, 1-p
+ else:
+ coeff, exp = 10**p-1, -p
+ else:
+ coeff, exp = _dexp(pc, -(p+1), p+1)
+ coeff = _div_nearest(coeff, 10)
+ exp += 1
+
+ return coeff, exp
+
+def _log10_lb(c, correction = {
+ '1': 100, '2': 70, '3': 53, '4': 40, '5': 31,
+ '6': 23, '7': 16, '8': 10, '9': 5}):
+ """Compute a lower bound for 100*log10(c) for a positive integer c."""
+ if c <= 0:
+ raise ValueError("The argument to _log10_lb should be nonnegative.")
+ str_c = str(c)
+ return 100*len(str_c) - correction[str_c[0]]
+
+##### Helper Functions ####################################################
+
+def _convert_other(other, raiseit=False, allow_float=False):
+ """Convert other to Decimal.
+
+ Verifies that it's ok to use in an implicit construction.
+ If allow_float is true, allow conversion from float; this
+ is used in the comparison methods (__eq__ and friends).
+
+ """
+ if isinstance(other, Decimal):
+ return other
+ if isinstance(other, int):
+ return Decimal(other)
+ if allow_float and isinstance(other, float):
+ return Decimal.from_float(other)
+
+ if raiseit:
+ raise TypeError("Unable to convert %s to Decimal" % other)
+ return NotImplemented
+
+def _convert_for_comparison(self, other, equality_op=False):
+ """Given a Decimal instance self and a Python object other, return
+ a pair (s, o) of Decimal instances such that "s op o" is
+ equivalent to "self op other" for any of the 6 comparison
+ operators "op".
+
+ """
+ if isinstance(other, Decimal):
+ return self, other
+
+ # Comparison with a Rational instance (also includes integers):
+ # self op n/d <=> self*d op n (for n and d integers, d positive).
+ # A NaN or infinity can be left unchanged without affecting the
+ # comparison result.
+ if isinstance(other, _numbers.Rational):
+ if not self._is_special:
+ self = _dec_from_triple(self._sign,
+ str(int(self._int) * other.denominator),
+ self._exp)
+ return self, Decimal(other.numerator)
+
+ # Comparisons with float and complex types. == and != comparisons
+ # with complex numbers should succeed, returning either True or False
+ # as appropriate. Other comparisons return NotImplemented.
+ if equality_op and isinstance(other, _numbers.Complex) and other.imag == 0:
+ other = other.real
+ if isinstance(other, float):
+ context = getcontext()
+ if equality_op:
+ context.flags[FloatOperation] = 1
+ else:
+ context._raise_error(FloatOperation,
+ "strict semantics for mixing floats and Decimals are enabled")
+ return self, Decimal.from_float(other)
+ return NotImplemented, NotImplemented
+
+
+##### Setup Specific Contexts ############################################
+
+# The default context prototype used by Context()
+# Is mutable, so that new contexts can have different default values
+
+DefaultContext = Context(
+ prec=28, rounding=ROUND_HALF_EVEN,
+ traps=[DivisionByZero, Overflow, InvalidOperation],
+ flags=[],
+ Emax=999999,
+ Emin=-999999,
+ capitals=1,
+ clamp=0
+)
+
+# Pre-made alternate contexts offered by the specification
+# Don't change these; the user should be able to select these
+# contexts and be able to reproduce results from other implementations
+# of the spec.
+
+BasicContext = Context(
+ prec=9, rounding=ROUND_HALF_UP,
+ traps=[DivisionByZero, Overflow, InvalidOperation, Clamped, Underflow],
+ flags=[],
+)
+
+ExtendedContext = Context(
+ prec=9, rounding=ROUND_HALF_EVEN,
+ traps=[],
+ flags=[],
+)
+
+
+##### crud for parsing strings #############################################
+#
+# Regular expression used for parsing numeric strings. Additional
+# comments:
+#
+# 1. Uncomment the two '\s*' lines to allow leading and/or trailing
+# whitespace. But note that the specification disallows whitespace in
+# a numeric string.
+#
+# 2. For finite numbers (not infinities and NaNs) the body of the
+# number between the optional sign and the optional exponent must have
+# at least one decimal digit, possibly after the decimal point. The
+# lookahead expression '(?=\d|\.\d)' checks this.
+
+import re
+_parser = re.compile(r""" # A numeric string consists of:
+# \s*
+ (?P<sign>[-+])? # an optional sign, followed by either...
+ (
+ (?=\d|\.\d) # ...a number (with at least one digit)
+ (?P<int>\d*) # having a (possibly empty) integer part
+ (\.(?P<frac>\d*))? # followed by an optional fractional part
+ (E(?P<exp>[-+]?\d+))? # followed by an optional exponent, or...
+ |
+ Inf(inity)? # ...an infinity, or...
+ |
+ (?P<signal>s)? # ...an (optionally signaling)
+ NaN # NaN
+ (?P<diag>\d*) # with (possibly empty) diagnostic info.
+ )
+# \s*
+ \Z
+""", re.VERBOSE | re.IGNORECASE).match
+
+_all_zeros = re.compile('0*$').match
+_exact_half = re.compile('50*$').match
+
+##### PEP3101 support functions ##############################################
+# The functions in this section have little to do with the Decimal
+# class, and could potentially be reused or adapted for other pure
+# Python numeric classes that want to implement __format__
+#
+# A format specifier for Decimal looks like:
+#
+# [[fill]align][sign][#][0][minimumwidth][,][.precision][type]
+
+_parse_format_specifier_regex = re.compile(r"""\A
+(?:
+ (?P<fill>.)?
+ (?P<align>[<>=^])
+)?
+(?P<sign>[-+ ])?
+(?P<alt>\#)?
+(?P<zeropad>0)?
+(?P<minimumwidth>(?!0)\d+)?
+(?P<thousands_sep>,)?
+(?:\.(?P<precision>0|(?!0)\d+))?
+(?P<type>[eEfFgGn%])?
+\Z
+""", re.VERBOSE|re.DOTALL)
+
+del re
+
+# The locale module is only needed for the 'n' format specifier. The
+# rest of the PEP 3101 code functions quite happily without it, so we
+# don't care too much if locale isn't present.
+try:
+ import locale as _locale
+except ImportError:
+ pass
+
+def _parse_format_specifier(format_spec, _localeconv=None):
+ """Parse and validate a format specifier.
+
+ Turns a standard numeric format specifier into a dict, with the
+ following entries:
+
+ fill: fill character to pad field to minimum width
+ align: alignment type, either '<', '>', '=' or '^'
+ sign: either '+', '-' or ' '
+ minimumwidth: nonnegative integer giving minimum width
+ zeropad: boolean, indicating whether to pad with zeros
+ thousands_sep: string to use as thousands separator, or ''
+ grouping: grouping for thousands separators, in format
+ used by localeconv
+ decimal_point: string to use for decimal point
+ precision: nonnegative integer giving precision, or None
+ type: one of the characters 'eEfFgG%', or None
+
+ """
+ m = _parse_format_specifier_regex.match(format_spec)
+ if m is None:
+ raise ValueError("Invalid format specifier: " + format_spec)
+
+ # get the dictionary
+ format_dict = m.groupdict()
+
+ # zeropad; defaults for fill and alignment. If zero padding
+ # is requested, the fill and align fields should be absent.
+ fill = format_dict['fill']
+ align = format_dict['align']
+ format_dict['zeropad'] = (format_dict['zeropad'] is not None)
+ if format_dict['zeropad']:
+ if fill is not None:
+ raise ValueError("Fill character conflicts with '0'"
+ " in format specifier: " + format_spec)
+ if align is not None:
+ raise ValueError("Alignment conflicts with '0' in "
+ "format specifier: " + format_spec)
+ format_dict['fill'] = fill or ' '
+ # PEP 3101 originally specified that the default alignment should
+ # be left; it was later agreed that right-aligned makes more sense
+ # for numeric types. See http://bugs.python.org/issue6857.
+ format_dict['align'] = align or '>'
+
+ # default sign handling: '-' for negative, '' for positive
+ if format_dict['sign'] is None:
+ format_dict['sign'] = '-'
+
+ # minimumwidth defaults to 0; precision remains None if not given
+ format_dict['minimumwidth'] = int(format_dict['minimumwidth'] or '0')
+ if format_dict['precision'] is not None:
+ format_dict['precision'] = int(format_dict['precision'])
+
+ # if format type is 'g' or 'G' then a precision of 0 makes little
+ # sense; convert it to 1. Same if format type is unspecified.
+ if format_dict['precision'] == 0:
+ if format_dict['type'] is None or format_dict['type'] in 'gGn':
+ format_dict['precision'] = 1
+
+ # determine thousands separator, grouping, and decimal separator, and
+ # add appropriate entries to format_dict
+ if format_dict['type'] == 'n':
+ # apart from separators, 'n' behaves just like 'g'
+ format_dict['type'] = 'g'
+ if _localeconv is None:
+ _localeconv = _locale.localeconv()
+ if format_dict['thousands_sep'] is not None:
+ raise ValueError("Explicit thousands separator conflicts with "
+ "'n' type in format specifier: " + format_spec)
+ format_dict['thousands_sep'] = _localeconv['thousands_sep']
+ format_dict['grouping'] = _localeconv['grouping']
+ format_dict['decimal_point'] = _localeconv['decimal_point']
+ else:
+ if format_dict['thousands_sep'] is None:
+ format_dict['thousands_sep'] = ''
+ format_dict['grouping'] = [3, 0]
+ format_dict['decimal_point'] = '.'
+
+ return format_dict
+
+def _format_align(sign, body, spec):
+ """Given an unpadded, non-aligned numeric string 'body' and sign
+ string 'sign', add padding and alignment conforming to the given
+ format specifier dictionary 'spec' (as produced by
+ parse_format_specifier).
+
+ """
+ # how much extra space do we have to play with?
+ minimumwidth = spec['minimumwidth']
+ fill = spec['fill']
+ padding = fill*(minimumwidth - len(sign) - len(body))
+
+ align = spec['align']
+ if align == '<':
+ result = sign + body + padding
+ elif align == '>':
+ result = padding + sign + body
+ elif align == '=':
+ result = sign + padding + body
+ elif align == '^':
+ half = len(padding)//2
+ result = padding[:half] + sign + body + padding[half:]
+ else:
+ raise ValueError('Unrecognised alignment field')
+
+ return result
+
+def _group_lengths(grouping):
+ """Convert a localeconv-style grouping into a (possibly infinite)
+ iterable of integers representing group lengths.
+
+ """
+ # The result from localeconv()['grouping'], and the input to this
+ # function, should be a list of integers in one of the
+ # following three forms:
+ #
+ # (1) an empty list, or
+ # (2) nonempty list of positive integers + [0]
+ # (3) list of positive integers + [locale.CHAR_MAX], or
+
+ from itertools import chain, repeat
+ if not grouping:
+ return []
+ elif grouping[-1] == 0 and len(grouping) >= 2:
+ return chain(grouping[:-1], repeat(grouping[-2]))
+ elif grouping[-1] == _locale.CHAR_MAX:
+ return grouping[:-1]
+ else:
+ raise ValueError('unrecognised format for grouping')
+
+def _insert_thousands_sep(digits, spec, min_width=1):
+ """Insert thousands separators into a digit string.
+
+ spec is a dictionary whose keys should include 'thousands_sep' and
+ 'grouping'; typically it's the result of parsing the format
+ specifier using _parse_format_specifier.
+
+ The min_width keyword argument gives the minimum length of the
+ result, which will be padded on the left with zeros if necessary.
+
+ If necessary, the zero padding adds an extra '0' on the left to
+ avoid a leading thousands separator. For example, inserting
+ commas every three digits in '123456', with min_width=8, gives
+ '0,123,456', even though that has length 9.
+
+ """
+
+ sep = spec['thousands_sep']
+ grouping = spec['grouping']
+
+ groups = []
+ for l in _group_lengths(grouping):
+ if l <= 0:
+ raise ValueError("group length should be positive")
+ # max(..., 1) forces at least 1 digit to the left of a separator
+ l = min(max(len(digits), min_width, 1), l)
+ groups.append('0'*(l - len(digits)) + digits[-l:])
+ digits = digits[:-l]
+ min_width -= l
+ if not digits and min_width <= 0:
+ break
+ min_width -= len(sep)
+ else:
+ l = max(len(digits), min_width, 1)
+ groups.append('0'*(l - len(digits)) + digits[-l:])
+ return sep.join(reversed(groups))
+
+def _format_sign(is_negative, spec):
+ """Determine sign character."""
+
+ if is_negative:
+ return '-'
+ elif spec['sign'] in ' +':
+ return spec['sign']
+ else:
+ return ''
+
+def _format_number(is_negative, intpart, fracpart, exp, spec):
+ """Format a number, given the following data:
+
+ is_negative: true if the number is negative, else false
+ intpart: string of digits that must appear before the decimal point
+ fracpart: string of digits that must come after the point
+ exp: exponent, as an integer
+ spec: dictionary resulting from parsing the format specifier
+
+ This function uses the information in spec to:
+ insert separators (decimal separator and thousands separators)
+ format the sign
+ format the exponent
+ add trailing '%' for the '%' type
+ zero-pad if necessary
+ fill and align if necessary
+ """
+
+ sign = _format_sign(is_negative, spec)
+
+ if fracpart or spec['alt']:
+ fracpart = spec['decimal_point'] + fracpart
+
+ if exp != 0 or spec['type'] in 'eE':
+ echar = {'E': 'E', 'e': 'e', 'G': 'E', 'g': 'e'}[spec['type']]
+ fracpart += "{0}{1:+}".format(echar, exp)
+ if spec['type'] == '%':
+ fracpart += '%'
+
+ if spec['zeropad']:
+ min_width = spec['minimumwidth'] - len(fracpart) - len(sign)
+ else:
+ min_width = 0
+ intpart = _insert_thousands_sep(intpart, spec, min_width)
+
+ return _format_align(sign, intpart+fracpart, spec)
+
+
+##### Useful Constants (internal use only) ################################
+
+# Reusable defaults
+_Infinity = Decimal('Inf')
+_NegativeInfinity = Decimal('-Inf')
+_NaN = Decimal('NaN')
+_Zero = Decimal(0)
+_One = Decimal(1)
+_NegativeOne = Decimal(-1)
+
+# _SignedInfinity[sign] is infinity w/ that sign
+_SignedInfinity = (_Infinity, _NegativeInfinity)
+
+# Constants related to the hash implementation; hash(x) is based
+# on the reduction of x modulo _PyHASH_MODULUS
+_PyHASH_MODULUS = sys.hash_info.modulus
+# hash values to use for positive and negative infinities, and nans
+_PyHASH_INF = sys.hash_info.inf
+_PyHASH_NAN = sys.hash_info.nan
+
+# _PyHASH_10INV is the inverse of 10 modulo the prime _PyHASH_MODULUS
+_PyHASH_10INV = pow(10, _PyHASH_MODULUS - 2, _PyHASH_MODULUS)
+del sys
diff --git a/Lib/_pyio.py b/Lib/_pyio.py
index c0b5fd1..f47df91 100644
--- a/Lib/_pyio.py
+++ b/Lib/_pyio.py
@@ -6,11 +6,18 @@ import os
import abc
import codecs
import errno
+import array
+import stat
+import sys
# Import _thread instead of threading to reduce startup cost
try:
from _thread import allocate_lock as Lock
except ImportError:
from _dummy_thread import allocate_lock as Lock
+if sys.platform in {'win32', 'cygwin'}:
+ from msvcrt import setmode as _setmode
+else:
+ _setmode = None
import io
from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END)
@@ -256,7 +263,7 @@ class OpenWrapper:
Trick so that open won't become a bound method when stored
as a class variable (as dbm.dumb does).
- See initstdio() in Python/pythonrun.c.
+ See initstdio() in Python/pylifecycle.c.
"""
__doc__ = DocDescriptor()
@@ -662,16 +669,33 @@ class BufferedIOBase(IOBase):
Raises BlockingIOError if the underlying raw stream has no
data at the moment.
"""
- # XXX This ought to work with anything that supports the buffer API
- data = self.read(len(b))
+
+ return self._readinto(b, read1=False)
+
+ def readinto1(self, b):
+ """Read up to len(b) bytes into *b*, using at most one system call
+
+ Returns an int representing the number of bytes read (0 for EOF).
+
+ Raises BlockingIOError if the underlying raw stream has no
+ data at the moment.
+ """
+
+ return self._readinto(b, read1=True)
+
+ def _readinto(self, b, read1):
+ if not isinstance(b, memoryview):
+ b = memoryview(b)
+ b = b.cast('B')
+
+ if read1:
+ data = self.read1(len(b))
+ else:
+ data = self.read(len(b))
n = len(data)
- try:
- b[:n] = data
- except TypeError as err:
- import array
- if not isinstance(b, array.array):
- raise err
- b[:n] = array.array('b', data)
+
+ b[:n] = data
+
return n
def write(self, b):
@@ -790,13 +814,14 @@ class _BufferedIOMixin(BufferedIOBase):
.format(self.__class__.__name__))
def __repr__(self):
- clsname = self.__class__.__name__
+ modname = self.__class__.__module__
+ clsname = self.__class__.__qualname__
try:
name = self.name
except Exception:
- return "<_pyio.{0}>".format(clsname)
+ return "<{}.{}>".format(modname, clsname)
else:
- return "<_pyio.{0} name={1!r}>".format(clsname, name)
+ return "<{}.{} name={!r}>".format(modname, clsname, name)
### Lower-level APIs ###
@@ -993,10 +1018,7 @@ class BufferedReader(_BufferedIOMixin):
current_size = 0
while True:
# Read until EOF or until read() would block.
- try:
- chunk = self.raw.read()
- except InterruptedError:
- continue
+ chunk = self.raw.read()
if chunk in empty_values:
nodata_val = chunk
break
@@ -1015,10 +1037,7 @@ class BufferedReader(_BufferedIOMixin):
chunks = [buf[pos:]]
wanted = max(self.buffer_size, n)
while avail < n:
- try:
- chunk = self.raw.read(wanted)
- except InterruptedError:
- continue
+ chunk = self.raw.read(wanted)
if chunk in empty_values:
nodata_val = chunk
break
@@ -1047,12 +1066,7 @@ class BufferedReader(_BufferedIOMixin):
have = len(self._read_buf) - self._read_pos
if have < want or have <= 0:
to_read = self.buffer_size - have
- while True:
- try:
- current = self.raw.read(to_read)
- except InterruptedError:
- continue
- break
+ current = self.raw.read(to_read)
if current:
self._read_buf = self._read_buf[self._read_pos:] + current
self._read_pos = 0
@@ -1071,6 +1085,58 @@ class BufferedReader(_BufferedIOMixin):
return self._read_unlocked(
min(size, len(self._read_buf) - self._read_pos))
+ # Implementing readinto() and readinto1() is not strictly necessary (we
+ # could rely on the base class that provides an implementation in terms of
+ # read() and read1()). We do it anyway to keep the _pyio implementation
+ # similar to the io implementation (which implements the methods for
+ # performance reasons).
+ def _readinto(self, buf, read1):
+ """Read data into *buf* with at most one system call."""
+
+ if len(buf) == 0:
+ return 0
+
+ # Need to create a memoryview object of type 'b', otherwise
+ # we may not be able to assign bytes to it, and slicing it
+ # would create a new object.
+ if not isinstance(buf, memoryview):
+ buf = memoryview(buf)
+ buf = buf.cast('B')
+
+ written = 0
+ with self._read_lock:
+ while written < len(buf):
+
+ # First try to read from internal buffer
+ avail = min(len(self._read_buf) - self._read_pos, len(buf))
+ if avail:
+ buf[written:written+avail] = \
+ self._read_buf[self._read_pos:self._read_pos+avail]
+ self._read_pos += avail
+ written += avail
+ if written == len(buf):
+ break
+
+ # If remaining space in callers buffer is larger than
+ # internal buffer, read directly into callers buffer
+ if len(buf) - written > self.buffer_size:
+ n = self.raw.readinto(buf[written:])
+ if not n:
+ break # eof
+ written += n
+
+ # Otherwise refill internal buffer - unless we're
+ # in read1 mode and already got some data
+ elif not (read1 and written):
+ if not self._peek_unlocked(1):
+ break # eof
+
+ # In readinto1 mode, return as soon as we have some data
+ if read1 and written:
+ break
+
+ return written
+
def tell(self):
return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos
@@ -1149,8 +1215,6 @@ class BufferedWriter(_BufferedIOMixin):
while self._write_buf:
try:
n = self.raw.write(self._write_buf)
- except InterruptedError:
- continue
except BlockingIOError:
raise RuntimeError("self.raw should implement RawIOBase: it "
"should not raise BlockingIOError")
@@ -1220,6 +1284,9 @@ class BufferedRWPair(BufferedIOBase):
def read1(self, size):
return self.reader.read1(size)
+ def readinto1(self, b):
+ return self.reader.readinto1(b)
+
def readable(self):
return self.reader.readable()
@@ -1304,6 +1371,10 @@ class BufferedRandom(BufferedWriter, BufferedReader):
self.flush()
return BufferedReader.read1(self, size)
+ def readinto1(self, b):
+ self.flush()
+ return BufferedReader.readinto1(self, b)
+
def write(self, b):
if self._read_buf:
# Undo readahead
@@ -1313,6 +1384,345 @@ class BufferedRandom(BufferedWriter, BufferedReader):
return BufferedWriter.write(self, b)
+class FileIO(RawIOBase):
+ _fd = -1
+ _created = False
+ _readable = False
+ _writable = False
+ _appending = False
+ _seekable = None
+ _closefd = True
+
+ def __init__(self, file, mode='r', closefd=True, opener=None):
+ """Open a file. The mode can be 'r' (default), 'w', 'x' or 'a' for reading,
+ writing, exclusive creation or appending. The file will be created if it
+ doesn't exist when opened for writing or appending; it will be truncated
+ when opened for writing. A FileExistsError will be raised if it already
+ exists when opened for creating. Opening a file for creating implies
+ writing so this mode behaves in a similar way to 'w'. Add a '+' to the mode
+ to allow simultaneous reading and writing. A custom opener can be used by
+ passing a callable as *opener*. The underlying file descriptor for the file
+ object is then obtained by calling opener with (*name*, *flags*).
+ *opener* must return an open file descriptor (passing os.open as *opener*
+ results in functionality similar to passing None).
+ """
+ if self._fd >= 0:
+ # Have to close the existing file first.
+ try:
+ if self._closefd:
+ os.close(self._fd)
+ finally:
+ self._fd = -1
+
+ if isinstance(file, float):
+ raise TypeError('integer argument expected, got float')
+ if isinstance(file, int):
+ fd = file
+ if fd < 0:
+ raise ValueError('negative file descriptor')
+ else:
+ fd = -1
+
+ if not isinstance(mode, str):
+ raise TypeError('invalid mode: %s' % (mode,))
+ if not set(mode) <= set('xrwab+'):
+ raise ValueError('invalid mode: %s' % (mode,))
+ if sum(c in 'rwax' for c in mode) != 1 or mode.count('+') > 1:
+ raise ValueError('Must have exactly one of create/read/write/append '
+ 'mode and at most one plus')
+
+ if 'x' in mode:
+ self._created = True
+ self._writable = True
+ flags = os.O_EXCL | os.O_CREAT
+ elif 'r' in mode:
+ self._readable = True
+ flags = 0
+ elif 'w' in mode:
+ self._writable = True
+ flags = os.O_CREAT | os.O_TRUNC
+ elif 'a' in mode:
+ self._writable = True
+ self._appending = True
+ flags = os.O_APPEND | os.O_CREAT
+
+ if '+' in mode:
+ self._readable = True
+ self._writable = True
+
+ if self._readable and self._writable:
+ flags |= os.O_RDWR
+ elif self._readable:
+ flags |= os.O_RDONLY
+ else:
+ flags |= os.O_WRONLY
+
+ flags |= getattr(os, 'O_BINARY', 0)
+
+ noinherit_flag = (getattr(os, 'O_NOINHERIT', 0) or
+ getattr(os, 'O_CLOEXEC', 0))
+ flags |= noinherit_flag
+
+ owned_fd = None
+ try:
+ if fd < 0:
+ if not closefd:
+ raise ValueError('Cannot use closefd=False with file name')
+ if opener is None:
+ fd = os.open(file, flags, 0o666)
+ else:
+ fd = opener(file, flags)
+ if not isinstance(fd, int):
+ raise TypeError('expected integer from opener')
+ if fd < 0:
+ raise OSError('Negative file descriptor')
+ owned_fd = fd
+ if not noinherit_flag:
+ os.set_inheritable(fd, False)
+
+ self._closefd = closefd
+ fdfstat = os.fstat(fd)
+ try:
+ if stat.S_ISDIR(fdfstat.st_mode):
+ raise IsADirectoryError(errno.EISDIR,
+ os.strerror(errno.EISDIR), file)
+ except AttributeError:
+ # Ignore the AttribueError if stat.S_ISDIR or errno.EISDIR
+ # don't exist.
+ pass
+ self._blksize = getattr(fdfstat, 'st_blksize', 0)
+ if self._blksize <= 1:
+ self._blksize = DEFAULT_BUFFER_SIZE
+
+ if _setmode:
+ # don't translate newlines (\r\n <=> \n)
+ _setmode(fd, os.O_BINARY)
+
+ self.name = file
+ if self._appending:
+ # For consistent behaviour, we explicitly seek to the
+ # end of file (otherwise, it might be done only on the
+ # first write()).
+ os.lseek(fd, 0, SEEK_END)
+ except:
+ if owned_fd is not None:
+ os.close(owned_fd)
+ raise
+ self._fd = fd
+
+ def __del__(self):
+ if self._fd >= 0 and self._closefd and not self.closed:
+ import warnings
+ warnings.warn('unclosed file %r' % (self,), ResourceWarning,
+ stacklevel=2)
+ self.close()
+
+ def __getstate__(self):
+ raise TypeError("cannot serialize '%s' object", self.__class__.__name__)
+
+ def __repr__(self):
+ class_name = '%s.%s' % (self.__class__.__module__,
+ self.__class__.__qualname__)
+ if self.closed:
+ return '<%s [closed]>' % class_name
+ try:
+ name = self.name
+ except AttributeError:
+ return ('<%s fd=%d mode=%r closefd=%r>' %
+ (class_name, self._fd, self.mode, self._closefd))
+ else:
+ return ('<%s name=%r mode=%r closefd=%r>' %
+ (class_name, name, self.mode, self._closefd))
+
+ def _checkReadable(self):
+ if not self._readable:
+ raise UnsupportedOperation('File not open for reading')
+
+ def _checkWritable(self, msg=None):
+ if not self._writable:
+ raise UnsupportedOperation('File not open for writing')
+
+ def read(self, size=None):
+ """Read at most size bytes, returned as bytes.
+
+ Only makes one system call, so less data may be returned than requested
+ In non-blocking mode, returns None if no data is available.
+ Return an empty bytes object at EOF.
+ """
+ self._checkClosed()
+ self._checkReadable()
+ if size is None or size < 0:
+ return self.readall()
+ try:
+ return os.read(self._fd, size)
+ except BlockingIOError:
+ return None
+
+ def readall(self):
+ """Read all data from the file, returned as bytes.
+
+ In non-blocking mode, returns as much as is immediately available,
+ or None if no data is available. Return an empty bytes object at EOF.
+ """
+ self._checkClosed()
+ self._checkReadable()
+ bufsize = DEFAULT_BUFFER_SIZE
+ try:
+ pos = os.lseek(self._fd, 0, SEEK_CUR)
+ end = os.fstat(self._fd).st_size
+ if end >= pos:
+ bufsize = end - pos + 1
+ except OSError:
+ pass
+
+ result = bytearray()
+ while True:
+ if len(result) >= bufsize:
+ bufsize = len(result)
+ bufsize += max(bufsize, DEFAULT_BUFFER_SIZE)
+ n = bufsize - len(result)
+ try:
+ chunk = os.read(self._fd, n)
+ except BlockingIOError:
+ if result:
+ break
+ return None
+ if not chunk: # reached the end of the file
+ break
+ result += chunk
+
+ return bytes(result)
+
+ def readinto(self, b):
+ """Same as RawIOBase.readinto()."""
+ m = memoryview(b).cast('B')
+ data = self.read(len(m))
+ n = len(data)
+ m[:n] = data
+ return n
+
+ def write(self, b):
+ """Write bytes b to file, return number written.
+
+ Only makes one system call, so not all of the data may be written.
+ The number of bytes actually written is returned. In non-blocking mode,
+ returns None if the write would block.
+ """
+ self._checkClosed()
+ self._checkWritable()
+ try:
+ return os.write(self._fd, b)
+ except BlockingIOError:
+ return None
+
+ def seek(self, pos, whence=SEEK_SET):
+ """Move to new file position.
+
+ Argument offset is a byte count. Optional argument whence defaults to
+ SEEK_SET or 0 (offset from start of file, offset should be >= 0); other values
+ are SEEK_CUR or 1 (move relative to current position, positive or negative),
+ and SEEK_END or 2 (move relative to end of file, usually negative, although
+ many platforms allow seeking beyond the end of a file).
+
+ Note that not all file objects are seekable.
+ """
+ if isinstance(pos, float):
+ raise TypeError('an integer is required')
+ self._checkClosed()
+ return os.lseek(self._fd, pos, whence)
+
+ def tell(self):
+ """tell() -> int. Current file position.
+
+ Can raise OSError for non seekable files."""
+ self._checkClosed()
+ return os.lseek(self._fd, 0, SEEK_CUR)
+
+ def truncate(self, size=None):
+ """Truncate the file to at most size bytes.
+
+ Size defaults to the current file position, as returned by tell().
+ The current file position is changed to the value of size.
+ """
+ self._checkClosed()
+ self._checkWritable()
+ if size is None:
+ size = self.tell()
+ os.ftruncate(self._fd, size)
+ return size
+
+ def close(self):
+ """Close the file.
+
+ A closed file cannot be used for further I/O operations. close() may be
+ called more than once without error.
+ """
+ if not self.closed:
+ try:
+ if self._closefd:
+ os.close(self._fd)
+ finally:
+ super().close()
+
+ def seekable(self):
+ """True if file supports random-access."""
+ self._checkClosed()
+ if self._seekable is None:
+ try:
+ self.tell()
+ except OSError:
+ self._seekable = False
+ else:
+ self._seekable = True
+ return self._seekable
+
+ def readable(self):
+ """True if file was opened in a read mode."""
+ self._checkClosed()
+ return self._readable
+
+ def writable(self):
+ """True if file was opened in a write mode."""
+ self._checkClosed()
+ return self._writable
+
+ def fileno(self):
+ """Return the underlying file descriptor (an integer)."""
+ self._checkClosed()
+ return self._fd
+
+ def isatty(self):
+ """True if the file is connected to a TTY device."""
+ self._checkClosed()
+ return os.isatty(self._fd)
+
+ @property
+ def closefd(self):
+ """True if the file descriptor will be closed by close()."""
+ return self._closefd
+
+ @property
+ def mode(self):
+ """String giving the file mode"""
+ if self._created:
+ if self._readable:
+ return 'xb+'
+ else:
+ return 'xb'
+ elif self._appending:
+ if self._readable:
+ return 'ab+'
+ else:
+ return 'ab'
+ elif self._readable:
+ if self._writable:
+ return 'rb+'
+ else:
+ return 'rb'
+ else:
+ return 'wb'
+
+
class TextIOBase(IOBase):
"""Base class for text I/O.
@@ -1566,7 +1976,8 @@ class TextIOWrapper(TextIOBase):
# - "chars_..." for integer variables that count decoded characters
def __repr__(self):
- result = "<_pyio.TextIOWrapper"
+ result = "<{}.{}".format(self.__class__.__module__,
+ self.__class__.__qualname__)
try:
name = self.name
except Exception:
diff --git a/Lib/_strptime.py b/Lib/_strptime.py
index f76fff1..374923d 100644
--- a/Lib/_strptime.py
+++ b/Lib/_strptime.py
@@ -167,9 +167,9 @@ class LocaleTime(object):
time.tzset()
except AttributeError:
pass
- no_saving = frozenset(["utc", "gmt", time.tzname[0].lower()])
+ no_saving = frozenset({"utc", "gmt", time.tzname[0].lower()})
if time.daylight:
- has_saving = frozenset([time.tzname[1].lower()])
+ has_saving = frozenset({time.tzname[1].lower()})
else:
has_saving = frozenset()
self.timezone = (no_saving, has_saving)
@@ -253,8 +253,8 @@ class TimeRE(dict):
# format directives (%m, etc.).
regex_chars = re_compile(r"([\\.^$*+?\(\){}\[\]|])")
format = regex_chars.sub(r"\\\1", format)
- whitespace_replacement = re_compile('\s+')
- format = whitespace_replacement.sub('\s+', format)
+ whitespace_replacement = re_compile(r'\s+')
+ format = whitespace_replacement.sub(r'\\s+', format)
while '%' in format:
directive_index = format.index('%')+1
processed_format = "%s%s%s" % (processed_format,
diff --git a/Lib/abc.py b/Lib/abc.py
index 0358a46..1cbf96a 100644
--- a/Lib/abc.py
+++ b/Lib/abc.py
@@ -168,7 +168,7 @@ class ABCMeta(type):
def _dump_registry(cls, file=None):
"""Debug helper to print the ABC registry."""
- print("Class: %s.%s" % (cls.__module__, cls.__name__), file=file)
+ print("Class: %s.%s" % (cls.__module__, cls.__qualname__), file=file)
print("Inv.counter: %s" % ABCMeta._abc_invalidation_counter, file=file)
for name in sorted(cls.__dict__.keys()):
if name.startswith("_abc_"):
diff --git a/Lib/argparse.py b/Lib/argparse.py
index be276bb..9a06719 100644
--- a/Lib/argparse.py
+++ b/Lib/argparse.py
@@ -1209,11 +1209,6 @@ class Namespace(_AttributeHolder):
return NotImplemented
return vars(self) == vars(other)
- def __ne__(self, other):
- if not isinstance(other, Namespace):
- return NotImplemented
- return not (self == other)
-
def __contains__(self, key):
return key in self.__dict__
@@ -1595,6 +1590,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
- argument_default -- The default value for all arguments
- conflict_handler -- String indicating how to handle conflicts
- add_help -- Add a -h/-help option
+ - allow_abbrev -- Allow long options to be abbreviated unambiguously
"""
def __init__(self,
@@ -1608,7 +1604,8 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
fromfile_prefix_chars=None,
argument_default=None,
conflict_handler='error',
- add_help=True):
+ add_help=True,
+ allow_abbrev=True):
superinit = super(ArgumentParser, self).__init__
superinit(description=description,
@@ -1626,6 +1623,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
self.formatter_class = formatter_class
self.fromfile_prefix_chars = fromfile_prefix_chars
self.add_help = add_help
+ self.allow_abbrev = allow_abbrev
add_group = self.add_argument_group
self._positionals = add_group(_('positional arguments'))
@@ -2103,23 +2101,24 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
action = self._option_string_actions[option_string]
return action, option_string, explicit_arg
- # search through all possible prefixes of the option string
- # and all actions in the parser for possible interpretations
- option_tuples = self._get_option_tuples(arg_string)
-
- # if multiple actions match, the option string was ambiguous
- if len(option_tuples) > 1:
- options = ', '.join([option_string
- for action, option_string, explicit_arg in option_tuples])
- args = {'option': arg_string, 'matches': options}
- msg = _('ambiguous option: %(option)s could match %(matches)s')
- self.error(msg % args)
-
- # if exactly one action matched, this segmentation is good,
- # so return the parsed action
- elif len(option_tuples) == 1:
- option_tuple, = option_tuples
- return option_tuple
+ if self.allow_abbrev:
+ # search through all possible prefixes of the option string
+ # and all actions in the parser for possible interpretations
+ option_tuples = self._get_option_tuples(arg_string)
+
+ # if multiple actions match, the option string was ambiguous
+ if len(option_tuples) > 1:
+ options = ', '.join([option_string
+ for action, option_string, explicit_arg in option_tuples])
+ args = {'option': arg_string, 'matches': options}
+ msg = _('ambiguous option: %(option)s could match %(matches)s')
+ self.error(msg % args)
+
+ # if exactly one action matched, this segmentation is good,
+ # so return the parsed action
+ elif len(option_tuples) == 1:
+ option_tuple, = option_tuples
+ return option_tuple
# if it was not found as an option, but it looks like a negative
# number, it was meant to be positional
diff --git a/Lib/ast.py b/Lib/ast.py
index 02c3b28..03c30f6 100644
--- a/Lib/ast.py
+++ b/Lib/ast.py
@@ -194,7 +194,7 @@ def get_docstring(node, clean=True):
be found. If the node provided does not have docstrings a TypeError
will be raised.
"""
- if not isinstance(node, (FunctionDef, ClassDef, Module)):
+ if not isinstance(node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)):
raise TypeError("%r can't have docstrings" % node.__class__.__name__)
if node.body and isinstance(node.body[0], Expr) and \
isinstance(node.body[0].value, Str):
diff --git a/Lib/asynchat.py b/Lib/asynchat.py
index 14c152f..f728d1b 100644
--- a/Lib/asynchat.py
+++ b/Lib/asynchat.py
@@ -287,6 +287,9 @@ class simple_producer:
class fifo:
def __init__(self, list=None):
+ import warnings
+ warnings.warn('fifo class will be removed in Python 3.6',
+ DeprecationWarning, stacklevel=2)
if not list:
self.list = deque()
else:
diff --git a/Lib/asyncore.py b/Lib/asyncore.py
index 00a6396..3b51f0f 100644
--- a/Lib/asyncore.py
+++ b/Lib/asyncore.py
@@ -57,8 +57,8 @@ from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \
ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \
errorcode
-_DISCONNECTED = frozenset((ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE,
- EBADF))
+_DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE,
+ EBADF})
try:
socket_map
@@ -141,10 +141,7 @@ def poll(timeout=0.0, map=None):
time.sleep(timeout)
return
- try:
- r, w, e = select.select(r, w, e, timeout)
- except InterruptedError:
- return
+ r, w, e = select.select(r, w, e, timeout)
for fd in r:
obj = map.get(fd)
@@ -182,10 +179,8 @@ def poll2(timeout=0.0, map=None):
flags |= select.POLLOUT
if flags:
pollster.register(fd, flags)
- try:
- r = pollster.poll(timeout)
- except InterruptedError:
- r = []
+
+ r = pollster.poll(timeout)
for fd, flags in r:
obj = map.get(fd)
if obj is None:
@@ -220,7 +215,7 @@ class dispatcher:
connecting = False
closing = False
addr = None
- ignore_log_types = frozenset(['warning'])
+ ignore_log_types = frozenset({'warning'})
def __init__(self, sock=None, map=None):
if map is None:
@@ -255,7 +250,7 @@ class dispatcher:
self.socket = None
def __repr__(self):
- status = [self.__class__.__module__+"."+self.__class__.__name__]
+ status = [self.__class__.__module__+"."+self.__class__.__qualname__]
if self.accepting and self.addr:
status.append('listening')
elif self.connected:
@@ -404,20 +399,6 @@ class dispatcher:
if why.args[0] not in (ENOTCONN, EBADF):
raise
- # cheap inheritance, used to pass all other attribute
- # references to the underlying socket object.
- def __getattr__(self, attr):
- try:
- retattr = getattr(self.socket, attr)
- except AttributeError:
- raise AttributeError("%s instance has no attribute '%s'"
- %(self.__class__.__name__, attr))
- else:
- msg = "%(me)s.%(attr)s is deprecated; use %(me)s.socket.%(attr)s " \
- "instead" % {'me' : self.__class__.__name__, 'attr' : attr}
- warnings.warn(msg, DeprecationWarning, stacklevel=2)
- return retattr
-
# log and log_info may be overridden to provide more sophisticated
# logging and warning methods. In general, log is for 'hit' logging
# and 'log_info' is for informational, warning and error logging.
@@ -604,8 +585,6 @@ def close_all(map=None, ignore_all=False):
# Regardless, this is useful for pipes, and stdin/stdout...
if os.name == 'posix':
- import fcntl
-
class file_wrapper:
# Here we override just enough to make a file
# look like a socket for the purposes of asyncore.
@@ -656,9 +635,7 @@ if os.name == 'posix':
pass
self.set_file(fd)
# set it to non-blocking mode
- flags = fcntl.fcntl(fd, fcntl.F_GETFL, 0)
- flags = flags | os.O_NONBLOCK
- fcntl.fcntl(fd, fcntl.F_SETFL, flags)
+ os.set_blocking(fd, False)
def set_file(self, fd):
self.socket = file_wrapper(fd)
diff --git a/Lib/binhex.py b/Lib/binhex.py
index 8272d5c..56b5f85 100644
--- a/Lib/binhex.py
+++ b/Lib/binhex.py
@@ -235,14 +235,13 @@ def binhex(inp, out):
finfo = getfileinfo(inp)
ofp = BinHex(finfo, out)
- ifp = io.open(inp, 'rb')
- # XXXX Do textfile translation on non-mac systems
- while True:
- d = ifp.read(128000)
- if not d: break
- ofp.write(d)
- ofp.close_data()
- ifp.close()
+ with io.open(inp, 'rb') as ifp:
+ # XXXX Do textfile translation on non-mac systems
+ while True:
+ d = ifp.read(128000)
+ if not d: break
+ ofp.write(d)
+ ofp.close_data()
ifp = openrsrc(inp, 'rb')
while True:
@@ -459,13 +458,12 @@ def hexbin(inp, out):
if not out:
out = ifp.FName
- ofp = io.open(out, 'wb')
- # XXXX Do translation on non-mac systems
- while True:
- d = ifp.read(128000)
- if not d: break
- ofp.write(d)
- ofp.close()
+ with io.open(out, 'wb') as ofp:
+ # XXXX Do translation on non-mac systems
+ while True:
+ d = ifp.read(128000)
+ if not d: break
+ ofp.write(d)
ifp.close_data()
d = ifp.read_rsrc(128000)
diff --git a/Lib/bz2.py b/Lib/bz2.py
index 6c5a60d..bc78c54 100644
--- a/Lib/bz2.py
+++ b/Lib/bz2.py
@@ -12,6 +12,7 @@ __author__ = "Nadeem Vawda <nadeem.vawda@gmail.com>"
from builtins import open as _builtin_open
import io
import warnings
+import _compression
try:
from threading import RLock
@@ -23,13 +24,11 @@ from _bz2 import BZ2Compressor, BZ2Decompressor
_MODE_CLOSED = 0
_MODE_READ = 1
-_MODE_READ_EOF = 2
+# Value 2 no longer used
_MODE_WRITE = 3
-_BUFFER_SIZE = 8192
-
-class BZ2File(io.BufferedIOBase):
+class BZ2File(_compression.BaseStream):
"""A file object providing transparent bzip2 (de)compression.
@@ -61,13 +60,11 @@ class BZ2File(io.BufferedIOBase):
multiple compressed streams.
"""
# This lock must be recursive, so that BufferedIOBase's
- # readline(), readlines() and writelines() don't deadlock.
+ # writelines() does not deadlock.
self._lock = RLock()
self._fp = None
self._closefp = False
self._mode = _MODE_CLOSED
- self._pos = 0
- self._size = -1
if buffering is not None:
warnings.warn("Use of 'buffering' argument is deprecated",
@@ -79,9 +76,6 @@ class BZ2File(io.BufferedIOBase):
if mode in ("", "r", "rb"):
mode = "rb"
mode_code = _MODE_READ
- self._decompressor = BZ2Decompressor()
- self._buffer = b""
- self._buffer_offset = 0
elif mode in ("w", "wb"):
mode = "wb"
mode_code = _MODE_WRITE
@@ -107,6 +101,13 @@ class BZ2File(io.BufferedIOBase):
else:
raise TypeError("filename must be a str or bytes object, or a file")
+ if self._mode == _MODE_READ:
+ raw = _compression.DecompressReader(self._fp,
+ BZ2Decompressor, trailing_error=OSError)
+ self._buffer = io.BufferedReader(raw)
+ else:
+ self._pos = 0
+
def close(self):
"""Flush and close the file.
@@ -117,8 +118,8 @@ class BZ2File(io.BufferedIOBase):
if self._mode == _MODE_CLOSED:
return
try:
- if self._mode in (_MODE_READ, _MODE_READ_EOF):
- self._decompressor = None
+ if self._mode == _MODE_READ:
+ self._buffer.close()
elif self._mode == _MODE_WRITE:
self._fp.write(self._compressor.flush())
self._compressor = None
@@ -130,8 +131,7 @@ class BZ2File(io.BufferedIOBase):
self._fp = None
self._closefp = False
self._mode = _MODE_CLOSED
- self._buffer = b""
- self._buffer_offset = 0
+ self._buffer = None
@property
def closed(self):
@@ -145,125 +145,18 @@ class BZ2File(io.BufferedIOBase):
def seekable(self):
"""Return whether the file supports seeking."""
- return self.readable() and self._fp.seekable()
+ return self.readable() and self._buffer.seekable()
def readable(self):
"""Return whether the file was opened for reading."""
self._check_not_closed()
- return self._mode in (_MODE_READ, _MODE_READ_EOF)
+ return self._mode == _MODE_READ
def writable(self):
"""Return whether the file was opened for writing."""
self._check_not_closed()
return self._mode == _MODE_WRITE
- # Mode-checking helper functions.
-
- def _check_not_closed(self):
- if self.closed:
- raise ValueError("I/O operation on closed file")
-
- def _check_can_read(self):
- if self._mode not in (_MODE_READ, _MODE_READ_EOF):
- self._check_not_closed()
- raise io.UnsupportedOperation("File not open for reading")
-
- def _check_can_write(self):
- if self._mode != _MODE_WRITE:
- self._check_not_closed()
- raise io.UnsupportedOperation("File not open for writing")
-
- def _check_can_seek(self):
- if self._mode not in (_MODE_READ, _MODE_READ_EOF):
- self._check_not_closed()
- raise io.UnsupportedOperation("Seeking is only supported "
- "on files open for reading")
- if not self._fp.seekable():
- raise io.UnsupportedOperation("The underlying file object "
- "does not support seeking")
-
- # Fill the readahead buffer if it is empty. Returns False on EOF.
- def _fill_buffer(self):
- if self._mode == _MODE_READ_EOF:
- return False
- # Depending on the input data, our call to the decompressor may not
- # return any data. In this case, try again after reading another block.
- while self._buffer_offset == len(self._buffer):
- rawblock = (self._decompressor.unused_data or
- self._fp.read(_BUFFER_SIZE))
-
- if not rawblock:
- if self._decompressor.eof:
- # End-of-stream marker and end of file. We're good.
- self._mode = _MODE_READ_EOF
- self._size = self._pos
- return False
- else:
- # Problem - we were expecting more compressed data.
- raise EOFError("Compressed file ended before the "
- "end-of-stream marker was reached")
-
- if self._decompressor.eof:
- # Continue to next stream.
- self._decompressor = BZ2Decompressor()
- try:
- self._buffer = self._decompressor.decompress(rawblock)
- except OSError:
- # Trailing data isn't a valid bzip2 stream. We're done here.
- self._mode = _MODE_READ_EOF
- self._size = self._pos
- return False
- else:
- self._buffer = self._decompressor.decompress(rawblock)
- self._buffer_offset = 0
- return True
-
- # Read data until EOF.
- # If return_data is false, consume the data without returning it.
- def _read_all(self, return_data=True):
- # The loop assumes that _buffer_offset is 0. Ensure that this is true.
- self._buffer = self._buffer[self._buffer_offset:]
- self._buffer_offset = 0
-
- blocks = []
- while self._fill_buffer():
- if return_data:
- blocks.append(self._buffer)
- self._pos += len(self._buffer)
- self._buffer = b""
- if return_data:
- return b"".join(blocks)
-
- # Read a block of up to n bytes.
- # If return_data is false, consume the data without returning it.
- def _read_block(self, n, return_data=True):
- # If we have enough data buffered, return immediately.
- end = self._buffer_offset + n
- if end <= len(self._buffer):
- data = self._buffer[self._buffer_offset : end]
- self._buffer_offset = end
- self._pos += len(data)
- return data if return_data else None
-
- # The loop assumes that _buffer_offset is 0. Ensure that this is true.
- self._buffer = self._buffer[self._buffer_offset:]
- self._buffer_offset = 0
-
- blocks = []
- while n > 0 and self._fill_buffer():
- if n < len(self._buffer):
- data = self._buffer[:n]
- self._buffer_offset = n
- else:
- data = self._buffer
- self._buffer = b""
- if return_data:
- blocks.append(data)
- self._pos += len(data)
- n -= len(data)
- if return_data:
- return b"".join(blocks)
-
def peek(self, n=0):
"""Return buffered data without advancing the file position.
@@ -272,9 +165,10 @@ class BZ2File(io.BufferedIOBase):
"""
with self._lock:
self._check_can_read()
- if not self._fill_buffer():
- return b""
- return self._buffer[self._buffer_offset:]
+ # Relies on the undocumented fact that BufferedReader.peek()
+ # always returns at least one byte (except at EOF), independent
+ # of the value of n
+ return self._buffer.peek(n)
def read(self, size=-1):
"""Read up to size uncompressed bytes from the file.
@@ -284,47 +178,29 @@ class BZ2File(io.BufferedIOBase):
"""
with self._lock:
self._check_can_read()
- if size == 0:
- return b""
- elif size < 0:
- return self._read_all()
- else:
- return self._read_block(size)
+ return self._buffer.read(size)
def read1(self, size=-1):
"""Read up to size uncompressed bytes, while trying to avoid
- making multiple reads from the underlying stream.
+ making multiple reads from the underlying stream. Reads up to a
+ buffer's worth of data if size is negative.
Returns b'' if the file is at EOF.
"""
- # Usually, read1() calls _fp.read() at most once. However, sometimes
- # this does not give enough data for the decompressor to make progress.
- # In this case we make multiple reads, to avoid returning b"".
with self._lock:
self._check_can_read()
- if (size == 0 or
- # Only call _fill_buffer() if the buffer is actually empty.
- # This gives a significant speedup if *size* is small.
- (self._buffer_offset == len(self._buffer) and not self._fill_buffer())):
- return b""
- if size > 0:
- data = self._buffer[self._buffer_offset :
- self._buffer_offset + size]
- self._buffer_offset += len(data)
- else:
- data = self._buffer[self._buffer_offset:]
- self._buffer = b""
- self._buffer_offset = 0
- self._pos += len(data)
- return data
+ if size < 0:
+ size = io.DEFAULT_BUFFER_SIZE
+ return self._buffer.read1(size)
def readinto(self, b):
- """Read up to len(b) bytes into b.
+ """Read bytes into b.
Returns the number of bytes read (0 for EOF).
"""
with self._lock:
- return io.BufferedIOBase.readinto(self, b)
+ self._check_can_read()
+ return self._buffer.readinto(b)
def readline(self, size=-1):
"""Read a line of uncompressed bytes from the file.
@@ -339,15 +215,7 @@ class BZ2File(io.BufferedIOBase):
size = size.__index__()
with self._lock:
self._check_can_read()
- # Shortcut for the common case - the whole line is in the buffer.
- if size < 0:
- end = self._buffer.find(b"\n", self._buffer_offset) + 1
- if end > 0:
- line = self._buffer[self._buffer_offset : end]
- self._buffer_offset = end
- self._pos += len(line)
- return line
- return io.BufferedIOBase.readline(self, size)
+ return self._buffer.readline(size)
def readlines(self, size=-1):
"""Read a list of lines of uncompressed bytes from the file.
@@ -361,7 +229,8 @@ class BZ2File(io.BufferedIOBase):
raise TypeError("Integer argument expected")
size = size.__index__()
with self._lock:
- return io.BufferedIOBase.readlines(self, size)
+ self._check_can_read()
+ return self._buffer.readlines(size)
def write(self, data):
"""Write a byte string to the file.
@@ -386,18 +255,9 @@ class BZ2File(io.BufferedIOBase):
Line separators are not added between the written byte strings.
"""
with self._lock:
- return io.BufferedIOBase.writelines(self, seq)
-
- # Rewind the file to the beginning of the data stream.
- def _rewind(self):
- self._fp.seek(0, 0)
- self._mode = _MODE_READ
- self._pos = 0
- self._decompressor = BZ2Decompressor()
- self._buffer = b""
- self._buffer_offset = 0
-
- def seek(self, offset, whence=0):
+ return _compression.BaseStream.writelines(self, seq)
+
+ def seek(self, offset, whence=io.SEEK_SET):
"""Change the file position.
The new position is specified by offset, relative to the
@@ -414,35 +274,14 @@ class BZ2File(io.BufferedIOBase):
"""
with self._lock:
self._check_can_seek()
-
- # Recalculate offset as an absolute file position.
- if whence == 0:
- pass
- elif whence == 1:
- offset = self._pos + offset
- elif whence == 2:
- # Seeking relative to EOF - we need to know the file's size.
- if self._size < 0:
- self._read_all(return_data=False)
- offset = self._size + offset
- else:
- raise ValueError("Invalid value for whence: %s" % (whence,))
-
- # Make it so that offset is the number of bytes to skip forward.
- if offset < self._pos:
- self._rewind()
- else:
- offset -= self._pos
-
- # Read and discard data until we reach the desired position.
- self._read_block(offset, return_data=False)
-
- return self._pos
+ return self._buffer.seek(offset, whence)
def tell(self):
"""Return the current file position."""
with self._lock:
self._check_not_closed()
+ if self._mode == _MODE_READ:
+ return self._buffer.tell()
return self._pos
diff --git a/Lib/cgi.py b/Lib/cgi.py
index 45badf6..26d2544 100755
--- a/Lib/cgi.py
+++ b/Lib/cgi.py
@@ -566,6 +566,12 @@ class FieldStorage:
except AttributeError:
pass
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ self.file.close()
+
def __repr__(self):
"""Return a printable representation."""
return "FieldStorage(%r, %r, %r)" % (
diff --git a/Lib/cgitb.py b/Lib/cgitb.py
index 6eb52e7..b291100 100644
--- a/Lib/cgitb.py
+++ b/Lib/cgitb.py
@@ -294,9 +294,8 @@ class Hook:
(fd, path) = tempfile.mkstemp(suffix=suffix, dir=self.logdir)
try:
- file = os.fdopen(fd, 'w')
- file.write(doc)
- file.close()
+ with os.fdopen(fd, 'w') as file:
+ file.write(doc)
msg = '%s contains the description of this error.' % path
except:
msg = 'Tried to save traceback to %s, but failed.' % path
diff --git a/Lib/code.py b/Lib/code.py
index f8184b6..53244e3 100644
--- a/Lib/code.py
+++ b/Lib/code.py
@@ -7,6 +7,7 @@
import sys
import traceback
+import argparse
from codeop import CommandCompiler, compile_command
__all__ = ["InteractiveInterpreter", "InteractiveConsole", "interact",
@@ -136,25 +137,18 @@ class InteractiveInterpreter:
The output is written by self.write(), below.
"""
+ sys.last_type, sys.last_value, last_tb = ei = sys.exc_info()
+ sys.last_traceback = last_tb
try:
- type, value, tb = sys.exc_info()
- sys.last_type = type
- sys.last_value = value
- sys.last_traceback = tb
- tblist = traceback.extract_tb(tb)
- del tblist[:1]
- lines = traceback.format_list(tblist)
- if lines:
- lines.insert(0, "Traceback (most recent call last):\n")
- lines.extend(traceback.format_exception_only(type, value))
+ lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next)
+ if sys.excepthook is sys.__excepthook__:
+ self.write(''.join(lines))
+ else:
+ # If someone has set sys.excepthook, we let that take precedence
+ # over self.write
+ sys.excepthook(ei[0], ei[1], last_tb)
finally:
- tblist = tb = None
- if sys.excepthook is sys.__excepthook__:
- self.write(''.join(lines))
- else:
- # If someone has set sys.excepthook, we let that take precedence
- # over self.write
- sys.excepthook(type, value, tb)
+ last_tb = ei = None
def write(self, data):
"""Write a string.
@@ -299,4 +293,12 @@ def interact(banner=None, readfunc=None, local=None):
if __name__ == "__main__":
- interact()
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-q', action='store_true',
+ help="don't print version and copyright messages")
+ args = parser.parse_args()
+ if args.q or sys.flags.quiet:
+ banner = ''
+ else:
+ banner = None
+ interact(banner)
diff --git a/Lib/codecs.py b/Lib/codecs.py
index 4a4d043..22d5f82 100644
--- a/Lib/codecs.py
+++ b/Lib/codecs.py
@@ -27,7 +27,8 @@ __all__ = ["register", "lookup", "open", "EncodedFile", "BOM", "BOM_BE",
"getincrementaldecoder", "getreader", "getwriter",
"encode", "decode", "iterencode", "iterdecode",
"strict_errors", "ignore_errors", "replace_errors",
- "xmlcharrefreplace_errors", "backslashreplace_errors",
+ "xmlcharrefreplace_errors",
+ "backslashreplace_errors", "namereplace_errors",
"register_error", "lookup_error"]
### Constants
@@ -105,8 +106,8 @@ class CodecInfo(tuple):
return self
def __repr__(self):
- return "<%s.%s object for encoding %s at 0x%x>" % \
- (self.__class__.__module__, self.__class__.__name__,
+ return "<%s.%s object for encoding %s at %#x>" % \
+ (self.__class__.__module__, self.__class__.__qualname__,
self.name, id(self))
class Codec:
@@ -126,7 +127,8 @@ class Codec:
'surrogateescape' - replace with private code points U+DCnn.
'xmlcharrefreplace' - Replace with the appropriate XML
character reference (only for encoding).
- 'backslashreplace' - Replace with backslashed escape sequences
+ 'backslashreplace' - Replace with backslashed escape sequences.
+ 'namereplace' - Replace with \\N{...} escape sequences
(only for encoding).
The set of allowed values can be extended via register_error.
@@ -358,7 +360,8 @@ class StreamWriter(Codec):
'xmlcharrefreplace' - Replace with the appropriate XML
character reference.
'backslashreplace' - Replace with backslashed escape
- sequences (only for encoding).
+ sequences.
+ 'namereplace' - Replace with \\N{...} escape sequences.
The set of allowed parameter values can be extended via
register_error.
@@ -428,7 +431,8 @@ class StreamReader(Codec):
'strict' - raise a ValueError (or a subclass)
'ignore' - ignore the character and continue with the next
- 'replace'- replace with a suitable replacement character;
+ 'replace'- replace with a suitable replacement character
+ 'backslashreplace' - Replace with backslashed escape sequences;
The set of allowed parameter values can be extended via
register_error.
@@ -1080,6 +1084,7 @@ try:
replace_errors = lookup_error("replace")
xmlcharrefreplace_errors = lookup_error("xmlcharrefreplace")
backslashreplace_errors = lookup_error("backslashreplace")
+ namereplace_errors = lookup_error("namereplace")
except LookupError:
# In --disable-unicode builds, these error handler are missing
strict_errors = None
@@ -1087,6 +1092,7 @@ except LookupError:
replace_errors = None
xmlcharrefreplace_errors = None
backslashreplace_errors = None
+ namereplace_errors = None
# Tell modulefinder that using codecs probably needs the encodings
# package
diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py
index 09afc8a..26e5d34 100644
--- a/Lib/collections/__init__.py
+++ b/Lib/collections/__init__.py
@@ -7,7 +7,6 @@ from _collections_abc import *
import _collections_abc
__all__ += _collections_abc.__all__
-from _collections import deque, defaultdict
from operator import itemgetter as _itemgetter, eq as _eq
from keyword import iskeyword as _iskeyword
import sys as _sys
@@ -16,10 +15,40 @@ from _weakref import proxy as _proxy
from itertools import repeat as _repeat, chain as _chain, starmap as _starmap
from reprlib import recursive_repr as _recursive_repr
+try:
+ from _collections import deque
+except ImportError:
+ pass
+else:
+ MutableSequence.register(deque)
+
+try:
+ from _collections import defaultdict
+except ImportError:
+ pass
+
+
################################################################################
### OrderedDict
################################################################################
+class _OrderedDictKeysView(KeysView):
+
+ def __reversed__(self):
+ yield from reversed(self._mapping)
+
+class _OrderedDictItemsView(ItemsView):
+
+ def __reversed__(self):
+ for key in reversed(self._mapping):
+ yield (key, self._mapping[key])
+
+class _OrderedDictValuesView(ValuesView):
+
+ def __reversed__(self):
+ for key in reversed(self._mapping):
+ yield self._mapping[key]
+
class _Link(object):
__slots__ = 'prev', 'next', 'key', '__weakref__'
@@ -83,6 +112,8 @@ class OrderedDict(dict):
link_next = link.next
link_prev.next = link_next
link_next.prev = link_prev
+ link.prev = None
+ link.next = None
def __iter__(self):
'od.__iter__() <==> iter(od)'
@@ -166,9 +197,19 @@ class OrderedDict(dict):
return size
update = __update = MutableMapping.update
- keys = MutableMapping.keys
- values = MutableMapping.values
- items = MutableMapping.items
+
+ def keys(self):
+ "D.keys() -> a set-like object providing a view on D's keys"
+ return _OrderedDictKeysView(self)
+
+ def items(self):
+ "D.items() -> a set-like object providing a view on D's items"
+ return _OrderedDictItemsView(self)
+
+ def values(self):
+ "D.values() -> an object providing a view on D's values"
+ return _OrderedDictValuesView(self)
+
__ne__ = MutableMapping.__ne__
__marker = object()
@@ -233,6 +274,13 @@ class OrderedDict(dict):
return dict.__eq__(self, other)
+try:
+ from _collections import OrderedDict
+except ImportError:
+ # Leave the pure Python version in place.
+ pass
+
+
################################################################################
### namedtuple
################################################################################
@@ -696,14 +744,22 @@ class Counter(dict):
def __pos__(self):
'Adds an empty counter, effectively stripping negative and zero counts'
- return self + Counter()
+ result = Counter()
+ for elem, count in self.items():
+ if count > 0:
+ result[elem] = count
+ return result
def __neg__(self):
'''Subtracts from an empty counter. Strips positive and zero counts,
and flips the sign on negative counts.
'''
- return Counter() - self
+ result = Counter()
+ for elem, count in self.items():
+ if count < 0:
+ result[elem] = 0 - count
+ return result
def _keep_positive(self):
'''Internal method to strip elements with a negative or zero count'''
@@ -949,7 +1005,6 @@ class UserList(MutableSequence):
def __lt__(self, other): return self.data < self.__cast(other)
def __le__(self, other): return self.data <= self.__cast(other)
def __eq__(self, other): return self.data == self.__cast(other)
- def __ne__(self, other): return self.data != self.__cast(other)
def __gt__(self, other): return self.data > self.__cast(other)
def __ge__(self, other): return self.data >= self.__cast(other)
def __cast(self, other):
@@ -1021,15 +1076,13 @@ class UserString(Sequence):
def __float__(self): return float(self.data)
def __complex__(self): return complex(self.data)
def __hash__(self): return hash(self.data)
+ def __getnewargs__(self):
+ return (self.data[:],)
def __eq__(self, string):
if isinstance(string, UserString):
return self.data == string.data
return self.data == string
- def __ne__(self, string):
- if isinstance(string, UserString):
- return self.data != string.data
- return self.data != string
def __lt__(self, string):
if isinstance(string, UserString):
return self.data < string.data
@@ -1069,9 +1122,13 @@ class UserString(Sequence):
__rmul__ = __mul__
def __mod__(self, args):
return self.__class__(self.data % args)
+ def __rmod__(self, format):
+ return self.__class__(format % args)
# the following methods are defined in alphabetical order:
def capitalize(self): return self.__class__(self.data.capitalize())
+ def casefold(self):
+ return self.__class__(self.data.casefold())
def center(self, width, *args):
return self.__class__(self.data.center(width, *args))
def count(self, sub, start=0, end=_sys.maxsize):
@@ -1094,6 +1151,8 @@ class UserString(Sequence):
return self.data.find(sub, start, end)
def format(self, *args, **kwds):
return self.data.format(*args, **kwds)
+ def format_map(self, mapping):
+ return self.data.format_map(mapping)
def index(self, sub, start=0, end=_sys.maxsize):
return self.data.index(sub, start, end)
def isalpha(self): return self.data.isalpha()
@@ -1103,6 +1162,7 @@ class UserString(Sequence):
def isidentifier(self): return self.data.isidentifier()
def islower(self): return self.data.islower()
def isnumeric(self): return self.data.isnumeric()
+ def isprintable(self): return self.data.isprintable()
def isspace(self): return self.data.isspace()
def istitle(self): return self.data.istitle()
def isupper(self): return self.data.isupper()
@@ -1111,6 +1171,7 @@ class UserString(Sequence):
return self.__class__(self.data.ljust(width, *args))
def lower(self): return self.__class__(self.data.lower())
def lstrip(self, chars=None): return self.__class__(self.data.lstrip(chars))
+ maketrans = str.maketrans
def partition(self, sep):
return self.data.partition(sep)
def replace(self, old, new, maxsplit=-1):
diff --git a/Lib/compileall.py b/Lib/compileall.py
index d957ee5..64c0a9a 100644
--- a/Lib/compileall.py
+++ b/Lib/compileall.py
@@ -1,4 +1,4 @@
-"""Module/script to byte-compile all .py files to .pyc (or .pyo) files.
+"""Module/script to byte-compile all .py files to .pyc files.
When called as a script with arguments, this compiles the directories
given as arguments recursively; the -l option prevents it from
@@ -16,32 +16,24 @@ import importlib.util
import py_compile
import struct
-__all__ = ["compile_dir","compile_file","compile_path"]
-
-def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
- quiet=False, legacy=False, optimize=-1):
- """Byte-compile all modules in the given directory tree.
+try:
+ from concurrent.futures import ProcessPoolExecutor
+except ImportError:
+ ProcessPoolExecutor = None
+from functools import partial
- Arguments (only dir is required):
+__all__ = ["compile_dir","compile_file","compile_path"]
- dir: the directory to byte-compile
- maxlevels: maximum recursion level (default 10)
- ddir: the directory that will be prepended to the path to the
- file as it is compiled into each byte-code file.
- force: if True, force compilation, even if timestamps are up-to-date
- quiet: if True, be quiet during compilation
- legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
- optimize: optimization level or -1 for level of the interpreter
- """
+def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0):
if not quiet:
print('Listing {!r}...'.format(dir))
try:
names = os.listdir(dir)
except OSError:
- print("Can't list {!r}".format(dir))
+ if quiet < 2:
+ print("Can't list {!r}".format(dir))
names = []
names.sort()
- success = 1
for name in names:
if name == '__pycache__':
continue
@@ -51,17 +43,53 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
else:
dfile = None
if not os.path.isdir(fullname):
- if not compile_file(fullname, ddir, force, rx, quiet,
- legacy, optimize):
- success = 0
+ yield fullname
elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
os.path.isdir(fullname) and not os.path.islink(fullname)):
- if not compile_dir(fullname, maxlevels - 1, dfile, force, rx,
- quiet, legacy, optimize):
+ yield from _walk_dir(fullname, ddir=dfile,
+ maxlevels=maxlevels - 1, quiet=quiet)
+
+def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
+ quiet=0, legacy=False, optimize=-1, workers=1):
+ """Byte-compile all modules in the given directory tree.
+
+ Arguments (only dir is required):
+
+ dir: the directory to byte-compile
+ maxlevels: maximum recursion level (default 10)
+ ddir: the directory that will be prepended to the path to the
+ file as it is compiled into each byte-code file.
+ force: if True, force compilation, even if timestamps are up-to-date
+ quiet: full output with False or 0, errors only with 1,
+ no output with 2
+ legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
+ optimize: optimization level or -1 for level of the interpreter
+ workers: maximum number of parallel workers
+ """
+ files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
+ ddir=ddir)
+ success = 1
+ if workers is not None and workers != 1 and ProcessPoolExecutor is not None:
+ if workers < 0:
+ raise ValueError('workers must be greater or equal to 0')
+
+ workers = workers or None
+ with ProcessPoolExecutor(max_workers=workers) as executor:
+ results = executor.map(partial(compile_file,
+ ddir=ddir, force=force,
+ rx=rx, quiet=quiet,
+ legacy=legacy,
+ optimize=optimize),
+ files)
+ success = min(results, default=1)
+ else:
+ for file in files:
+ if not compile_file(file, ddir, force, rx, quiet,
+ legacy, optimize):
success = 0
return success
-def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False,
+def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
legacy=False, optimize=-1):
"""Byte-compile one file.
@@ -71,7 +99,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False,
ddir: if given, the directory name compiled in to the
byte-code file.
force: if True, force compilation, even if timestamps are up-to-date
- quiet: if True, be quiet during compilation
+ quiet: full output with False or 0, errors only with 1,
+ no output with 2
legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
optimize: optimization level or -1 for level of the interpreter
"""
@@ -87,11 +116,12 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False,
return success
if os.path.isfile(fullname):
if legacy:
- cfile = fullname + ('c' if __debug__ else 'o')
+ cfile = fullname + 'c'
else:
if optimize >= 0:
+ opt = optimize if optimize >= 1 else ''
cfile = importlib.util.cache_from_source(
- fullname, debug_override=not optimize)
+ fullname, optimization=opt)
else:
cfile = importlib.util.cache_from_source(fullname)
cache_dir = os.path.dirname(cfile)
@@ -114,7 +144,10 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False,
ok = py_compile.compile(fullname, cfile, dfile, True,
optimize=optimize)
except py_compile.PyCompileError as err:
- if quiet:
+ success = 0
+ if quiet >= 2:
+ return success
+ elif quiet:
print('*** Error compiling {!r}...'.format(fullname))
else:
print('*** ', end='')
@@ -123,20 +156,21 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False,
errors='backslashreplace')
msg = msg.decode(sys.stdout.encoding)
print(msg)
- success = 0
except (SyntaxError, UnicodeError, OSError) as e:
- if quiet:
+ success = 0
+ if quiet >= 2:
+ return success
+ elif quiet:
print('*** Error compiling {!r}...'.format(fullname))
else:
print('*** ', end='')
print(e.__class__.__name__ + ':', e)
- success = 0
else:
if ok == 0:
success = 0
return success
-def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=False,
+def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0,
legacy=False, optimize=-1):
"""Byte-compile all module on sys.path.
@@ -145,14 +179,15 @@ def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=False,
skip_curdir: if true, skip current directory (default True)
maxlevels: max recursion level (default 0)
force: as for compile_dir() (default False)
- quiet: as for compile_dir() (default False)
+ quiet: as for compile_dir() (default 0)
legacy: as for compile_dir() (default False)
optimize: as for compile_dir() (default -1)
"""
success = 1
for dir in sys.path:
if (not dir or dir == os.curdir) and skip_curdir:
- print('Skipping current directory')
+ if quiet < 2:
+ print('Skipping current directory')
else:
success = success and compile_dir(dir, maxlevels, None,
force, quiet=quiet,
@@ -169,10 +204,15 @@ def main():
parser.add_argument('-l', action='store_const', const=0,
default=10, dest='maxlevels',
help="don't recurse into subdirectories")
+ parser.add_argument('-r', type=int, dest='recursion',
+ help=('control the maximum recursion level. '
+ 'if `-l` and `-r` options are specified, '
+ 'then `-r` takes precedence.'))
parser.add_argument('-f', action='store_true', dest='force',
help='force rebuild even if timestamps are up to date')
- parser.add_argument('-q', action='store_true', dest='quiet',
- help='output only error messages')
+ parser.add_argument('-q', action='count', dest='quiet', default=0,
+ help='output only error messages; -qq will suppress '
+ 'the error messages as well.')
parser.add_argument('-b', action='store_true', dest='legacy',
help='use legacy (pre-PEP3147) compiled file locations')
parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None,
@@ -192,8 +232,10 @@ def main():
help=('zero or more file and directory names '
'to compile; if no arguments given, defaults '
'to the equivalent of -l sys.path'))
- args = parser.parse_args()
+ parser.add_argument('-j', '--workers', default=1,
+ type=int, help='Run compileall concurrently')
+ args = parser.parse_args()
compile_dests = args.compile_dest
if (args.ddir and (len(compile_dests) != 1
@@ -203,6 +245,12 @@ def main():
import re
args.rx = re.compile(args.rx)
+
+ if args.recursion is not None:
+ maxlevels = args.recursion
+ else:
+ maxlevels = args.maxlevels
+
# if flist is provided then load it
if args.flist:
try:
@@ -210,9 +258,13 @@ def main():
for line in f:
compile_dests.append(line.strip())
except OSError:
- print("Error reading file list {}".format(args.flist))
+ if args.quiet < 2:
+ print("Error reading file list {}".format(args.flist))
return False
+ if args.workers is not None:
+ args.workers = args.workers or None
+
success = True
try:
if compile_dests:
@@ -222,16 +274,17 @@ def main():
args.quiet, args.legacy):
success = False
else:
- if not compile_dir(dest, args.maxlevels, args.ddir,
+ if not compile_dir(dest, maxlevels, args.ddir,
args.force, args.rx, args.quiet,
- args.legacy):
+ args.legacy, workers=args.workers):
success = False
return success
else:
return compile_path(legacy=args.legacy, force=args.force,
quiet=args.quiet)
except KeyboardInterrupt:
- print("\n[interrupted]")
+ if args.quiet < 2:
+ print("\n[interrupted]")
return False
return True
diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py
index acd05d0..9e44713 100644
--- a/Lib/concurrent/futures/_base.py
+++ b/Lib/concurrent/futures/_base.py
@@ -302,17 +302,20 @@ class Future(object):
with self._condition:
if self._state == FINISHED:
if self._exception:
- return '<Future at %s state=%s raised %s>' % (
- hex(id(self)),
+ return '<%s at %#x state=%s raised %s>' % (
+ self.__class__.__name__,
+ id(self),
_STATE_TO_DESCRIPTION_MAP[self._state],
self._exception.__class__.__name__)
else:
- return '<Future at %s state=%s returned %s>' % (
- hex(id(self)),
+ return '<%s at %#x state=%s returned %s>' % (
+ self.__class__.__name__,
+ id(self),
_STATE_TO_DESCRIPTION_MAP[self._state],
self._result.__class__.__name__)
- return '<Future at %s state=%s>' % (
- hex(id(self)),
+ return '<%s at %#x state=%s>' % (
+ self.__class__.__name__,
+ id(self),
_STATE_TO_DESCRIPTION_MAP[self._state])
def cancel(self):
@@ -517,7 +520,7 @@ class Executor(object):
"""
raise NotImplementedError()
- def map(self, fn, *iterables, timeout=None):
+ def map(self, fn, *iterables, timeout=None, chunksize=1):
"""Returns a iterator equivalent to map(fn, iter).
Args:
@@ -525,6 +528,10 @@ class Executor(object):
passed iterables.
timeout: The maximum number of seconds to wait. If None, then there
is no limit on the wait time.
+ chunksize: The size of the chunks the iterable will be broken into
+ before being passed to a child process. This argument is only
+ used by ProcessPoolExecutor; it is ignored by
+ ThreadPoolExecutor.
Returns:
An iterator equivalent to: map(func, *iterables) but the calls may
diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py
index 07b5225..3dd6da1 100644
--- a/Lib/concurrent/futures/process.py
+++ b/Lib/concurrent/futures/process.py
@@ -55,6 +55,9 @@ from multiprocessing import SimpleQueue
from multiprocessing.connection import wait
import threading
import weakref
+from functools import partial
+import itertools
+import traceback
# Workers are created as daemon threads and processes. This is done to allow the
# interpreter to exit when there are still idle processes in a
@@ -88,6 +91,27 @@ def _python_exit():
# (Futures in the call queue cannot be cancelled).
EXTRA_QUEUED_CALLS = 1
+# Hack to embed stringification of remote traceback in local traceback
+
+class _RemoteTraceback(Exception):
+ def __init__(self, tb):
+ self.tb = tb
+ def __str__(self):
+ return self.tb
+
+class _ExceptionWithTraceback:
+ def __init__(self, exc, tb):
+ tb = traceback.format_exception(type(exc), exc, tb)
+ tb = ''.join(tb)
+ self.exc = exc
+ self.tb = '\n"""\n%s"""' % tb
+ def __reduce__(self):
+ return _rebuild_exc, (self.exc, self.tb)
+
+def _rebuild_exc(exc, tb):
+ exc.__cause__ = _RemoteTraceback(tb)
+ return exc
+
class _WorkItem(object):
def __init__(self, future, fn, args, kwargs):
self.future = future
@@ -108,6 +132,26 @@ class _CallItem(object):
self.args = args
self.kwargs = kwargs
+def _get_chunks(*iterables, chunksize):
+ """ Iterates over zip()ed iterables in chunks. """
+ it = zip(*iterables)
+ while True:
+ chunk = tuple(itertools.islice(it, chunksize))
+ if not chunk:
+ return
+ yield chunk
+
+def _process_chunk(fn, chunk):
+ """ Processes a chunk of an iterable passed to map.
+
+ Runs the function passed to map() on a chunk of the
+ iterable passed to map.
+
+ This function is run in a separate process.
+
+ """
+ return [fn(*args) for args in chunk]
+
def _process_worker(call_queue, result_queue):
"""Evaluates calls from call_queue and places the results in result_queue.
@@ -130,8 +174,8 @@ def _process_worker(call_queue, result_queue):
try:
r = call_item.fn(*call_item.args, **call_item.kwargs)
except BaseException as e:
- result_queue.put(_ResultItem(call_item.work_id,
- exception=e))
+ exc = _ExceptionWithTraceback(e, e.__traceback__)
+ result_queue.put(_ResultItem(call_item.work_id, exception=exc))
else:
result_queue.put(_ResultItem(call_item.work_id,
result=r))
@@ -334,6 +378,9 @@ class ProcessPoolExecutor(_base.Executor):
if max_workers is None:
self._max_workers = os.cpu_count() or 1
else:
+ if max_workers <= 0:
+ raise ValueError("max_workers must be greater than 0")
+
self._max_workers = max_workers
# Make the call queue slightly larger than the number of processes to
@@ -408,6 +455,35 @@ class ProcessPoolExecutor(_base.Executor):
return f
submit.__doc__ = _base.Executor.submit.__doc__
+ def map(self, fn, *iterables, timeout=None, chunksize=1):
+ """Returns a iterator equivalent to map(fn, iter).
+
+ Args:
+ fn: A callable that will take as many arguments as there are
+ passed iterables.
+ timeout: The maximum number of seconds to wait. If None, then there
+ is no limit on the wait time.
+ chunksize: If greater than one, the iterables will be chopped into
+ chunks of size chunksize and submitted to the process pool.
+ If set to one, the items in the list will be sent one at a time.
+
+ Returns:
+ An iterator equivalent to: map(func, *iterables) but the calls may
+ be evaluated out-of-order.
+
+ Raises:
+ TimeoutError: If the entire result iterator could not be generated
+ before the given timeout.
+ Exception: If fn(*args) raises for any values.
+ """
+ if chunksize < 1:
+ raise ValueError("chunksize must be >= 1.")
+
+ results = super().map(partial(_process_chunk, fn),
+ _get_chunks(*iterables, chunksize=chunksize),
+ timeout=timeout)
+ return itertools.chain.from_iterable(results)
+
def shutdown(self, wait=True):
with self._shutdown_lock:
self._shutdown_thread = True
diff --git a/Lib/concurrent/futures/thread.py b/Lib/concurrent/futures/thread.py
index f9beb0f..3ae442d 100644
--- a/Lib/concurrent/futures/thread.py
+++ b/Lib/concurrent/futures/thread.py
@@ -10,6 +10,7 @@ from concurrent.futures import _base
import queue
import threading
import weakref
+import os
# Workers are created as daemon threads. This is done to allow the interpreter
# to exit when there are still idle threads in a ThreadPoolExecutor's thread
@@ -80,13 +81,20 @@ def _worker(executor_reference, work_queue):
_base.LOGGER.critical('Exception in worker', exc_info=True)
class ThreadPoolExecutor(_base.Executor):
- def __init__(self, max_workers):
+ def __init__(self, max_workers=None):
"""Initializes a new ThreadPoolExecutor instance.
Args:
max_workers: The maximum number of threads that can be used to
execute the given calls.
"""
+ if max_workers is None:
+ # Use this number because ThreadPoolExecutor is often
+ # used to overlap I/O instead of CPU work.
+ max_workers = (os.cpu_count() or 1) * 5
+ if max_workers <= 0:
+ raise ValueError("max_workers must be greater than 0")
+
self._max_workers = max_workers
self._work_queue = queue.Queue()
self._threads = set()
diff --git a/Lib/configparser.py b/Lib/configparser.py
index dcb7ec4..3a9fb56 100644
--- a/Lib/configparser.py
+++ b/Lib/configparser.py
@@ -17,7 +17,8 @@ ConfigParser -- responsible for parsing a list of
__init__(defaults=None, dict_type=_default_dict, allow_no_value=False,
delimiters=('=', ':'), comment_prefixes=('#', ';'),
inline_comment_prefixes=None, strict=True,
- empty_lines_in_values=True):
+ empty_lines_in_values=True, default_section='DEFAULT',
+ interpolation=<unset>, converters=<unset>):
Create the parser. When `defaults' is given, it is initialized into the
dictionary or intrinsic defaults. The keys must be strings, the values
must be appropriate for %()s string interpolation.
@@ -47,6 +48,25 @@ ConfigParser -- responsible for parsing a list of
When `allow_no_value' is True (default: False), options without
values are accepted; the value presented for these is None.
+ When `default_section' is given, the name of the special section is
+ named accordingly. By default it is called ``"DEFAULT"`` but this can
+ be customized to point to any other valid section name. Its current
+ value can be retrieved using the ``parser_instance.default_section``
+ attribute and may be modified at runtime.
+
+ When `interpolation` is given, it should be an Interpolation subclass
+ instance. It will be used as the handler for option value
+ pre-processing when using getters. RawConfigParser object s don't do
+ any sort of interpolation, whereas ConfigParser uses an instance of
+ BasicInterpolation. The library also provides a ``zc.buildbot``
+ inspired ExtendedInterpolation implementation.
+
+ When `converters` is given, it should be a dictionary where each key
+ represents the name of a type converter and each value is a callable
+ implementing the conversion from string to the desired datatype. Every
+ converter gets its corresponding get*() method on the parser object and
+ section proxies.
+
sections()
Return all the configuration section names, sans DEFAULT.
@@ -129,9 +149,11 @@ import warnings
__all__ = ["NoSectionError", "DuplicateOptionError", "DuplicateSectionError",
"NoOptionError", "InterpolationError", "InterpolationDepthError",
- "InterpolationSyntaxError", "ParsingError",
- "MissingSectionHeaderError",
+ "InterpolationMissingOptionError", "InterpolationSyntaxError",
+ "ParsingError", "MissingSectionHeaderError",
"ConfigParser", "SafeConfigParser", "RawConfigParser",
+ "Interpolation", "BasicInterpolation", "ExtendedInterpolation",
+ "LegacyInterpolation", "SectionProxy", "ConverterMapping",
"DEFAULTSECT", "MAX_INTERPOLATION_DEPTH"]
DEFAULTSECT = "DEFAULT"
@@ -408,7 +430,7 @@ class BasicInterpolation(Interpolation):
v = map[var]
except KeyError:
raise InterpolationMissingOptionError(
- option, section, rawval, var)
+ option, section, rawval, var) from None
if "%" in v:
self._interpolate_some(parser, option, accum, v,
section, map, depth + 1)
@@ -481,7 +503,7 @@ class ExtendedInterpolation(Interpolation):
"More than one ':' found: %r" % (rest,))
except (KeyError, NoSectionError, NoOptionError):
raise InterpolationMissingOptionError(
- option, section, rawval, ":".join(path))
+ option, section, rawval, ":".join(path)) from None
if "$" in v:
self._interpolate_some(parser, opt, accum, v, sect,
dict(parser.items(sect, raw=True)),
@@ -514,7 +536,7 @@ class LegacyInterpolation(Interpolation):
value = value % vars
except KeyError as e:
raise InterpolationMissingOptionError(
- option, section, rawval, e.args[0])
+ option, section, rawval, e.args[0]) from None
else:
break
if value and "%(" in value:
@@ -579,11 +601,12 @@ class RawConfigParser(MutableMapping):
comment_prefixes=('#', ';'), inline_comment_prefixes=None,
strict=True, empty_lines_in_values=True,
default_section=DEFAULTSECT,
- interpolation=_UNSET):
+ interpolation=_UNSET, converters=_UNSET):
self._dict = dict_type
self._sections = self._dict()
self._defaults = self._dict()
+ self._converters = ConverterMapping(self)
self._proxies = self._dict()
self._proxies[default_section] = SectionProxy(self, default_section)
if defaults:
@@ -611,6 +634,8 @@ class RawConfigParser(MutableMapping):
self._interpolation = self._DEFAULT_INTERPOLATION
if self._interpolation is None:
self._interpolation = Interpolation()
+ if converters is not _UNSET:
+ self._converters.update(converters)
def defaults(self):
return self._defaults
@@ -646,7 +671,7 @@ class RawConfigParser(MutableMapping):
try:
opts = self._sections[section].copy()
except KeyError:
- raise NoSectionError(section)
+ raise NoSectionError(section) from None
opts.update(self._defaults)
return list(opts.keys())
@@ -774,36 +799,31 @@ class RawConfigParser(MutableMapping):
def _get(self, section, conv, option, **kwargs):
return conv(self.get(section, option, **kwargs))
- def getint(self, section, option, *, raw=False, vars=None,
- fallback=_UNSET):
+ def _get_conv(self, section, option, conv, *, raw=False, vars=None,
+ fallback=_UNSET, **kwargs):
try:
- return self._get(section, int, option, raw=raw, vars=vars)
+ return self._get(section, conv, option, raw=raw, vars=vars,
+ **kwargs)
except (NoSectionError, NoOptionError):
if fallback is _UNSET:
raise
- else:
- return fallback
+ return fallback
+
+ # getint, getfloat and getboolean provided directly for backwards compat
+ def getint(self, section, option, *, raw=False, vars=None,
+ fallback=_UNSET, **kwargs):
+ return self._get_conv(section, option, int, raw=raw, vars=vars,
+ fallback=fallback, **kwargs)
def getfloat(self, section, option, *, raw=False, vars=None,
- fallback=_UNSET):
- try:
- return self._get(section, float, option, raw=raw, vars=vars)
- except (NoSectionError, NoOptionError):
- if fallback is _UNSET:
- raise
- else:
- return fallback
+ fallback=_UNSET, **kwargs):
+ return self._get_conv(section, option, float, raw=raw, vars=vars,
+ fallback=fallback, **kwargs)
def getboolean(self, section, option, *, raw=False, vars=None,
- fallback=_UNSET):
- try:
- return self._get(section, self._convert_to_boolean, option,
- raw=raw, vars=vars)
- except (NoSectionError, NoOptionError):
- if fallback is _UNSET:
- raise
- else:
- return fallback
+ fallback=_UNSET, **kwargs):
+ return self._get_conv(section, option, self._convert_to_boolean,
+ raw=raw, vars=vars, fallback=fallback, **kwargs)
def items(self, section=_UNSET, raw=False, vars=None):
"""Return a list of (name, value) tuples for each option in a section.
@@ -875,7 +895,7 @@ class RawConfigParser(MutableMapping):
try:
sectdict = self._sections[section]
except KeyError:
- raise NoSectionError(section)
+ raise NoSectionError(section) from None
sectdict[self.optionxform(option)] = value
def write(self, fp, space_around_delimiters=True):
@@ -916,7 +936,7 @@ class RawConfigParser(MutableMapping):
try:
sectdict = self._sections[section]
except KeyError:
- raise NoSectionError(section)
+ raise NoSectionError(section) from None
option = self.optionxform(option)
existed = option in sectdict
if existed:
@@ -1153,6 +1173,10 @@ class RawConfigParser(MutableMapping):
if not isinstance(value, str):
raise TypeError("option values must be strings")
+ @property
+ def converters(self):
+ return self._converters
+
class ConfigParser(RawConfigParser):
"""ConfigParser implementing interpolation."""
@@ -1193,6 +1217,10 @@ class SectionProxy(MutableMapping):
"""Creates a view on a section of the specified `name` in `parser`."""
self._parser = parser
self._name = name
+ for conv in parser.converters:
+ key = 'get' + conv
+ getter = functools.partial(self.get, _impl=getattr(parser, key))
+ setattr(self, key, getter)
def __repr__(self):
return '<Section: {}>'.format(self._name)
@@ -1226,22 +1254,6 @@ class SectionProxy(MutableMapping):
else:
return self._parser.defaults()
- def get(self, option, fallback=None, *, raw=False, vars=None):
- return self._parser.get(self._name, option, raw=raw, vars=vars,
- fallback=fallback)
-
- def getint(self, option, fallback=None, *, raw=False, vars=None):
- return self._parser.getint(self._name, option, raw=raw, vars=vars,
- fallback=fallback)
-
- def getfloat(self, option, fallback=None, *, raw=False, vars=None):
- return self._parser.getfloat(self._name, option, raw=raw, vars=vars,
- fallback=fallback)
-
- def getboolean(self, option, fallback=None, *, raw=False, vars=None):
- return self._parser.getboolean(self._name, option, raw=raw, vars=vars,
- fallback=fallback)
-
@property
def parser(self):
# The parser object of the proxy is read-only.
@@ -1251,3 +1263,77 @@ class SectionProxy(MutableMapping):
def name(self):
# The name of the section on a proxy is read-only.
return self._name
+
+ def get(self, option, fallback=None, *, raw=False, vars=None,
+ _impl=None, **kwargs):
+ """Get an option value.
+
+ Unless `fallback` is provided, `None` will be returned if the option
+ is not found.
+
+ """
+ # If `_impl` is provided, it should be a getter method on the parser
+ # object that provides the desired type conversion.
+ if not _impl:
+ _impl = self._parser.get
+ return _impl(self._name, option, raw=raw, vars=vars,
+ fallback=fallback, **kwargs)
+
+
+class ConverterMapping(MutableMapping):
+ """Enables reuse of get*() methods between the parser and section proxies.
+
+ If a parser class implements a getter directly, the value for the given
+ key will be ``None``. The presence of the converter name here enables
+ section proxies to find and use the implementation on the parser class.
+ """
+
+ GETTERCRE = re.compile(r"^get(?P<name>.+)$")
+
+ def __init__(self, parser):
+ self._parser = parser
+ self._data = {}
+ for getter in dir(self._parser):
+ m = self.GETTERCRE.match(getter)
+ if not m or not callable(getattr(self._parser, getter)):
+ continue
+ self._data[m.group('name')] = None # See class docstring.
+
+ def __getitem__(self, key):
+ return self._data[key]
+
+ def __setitem__(self, key, value):
+ try:
+ k = 'get' + key
+ except TypeError:
+ raise ValueError('Incompatible key: {} (type: {})'
+ ''.format(key, type(key)))
+ if k == 'get':
+ raise ValueError('Incompatible key: cannot use "" as a name')
+ self._data[key] = value
+ func = functools.partial(self._parser._get_conv, conv=value)
+ func.converter = value
+ setattr(self._parser, k, func)
+ for proxy in self._parser.values():
+ getter = functools.partial(proxy.get, _impl=func)
+ setattr(proxy, k, getter)
+
+ def __delitem__(self, key):
+ try:
+ k = 'get' + (key or None)
+ except TypeError:
+ raise KeyError(key)
+ del self._data[key]
+ for inst in itertools.chain((self._parser,), self._parser.values()):
+ try:
+ delattr(inst, k)
+ except AttributeError:
+ # don't raise since the entry was present in _data, silently
+ # clean up
+ continue
+
+ def __iter__(self):
+ return iter(self._data)
+
+ def __len__(self):
+ return len(self._data)
diff --git a/Lib/contextlib.py b/Lib/contextlib.py
index 07b2261..5377987 100644
--- a/Lib/contextlib.py
+++ b/Lib/contextlib.py
@@ -5,7 +5,7 @@ from collections import deque
from functools import wraps
__all__ = ["contextmanager", "closing", "ContextDecorator", "ExitStack",
- "redirect_stdout", "suppress"]
+ "redirect_stdout", "redirect_stderr", "suppress"]
class ContextDecorator(object):
@@ -77,10 +77,17 @@ class _GeneratorContextManager(ContextDecorator):
self.gen.throw(type, value, traceback)
raise RuntimeError("generator didn't stop after throw()")
except StopIteration as exc:
- # Suppress the exception *unless* it's the same exception that
+ # Suppress StopIteration *unless* it's the same exception that
# was passed to throw(). This prevents a StopIteration
- # raised inside the "with" statement from being suppressed
+ # raised inside the "with" statement from being suppressed.
return exc is not value
+ except RuntimeError as exc:
+ # Likewise, avoid suppressing if a StopIteration exception
+ # was passed to throw() and later wrapped into a RuntimeError
+ # (see PEP 479).
+ if exc.__cause__ is value:
+ return False
+ raise
except:
# only re-raise if it's *not* the exception that was
# passed to throw(), because __exit__() must not raise
@@ -151,8 +158,27 @@ class closing(object):
def __exit__(self, *exc_info):
self.thing.close()
-class redirect_stdout:
- """Context manager for temporarily redirecting stdout to another file
+
+class _RedirectStream:
+
+ _stream = None
+
+ def __init__(self, new_target):
+ self._new_target = new_target
+ # We use a list of old targets to make this CM re-entrant
+ self._old_targets = []
+
+ def __enter__(self):
+ self._old_targets.append(getattr(sys, self._stream))
+ setattr(sys, self._stream, self._new_target)
+ return self._new_target
+
+ def __exit__(self, exctype, excinst, exctb):
+ setattr(sys, self._stream, self._old_targets.pop())
+
+
+class redirect_stdout(_RedirectStream):
+ """Context manager for temporarily redirecting stdout to another file.
# How to send help() to stderr
with redirect_stdout(sys.stderr):
@@ -164,18 +190,13 @@ class redirect_stdout:
help(pow)
"""
- def __init__(self, new_target):
- self._new_target = new_target
- # We use a list of old targets to make this CM re-entrant
- self._old_targets = []
+ _stream = "stdout"
- def __enter__(self):
- self._old_targets.append(sys.stdout)
- sys.stdout = self._new_target
- return self._new_target
- def __exit__(self, exctype, excinst, exctb):
- sys.stdout = self._old_targets.pop()
+class redirect_stderr(_RedirectStream):
+ """Context manager for temporarily redirecting stderr to another file."""
+
+ _stream = "stderr"
class suppress:
diff --git a/Lib/copy.py b/Lib/copy.py
index bb8840e..3a45fdf 100644
--- a/Lib/copy.py
+++ b/Lib/copy.py
@@ -94,7 +94,7 @@ def copy(x):
else:
reductor = getattr(x, "__reduce_ex__", None)
if reductor:
- rv = reductor(2)
+ rv = reductor(4)
else:
reductor = getattr(x, "__reduce__", None)
if reductor:
@@ -171,7 +171,7 @@ def deepcopy(x, memo=None, _nil=[]):
else:
reductor = getattr(x, "__reduce_ex__", None)
if reductor:
- rv = reductor(2)
+ rv = reductor(4)
else:
reductor = getattr(x, "__reduce__", None)
if reductor:
@@ -221,17 +221,15 @@ def _deepcopy_list(x, memo):
d[list] = _deepcopy_list
def _deepcopy_tuple(x, memo):
- y = []
- for a in x:
- y.append(deepcopy(a, memo))
+ y = [deepcopy(a, memo) for a in x]
# We're not going to put the tuple in the memo, but it's still important we
# check for it, in case the tuple contains recursive mutable structures.
try:
return memo[id(x)]
except KeyError:
pass
- for i in range(len(x)):
- if x[i] is not y[i]:
+ for k, j in zip(x, y):
+ if k is not j:
y = tuple(y)
break
else:
diff --git a/Lib/csv.py b/Lib/csv.py
index a56eed8..ca40e5e 100644
--- a/Lib/csv.py
+++ b/Lib/csv.py
@@ -147,16 +147,13 @@ class DictWriter:
if wrong_fields:
raise ValueError("dict contains fields not in fieldnames: "
+ ", ".join([repr(x) for x in wrong_fields]))
- return [rowdict.get(key, self.restval) for key in self.fieldnames]
+ return (rowdict.get(key, self.restval) for key in self.fieldnames)
def writerow(self, rowdict):
return self.writer.writerow(self._dict_to_list(rowdict))
def writerows(self, rowdicts):
- rows = []
- for rowdict in rowdicts:
- rows.append(self._dict_to_list(rowdict))
- return self.writer.writerows(rows)
+ return self.writer.writerows(map(self._dict_to_list, rowdicts))
# Guard Sniffer's type checking against builds that exclude complex()
try:
@@ -231,20 +228,21 @@ class Sniffer:
quotes = {}
delims = {}
spaces = 0
+ groupindex = regexp.groupindex
for m in matches:
- n = regexp.groupindex['quote'] - 1
+ n = groupindex['quote'] - 1
key = m[n]
if key:
quotes[key] = quotes.get(key, 0) + 1
try:
- n = regexp.groupindex['delim'] - 1
+ n = groupindex['delim'] - 1
key = m[n]
except KeyError:
continue
if key and (delimiters is None or key in delimiters):
delims[key] = delims.get(key, 0) + 1
try:
- n = regexp.groupindex['space'] - 1
+ n = groupindex['space'] - 1
except KeyError:
continue
if m[n]:
diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py
index 5c803ff..4cb6d0d 100644
--- a/Lib/ctypes/__init__.py
+++ b/Lib/ctypes/__init__.py
@@ -237,14 +237,8 @@ _check_size(c_char)
class c_char_p(_SimpleCData):
_type_ = "z"
- if _os.name == "nt":
- def __repr__(self):
- if not windll.kernel32.IsBadStringPtrA(self, -1):
- return "%s(%r)" % (self.__class__.__name__, self.value)
- return "%s(%s)" % (self.__class__.__name__, cast(self, c_void_p).value)
- else:
- def __repr__(self):
- return "%s(%s)" % (self.__class__.__name__, cast(self, c_void_p).value)
+ def __repr__(self):
+ return "%s(%s)" % (self.__class__.__name__, c_void_p.from_buffer(self).value)
_check_size(c_char_p, "P")
class c_void_p(_SimpleCData):
@@ -259,6 +253,8 @@ from _ctypes import POINTER, pointer, _pointer_type_cache
class c_wchar_p(_SimpleCData):
_type_ = "Z"
+ def __repr__(self):
+ return "%s(%s)" % (self.__class__.__name__, c_void_p.from_buffer(self).value)
class c_wchar(_SimpleCData):
_type_ = "u"
@@ -353,7 +349,7 @@ class CDLL(object):
self._handle = handle
def __repr__(self):
- return "<%s '%s', handle %x at %x>" % \
+ return "<%s '%s', handle %x at %#x>" % \
(self.__class__.__name__, self._name,
(self._handle & (_sys.maxsize*2 + 1)),
id(self) & (_sys.maxsize*2 + 1))
diff --git a/Lib/ctypes/_endian.py b/Lib/ctypes/_endian.py
index dae65fc..37444bd 100644
--- a/Lib/ctypes/_endian.py
+++ b/Lib/ctypes/_endian.py
@@ -45,6 +45,7 @@ if sys.byteorder == "little":
class BigEndianStructure(Structure, metaclass=_swapped_meta):
"""Structure with big endian byte order"""
+ __slots__ = ()
_swappedbytes_ = None
elif sys.byteorder == "big":
@@ -53,6 +54,7 @@ elif sys.byteorder == "big":
BigEndianStructure = Structure
class LittleEndianStructure(Structure, metaclass=_swapped_meta):
"""Structure with little endian byte order"""
+ __slots__ = ()
_swappedbytes_ = None
else:
diff --git a/Lib/ctypes/test/test_as_parameter.py b/Lib/ctypes/test/test_as_parameter.py
index 948b463..2a3484b 100644
--- a/Lib/ctypes/test/test_as_parameter.py
+++ b/Lib/ctypes/test/test_as_parameter.py
@@ -194,7 +194,7 @@ class BasicWrapTestCase(unittest.TestCase):
a = A()
a._as_parameter_ = a
- with self.assertRaises(RuntimeError):
+ with self.assertRaises(RecursionError):
c_int.from_param(a)
diff --git a/Lib/ctypes/test/test_byteswap.py b/Lib/ctypes/test/test_byteswap.py
index 427bb8b..01c97e8 100644
--- a/Lib/ctypes/test/test_byteswap.py
+++ b/Lib/ctypes/test/test_byteswap.py
@@ -22,6 +22,26 @@ class Test(unittest.TestCase):
setattr(bits, "i%s" % i, 1)
dump(bits)
+ def test_slots(self):
+ class BigPoint(BigEndianStructure):
+ __slots__ = ()
+ _fields_ = [("x", c_int), ("y", c_int)]
+
+ class LowPoint(LittleEndianStructure):
+ __slots__ = ()
+ _fields_ = [("x", c_int), ("y", c_int)]
+
+ big = BigPoint()
+ little = LowPoint()
+ big.x = 4
+ big.y = 2
+ little.x = 2
+ little.y = 4
+ with self.assertRaises(AttributeError):
+ big.z = 42
+ with self.assertRaises(AttributeError):
+ little.z = 24
+
def test_endian_short(self):
if sys.byteorder == "little":
self.assertIs(c_short.__ctype_le__, c_short)
diff --git a/Lib/ctypes/test/test_loading.py b/Lib/ctypes/test/test_loading.py
index 4fb8964..28468c1 100644
--- a/Lib/ctypes/test/test_loading.py
+++ b/Lib/ctypes/test/test_loading.py
@@ -52,7 +52,9 @@ class LoaderTest(unittest.TestCase):
@unittest.skipUnless(os.name in ("nt", "ce"),
'test specific to Windows (NT/CE)')
def test_load_library(self):
- self.assertIsNotNone(libc_name)
+ # CRT is no longer directly loadable. See issue23606 for the
+ # discussion about alternative approaches.
+ #self.assertIsNotNone(libc_name)
if test.support.verbose:
print(find_library("kernel32"))
print(find_library("user32"))
diff --git a/Lib/ctypes/test/test_pointers.py b/Lib/ctypes/test/test_pointers.py
index e24a520..40738f7 100644
--- a/Lib/ctypes/test/test_pointers.py
+++ b/Lib/ctypes/test/test_pointers.py
@@ -22,7 +22,10 @@ class PointersTestCase(unittest.TestCase):
def test_pass_pointers(self):
dll = CDLL(_ctypes_test.__file__)
func = dll._testfunc_p_p
- func.restype = c_long
+ if sizeof(c_longlong) == sizeof(c_void_p):
+ func.restype = c_longlong
+ else:
+ func.restype = c_long
i = c_int(12345678)
## func.argtypes = (POINTER(c_int),)
diff --git a/Lib/ctypes/test/test_prototypes.py b/Lib/ctypes/test/test_prototypes.py
index 818c111..cd0c649 100644
--- a/Lib/ctypes/test/test_prototypes.py
+++ b/Lib/ctypes/test/test_prototypes.py
@@ -69,7 +69,10 @@ class CharPointersTestCase(unittest.TestCase):
def test_int_pointer_arg(self):
func = testdll._testfunc_p_p
- func.restype = c_long
+ if sizeof(c_longlong) == sizeof(c_void_p):
+ func.restype = c_longlong
+ else:
+ func.restype = c_long
self.assertEqual(0, func(0))
ci = c_int(0)
diff --git a/Lib/ctypes/test/test_values.py b/Lib/ctypes/test/test_values.py
index 1c1fd7d..9551e7a 100644
--- a/Lib/ctypes/test/test_values.py
+++ b/Lib/ctypes/test/test_values.py
@@ -59,8 +59,13 @@ class Win_ValuesTestCase(unittest.TestCase):
items = []
# _frozen_importlib changes size whenever importlib._bootstrap
# changes, so it gets a special case. We should make sure it's
- # found, but don't worry about its size too much.
- _fzn_implib_seen = False
+ # found, but don't worry about its size too much. The same
+ # applies to _frozen_importlib_external.
+ bootstrap_seen = []
+ bootstrap_expected = [
+ b'_frozen_importlib',
+ b'_frozen_importlib_external',
+ ]
for entry in ft:
# This is dangerous. We *can* iterate over a pointer, but
# the loop will not terminate (maybe with an access
@@ -68,10 +73,10 @@ class Win_ValuesTestCase(unittest.TestCase):
if entry.name is None:
break
- if entry.name == b'_frozen_importlib':
- _fzn_implib_seen = True
+ if entry.name in bootstrap_expected:
+ bootstrap_seen.append(entry.name)
self.assertTrue(entry.size,
- "_frozen_importlib was reported as having no size")
+ "{} was reported as having no size".format(entry.name))
continue
items.append((entry.name, entry.size))
@@ -81,8 +86,8 @@ class Win_ValuesTestCase(unittest.TestCase):
]
self.assertEqual(items, expected)
- self.assertTrue(_fzn_implib_seen,
- "_frozen_importlib wasn't found in PyImport_FrozenModules")
+ self.assertEqual(sorted(bootstrap_seen), bootstrap_expected,
+ "frozen bootstrap modules did not match PyImport_FrozenModules")
from ctypes import _pointer_type_cache
del _pointer_type_cache[struct_frozen]
diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py
index 595113b..9e74ccd 100644
--- a/Lib/ctypes/util.py
+++ b/Lib/ctypes/util.py
@@ -19,6 +19,8 @@ if os.name == "nt":
i = i + len(prefix)
s, rest = sys.version[i:].split(" ", 1)
majorVersion = int(s[:-2]) - 6
+ if majorVersion >= 13:
+ majorVersion += 1
minorVersion = int(s[2:3]) / 10.0
# I don't think paths are affected by minor version in version 6
if majorVersion == 6:
@@ -36,8 +38,12 @@ if os.name == "nt":
return None
if version <= 6:
clibname = 'msvcrt'
- else:
+ elif version <= 13:
clibname = 'msvcr%d' % (version * 10)
+ else:
+ # CRT is no longer directly loadable. See issue23606 for the
+ # discussion about alternative approaches.
+ return None
# If python was built with in debug mode
import importlib.machinery
diff --git a/Lib/datetime.py b/Lib/datetime.py
index 34e5d38..db13b12 100644
--- a/Lib/datetime.py
+++ b/Lib/datetime.py
@@ -12,7 +12,7 @@ def _cmp(x, y):
MINYEAR = 1
MAXYEAR = 9999
-_MAXORDINAL = 3652059 # date.max.toordinal()
+_MAXORDINAL = 3652059 # date.max.toordinal()
# Utility functions, adapted from Python's Demo/classes/Dates.py, which
# also assumes the current Gregorian calendar indefinitely extended in
@@ -26,7 +26,7 @@ _MAXORDINAL = 3652059 # date.max.toordinal()
# -1 is a placeholder for indexing purposes.
_DAYS_IN_MONTH = [-1, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
-_DAYS_BEFORE_MONTH = [-1] # -1 is a placeholder for indexing purposes.
+_DAYS_BEFORE_MONTH = [-1] # -1 is a placeholder for indexing purposes.
dbm = 0
for dim in _DAYS_IN_MONTH[1:]:
_DAYS_BEFORE_MONTH.append(dbm)
@@ -162,9 +162,9 @@ def _format_time(hh, mm, ss, us):
# Correctly substitute for %z and %Z escapes in strftime formats.
def _wrap_strftime(object, format, timetuple):
# Don't call utcoffset() or tzname() unless actually needed.
- freplace = None # the string to use for %f
- zreplace = None # the string to use for %z
- Zreplace = None # the string to use for %Z
+ freplace = None # the string to use for %f
+ zreplace = None # the string to use for %z
+ Zreplace = None # the string to use for %Z
# Scan format for %z and %Z escapes, replacing as needed.
newformat = []
@@ -217,11 +217,6 @@ def _wrap_strftime(object, format, timetuple):
newformat = "".join(newformat)
return _time.strftime(newformat, timetuple)
-def _call_tzinfo_method(tzinfo, methname, tzinfoarg):
- if tzinfo is None:
- return None
- return getattr(tzinfo, methname)(tzinfoarg)
-
# Just raise TypeError if the arg isn't None or a string.
def _check_tzname(name):
if name is not None and not isinstance(name, str):
@@ -245,13 +240,31 @@ def _check_utc_offset(name, offset):
raise ValueError("tzinfo.%s() must return a whole number "
"of minutes, got %s" % (name, offset))
if not -timedelta(1) < offset < timedelta(1):
- raise ValueError("%s()=%s, must be must be strictly between"
- " -timedelta(hours=24) and timedelta(hours=24)"
- % (name, offset))
+ raise ValueError("%s()=%s, must be must be strictly between "
+ "-timedelta(hours=24) and timedelta(hours=24)" %
+ (name, offset))
+
+def _check_int_field(value):
+ if isinstance(value, int):
+ return value
+ if not isinstance(value, float):
+ try:
+ value = value.__int__()
+ except AttributeError:
+ pass
+ else:
+ if isinstance(value, int):
+ return value
+ raise TypeError('__int__ returned non-int (type %s)' %
+ type(value).__name__)
+ raise TypeError('an integer is required (got type %s)' %
+ type(value).__name__)
+ raise TypeError('integer argument expected, got float')
def _check_date_fields(year, month, day):
- if not isinstance(year, int):
- raise TypeError('int expected')
+ year = _check_int_field(year)
+ month = _check_int_field(month)
+ day = _check_int_field(day)
if not MINYEAR <= year <= MAXYEAR:
raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year)
if not 1 <= month <= 12:
@@ -259,10 +272,13 @@ def _check_date_fields(year, month, day):
dim = _days_in_month(year, month)
if not 1 <= day <= dim:
raise ValueError('day must be in 1..%d' % dim, day)
+ return year, month, day
def _check_time_fields(hour, minute, second, microsecond):
- if not isinstance(hour, int):
- raise TypeError('int expected')
+ hour = _check_int_field(hour)
+ minute = _check_int_field(minute)
+ second = _check_int_field(second)
+ microsecond = _check_int_field(microsecond)
if not 0 <= hour <= 23:
raise ValueError('hour must be in 0..23', hour)
if not 0 <= minute <= 59:
@@ -271,6 +287,7 @@ def _check_time_fields(hour, minute, second, microsecond):
raise ValueError('second must be in 0..59', second)
if not 0 <= microsecond <= 999999:
raise ValueError('microsecond must be in 0..999999', microsecond)
+ return hour, minute, second, microsecond
def _check_tzinfo_arg(tz):
if tz is not None and not isinstance(tz, tzinfo):
@@ -316,7 +333,7 @@ class timedelta:
Representation: (days, seconds, microseconds). Why? Because I
felt like it.
"""
- __slots__ = '_days', '_seconds', '_microseconds'
+ __slots__ = '_days', '_seconds', '_microseconds', '_hashcode'
def __new__(cls, days=0, seconds=0, microseconds=0,
milliseconds=0, minutes=0, hours=0, weeks=0):
@@ -382,38 +399,26 @@ class timedelta:
# secondsfrac isn't referenced again
if isinstance(microseconds, float):
- microseconds += usdouble
- microseconds = round(microseconds, 0)
- seconds, microseconds = divmod(microseconds, 1e6)
- assert microseconds == int(microseconds)
- assert seconds == int(seconds)
- days, seconds = divmod(seconds, 24.*3600.)
- assert days == int(days)
- assert seconds == int(seconds)
- d += int(days)
- s += int(seconds) # can't overflow
- assert isinstance(s, int)
- assert abs(s) <= 3 * 24 * 3600
+ microseconds = round(microseconds + usdouble)
+ seconds, microseconds = divmod(microseconds, 1000000)
+ days, seconds = divmod(seconds, 24*3600)
+ d += days
+ s += seconds
else:
+ microseconds = int(microseconds)
seconds, microseconds = divmod(microseconds, 1000000)
days, seconds = divmod(seconds, 24*3600)
d += days
- s += int(seconds) # can't overflow
- assert isinstance(s, int)
- assert abs(s) <= 3 * 24 * 3600
- microseconds = float(microseconds)
- microseconds += usdouble
- microseconds = round(microseconds, 0)
+ s += seconds
+ microseconds = round(microseconds + usdouble)
+ assert isinstance(s, int)
+ assert isinstance(microseconds, int)
assert abs(s) <= 3 * 24 * 3600
assert abs(microseconds) < 3.1e6
# Just a little bit of carrying possible for microseconds and seconds.
- assert isinstance(microseconds, float)
- assert int(microseconds) == microseconds
- us = int(microseconds)
- seconds, us = divmod(us, 1000000)
- s += seconds # cant't overflow
- assert isinstance(s, int)
+ seconds, us = divmod(microseconds, 1000000)
+ s += seconds
days, s = divmod(s, 24*3600)
d += days
@@ -421,27 +426,31 @@ class timedelta:
assert isinstance(s, int) and 0 <= s < 24*3600
assert isinstance(us, int) and 0 <= us < 1000000
- self = object.__new__(cls)
+ if abs(d) > 999999999:
+ raise OverflowError("timedelta # of days is too large: %d" % d)
+ self = object.__new__(cls)
self._days = d
self._seconds = s
self._microseconds = us
- if abs(d) > 999999999:
- raise OverflowError("timedelta # of days is too large: %d" % d)
-
+ self._hashcode = -1
return self
def __repr__(self):
if self._microseconds:
- return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__,
- self._days,
- self._seconds,
- self._microseconds)
+ return "%s.%s(%d, %d, %d)" % (self.__class__.__module__,
+ self.__class__.__qualname__,
+ self._days,
+ self._seconds,
+ self._microseconds)
if self._seconds:
- return "%s(%d, %d)" % ('datetime.' + self.__class__.__name__,
- self._days,
- self._seconds)
- return "%s(%d)" % ('datetime.' + self.__class__.__name__, self._days)
+ return "%s.%s(%d, %d)" % (self.__class__.__module__,
+ self.__class__.__qualname__,
+ self._days,
+ self._seconds)
+ return "%s.%s(%d)" % (self.__class__.__module__,
+ self.__class__.__qualname__,
+ self._days)
def __str__(self):
mm, ss = divmod(self._seconds, 60)
@@ -457,7 +466,7 @@ class timedelta:
def total_seconds(self):
"""Total seconds in the duration."""
- return ((self.days * 86400 + self.seconds)*10**6 +
+ return ((self.days * 86400 + self.seconds) * 10**6 +
self.microseconds) / 10**6
# Read-only field accessors
@@ -578,12 +587,6 @@ class timedelta:
else:
return False
- def __ne__(self, other):
- if isinstance(other, timedelta):
- return self._cmp(other) != 0
- else:
- return True
-
def __le__(self, other):
if isinstance(other, timedelta):
return self._cmp(other) <= 0
@@ -613,7 +616,9 @@ class timedelta:
return _cmp(self._getstate(), other._getstate())
def __hash__(self):
- return hash(self._getstate())
+ if self._hashcode == -1:
+ self._hashcode = hash(self._getstate())
+ return self._hashcode
def __bool__(self):
return (self._days != 0 or
@@ -661,7 +666,7 @@ class date:
Properties (readonly):
year, month, day
"""
- __slots__ = '_year', '_month', '_day'
+ __slots__ = '_year', '_month', '_day', '_hashcode'
def __new__(cls, year, month=None, day=None):
"""Constructor.
@@ -670,17 +675,19 @@ class date:
year, month, day (required, base 1)
"""
- if (isinstance(year, bytes) and len(year) == 4 and
- 1 <= year[2] <= 12 and month is None): # Month is sane
+ if month is None and isinstance(year, bytes) and len(year) == 4 and \
+ 1 <= year[2] <= 12:
# Pickle support
self = object.__new__(cls)
self.__setstate(year)
+ self._hashcode = -1
return self
- _check_date_fields(year, month, day)
+ year, month, day = _check_date_fields(year, month, day)
self = object.__new__(cls)
self._year = year
self._month = month
self._day = day
+ self._hashcode = -1
return self
# Additional constructors
@@ -720,10 +727,11 @@ class date:
>>> repr(dt)
'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)'
"""
- return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__,
- self._year,
- self._month,
- self._day)
+ return "%s.%s(%d, %d, %d)" % (self.__class__.__module__,
+ self.__class__.__qualname__,
+ self._year,
+ self._month,
+ self._day)
# XXX These shouldn't depend on time.localtime(), because that
# clips the usable dates to [1970 .. 2038). At least ctime() is
# easily done without using strftime() -- that's better too because
@@ -743,6 +751,8 @@ class date:
return _wrap_strftime(self, fmt, self.timetuple())
def __format__(self, fmt):
+ if not isinstance(fmt, str):
+ raise TypeError("must be str, not %s" % type(fmt).__name__)
if len(fmt) != 0:
return self.strftime(fmt)
return str(self)
@@ -800,7 +810,6 @@ class date:
month = self._month
if day is None:
day = self._day
- _check_date_fields(year, month, day)
return date(year, month, day)
# Comparisons of date objects with other.
@@ -810,11 +819,6 @@ class date:
return self._cmp(other) == 0
return NotImplemented
- def __ne__(self, other):
- if isinstance(other, date):
- return self._cmp(other) != 0
- return NotImplemented
-
def __le__(self, other):
if isinstance(other, date):
return self._cmp(other) <= 0
@@ -843,7 +847,9 @@ class date:
def __hash__(self):
"Hash."
- return hash(self._getstate())
+ if self._hashcode == -1:
+ self._hashcode = hash(self._getstate())
+ return self._hashcode
# Computations
@@ -913,8 +919,6 @@ class date:
return bytes([yhi, ylo, self._month, self._day]),
def __setstate(self, string):
- if len(string) != 4 or not (1 <= string[2] <= 12):
- raise TypeError("not enough arguments")
yhi, ylo, self._month, self._day = string
self._year = yhi * 256 + ylo
@@ -933,6 +937,7 @@ class tzinfo:
Subclasses must override the name(), utcoffset() and dst() methods.
"""
__slots__ = ()
+
def tzname(self, dt):
"datetime -> string name of time zone."
raise NotImplementedError("tzinfo subclass must override tzname()")
@@ -1019,6 +1024,7 @@ class time:
Properties (readonly):
hour, minute, second, microsecond, tzinfo
"""
+ __slots__ = '_hour', '_minute', '_second', '_microsecond', '_tzinfo', '_hashcode'
def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None):
"""Constructor.
@@ -1029,18 +1035,22 @@ class time:
second, microsecond (default to zero)
tzinfo (default to None)
"""
- self = object.__new__(cls)
- if isinstance(hour, bytes) and len(hour) == 6:
+ if isinstance(hour, bytes) and len(hour) == 6 and hour[0] < 24:
# Pickle support
+ self = object.__new__(cls)
self.__setstate(hour, minute or None)
+ self._hashcode = -1
return self
+ hour, minute, second, microsecond = _check_time_fields(
+ hour, minute, second, microsecond)
_check_tzinfo_arg(tzinfo)
- _check_time_fields(hour, minute, second, microsecond)
+ self = object.__new__(cls)
self._hour = hour
self._minute = minute
self._second = second
self._microsecond = microsecond
self._tzinfo = tzinfo
+ self._hashcode = -1
return self
# Read-only field accessors
@@ -1079,12 +1089,6 @@ class time:
else:
return False
- def __ne__(self, other):
- if isinstance(other, time):
- return self._cmp(other, allow_mixed=True) != 0
- else:
- return True
-
def __le__(self, other):
if isinstance(other, time):
return self._cmp(other) <= 0
@@ -1125,8 +1129,8 @@ class time:
if base_compare:
return _cmp((self._hour, self._minute, self._second,
self._microsecond),
- (other._hour, other._minute, other._second,
- other._microsecond))
+ (other._hour, other._minute, other._second,
+ other._microsecond))
if myoff is None or otoff is None:
if allow_mixed:
return 2 # arbitrary non-zero value
@@ -1139,16 +1143,20 @@ class time:
def __hash__(self):
"""Hash."""
- tzoff = self.utcoffset()
- if not tzoff: # zero or None
- return hash(self._getstate()[0])
- h, m = divmod(timedelta(hours=self.hour, minutes=self.minute) - tzoff,
- timedelta(hours=1))
- assert not m % timedelta(minutes=1), "whole minute"
- m //= timedelta(minutes=1)
- if 0 <= h < 24:
- return hash(time(h, m, self.second, self.microsecond))
- return hash((h, m, self.second, self.microsecond))
+ if self._hashcode == -1:
+ tzoff = self.utcoffset()
+ if not tzoff: # zero or None
+ self._hashcode = hash(self._getstate()[0])
+ else:
+ h, m = divmod(timedelta(hours=self.hour, minutes=self.minute) - tzoff,
+ timedelta(hours=1))
+ assert not m % timedelta(minutes=1), "whole minute"
+ m //= timedelta(minutes=1)
+ if 0 <= h < 24:
+ self._hashcode = hash(time(h, m, self.second, self.microsecond))
+ else:
+ self._hashcode = hash((h, m, self.second, self.microsecond))
+ return self._hashcode
# Conversion to string
@@ -1176,8 +1184,9 @@ class time:
s = ", %d" % self._second
else:
s = ""
- s= "%s(%d, %d%s)" % ('datetime.' + self.__class__.__name__,
- self._hour, self._minute, s)
+ s= "%s.%s(%d, %d%s)" % (self.__class__.__module__,
+ self.__class__.__qualname__,
+ self._hour, self._minute, s)
if self._tzinfo is not None:
assert s[-1:] == ")"
s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")"
@@ -1210,6 +1219,8 @@ class time:
return _wrap_strftime(self, fmt, timetuple)
def __format__(self, fmt):
+ if not isinstance(fmt, str):
+ raise TypeError("must be str, not %s" % type(fmt).__name__)
if len(fmt) != 0:
return self.strftime(fmt)
return str(self)
@@ -1266,16 +1277,8 @@ class time:
microsecond = self.microsecond
if tzinfo is True:
tzinfo = self.tzinfo
- _check_time_fields(hour, minute, second, microsecond)
- _check_tzinfo_arg(tzinfo)
return time(hour, minute, second, microsecond, tzinfo)
- def __bool__(self):
- if self.second or self.microsecond:
- return True
- offset = self.utcoffset() or timedelta(0)
- return timedelta(hours=self.hour, minutes=self.minute) != offset
-
# Pickle support.
def _getstate(self):
@@ -1289,15 +1292,11 @@ class time:
return (basestate, self._tzinfo)
def __setstate(self, string, tzinfo):
- if len(string) != 6 or string[0] >= 24:
- raise TypeError("an integer is required")
- (self._hour, self._minute, self._second,
- us1, us2, us3) = string
+ if tzinfo is not None and not isinstance(tzinfo, _tzinfo_class):
+ raise TypeError("bad tzinfo state arg")
+ self._hour, self._minute, self._second, us1, us2, us3 = string
self._microsecond = (((us1 << 8) | us2) << 8) | us3
- if tzinfo is None or isinstance(tzinfo, _tzinfo_class):
- self._tzinfo = tzinfo
- else:
- raise TypeError("bad tzinfo state arg %r" % tzinfo)
+ self._tzinfo = tzinfo
def __reduce__(self):
return (time, self._getstate())
@@ -1314,25 +1313,30 @@ class datetime(date):
The year, month and day arguments are required. tzinfo may be None, or an
instance of a tzinfo subclass. The remaining arguments may be ints.
"""
+ __slots__ = date.__slots__ + time.__slots__
- __slots__ = date.__slots__ + (
- '_hour', '_minute', '_second',
- '_microsecond', '_tzinfo')
def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0,
microsecond=0, tzinfo=None):
- if isinstance(year, bytes) and len(year) == 10:
+ if isinstance(year, bytes) and len(year) == 10 and 1 <= year[2] <= 12:
# Pickle support
- self = date.__new__(cls, year[:4])
+ self = object.__new__(cls)
self.__setstate(year, month)
+ self._hashcode = -1
return self
+ year, month, day = _check_date_fields(year, month, day)
+ hour, minute, second, microsecond = _check_time_fields(
+ hour, minute, second, microsecond)
_check_tzinfo_arg(tzinfo)
- _check_time_fields(hour, minute, second, microsecond)
- self = date.__new__(cls, year, month, day)
+ self = object.__new__(cls)
+ self._year = year
+ self._month = month
+ self._day = day
self._hour = hour
self._minute = minute
self._second = second
self._microsecond = microsecond
self._tzinfo = tzinfo
+ self._hashcode = -1
return self
# Read-only field accessors
@@ -1367,7 +1371,6 @@ class datetime(date):
A timezone info object may be passed in as well.
"""
-
_check_tzinfo_arg(tz)
converter = _time.localtime if tz is None else _time.gmtime
@@ -1391,7 +1394,7 @@ class datetime(date):
@classmethod
def utcfromtimestamp(cls, t):
- "Construct a UTC datetime from a POSIX timestamp (like time.time())."
+ """Construct a naive UTC datetime from a POSIX timestamp."""
t, frac = divmod(t, 1.0)
us = int(frac * 1e6)
@@ -1406,11 +1409,6 @@ class datetime(date):
ss = min(ss, 59) # clamp out leap seconds if the platform has them
return cls(y, m, d, hh, mm, ss, us)
- # XXX This is supposed to do better than we *can* do by using time.time(),
- # XXX if the platform supports a more accurate way. The C implementation
- # XXX uses gettimeofday on platforms that have it, but that isn't
- # XXX available from Python. So now() may return different results
- # XXX across the implementations.
@classmethod
def now(cls, tz=None):
"Construct a datetime from time.time() and optional time zone info."
@@ -1497,11 +1495,8 @@ class datetime(date):
microsecond = self.microsecond
if tzinfo is True:
tzinfo = self.tzinfo
- _check_date_fields(year, month, day)
- _check_time_fields(hour, minute, second, microsecond)
- _check_tzinfo_arg(tzinfo)
- return datetime(year, month, day, hour, minute, second,
- microsecond, tzinfo)
+ return datetime(year, month, day, hour, minute, second, microsecond,
+ tzinfo)
def astimezone(self, tz=None):
if tz is None:
@@ -1571,10 +1566,9 @@ class datetime(date):
Optional argument sep specifies the separator between date and
time, default 'T'.
"""
- s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day,
- sep) +
- _format_time(self._hour, self._minute, self._second,
- self._microsecond))
+ s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day, sep) +
+ _format_time(self._hour, self._minute, self._second,
+ self._microsecond))
off = self.utcoffset()
if off is not None:
if off.days < 0:
@@ -1590,14 +1584,15 @@ class datetime(date):
def __repr__(self):
"""Convert to formal string, for repr()."""
- L = [self._year, self._month, self._day, # These are never zero
+ L = [self._year, self._month, self._day, # These are never zero
self._hour, self._minute, self._second, self._microsecond]
if L[-1] == 0:
del L[-1]
if L[-1] == 0:
del L[-1]
- s = ", ".join(map(str, L))
- s = "%s(%s)" % ('datetime.' + self.__class__.__name__, s)
+ s = "%s.%s(%s)" % (self.__class__.__module__,
+ self.__class__.__qualname__,
+ ", ".join(map(str, L)))
if self._tzinfo is not None:
assert s[-1:] == ")"
s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")"
@@ -1629,7 +1624,9 @@ class datetime(date):
it mean anything in particular. For example, "GMT", "UTC", "-500",
"-5:00", "EDT", "US/Eastern", "America/New York" are all valid replies.
"""
- name = _call_tzinfo_method(self._tzinfo, "tzname", self)
+ if self._tzinfo is None:
+ return None
+ name = self._tzinfo.tzname(self)
_check_tzname(name)
return name
@@ -1658,14 +1655,6 @@ class datetime(date):
else:
return False
- def __ne__(self, other):
- if isinstance(other, datetime):
- return self._cmp(other, allow_mixed=True) != 0
- elif not isinstance(other, date):
- return NotImplemented
- else:
- return True
-
def __le__(self, other):
if isinstance(other, datetime):
return self._cmp(other) <= 0
@@ -1715,9 +1704,9 @@ class datetime(date):
return _cmp((self._year, self._month, self._day,
self._hour, self._minute, self._second,
self._microsecond),
- (other._year, other._month, other._day,
- other._hour, other._minute, other._second,
- other._microsecond))
+ (other._year, other._month, other._day,
+ other._hour, other._minute, other._second,
+ other._microsecond))
if myoff is None or otoff is None:
if allow_mixed:
return 2 # arbitrary non-zero value
@@ -1775,12 +1764,15 @@ class datetime(date):
return base + otoff - myoff
def __hash__(self):
- tzoff = self.utcoffset()
- if tzoff is None:
- return hash(self._getstate()[0])
- days = _ymd2ord(self.year, self.month, self.day)
- seconds = self.hour * 3600 + self.minute * 60 + self.second
- return hash(timedelta(days, seconds, self.microsecond) - tzoff)
+ if self._hashcode == -1:
+ tzoff = self.utcoffset()
+ if tzoff is None:
+ self._hashcode = hash(self._getstate()[0])
+ else:
+ days = _ymd2ord(self.year, self.month, self.day)
+ seconds = self.hour * 3600 + self.minute * 60 + self.second
+ self._hashcode = hash(timedelta(days, seconds, self.microsecond) - tzoff)
+ return self._hashcode
# Pickle support.
@@ -1797,14 +1789,13 @@ class datetime(date):
return (basestate, self._tzinfo)
def __setstate(self, string, tzinfo):
+ if tzinfo is not None and not isinstance(tzinfo, _tzinfo_class):
+ raise TypeError("bad tzinfo state arg")
(yhi, ylo, self._month, self._day, self._hour,
self._minute, self._second, us1, us2, us3) = string
self._year = yhi * 256 + ylo
self._microsecond = (((us1 << 8) | us2) << 8) | us3
- if tzinfo is None or isinstance(tzinfo, _tzinfo_class):
- self._tzinfo = tzinfo
- else:
- raise TypeError("bad tzinfo state arg %r" % tzinfo)
+ self._tzinfo = tzinfo
def __reduce__(self):
return (self.__class__, self._getstate())
@@ -1820,7 +1811,7 @@ def _isoweek1monday(year):
# XXX This could be done more efficiently
THURSDAY = 3
firstday = _ymd2ord(year, 1, 1)
- firstweekday = (firstday + 6) % 7 # See weekday() above
+ firstweekday = (firstday + 6) % 7 # See weekday() above
week1monday = firstday - firstweekday
if firstweekday > THURSDAY:
week1monday += 7
@@ -1841,13 +1832,12 @@ class timezone(tzinfo):
elif not isinstance(name, str):
raise TypeError("name must be a string")
if not cls._minoffset <= offset <= cls._maxoffset:
- raise ValueError("offset must be a timedelta"
- " strictly between -timedelta(hours=24) and"
- " timedelta(hours=24).")
- if (offset.microseconds != 0 or
- offset.seconds % 60 != 0):
- raise ValueError("offset must be a timedelta"
- " representing a whole number of minutes")
+ raise ValueError("offset must be a timedelta "
+ "strictly between -timedelta(hours=24) and "
+ "timedelta(hours=24).")
+ if (offset.microseconds != 0 or offset.seconds % 60 != 0):
+ raise ValueError("offset must be a timedelta "
+ "representing a whole number of minutes")
return cls._create(offset, name)
@classmethod
@@ -1884,10 +1874,12 @@ class timezone(tzinfo):
if self is self.utc:
return 'datetime.timezone.utc'
if self._name is None:
- return "%s(%r)" % ('datetime.' + self.__class__.__name__,
- self._offset)
- return "%s(%r, %r)" % ('datetime.' + self.__class__.__name__,
- self._offset, self._name)
+ return "%s.%s(%r)" % (self.__class__.__module__,
+ self.__class__.__qualname__,
+ self._offset)
+ return "%s.%s(%r, %r)" % (self.__class__.__module__,
+ self.__class__.__qualname__,
+ self._offset, self._name)
def __str__(self):
return self.tzname(None)
@@ -2142,14 +2134,13 @@ except ImportError:
pass
else:
# Clean up unused names
- del (_DAYNAMES, _DAYS_BEFORE_MONTH, _DAYS_IN_MONTH,
- _DI100Y, _DI400Y, _DI4Y, _MAXORDINAL, _MONTHNAMES,
- _build_struct_time, _call_tzinfo_method, _check_date_fields,
- _check_time_fields, _check_tzinfo_arg, _check_tzname,
- _check_utc_offset, _cmp, _cmperror, _date_class, _days_before_month,
- _days_before_year, _days_in_month, _format_time, _is_leap,
- _isoweek1monday, _math, _ord2ymd, _time, _time_class, _tzinfo_class,
- _wrap_strftime, _ymd2ord)
+ del (_DAYNAMES, _DAYS_BEFORE_MONTH, _DAYS_IN_MONTH, _DI100Y, _DI400Y,
+ _DI4Y, _EPOCH, _MAXORDINAL, _MONTHNAMES, _build_struct_time,
+ _check_date_fields, _check_int_field, _check_time_fields,
+ _check_tzinfo_arg, _check_tzname, _check_utc_offset, _cmp, _cmperror,
+ _date_class, _days_before_month, _days_before_year, _days_in_month,
+ _format_time, _is_leap, _isoweek1monday, _math, _ord2ymd,
+ _time, _time_class, _tzinfo_class, _wrap_strftime, _ymd2ord)
# XXX Since import * above excludes names that start with _,
# docstring does not get overwritten. In the future, it may be
# appropriate to maintain a single module level docstring and
diff --git a/Lib/dbm/__init__.py b/Lib/dbm/__init__.py
index 5f4664a..6831a84 100644
--- a/Lib/dbm/__init__.py
+++ b/Lib/dbm/__init__.py
@@ -153,9 +153,9 @@ def whichdb(filename):
except OSError:
return None
- # Read the start of the file -- the magic number
- s16 = f.read(16)
- f.close()
+ with f:
+ # Read the start of the file -- the magic number
+ s16 = f.read(16)
s = s16[0:4]
# Return "" if not at least 4 bytes
diff --git a/Lib/dbm/dumb.py b/Lib/dbm/dumb.py
index 63bc329..7777a7c 100644
--- a/Lib/dbm/dumb.py
+++ b/Lib/dbm/dumb.py
@@ -45,7 +45,7 @@ class _Database(collections.MutableMapping):
_os = _os # for _commit()
_io = _io # for _commit()
- def __init__(self, filebasename, mode):
+ def __init__(self, filebasename, mode, flag='c'):
self._mode = mode
# The directory file is a text file. Each line looks like
@@ -65,6 +65,17 @@ class _Database(collections.MutableMapping):
# The index is an in-memory dict, mirroring the directory file.
self._index = None # maps keys to (pos, siz) pairs
+ # Handle the creation
+ self._create(flag)
+ self._update()
+
+ def _create(self, flag):
+ if flag == 'n':
+ for filename in (self._datfile, self._bakfile, self._dirfile):
+ try:
+ _os.remove(filename)
+ except OSError:
+ pass
# Mod by Jack: create data file if needed
try:
f = _io.open(self._datfile, 'r', encoding="Latin-1")
@@ -73,7 +84,6 @@ class _Database(collections.MutableMapping):
self._chmod(self._datfile)
else:
f.close()
- self._update()
# Read directory file into the in-memory index dict.
def _update(self):
@@ -266,20 +276,20 @@ class _Database(collections.MutableMapping):
self.close()
-def open(file, flag=None, mode=0o666):
+def open(file, flag='c', mode=0o666):
"""Open the database file, filename, and return corresponding object.
The flag argument, used to control how the database is opened in the
- other DBM implementations, is ignored in the dbm.dumb module; the
- database is always opened for update, and will be created if it does
- not exist.
+ other DBM implementations, supports only the semantics of 'c' and 'n'
+ values. Other values will default to the semantics of 'c' value:
+ the database will always opened for update and will be created if it
+ does not exist.
The optional mode argument is the UNIX mode of the file, used only when
the database has to be created. It defaults to octal code 0o666 (and
will be modified by the prevailing umask).
"""
- # flag argument is currently ignored
# Modify mode depending on the umask
try:
@@ -290,5 +300,4 @@ def open(file, flag=None, mode=0o666):
else:
# Turn off any bits that are set in the umask
mode = mode & (~um)
-
- return _Database(file, mode)
+ return _Database(file, mode, flag=flag)
diff --git a/Lib/decimal.py b/Lib/decimal.py
index 324e4f9..7746ea2 100644
--- a/Lib/decimal.py
+++ b/Lib/decimal.py
@@ -1,6413 +1,11 @@
-# Copyright (c) 2004 Python Software Foundation.
-# All rights reserved.
-
-# Written by Eric Price <eprice at tjhsst.edu>
-# and Facundo Batista <facundo at taniquetil.com.ar>
-# and Raymond Hettinger <python at rcn.com>
-# and Aahz <aahz at pobox.com>
-# and Tim Peters
-
-# This module should be kept in sync with the latest updates of the
-# IBM specification as it evolves. Those updates will be treated
-# as bug fixes (deviation from the spec is a compatibility, usability
-# bug) and will be backported. At this point the spec is stabilizing
-# and the updates are becoming fewer, smaller, and less significant.
-
-"""
-This is an implementation of decimal floating point arithmetic based on
-the General Decimal Arithmetic Specification:
-
- http://speleotrove.com/decimal/decarith.html
-
-and IEEE standard 854-1987:
-
- http://en.wikipedia.org/wiki/IEEE_854-1987
-
-Decimal floating point has finite precision with arbitrarily large bounds.
-
-The purpose of this module is to support arithmetic using familiar
-"schoolhouse" rules and to avoid some of the tricky representation
-issues associated with binary floating point. The package is especially
-useful for financial applications or for contexts where users have
-expectations that are at odds with binary floating point (for instance,
-in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead
-of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected
-Decimal('0.00')).
-
-Here are some examples of using the decimal module:
-
->>> from decimal import *
->>> setcontext(ExtendedContext)
->>> Decimal(0)
-Decimal('0')
->>> Decimal('1')
-Decimal('1')
->>> Decimal('-.0123')
-Decimal('-0.0123')
->>> Decimal(123456)
-Decimal('123456')
->>> Decimal('123.45e12345678')
-Decimal('1.2345E+12345680')
->>> Decimal('1.33') + Decimal('1.27')
-Decimal('2.60')
->>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41')
-Decimal('-2.20')
->>> dig = Decimal(1)
->>> print(dig / Decimal(3))
-0.333333333
->>> getcontext().prec = 18
->>> print(dig / Decimal(3))
-0.333333333333333333
->>> print(dig.sqrt())
-1
->>> print(Decimal(3).sqrt())
-1.73205080756887729
->>> print(Decimal(3) ** 123)
-4.85192780976896427E+58
->>> inf = Decimal(1) / Decimal(0)
->>> print(inf)
-Infinity
->>> neginf = Decimal(-1) / Decimal(0)
->>> print(neginf)
--Infinity
->>> print(neginf + inf)
-NaN
->>> print(neginf * inf)
--Infinity
->>> print(dig / 0)
-Infinity
->>> getcontext().traps[DivisionByZero] = 1
->>> print(dig / 0)
-Traceback (most recent call last):
- ...
- ...
- ...
-decimal.DivisionByZero: x / 0
->>> c = Context()
->>> c.traps[InvalidOperation] = 0
->>> print(c.flags[InvalidOperation])
-0
->>> c.divide(Decimal(0), Decimal(0))
-Decimal('NaN')
->>> c.traps[InvalidOperation] = 1
->>> print(c.flags[InvalidOperation])
-1
->>> c.flags[InvalidOperation] = 0
->>> print(c.flags[InvalidOperation])
-0
->>> print(c.divide(Decimal(0), Decimal(0)))
-Traceback (most recent call last):
- ...
- ...
- ...
-decimal.InvalidOperation: 0 / 0
->>> print(c.flags[InvalidOperation])
-1
->>> c.flags[InvalidOperation] = 0
->>> c.traps[InvalidOperation] = 0
->>> print(c.divide(Decimal(0), Decimal(0)))
-NaN
->>> print(c.flags[InvalidOperation])
-1
->>>
-"""
-
-__all__ = [
- # Two major classes
- 'Decimal', 'Context',
-
- # Named tuple representation
- 'DecimalTuple',
-
- # Contexts
- 'DefaultContext', 'BasicContext', 'ExtendedContext',
-
- # Exceptions
- 'DecimalException', 'Clamped', 'InvalidOperation', 'DivisionByZero',
- 'Inexact', 'Rounded', 'Subnormal', 'Overflow', 'Underflow',
- 'FloatOperation',
-
- # Exceptional conditions that trigger InvalidOperation
- 'DivisionImpossible', 'InvalidContext', 'ConversionSyntax', 'DivisionUndefined',
-
- # Constants for use in setting up contexts
- 'ROUND_DOWN', 'ROUND_HALF_UP', 'ROUND_HALF_EVEN', 'ROUND_CEILING',
- 'ROUND_FLOOR', 'ROUND_UP', 'ROUND_HALF_DOWN', 'ROUND_05UP',
-
- # Functions for manipulating contexts
- 'setcontext', 'getcontext', 'localcontext',
-
- # Limits for the C version for compatibility
- 'MAX_PREC', 'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY',
-
- # C version: compile time choice that enables the thread local context
- 'HAVE_THREADS'
-]
-
-__version__ = '1.70' # Highest version of the spec this complies with
- # See http://speleotrove.com/decimal/
-__libmpdec_version__ = "2.4.1" # compatible libmpdec version
-
-import math as _math
-import numbers as _numbers
-import sys
-
-try:
- from collections import namedtuple as _namedtuple
- DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent')
-except ImportError:
- DecimalTuple = lambda *args: args
-
-# Rounding
-ROUND_DOWN = 'ROUND_DOWN'
-ROUND_HALF_UP = 'ROUND_HALF_UP'
-ROUND_HALF_EVEN = 'ROUND_HALF_EVEN'
-ROUND_CEILING = 'ROUND_CEILING'
-ROUND_FLOOR = 'ROUND_FLOOR'
-ROUND_UP = 'ROUND_UP'
-ROUND_HALF_DOWN = 'ROUND_HALF_DOWN'
-ROUND_05UP = 'ROUND_05UP'
-
-# Compatibility with the C version
-HAVE_THREADS = True
-if sys.maxsize == 2**63-1:
- MAX_PREC = 999999999999999999
- MAX_EMAX = 999999999999999999
- MIN_EMIN = -999999999999999999
-else:
- MAX_PREC = 425000000
- MAX_EMAX = 425000000
- MIN_EMIN = -425000000
-
-MIN_ETINY = MIN_EMIN - (MAX_PREC-1)
-
-# Errors
-
-class DecimalException(ArithmeticError):
- """Base exception class.
-
- Used exceptions derive from this.
- If an exception derives from another exception besides this (such as
- Underflow (Inexact, Rounded, Subnormal) that indicates that it is only
- called if the others are present. This isn't actually used for
- anything, though.
-
- handle -- Called when context._raise_error is called and the
- trap_enabler is not set. First argument is self, second is the
- context. More arguments can be given, those being after
- the explanation in _raise_error (For example,
- context._raise_error(NewError, '(-x)!', self._sign) would
- call NewError().handle(context, self._sign).)
-
- To define a new exception, it should be sufficient to have it derive
- from DecimalException.
- """
- def handle(self, context, *args):
- pass
-
-
-class Clamped(DecimalException):
- """Exponent of a 0 changed to fit bounds.
-
- This occurs and signals clamped if the exponent of a result has been
- altered in order to fit the constraints of a specific concrete
- representation. This may occur when the exponent of a zero result would
- be outside the bounds of a representation, or when a large normal
- number would have an encoded exponent that cannot be represented. In
- this latter case, the exponent is reduced to fit and the corresponding
- number of zero digits are appended to the coefficient ("fold-down").
- """
-
-class InvalidOperation(DecimalException):
- """An invalid operation was performed.
-
- Various bad things cause this:
-
- Something creates a signaling NaN
- -INF + INF
- 0 * (+-)INF
- (+-)INF / (+-)INF
- x % 0
- (+-)INF % x
- x._rescale( non-integer )
- sqrt(-x) , x > 0
- 0 ** 0
- x ** (non-integer)
- x ** (+-)INF
- An operand is invalid
-
- The result of the operation after these is a quiet positive NaN,
- except when the cause is a signaling NaN, in which case the result is
- also a quiet NaN, but with the original sign, and an optional
- diagnostic information.
- """
- def handle(self, context, *args):
- if args:
- ans = _dec_from_triple(args[0]._sign, args[0]._int, 'n', True)
- return ans._fix_nan(context)
- return _NaN
-
-class ConversionSyntax(InvalidOperation):
- """Trying to convert badly formed string.
-
- This occurs and signals invalid-operation if an string is being
- converted to a number and it does not conform to the numeric string
- syntax. The result is [0,qNaN].
- """
- def handle(self, context, *args):
- return _NaN
-
-class DivisionByZero(DecimalException, ZeroDivisionError):
- """Division by 0.
-
- This occurs and signals division-by-zero if division of a finite number
- by zero was attempted (during a divide-integer or divide operation, or a
- power operation with negative right-hand operand), and the dividend was
- not zero.
-
- The result of the operation is [sign,inf], where sign is the exclusive
- or of the signs of the operands for divide, or is 1 for an odd power of
- -0, for power.
- """
-
- def handle(self, context, sign, *args):
- return _SignedInfinity[sign]
-
-class DivisionImpossible(InvalidOperation):
- """Cannot perform the division adequately.
-
- This occurs and signals invalid-operation if the integer result of a
- divide-integer or remainder operation had too many digits (would be
- longer than precision). The result is [0,qNaN].
- """
-
- def handle(self, context, *args):
- return _NaN
-
-class DivisionUndefined(InvalidOperation, ZeroDivisionError):
- """Undefined result of division.
-
- This occurs and signals invalid-operation if division by zero was
- attempted (during a divide-integer, divide, or remainder operation), and
- the dividend is also zero. The result is [0,qNaN].
- """
-
- def handle(self, context, *args):
- return _NaN
-
-class Inexact(DecimalException):
- """Had to round, losing information.
-
- This occurs and signals inexact whenever the result of an operation is
- not exact (that is, it needed to be rounded and any discarded digits
- were non-zero), or if an overflow or underflow condition occurs. The
- result in all cases is unchanged.
-
- The inexact signal may be tested (or trapped) to determine if a given
- operation (or sequence of operations) was inexact.
- """
-
-class InvalidContext(InvalidOperation):
- """Invalid context. Unknown rounding, for example.
-
- This occurs and signals invalid-operation if an invalid context was
- detected during an operation. This can occur if contexts are not checked
- on creation and either the precision exceeds the capability of the
- underlying concrete representation or an unknown or unsupported rounding
- was specified. These aspects of the context need only be checked when
- the values are required to be used. The result is [0,qNaN].
- """
-
- def handle(self, context, *args):
- return _NaN
-
-class Rounded(DecimalException):
- """Number got rounded (not necessarily changed during rounding).
-
- This occurs and signals rounded whenever the result of an operation is
- rounded (that is, some zero or non-zero digits were discarded from the
- coefficient), or if an overflow or underflow condition occurs. The
- result in all cases is unchanged.
-
- The rounded signal may be tested (or trapped) to determine if a given
- operation (or sequence of operations) caused a loss of precision.
- """
-
-class Subnormal(DecimalException):
- """Exponent < Emin before rounding.
-
- This occurs and signals subnormal whenever the result of a conversion or
- operation is subnormal (that is, its adjusted exponent is less than
- Emin, before any rounding). The result in all cases is unchanged.
-
- The subnormal signal may be tested (or trapped) to determine if a given
- or operation (or sequence of operations) yielded a subnormal result.
- """
-
-class Overflow(Inexact, Rounded):
- """Numerical overflow.
-
- This occurs and signals overflow if the adjusted exponent of a result
- (from a conversion or from an operation that is not an attempt to divide
- by zero), after rounding, would be greater than the largest value that
- can be handled by the implementation (the value Emax).
-
- The result depends on the rounding mode:
-
- For round-half-up and round-half-even (and for round-half-down and
- round-up, if implemented), the result of the operation is [sign,inf],
- where sign is the sign of the intermediate result. For round-down, the
- result is the largest finite number that can be represented in the
- current precision, with the sign of the intermediate result. For
- round-ceiling, the result is the same as for round-down if the sign of
- the intermediate result is 1, or is [0,inf] otherwise. For round-floor,
- the result is the same as for round-down if the sign of the intermediate
- result is 0, or is [1,inf] otherwise. In all cases, Inexact and Rounded
- will also be raised.
- """
-
- def handle(self, context, sign, *args):
- if context.rounding in (ROUND_HALF_UP, ROUND_HALF_EVEN,
- ROUND_HALF_DOWN, ROUND_UP):
- return _SignedInfinity[sign]
- if sign == 0:
- if context.rounding == ROUND_CEILING:
- return _SignedInfinity[sign]
- return _dec_from_triple(sign, '9'*context.prec,
- context.Emax-context.prec+1)
- if sign == 1:
- if context.rounding == ROUND_FLOOR:
- return _SignedInfinity[sign]
- return _dec_from_triple(sign, '9'*context.prec,
- context.Emax-context.prec+1)
-
-
-class Underflow(Inexact, Rounded, Subnormal):
- """Numerical underflow with result rounded to 0.
-
- This occurs and signals underflow if a result is inexact and the
- adjusted exponent of the result would be smaller (more negative) than
- the smallest value that can be handled by the implementation (the value
- Emin). That is, the result is both inexact and subnormal.
-
- The result after an underflow will be a subnormal number rounded, if
- necessary, so that its exponent is not less than Etiny. This may result
- in 0 with the sign of the intermediate result and an exponent of Etiny.
-
- In all cases, Inexact, Rounded, and Subnormal will also be raised.
- """
-
-class FloatOperation(DecimalException, TypeError):
- """Enable stricter semantics for mixing floats and Decimals.
-
- If the signal is not trapped (default), mixing floats and Decimals is
- permitted in the Decimal() constructor, context.create_decimal() and
- all comparison operators. Both conversion and comparisons are exact.
- Any occurrence of a mixed operation is silently recorded by setting
- FloatOperation in the context flags. Explicit conversions with
- Decimal.from_float() or context.create_decimal_from_float() do not
- set the flag.
-
- Otherwise (the signal is trapped), only equality comparisons and explicit
- conversions are silent. All other mixed operations raise FloatOperation.
- """
-
-# List of public traps and flags
-_signals = [Clamped, DivisionByZero, Inexact, Overflow, Rounded,
- Underflow, InvalidOperation, Subnormal, FloatOperation]
-
-# Map conditions (per the spec) to signals
-_condition_map = {ConversionSyntax:InvalidOperation,
- DivisionImpossible:InvalidOperation,
- DivisionUndefined:InvalidOperation,
- InvalidContext:InvalidOperation}
-
-# Valid rounding modes
-_rounding_modes = (ROUND_DOWN, ROUND_HALF_UP, ROUND_HALF_EVEN, ROUND_CEILING,
- ROUND_FLOOR, ROUND_UP, ROUND_HALF_DOWN, ROUND_05UP)
-
-##### Context Functions ##################################################
-
-# The getcontext() and setcontext() function manage access to a thread-local
-# current context. Py2.4 offers direct support for thread locals. If that
-# is not available, use threading.current_thread() which is slower but will
-# work for older Pythons. If threads are not part of the build, create a
-# mock threading object with threading.local() returning the module namespace.
-
-try:
- import threading
-except ImportError:
- # Python was compiled without threads; create a mock object instead
- class MockThreading(object):
- def local(self, sys=sys):
- return sys.modules[__name__]
- threading = MockThreading()
- del MockThreading
-
-try:
- threading.local
-
-except AttributeError:
-
- # To fix reloading, force it to create a new context
- # Old contexts have different exceptions in their dicts, making problems.
- if hasattr(threading.current_thread(), '__decimal_context__'):
- del threading.current_thread().__decimal_context__
-
- def setcontext(context):
- """Set this thread's context to context."""
- if context in (DefaultContext, BasicContext, ExtendedContext):
- context = context.copy()
- context.clear_flags()
- threading.current_thread().__decimal_context__ = context
-
- def getcontext():
- """Returns this thread's context.
-
- If this thread does not yet have a context, returns
- a new context and sets this thread's context.
- New contexts are copies of DefaultContext.
- """
- try:
- return threading.current_thread().__decimal_context__
- except AttributeError:
- context = Context()
- threading.current_thread().__decimal_context__ = context
- return context
-
-else:
-
- local = threading.local()
- if hasattr(local, '__decimal_context__'):
- del local.__decimal_context__
-
- def getcontext(_local=local):
- """Returns this thread's context.
-
- If this thread does not yet have a context, returns
- a new context and sets this thread's context.
- New contexts are copies of DefaultContext.
- """
- try:
- return _local.__decimal_context__
- except AttributeError:
- context = Context()
- _local.__decimal_context__ = context
- return context
-
- def setcontext(context, _local=local):
- """Set this thread's context to context."""
- if context in (DefaultContext, BasicContext, ExtendedContext):
- context = context.copy()
- context.clear_flags()
- _local.__decimal_context__ = context
-
- del threading, local # Don't contaminate the namespace
-
-def localcontext(ctx=None):
- """Return a context manager for a copy of the supplied context
-
- Uses a copy of the current context if no context is specified
- The returned context manager creates a local decimal context
- in a with statement:
- def sin(x):
- with localcontext() as ctx:
- ctx.prec += 2
- # Rest of sin calculation algorithm
- # uses a precision 2 greater than normal
- return +s # Convert result to normal precision
-
- def sin(x):
- with localcontext(ExtendedContext):
- # Rest of sin calculation algorithm
- # uses the Extended Context from the
- # General Decimal Arithmetic Specification
- return +s # Convert result to normal context
-
- >>> setcontext(DefaultContext)
- >>> print(getcontext().prec)
- 28
- >>> with localcontext():
- ... ctx = getcontext()
- ... ctx.prec += 2
- ... print(ctx.prec)
- ...
- 30
- >>> with localcontext(ExtendedContext):
- ... print(getcontext().prec)
- ...
- 9
- >>> print(getcontext().prec)
- 28
- """
- if ctx is None: ctx = getcontext()
- return _ContextManager(ctx)
-
-
-##### Decimal class #######################################################
-
-# Do not subclass Decimal from numbers.Real and do not register it as such
-# (because Decimals are not interoperable with floats). See the notes in
-# numbers.py for more detail.
-
-class Decimal(object):
- """Floating point class for decimal arithmetic."""
-
- __slots__ = ('_exp','_int','_sign', '_is_special')
- # Generally, the value of the Decimal instance is given by
- # (-1)**_sign * _int * 10**_exp
- # Special values are signified by _is_special == True
-
- # We're immutable, so use __new__ not __init__
- def __new__(cls, value="0", context=None):
- """Create a decimal point instance.
-
- >>> Decimal('3.14') # string input
- Decimal('3.14')
- >>> Decimal((0, (3, 1, 4), -2)) # tuple (sign, digit_tuple, exponent)
- Decimal('3.14')
- >>> Decimal(314) # int
- Decimal('314')
- >>> Decimal(Decimal(314)) # another decimal instance
- Decimal('314')
- >>> Decimal(' 3.14 \\n') # leading and trailing whitespace okay
- Decimal('3.14')
- """
-
- # Note that the coefficient, self._int, is actually stored as
- # a string rather than as a tuple of digits. This speeds up
- # the "digits to integer" and "integer to digits" conversions
- # that are used in almost every arithmetic operation on
- # Decimals. This is an internal detail: the as_tuple function
- # and the Decimal constructor still deal with tuples of
- # digits.
-
- self = object.__new__(cls)
-
- # From a string
- # REs insist on real strings, so we can too.
- if isinstance(value, str):
- m = _parser(value.strip())
- if m is None:
- if context is None:
- context = getcontext()
- return context._raise_error(ConversionSyntax,
- "Invalid literal for Decimal: %r" % value)
-
- if m.group('sign') == "-":
- self._sign = 1
- else:
- self._sign = 0
- intpart = m.group('int')
- if intpart is not None:
- # finite number
- fracpart = m.group('frac') or ''
- exp = int(m.group('exp') or '0')
- self._int = str(int(intpart+fracpart))
- self._exp = exp - len(fracpart)
- self._is_special = False
- else:
- diag = m.group('diag')
- if diag is not None:
- # NaN
- self._int = str(int(diag or '0')).lstrip('0')
- if m.group('signal'):
- self._exp = 'N'
- else:
- self._exp = 'n'
- else:
- # infinity
- self._int = '0'
- self._exp = 'F'
- self._is_special = True
- return self
-
- # From an integer
- if isinstance(value, int):
- if value >= 0:
- self._sign = 0
- else:
- self._sign = 1
- self._exp = 0
- self._int = str(abs(value))
- self._is_special = False
- return self
-
- # From another decimal
- if isinstance(value, Decimal):
- self._exp = value._exp
- self._sign = value._sign
- self._int = value._int
- self._is_special = value._is_special
- return self
-
- # From an internal working value
- if isinstance(value, _WorkRep):
- self._sign = value.sign
- self._int = str(value.int)
- self._exp = int(value.exp)
- self._is_special = False
- return self
-
- # tuple/list conversion (possibly from as_tuple())
- if isinstance(value, (list,tuple)):
- if len(value) != 3:
- raise ValueError('Invalid tuple size in creation of Decimal '
- 'from list or tuple. The list or tuple '
- 'should have exactly three elements.')
- # process sign. The isinstance test rejects floats
- if not (isinstance(value[0], int) and value[0] in (0,1)):
- raise ValueError("Invalid sign. The first value in the tuple "
- "should be an integer; either 0 for a "
- "positive number or 1 for a negative number.")
- self._sign = value[0]
- if value[2] == 'F':
- # infinity: value[1] is ignored
- self._int = '0'
- self._exp = value[2]
- self._is_special = True
- else:
- # process and validate the digits in value[1]
- digits = []
- for digit in value[1]:
- if isinstance(digit, int) and 0 <= digit <= 9:
- # skip leading zeros
- if digits or digit != 0:
- digits.append(digit)
- else:
- raise ValueError("The second value in the tuple must "
- "be composed of integers in the range "
- "0 through 9.")
- if value[2] in ('n', 'N'):
- # NaN: digits form the diagnostic
- self._int = ''.join(map(str, digits))
- self._exp = value[2]
- self._is_special = True
- elif isinstance(value[2], int):
- # finite number: digits give the coefficient
- self._int = ''.join(map(str, digits or [0]))
- self._exp = value[2]
- self._is_special = False
- else:
- raise ValueError("The third value in the tuple must "
- "be an integer, or one of the "
- "strings 'F', 'n', 'N'.")
- return self
-
- if isinstance(value, float):
- if context is None:
- context = getcontext()
- context._raise_error(FloatOperation,
- "strict semantics for mixing floats and Decimals are "
- "enabled")
- value = Decimal.from_float(value)
- self._exp = value._exp
- self._sign = value._sign
- self._int = value._int
- self._is_special = value._is_special
- return self
-
- raise TypeError("Cannot convert %r to Decimal" % value)
-
- @classmethod
- def from_float(cls, f):
- """Converts a float to a decimal number, exactly.
-
- Note that Decimal.from_float(0.1) is not the same as Decimal('0.1').
- Since 0.1 is not exactly representable in binary floating point, the
- value is stored as the nearest representable value which is
- 0x1.999999999999ap-4. The exact equivalent of the value in decimal
- is 0.1000000000000000055511151231257827021181583404541015625.
-
- >>> Decimal.from_float(0.1)
- Decimal('0.1000000000000000055511151231257827021181583404541015625')
- >>> Decimal.from_float(float('nan'))
- Decimal('NaN')
- >>> Decimal.from_float(float('inf'))
- Decimal('Infinity')
- >>> Decimal.from_float(-float('inf'))
- Decimal('-Infinity')
- >>> Decimal.from_float(-0.0)
- Decimal('-0')
-
- """
- if isinstance(f, int): # handle integer inputs
- return cls(f)
- if not isinstance(f, float):
- raise TypeError("argument must be int or float.")
- if _math.isinf(f) or _math.isnan(f):
- return cls(repr(f))
- if _math.copysign(1.0, f) == 1.0:
- sign = 0
- else:
- sign = 1
- n, d = abs(f).as_integer_ratio()
- k = d.bit_length() - 1
- result = _dec_from_triple(sign, str(n*5**k), -k)
- if cls is Decimal:
- return result
- else:
- return cls(result)
-
- def _isnan(self):
- """Returns whether the number is not actually one.
-
- 0 if a number
- 1 if NaN
- 2 if sNaN
- """
- if self._is_special:
- exp = self._exp
- if exp == 'n':
- return 1
- elif exp == 'N':
- return 2
- return 0
-
- def _isinfinity(self):
- """Returns whether the number is infinite
-
- 0 if finite or not a number
- 1 if +INF
- -1 if -INF
- """
- if self._exp == 'F':
- if self._sign:
- return -1
- return 1
- return 0
-
- def _check_nans(self, other=None, context=None):
- """Returns whether the number is not actually one.
-
- if self, other are sNaN, signal
- if self, other are NaN return nan
- return 0
-
- Done before operations.
- """
-
- self_is_nan = self._isnan()
- if other is None:
- other_is_nan = False
- else:
- other_is_nan = other._isnan()
-
- if self_is_nan or other_is_nan:
- if context is None:
- context = getcontext()
-
- if self_is_nan == 2:
- return context._raise_error(InvalidOperation, 'sNaN',
- self)
- if other_is_nan == 2:
- return context._raise_error(InvalidOperation, 'sNaN',
- other)
- if self_is_nan:
- return self._fix_nan(context)
-
- return other._fix_nan(context)
- return 0
-
- def _compare_check_nans(self, other, context):
- """Version of _check_nans used for the signaling comparisons
- compare_signal, __le__, __lt__, __ge__, __gt__.
-
- Signal InvalidOperation if either self or other is a (quiet
- or signaling) NaN. Signaling NaNs take precedence over quiet
- NaNs.
-
- Return 0 if neither operand is a NaN.
-
- """
- if context is None:
- context = getcontext()
-
- if self._is_special or other._is_special:
- if self.is_snan():
- return context._raise_error(InvalidOperation,
- 'comparison involving sNaN',
- self)
- elif other.is_snan():
- return context._raise_error(InvalidOperation,
- 'comparison involving sNaN',
- other)
- elif self.is_qnan():
- return context._raise_error(InvalidOperation,
- 'comparison involving NaN',
- self)
- elif other.is_qnan():
- return context._raise_error(InvalidOperation,
- 'comparison involving NaN',
- other)
- return 0
-
- def __bool__(self):
- """Return True if self is nonzero; otherwise return False.
-
- NaNs and infinities are considered nonzero.
- """
- return self._is_special or self._int != '0'
-
- def _cmp(self, other):
- """Compare the two non-NaN decimal instances self and other.
-
- Returns -1 if self < other, 0 if self == other and 1
- if self > other. This routine is for internal use only."""
-
- if self._is_special or other._is_special:
- self_inf = self._isinfinity()
- other_inf = other._isinfinity()
- if self_inf == other_inf:
- return 0
- elif self_inf < other_inf:
- return -1
- else:
- return 1
-
- # check for zeros; Decimal('0') == Decimal('-0')
- if not self:
- if not other:
- return 0
- else:
- return -((-1)**other._sign)
- if not other:
- return (-1)**self._sign
-
- # If different signs, neg one is less
- if other._sign < self._sign:
- return -1
- if self._sign < other._sign:
- return 1
-
- self_adjusted = self.adjusted()
- other_adjusted = other.adjusted()
- if self_adjusted == other_adjusted:
- self_padded = self._int + '0'*(self._exp - other._exp)
- other_padded = other._int + '0'*(other._exp - self._exp)
- if self_padded == other_padded:
- return 0
- elif self_padded < other_padded:
- return -(-1)**self._sign
- else:
- return (-1)**self._sign
- elif self_adjusted > other_adjusted:
- return (-1)**self._sign
- else: # self_adjusted < other_adjusted
- return -((-1)**self._sign)
-
- # Note: The Decimal standard doesn't cover rich comparisons for
- # Decimals. In particular, the specification is silent on the
- # subject of what should happen for a comparison involving a NaN.
- # We take the following approach:
- #
- # == comparisons involving a quiet NaN always return False
- # != comparisons involving a quiet NaN always return True
- # == or != comparisons involving a signaling NaN signal
- # InvalidOperation, and return False or True as above if the
- # InvalidOperation is not trapped.
- # <, >, <= and >= comparisons involving a (quiet or signaling)
- # NaN signal InvalidOperation, and return False if the
- # InvalidOperation is not trapped.
- #
- # This behavior is designed to conform as closely as possible to
- # that specified by IEEE 754.
-
- def __eq__(self, other, context=None):
- self, other = _convert_for_comparison(self, other, equality_op=True)
- if other is NotImplemented:
- return other
- if self._check_nans(other, context):
- return False
- return self._cmp(other) == 0
-
- def __ne__(self, other, context=None):
- self, other = _convert_for_comparison(self, other, equality_op=True)
- if other is NotImplemented:
- return other
- if self._check_nans(other, context):
- return True
- return self._cmp(other) != 0
-
-
- def __lt__(self, other, context=None):
- self, other = _convert_for_comparison(self, other)
- if other is NotImplemented:
- return other
- ans = self._compare_check_nans(other, context)
- if ans:
- return False
- return self._cmp(other) < 0
-
- def __le__(self, other, context=None):
- self, other = _convert_for_comparison(self, other)
- if other is NotImplemented:
- return other
- ans = self._compare_check_nans(other, context)
- if ans:
- return False
- return self._cmp(other) <= 0
-
- def __gt__(self, other, context=None):
- self, other = _convert_for_comparison(self, other)
- if other is NotImplemented:
- return other
- ans = self._compare_check_nans(other, context)
- if ans:
- return False
- return self._cmp(other) > 0
-
- def __ge__(self, other, context=None):
- self, other = _convert_for_comparison(self, other)
- if other is NotImplemented:
- return other
- ans = self._compare_check_nans(other, context)
- if ans:
- return False
- return self._cmp(other) >= 0
-
- def compare(self, other, context=None):
- """Compare self to other. Return a decimal value:
-
- a or b is a NaN ==> Decimal('NaN')
- a < b ==> Decimal('-1')
- a == b ==> Decimal('0')
- a > b ==> Decimal('1')
- """
- other = _convert_other(other, raiseit=True)
-
- # Compare(NaN, NaN) = NaN
- if (self._is_special or other and other._is_special):
- ans = self._check_nans(other, context)
- if ans:
- return ans
-
- return Decimal(self._cmp(other))
-
- def __hash__(self):
- """x.__hash__() <==> hash(x)"""
-
- # In order to make sure that the hash of a Decimal instance
- # agrees with the hash of a numerically equal integer, float
- # or Fraction, we follow the rules for numeric hashes outlined
- # in the documentation. (See library docs, 'Built-in Types').
- if self._is_special:
- if self.is_snan():
- raise TypeError('Cannot hash a signaling NaN value.')
- elif self.is_nan():
- return _PyHASH_NAN
- else:
- if self._sign:
- return -_PyHASH_INF
- else:
- return _PyHASH_INF
-
- if self._exp >= 0:
- exp_hash = pow(10, self._exp, _PyHASH_MODULUS)
- else:
- exp_hash = pow(_PyHASH_10INV, -self._exp, _PyHASH_MODULUS)
- hash_ = int(self._int) * exp_hash % _PyHASH_MODULUS
- ans = hash_ if self >= 0 else -hash_
- return -2 if ans == -1 else ans
-
- def as_tuple(self):
- """Represents the number as a triple tuple.
-
- To show the internals exactly as they are.
- """
- return DecimalTuple(self._sign, tuple(map(int, self._int)), self._exp)
-
- def __repr__(self):
- """Represents the number as an instance of Decimal."""
- # Invariant: eval(repr(d)) == d
- return "Decimal('%s')" % str(self)
-
- def __str__(self, eng=False, context=None):
- """Return string representation of the number in scientific notation.
-
- Captures all of the information in the underlying representation.
- """
-
- sign = ['', '-'][self._sign]
- if self._is_special:
- if self._exp == 'F':
- return sign + 'Infinity'
- elif self._exp == 'n':
- return sign + 'NaN' + self._int
- else: # self._exp == 'N'
- return sign + 'sNaN' + self._int
-
- # number of digits of self._int to left of decimal point
- leftdigits = self._exp + len(self._int)
-
- # dotplace is number of digits of self._int to the left of the
- # decimal point in the mantissa of the output string (that is,
- # after adjusting the exponent)
- if self._exp <= 0 and leftdigits > -6:
- # no exponent required
- dotplace = leftdigits
- elif not eng:
- # usual scientific notation: 1 digit on left of the point
- dotplace = 1
- elif self._int == '0':
- # engineering notation, zero
- dotplace = (leftdigits + 1) % 3 - 1
- else:
- # engineering notation, nonzero
- dotplace = (leftdigits - 1) % 3 + 1
-
- if dotplace <= 0:
- intpart = '0'
- fracpart = '.' + '0'*(-dotplace) + self._int
- elif dotplace >= len(self._int):
- intpart = self._int+'0'*(dotplace-len(self._int))
- fracpart = ''
- else:
- intpart = self._int[:dotplace]
- fracpart = '.' + self._int[dotplace:]
- if leftdigits == dotplace:
- exp = ''
- else:
- if context is None:
- context = getcontext()
- exp = ['e', 'E'][context.capitals] + "%+d" % (leftdigits-dotplace)
-
- return sign + intpart + fracpart + exp
-
- def to_eng_string(self, context=None):
- """Convert to engineering-type string.
-
- Engineering notation has an exponent which is a multiple of 3, so there
- are up to 3 digits left of the decimal place.
-
- Same rules for when in exponential and when as a value as in __str__.
- """
- return self.__str__(eng=True, context=context)
-
- def __neg__(self, context=None):
- """Returns a copy with the sign switched.
-
- Rounds, if it has reason.
- """
- if self._is_special:
- ans = self._check_nans(context=context)
- if ans:
- return ans
-
- if context is None:
- context = getcontext()
-
- if not self and context.rounding != ROUND_FLOOR:
- # -Decimal('0') is Decimal('0'), not Decimal('-0'), except
- # in ROUND_FLOOR rounding mode.
- ans = self.copy_abs()
- else:
- ans = self.copy_negate()
-
- return ans._fix(context)
-
- def __pos__(self, context=None):
- """Returns a copy, unless it is a sNaN.
-
- Rounds the number (if more then precision digits)
- """
- if self._is_special:
- ans = self._check_nans(context=context)
- if ans:
- return ans
-
- if context is None:
- context = getcontext()
-
- if not self and context.rounding != ROUND_FLOOR:
- # + (-0) = 0, except in ROUND_FLOOR rounding mode.
- ans = self.copy_abs()
- else:
- ans = Decimal(self)
-
- return ans._fix(context)
-
- def __abs__(self, round=True, context=None):
- """Returns the absolute value of self.
-
- If the keyword argument 'round' is false, do not round. The
- expression self.__abs__(round=False) is equivalent to
- self.copy_abs().
- """
- if not round:
- return self.copy_abs()
-
- if self._is_special:
- ans = self._check_nans(context=context)
- if ans:
- return ans
-
- if self._sign:
- ans = self.__neg__(context=context)
- else:
- ans = self.__pos__(context=context)
-
- return ans
-
- def __add__(self, other, context=None):
- """Returns self + other.
-
- -INF + INF (or the reverse) cause InvalidOperation errors.
- """
- other = _convert_other(other)
- if other is NotImplemented:
- return other
-
- if context is None:
- context = getcontext()
-
- if self._is_special or other._is_special:
- ans = self._check_nans(other, context)
- if ans:
- return ans
-
- if self._isinfinity():
- # If both INF, same sign => same as both, opposite => error.
- if self._sign != other._sign and other._isinfinity():
- return context._raise_error(InvalidOperation, '-INF + INF')
- return Decimal(self)
- if other._isinfinity():
- return Decimal(other) # Can't both be infinity here
-
- exp = min(self._exp, other._exp)
- negativezero = 0
- if context.rounding == ROUND_FLOOR and self._sign != other._sign:
- # If the answer is 0, the sign should be negative, in this case.
- negativezero = 1
-
- if not self and not other:
- sign = min(self._sign, other._sign)
- if negativezero:
- sign = 1
- ans = _dec_from_triple(sign, '0', exp)
- ans = ans._fix(context)
- return ans
- if not self:
- exp = max(exp, other._exp - context.prec-1)
- ans = other._rescale(exp, context.rounding)
- ans = ans._fix(context)
- return ans
- if not other:
- exp = max(exp, self._exp - context.prec-1)
- ans = self._rescale(exp, context.rounding)
- ans = ans._fix(context)
- return ans
-
- op1 = _WorkRep(self)
- op2 = _WorkRep(other)
- op1, op2 = _normalize(op1, op2, context.prec)
-
- result = _WorkRep()
- if op1.sign != op2.sign:
- # Equal and opposite
- if op1.int == op2.int:
- ans = _dec_from_triple(negativezero, '0', exp)
- ans = ans._fix(context)
- return ans
- if op1.int < op2.int:
- op1, op2 = op2, op1
- # OK, now abs(op1) > abs(op2)
- if op1.sign == 1:
- result.sign = 1
- op1.sign, op2.sign = op2.sign, op1.sign
- else:
- result.sign = 0
- # So we know the sign, and op1 > 0.
- elif op1.sign == 1:
- result.sign = 1
- op1.sign, op2.sign = (0, 0)
- else:
- result.sign = 0
- # Now, op1 > abs(op2) > 0
-
- if op2.sign == 0:
- result.int = op1.int + op2.int
- else:
- result.int = op1.int - op2.int
-
- result.exp = op1.exp
- ans = Decimal(result)
- ans = ans._fix(context)
- return ans
-
- __radd__ = __add__
-
- def __sub__(self, other, context=None):
- """Return self - other"""
- other = _convert_other(other)
- if other is NotImplemented:
- return other
-
- if self._is_special or other._is_special:
- ans = self._check_nans(other, context=context)
- if ans:
- return ans
-
- # self - other is computed as self + other.copy_negate()
- return self.__add__(other.copy_negate(), context=context)
-
- def __rsub__(self, other, context=None):
- """Return other - self"""
- other = _convert_other(other)
- if other is NotImplemented:
- return other
-
- return other.__sub__(self, context=context)
-
- def __mul__(self, other, context=None):
- """Return self * other.
-
- (+-) INF * 0 (or its reverse) raise InvalidOperation.
- """
- other = _convert_other(other)
- if other is NotImplemented:
- return other
-
- if context is None:
- context = getcontext()
-
- resultsign = self._sign ^ other._sign
-
- if self._is_special or other._is_special:
- ans = self._check_nans(other, context)
- if ans:
- return ans
-
- if self._isinfinity():
- if not other:
- return context._raise_error(InvalidOperation, '(+-)INF * 0')
- return _SignedInfinity[resultsign]
-
- if other._isinfinity():
- if not self:
- return context._raise_error(InvalidOperation, '0 * (+-)INF')
- return _SignedInfinity[resultsign]
-
- resultexp = self._exp + other._exp
-
- # Special case for multiplying by zero
- if not self or not other:
- ans = _dec_from_triple(resultsign, '0', resultexp)
- # Fixing in case the exponent is out of bounds
- ans = ans._fix(context)
- return ans
-
- # Special case for multiplying by power of 10
- if self._int == '1':
- ans = _dec_from_triple(resultsign, other._int, resultexp)
- ans = ans._fix(context)
- return ans
- if other._int == '1':
- ans = _dec_from_triple(resultsign, self._int, resultexp)
- ans = ans._fix(context)
- return ans
-
- op1 = _WorkRep(self)
- op2 = _WorkRep(other)
-
- ans = _dec_from_triple(resultsign, str(op1.int * op2.int), resultexp)
- ans = ans._fix(context)
-
- return ans
- __rmul__ = __mul__
-
- def __truediv__(self, other, context=None):
- """Return self / other."""
- other = _convert_other(other)
- if other is NotImplemented:
- return NotImplemented
-
- if context is None:
- context = getcontext()
-
- sign = self._sign ^ other._sign
-
- if self._is_special or other._is_special:
- ans = self._check_nans(other, context)
- if ans:
- return ans
-
- if self._isinfinity() and other._isinfinity():
- return context._raise_error(InvalidOperation, '(+-)INF/(+-)INF')
-
- if self._isinfinity():
- return _SignedInfinity[sign]
-
- if other._isinfinity():
- context._raise_error(Clamped, 'Division by infinity')
- return _dec_from_triple(sign, '0', context.Etiny())
-
- # Special cases for zeroes
- if not other:
- if not self:
- return context._raise_error(DivisionUndefined, '0 / 0')
- return context._raise_error(DivisionByZero, 'x / 0', sign)
-
- if not self:
- exp = self._exp - other._exp
- coeff = 0
- else:
- # OK, so neither = 0, INF or NaN
- shift = len(other._int) - len(self._int) + context.prec + 1
- exp = self._exp - other._exp - shift
- op1 = _WorkRep(self)
- op2 = _WorkRep(other)
- if shift >= 0:
- coeff, remainder = divmod(op1.int * 10**shift, op2.int)
- else:
- coeff, remainder = divmod(op1.int, op2.int * 10**-shift)
- if remainder:
- # result is not exact; adjust to ensure correct rounding
- if coeff % 5 == 0:
- coeff += 1
- else:
- # result is exact; get as close to ideal exponent as possible
- ideal_exp = self._exp - other._exp
- while exp < ideal_exp and coeff % 10 == 0:
- coeff //= 10
- exp += 1
-
- ans = _dec_from_triple(sign, str(coeff), exp)
- return ans._fix(context)
-
- def _divide(self, other, context):
- """Return (self // other, self % other), to context.prec precision.
-
- Assumes that neither self nor other is a NaN, that self is not
- infinite and that other is nonzero.
- """
- sign = self._sign ^ other._sign
- if other._isinfinity():
- ideal_exp = self._exp
- else:
- ideal_exp = min(self._exp, other._exp)
-
- expdiff = self.adjusted() - other.adjusted()
- if not self or other._isinfinity() or expdiff <= -2:
- return (_dec_from_triple(sign, '0', 0),
- self._rescale(ideal_exp, context.rounding))
- if expdiff <= context.prec:
- op1 = _WorkRep(self)
- op2 = _WorkRep(other)
- if op1.exp >= op2.exp:
- op1.int *= 10**(op1.exp - op2.exp)
- else:
- op2.int *= 10**(op2.exp - op1.exp)
- q, r = divmod(op1.int, op2.int)
- if q < 10**context.prec:
- return (_dec_from_triple(sign, str(q), 0),
- _dec_from_triple(self._sign, str(r), ideal_exp))
-
- # Here the quotient is too large to be representable
- ans = context._raise_error(DivisionImpossible,
- 'quotient too large in //, % or divmod')
- return ans, ans
-
- def __rtruediv__(self, other, context=None):
- """Swaps self/other and returns __truediv__."""
- other = _convert_other(other)
- if other is NotImplemented:
- return other
- return other.__truediv__(self, context=context)
-
- def __divmod__(self, other, context=None):
- """
- Return (self // other, self % other)
- """
- other = _convert_other(other)
- if other is NotImplemented:
- return other
-
- if context is None:
- context = getcontext()
-
- ans = self._check_nans(other, context)
- if ans:
- return (ans, ans)
-
- sign = self._sign ^ other._sign
- if self._isinfinity():
- if other._isinfinity():
- ans = context._raise_error(InvalidOperation, 'divmod(INF, INF)')
- return ans, ans
- else:
- return (_SignedInfinity[sign],
- context._raise_error(InvalidOperation, 'INF % x'))
-
- if not other:
- if not self:
- ans = context._raise_error(DivisionUndefined, 'divmod(0, 0)')
- return ans, ans
- else:
- return (context._raise_error(DivisionByZero, 'x // 0', sign),
- context._raise_error(InvalidOperation, 'x % 0'))
-
- quotient, remainder = self._divide(other, context)
- remainder = remainder._fix(context)
- return quotient, remainder
-
- def __rdivmod__(self, other, context=None):
- """Swaps self/other and returns __divmod__."""
- other = _convert_other(other)
- if other is NotImplemented:
- return other
- return other.__divmod__(self, context=context)
-
- def __mod__(self, other, context=None):
- """
- self % other
- """
- other = _convert_other(other)
- if other is NotImplemented:
- return other
-
- if context is None:
- context = getcontext()
-
- ans = self._check_nans(other, context)
- if ans:
- return ans
-
- if self._isinfinity():
- return context._raise_error(InvalidOperation, 'INF % x')
- elif not other:
- if self:
- return context._raise_error(InvalidOperation, 'x % 0')
- else:
- return context._raise_error(DivisionUndefined, '0 % 0')
-
- remainder = self._divide(other, context)[1]
- remainder = remainder._fix(context)
- return remainder
-
- def __rmod__(self, other, context=None):
- """Swaps self/other and returns __mod__."""
- other = _convert_other(other)
- if other is NotImplemented:
- return other
- return other.__mod__(self, context=context)
-
- def remainder_near(self, other, context=None):
- """
- Remainder nearest to 0- abs(remainder-near) <= other/2
- """
- if context is None:
- context = getcontext()
-
- other = _convert_other(other, raiseit=True)
-
- ans = self._check_nans(other, context)
- if ans:
- return ans
-
- # self == +/-infinity -> InvalidOperation
- if self._isinfinity():
- return context._raise_error(InvalidOperation,
- 'remainder_near(infinity, x)')
-
- # other == 0 -> either InvalidOperation or DivisionUndefined
- if not other:
- if self:
- return context._raise_error(InvalidOperation,
- 'remainder_near(x, 0)')
- else:
- return context._raise_error(DivisionUndefined,
- 'remainder_near(0, 0)')
-
- # other = +/-infinity -> remainder = self
- if other._isinfinity():
- ans = Decimal(self)
- return ans._fix(context)
-
- # self = 0 -> remainder = self, with ideal exponent
- ideal_exponent = min(self._exp, other._exp)
- if not self:
- ans = _dec_from_triple(self._sign, '0', ideal_exponent)
- return ans._fix(context)
-
- # catch most cases of large or small quotient
- expdiff = self.adjusted() - other.adjusted()
- if expdiff >= context.prec + 1:
- # expdiff >= prec+1 => abs(self/other) > 10**prec
- return context._raise_error(DivisionImpossible)
- if expdiff <= -2:
- # expdiff <= -2 => abs(self/other) < 0.1
- ans = self._rescale(ideal_exponent, context.rounding)
- return ans._fix(context)
-
- # adjust both arguments to have the same exponent, then divide
- op1 = _WorkRep(self)
- op2 = _WorkRep(other)
- if op1.exp >= op2.exp:
- op1.int *= 10**(op1.exp - op2.exp)
- else:
- op2.int *= 10**(op2.exp - op1.exp)
- q, r = divmod(op1.int, op2.int)
- # remainder is r*10**ideal_exponent; other is +/-op2.int *
- # 10**ideal_exponent. Apply correction to ensure that
- # abs(remainder) <= abs(other)/2
- if 2*r + (q&1) > op2.int:
- r -= op2.int
- q += 1
-
- if q >= 10**context.prec:
- return context._raise_error(DivisionImpossible)
-
- # result has same sign as self unless r is negative
- sign = self._sign
- if r < 0:
- sign = 1-sign
- r = -r
-
- ans = _dec_from_triple(sign, str(r), ideal_exponent)
- return ans._fix(context)
-
- def __floordiv__(self, other, context=None):
- """self // other"""
- other = _convert_other(other)
- if other is NotImplemented:
- return other
-
- if context is None:
- context = getcontext()
-
- ans = self._check_nans(other, context)
- if ans:
- return ans
-
- if self._isinfinity():
- if other._isinfinity():
- return context._raise_error(InvalidOperation, 'INF // INF')
- else:
- return _SignedInfinity[self._sign ^ other._sign]
-
- if not other:
- if self:
- return context._raise_error(DivisionByZero, 'x // 0',
- self._sign ^ other._sign)
- else:
- return context._raise_error(DivisionUndefined, '0 // 0')
-
- return self._divide(other, context)[0]
-
- def __rfloordiv__(self, other, context=None):
- """Swaps self/other and returns __floordiv__."""
- other = _convert_other(other)
- if other is NotImplemented:
- return other
- return other.__floordiv__(self, context=context)
-
- def __float__(self):
- """Float representation."""
- if self._isnan():
- if self.is_snan():
- raise ValueError("Cannot convert signaling NaN to float")
- s = "-nan" if self._sign else "nan"
- else:
- s = str(self)
- return float(s)
-
- def __int__(self):
- """Converts self to an int, truncating if necessary."""
- if self._is_special:
- if self._isnan():
- raise ValueError("Cannot convert NaN to integer")
- elif self._isinfinity():
- raise OverflowError("Cannot convert infinity to integer")
- s = (-1)**self._sign
- if self._exp >= 0:
- return s*int(self._int)*10**self._exp
- else:
- return s*int(self._int[:self._exp] or '0')
-
- __trunc__ = __int__
-
- def real(self):
- return self
- real = property(real)
-
- def imag(self):
- return Decimal(0)
- imag = property(imag)
-
- def conjugate(self):
- return self
-
- def __complex__(self):
- return complex(float(self))
-
- def _fix_nan(self, context):
- """Decapitate the payload of a NaN to fit the context"""
- payload = self._int
-
- # maximum length of payload is precision if clamp=0,
- # precision-1 if clamp=1.
- max_payload_len = context.prec - context.clamp
- if len(payload) > max_payload_len:
- payload = payload[len(payload)-max_payload_len:].lstrip('0')
- return _dec_from_triple(self._sign, payload, self._exp, True)
- return Decimal(self)
-
- def _fix(self, context):
- """Round if it is necessary to keep self within prec precision.
-
- Rounds and fixes the exponent. Does not raise on a sNaN.
-
- Arguments:
- self - Decimal instance
- context - context used.
- """
-
- if self._is_special:
- if self._isnan():
- # decapitate payload if necessary
- return self._fix_nan(context)
- else:
- # self is +/-Infinity; return unaltered
- return Decimal(self)
-
- # if self is zero then exponent should be between Etiny and
- # Emax if clamp==0, and between Etiny and Etop if clamp==1.
- Etiny = context.Etiny()
- Etop = context.Etop()
- if not self:
- exp_max = [context.Emax, Etop][context.clamp]
- new_exp = min(max(self._exp, Etiny), exp_max)
- if new_exp != self._exp:
- context._raise_error(Clamped)
- return _dec_from_triple(self._sign, '0', new_exp)
- else:
- return Decimal(self)
-
- # exp_min is the smallest allowable exponent of the result,
- # equal to max(self.adjusted()-context.prec+1, Etiny)
- exp_min = len(self._int) + self._exp - context.prec
- if exp_min > Etop:
- # overflow: exp_min > Etop iff self.adjusted() > Emax
- ans = context._raise_error(Overflow, 'above Emax', self._sign)
- context._raise_error(Inexact)
- context._raise_error(Rounded)
- return ans
-
- self_is_subnormal = exp_min < Etiny
- if self_is_subnormal:
- exp_min = Etiny
-
- # round if self has too many digits
- if self._exp < exp_min:
- digits = len(self._int) + self._exp - exp_min
- if digits < 0:
- self = _dec_from_triple(self._sign, '1', exp_min-1)
- digits = 0
- rounding_method = self._pick_rounding_function[context.rounding]
- changed = rounding_method(self, digits)
- coeff = self._int[:digits] or '0'
- if changed > 0:
- coeff = str(int(coeff)+1)
- if len(coeff) > context.prec:
- coeff = coeff[:-1]
- exp_min += 1
-
- # check whether the rounding pushed the exponent out of range
- if exp_min > Etop:
- ans = context._raise_error(Overflow, 'above Emax', self._sign)
- else:
- ans = _dec_from_triple(self._sign, coeff, exp_min)
-
- # raise the appropriate signals, taking care to respect
- # the precedence described in the specification
- if changed and self_is_subnormal:
- context._raise_error(Underflow)
- if self_is_subnormal:
- context._raise_error(Subnormal)
- if changed:
- context._raise_error(Inexact)
- context._raise_error(Rounded)
- if not ans:
- # raise Clamped on underflow to 0
- context._raise_error(Clamped)
- return ans
-
- if self_is_subnormal:
- context._raise_error(Subnormal)
-
- # fold down if clamp == 1 and self has too few digits
- if context.clamp == 1 and self._exp > Etop:
- context._raise_error(Clamped)
- self_padded = self._int + '0'*(self._exp - Etop)
- return _dec_from_triple(self._sign, self_padded, Etop)
-
- # here self was representable to begin with; return unchanged
- return Decimal(self)
-
- # for each of the rounding functions below:
- # self is a finite, nonzero Decimal
- # prec is an integer satisfying 0 <= prec < len(self._int)
- #
- # each function returns either -1, 0, or 1, as follows:
- # 1 indicates that self should be rounded up (away from zero)
- # 0 indicates that self should be truncated, and that all the
- # digits to be truncated are zeros (so the value is unchanged)
- # -1 indicates that there are nonzero digits to be truncated
-
- def _round_down(self, prec):
- """Also known as round-towards-0, truncate."""
- if _all_zeros(self._int, prec):
- return 0
- else:
- return -1
-
- def _round_up(self, prec):
- """Rounds away from 0."""
- return -self._round_down(prec)
-
- def _round_half_up(self, prec):
- """Rounds 5 up (away from 0)"""
- if self._int[prec] in '56789':
- return 1
- elif _all_zeros(self._int, prec):
- return 0
- else:
- return -1
-
- def _round_half_down(self, prec):
- """Round 5 down"""
- if _exact_half(self._int, prec):
- return -1
- else:
- return self._round_half_up(prec)
-
- def _round_half_even(self, prec):
- """Round 5 to even, rest to nearest."""
- if _exact_half(self._int, prec) and \
- (prec == 0 or self._int[prec-1] in '02468'):
- return -1
- else:
- return self._round_half_up(prec)
-
- def _round_ceiling(self, prec):
- """Rounds up (not away from 0 if negative.)"""
- if self._sign:
- return self._round_down(prec)
- else:
- return -self._round_down(prec)
-
- def _round_floor(self, prec):
- """Rounds down (not towards 0 if negative)"""
- if not self._sign:
- return self._round_down(prec)
- else:
- return -self._round_down(prec)
-
- def _round_05up(self, prec):
- """Round down unless digit prec-1 is 0 or 5."""
- if prec and self._int[prec-1] not in '05':
- return self._round_down(prec)
- else:
- return -self._round_down(prec)
-
- _pick_rounding_function = dict(
- ROUND_DOWN = _round_down,
- ROUND_UP = _round_up,
- ROUND_HALF_UP = _round_half_up,
- ROUND_HALF_DOWN = _round_half_down,
- ROUND_HALF_EVEN = _round_half_even,
- ROUND_CEILING = _round_ceiling,
- ROUND_FLOOR = _round_floor,
- ROUND_05UP = _round_05up,
- )
-
- def __round__(self, n=None):
- """Round self to the nearest integer, or to a given precision.
-
- If only one argument is supplied, round a finite Decimal
- instance self to the nearest integer. If self is infinite or
- a NaN then a Python exception is raised. If self is finite
- and lies exactly halfway between two integers then it is
- rounded to the integer with even last digit.
-
- >>> round(Decimal('123.456'))
- 123
- >>> round(Decimal('-456.789'))
- -457
- >>> round(Decimal('-3.0'))
- -3
- >>> round(Decimal('2.5'))
- 2
- >>> round(Decimal('3.5'))
- 4
- >>> round(Decimal('Inf'))
- Traceback (most recent call last):
- ...
- OverflowError: cannot round an infinity
- >>> round(Decimal('NaN'))
- Traceback (most recent call last):
- ...
- ValueError: cannot round a NaN
-
- If a second argument n is supplied, self is rounded to n
- decimal places using the rounding mode for the current
- context.
-
- For an integer n, round(self, -n) is exactly equivalent to
- self.quantize(Decimal('1En')).
-
- >>> round(Decimal('123.456'), 0)
- Decimal('123')
- >>> round(Decimal('123.456'), 2)
- Decimal('123.46')
- >>> round(Decimal('123.456'), -2)
- Decimal('1E+2')
- >>> round(Decimal('-Infinity'), 37)
- Decimal('NaN')
- >>> round(Decimal('sNaN123'), 0)
- Decimal('NaN123')
-
- """
- if n is not None:
- # two-argument form: use the equivalent quantize call
- if not isinstance(n, int):
- raise TypeError('Second argument to round should be integral')
- exp = _dec_from_triple(0, '1', -n)
- return self.quantize(exp)
-
- # one-argument form
- if self._is_special:
- if self.is_nan():
- raise ValueError("cannot round a NaN")
- else:
- raise OverflowError("cannot round an infinity")
- return int(self._rescale(0, ROUND_HALF_EVEN))
-
- def __floor__(self):
- """Return the floor of self, as an integer.
-
- For a finite Decimal instance self, return the greatest
- integer n such that n <= self. If self is infinite or a NaN
- then a Python exception is raised.
-
- """
- if self._is_special:
- if self.is_nan():
- raise ValueError("cannot round a NaN")
- else:
- raise OverflowError("cannot round an infinity")
- return int(self._rescale(0, ROUND_FLOOR))
-
- def __ceil__(self):
- """Return the ceiling of self, as an integer.
-
- For a finite Decimal instance self, return the least integer n
- such that n >= self. If self is infinite or a NaN then a
- Python exception is raised.
-
- """
- if self._is_special:
- if self.is_nan():
- raise ValueError("cannot round a NaN")
- else:
- raise OverflowError("cannot round an infinity")
- return int(self._rescale(0, ROUND_CEILING))
-
- def fma(self, other, third, context=None):
- """Fused multiply-add.
-
- Returns self*other+third with no rounding of the intermediate
- product self*other.
-
- self and other are multiplied together, with no rounding of
- the result. The third operand is then added to the result,
- and a single final rounding is performed.
- """
-
- other = _convert_other(other, raiseit=True)
- third = _convert_other(third, raiseit=True)
-
- # compute product; raise InvalidOperation if either operand is
- # a signaling NaN or if the product is zero times infinity.
- if self._is_special or other._is_special:
- if context is None:
- context = getcontext()
- if self._exp == 'N':
- return context._raise_error(InvalidOperation, 'sNaN', self)
- if other._exp == 'N':
- return context._raise_error(InvalidOperation, 'sNaN', other)
- if self._exp == 'n':
- product = self
- elif other._exp == 'n':
- product = other
- elif self._exp == 'F':
- if not other:
- return context._raise_error(InvalidOperation,
- 'INF * 0 in fma')
- product = _SignedInfinity[self._sign ^ other._sign]
- elif other._exp == 'F':
- if not self:
- return context._raise_error(InvalidOperation,
- '0 * INF in fma')
- product = _SignedInfinity[self._sign ^ other._sign]
- else:
- product = _dec_from_triple(self._sign ^ other._sign,
- str(int(self._int) * int(other._int)),
- self._exp + other._exp)
-
- return product.__add__(third, context)
-
- def _power_modulo(self, other, modulo, context=None):
- """Three argument version of __pow__"""
-
- other = _convert_other(other)
- if other is NotImplemented:
- return other
- modulo = _convert_other(modulo)
- if modulo is NotImplemented:
- return modulo
-
- if context is None:
- context = getcontext()
-
- # deal with NaNs: if there are any sNaNs then first one wins,
- # (i.e. behaviour for NaNs is identical to that of fma)
- self_is_nan = self._isnan()
- other_is_nan = other._isnan()
- modulo_is_nan = modulo._isnan()
- if self_is_nan or other_is_nan or modulo_is_nan:
- if self_is_nan == 2:
- return context._raise_error(InvalidOperation, 'sNaN',
- self)
- if other_is_nan == 2:
- return context._raise_error(InvalidOperation, 'sNaN',
- other)
- if modulo_is_nan == 2:
- return context._raise_error(InvalidOperation, 'sNaN',
- modulo)
- if self_is_nan:
- return self._fix_nan(context)
- if other_is_nan:
- return other._fix_nan(context)
- return modulo._fix_nan(context)
-
- # check inputs: we apply same restrictions as Python's pow()
- if not (self._isinteger() and
- other._isinteger() and
- modulo._isinteger()):
- return context._raise_error(InvalidOperation,
- 'pow() 3rd argument not allowed '
- 'unless all arguments are integers')
- if other < 0:
- return context._raise_error(InvalidOperation,
- 'pow() 2nd argument cannot be '
- 'negative when 3rd argument specified')
- if not modulo:
- return context._raise_error(InvalidOperation,
- 'pow() 3rd argument cannot be 0')
-
- # additional restriction for decimal: the modulus must be less
- # than 10**prec in absolute value
- if modulo.adjusted() >= context.prec:
- return context._raise_error(InvalidOperation,
- 'insufficient precision: pow() 3rd '
- 'argument must not have more than '
- 'precision digits')
-
- # define 0**0 == NaN, for consistency with two-argument pow
- # (even though it hurts!)
- if not other and not self:
- return context._raise_error(InvalidOperation,
- 'at least one of pow() 1st argument '
- 'and 2nd argument must be nonzero ;'
- '0**0 is not defined')
-
- # compute sign of result
- if other._iseven():
- sign = 0
- else:
- sign = self._sign
-
- # convert modulo to a Python integer, and self and other to
- # Decimal integers (i.e. force their exponents to be >= 0)
- modulo = abs(int(modulo))
- base = _WorkRep(self.to_integral_value())
- exponent = _WorkRep(other.to_integral_value())
-
- # compute result using integer pow()
- base = (base.int % modulo * pow(10, base.exp, modulo)) % modulo
- for i in range(exponent.exp):
- base = pow(base, 10, modulo)
- base = pow(base, exponent.int, modulo)
-
- return _dec_from_triple(sign, str(base), 0)
-
- def _power_exact(self, other, p):
- """Attempt to compute self**other exactly.
-
- Given Decimals self and other and an integer p, attempt to
- compute an exact result for the power self**other, with p
- digits of precision. Return None if self**other is not
- exactly representable in p digits.
-
- Assumes that elimination of special cases has already been
- performed: self and other must both be nonspecial; self must
- be positive and not numerically equal to 1; other must be
- nonzero. For efficiency, other._exp should not be too large,
- so that 10**abs(other._exp) is a feasible calculation."""
-
- # In the comments below, we write x for the value of self and y for the
- # value of other. Write x = xc*10**xe and abs(y) = yc*10**ye, with xc
- # and yc positive integers not divisible by 10.
-
- # The main purpose of this method is to identify the *failure*
- # of x**y to be exactly representable with as little effort as
- # possible. So we look for cheap and easy tests that
- # eliminate the possibility of x**y being exact. Only if all
- # these tests are passed do we go on to actually compute x**y.
-
- # Here's the main idea. Express y as a rational number m/n, with m and
- # n relatively prime and n>0. Then for x**y to be exactly
- # representable (at *any* precision), xc must be the nth power of a
- # positive integer and xe must be divisible by n. If y is negative
- # then additionally xc must be a power of either 2 or 5, hence a power
- # of 2**n or 5**n.
- #
- # There's a limit to how small |y| can be: if y=m/n as above
- # then:
- #
- # (1) if xc != 1 then for the result to be representable we
- # need xc**(1/n) >= 2, and hence also xc**|y| >= 2. So
- # if |y| <= 1/nbits(xc) then xc < 2**nbits(xc) <=
- # 2**(1/|y|), hence xc**|y| < 2 and the result is not
- # representable.
- #
- # (2) if xe != 0, |xe|*(1/n) >= 1, so |xe|*|y| >= 1. Hence if
- # |y| < 1/|xe| then the result is not representable.
- #
- # Note that since x is not equal to 1, at least one of (1) and
- # (2) must apply. Now |y| < 1/nbits(xc) iff |yc|*nbits(xc) <
- # 10**-ye iff len(str(|yc|*nbits(xc)) <= -ye.
- #
- # There's also a limit to how large y can be, at least if it's
- # positive: the normalized result will have coefficient xc**y,
- # so if it's representable then xc**y < 10**p, and y <
- # p/log10(xc). Hence if y*log10(xc) >= p then the result is
- # not exactly representable.
-
- # if len(str(abs(yc*xe)) <= -ye then abs(yc*xe) < 10**-ye,
- # so |y| < 1/xe and the result is not representable.
- # Similarly, len(str(abs(yc)*xc_bits)) <= -ye implies |y|
- # < 1/nbits(xc).
-
- x = _WorkRep(self)
- xc, xe = x.int, x.exp
- while xc % 10 == 0:
- xc //= 10
- xe += 1
-
- y = _WorkRep(other)
- yc, ye = y.int, y.exp
- while yc % 10 == 0:
- yc //= 10
- ye += 1
-
- # case where xc == 1: result is 10**(xe*y), with xe*y
- # required to be an integer
- if xc == 1:
- xe *= yc
- # result is now 10**(xe * 10**ye); xe * 10**ye must be integral
- while xe % 10 == 0:
- xe //= 10
- ye += 1
- if ye < 0:
- return None
- exponent = xe * 10**ye
- if y.sign == 1:
- exponent = -exponent
- # if other is a nonnegative integer, use ideal exponent
- if other._isinteger() and other._sign == 0:
- ideal_exponent = self._exp*int(other)
- zeros = min(exponent-ideal_exponent, p-1)
- else:
- zeros = 0
- return _dec_from_triple(0, '1' + '0'*zeros, exponent-zeros)
-
- # case where y is negative: xc must be either a power
- # of 2 or a power of 5.
- if y.sign == 1:
- last_digit = xc % 10
- if last_digit in (2,4,6,8):
- # quick test for power of 2
- if xc & -xc != xc:
- return None
- # now xc is a power of 2; e is its exponent
- e = _nbits(xc)-1
-
- # We now have:
- #
- # x = 2**e * 10**xe, e > 0, and y < 0.
- #
- # The exact result is:
- #
- # x**y = 5**(-e*y) * 10**(e*y + xe*y)
- #
- # provided that both e*y and xe*y are integers. Note that if
- # 5**(-e*y) >= 10**p, then the result can't be expressed
- # exactly with p digits of precision.
- #
- # Using the above, we can guard against large values of ye.
- # 93/65 is an upper bound for log(10)/log(5), so if
- #
- # ye >= len(str(93*p//65))
- #
- # then
- #
- # -e*y >= -y >= 10**ye > 93*p/65 > p*log(10)/log(5),
- #
- # so 5**(-e*y) >= 10**p, and the coefficient of the result
- # can't be expressed in p digits.
-
- # emax >= largest e such that 5**e < 10**p.
- emax = p*93//65
- if ye >= len(str(emax)):
- return None
-
- # Find -e*y and -xe*y; both must be integers
- e = _decimal_lshift_exact(e * yc, ye)
- xe = _decimal_lshift_exact(xe * yc, ye)
- if e is None or xe is None:
- return None
-
- if e > emax:
- return None
- xc = 5**e
-
- elif last_digit == 5:
- # e >= log_5(xc) if xc is a power of 5; we have
- # equality all the way up to xc=5**2658
- e = _nbits(xc)*28//65
- xc, remainder = divmod(5**e, xc)
- if remainder:
- return None
- while xc % 5 == 0:
- xc //= 5
- e -= 1
-
- # Guard against large values of ye, using the same logic as in
- # the 'xc is a power of 2' branch. 10/3 is an upper bound for
- # log(10)/log(2).
- emax = p*10//3
- if ye >= len(str(emax)):
- return None
-
- e = _decimal_lshift_exact(e * yc, ye)
- xe = _decimal_lshift_exact(xe * yc, ye)
- if e is None or xe is None:
- return None
-
- if e > emax:
- return None
- xc = 2**e
- else:
- return None
-
- if xc >= 10**p:
- return None
- xe = -e-xe
- return _dec_from_triple(0, str(xc), xe)
-
- # now y is positive; find m and n such that y = m/n
- if ye >= 0:
- m, n = yc*10**ye, 1
- else:
- if xe != 0 and len(str(abs(yc*xe))) <= -ye:
- return None
- xc_bits = _nbits(xc)
- if xc != 1 and len(str(abs(yc)*xc_bits)) <= -ye:
- return None
- m, n = yc, 10**(-ye)
- while m % 2 == n % 2 == 0:
- m //= 2
- n //= 2
- while m % 5 == n % 5 == 0:
- m //= 5
- n //= 5
-
- # compute nth root of xc*10**xe
- if n > 1:
- # if 1 < xc < 2**n then xc isn't an nth power
- if xc != 1 and xc_bits <= n:
- return None
-
- xe, rem = divmod(xe, n)
- if rem != 0:
- return None
-
- # compute nth root of xc using Newton's method
- a = 1 << -(-_nbits(xc)//n) # initial estimate
- while True:
- q, r = divmod(xc, a**(n-1))
- if a <= q:
- break
- else:
- a = (a*(n-1) + q)//n
- if not (a == q and r == 0):
- return None
- xc = a
-
- # now xc*10**xe is the nth root of the original xc*10**xe
- # compute mth power of xc*10**xe
-
- # if m > p*100//_log10_lb(xc) then m > p/log10(xc), hence xc**m >
- # 10**p and the result is not representable.
- if xc > 1 and m > p*100//_log10_lb(xc):
- return None
- xc = xc**m
- xe *= m
- if xc > 10**p:
- return None
-
- # by this point the result *is* exactly representable
- # adjust the exponent to get as close as possible to the ideal
- # exponent, if necessary
- str_xc = str(xc)
- if other._isinteger() and other._sign == 0:
- ideal_exponent = self._exp*int(other)
- zeros = min(xe-ideal_exponent, p-len(str_xc))
- else:
- zeros = 0
- return _dec_from_triple(0, str_xc+'0'*zeros, xe-zeros)
-
- def __pow__(self, other, modulo=None, context=None):
- """Return self ** other [ % modulo].
-
- With two arguments, compute self**other.
-
- With three arguments, compute (self**other) % modulo. For the
- three argument form, the following restrictions on the
- arguments hold:
-
- - all three arguments must be integral
- - other must be nonnegative
- - either self or other (or both) must be nonzero
- - modulo must be nonzero and must have at most p digits,
- where p is the context precision.
-
- If any of these restrictions is violated the InvalidOperation
- flag is raised.
-
- The result of pow(self, other, modulo) is identical to the
- result that would be obtained by computing (self**other) %
- modulo with unbounded precision, but is computed more
- efficiently. It is always exact.
- """
-
- if modulo is not None:
- return self._power_modulo(other, modulo, context)
-
- other = _convert_other(other)
- if other is NotImplemented:
- return other
-
- if context is None:
- context = getcontext()
-
- # either argument is a NaN => result is NaN
- ans = self._check_nans(other, context)
- if ans:
- return ans
-
- # 0**0 = NaN (!), x**0 = 1 for nonzero x (including +/-Infinity)
- if not other:
- if not self:
- return context._raise_error(InvalidOperation, '0 ** 0')
- else:
- return _One
-
- # result has sign 1 iff self._sign is 1 and other is an odd integer
- result_sign = 0
- if self._sign == 1:
- if other._isinteger():
- if not other._iseven():
- result_sign = 1
- else:
- # -ve**noninteger = NaN
- # (-0)**noninteger = 0**noninteger
- if self:
- return context._raise_error(InvalidOperation,
- 'x ** y with x negative and y not an integer')
- # negate self, without doing any unwanted rounding
- self = self.copy_negate()
-
- # 0**(+ve or Inf)= 0; 0**(-ve or -Inf) = Infinity
- if not self:
- if other._sign == 0:
- return _dec_from_triple(result_sign, '0', 0)
- else:
- return _SignedInfinity[result_sign]
-
- # Inf**(+ve or Inf) = Inf; Inf**(-ve or -Inf) = 0
- if self._isinfinity():
- if other._sign == 0:
- return _SignedInfinity[result_sign]
- else:
- return _dec_from_triple(result_sign, '0', 0)
-
- # 1**other = 1, but the choice of exponent and the flags
- # depend on the exponent of self, and on whether other is a
- # positive integer, a negative integer, or neither
- if self == _One:
- if other._isinteger():
- # exp = max(self._exp*max(int(other), 0),
- # 1-context.prec) but evaluating int(other) directly
- # is dangerous until we know other is small (other
- # could be 1e999999999)
- if other._sign == 1:
- multiplier = 0
- elif other > context.prec:
- multiplier = context.prec
- else:
- multiplier = int(other)
-
- exp = self._exp * multiplier
- if exp < 1-context.prec:
- exp = 1-context.prec
- context._raise_error(Rounded)
- else:
- context._raise_error(Inexact)
- context._raise_error(Rounded)
- exp = 1-context.prec
-
- return _dec_from_triple(result_sign, '1'+'0'*-exp, exp)
-
- # compute adjusted exponent of self
- self_adj = self.adjusted()
-
- # self ** infinity is infinity if self > 1, 0 if self < 1
- # self ** -infinity is infinity if self < 1, 0 if self > 1
- if other._isinfinity():
- if (other._sign == 0) == (self_adj < 0):
- return _dec_from_triple(result_sign, '0', 0)
- else:
- return _SignedInfinity[result_sign]
-
- # from here on, the result always goes through the call
- # to _fix at the end of this function.
- ans = None
- exact = False
-
- # crude test to catch cases of extreme overflow/underflow. If
- # log10(self)*other >= 10**bound and bound >= len(str(Emax))
- # then 10**bound >= 10**len(str(Emax)) >= Emax+1 and hence
- # self**other >= 10**(Emax+1), so overflow occurs. The test
- # for underflow is similar.
- bound = self._log10_exp_bound() + other.adjusted()
- if (self_adj >= 0) == (other._sign == 0):
- # self > 1 and other +ve, or self < 1 and other -ve
- # possibility of overflow
- if bound >= len(str(context.Emax)):
- ans = _dec_from_triple(result_sign, '1', context.Emax+1)
- else:
- # self > 1 and other -ve, or self < 1 and other +ve
- # possibility of underflow to 0
- Etiny = context.Etiny()
- if bound >= len(str(-Etiny)):
- ans = _dec_from_triple(result_sign, '1', Etiny-1)
-
- # try for an exact result with precision +1
- if ans is None:
- ans = self._power_exact(other, context.prec + 1)
- if ans is not None:
- if result_sign == 1:
- ans = _dec_from_triple(1, ans._int, ans._exp)
- exact = True
-
- # usual case: inexact result, x**y computed directly as exp(y*log(x))
- if ans is None:
- p = context.prec
- x = _WorkRep(self)
- xc, xe = x.int, x.exp
- y = _WorkRep(other)
- yc, ye = y.int, y.exp
- if y.sign == 1:
- yc = -yc
-
- # compute correctly rounded result: start with precision +3,
- # then increase precision until result is unambiguously roundable
- extra = 3
- while True:
- coeff, exp = _dpower(xc, xe, yc, ye, p+extra)
- if coeff % (5*10**(len(str(coeff))-p-1)):
- break
- extra += 3
-
- ans = _dec_from_triple(result_sign, str(coeff), exp)
-
- # unlike exp, ln and log10, the power function respects the
- # rounding mode; no need to switch to ROUND_HALF_EVEN here
-
- # There's a difficulty here when 'other' is not an integer and
- # the result is exact. In this case, the specification
- # requires that the Inexact flag be raised (in spite of
- # exactness), but since the result is exact _fix won't do this
- # for us. (Correspondingly, the Underflow signal should also
- # be raised for subnormal results.) We can't directly raise
- # these signals either before or after calling _fix, since
- # that would violate the precedence for signals. So we wrap
- # the ._fix call in a temporary context, and reraise
- # afterwards.
- if exact and not other._isinteger():
- # pad with zeros up to length context.prec+1 if necessary; this
- # ensures that the Rounded signal will be raised.
- if len(ans._int) <= context.prec:
- expdiff = context.prec + 1 - len(ans._int)
- ans = _dec_from_triple(ans._sign, ans._int+'0'*expdiff,
- ans._exp-expdiff)
-
- # create a copy of the current context, with cleared flags/traps
- newcontext = context.copy()
- newcontext.clear_flags()
- for exception in _signals:
- newcontext.traps[exception] = 0
-
- # round in the new context
- ans = ans._fix(newcontext)
-
- # raise Inexact, and if necessary, Underflow
- newcontext._raise_error(Inexact)
- if newcontext.flags[Subnormal]:
- newcontext._raise_error(Underflow)
-
- # propagate signals to the original context; _fix could
- # have raised any of Overflow, Underflow, Subnormal,
- # Inexact, Rounded, Clamped. Overflow needs the correct
- # arguments. Note that the order of the exceptions is
- # important here.
- if newcontext.flags[Overflow]:
- context._raise_error(Overflow, 'above Emax', ans._sign)
- for exception in Underflow, Subnormal, Inexact, Rounded, Clamped:
- if newcontext.flags[exception]:
- context._raise_error(exception)
-
- else:
- ans = ans._fix(context)
-
- return ans
-
- def __rpow__(self, other, context=None):
- """Swaps self/other and returns __pow__."""
- other = _convert_other(other)
- if other is NotImplemented:
- return other
- return other.__pow__(self, context=context)
-
- def normalize(self, context=None):
- """Normalize- strip trailing 0s, change anything equal to 0 to 0e0"""
-
- if context is None:
- context = getcontext()
-
- if self._is_special:
- ans = self._check_nans(context=context)
- if ans:
- return ans
-
- dup = self._fix(context)
- if dup._isinfinity():
- return dup
-
- if not dup:
- return _dec_from_triple(dup._sign, '0', 0)
- exp_max = [context.Emax, context.Etop()][context.clamp]
- end = len(dup._int)
- exp = dup._exp
- while dup._int[end-1] == '0' and exp < exp_max:
- exp += 1
- end -= 1
- return _dec_from_triple(dup._sign, dup._int[:end], exp)
-
- def quantize(self, exp, rounding=None, context=None, watchexp=True):
- """Quantize self so its exponent is the same as that of exp.
-
- Similar to self._rescale(exp._exp) but with error checking.
- """
- exp = _convert_other(exp, raiseit=True)
-
- if context is None:
- context = getcontext()
- if rounding is None:
- rounding = context.rounding
-
- if self._is_special or exp._is_special:
- ans = self._check_nans(exp, context)
- if ans:
- return ans
-
- if exp._isinfinity() or self._isinfinity():
- if exp._isinfinity() and self._isinfinity():
- return Decimal(self) # if both are inf, it is OK
- return context._raise_error(InvalidOperation,
- 'quantize with one INF')
-
- # if we're not watching exponents, do a simple rescale
- if not watchexp:
- ans = self._rescale(exp._exp, rounding)
- # raise Inexact and Rounded where appropriate
- if ans._exp > self._exp:
- context._raise_error(Rounded)
- if ans != self:
- context._raise_error(Inexact)
- return ans
-
- # exp._exp should be between Etiny and Emax
- if not (context.Etiny() <= exp._exp <= context.Emax):
- return context._raise_error(InvalidOperation,
- 'target exponent out of bounds in quantize')
-
- if not self:
- ans = _dec_from_triple(self._sign, '0', exp._exp)
- return ans._fix(context)
-
- self_adjusted = self.adjusted()
- if self_adjusted > context.Emax:
- return context._raise_error(InvalidOperation,
- 'exponent of quantize result too large for current context')
- if self_adjusted - exp._exp + 1 > context.prec:
- return context._raise_error(InvalidOperation,
- 'quantize result has too many digits for current context')
-
- ans = self._rescale(exp._exp, rounding)
- if ans.adjusted() > context.Emax:
- return context._raise_error(InvalidOperation,
- 'exponent of quantize result too large for current context')
- if len(ans._int) > context.prec:
- return context._raise_error(InvalidOperation,
- 'quantize result has too many digits for current context')
-
- # raise appropriate flags
- if ans and ans.adjusted() < context.Emin:
- context._raise_error(Subnormal)
- if ans._exp > self._exp:
- if ans != self:
- context._raise_error(Inexact)
- context._raise_error(Rounded)
-
- # call to fix takes care of any necessary folddown, and
- # signals Clamped if necessary
- ans = ans._fix(context)
- return ans
-
- def same_quantum(self, other, context=None):
- """Return True if self and other have the same exponent; otherwise
- return False.
-
- If either operand is a special value, the following rules are used:
- * return True if both operands are infinities
- * return True if both operands are NaNs
- * otherwise, return False.
- """
- other = _convert_other(other, raiseit=True)
- if self._is_special or other._is_special:
- return (self.is_nan() and other.is_nan() or
- self.is_infinite() and other.is_infinite())
- return self._exp == other._exp
-
- def _rescale(self, exp, rounding):
- """Rescale self so that the exponent is exp, either by padding with zeros
- or by truncating digits, using the given rounding mode.
-
- Specials are returned without change. This operation is
- quiet: it raises no flags, and uses no information from the
- context.
-
- exp = exp to scale to (an integer)
- rounding = rounding mode
- """
- if self._is_special:
- return Decimal(self)
- if not self:
- return _dec_from_triple(self._sign, '0', exp)
-
- if self._exp >= exp:
- # pad answer with zeros if necessary
- return _dec_from_triple(self._sign,
- self._int + '0'*(self._exp - exp), exp)
-
- # too many digits; round and lose data. If self.adjusted() <
- # exp-1, replace self by 10**(exp-1) before rounding
- digits = len(self._int) + self._exp - exp
- if digits < 0:
- self = _dec_from_triple(self._sign, '1', exp-1)
- digits = 0
- this_function = self._pick_rounding_function[rounding]
- changed = this_function(self, digits)
- coeff = self._int[:digits] or '0'
- if changed == 1:
- coeff = str(int(coeff)+1)
- return _dec_from_triple(self._sign, coeff, exp)
-
- def _round(self, places, rounding):
- """Round a nonzero, nonspecial Decimal to a fixed number of
- significant figures, using the given rounding mode.
-
- Infinities, NaNs and zeros are returned unaltered.
-
- This operation is quiet: it raises no flags, and uses no
- information from the context.
-
- """
- if places <= 0:
- raise ValueError("argument should be at least 1 in _round")
- if self._is_special or not self:
- return Decimal(self)
- ans = self._rescale(self.adjusted()+1-places, rounding)
- # it can happen that the rescale alters the adjusted exponent;
- # for example when rounding 99.97 to 3 significant figures.
- # When this happens we end up with an extra 0 at the end of
- # the number; a second rescale fixes this.
- if ans.adjusted() != self.adjusted():
- ans = ans._rescale(ans.adjusted()+1-places, rounding)
- return ans
-
- def to_integral_exact(self, rounding=None, context=None):
- """Rounds to a nearby integer.
-
- If no rounding mode is specified, take the rounding mode from
- the context. This method raises the Rounded and Inexact flags
- when appropriate.
-
- See also: to_integral_value, which does exactly the same as
- this method except that it doesn't raise Inexact or Rounded.
- """
- if self._is_special:
- ans = self._check_nans(context=context)
- if ans:
- return ans
- return Decimal(self)
- if self._exp >= 0:
- return Decimal(self)
- if not self:
- return _dec_from_triple(self._sign, '0', 0)
- if context is None:
- context = getcontext()
- if rounding is None:
- rounding = context.rounding
- ans = self._rescale(0, rounding)
- if ans != self:
- context._raise_error(Inexact)
- context._raise_error(Rounded)
- return ans
-
- def to_integral_value(self, rounding=None, context=None):
- """Rounds to the nearest integer, without raising inexact, rounded."""
- if context is None:
- context = getcontext()
- if rounding is None:
- rounding = context.rounding
- if self._is_special:
- ans = self._check_nans(context=context)
- if ans:
- return ans
- return Decimal(self)
- if self._exp >= 0:
- return Decimal(self)
- else:
- return self._rescale(0, rounding)
-
- # the method name changed, but we provide also the old one, for compatibility
- to_integral = to_integral_value
-
- def sqrt(self, context=None):
- """Return the square root of self."""
- if context is None:
- context = getcontext()
-
- if self._is_special:
- ans = self._check_nans(context=context)
- if ans:
- return ans
-
- if self._isinfinity() and self._sign == 0:
- return Decimal(self)
-
- if not self:
- # exponent = self._exp // 2. sqrt(-0) = -0
- ans = _dec_from_triple(self._sign, '0', self._exp // 2)
- return ans._fix(context)
-
- if self._sign == 1:
- return context._raise_error(InvalidOperation, 'sqrt(-x), x > 0')
-
- # At this point self represents a positive number. Let p be
- # the desired precision and express self in the form c*100**e
- # with c a positive real number and e an integer, c and e
- # being chosen so that 100**(p-1) <= c < 100**p. Then the
- # (exact) square root of self is sqrt(c)*10**e, and 10**(p-1)
- # <= sqrt(c) < 10**p, so the closest representable Decimal at
- # precision p is n*10**e where n = round_half_even(sqrt(c)),
- # the closest integer to sqrt(c) with the even integer chosen
- # in the case of a tie.
- #
- # To ensure correct rounding in all cases, we use the
- # following trick: we compute the square root to an extra
- # place (precision p+1 instead of precision p), rounding down.
- # Then, if the result is inexact and its last digit is 0 or 5,
- # we increase the last digit to 1 or 6 respectively; if it's
- # exact we leave the last digit alone. Now the final round to
- # p places (or fewer in the case of underflow) will round
- # correctly and raise the appropriate flags.
-
- # use an extra digit of precision
- prec = context.prec+1
-
- # write argument in the form c*100**e where e = self._exp//2
- # is the 'ideal' exponent, to be used if the square root is
- # exactly representable. l is the number of 'digits' of c in
- # base 100, so that 100**(l-1) <= c < 100**l.
- op = _WorkRep(self)
- e = op.exp >> 1
- if op.exp & 1:
- c = op.int * 10
- l = (len(self._int) >> 1) + 1
- else:
- c = op.int
- l = len(self._int)+1 >> 1
-
- # rescale so that c has exactly prec base 100 'digits'
- shift = prec-l
- if shift >= 0:
- c *= 100**shift
- exact = True
- else:
- c, remainder = divmod(c, 100**-shift)
- exact = not remainder
- e -= shift
-
- # find n = floor(sqrt(c)) using Newton's method
- n = 10**prec
- while True:
- q = c//n
- if n <= q:
- break
- else:
- n = n + q >> 1
- exact = exact and n*n == c
-
- if exact:
- # result is exact; rescale to use ideal exponent e
- if shift >= 0:
- # assert n % 10**shift == 0
- n //= 10**shift
- else:
- n *= 10**-shift
- e += shift
- else:
- # result is not exact; fix last digit as described above
- if n % 5 == 0:
- n += 1
-
- ans = _dec_from_triple(0, str(n), e)
-
- # round, and fit to current context
- context = context._shallow_copy()
- rounding = context._set_rounding(ROUND_HALF_EVEN)
- ans = ans._fix(context)
- context.rounding = rounding
-
- return ans
-
- def max(self, other, context=None):
- """Returns the larger value.
-
- Like max(self, other) except if one is not a number, returns
- NaN (and signals if one is sNaN). Also rounds.
- """
- other = _convert_other(other, raiseit=True)
-
- if context is None:
- context = getcontext()
-
- if self._is_special or other._is_special:
- # If one operand is a quiet NaN and the other is number, then the
- # number is always returned
- sn = self._isnan()
- on = other._isnan()
- if sn or on:
- if on == 1 and sn == 0:
- return self._fix(context)
- if sn == 1 and on == 0:
- return other._fix(context)
- return self._check_nans(other, context)
-
- c = self._cmp(other)
- if c == 0:
- # If both operands are finite and equal in numerical value
- # then an ordering is applied:
- #
- # If the signs differ then max returns the operand with the
- # positive sign and min returns the operand with the negative sign
- #
- # If the signs are the same then the exponent is used to select
- # the result. This is exactly the ordering used in compare_total.
- c = self.compare_total(other)
-
- if c == -1:
- ans = other
- else:
- ans = self
-
- return ans._fix(context)
-
- def min(self, other, context=None):
- """Returns the smaller value.
-
- Like min(self, other) except if one is not a number, returns
- NaN (and signals if one is sNaN). Also rounds.
- """
- other = _convert_other(other, raiseit=True)
-
- if context is None:
- context = getcontext()
-
- if self._is_special or other._is_special:
- # If one operand is a quiet NaN and the other is number, then the
- # number is always returned
- sn = self._isnan()
- on = other._isnan()
- if sn or on:
- if on == 1 and sn == 0:
- return self._fix(context)
- if sn == 1 and on == 0:
- return other._fix(context)
- return self._check_nans(other, context)
-
- c = self._cmp(other)
- if c == 0:
- c = self.compare_total(other)
-
- if c == -1:
- ans = self
- else:
- ans = other
-
- return ans._fix(context)
-
- def _isinteger(self):
- """Returns whether self is an integer"""
- if self._is_special:
- return False
- if self._exp >= 0:
- return True
- rest = self._int[self._exp:]
- return rest == '0'*len(rest)
-
- def _iseven(self):
- """Returns True if self is even. Assumes self is an integer."""
- if not self or self._exp > 0:
- return True
- return self._int[-1+self._exp] in '02468'
-
- def adjusted(self):
- """Return the adjusted exponent of self"""
- try:
- return self._exp + len(self._int) - 1
- # If NaN or Infinity, self._exp is string
- except TypeError:
- return 0
-
- def canonical(self):
- """Returns the same Decimal object.
-
- As we do not have different encodings for the same number, the
- received object already is in its canonical form.
- """
- return self
-
- def compare_signal(self, other, context=None):
- """Compares self to the other operand numerically.
-
- It's pretty much like compare(), but all NaNs signal, with signaling
- NaNs taking precedence over quiet NaNs.
- """
- other = _convert_other(other, raiseit = True)
- ans = self._compare_check_nans(other, context)
- if ans:
- return ans
- return self.compare(other, context=context)
-
- def compare_total(self, other, context=None):
- """Compares self to other using the abstract representations.
-
- This is not like the standard compare, which use their numerical
- value. Note that a total ordering is defined for all possible abstract
- representations.
- """
- other = _convert_other(other, raiseit=True)
-
- # if one is negative and the other is positive, it's easy
- if self._sign and not other._sign:
- return _NegativeOne
- if not self._sign and other._sign:
- return _One
- sign = self._sign
-
- # let's handle both NaN types
- self_nan = self._isnan()
- other_nan = other._isnan()
- if self_nan or other_nan:
- if self_nan == other_nan:
- # compare payloads as though they're integers
- self_key = len(self._int), self._int
- other_key = len(other._int), other._int
- if self_key < other_key:
- if sign:
- return _One
- else:
- return _NegativeOne
- if self_key > other_key:
- if sign:
- return _NegativeOne
- else:
- return _One
- return _Zero
-
- if sign:
- if self_nan == 1:
- return _NegativeOne
- if other_nan == 1:
- return _One
- if self_nan == 2:
- return _NegativeOne
- if other_nan == 2:
- return _One
- else:
- if self_nan == 1:
- return _One
- if other_nan == 1:
- return _NegativeOne
- if self_nan == 2:
- return _One
- if other_nan == 2:
- return _NegativeOne
-
- if self < other:
- return _NegativeOne
- if self > other:
- return _One
-
- if self._exp < other._exp:
- if sign:
- return _One
- else:
- return _NegativeOne
- if self._exp > other._exp:
- if sign:
- return _NegativeOne
- else:
- return _One
- return _Zero
-
-
- def compare_total_mag(self, other, context=None):
- """Compares self to other using abstract repr., ignoring sign.
-
- Like compare_total, but with operand's sign ignored and assumed to be 0.
- """
- other = _convert_other(other, raiseit=True)
-
- s = self.copy_abs()
- o = other.copy_abs()
- return s.compare_total(o)
-
- def copy_abs(self):
- """Returns a copy with the sign set to 0. """
- return _dec_from_triple(0, self._int, self._exp, self._is_special)
-
- def copy_negate(self):
- """Returns a copy with the sign inverted."""
- if self._sign:
- return _dec_from_triple(0, self._int, self._exp, self._is_special)
- else:
- return _dec_from_triple(1, self._int, self._exp, self._is_special)
-
- def copy_sign(self, other, context=None):
- """Returns self with the sign of other."""
- other = _convert_other(other, raiseit=True)
- return _dec_from_triple(other._sign, self._int,
- self._exp, self._is_special)
-
- def exp(self, context=None):
- """Returns e ** self."""
-
- if context is None:
- context = getcontext()
-
- # exp(NaN) = NaN
- ans = self._check_nans(context=context)
- if ans:
- return ans
-
- # exp(-Infinity) = 0
- if self._isinfinity() == -1:
- return _Zero
-
- # exp(0) = 1
- if not self:
- return _One
-
- # exp(Infinity) = Infinity
- if self._isinfinity() == 1:
- return Decimal(self)
-
- # the result is now guaranteed to be inexact (the true
- # mathematical result is transcendental). There's no need to
- # raise Rounded and Inexact here---they'll always be raised as
- # a result of the call to _fix.
- p = context.prec
- adj = self.adjusted()
-
- # we only need to do any computation for quite a small range
- # of adjusted exponents---for example, -29 <= adj <= 10 for
- # the default context. For smaller exponent the result is
- # indistinguishable from 1 at the given precision, while for
- # larger exponent the result either overflows or underflows.
- if self._sign == 0 and adj > len(str((context.Emax+1)*3)):
- # overflow
- ans = _dec_from_triple(0, '1', context.Emax+1)
- elif self._sign == 1 and adj > len(str((-context.Etiny()+1)*3)):
- # underflow to 0
- ans = _dec_from_triple(0, '1', context.Etiny()-1)
- elif self._sign == 0 and adj < -p:
- # p+1 digits; final round will raise correct flags
- ans = _dec_from_triple(0, '1' + '0'*(p-1) + '1', -p)
- elif self._sign == 1 and adj < -p-1:
- # p+1 digits; final round will raise correct flags
- ans = _dec_from_triple(0, '9'*(p+1), -p-1)
- # general case
- else:
- op = _WorkRep(self)
- c, e = op.int, op.exp
- if op.sign == 1:
- c = -c
-
- # compute correctly rounded result: increase precision by
- # 3 digits at a time until we get an unambiguously
- # roundable result
- extra = 3
- while True:
- coeff, exp = _dexp(c, e, p+extra)
- if coeff % (5*10**(len(str(coeff))-p-1)):
- break
- extra += 3
-
- ans = _dec_from_triple(0, str(coeff), exp)
-
- # at this stage, ans should round correctly with *any*
- # rounding mode, not just with ROUND_HALF_EVEN
- context = context._shallow_copy()
- rounding = context._set_rounding(ROUND_HALF_EVEN)
- ans = ans._fix(context)
- context.rounding = rounding
-
- return ans
-
- def is_canonical(self):
- """Return True if self is canonical; otherwise return False.
-
- Currently, the encoding of a Decimal instance is always
- canonical, so this method returns True for any Decimal.
- """
- return True
-
- def is_finite(self):
- """Return True if self is finite; otherwise return False.
-
- A Decimal instance is considered finite if it is neither
- infinite nor a NaN.
- """
- return not self._is_special
-
- def is_infinite(self):
- """Return True if self is infinite; otherwise return False."""
- return self._exp == 'F'
-
- def is_nan(self):
- """Return True if self is a qNaN or sNaN; otherwise return False."""
- return self._exp in ('n', 'N')
-
- def is_normal(self, context=None):
- """Return True if self is a normal number; otherwise return False."""
- if self._is_special or not self:
- return False
- if context is None:
- context = getcontext()
- return context.Emin <= self.adjusted()
-
- def is_qnan(self):
- """Return True if self is a quiet NaN; otherwise return False."""
- return self._exp == 'n'
-
- def is_signed(self):
- """Return True if self is negative; otherwise return False."""
- return self._sign == 1
-
- def is_snan(self):
- """Return True if self is a signaling NaN; otherwise return False."""
- return self._exp == 'N'
-
- def is_subnormal(self, context=None):
- """Return True if self is subnormal; otherwise return False."""
- if self._is_special or not self:
- return False
- if context is None:
- context = getcontext()
- return self.adjusted() < context.Emin
-
- def is_zero(self):
- """Return True if self is a zero; otherwise return False."""
- return not self._is_special and self._int == '0'
-
- def _ln_exp_bound(self):
- """Compute a lower bound for the adjusted exponent of self.ln().
- In other words, compute r such that self.ln() >= 10**r. Assumes
- that self is finite and positive and that self != 1.
- """
-
- # for 0.1 <= x <= 10 we use the inequalities 1-1/x <= ln(x) <= x-1
- adj = self._exp + len(self._int) - 1
- if adj >= 1:
- # argument >= 10; we use 23/10 = 2.3 as a lower bound for ln(10)
- return len(str(adj*23//10)) - 1
- if adj <= -2:
- # argument <= 0.1
- return len(str((-1-adj)*23//10)) - 1
- op = _WorkRep(self)
- c, e = op.int, op.exp
- if adj == 0:
- # 1 < self < 10
- num = str(c-10**-e)
- den = str(c)
- return len(num) - len(den) - (num < den)
- # adj == -1, 0.1 <= self < 1
- return e + len(str(10**-e - c)) - 1
-
-
- def ln(self, context=None):
- """Returns the natural (base e) logarithm of self."""
-
- if context is None:
- context = getcontext()
-
- # ln(NaN) = NaN
- ans = self._check_nans(context=context)
- if ans:
- return ans
-
- # ln(0.0) == -Infinity
- if not self:
- return _NegativeInfinity
-
- # ln(Infinity) = Infinity
- if self._isinfinity() == 1:
- return _Infinity
-
- # ln(1.0) == 0.0
- if self == _One:
- return _Zero
-
- # ln(negative) raises InvalidOperation
- if self._sign == 1:
- return context._raise_error(InvalidOperation,
- 'ln of a negative value')
-
- # result is irrational, so necessarily inexact
- op = _WorkRep(self)
- c, e = op.int, op.exp
- p = context.prec
-
- # correctly rounded result: repeatedly increase precision by 3
- # until we get an unambiguously roundable result
- places = p - self._ln_exp_bound() + 2 # at least p+3 places
- while True:
- coeff = _dlog(c, e, places)
- # assert len(str(abs(coeff)))-p >= 1
- if coeff % (5*10**(len(str(abs(coeff)))-p-1)):
- break
- places += 3
- ans = _dec_from_triple(int(coeff<0), str(abs(coeff)), -places)
-
- context = context._shallow_copy()
- rounding = context._set_rounding(ROUND_HALF_EVEN)
- ans = ans._fix(context)
- context.rounding = rounding
- return ans
-
- def _log10_exp_bound(self):
- """Compute a lower bound for the adjusted exponent of self.log10().
- In other words, find r such that self.log10() >= 10**r.
- Assumes that self is finite and positive and that self != 1.
- """
-
- # For x >= 10 or x < 0.1 we only need a bound on the integer
- # part of log10(self), and this comes directly from the
- # exponent of x. For 0.1 <= x <= 10 we use the inequalities
- # 1-1/x <= log(x) <= x-1. If x > 1 we have |log10(x)| >
- # (1-1/x)/2.31 > 0. If x < 1 then |log10(x)| > (1-x)/2.31 > 0
-
- adj = self._exp + len(self._int) - 1
- if adj >= 1:
- # self >= 10
- return len(str(adj))-1
- if adj <= -2:
- # self < 0.1
- return len(str(-1-adj))-1
- op = _WorkRep(self)
- c, e = op.int, op.exp
- if adj == 0:
- # 1 < self < 10
- num = str(c-10**-e)
- den = str(231*c)
- return len(num) - len(den) - (num < den) + 2
- # adj == -1, 0.1 <= self < 1
- num = str(10**-e-c)
- return len(num) + e - (num < "231") - 1
-
- def log10(self, context=None):
- """Returns the base 10 logarithm of self."""
-
- if context is None:
- context = getcontext()
-
- # log10(NaN) = NaN
- ans = self._check_nans(context=context)
- if ans:
- return ans
-
- # log10(0.0) == -Infinity
- if not self:
- return _NegativeInfinity
-
- # log10(Infinity) = Infinity
- if self._isinfinity() == 1:
- return _Infinity
-
- # log10(negative or -Infinity) raises InvalidOperation
- if self._sign == 1:
- return context._raise_error(InvalidOperation,
- 'log10 of a negative value')
-
- # log10(10**n) = n
- if self._int[0] == '1' and self._int[1:] == '0'*(len(self._int) - 1):
- # answer may need rounding
- ans = Decimal(self._exp + len(self._int) - 1)
- else:
- # result is irrational, so necessarily inexact
- op = _WorkRep(self)
- c, e = op.int, op.exp
- p = context.prec
-
- # correctly rounded result: repeatedly increase precision
- # until result is unambiguously roundable
- places = p-self._log10_exp_bound()+2
- while True:
- coeff = _dlog10(c, e, places)
- # assert len(str(abs(coeff)))-p >= 1
- if coeff % (5*10**(len(str(abs(coeff)))-p-1)):
- break
- places += 3
- ans = _dec_from_triple(int(coeff<0), str(abs(coeff)), -places)
-
- context = context._shallow_copy()
- rounding = context._set_rounding(ROUND_HALF_EVEN)
- ans = ans._fix(context)
- context.rounding = rounding
- return ans
-
- def logb(self, context=None):
- """ Returns the exponent of the magnitude of self's MSD.
-
- The result is the integer which is the exponent of the magnitude
- of the most significant digit of self (as though it were truncated
- to a single digit while maintaining the value of that digit and
- without limiting the resulting exponent).
- """
- # logb(NaN) = NaN
- ans = self._check_nans(context=context)
- if ans:
- return ans
-
- if context is None:
- context = getcontext()
-
- # logb(+/-Inf) = +Inf
- if self._isinfinity():
- return _Infinity
-
- # logb(0) = -Inf, DivisionByZero
- if not self:
- return context._raise_error(DivisionByZero, 'logb(0)', 1)
-
- # otherwise, simply return the adjusted exponent of self, as a
- # Decimal. Note that no attempt is made to fit the result
- # into the current context.
- ans = Decimal(self.adjusted())
- return ans._fix(context)
-
- def _islogical(self):
- """Return True if self is a logical operand.
-
- For being logical, it must be a finite number with a sign of 0,
- an exponent of 0, and a coefficient whose digits must all be
- either 0 or 1.
- """
- if self._sign != 0 or self._exp != 0:
- return False
- for dig in self._int:
- if dig not in '01':
- return False
- return True
-
- def _fill_logical(self, context, opa, opb):
- dif = context.prec - len(opa)
- if dif > 0:
- opa = '0'*dif + opa
- elif dif < 0:
- opa = opa[-context.prec:]
- dif = context.prec - len(opb)
- if dif > 0:
- opb = '0'*dif + opb
- elif dif < 0:
- opb = opb[-context.prec:]
- return opa, opb
-
- def logical_and(self, other, context=None):
- """Applies an 'and' operation between self and other's digits."""
- if context is None:
- context = getcontext()
-
- other = _convert_other(other, raiseit=True)
-
- if not self._islogical() or not other._islogical():
- return context._raise_error(InvalidOperation)
-
- # fill to context.prec
- (opa, opb) = self._fill_logical(context, self._int, other._int)
-
- # make the operation, and clean starting zeroes
- result = "".join([str(int(a)&int(b)) for a,b in zip(opa,opb)])
- return _dec_from_triple(0, result.lstrip('0') or '0', 0)
-
- def logical_invert(self, context=None):
- """Invert all its digits."""
- if context is None:
- context = getcontext()
- return self.logical_xor(_dec_from_triple(0,'1'*context.prec,0),
- context)
-
- def logical_or(self, other, context=None):
- """Applies an 'or' operation between self and other's digits."""
- if context is None:
- context = getcontext()
-
- other = _convert_other(other, raiseit=True)
-
- if not self._islogical() or not other._islogical():
- return context._raise_error(InvalidOperation)
-
- # fill to context.prec
- (opa, opb) = self._fill_logical(context, self._int, other._int)
-
- # make the operation, and clean starting zeroes
- result = "".join([str(int(a)|int(b)) for a,b in zip(opa,opb)])
- return _dec_from_triple(0, result.lstrip('0') or '0', 0)
-
- def logical_xor(self, other, context=None):
- """Applies an 'xor' operation between self and other's digits."""
- if context is None:
- context = getcontext()
-
- other = _convert_other(other, raiseit=True)
-
- if not self._islogical() or not other._islogical():
- return context._raise_error(InvalidOperation)
-
- # fill to context.prec
- (opa, opb) = self._fill_logical(context, self._int, other._int)
-
- # make the operation, and clean starting zeroes
- result = "".join([str(int(a)^int(b)) for a,b in zip(opa,opb)])
- return _dec_from_triple(0, result.lstrip('0') or '0', 0)
-
- def max_mag(self, other, context=None):
- """Compares the values numerically with their sign ignored."""
- other = _convert_other(other, raiseit=True)
-
- if context is None:
- context = getcontext()
-
- if self._is_special or other._is_special:
- # If one operand is a quiet NaN and the other is number, then the
- # number is always returned
- sn = self._isnan()
- on = other._isnan()
- if sn or on:
- if on == 1 and sn == 0:
- return self._fix(context)
- if sn == 1 and on == 0:
- return other._fix(context)
- return self._check_nans(other, context)
-
- c = self.copy_abs()._cmp(other.copy_abs())
- if c == 0:
- c = self.compare_total(other)
-
- if c == -1:
- ans = other
- else:
- ans = self
-
- return ans._fix(context)
-
- def min_mag(self, other, context=None):
- """Compares the values numerically with their sign ignored."""
- other = _convert_other(other, raiseit=True)
-
- if context is None:
- context = getcontext()
-
- if self._is_special or other._is_special:
- # If one operand is a quiet NaN and the other is number, then the
- # number is always returned
- sn = self._isnan()
- on = other._isnan()
- if sn or on:
- if on == 1 and sn == 0:
- return self._fix(context)
- if sn == 1 and on == 0:
- return other._fix(context)
- return self._check_nans(other, context)
-
- c = self.copy_abs()._cmp(other.copy_abs())
- if c == 0:
- c = self.compare_total(other)
-
- if c == -1:
- ans = self
- else:
- ans = other
-
- return ans._fix(context)
-
- def next_minus(self, context=None):
- """Returns the largest representable number smaller than itself."""
- if context is None:
- context = getcontext()
-
- ans = self._check_nans(context=context)
- if ans:
- return ans
-
- if self._isinfinity() == -1:
- return _NegativeInfinity
- if self._isinfinity() == 1:
- return _dec_from_triple(0, '9'*context.prec, context.Etop())
-
- context = context.copy()
- context._set_rounding(ROUND_FLOOR)
- context._ignore_all_flags()
- new_self = self._fix(context)
- if new_self != self:
- return new_self
- return self.__sub__(_dec_from_triple(0, '1', context.Etiny()-1),
- context)
-
- def next_plus(self, context=None):
- """Returns the smallest representable number larger than itself."""
- if context is None:
- context = getcontext()
-
- ans = self._check_nans(context=context)
- if ans:
- return ans
-
- if self._isinfinity() == 1:
- return _Infinity
- if self._isinfinity() == -1:
- return _dec_from_triple(1, '9'*context.prec, context.Etop())
-
- context = context.copy()
- context._set_rounding(ROUND_CEILING)
- context._ignore_all_flags()
- new_self = self._fix(context)
- if new_self != self:
- return new_self
- return self.__add__(_dec_from_triple(0, '1', context.Etiny()-1),
- context)
-
- def next_toward(self, other, context=None):
- """Returns the number closest to self, in the direction towards other.
-
- The result is the closest representable number to self
- (excluding self) that is in the direction towards other,
- unless both have the same value. If the two operands are
- numerically equal, then the result is a copy of self with the
- sign set to be the same as the sign of other.
- """
- other = _convert_other(other, raiseit=True)
-
- if context is None:
- context = getcontext()
-
- ans = self._check_nans(other, context)
- if ans:
- return ans
-
- comparison = self._cmp(other)
- if comparison == 0:
- return self.copy_sign(other)
-
- if comparison == -1:
- ans = self.next_plus(context)
- else: # comparison == 1
- ans = self.next_minus(context)
-
- # decide which flags to raise using value of ans
- if ans._isinfinity():
- context._raise_error(Overflow,
- 'Infinite result from next_toward',
- ans._sign)
- context._raise_error(Inexact)
- context._raise_error(Rounded)
- elif ans.adjusted() < context.Emin:
- context._raise_error(Underflow)
- context._raise_error(Subnormal)
- context._raise_error(Inexact)
- context._raise_error(Rounded)
- # if precision == 1 then we don't raise Clamped for a
- # result 0E-Etiny.
- if not ans:
- context._raise_error(Clamped)
-
- return ans
-
- def number_class(self, context=None):
- """Returns an indication of the class of self.
-
- The class is one of the following strings:
- sNaN
- NaN
- -Infinity
- -Normal
- -Subnormal
- -Zero
- +Zero
- +Subnormal
- +Normal
- +Infinity
- """
- if self.is_snan():
- return "sNaN"
- if self.is_qnan():
- return "NaN"
- inf = self._isinfinity()
- if inf == 1:
- return "+Infinity"
- if inf == -1:
- return "-Infinity"
- if self.is_zero():
- if self._sign:
- return "-Zero"
- else:
- return "+Zero"
- if context is None:
- context = getcontext()
- if self.is_subnormal(context=context):
- if self._sign:
- return "-Subnormal"
- else:
- return "+Subnormal"
- # just a normal, regular, boring number, :)
- if self._sign:
- return "-Normal"
- else:
- return "+Normal"
-
- def radix(self):
- """Just returns 10, as this is Decimal, :)"""
- return Decimal(10)
-
- def rotate(self, other, context=None):
- """Returns a rotated copy of self, value-of-other times."""
- if context is None:
- context = getcontext()
-
- other = _convert_other(other, raiseit=True)
-
- ans = self._check_nans(other, context)
- if ans:
- return ans
-
- if other._exp != 0:
- return context._raise_error(InvalidOperation)
- if not (-context.prec <= int(other) <= context.prec):
- return context._raise_error(InvalidOperation)
-
- if self._isinfinity():
- return Decimal(self)
-
- # get values, pad if necessary
- torot = int(other)
- rotdig = self._int
- topad = context.prec - len(rotdig)
- if topad > 0:
- rotdig = '0'*topad + rotdig
- elif topad < 0:
- rotdig = rotdig[-topad:]
-
- # let's rotate!
- rotated = rotdig[torot:] + rotdig[:torot]
- return _dec_from_triple(self._sign,
- rotated.lstrip('0') or '0', self._exp)
-
- def scaleb(self, other, context=None):
- """Returns self operand after adding the second value to its exp."""
- if context is None:
- context = getcontext()
-
- other = _convert_other(other, raiseit=True)
-
- ans = self._check_nans(other, context)
- if ans:
- return ans
-
- if other._exp != 0:
- return context._raise_error(InvalidOperation)
- liminf = -2 * (context.Emax + context.prec)
- limsup = 2 * (context.Emax + context.prec)
- if not (liminf <= int(other) <= limsup):
- return context._raise_error(InvalidOperation)
-
- if self._isinfinity():
- return Decimal(self)
-
- d = _dec_from_triple(self._sign, self._int, self._exp + int(other))
- d = d._fix(context)
- return d
-
- def shift(self, other, context=None):
- """Returns a shifted copy of self, value-of-other times."""
- if context is None:
- context = getcontext()
-
- other = _convert_other(other, raiseit=True)
-
- ans = self._check_nans(other, context)
- if ans:
- return ans
-
- if other._exp != 0:
- return context._raise_error(InvalidOperation)
- if not (-context.prec <= int(other) <= context.prec):
- return context._raise_error(InvalidOperation)
-
- if self._isinfinity():
- return Decimal(self)
-
- # get values, pad if necessary
- torot = int(other)
- rotdig = self._int
- topad = context.prec - len(rotdig)
- if topad > 0:
- rotdig = '0'*topad + rotdig
- elif topad < 0:
- rotdig = rotdig[-topad:]
-
- # let's shift!
- if torot < 0:
- shifted = rotdig[:torot]
- else:
- shifted = rotdig + '0'*torot
- shifted = shifted[-context.prec:]
-
- return _dec_from_triple(self._sign,
- shifted.lstrip('0') or '0', self._exp)
-
- # Support for pickling, copy, and deepcopy
- def __reduce__(self):
- return (self.__class__, (str(self),))
-
- def __copy__(self):
- if type(self) is Decimal:
- return self # I'm immutable; therefore I am my own clone
- return self.__class__(str(self))
-
- def __deepcopy__(self, memo):
- if type(self) is Decimal:
- return self # My components are also immutable
- return self.__class__(str(self))
-
- # PEP 3101 support. the _localeconv keyword argument should be
- # considered private: it's provided for ease of testing only.
- def __format__(self, specifier, context=None, _localeconv=None):
- """Format a Decimal instance according to the given specifier.
-
- The specifier should be a standard format specifier, with the
- form described in PEP 3101. Formatting types 'e', 'E', 'f',
- 'F', 'g', 'G', 'n' and '%' are supported. If the formatting
- type is omitted it defaults to 'g' or 'G', depending on the
- value of context.capitals.
- """
-
- # Note: PEP 3101 says that if the type is not present then
- # there should be at least one digit after the decimal point.
- # We take the liberty of ignoring this requirement for
- # Decimal---it's presumably there to make sure that
- # format(float, '') behaves similarly to str(float).
- if context is None:
- context = getcontext()
-
- spec = _parse_format_specifier(specifier, _localeconv=_localeconv)
-
- # special values don't care about the type or precision
- if self._is_special:
- sign = _format_sign(self._sign, spec)
- body = str(self.copy_abs())
- if spec['type'] == '%':
- body += '%'
- return _format_align(sign, body, spec)
-
- # a type of None defaults to 'g' or 'G', depending on context
- if spec['type'] is None:
- spec['type'] = ['g', 'G'][context.capitals]
-
- # if type is '%', adjust exponent of self accordingly
- if spec['type'] == '%':
- self = _dec_from_triple(self._sign, self._int, self._exp+2)
-
- # round if necessary, taking rounding mode from the context
- rounding = context.rounding
- precision = spec['precision']
- if precision is not None:
- if spec['type'] in 'eE':
- self = self._round(precision+1, rounding)
- elif spec['type'] in 'fF%':
- self = self._rescale(-precision, rounding)
- elif spec['type'] in 'gG' and len(self._int) > precision:
- self = self._round(precision, rounding)
- # special case: zeros with a positive exponent can't be
- # represented in fixed point; rescale them to 0e0.
- if not self and self._exp > 0 and spec['type'] in 'fF%':
- self = self._rescale(0, rounding)
-
- # figure out placement of the decimal point
- leftdigits = self._exp + len(self._int)
- if spec['type'] in 'eE':
- if not self and precision is not None:
- dotplace = 1 - precision
- else:
- dotplace = 1
- elif spec['type'] in 'fF%':
- dotplace = leftdigits
- elif spec['type'] in 'gG':
- if self._exp <= 0 and leftdigits > -6:
- dotplace = leftdigits
- else:
- dotplace = 1
-
- # find digits before and after decimal point, and get exponent
- if dotplace < 0:
- intpart = '0'
- fracpart = '0'*(-dotplace) + self._int
- elif dotplace > len(self._int):
- intpart = self._int + '0'*(dotplace-len(self._int))
- fracpart = ''
- else:
- intpart = self._int[:dotplace] or '0'
- fracpart = self._int[dotplace:]
- exp = leftdigits-dotplace
-
- # done with the decimal-specific stuff; hand over the rest
- # of the formatting to the _format_number function
- return _format_number(self._sign, intpart, fracpart, exp, spec)
-
-def _dec_from_triple(sign, coefficient, exponent, special=False):
- """Create a decimal instance directly, without any validation,
- normalization (e.g. removal of leading zeros) or argument
- conversion.
-
- This function is for *internal use only*.
- """
-
- self = object.__new__(Decimal)
- self._sign = sign
- self._int = coefficient
- self._exp = exponent
- self._is_special = special
-
- return self
-
-# Register Decimal as a kind of Number (an abstract base class).
-# However, do not register it as Real (because Decimals are not
-# interoperable with floats).
-_numbers.Number.register(Decimal)
-
-
-##### Context class #######################################################
-
-class _ContextManager(object):
- """Context manager class to support localcontext().
-
- Sets a copy of the supplied context in __enter__() and restores
- the previous decimal context in __exit__()
- """
- def __init__(self, new_context):
- self.new_context = new_context.copy()
- def __enter__(self):
- self.saved_context = getcontext()
- setcontext(self.new_context)
- return self.new_context
- def __exit__(self, t, v, tb):
- setcontext(self.saved_context)
-
-class Context(object):
- """Contains the context for a Decimal instance.
-
- Contains:
- prec - precision (for use in rounding, division, square roots..)
- rounding - rounding type (how you round)
- traps - If traps[exception] = 1, then the exception is
- raised when it is caused. Otherwise, a value is
- substituted in.
- flags - When an exception is caused, flags[exception] is set.
- (Whether or not the trap_enabler is set)
- Should be reset by user of Decimal instance.
- Emin - Minimum exponent
- Emax - Maximum exponent
- capitals - If 1, 1*10^1 is printed as 1E+1.
- If 0, printed as 1e1
- clamp - If 1, change exponents if too high (Default 0)
- """
-
- def __init__(self, prec=None, rounding=None, Emin=None, Emax=None,
- capitals=None, clamp=None, flags=None, traps=None,
- _ignored_flags=None):
- # Set defaults; for everything except flags and _ignored_flags,
- # inherit from DefaultContext.
- try:
- dc = DefaultContext
- except NameError:
- pass
-
- self.prec = prec if prec is not None else dc.prec
- self.rounding = rounding if rounding is not None else dc.rounding
- self.Emin = Emin if Emin is not None else dc.Emin
- self.Emax = Emax if Emax is not None else dc.Emax
- self.capitals = capitals if capitals is not None else dc.capitals
- self.clamp = clamp if clamp is not None else dc.clamp
-
- if _ignored_flags is None:
- self._ignored_flags = []
- else:
- self._ignored_flags = _ignored_flags
-
- if traps is None:
- self.traps = dc.traps.copy()
- elif not isinstance(traps, dict):
- self.traps = dict((s, int(s in traps)) for s in _signals + traps)
- else:
- self.traps = traps
-
- if flags is None:
- self.flags = dict.fromkeys(_signals, 0)
- elif not isinstance(flags, dict):
- self.flags = dict((s, int(s in flags)) for s in _signals + flags)
- else:
- self.flags = flags
-
- def _set_integer_check(self, name, value, vmin, vmax):
- if not isinstance(value, int):
- raise TypeError("%s must be an integer" % name)
- if vmin == '-inf':
- if value > vmax:
- raise ValueError("%s must be in [%s, %d]. got: %s" % (name, vmin, vmax, value))
- elif vmax == 'inf':
- if value < vmin:
- raise ValueError("%s must be in [%d, %s]. got: %s" % (name, vmin, vmax, value))
- else:
- if value < vmin or value > vmax:
- raise ValueError("%s must be in [%d, %d]. got %s" % (name, vmin, vmax, value))
- return object.__setattr__(self, name, value)
-
- def _set_signal_dict(self, name, d):
- if not isinstance(d, dict):
- raise TypeError("%s must be a signal dict" % d)
- for key in d:
- if not key in _signals:
- raise KeyError("%s is not a valid signal dict" % d)
- for key in _signals:
- if not key in d:
- raise KeyError("%s is not a valid signal dict" % d)
- return object.__setattr__(self, name, d)
-
- def __setattr__(self, name, value):
- if name == 'prec':
- return self._set_integer_check(name, value, 1, 'inf')
- elif name == 'Emin':
- return self._set_integer_check(name, value, '-inf', 0)
- elif name == 'Emax':
- return self._set_integer_check(name, value, 0, 'inf')
- elif name == 'capitals':
- return self._set_integer_check(name, value, 0, 1)
- elif name == 'clamp':
- return self._set_integer_check(name, value, 0, 1)
- elif name == 'rounding':
- if not value in _rounding_modes:
- # raise TypeError even for strings to have consistency
- # among various implementations.
- raise TypeError("%s: invalid rounding mode" % value)
- return object.__setattr__(self, name, value)
- elif name == 'flags' or name == 'traps':
- return self._set_signal_dict(name, value)
- elif name == '_ignored_flags':
- return object.__setattr__(self, name, value)
- else:
- raise AttributeError(
- "'decimal.Context' object has no attribute '%s'" % name)
-
- def __delattr__(self, name):
- raise AttributeError("%s cannot be deleted" % name)
-
- # Support for pickling, copy, and deepcopy
- def __reduce__(self):
- flags = [sig for sig, v in self.flags.items() if v]
- traps = [sig for sig, v in self.traps.items() if v]
- return (self.__class__,
- (self.prec, self.rounding, self.Emin, self.Emax,
- self.capitals, self.clamp, flags, traps))
-
- def __repr__(self):
- """Show the current context."""
- s = []
- s.append('Context(prec=%(prec)d, rounding=%(rounding)s, '
- 'Emin=%(Emin)d, Emax=%(Emax)d, capitals=%(capitals)d, '
- 'clamp=%(clamp)d'
- % vars(self))
- names = [f.__name__ for f, v in self.flags.items() if v]
- s.append('flags=[' + ', '.join(names) + ']')
- names = [t.__name__ for t, v in self.traps.items() if v]
- s.append('traps=[' + ', '.join(names) + ']')
- return ', '.join(s) + ')'
-
- def clear_flags(self):
- """Reset all flags to zero"""
- for flag in self.flags:
- self.flags[flag] = 0
-
- def clear_traps(self):
- """Reset all traps to zero"""
- for flag in self.traps:
- self.traps[flag] = 0
-
- def _shallow_copy(self):
- """Returns a shallow copy from self."""
- nc = Context(self.prec, self.rounding, self.Emin, self.Emax,
- self.capitals, self.clamp, self.flags, self.traps,
- self._ignored_flags)
- return nc
-
- def copy(self):
- """Returns a deep copy from self."""
- nc = Context(self.prec, self.rounding, self.Emin, self.Emax,
- self.capitals, self.clamp,
- self.flags.copy(), self.traps.copy(),
- self._ignored_flags)
- return nc
- __copy__ = copy
-
- def _raise_error(self, condition, explanation = None, *args):
- """Handles an error
-
- If the flag is in _ignored_flags, returns the default response.
- Otherwise, it sets the flag, then, if the corresponding
- trap_enabler is set, it reraises the exception. Otherwise, it returns
- the default value after setting the flag.
- """
- error = _condition_map.get(condition, condition)
- if error in self._ignored_flags:
- # Don't touch the flag
- return error().handle(self, *args)
-
- self.flags[error] = 1
- if not self.traps[error]:
- # The errors define how to handle themselves.
- return condition().handle(self, *args)
-
- # Errors should only be risked on copies of the context
- # self._ignored_flags = []
- raise error(explanation)
-
- def _ignore_all_flags(self):
- """Ignore all flags, if they are raised"""
- return self._ignore_flags(*_signals)
-
- def _ignore_flags(self, *flags):
- """Ignore the flags, if they are raised"""
- # Do not mutate-- This way, copies of a context leave the original
- # alone.
- self._ignored_flags = (self._ignored_flags + list(flags))
- return list(flags)
-
- def _regard_flags(self, *flags):
- """Stop ignoring the flags, if they are raised"""
- if flags and isinstance(flags[0], (tuple,list)):
- flags = flags[0]
- for flag in flags:
- self._ignored_flags.remove(flag)
-
- # We inherit object.__hash__, so we must deny this explicitly
- __hash__ = None
-
- def Etiny(self):
- """Returns Etiny (= Emin - prec + 1)"""
- return int(self.Emin - self.prec + 1)
-
- def Etop(self):
- """Returns maximum exponent (= Emax - prec + 1)"""
- return int(self.Emax - self.prec + 1)
-
- def _set_rounding(self, type):
- """Sets the rounding type.
-
- Sets the rounding type, and returns the current (previous)
- rounding type. Often used like:
-
- context = context.copy()
- # so you don't change the calling context
- # if an error occurs in the middle.
- rounding = context._set_rounding(ROUND_UP)
- val = self.__sub__(other, context=context)
- context._set_rounding(rounding)
-
- This will make it round up for that operation.
- """
- rounding = self.rounding
- self.rounding= type
- return rounding
-
- def create_decimal(self, num='0'):
- """Creates a new Decimal instance but using self as context.
-
- This method implements the to-number operation of the
- IBM Decimal specification."""
-
- if isinstance(num, str) and num != num.strip():
- return self._raise_error(ConversionSyntax,
- "no trailing or leading whitespace is "
- "permitted.")
-
- d = Decimal(num, context=self)
- if d._isnan() and len(d._int) > self.prec - self.clamp:
- return self._raise_error(ConversionSyntax,
- "diagnostic info too long in NaN")
- return d._fix(self)
-
- def create_decimal_from_float(self, f):
- """Creates a new Decimal instance from a float but rounding using self
- as the context.
-
- >>> context = Context(prec=5, rounding=ROUND_DOWN)
- >>> context.create_decimal_from_float(3.1415926535897932)
- Decimal('3.1415')
- >>> context = Context(prec=5, traps=[Inexact])
- >>> context.create_decimal_from_float(3.1415926535897932)
- Traceback (most recent call last):
- ...
- decimal.Inexact: None
-
- """
- d = Decimal.from_float(f) # An exact conversion
- return d._fix(self) # Apply the context rounding
-
- # Methods
- def abs(self, a):
- """Returns the absolute value of the operand.
-
- If the operand is negative, the result is the same as using the minus
- operation on the operand. Otherwise, the result is the same as using
- the plus operation on the operand.
-
- >>> ExtendedContext.abs(Decimal('2.1'))
- Decimal('2.1')
- >>> ExtendedContext.abs(Decimal('-100'))
- Decimal('100')
- >>> ExtendedContext.abs(Decimal('101.5'))
- Decimal('101.5')
- >>> ExtendedContext.abs(Decimal('-101.5'))
- Decimal('101.5')
- >>> ExtendedContext.abs(-1)
- Decimal('1')
- """
- a = _convert_other(a, raiseit=True)
- return a.__abs__(context=self)
-
- def add(self, a, b):
- """Return the sum of the two operands.
-
- >>> ExtendedContext.add(Decimal('12'), Decimal('7.00'))
- Decimal('19.00')
- >>> ExtendedContext.add(Decimal('1E+2'), Decimal('1.01E+4'))
- Decimal('1.02E+4')
- >>> ExtendedContext.add(1, Decimal(2))
- Decimal('3')
- >>> ExtendedContext.add(Decimal(8), 5)
- Decimal('13')
- >>> ExtendedContext.add(5, 5)
- Decimal('10')
- """
- a = _convert_other(a, raiseit=True)
- r = a.__add__(b, context=self)
- if r is NotImplemented:
- raise TypeError("Unable to convert %s to Decimal" % b)
- else:
- return r
-
- def _apply(self, a):
- return str(a._fix(self))
-
- def canonical(self, a):
- """Returns the same Decimal object.
-
- As we do not have different encodings for the same number, the
- received object already is in its canonical form.
-
- >>> ExtendedContext.canonical(Decimal('2.50'))
- Decimal('2.50')
- """
- if not isinstance(a, Decimal):
- raise TypeError("canonical requires a Decimal as an argument.")
- return a.canonical()
-
- def compare(self, a, b):
- """Compares values numerically.
-
- If the signs of the operands differ, a value representing each operand
- ('-1' if the operand is less than zero, '0' if the operand is zero or
- negative zero, or '1' if the operand is greater than zero) is used in
- place of that operand for the comparison instead of the actual
- operand.
-
- The comparison is then effected by subtracting the second operand from
- the first and then returning a value according to the result of the
- subtraction: '-1' if the result is less than zero, '0' if the result is
- zero or negative zero, or '1' if the result is greater than zero.
-
- >>> ExtendedContext.compare(Decimal('2.1'), Decimal('3'))
- Decimal('-1')
- >>> ExtendedContext.compare(Decimal('2.1'), Decimal('2.1'))
- Decimal('0')
- >>> ExtendedContext.compare(Decimal('2.1'), Decimal('2.10'))
- Decimal('0')
- >>> ExtendedContext.compare(Decimal('3'), Decimal('2.1'))
- Decimal('1')
- >>> ExtendedContext.compare(Decimal('2.1'), Decimal('-3'))
- Decimal('1')
- >>> ExtendedContext.compare(Decimal('-3'), Decimal('2.1'))
- Decimal('-1')
- >>> ExtendedContext.compare(1, 2)
- Decimal('-1')
- >>> ExtendedContext.compare(Decimal(1), 2)
- Decimal('-1')
- >>> ExtendedContext.compare(1, Decimal(2))
- Decimal('-1')
- """
- a = _convert_other(a, raiseit=True)
- return a.compare(b, context=self)
-
- def compare_signal(self, a, b):
- """Compares the values of the two operands numerically.
-
- It's pretty much like compare(), but all NaNs signal, with signaling
- NaNs taking precedence over quiet NaNs.
-
- >>> c = ExtendedContext
- >>> c.compare_signal(Decimal('2.1'), Decimal('3'))
- Decimal('-1')
- >>> c.compare_signal(Decimal('2.1'), Decimal('2.1'))
- Decimal('0')
- >>> c.flags[InvalidOperation] = 0
- >>> print(c.flags[InvalidOperation])
- 0
- >>> c.compare_signal(Decimal('NaN'), Decimal('2.1'))
- Decimal('NaN')
- >>> print(c.flags[InvalidOperation])
- 1
- >>> c.flags[InvalidOperation] = 0
- >>> print(c.flags[InvalidOperation])
- 0
- >>> c.compare_signal(Decimal('sNaN'), Decimal('2.1'))
- Decimal('NaN')
- >>> print(c.flags[InvalidOperation])
- 1
- >>> c.compare_signal(-1, 2)
- Decimal('-1')
- >>> c.compare_signal(Decimal(-1), 2)
- Decimal('-1')
- >>> c.compare_signal(-1, Decimal(2))
- Decimal('-1')
- """
- a = _convert_other(a, raiseit=True)
- return a.compare_signal(b, context=self)
-
- def compare_total(self, a, b):
- """Compares two operands using their abstract representation.
-
- This is not like the standard compare, which use their numerical
- value. Note that a total ordering is defined for all possible abstract
- representations.
-
- >>> ExtendedContext.compare_total(Decimal('12.73'), Decimal('127.9'))
- Decimal('-1')
- >>> ExtendedContext.compare_total(Decimal('-127'), Decimal('12'))
- Decimal('-1')
- >>> ExtendedContext.compare_total(Decimal('12.30'), Decimal('12.3'))
- Decimal('-1')
- >>> ExtendedContext.compare_total(Decimal('12.30'), Decimal('12.30'))
- Decimal('0')
- >>> ExtendedContext.compare_total(Decimal('12.3'), Decimal('12.300'))
- Decimal('1')
- >>> ExtendedContext.compare_total(Decimal('12.3'), Decimal('NaN'))
- Decimal('-1')
- >>> ExtendedContext.compare_total(1, 2)
- Decimal('-1')
- >>> ExtendedContext.compare_total(Decimal(1), 2)
- Decimal('-1')
- >>> ExtendedContext.compare_total(1, Decimal(2))
- Decimal('-1')
- """
- a = _convert_other(a, raiseit=True)
- return a.compare_total(b)
-
- def compare_total_mag(self, a, b):
- """Compares two operands using their abstract representation ignoring sign.
-
- Like compare_total, but with operand's sign ignored and assumed to be 0.
- """
- a = _convert_other(a, raiseit=True)
- return a.compare_total_mag(b)
-
- def copy_abs(self, a):
- """Returns a copy of the operand with the sign set to 0.
-
- >>> ExtendedContext.copy_abs(Decimal('2.1'))
- Decimal('2.1')
- >>> ExtendedContext.copy_abs(Decimal('-100'))
- Decimal('100')
- >>> ExtendedContext.copy_abs(-1)
- Decimal('1')
- """
- a = _convert_other(a, raiseit=True)
- return a.copy_abs()
-
- def copy_decimal(self, a):
- """Returns a copy of the decimal object.
-
- >>> ExtendedContext.copy_decimal(Decimal('2.1'))
- Decimal('2.1')
- >>> ExtendedContext.copy_decimal(Decimal('-1.00'))
- Decimal('-1.00')
- >>> ExtendedContext.copy_decimal(1)
- Decimal('1')
- """
- a = _convert_other(a, raiseit=True)
- return Decimal(a)
-
- def copy_negate(self, a):
- """Returns a copy of the operand with the sign inverted.
-
- >>> ExtendedContext.copy_negate(Decimal('101.5'))
- Decimal('-101.5')
- >>> ExtendedContext.copy_negate(Decimal('-101.5'))
- Decimal('101.5')
- >>> ExtendedContext.copy_negate(1)
- Decimal('-1')
- """
- a = _convert_other(a, raiseit=True)
- return a.copy_negate()
-
- def copy_sign(self, a, b):
- """Copies the second operand's sign to the first one.
-
- In detail, it returns a copy of the first operand with the sign
- equal to the sign of the second operand.
-
- >>> ExtendedContext.copy_sign(Decimal( '1.50'), Decimal('7.33'))
- Decimal('1.50')
- >>> ExtendedContext.copy_sign(Decimal('-1.50'), Decimal('7.33'))
- Decimal('1.50')
- >>> ExtendedContext.copy_sign(Decimal( '1.50'), Decimal('-7.33'))
- Decimal('-1.50')
- >>> ExtendedContext.copy_sign(Decimal('-1.50'), Decimal('-7.33'))
- Decimal('-1.50')
- >>> ExtendedContext.copy_sign(1, -2)
- Decimal('-1')
- >>> ExtendedContext.copy_sign(Decimal(1), -2)
- Decimal('-1')
- >>> ExtendedContext.copy_sign(1, Decimal(-2))
- Decimal('-1')
- """
- a = _convert_other(a, raiseit=True)
- return a.copy_sign(b)
-
- def divide(self, a, b):
- """Decimal division in a specified context.
-
- >>> ExtendedContext.divide(Decimal('1'), Decimal('3'))
- Decimal('0.333333333')
- >>> ExtendedContext.divide(Decimal('2'), Decimal('3'))
- Decimal('0.666666667')
- >>> ExtendedContext.divide(Decimal('5'), Decimal('2'))
- Decimal('2.5')
- >>> ExtendedContext.divide(Decimal('1'), Decimal('10'))
- Decimal('0.1')
- >>> ExtendedContext.divide(Decimal('12'), Decimal('12'))
- Decimal('1')
- >>> ExtendedContext.divide(Decimal('8.00'), Decimal('2'))
- Decimal('4.00')
- >>> ExtendedContext.divide(Decimal('2.400'), Decimal('2.0'))
- Decimal('1.20')
- >>> ExtendedContext.divide(Decimal('1000'), Decimal('100'))
- Decimal('10')
- >>> ExtendedContext.divide(Decimal('1000'), Decimal('1'))
- Decimal('1000')
- >>> ExtendedContext.divide(Decimal('2.40E+6'), Decimal('2'))
- Decimal('1.20E+6')
- >>> ExtendedContext.divide(5, 5)
- Decimal('1')
- >>> ExtendedContext.divide(Decimal(5), 5)
- Decimal('1')
- >>> ExtendedContext.divide(5, Decimal(5))
- Decimal('1')
- """
- a = _convert_other(a, raiseit=True)
- r = a.__truediv__(b, context=self)
- if r is NotImplemented:
- raise TypeError("Unable to convert %s to Decimal" % b)
- else:
- return r
-
- def divide_int(self, a, b):
- """Divides two numbers and returns the integer part of the result.
-
- >>> ExtendedContext.divide_int(Decimal('2'), Decimal('3'))
- Decimal('0')
- >>> ExtendedContext.divide_int(Decimal('10'), Decimal('3'))
- Decimal('3')
- >>> ExtendedContext.divide_int(Decimal('1'), Decimal('0.3'))
- Decimal('3')
- >>> ExtendedContext.divide_int(10, 3)
- Decimal('3')
- >>> ExtendedContext.divide_int(Decimal(10), 3)
- Decimal('3')
- >>> ExtendedContext.divide_int(10, Decimal(3))
- Decimal('3')
- """
- a = _convert_other(a, raiseit=True)
- r = a.__floordiv__(b, context=self)
- if r is NotImplemented:
- raise TypeError("Unable to convert %s to Decimal" % b)
- else:
- return r
-
- def divmod(self, a, b):
- """Return (a // b, a % b).
-
- >>> ExtendedContext.divmod(Decimal(8), Decimal(3))
- (Decimal('2'), Decimal('2'))
- >>> ExtendedContext.divmod(Decimal(8), Decimal(4))
- (Decimal('2'), Decimal('0'))
- >>> ExtendedContext.divmod(8, 4)
- (Decimal('2'), Decimal('0'))
- >>> ExtendedContext.divmod(Decimal(8), 4)
- (Decimal('2'), Decimal('0'))
- >>> ExtendedContext.divmod(8, Decimal(4))
- (Decimal('2'), Decimal('0'))
- """
- a = _convert_other(a, raiseit=True)
- r = a.__divmod__(b, context=self)
- if r is NotImplemented:
- raise TypeError("Unable to convert %s to Decimal" % b)
- else:
- return r
-
- def exp(self, a):
- """Returns e ** a.
-
- >>> c = ExtendedContext.copy()
- >>> c.Emin = -999
- >>> c.Emax = 999
- >>> c.exp(Decimal('-Infinity'))
- Decimal('0')
- >>> c.exp(Decimal('-1'))
- Decimal('0.367879441')
- >>> c.exp(Decimal('0'))
- Decimal('1')
- >>> c.exp(Decimal('1'))
- Decimal('2.71828183')
- >>> c.exp(Decimal('0.693147181'))
- Decimal('2.00000000')
- >>> c.exp(Decimal('+Infinity'))
- Decimal('Infinity')
- >>> c.exp(10)
- Decimal('22026.4658')
- """
- a =_convert_other(a, raiseit=True)
- return a.exp(context=self)
-
- def fma(self, a, b, c):
- """Returns a multiplied by b, plus c.
-
- The first two operands are multiplied together, using multiply,
- the third operand is then added to the result of that
- multiplication, using add, all with only one final rounding.
-
- >>> ExtendedContext.fma(Decimal('3'), Decimal('5'), Decimal('7'))
- Decimal('22')
- >>> ExtendedContext.fma(Decimal('3'), Decimal('-5'), Decimal('7'))
- Decimal('-8')
- >>> ExtendedContext.fma(Decimal('888565290'), Decimal('1557.96930'), Decimal('-86087.7578'))
- Decimal('1.38435736E+12')
- >>> ExtendedContext.fma(1, 3, 4)
- Decimal('7')
- >>> ExtendedContext.fma(1, Decimal(3), 4)
- Decimal('7')
- >>> ExtendedContext.fma(1, 3, Decimal(4))
- Decimal('7')
- """
- a = _convert_other(a, raiseit=True)
- return a.fma(b, c, context=self)
-
- def is_canonical(self, a):
- """Return True if the operand is canonical; otherwise return False.
-
- Currently, the encoding of a Decimal instance is always
- canonical, so this method returns True for any Decimal.
-
- >>> ExtendedContext.is_canonical(Decimal('2.50'))
- True
- """
- if not isinstance(a, Decimal):
- raise TypeError("is_canonical requires a Decimal as an argument.")
- return a.is_canonical()
-
- def is_finite(self, a):
- """Return True if the operand is finite; otherwise return False.
-
- A Decimal instance is considered finite if it is neither
- infinite nor a NaN.
-
- >>> ExtendedContext.is_finite(Decimal('2.50'))
- True
- >>> ExtendedContext.is_finite(Decimal('-0.3'))
- True
- >>> ExtendedContext.is_finite(Decimal('0'))
- True
- >>> ExtendedContext.is_finite(Decimal('Inf'))
- False
- >>> ExtendedContext.is_finite(Decimal('NaN'))
- False
- >>> ExtendedContext.is_finite(1)
- True
- """
- a = _convert_other(a, raiseit=True)
- return a.is_finite()
-
- def is_infinite(self, a):
- """Return True if the operand is infinite; otherwise return False.
-
- >>> ExtendedContext.is_infinite(Decimal('2.50'))
- False
- >>> ExtendedContext.is_infinite(Decimal('-Inf'))
- True
- >>> ExtendedContext.is_infinite(Decimal('NaN'))
- False
- >>> ExtendedContext.is_infinite(1)
- False
- """
- a = _convert_other(a, raiseit=True)
- return a.is_infinite()
-
- def is_nan(self, a):
- """Return True if the operand is a qNaN or sNaN;
- otherwise return False.
-
- >>> ExtendedContext.is_nan(Decimal('2.50'))
- False
- >>> ExtendedContext.is_nan(Decimal('NaN'))
- True
- >>> ExtendedContext.is_nan(Decimal('-sNaN'))
- True
- >>> ExtendedContext.is_nan(1)
- False
- """
- a = _convert_other(a, raiseit=True)
- return a.is_nan()
-
- def is_normal(self, a):
- """Return True if the operand is a normal number;
- otherwise return False.
-
- >>> c = ExtendedContext.copy()
- >>> c.Emin = -999
- >>> c.Emax = 999
- >>> c.is_normal(Decimal('2.50'))
- True
- >>> c.is_normal(Decimal('0.1E-999'))
- False
- >>> c.is_normal(Decimal('0.00'))
- False
- >>> c.is_normal(Decimal('-Inf'))
- False
- >>> c.is_normal(Decimal('NaN'))
- False
- >>> c.is_normal(1)
- True
- """
- a = _convert_other(a, raiseit=True)
- return a.is_normal(context=self)
-
- def is_qnan(self, a):
- """Return True if the operand is a quiet NaN; otherwise return False.
-
- >>> ExtendedContext.is_qnan(Decimal('2.50'))
- False
- >>> ExtendedContext.is_qnan(Decimal('NaN'))
- True
- >>> ExtendedContext.is_qnan(Decimal('sNaN'))
- False
- >>> ExtendedContext.is_qnan(1)
- False
- """
- a = _convert_other(a, raiseit=True)
- return a.is_qnan()
-
- def is_signed(self, a):
- """Return True if the operand is negative; otherwise return False.
-
- >>> ExtendedContext.is_signed(Decimal('2.50'))
- False
- >>> ExtendedContext.is_signed(Decimal('-12'))
- True
- >>> ExtendedContext.is_signed(Decimal('-0'))
- True
- >>> ExtendedContext.is_signed(8)
- False
- >>> ExtendedContext.is_signed(-8)
- True
- """
- a = _convert_other(a, raiseit=True)
- return a.is_signed()
-
- def is_snan(self, a):
- """Return True if the operand is a signaling NaN;
- otherwise return False.
-
- >>> ExtendedContext.is_snan(Decimal('2.50'))
- False
- >>> ExtendedContext.is_snan(Decimal('NaN'))
- False
- >>> ExtendedContext.is_snan(Decimal('sNaN'))
- True
- >>> ExtendedContext.is_snan(1)
- False
- """
- a = _convert_other(a, raiseit=True)
- return a.is_snan()
-
- def is_subnormal(self, a):
- """Return True if the operand is subnormal; otherwise return False.
-
- >>> c = ExtendedContext.copy()
- >>> c.Emin = -999
- >>> c.Emax = 999
- >>> c.is_subnormal(Decimal('2.50'))
- False
- >>> c.is_subnormal(Decimal('0.1E-999'))
- True
- >>> c.is_subnormal(Decimal('0.00'))
- False
- >>> c.is_subnormal(Decimal('-Inf'))
- False
- >>> c.is_subnormal(Decimal('NaN'))
- False
- >>> c.is_subnormal(1)
- False
- """
- a = _convert_other(a, raiseit=True)
- return a.is_subnormal(context=self)
-
- def is_zero(self, a):
- """Return True if the operand is a zero; otherwise return False.
-
- >>> ExtendedContext.is_zero(Decimal('0'))
- True
- >>> ExtendedContext.is_zero(Decimal('2.50'))
- False
- >>> ExtendedContext.is_zero(Decimal('-0E+2'))
- True
- >>> ExtendedContext.is_zero(1)
- False
- >>> ExtendedContext.is_zero(0)
- True
- """
- a = _convert_other(a, raiseit=True)
- return a.is_zero()
-
- def ln(self, a):
- """Returns the natural (base e) logarithm of the operand.
-
- >>> c = ExtendedContext.copy()
- >>> c.Emin = -999
- >>> c.Emax = 999
- >>> c.ln(Decimal('0'))
- Decimal('-Infinity')
- >>> c.ln(Decimal('1.000'))
- Decimal('0')
- >>> c.ln(Decimal('2.71828183'))
- Decimal('1.00000000')
- >>> c.ln(Decimal('10'))
- Decimal('2.30258509')
- >>> c.ln(Decimal('+Infinity'))
- Decimal('Infinity')
- >>> c.ln(1)
- Decimal('0')
- """
- a = _convert_other(a, raiseit=True)
- return a.ln(context=self)
-
- def log10(self, a):
- """Returns the base 10 logarithm of the operand.
-
- >>> c = ExtendedContext.copy()
- >>> c.Emin = -999
- >>> c.Emax = 999
- >>> c.log10(Decimal('0'))
- Decimal('-Infinity')
- >>> c.log10(Decimal('0.001'))
- Decimal('-3')
- >>> c.log10(Decimal('1.000'))
- Decimal('0')
- >>> c.log10(Decimal('2'))
- Decimal('0.301029996')
- >>> c.log10(Decimal('10'))
- Decimal('1')
- >>> c.log10(Decimal('70'))
- Decimal('1.84509804')
- >>> c.log10(Decimal('+Infinity'))
- Decimal('Infinity')
- >>> c.log10(0)
- Decimal('-Infinity')
- >>> c.log10(1)
- Decimal('0')
- """
- a = _convert_other(a, raiseit=True)
- return a.log10(context=self)
-
- def logb(self, a):
- """ Returns the exponent of the magnitude of the operand's MSD.
-
- The result is the integer which is the exponent of the magnitude
- of the most significant digit of the operand (as though the
- operand were truncated to a single digit while maintaining the
- value of that digit and without limiting the resulting exponent).
-
- >>> ExtendedContext.logb(Decimal('250'))
- Decimal('2')
- >>> ExtendedContext.logb(Decimal('2.50'))
- Decimal('0')
- >>> ExtendedContext.logb(Decimal('0.03'))
- Decimal('-2')
- >>> ExtendedContext.logb(Decimal('0'))
- Decimal('-Infinity')
- >>> ExtendedContext.logb(1)
- Decimal('0')
- >>> ExtendedContext.logb(10)
- Decimal('1')
- >>> ExtendedContext.logb(100)
- Decimal('2')
- """
- a = _convert_other(a, raiseit=True)
- return a.logb(context=self)
-
- def logical_and(self, a, b):
- """Applies the logical operation 'and' between each operand's digits.
-
- The operands must be both logical numbers.
-
- >>> ExtendedContext.logical_and(Decimal('0'), Decimal('0'))
- Decimal('0')
- >>> ExtendedContext.logical_and(Decimal('0'), Decimal('1'))
- Decimal('0')
- >>> ExtendedContext.logical_and(Decimal('1'), Decimal('0'))
- Decimal('0')
- >>> ExtendedContext.logical_and(Decimal('1'), Decimal('1'))
- Decimal('1')
- >>> ExtendedContext.logical_and(Decimal('1100'), Decimal('1010'))
- Decimal('1000')
- >>> ExtendedContext.logical_and(Decimal('1111'), Decimal('10'))
- Decimal('10')
- >>> ExtendedContext.logical_and(110, 1101)
- Decimal('100')
- >>> ExtendedContext.logical_and(Decimal(110), 1101)
- Decimal('100')
- >>> ExtendedContext.logical_and(110, Decimal(1101))
- Decimal('100')
- """
- a = _convert_other(a, raiseit=True)
- return a.logical_and(b, context=self)
-
- def logical_invert(self, a):
- """Invert all the digits in the operand.
-
- The operand must be a logical number.
-
- >>> ExtendedContext.logical_invert(Decimal('0'))
- Decimal('111111111')
- >>> ExtendedContext.logical_invert(Decimal('1'))
- Decimal('111111110')
- >>> ExtendedContext.logical_invert(Decimal('111111111'))
- Decimal('0')
- >>> ExtendedContext.logical_invert(Decimal('101010101'))
- Decimal('10101010')
- >>> ExtendedContext.logical_invert(1101)
- Decimal('111110010')
- """
- a = _convert_other(a, raiseit=True)
- return a.logical_invert(context=self)
-
- def logical_or(self, a, b):
- """Applies the logical operation 'or' between each operand's digits.
-
- The operands must be both logical numbers.
-
- >>> ExtendedContext.logical_or(Decimal('0'), Decimal('0'))
- Decimal('0')
- >>> ExtendedContext.logical_or(Decimal('0'), Decimal('1'))
- Decimal('1')
- >>> ExtendedContext.logical_or(Decimal('1'), Decimal('0'))
- Decimal('1')
- >>> ExtendedContext.logical_or(Decimal('1'), Decimal('1'))
- Decimal('1')
- >>> ExtendedContext.logical_or(Decimal('1100'), Decimal('1010'))
- Decimal('1110')
- >>> ExtendedContext.logical_or(Decimal('1110'), Decimal('10'))
- Decimal('1110')
- >>> ExtendedContext.logical_or(110, 1101)
- Decimal('1111')
- >>> ExtendedContext.logical_or(Decimal(110), 1101)
- Decimal('1111')
- >>> ExtendedContext.logical_or(110, Decimal(1101))
- Decimal('1111')
- """
- a = _convert_other(a, raiseit=True)
- return a.logical_or(b, context=self)
-
- def logical_xor(self, a, b):
- """Applies the logical operation 'xor' between each operand's digits.
-
- The operands must be both logical numbers.
-
- >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('0'))
- Decimal('0')
- >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('1'))
- Decimal('1')
- >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('0'))
- Decimal('1')
- >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('1'))
- Decimal('0')
- >>> ExtendedContext.logical_xor(Decimal('1100'), Decimal('1010'))
- Decimal('110')
- >>> ExtendedContext.logical_xor(Decimal('1111'), Decimal('10'))
- Decimal('1101')
- >>> ExtendedContext.logical_xor(110, 1101)
- Decimal('1011')
- >>> ExtendedContext.logical_xor(Decimal(110), 1101)
- Decimal('1011')
- >>> ExtendedContext.logical_xor(110, Decimal(1101))
- Decimal('1011')
- """
- a = _convert_other(a, raiseit=True)
- return a.logical_xor(b, context=self)
-
- def max(self, a, b):
- """max compares two values numerically and returns the maximum.
-
- If either operand is a NaN then the general rules apply.
- Otherwise, the operands are compared as though by the compare
- operation. If they are numerically equal then the left-hand operand
- is chosen as the result. Otherwise the maximum (closer to positive
- infinity) of the two operands is chosen as the result.
-
- >>> ExtendedContext.max(Decimal('3'), Decimal('2'))
- Decimal('3')
- >>> ExtendedContext.max(Decimal('-10'), Decimal('3'))
- Decimal('3')
- >>> ExtendedContext.max(Decimal('1.0'), Decimal('1'))
- Decimal('1')
- >>> ExtendedContext.max(Decimal('7'), Decimal('NaN'))
- Decimal('7')
- >>> ExtendedContext.max(1, 2)
- Decimal('2')
- >>> ExtendedContext.max(Decimal(1), 2)
- Decimal('2')
- >>> ExtendedContext.max(1, Decimal(2))
- Decimal('2')
- """
- a = _convert_other(a, raiseit=True)
- return a.max(b, context=self)
-
- def max_mag(self, a, b):
- """Compares the values numerically with their sign ignored.
-
- >>> ExtendedContext.max_mag(Decimal('7'), Decimal('NaN'))
- Decimal('7')
- >>> ExtendedContext.max_mag(Decimal('7'), Decimal('-10'))
- Decimal('-10')
- >>> ExtendedContext.max_mag(1, -2)
- Decimal('-2')
- >>> ExtendedContext.max_mag(Decimal(1), -2)
- Decimal('-2')
- >>> ExtendedContext.max_mag(1, Decimal(-2))
- Decimal('-2')
- """
- a = _convert_other(a, raiseit=True)
- return a.max_mag(b, context=self)
-
- def min(self, a, b):
- """min compares two values numerically and returns the minimum.
-
- If either operand is a NaN then the general rules apply.
- Otherwise, the operands are compared as though by the compare
- operation. If they are numerically equal then the left-hand operand
- is chosen as the result. Otherwise the minimum (closer to negative
- infinity) of the two operands is chosen as the result.
-
- >>> ExtendedContext.min(Decimal('3'), Decimal('2'))
- Decimal('2')
- >>> ExtendedContext.min(Decimal('-10'), Decimal('3'))
- Decimal('-10')
- >>> ExtendedContext.min(Decimal('1.0'), Decimal('1'))
- Decimal('1.0')
- >>> ExtendedContext.min(Decimal('7'), Decimal('NaN'))
- Decimal('7')
- >>> ExtendedContext.min(1, 2)
- Decimal('1')
- >>> ExtendedContext.min(Decimal(1), 2)
- Decimal('1')
- >>> ExtendedContext.min(1, Decimal(29))
- Decimal('1')
- """
- a = _convert_other(a, raiseit=True)
- return a.min(b, context=self)
-
- def min_mag(self, a, b):
- """Compares the values numerically with their sign ignored.
-
- >>> ExtendedContext.min_mag(Decimal('3'), Decimal('-2'))
- Decimal('-2')
- >>> ExtendedContext.min_mag(Decimal('-3'), Decimal('NaN'))
- Decimal('-3')
- >>> ExtendedContext.min_mag(1, -2)
- Decimal('1')
- >>> ExtendedContext.min_mag(Decimal(1), -2)
- Decimal('1')
- >>> ExtendedContext.min_mag(1, Decimal(-2))
- Decimal('1')
- """
- a = _convert_other(a, raiseit=True)
- return a.min_mag(b, context=self)
-
- def minus(self, a):
- """Minus corresponds to unary prefix minus in Python.
-
- The operation is evaluated using the same rules as subtract; the
- operation minus(a) is calculated as subtract('0', a) where the '0'
- has the same exponent as the operand.
-
- >>> ExtendedContext.minus(Decimal('1.3'))
- Decimal('-1.3')
- >>> ExtendedContext.minus(Decimal('-1.3'))
- Decimal('1.3')
- >>> ExtendedContext.minus(1)
- Decimal('-1')
- """
- a = _convert_other(a, raiseit=True)
- return a.__neg__(context=self)
-
- def multiply(self, a, b):
- """multiply multiplies two operands.
-
- If either operand is a special value then the general rules apply.
- Otherwise, the operands are multiplied together
- ('long multiplication'), resulting in a number which may be as long as
- the sum of the lengths of the two operands.
-
- >>> ExtendedContext.multiply(Decimal('1.20'), Decimal('3'))
- Decimal('3.60')
- >>> ExtendedContext.multiply(Decimal('7'), Decimal('3'))
- Decimal('21')
- >>> ExtendedContext.multiply(Decimal('0.9'), Decimal('0.8'))
- Decimal('0.72')
- >>> ExtendedContext.multiply(Decimal('0.9'), Decimal('-0'))
- Decimal('-0.0')
- >>> ExtendedContext.multiply(Decimal('654321'), Decimal('654321'))
- Decimal('4.28135971E+11')
- >>> ExtendedContext.multiply(7, 7)
- Decimal('49')
- >>> ExtendedContext.multiply(Decimal(7), 7)
- Decimal('49')
- >>> ExtendedContext.multiply(7, Decimal(7))
- Decimal('49')
- """
- a = _convert_other(a, raiseit=True)
- r = a.__mul__(b, context=self)
- if r is NotImplemented:
- raise TypeError("Unable to convert %s to Decimal" % b)
- else:
- return r
-
- def next_minus(self, a):
- """Returns the largest representable number smaller than a.
-
- >>> c = ExtendedContext.copy()
- >>> c.Emin = -999
- >>> c.Emax = 999
- >>> ExtendedContext.next_minus(Decimal('1'))
- Decimal('0.999999999')
- >>> c.next_minus(Decimal('1E-1007'))
- Decimal('0E-1007')
- >>> ExtendedContext.next_minus(Decimal('-1.00000003'))
- Decimal('-1.00000004')
- >>> c.next_minus(Decimal('Infinity'))
- Decimal('9.99999999E+999')
- >>> c.next_minus(1)
- Decimal('0.999999999')
- """
- a = _convert_other(a, raiseit=True)
- return a.next_minus(context=self)
-
- def next_plus(self, a):
- """Returns the smallest representable number larger than a.
-
- >>> c = ExtendedContext.copy()
- >>> c.Emin = -999
- >>> c.Emax = 999
- >>> ExtendedContext.next_plus(Decimal('1'))
- Decimal('1.00000001')
- >>> c.next_plus(Decimal('-1E-1007'))
- Decimal('-0E-1007')
- >>> ExtendedContext.next_plus(Decimal('-1.00000003'))
- Decimal('-1.00000002')
- >>> c.next_plus(Decimal('-Infinity'))
- Decimal('-9.99999999E+999')
- >>> c.next_plus(1)
- Decimal('1.00000001')
- """
- a = _convert_other(a, raiseit=True)
- return a.next_plus(context=self)
-
- def next_toward(self, a, b):
- """Returns the number closest to a, in direction towards b.
-
- The result is the closest representable number from the first
- operand (but not the first operand) that is in the direction
- towards the second operand, unless the operands have the same
- value.
-
- >>> c = ExtendedContext.copy()
- >>> c.Emin = -999
- >>> c.Emax = 999
- >>> c.next_toward(Decimal('1'), Decimal('2'))
- Decimal('1.00000001')
- >>> c.next_toward(Decimal('-1E-1007'), Decimal('1'))
- Decimal('-0E-1007')
- >>> c.next_toward(Decimal('-1.00000003'), Decimal('0'))
- Decimal('-1.00000002')
- >>> c.next_toward(Decimal('1'), Decimal('0'))
- Decimal('0.999999999')
- >>> c.next_toward(Decimal('1E-1007'), Decimal('-100'))
- Decimal('0E-1007')
- >>> c.next_toward(Decimal('-1.00000003'), Decimal('-10'))
- Decimal('-1.00000004')
- >>> c.next_toward(Decimal('0.00'), Decimal('-0.0000'))
- Decimal('-0.00')
- >>> c.next_toward(0, 1)
- Decimal('1E-1007')
- >>> c.next_toward(Decimal(0), 1)
- Decimal('1E-1007')
- >>> c.next_toward(0, Decimal(1))
- Decimal('1E-1007')
- """
- a = _convert_other(a, raiseit=True)
- return a.next_toward(b, context=self)
-
- def normalize(self, a):
- """normalize reduces an operand to its simplest form.
-
- Essentially a plus operation with all trailing zeros removed from the
- result.
-
- >>> ExtendedContext.normalize(Decimal('2.1'))
- Decimal('2.1')
- >>> ExtendedContext.normalize(Decimal('-2.0'))
- Decimal('-2')
- >>> ExtendedContext.normalize(Decimal('1.200'))
- Decimal('1.2')
- >>> ExtendedContext.normalize(Decimal('-120'))
- Decimal('-1.2E+2')
- >>> ExtendedContext.normalize(Decimal('120.00'))
- Decimal('1.2E+2')
- >>> ExtendedContext.normalize(Decimal('0.00'))
- Decimal('0')
- >>> ExtendedContext.normalize(6)
- Decimal('6')
- """
- a = _convert_other(a, raiseit=True)
- return a.normalize(context=self)
-
- def number_class(self, a):
- """Returns an indication of the class of the operand.
-
- The class is one of the following strings:
- -sNaN
- -NaN
- -Infinity
- -Normal
- -Subnormal
- -Zero
- +Zero
- +Subnormal
- +Normal
- +Infinity
-
- >>> c = ExtendedContext.copy()
- >>> c.Emin = -999
- >>> c.Emax = 999
- >>> c.number_class(Decimal('Infinity'))
- '+Infinity'
- >>> c.number_class(Decimal('1E-10'))
- '+Normal'
- >>> c.number_class(Decimal('2.50'))
- '+Normal'
- >>> c.number_class(Decimal('0.1E-999'))
- '+Subnormal'
- >>> c.number_class(Decimal('0'))
- '+Zero'
- >>> c.number_class(Decimal('-0'))
- '-Zero'
- >>> c.number_class(Decimal('-0.1E-999'))
- '-Subnormal'
- >>> c.number_class(Decimal('-1E-10'))
- '-Normal'
- >>> c.number_class(Decimal('-2.50'))
- '-Normal'
- >>> c.number_class(Decimal('-Infinity'))
- '-Infinity'
- >>> c.number_class(Decimal('NaN'))
- 'NaN'
- >>> c.number_class(Decimal('-NaN'))
- 'NaN'
- >>> c.number_class(Decimal('sNaN'))
- 'sNaN'
- >>> c.number_class(123)
- '+Normal'
- """
- a = _convert_other(a, raiseit=True)
- return a.number_class(context=self)
-
- def plus(self, a):
- """Plus corresponds to unary prefix plus in Python.
-
- The operation is evaluated using the same rules as add; the
- operation plus(a) is calculated as add('0', a) where the '0'
- has the same exponent as the operand.
-
- >>> ExtendedContext.plus(Decimal('1.3'))
- Decimal('1.3')
- >>> ExtendedContext.plus(Decimal('-1.3'))
- Decimal('-1.3')
- >>> ExtendedContext.plus(-1)
- Decimal('-1')
- """
- a = _convert_other(a, raiseit=True)
- return a.__pos__(context=self)
-
- def power(self, a, b, modulo=None):
- """Raises a to the power of b, to modulo if given.
-
- With two arguments, compute a**b. If a is negative then b
- must be integral. The result will be inexact unless b is
- integral and the result is finite and can be expressed exactly
- in 'precision' digits.
-
- With three arguments, compute (a**b) % modulo. For the
- three argument form, the following restrictions on the
- arguments hold:
-
- - all three arguments must be integral
- - b must be nonnegative
- - at least one of a or b must be nonzero
- - modulo must be nonzero and have at most 'precision' digits
-
- The result of pow(a, b, modulo) is identical to the result
- that would be obtained by computing (a**b) % modulo with
- unbounded precision, but is computed more efficiently. It is
- always exact.
-
- >>> c = ExtendedContext.copy()
- >>> c.Emin = -999
- >>> c.Emax = 999
- >>> c.power(Decimal('2'), Decimal('3'))
- Decimal('8')
- >>> c.power(Decimal('-2'), Decimal('3'))
- Decimal('-8')
- >>> c.power(Decimal('2'), Decimal('-3'))
- Decimal('0.125')
- >>> c.power(Decimal('1.7'), Decimal('8'))
- Decimal('69.7575744')
- >>> c.power(Decimal('10'), Decimal('0.301029996'))
- Decimal('2.00000000')
- >>> c.power(Decimal('Infinity'), Decimal('-1'))
- Decimal('0')
- >>> c.power(Decimal('Infinity'), Decimal('0'))
- Decimal('1')
- >>> c.power(Decimal('Infinity'), Decimal('1'))
- Decimal('Infinity')
- >>> c.power(Decimal('-Infinity'), Decimal('-1'))
- Decimal('-0')
- >>> c.power(Decimal('-Infinity'), Decimal('0'))
- Decimal('1')
- >>> c.power(Decimal('-Infinity'), Decimal('1'))
- Decimal('-Infinity')
- >>> c.power(Decimal('-Infinity'), Decimal('2'))
- Decimal('Infinity')
- >>> c.power(Decimal('0'), Decimal('0'))
- Decimal('NaN')
-
- >>> c.power(Decimal('3'), Decimal('7'), Decimal('16'))
- Decimal('11')
- >>> c.power(Decimal('-3'), Decimal('7'), Decimal('16'))
- Decimal('-11')
- >>> c.power(Decimal('-3'), Decimal('8'), Decimal('16'))
- Decimal('1')
- >>> c.power(Decimal('3'), Decimal('7'), Decimal('-16'))
- Decimal('11')
- >>> c.power(Decimal('23E12345'), Decimal('67E189'), Decimal('123456789'))
- Decimal('11729830')
- >>> c.power(Decimal('-0'), Decimal('17'), Decimal('1729'))
- Decimal('-0')
- >>> c.power(Decimal('-23'), Decimal('0'), Decimal('65537'))
- Decimal('1')
- >>> ExtendedContext.power(7, 7)
- Decimal('823543')
- >>> ExtendedContext.power(Decimal(7), 7)
- Decimal('823543')
- >>> ExtendedContext.power(7, Decimal(7), 2)
- Decimal('1')
- """
- a = _convert_other(a, raiseit=True)
- r = a.__pow__(b, modulo, context=self)
- if r is NotImplemented:
- raise TypeError("Unable to convert %s to Decimal" % b)
- else:
- return r
-
- def quantize(self, a, b):
- """Returns a value equal to 'a' (rounded), having the exponent of 'b'.
-
- The coefficient of the result is derived from that of the left-hand
- operand. It may be rounded using the current rounding setting (if the
- exponent is being increased), multiplied by a positive power of ten (if
- the exponent is being decreased), or is unchanged (if the exponent is
- already equal to that of the right-hand operand).
-
- Unlike other operations, if the length of the coefficient after the
- quantize operation would be greater than precision then an Invalid
- operation condition is raised. This guarantees that, unless there is
- an error condition, the exponent of the result of a quantize is always
- equal to that of the right-hand operand.
-
- Also unlike other operations, quantize will never raise Underflow, even
- if the result is subnormal and inexact.
-
- >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.001'))
- Decimal('2.170')
- >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.01'))
- Decimal('2.17')
- >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.1'))
- Decimal('2.2')
- >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('1e+0'))
- Decimal('2')
- >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('1e+1'))
- Decimal('0E+1')
- >>> ExtendedContext.quantize(Decimal('-Inf'), Decimal('Infinity'))
- Decimal('-Infinity')
- >>> ExtendedContext.quantize(Decimal('2'), Decimal('Infinity'))
- Decimal('NaN')
- >>> ExtendedContext.quantize(Decimal('-0.1'), Decimal('1'))
- Decimal('-0')
- >>> ExtendedContext.quantize(Decimal('-0'), Decimal('1e+5'))
- Decimal('-0E+5')
- >>> ExtendedContext.quantize(Decimal('+35236450.6'), Decimal('1e-2'))
- Decimal('NaN')
- >>> ExtendedContext.quantize(Decimal('-35236450.6'), Decimal('1e-2'))
- Decimal('NaN')
- >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e-1'))
- Decimal('217.0')
- >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e-0'))
- Decimal('217')
- >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e+1'))
- Decimal('2.2E+2')
- >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e+2'))
- Decimal('2E+2')
- >>> ExtendedContext.quantize(1, 2)
- Decimal('1')
- >>> ExtendedContext.quantize(Decimal(1), 2)
- Decimal('1')
- >>> ExtendedContext.quantize(1, Decimal(2))
- Decimal('1')
- """
- a = _convert_other(a, raiseit=True)
- return a.quantize(b, context=self)
-
- def radix(self):
- """Just returns 10, as this is Decimal, :)
-
- >>> ExtendedContext.radix()
- Decimal('10')
- """
- return Decimal(10)
-
- def remainder(self, a, b):
- """Returns the remainder from integer division.
-
- The result is the residue of the dividend after the operation of
- calculating integer division as described for divide-integer, rounded
- to precision digits if necessary. The sign of the result, if
- non-zero, is the same as that of the original dividend.
-
- This operation will fail under the same conditions as integer division
- (that is, if integer division on the same two operands would fail, the
- remainder cannot be calculated).
-
- >>> ExtendedContext.remainder(Decimal('2.1'), Decimal('3'))
- Decimal('2.1')
- >>> ExtendedContext.remainder(Decimal('10'), Decimal('3'))
- Decimal('1')
- >>> ExtendedContext.remainder(Decimal('-10'), Decimal('3'))
- Decimal('-1')
- >>> ExtendedContext.remainder(Decimal('10.2'), Decimal('1'))
- Decimal('0.2')
- >>> ExtendedContext.remainder(Decimal('10'), Decimal('0.3'))
- Decimal('0.1')
- >>> ExtendedContext.remainder(Decimal('3.6'), Decimal('1.3'))
- Decimal('1.0')
- >>> ExtendedContext.remainder(22, 6)
- Decimal('4')
- >>> ExtendedContext.remainder(Decimal(22), 6)
- Decimal('4')
- >>> ExtendedContext.remainder(22, Decimal(6))
- Decimal('4')
- """
- a = _convert_other(a, raiseit=True)
- r = a.__mod__(b, context=self)
- if r is NotImplemented:
- raise TypeError("Unable to convert %s to Decimal" % b)
- else:
- return r
-
- def remainder_near(self, a, b):
- """Returns to be "a - b * n", where n is the integer nearest the exact
- value of "x / b" (if two integers are equally near then the even one
- is chosen). If the result is equal to 0 then its sign will be the
- sign of a.
-
- This operation will fail under the same conditions as integer division
- (that is, if integer division on the same two operands would fail, the
- remainder cannot be calculated).
-
- >>> ExtendedContext.remainder_near(Decimal('2.1'), Decimal('3'))
- Decimal('-0.9')
- >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('6'))
- Decimal('-2')
- >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('3'))
- Decimal('1')
- >>> ExtendedContext.remainder_near(Decimal('-10'), Decimal('3'))
- Decimal('-1')
- >>> ExtendedContext.remainder_near(Decimal('10.2'), Decimal('1'))
- Decimal('0.2')
- >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('0.3'))
- Decimal('0.1')
- >>> ExtendedContext.remainder_near(Decimal('3.6'), Decimal('1.3'))
- Decimal('-0.3')
- >>> ExtendedContext.remainder_near(3, 11)
- Decimal('3')
- >>> ExtendedContext.remainder_near(Decimal(3), 11)
- Decimal('3')
- >>> ExtendedContext.remainder_near(3, Decimal(11))
- Decimal('3')
- """
- a = _convert_other(a, raiseit=True)
- return a.remainder_near(b, context=self)
-
- def rotate(self, a, b):
- """Returns a rotated copy of a, b times.
-
- The coefficient of the result is a rotated copy of the digits in
- the coefficient of the first operand. The number of places of
- rotation is taken from the absolute value of the second operand,
- with the rotation being to the left if the second operand is
- positive or to the right otherwise.
-
- >>> ExtendedContext.rotate(Decimal('34'), Decimal('8'))
- Decimal('400000003')
- >>> ExtendedContext.rotate(Decimal('12'), Decimal('9'))
- Decimal('12')
- >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('-2'))
- Decimal('891234567')
- >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('0'))
- Decimal('123456789')
- >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('+2'))
- Decimal('345678912')
- >>> ExtendedContext.rotate(1333333, 1)
- Decimal('13333330')
- >>> ExtendedContext.rotate(Decimal(1333333), 1)
- Decimal('13333330')
- >>> ExtendedContext.rotate(1333333, Decimal(1))
- Decimal('13333330')
- """
- a = _convert_other(a, raiseit=True)
- return a.rotate(b, context=self)
-
- def same_quantum(self, a, b):
- """Returns True if the two operands have the same exponent.
-
- The result is never affected by either the sign or the coefficient of
- either operand.
-
- >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.001'))
- False
- >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.01'))
- True
- >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('1'))
- False
- >>> ExtendedContext.same_quantum(Decimal('Inf'), Decimal('-Inf'))
- True
- >>> ExtendedContext.same_quantum(10000, -1)
- True
- >>> ExtendedContext.same_quantum(Decimal(10000), -1)
- True
- >>> ExtendedContext.same_quantum(10000, Decimal(-1))
- True
- """
- a = _convert_other(a, raiseit=True)
- return a.same_quantum(b)
-
- def scaleb (self, a, b):
- """Returns the first operand after adding the second value its exp.
-
- >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('-2'))
- Decimal('0.0750')
- >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('0'))
- Decimal('7.50')
- >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('3'))
- Decimal('7.50E+3')
- >>> ExtendedContext.scaleb(1, 4)
- Decimal('1E+4')
- >>> ExtendedContext.scaleb(Decimal(1), 4)
- Decimal('1E+4')
- >>> ExtendedContext.scaleb(1, Decimal(4))
- Decimal('1E+4')
- """
- a = _convert_other(a, raiseit=True)
- return a.scaleb(b, context=self)
-
- def shift(self, a, b):
- """Returns a shifted copy of a, b times.
-
- The coefficient of the result is a shifted copy of the digits
- in the coefficient of the first operand. The number of places
- to shift is taken from the absolute value of the second operand,
- with the shift being to the left if the second operand is
- positive or to the right otherwise. Digits shifted into the
- coefficient are zeros.
-
- >>> ExtendedContext.shift(Decimal('34'), Decimal('8'))
- Decimal('400000000')
- >>> ExtendedContext.shift(Decimal('12'), Decimal('9'))
- Decimal('0')
- >>> ExtendedContext.shift(Decimal('123456789'), Decimal('-2'))
- Decimal('1234567')
- >>> ExtendedContext.shift(Decimal('123456789'), Decimal('0'))
- Decimal('123456789')
- >>> ExtendedContext.shift(Decimal('123456789'), Decimal('+2'))
- Decimal('345678900')
- >>> ExtendedContext.shift(88888888, 2)
- Decimal('888888800')
- >>> ExtendedContext.shift(Decimal(88888888), 2)
- Decimal('888888800')
- >>> ExtendedContext.shift(88888888, Decimal(2))
- Decimal('888888800')
- """
- a = _convert_other(a, raiseit=True)
- return a.shift(b, context=self)
-
- def sqrt(self, a):
- """Square root of a non-negative number to context precision.
-
- If the result must be inexact, it is rounded using the round-half-even
- algorithm.
-
- >>> ExtendedContext.sqrt(Decimal('0'))
- Decimal('0')
- >>> ExtendedContext.sqrt(Decimal('-0'))
- Decimal('-0')
- >>> ExtendedContext.sqrt(Decimal('0.39'))
- Decimal('0.624499800')
- >>> ExtendedContext.sqrt(Decimal('100'))
- Decimal('10')
- >>> ExtendedContext.sqrt(Decimal('1'))
- Decimal('1')
- >>> ExtendedContext.sqrt(Decimal('1.0'))
- Decimal('1.0')
- >>> ExtendedContext.sqrt(Decimal('1.00'))
- Decimal('1.0')
- >>> ExtendedContext.sqrt(Decimal('7'))
- Decimal('2.64575131')
- >>> ExtendedContext.sqrt(Decimal('10'))
- Decimal('3.16227766')
- >>> ExtendedContext.sqrt(2)
- Decimal('1.41421356')
- >>> ExtendedContext.prec
- 9
- """
- a = _convert_other(a, raiseit=True)
- return a.sqrt(context=self)
-
- def subtract(self, a, b):
- """Return the difference between the two operands.
-
- >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('1.07'))
- Decimal('0.23')
- >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('1.30'))
- Decimal('0.00')
- >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('2.07'))
- Decimal('-0.77')
- >>> ExtendedContext.subtract(8, 5)
- Decimal('3')
- >>> ExtendedContext.subtract(Decimal(8), 5)
- Decimal('3')
- >>> ExtendedContext.subtract(8, Decimal(5))
- Decimal('3')
- """
- a = _convert_other(a, raiseit=True)
- r = a.__sub__(b, context=self)
- if r is NotImplemented:
- raise TypeError("Unable to convert %s to Decimal" % b)
- else:
- return r
-
- def to_eng_string(self, a):
- """Converts a number to a string, using scientific notation.
-
- The operation is not affected by the context.
- """
- a = _convert_other(a, raiseit=True)
- return a.to_eng_string(context=self)
-
- def to_sci_string(self, a):
- """Converts a number to a string, using scientific notation.
-
- The operation is not affected by the context.
- """
- a = _convert_other(a, raiseit=True)
- return a.__str__(context=self)
-
- def to_integral_exact(self, a):
- """Rounds to an integer.
-
- When the operand has a negative exponent, the result is the same
- as using the quantize() operation using the given operand as the
- left-hand-operand, 1E+0 as the right-hand-operand, and the precision
- of the operand as the precision setting; Inexact and Rounded flags
- are allowed in this operation. The rounding mode is taken from the
- context.
-
- >>> ExtendedContext.to_integral_exact(Decimal('2.1'))
- Decimal('2')
- >>> ExtendedContext.to_integral_exact(Decimal('100'))
- Decimal('100')
- >>> ExtendedContext.to_integral_exact(Decimal('100.0'))
- Decimal('100')
- >>> ExtendedContext.to_integral_exact(Decimal('101.5'))
- Decimal('102')
- >>> ExtendedContext.to_integral_exact(Decimal('-101.5'))
- Decimal('-102')
- >>> ExtendedContext.to_integral_exact(Decimal('10E+5'))
- Decimal('1.0E+6')
- >>> ExtendedContext.to_integral_exact(Decimal('7.89E+77'))
- Decimal('7.89E+77')
- >>> ExtendedContext.to_integral_exact(Decimal('-Inf'))
- Decimal('-Infinity')
- """
- a = _convert_other(a, raiseit=True)
- return a.to_integral_exact(context=self)
-
- def to_integral_value(self, a):
- """Rounds to an integer.
-
- When the operand has a negative exponent, the result is the same
- as using the quantize() operation using the given operand as the
- left-hand-operand, 1E+0 as the right-hand-operand, and the precision
- of the operand as the precision setting, except that no flags will
- be set. The rounding mode is taken from the context.
-
- >>> ExtendedContext.to_integral_value(Decimal('2.1'))
- Decimal('2')
- >>> ExtendedContext.to_integral_value(Decimal('100'))
- Decimal('100')
- >>> ExtendedContext.to_integral_value(Decimal('100.0'))
- Decimal('100')
- >>> ExtendedContext.to_integral_value(Decimal('101.5'))
- Decimal('102')
- >>> ExtendedContext.to_integral_value(Decimal('-101.5'))
- Decimal('-102')
- >>> ExtendedContext.to_integral_value(Decimal('10E+5'))
- Decimal('1.0E+6')
- >>> ExtendedContext.to_integral_value(Decimal('7.89E+77'))
- Decimal('7.89E+77')
- >>> ExtendedContext.to_integral_value(Decimal('-Inf'))
- Decimal('-Infinity')
- """
- a = _convert_other(a, raiseit=True)
- return a.to_integral_value(context=self)
-
- # the method name changed, but we provide also the old one, for compatibility
- to_integral = to_integral_value
-
-class _WorkRep(object):
- __slots__ = ('sign','int','exp')
- # sign: 0 or 1
- # int: int
- # exp: None, int, or string
-
- def __init__(self, value=None):
- if value is None:
- self.sign = None
- self.int = 0
- self.exp = None
- elif isinstance(value, Decimal):
- self.sign = value._sign
- self.int = int(value._int)
- self.exp = value._exp
- else:
- # assert isinstance(value, tuple)
- self.sign = value[0]
- self.int = value[1]
- self.exp = value[2]
-
- def __repr__(self):
- return "(%r, %r, %r)" % (self.sign, self.int, self.exp)
-
- __str__ = __repr__
-
-
-
-def _normalize(op1, op2, prec = 0):
- """Normalizes op1, op2 to have the same exp and length of coefficient.
-
- Done during addition.
- """
- if op1.exp < op2.exp:
- tmp = op2
- other = op1
- else:
- tmp = op1
- other = op2
-
- # Let exp = min(tmp.exp - 1, tmp.adjusted() - precision - 1).
- # Then adding 10**exp to tmp has the same effect (after rounding)
- # as adding any positive quantity smaller than 10**exp; similarly
- # for subtraction. So if other is smaller than 10**exp we replace
- # it with 10**exp. This avoids tmp.exp - other.exp getting too large.
- tmp_len = len(str(tmp.int))
- other_len = len(str(other.int))
- exp = tmp.exp + min(-1, tmp_len - prec - 2)
- if other_len + other.exp - 1 < exp:
- other.int = 1
- other.exp = exp
-
- tmp.int *= 10 ** (tmp.exp - other.exp)
- tmp.exp = other.exp
- return op1, op2
-
-##### Integer arithmetic functions used by ln, log10, exp and __pow__ #####
-
-_nbits = int.bit_length
-
-def _decimal_lshift_exact(n, e):
- """ Given integers n and e, return n * 10**e if it's an integer, else None.
-
- The computation is designed to avoid computing large powers of 10
- unnecessarily.
-
- >>> _decimal_lshift_exact(3, 4)
- 30000
- >>> _decimal_lshift_exact(300, -999999999) # returns None
-
- """
- if n == 0:
- return 0
- elif e >= 0:
- return n * 10**e
- else:
- # val_n = largest power of 10 dividing n.
- str_n = str(abs(n))
- val_n = len(str_n) - len(str_n.rstrip('0'))
- return None if val_n < -e else n // 10**-e
-
-def _sqrt_nearest(n, a):
- """Closest integer to the square root of the positive integer n. a is
- an initial approximation to the square root. Any positive integer
- will do for a, but the closer a is to the square root of n the
- faster convergence will be.
-
- """
- if n <= 0 or a <= 0:
- raise ValueError("Both arguments to _sqrt_nearest should be positive.")
-
- b=0
- while a != b:
- b, a = a, a--n//a>>1
- return a
-
-def _rshift_nearest(x, shift):
- """Given an integer x and a nonnegative integer shift, return closest
- integer to x / 2**shift; use round-to-even in case of a tie.
-
- """
- b, q = 1 << shift, x >> shift
- return q + (2*(x & (b-1)) + (q&1) > b)
-
-def _div_nearest(a, b):
- """Closest integer to a/b, a and b positive integers; rounds to even
- in the case of a tie.
-
- """
- q, r = divmod(a, b)
- return q + (2*r + (q&1) > b)
-
-def _ilog(x, M, L = 8):
- """Integer approximation to M*log(x/M), with absolute error boundable
- in terms only of x/M.
-
- Given positive integers x and M, return an integer approximation to
- M * log(x/M). For L = 8 and 0.1 <= x/M <= 10 the difference
- between the approximation and the exact result is at most 22. For
- L = 8 and 1.0 <= x/M <= 10.0 the difference is at most 15. In
- both cases these are upper bounds on the error; it will usually be
- much smaller."""
-
- # The basic algorithm is the following: let log1p be the function
- # log1p(x) = log(1+x). Then log(x/M) = log1p((x-M)/M). We use
- # the reduction
- #
- # log1p(y) = 2*log1p(y/(1+sqrt(1+y)))
- #
- # repeatedly until the argument to log1p is small (< 2**-L in
- # absolute value). For small y we can use the Taylor series
- # expansion
- #
- # log1p(y) ~ y - y**2/2 + y**3/3 - ... - (-y)**T/T
- #
- # truncating at T such that y**T is small enough. The whole
- # computation is carried out in a form of fixed-point arithmetic,
- # with a real number z being represented by an integer
- # approximation to z*M. To avoid loss of precision, the y below
- # is actually an integer approximation to 2**R*y*M, where R is the
- # number of reductions performed so far.
-
- y = x-M
- # argument reduction; R = number of reductions performed
- R = 0
- while (R <= L and abs(y) << L-R >= M or
- R > L and abs(y) >> R-L >= M):
- y = _div_nearest((M*y) << 1,
- M + _sqrt_nearest(M*(M+_rshift_nearest(y, R)), M))
- R += 1
-
- # Taylor series with T terms
- T = -int(-10*len(str(M))//(3*L))
- yshift = _rshift_nearest(y, R)
- w = _div_nearest(M, T)
- for k in range(T-1, 0, -1):
- w = _div_nearest(M, k) - _div_nearest(yshift*w, M)
-
- return _div_nearest(w*y, M)
-
-def _dlog10(c, e, p):
- """Given integers c, e and p with c > 0, p >= 0, compute an integer
- approximation to 10**p * log10(c*10**e), with an absolute error of
- at most 1. Assumes that c*10**e is not exactly 1."""
-
- # increase precision by 2; compensate for this by dividing
- # final result by 100
- p += 2
-
- # write c*10**e as d*10**f with either:
- # f >= 0 and 1 <= d <= 10, or
- # f <= 0 and 0.1 <= d <= 1.
- # Thus for c*10**e close to 1, f = 0
- l = len(str(c))
- f = e+l - (e+l >= 1)
-
- if p > 0:
- M = 10**p
- k = e+p-f
- if k >= 0:
- c *= 10**k
- else:
- c = _div_nearest(c, 10**-k)
-
- log_d = _ilog(c, M) # error < 5 + 22 = 27
- log_10 = _log10_digits(p) # error < 1
- log_d = _div_nearest(log_d*M, log_10)
- log_tenpower = f*M # exact
- else:
- log_d = 0 # error < 2.31
- log_tenpower = _div_nearest(f, 10**-p) # error < 0.5
-
- return _div_nearest(log_tenpower+log_d, 100)
-
-def _dlog(c, e, p):
- """Given integers c, e and p with c > 0, compute an integer
- approximation to 10**p * log(c*10**e), with an absolute error of
- at most 1. Assumes that c*10**e is not exactly 1."""
-
- # Increase precision by 2. The precision increase is compensated
- # for at the end with a division by 100.
- p += 2
-
- # rewrite c*10**e as d*10**f with either f >= 0 and 1 <= d <= 10,
- # or f <= 0 and 0.1 <= d <= 1. Then we can compute 10**p * log(c*10**e)
- # as 10**p * log(d) + 10**p*f * log(10).
- l = len(str(c))
- f = e+l - (e+l >= 1)
-
- # compute approximation to 10**p*log(d), with error < 27
- if p > 0:
- k = e+p-f
- if k >= 0:
- c *= 10**k
- else:
- c = _div_nearest(c, 10**-k) # error of <= 0.5 in c
-
- # _ilog magnifies existing error in c by a factor of at most 10
- log_d = _ilog(c, 10**p) # error < 5 + 22 = 27
- else:
- # p <= 0: just approximate the whole thing by 0; error < 2.31
- log_d = 0
-
- # compute approximation to f*10**p*log(10), with error < 11.
- if f:
- extra = len(str(abs(f)))-1
- if p + extra >= 0:
- # error in f * _log10_digits(p+extra) < |f| * 1 = |f|
- # after division, error < |f|/10**extra + 0.5 < 10 + 0.5 < 11
- f_log_ten = _div_nearest(f*_log10_digits(p+extra), 10**extra)
- else:
- f_log_ten = 0
- else:
- f_log_ten = 0
-
- # error in sum < 11+27 = 38; error after division < 0.38 + 0.5 < 1
- return _div_nearest(f_log_ten + log_d, 100)
-
-class _Log10Memoize(object):
- """Class to compute, store, and allow retrieval of, digits of the
- constant log(10) = 2.302585.... This constant is needed by
- Decimal.ln, Decimal.log10, Decimal.exp and Decimal.__pow__."""
- def __init__(self):
- self.digits = "23025850929940456840179914546843642076011014886"
-
- def getdigits(self, p):
- """Given an integer p >= 0, return floor(10**p)*log(10).
-
- For example, self.getdigits(3) returns 2302.
- """
- # digits are stored as a string, for quick conversion to
- # integer in the case that we've already computed enough
- # digits; the stored digits should always be correct
- # (truncated, not rounded to nearest).
- if p < 0:
- raise ValueError("p should be nonnegative")
-
- if p >= len(self.digits):
- # compute p+3, p+6, p+9, ... digits; continue until at
- # least one of the extra digits is nonzero
- extra = 3
- while True:
- # compute p+extra digits, correct to within 1ulp
- M = 10**(p+extra+2)
- digits = str(_div_nearest(_ilog(10*M, M), 100))
- if digits[-extra:] != '0'*extra:
- break
- extra += 3
- # keep all reliable digits so far; remove trailing zeros
- # and next nonzero digit
- self.digits = digits.rstrip('0')[:-1]
- return int(self.digits[:p+1])
-
-_log10_digits = _Log10Memoize().getdigits
-
-def _iexp(x, M, L=8):
- """Given integers x and M, M > 0, such that x/M is small in absolute
- value, compute an integer approximation to M*exp(x/M). For 0 <=
- x/M <= 2.4, the absolute error in the result is bounded by 60 (and
- is usually much smaller)."""
-
- # Algorithm: to compute exp(z) for a real number z, first divide z
- # by a suitable power R of 2 so that |z/2**R| < 2**-L. Then
- # compute expm1(z/2**R) = exp(z/2**R) - 1 using the usual Taylor
- # series
- #
- # expm1(x) = x + x**2/2! + x**3/3! + ...
- #
- # Now use the identity
- #
- # expm1(2x) = expm1(x)*(expm1(x)+2)
- #
- # R times to compute the sequence expm1(z/2**R),
- # expm1(z/2**(R-1)), ... , exp(z/2), exp(z).
-
- # Find R such that x/2**R/M <= 2**-L
- R = _nbits((x<<L)//M)
-
- # Taylor series. (2**L)**T > M
- T = -int(-10*len(str(M))//(3*L))
- y = _div_nearest(x, T)
- Mshift = M<<R
- for i in range(T-1, 0, -1):
- y = _div_nearest(x*(Mshift + y), Mshift * i)
-
- # Expansion
- for k in range(R-1, -1, -1):
- Mshift = M<<(k+2)
- y = _div_nearest(y*(y+Mshift), Mshift)
-
- return M+y
-
-def _dexp(c, e, p):
- """Compute an approximation to exp(c*10**e), with p decimal places of
- precision.
-
- Returns integers d, f such that:
-
- 10**(p-1) <= d <= 10**p, and
- (d-1)*10**f < exp(c*10**e) < (d+1)*10**f
-
- In other words, d*10**f is an approximation to exp(c*10**e) with p
- digits of precision, and with an error in d of at most 1. This is
- almost, but not quite, the same as the error being < 1ulp: when d
- = 10**(p-1) the error could be up to 10 ulp."""
-
- # we'll call iexp with M = 10**(p+2), giving p+3 digits of precision
- p += 2
-
- # compute log(10) with extra precision = adjusted exponent of c*10**e
- extra = max(0, e + len(str(c)) - 1)
- q = p + extra
-
- # compute quotient c*10**e/(log(10)) = c*10**(e+q)/(log(10)*10**q),
- # rounding down
- shift = e+q
- if shift >= 0:
- cshift = c*10**shift
- else:
- cshift = c//10**-shift
- quot, rem = divmod(cshift, _log10_digits(q))
-
- # reduce remainder back to original precision
- rem = _div_nearest(rem, 10**extra)
-
- # error in result of _iexp < 120; error after division < 0.62
- return _div_nearest(_iexp(rem, 10**p), 1000), quot - p + 3
-
-def _dpower(xc, xe, yc, ye, p):
- """Given integers xc, xe, yc and ye representing Decimals x = xc*10**xe and
- y = yc*10**ye, compute x**y. Returns a pair of integers (c, e) such that:
-
- 10**(p-1) <= c <= 10**p, and
- (c-1)*10**e < x**y < (c+1)*10**e
-
- in other words, c*10**e is an approximation to x**y with p digits
- of precision, and with an error in c of at most 1. (This is
- almost, but not quite, the same as the error being < 1ulp: when c
- == 10**(p-1) we can only guarantee error < 10ulp.)
-
- We assume that: x is positive and not equal to 1, and y is nonzero.
- """
-
- # Find b such that 10**(b-1) <= |y| <= 10**b
- b = len(str(abs(yc))) + ye
-
- # log(x) = lxc*10**(-p-b-1), to p+b+1 places after the decimal point
- lxc = _dlog(xc, xe, p+b+1)
-
- # compute product y*log(x) = yc*lxc*10**(-p-b-1+ye) = pc*10**(-p-1)
- shift = ye-b
- if shift >= 0:
- pc = lxc*yc*10**shift
- else:
- pc = _div_nearest(lxc*yc, 10**-shift)
-
- if pc == 0:
- # we prefer a result that isn't exactly 1; this makes it
- # easier to compute a correctly rounded result in __pow__
- if ((len(str(xc)) + xe >= 1) == (yc > 0)): # if x**y > 1:
- coeff, exp = 10**(p-1)+1, 1-p
- else:
- coeff, exp = 10**p-1, -p
- else:
- coeff, exp = _dexp(pc, -(p+1), p+1)
- coeff = _div_nearest(coeff, 10)
- exp += 1
-
- return coeff, exp
-
-def _log10_lb(c, correction = {
- '1': 100, '2': 70, '3': 53, '4': 40, '5': 31,
- '6': 23, '7': 16, '8': 10, '9': 5}):
- """Compute a lower bound for 100*log10(c) for a positive integer c."""
- if c <= 0:
- raise ValueError("The argument to _log10_lb should be nonnegative.")
- str_c = str(c)
- return 100*len(str_c) - correction[str_c[0]]
-
-##### Helper Functions ####################################################
-
-def _convert_other(other, raiseit=False, allow_float=False):
- """Convert other to Decimal.
-
- Verifies that it's ok to use in an implicit construction.
- If allow_float is true, allow conversion from float; this
- is used in the comparison methods (__eq__ and friends).
-
- """
- if isinstance(other, Decimal):
- return other
- if isinstance(other, int):
- return Decimal(other)
- if allow_float and isinstance(other, float):
- return Decimal.from_float(other)
-
- if raiseit:
- raise TypeError("Unable to convert %s to Decimal" % other)
- return NotImplemented
-
-def _convert_for_comparison(self, other, equality_op=False):
- """Given a Decimal instance self and a Python object other, return
- a pair (s, o) of Decimal instances such that "s op o" is
- equivalent to "self op other" for any of the 6 comparison
- operators "op".
-
- """
- if isinstance(other, Decimal):
- return self, other
-
- # Comparison with a Rational instance (also includes integers):
- # self op n/d <=> self*d op n (for n and d integers, d positive).
- # A NaN or infinity can be left unchanged without affecting the
- # comparison result.
- if isinstance(other, _numbers.Rational):
- if not self._is_special:
- self = _dec_from_triple(self._sign,
- str(int(self._int) * other.denominator),
- self._exp)
- return self, Decimal(other.numerator)
-
- # Comparisons with float and complex types. == and != comparisons
- # with complex numbers should succeed, returning either True or False
- # as appropriate. Other comparisons return NotImplemented.
- if equality_op and isinstance(other, _numbers.Complex) and other.imag == 0:
- other = other.real
- if isinstance(other, float):
- context = getcontext()
- if equality_op:
- context.flags[FloatOperation] = 1
- else:
- context._raise_error(FloatOperation,
- "strict semantics for mixing floats and Decimals are enabled")
- return self, Decimal.from_float(other)
- return NotImplemented, NotImplemented
-
-
-##### Setup Specific Contexts ############################################
-
-# The default context prototype used by Context()
-# Is mutable, so that new contexts can have different default values
-
-DefaultContext = Context(
- prec=28, rounding=ROUND_HALF_EVEN,
- traps=[DivisionByZero, Overflow, InvalidOperation],
- flags=[],
- Emax=999999,
- Emin=-999999,
- capitals=1,
- clamp=0
-)
-
-# Pre-made alternate contexts offered by the specification
-# Don't change these; the user should be able to select these
-# contexts and be able to reproduce results from other implementations
-# of the spec.
-
-BasicContext = Context(
- prec=9, rounding=ROUND_HALF_UP,
- traps=[DivisionByZero, Overflow, InvalidOperation, Clamped, Underflow],
- flags=[],
-)
-
-ExtendedContext = Context(
- prec=9, rounding=ROUND_HALF_EVEN,
- traps=[],
- flags=[],
-)
-
-
-##### crud for parsing strings #############################################
-#
-# Regular expression used for parsing numeric strings. Additional
-# comments:
-#
-# 1. Uncomment the two '\s*' lines to allow leading and/or trailing
-# whitespace. But note that the specification disallows whitespace in
-# a numeric string.
-#
-# 2. For finite numbers (not infinities and NaNs) the body of the
-# number between the optional sign and the optional exponent must have
-# at least one decimal digit, possibly after the decimal point. The
-# lookahead expression '(?=\d|\.\d)' checks this.
-
-import re
-_parser = re.compile(r""" # A numeric string consists of:
-# \s*
- (?P<sign>[-+])? # an optional sign, followed by either...
- (
- (?=\d|\.\d) # ...a number (with at least one digit)
- (?P<int>\d*) # having a (possibly empty) integer part
- (\.(?P<frac>\d*))? # followed by an optional fractional part
- (E(?P<exp>[-+]?\d+))? # followed by an optional exponent, or...
- |
- Inf(inity)? # ...an infinity, or...
- |
- (?P<signal>s)? # ...an (optionally signaling)
- NaN # NaN
- (?P<diag>\d*) # with (possibly empty) diagnostic info.
- )
-# \s*
- \Z
-""", re.VERBOSE | re.IGNORECASE).match
-
-_all_zeros = re.compile('0*$').match
-_exact_half = re.compile('50*$').match
-
-##### PEP3101 support functions ##############################################
-# The functions in this section have little to do with the Decimal
-# class, and could potentially be reused or adapted for other pure
-# Python numeric classes that want to implement __format__
-#
-# A format specifier for Decimal looks like:
-#
-# [[fill]align][sign][#][0][minimumwidth][,][.precision][type]
-
-_parse_format_specifier_regex = re.compile(r"""\A
-(?:
- (?P<fill>.)?
- (?P<align>[<>=^])
-)?
-(?P<sign>[-+ ])?
-(?P<alt>\#)?
-(?P<zeropad>0)?
-(?P<minimumwidth>(?!0)\d+)?
-(?P<thousands_sep>,)?
-(?:\.(?P<precision>0|(?!0)\d+))?
-(?P<type>[eEfFgGn%])?
-\Z
-""", re.VERBOSE|re.DOTALL)
-
-del re
-
-# The locale module is only needed for the 'n' format specifier. The
-# rest of the PEP 3101 code functions quite happily without it, so we
-# don't care too much if locale isn't present.
-try:
- import locale as _locale
-except ImportError:
- pass
-
-def _parse_format_specifier(format_spec, _localeconv=None):
- """Parse and validate a format specifier.
-
- Turns a standard numeric format specifier into a dict, with the
- following entries:
-
- fill: fill character to pad field to minimum width
- align: alignment type, either '<', '>', '=' or '^'
- sign: either '+', '-' or ' '
- minimumwidth: nonnegative integer giving minimum width
- zeropad: boolean, indicating whether to pad with zeros
- thousands_sep: string to use as thousands separator, or ''
- grouping: grouping for thousands separators, in format
- used by localeconv
- decimal_point: string to use for decimal point
- precision: nonnegative integer giving precision, or None
- type: one of the characters 'eEfFgG%', or None
-
- """
- m = _parse_format_specifier_regex.match(format_spec)
- if m is None:
- raise ValueError("Invalid format specifier: " + format_spec)
-
- # get the dictionary
- format_dict = m.groupdict()
-
- # zeropad; defaults for fill and alignment. If zero padding
- # is requested, the fill and align fields should be absent.
- fill = format_dict['fill']
- align = format_dict['align']
- format_dict['zeropad'] = (format_dict['zeropad'] is not None)
- if format_dict['zeropad']:
- if fill is not None:
- raise ValueError("Fill character conflicts with '0'"
- " in format specifier: " + format_spec)
- if align is not None:
- raise ValueError("Alignment conflicts with '0' in "
- "format specifier: " + format_spec)
- format_dict['fill'] = fill or ' '
- # PEP 3101 originally specified that the default alignment should
- # be left; it was later agreed that right-aligned makes more sense
- # for numeric types. See http://bugs.python.org/issue6857.
- format_dict['align'] = align or '>'
-
- # default sign handling: '-' for negative, '' for positive
- if format_dict['sign'] is None:
- format_dict['sign'] = '-'
-
- # minimumwidth defaults to 0; precision remains None if not given
- format_dict['minimumwidth'] = int(format_dict['minimumwidth'] or '0')
- if format_dict['precision'] is not None:
- format_dict['precision'] = int(format_dict['precision'])
-
- # if format type is 'g' or 'G' then a precision of 0 makes little
- # sense; convert it to 1. Same if format type is unspecified.
- if format_dict['precision'] == 0:
- if format_dict['type'] is None or format_dict['type'] in 'gGn':
- format_dict['precision'] = 1
-
- # determine thousands separator, grouping, and decimal separator, and
- # add appropriate entries to format_dict
- if format_dict['type'] == 'n':
- # apart from separators, 'n' behaves just like 'g'
- format_dict['type'] = 'g'
- if _localeconv is None:
- _localeconv = _locale.localeconv()
- if format_dict['thousands_sep'] is not None:
- raise ValueError("Explicit thousands separator conflicts with "
- "'n' type in format specifier: " + format_spec)
- format_dict['thousands_sep'] = _localeconv['thousands_sep']
- format_dict['grouping'] = _localeconv['grouping']
- format_dict['decimal_point'] = _localeconv['decimal_point']
- else:
- if format_dict['thousands_sep'] is None:
- format_dict['thousands_sep'] = ''
- format_dict['grouping'] = [3, 0]
- format_dict['decimal_point'] = '.'
-
- return format_dict
-
-def _format_align(sign, body, spec):
- """Given an unpadded, non-aligned numeric string 'body' and sign
- string 'sign', add padding and alignment conforming to the given
- format specifier dictionary 'spec' (as produced by
- parse_format_specifier).
-
- """
- # how much extra space do we have to play with?
- minimumwidth = spec['minimumwidth']
- fill = spec['fill']
- padding = fill*(minimumwidth - len(sign) - len(body))
-
- align = spec['align']
- if align == '<':
- result = sign + body + padding
- elif align == '>':
- result = padding + sign + body
- elif align == '=':
- result = sign + padding + body
- elif align == '^':
- half = len(padding)//2
- result = padding[:half] + sign + body + padding[half:]
- else:
- raise ValueError('Unrecognised alignment field')
-
- return result
-
-def _group_lengths(grouping):
- """Convert a localeconv-style grouping into a (possibly infinite)
- iterable of integers representing group lengths.
-
- """
- # The result from localeconv()['grouping'], and the input to this
- # function, should be a list of integers in one of the
- # following three forms:
- #
- # (1) an empty list, or
- # (2) nonempty list of positive integers + [0]
- # (3) list of positive integers + [locale.CHAR_MAX], or
-
- from itertools import chain, repeat
- if not grouping:
- return []
- elif grouping[-1] == 0 and len(grouping) >= 2:
- return chain(grouping[:-1], repeat(grouping[-2]))
- elif grouping[-1] == _locale.CHAR_MAX:
- return grouping[:-1]
- else:
- raise ValueError('unrecognised format for grouping')
-
-def _insert_thousands_sep(digits, spec, min_width=1):
- """Insert thousands separators into a digit string.
-
- spec is a dictionary whose keys should include 'thousands_sep' and
- 'grouping'; typically it's the result of parsing the format
- specifier using _parse_format_specifier.
-
- The min_width keyword argument gives the minimum length of the
- result, which will be padded on the left with zeros if necessary.
-
- If necessary, the zero padding adds an extra '0' on the left to
- avoid a leading thousands separator. For example, inserting
- commas every three digits in '123456', with min_width=8, gives
- '0,123,456', even though that has length 9.
-
- """
-
- sep = spec['thousands_sep']
- grouping = spec['grouping']
-
- groups = []
- for l in _group_lengths(grouping):
- if l <= 0:
- raise ValueError("group length should be positive")
- # max(..., 1) forces at least 1 digit to the left of a separator
- l = min(max(len(digits), min_width, 1), l)
- groups.append('0'*(l - len(digits)) + digits[-l:])
- digits = digits[:-l]
- min_width -= l
- if not digits and min_width <= 0:
- break
- min_width -= len(sep)
- else:
- l = max(len(digits), min_width, 1)
- groups.append('0'*(l - len(digits)) + digits[-l:])
- return sep.join(reversed(groups))
-
-def _format_sign(is_negative, spec):
- """Determine sign character."""
-
- if is_negative:
- return '-'
- elif spec['sign'] in ' +':
- return spec['sign']
- else:
- return ''
-
-def _format_number(is_negative, intpart, fracpart, exp, spec):
- """Format a number, given the following data:
-
- is_negative: true if the number is negative, else false
- intpart: string of digits that must appear before the decimal point
- fracpart: string of digits that must come after the point
- exp: exponent, as an integer
- spec: dictionary resulting from parsing the format specifier
-
- This function uses the information in spec to:
- insert separators (decimal separator and thousands separators)
- format the sign
- format the exponent
- add trailing '%' for the '%' type
- zero-pad if necessary
- fill and align if necessary
- """
-
- sign = _format_sign(is_negative, spec)
-
- if fracpart or spec['alt']:
- fracpart = spec['decimal_point'] + fracpart
-
- if exp != 0 or spec['type'] in 'eE':
- echar = {'E': 'E', 'e': 'e', 'G': 'E', 'g': 'e'}[spec['type']]
- fracpart += "{0}{1:+}".format(echar, exp)
- if spec['type'] == '%':
- fracpart += '%'
-
- if spec['zeropad']:
- min_width = spec['minimumwidth'] - len(fracpart) - len(sign)
- else:
- min_width = 0
- intpart = _insert_thousands_sep(intpart, spec, min_width)
-
- return _format_align(sign, intpart+fracpart, spec)
-
-
-##### Useful Constants (internal use only) ################################
-
-# Reusable defaults
-_Infinity = Decimal('Inf')
-_NegativeInfinity = Decimal('-Inf')
-_NaN = Decimal('NaN')
-_Zero = Decimal(0)
-_One = Decimal(1)
-_NegativeOne = Decimal(-1)
-
-# _SignedInfinity[sign] is infinity w/ that sign
-_SignedInfinity = (_Infinity, _NegativeInfinity)
-
-# Constants related to the hash implementation; hash(x) is based
-# on the reduction of x modulo _PyHASH_MODULUS
-_PyHASH_MODULUS = sys.hash_info.modulus
-# hash values to use for positive and negative infinities, and nans
-_PyHASH_INF = sys.hash_info.inf
-_PyHASH_NAN = sys.hash_info.nan
-
-# _PyHASH_10INV is the inverse of 10 modulo the prime _PyHASH_MODULUS
-_PyHASH_10INV = pow(10, _PyHASH_MODULUS - 2, _PyHASH_MODULUS)
-del sys
try:
- import _decimal
-except ImportError:
- pass
-else:
- s1 = set(dir())
- s2 = set(dir(_decimal))
- for name in s1 - s2:
- del globals()[name]
- del s1, s2, name
from _decimal import *
-
-if __name__ == '__main__':
- import doctest, decimal
- doctest.testmod(decimal)
+ from _decimal import __doc__
+ from _decimal import __version__
+ from _decimal import __libmpdec_version__
+except ImportError:
+ from _pydecimal import *
+ from _pydecimal import __doc__
+ from _pydecimal import __version__
+ from _pydecimal import __libmpdec_version__
diff --git a/Lib/difflib.py b/Lib/difflib.py
index 7eb42a9..22d9145 100644
--- a/Lib/difflib.py
+++ b/Lib/difflib.py
@@ -28,9 +28,9 @@ Class HtmlDiff:
__all__ = ['get_close_matches', 'ndiff', 'restore', 'SequenceMatcher',
'Differ','IS_CHARACTER_JUNK', 'IS_LINE_JUNK', 'context_diff',
- 'unified_diff', 'HtmlDiff', 'Match']
+ 'unified_diff', 'diff_bytes', 'HtmlDiff', 'Match']
-import heapq
+from heapq import nlargest as _nlargest
from collections import namedtuple as _namedtuple
Match = _namedtuple('Match', 'a b size')
@@ -729,7 +729,7 @@ def get_close_matches(word, possibilities, n=3, cutoff=0.6):
result.append((s.ratio(), x))
# Move the best scorers to head of list
- result = heapq.nlargest(n, result)
+ result = _nlargest(n, result)
# Strip scores for the best n matches
return [x for score, x in result]
@@ -852,10 +852,9 @@ class Differ:
and return true iff the string is junk. The module-level function
`IS_LINE_JUNK` may be used to filter out lines without visible
characters, except for at most one splat ('#'). It is recommended
- to leave linejunk None; as of Python 2.3, the underlying
- SequenceMatcher class has grown an adaptive notion of "noise" lines
- that's better than any static definition the author has ever been
- able to craft.
+ to leave linejunk None; the underlying SequenceMatcher class has
+ an adaptive notion of "noise" lines that's better than any static
+ definition the author has ever been able to craft.
- `charjunk`: A function that should accept a string of length 1. The
module-level function `IS_CHARACTER_JUNK` may be used to filter out
@@ -1175,6 +1174,7 @@ def unified_diff(a, b, fromfile='', tofile='', fromfiledate='',
four
"""
+ _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm)
started = False
for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n):
if not started:
@@ -1262,6 +1262,7 @@ def context_diff(a, b, fromfile='', tofile='',
four
"""
+ _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm)
prefix = dict(insert='+ ', delete='- ', replace='! ', equal=' ')
started = False
for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n):
@@ -1293,22 +1294,70 @@ def context_diff(a, b, fromfile='', tofile='',
for line in b[j1:j2]:
yield prefix[tag] + line
+def _check_types(a, b, *args):
+ # Checking types is weird, but the alternative is garbled output when
+ # someone passes mixed bytes and str to {unified,context}_diff(). E.g.
+ # without this check, passing filenames as bytes results in output like
+ # --- b'oldfile.txt'
+ # +++ b'newfile.txt'
+ # because of how str.format() incorporates bytes objects.
+ if a and not isinstance(a[0], str):
+ raise TypeError('lines to compare must be str, not %s (%r)' %
+ (type(a[0]).__name__, a[0]))
+ if b and not isinstance(b[0], str):
+ raise TypeError('lines to compare must be str, not %s (%r)' %
+ (type(b[0]).__name__, b[0]))
+ for arg in args:
+ if not isinstance(arg, str):
+ raise TypeError('all arguments must be str, not: %r' % (arg,))
+
+def diff_bytes(dfunc, a, b, fromfile=b'', tofile=b'',
+ fromfiledate=b'', tofiledate=b'', n=3, lineterm=b'\n'):
+ r"""
+ Compare `a` and `b`, two sequences of lines represented as bytes rather
+ than str. This is a wrapper for `dfunc`, which is typically either
+ unified_diff() or context_diff(). Inputs are losslessly converted to
+ strings so that `dfunc` only has to worry about strings, and encoded
+ back to bytes on return. This is necessary to compare files with
+ unknown or inconsistent encoding. All other inputs (except `n`) must be
+ bytes rather than str.
+ """
+ def decode(s):
+ try:
+ return s.decode('ascii', 'surrogateescape')
+ except AttributeError as err:
+ msg = ('all arguments must be bytes, not %s (%r)' %
+ (type(s).__name__, s))
+ raise TypeError(msg) from err
+ a = list(map(decode, a))
+ b = list(map(decode, b))
+ fromfile = decode(fromfile)
+ tofile = decode(tofile)
+ fromfiledate = decode(fromfiledate)
+ tofiledate = decode(tofiledate)
+ lineterm = decode(lineterm)
+
+ lines = dfunc(a, b, fromfile, tofile, fromfiledate, tofiledate, n, lineterm)
+ for line in lines:
+ yield line.encode('ascii', 'surrogateescape')
+
def ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK):
r"""
Compare `a` and `b` (lists of strings); return a `Differ`-style delta.
Optional keyword parameters `linejunk` and `charjunk` are for filter
- functions (or None):
+ functions, or can be None:
- - linejunk: A function that should accept a single string argument, and
+ - linejunk: A function that should accept a single string argument and
return true iff the string is junk. The default is None, and is
- recommended; as of Python 2.3, an adaptive notion of "noise" lines is
- used that does a good job on its own.
+ recommended; the underlying SequenceMatcher class has an adaptive
+ notion of "noise" lines.
- - charjunk: A function that should accept a string of length 1. The
- default is module-level function IS_CHARACTER_JUNK, which filters out
- whitespace characters (a blank or tab; note: bad idea to include newline
- in this!).
+ - charjunk: A function that accepts a character (string of length
+ 1), and returns true iff the character is junk. The default is
+ the module-level function IS_CHARACTER_JUNK, which filters out
+ whitespace characters (a blank or tab; note: it's a bad idea to
+ include newline in this!).
Tools/scripts/ndiff.py is a command-line front-end to this function.
@@ -1410,7 +1459,7 @@ def _mdiff(fromlines, tolines, context=None, linejunk=None,
change_re.sub(record_sub_info,markers)
# process each tuple inserting our special marks that won't be
# noticed by an xml/html escaper.
- for key,(begin,end) in sub_info[::-1]:
+ for key,(begin,end) in reversed(sub_info):
text = text[0:begin]+'\0'+key+text[begin:end]+'\1'+text[end:]
text = text[2:]
# Handle case of add/delete entire line
@@ -1448,10 +1497,7 @@ def _mdiff(fromlines, tolines, context=None, linejunk=None,
# are a concatenation of the first character of each of the 4 lines
# so we can do some very readable comparisons.
while len(lines) < 4:
- try:
- lines.append(next(diff_lines_iterator))
- except StopIteration:
- lines.append('X')
+ lines.append(next(diff_lines_iterator, 'X'))
s = ''.join([line[0] for line in lines])
if s.startswith('X'):
# When no more lines, pump out any remaining blank lines so the
@@ -1514,7 +1560,7 @@ def _mdiff(fromlines, tolines, context=None, linejunk=None,
num_blanks_to_yield -= 1
yield ('','\n'),None,True
if s.startswith('X'):
- raise StopIteration
+ return
else:
yield from_line,to_line,True
@@ -1536,7 +1582,10 @@ def _mdiff(fromlines, tolines, context=None, linejunk=None,
while True:
# Collecting lines of text until we have a from/to pair
while (len(fromlines)==0 or len(tolines)==0):
- from_line, to_line, found_diff = next(line_iterator)
+ try:
+ from_line, to_line, found_diff = next(line_iterator)
+ except StopIteration:
+ return
if from_line is not None:
fromlines.append((from_line,found_diff))
if to_line is not None:
@@ -1550,8 +1599,7 @@ def _mdiff(fromlines, tolines, context=None, linejunk=None,
# them up without doing anything else with them.
line_pair_iterator = _line_pair_iterator()
if context is None:
- while True:
- yield next(line_pair_iterator)
+ yield from line_pair_iterator
# Handle case where user wants context differencing. We must do some
# storage of lines until we know for sure that they are to be yielded.
else:
@@ -1564,7 +1612,10 @@ def _mdiff(fromlines, tolines, context=None, linejunk=None,
index, contextLines = 0, [None]*(context)
found_diff = False
while(found_diff is False):
- from_line, to_line, found_diff = next(line_pair_iterator)
+ try:
+ from_line, to_line, found_diff = next(line_pair_iterator)
+ except StopIteration:
+ return
i = index % context
contextLines[i] = (from_line, to_line, found_diff)
index += 1
@@ -1601,7 +1652,7 @@ _file_template = """
<head>
<meta http-equiv="Content-Type"
- content="text/html; charset=ISO-8859-1" />
+ content="text/html; charset=%(charset)s" />
<title></title>
<style type="text/css">%(styles)s
</style>
@@ -1679,7 +1730,7 @@ class HtmlDiff(object):
tabsize -- tab stop spacing, defaults to 8.
wrapcolumn -- column number where lines are broken and wrapped,
defaults to None where lines are not wrapped.
- linejunk,charjunk -- keyword arguments passed into ndiff() (used to by
+ linejunk,charjunk -- keyword arguments passed into ndiff() (used by
HtmlDiff() to generate the side by side HTML differences). See
ndiff() documentation for argument default values and descriptions.
"""
@@ -1688,8 +1739,8 @@ class HtmlDiff(object):
self._linejunk = linejunk
self._charjunk = charjunk
- def make_file(self,fromlines,tolines,fromdesc='',todesc='',context=False,
- numlines=5):
+ def make_file(self, fromlines, tolines, fromdesc='', todesc='',
+ context=False, numlines=5, *, charset='utf-8'):
"""Returns HTML file of side by side comparison with change highlights
Arguments:
@@ -1704,13 +1755,16 @@ class HtmlDiff(object):
When context is False, controls the number of lines to place
the "next" link anchors before the next change (so click of
"next" link jumps to just before the change).
+ charset -- charset of the HTML document
"""
- return self._file_template % dict(
- styles = self._styles,
- legend = self._legend,
- table = self.make_table(fromlines,tolines,fromdesc,todesc,
- context=context,numlines=numlines))
+ return (self._file_template % dict(
+ styles=self._styles,
+ legend=self._legend,
+ table=self.make_table(fromlines, tolines, fromdesc, todesc,
+ context=context, numlines=numlines),
+ charset=charset
+ )).encode(charset, 'xmlcharrefreplace').decode(charset)
def _tab_newline_replace(self,fromlines,tolines):
"""Returns from/to line lists with tabs expanded and newlines removed.
diff --git a/Lib/dis.py b/Lib/dis.py
index 81cbe7f..af37cdf 100644
--- a/Lib/dis.py
+++ b/Lib/dis.py
@@ -29,7 +29,7 @@ def _try_compile(source, name):
return c
def dis(x=None, *, file=None):
- """Disassemble classes, methods, functions, or code.
+ """Disassemble classes, methods, functions, generators, or code.
With no argument, disassemble the last traceback.
@@ -41,6 +41,8 @@ def dis(x=None, *, file=None):
x = x.__func__
if hasattr(x, '__code__'): # Function
x = x.__code__
+ if hasattr(x, 'gi_code'): # Generator
+ x = x.gi_code
if hasattr(x, '__dict__'): # Class or module
items = sorted(x.__dict__.items())
for name, x1 in items:
@@ -82,6 +84,8 @@ COMPILER_FLAG_NAMES = {
16: "NESTED",
32: "GENERATOR",
64: "NOFREE",
+ 128: "COROUTINE",
+ 256: "ITERABLE_COROUTINE",
}
def pretty_flags(flags):
@@ -99,11 +103,13 @@ def pretty_flags(flags):
return ", ".join(names)
def _get_code_object(x):
- """Helper to handle methods, functions, strings and raw code objects"""
+ """Helper to handle methods, functions, generators, strings and raw code objects"""
if hasattr(x, '__func__'): # Method
x = x.__func__
if hasattr(x, '__code__'): # Function
x = x.__code__
+ if hasattr(x, 'gi_code'): # Generator
+ x = x.gi_code
if isinstance(x, str): # Source code
x = _try_compile(x, "<disassembly>")
if hasattr(x, 'co_code'): # Code object
diff --git a/Lib/distutils/_msvccompiler.py b/Lib/distutils/_msvccompiler.py
new file mode 100644
index 0000000..b344616
--- /dev/null
+++ b/Lib/distutils/_msvccompiler.py
@@ -0,0 +1,494 @@
+"""distutils._msvccompiler
+
+Contains MSVCCompiler, an implementation of the abstract CCompiler class
+for Microsoft Visual Studio 2015.
+
+The module is compatible with VS 2015 and later. You can find legacy support
+for older versions in distutils.msvc9compiler and distutils.msvccompiler.
+"""
+
+# Written by Perry Stoll
+# hacked by Robin Becker and Thomas Heller to do a better job of
+# finding DevStudio (through the registry)
+# ported to VS 2005 and VS 2008 by Christian Heimes
+# ported to VS 2015 by Steve Dower
+
+import os
+import subprocess
+
+from distutils.errors import DistutilsExecError, DistutilsPlatformError, \
+ CompileError, LibError, LinkError
+from distutils.ccompiler import CCompiler, gen_lib_options
+from distutils import log
+from distutils.util import get_platform
+
+import winreg
+from itertools import count
+
+def _find_vcvarsall():
+ with winreg.OpenKeyEx(
+ winreg.HKEY_LOCAL_MACHINE,
+ r"Software\Microsoft\VisualStudio\SxS\VC7",
+ access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY
+ ) as key:
+ if not key:
+ log.debug("Visual C++ is not registered")
+ return None
+
+ best_version = 0
+ best_dir = None
+ for i in count():
+ try:
+ v, vc_dir, vt = winreg.EnumValue(key, i)
+ except OSError:
+ break
+ if v and vt == winreg.REG_SZ and os.path.isdir(vc_dir):
+ try:
+ version = int(float(v))
+ except (ValueError, TypeError):
+ continue
+ if version >= 14 and version > best_version:
+ best_version, best_dir = version, vc_dir
+ if not best_version:
+ log.debug("No suitable Visual C++ version found")
+ return None
+
+ vcvarsall = os.path.join(best_dir, "vcvarsall.bat")
+ if not os.path.isfile(vcvarsall):
+ log.debug("%s cannot be found", vcvarsall)
+ return None
+
+ return vcvarsall
+
+def _get_vc_env(plat_spec):
+ if os.getenv("DISTUTILS_USE_SDK"):
+ return {
+ key.lower(): value
+ for key, value in os.environ.items()
+ }
+
+ vcvarsall = _find_vcvarsall()
+ if not vcvarsall:
+ raise DistutilsPlatformError("Unable to find vcvarsall.bat")
+
+ try:
+ out = subprocess.check_output(
+ '"{}" {} && set'.format(vcvarsall, plat_spec),
+ shell=True,
+ stderr=subprocess.STDOUT,
+ universal_newlines=True,
+ )
+ except subprocess.CalledProcessError as exc:
+ log.error(exc.output)
+ raise DistutilsPlatformError("Error executing {}"
+ .format(exc.cmd))
+
+ return {
+ key.lower(): value
+ for key, _, value in
+ (line.partition('=') for line in out.splitlines())
+ if key and value
+ }
+
+def _find_exe(exe, paths=None):
+ """Return path to an MSVC executable program.
+
+ Tries to find the program in several places: first, one of the
+ MSVC program search paths from the registry; next, the directories
+ in the PATH environment variable. If any of those work, return an
+ absolute path that is known to exist. If none of them work, just
+ return the original program name, 'exe'.
+ """
+ if not paths:
+ paths = os.getenv('path').split(os.pathsep)
+ for p in paths:
+ fn = os.path.join(os.path.abspath(p), exe)
+ if os.path.isfile(fn):
+ return fn
+ return exe
+
+# A map keyed by get_platform() return values to values accepted by
+# 'vcvarsall.bat'. Note a cross-compile may combine these (eg, 'x86_amd64' is
+# the param to cross-compile on x86 targetting amd64.)
+PLAT_TO_VCVARS = {
+ 'win32' : 'x86',
+ 'win-amd64' : 'amd64',
+}
+
+class MSVCCompiler(CCompiler) :
+ """Concrete class that implements an interface to Microsoft Visual C++,
+ as defined by the CCompiler abstract class."""
+
+ compiler_type = 'msvc'
+
+ # Just set this so CCompiler's constructor doesn't barf. We currently
+ # don't use the 'set_executables()' bureaucracy provided by CCompiler,
+ # as it really isn't necessary for this sort of single-compiler class.
+ # Would be nice to have a consistent interface with UnixCCompiler,
+ # though, so it's worth thinking about.
+ executables = {}
+
+ # Private class data (need to distinguish C from C++ source for compiler)
+ _c_extensions = ['.c']
+ _cpp_extensions = ['.cc', '.cpp', '.cxx']
+ _rc_extensions = ['.rc']
+ _mc_extensions = ['.mc']
+
+ # Needed for the filename generation methods provided by the
+ # base class, CCompiler.
+ src_extensions = (_c_extensions + _cpp_extensions +
+ _rc_extensions + _mc_extensions)
+ res_extension = '.res'
+ obj_extension = '.obj'
+ static_lib_extension = '.lib'
+ shared_lib_extension = '.dll'
+ static_lib_format = shared_lib_format = '%s%s'
+ exe_extension = '.exe'
+
+
+ def __init__(self, verbose=0, dry_run=0, force=0):
+ CCompiler.__init__ (self, verbose, dry_run, force)
+ # target platform (.plat_name is consistent with 'bdist')
+ self.plat_name = None
+ self.initialized = False
+
+ def initialize(self, plat_name=None):
+ # multi-init means we would need to check platform same each time...
+ assert not self.initialized, "don't init multiple times"
+ if plat_name is None:
+ plat_name = get_platform()
+ # sanity check for platforms to prevent obscure errors later.
+ if plat_name not in PLAT_TO_VCVARS:
+ raise DistutilsPlatformError("--plat-name must be one of {}"
+ .format(tuple(PLAT_TO_VCVARS)))
+
+ # On x86, 'vcvarsall.bat amd64' creates an env that doesn't work;
+ # to cross compile, you use 'x86_amd64'.
+ # On AMD64, 'vcvarsall.bat amd64' is a native build env; to cross
+ # compile use 'x86' (ie, it runs the x86 compiler directly)
+ if plat_name == get_platform() or plat_name == 'win32':
+ # native build or cross-compile to win32
+ plat_spec = PLAT_TO_VCVARS[plat_name]
+ else:
+ # cross compile from win32 -> some 64bit
+ plat_spec = '{}_{}'.format(
+ PLAT_TO_VCVARS[get_platform()],
+ PLAT_TO_VCVARS[plat_name]
+ )
+
+ vc_env = _get_vc_env(plat_spec)
+ if not vc_env:
+ raise DistutilsPlatformError("Unable to find a compatible "
+ "Visual Studio installation.")
+
+ self._paths = vc_env.get('path', '')
+ paths = self._paths.split(os.pathsep)
+ self.cc = _find_exe("cl.exe", paths)
+ self.linker = _find_exe("link.exe", paths)
+ self.lib = _find_exe("lib.exe", paths)
+ self.rc = _find_exe("rc.exe", paths) # resource compiler
+ self.mc = _find_exe("mc.exe", paths) # message compiler
+ self.mt = _find_exe("mt.exe", paths) # message compiler
+
+ for dir in vc_env.get('include', '').split(os.pathsep):
+ if dir:
+ self.add_include_dir(dir)
+
+ for dir in vc_env.get('lib', '').split(os.pathsep):
+ if dir:
+ self.add_library_dir(dir)
+
+ self.preprocess_options = None
+ # Use /MT[d] to build statically, then switch from libucrt[d].lib to ucrt[d].lib
+ # later to dynamically link to ucrtbase but not vcruntime.
+ self.compile_options = [
+ '/nologo', '/Ox', '/MT', '/W3', '/GL', '/DNDEBUG'
+ ]
+ self.compile_options_debug = [
+ '/nologo', '/Od', '/MTd', '/Zi', '/W3', '/D_DEBUG'
+ ]
+
+ ldflags = [
+ '/nologo', '/INCREMENTAL:NO', '/LTCG', '/nodefaultlib:libucrt.lib', 'ucrt.lib',
+ ]
+ ldflags_debug = [
+ '/nologo', '/INCREMENTAL:NO', '/LTCG', '/DEBUG:FULL', '/nodefaultlib:libucrtd.lib', 'ucrtd.lib',
+ ]
+
+ self.ldflags_exe = [*ldflags, '/MANIFEST:EMBED,ID=1']
+ self.ldflags_exe_debug = [*ldflags_debug, '/MANIFEST:EMBED,ID=1']
+ self.ldflags_shared = [*ldflags, '/DLL', '/MANIFEST:EMBED,ID=2', '/MANIFESTUAC:NO']
+ self.ldflags_shared_debug = [*ldflags_debug, '/DLL', '/MANIFEST:EMBED,ID=2', '/MANIFESTUAC:NO']
+ self.ldflags_static = [*ldflags]
+ self.ldflags_static_debug = [*ldflags_debug]
+
+ self._ldflags = {
+ (CCompiler.EXECUTABLE, None): self.ldflags_exe,
+ (CCompiler.EXECUTABLE, False): self.ldflags_exe,
+ (CCompiler.EXECUTABLE, True): self.ldflags_exe_debug,
+ (CCompiler.SHARED_OBJECT, None): self.ldflags_shared,
+ (CCompiler.SHARED_OBJECT, False): self.ldflags_shared,
+ (CCompiler.SHARED_OBJECT, True): self.ldflags_shared_debug,
+ (CCompiler.SHARED_LIBRARY, None): self.ldflags_static,
+ (CCompiler.SHARED_LIBRARY, False): self.ldflags_static,
+ (CCompiler.SHARED_LIBRARY, True): self.ldflags_static_debug,
+ }
+
+ self.initialized = True
+
+ # -- Worker methods ------------------------------------------------
+
+ def object_filenames(self,
+ source_filenames,
+ strip_dir=0,
+ output_dir=''):
+ ext_map = {
+ **{ext: self.obj_extension for ext in self.src_extensions},
+ **{ext: self.res_extension for ext in self._rc_extensions + self._mc_extensions},
+ }
+
+ output_dir = output_dir or ''
+
+ def make_out_path(p):
+ base, ext = os.path.splitext(p)
+ if strip_dir:
+ base = os.path.basename(base)
+ else:
+ _, base = os.path.splitdrive(base)
+ if base.startswith((os.path.sep, os.path.altsep)):
+ base = base[1:]
+ try:
+ # XXX: This may produce absurdly long paths. We should check
+ # the length of the result and trim base until we fit within
+ # 260 characters.
+ return os.path.join(output_dir, base + ext_map[ext])
+ except LookupError:
+ # Better to raise an exception instead of silently continuing
+ # and later complain about sources and targets having
+ # different lengths
+ raise CompileError("Don't know how to compile {}".format(p))
+
+ return list(map(make_out_path, source_filenames))
+
+
+ def compile(self, sources,
+ output_dir=None, macros=None, include_dirs=None, debug=0,
+ extra_preargs=None, extra_postargs=None, depends=None):
+
+ if not self.initialized:
+ self.initialize()
+ compile_info = self._setup_compile(output_dir, macros, include_dirs,
+ sources, depends, extra_postargs)
+ macros, objects, extra_postargs, pp_opts, build = compile_info
+
+ compile_opts = extra_preargs or []
+ compile_opts.append('/c')
+ if debug:
+ compile_opts.extend(self.compile_options_debug)
+ else:
+ compile_opts.extend(self.compile_options)
+
+
+ add_cpp_opts = False
+
+ for obj in objects:
+ try:
+ src, ext = build[obj]
+ except KeyError:
+ continue
+ if debug:
+ # pass the full pathname to MSVC in debug mode,
+ # this allows the debugger to find the source file
+ # without asking the user to browse for it
+ src = os.path.abspath(src)
+
+ if ext in self._c_extensions:
+ input_opt = "/Tc" + src
+ elif ext in self._cpp_extensions:
+ input_opt = "/Tp" + src
+ add_cpp_opts = True
+ elif ext in self._rc_extensions:
+ # compile .RC to .RES file
+ input_opt = src
+ output_opt = "/fo" + obj
+ try:
+ self.spawn([self.rc] + pp_opts + [output_opt, input_opt])
+ except DistutilsExecError as msg:
+ raise CompileError(msg)
+ continue
+ elif ext in self._mc_extensions:
+ # Compile .MC to .RC file to .RES file.
+ # * '-h dir' specifies the directory for the
+ # generated include file
+ # * '-r dir' specifies the target directory of the
+ # generated RC file and the binary message resource
+ # it includes
+ #
+ # For now (since there are no options to change this),
+ # we use the source-directory for the include file and
+ # the build directory for the RC file and message
+ # resources. This works at least for win32all.
+ h_dir = os.path.dirname(src)
+ rc_dir = os.path.dirname(obj)
+ try:
+ # first compile .MC to .RC and .H file
+ self.spawn([self.mc, '-h', h_dir, '-r', rc_dir, src])
+ base, _ = os.path.splitext(os.path.basename (src))
+ rc_file = os.path.join(rc_dir, base + '.rc')
+ # then compile .RC to .RES file
+ self.spawn([self.rc, "/fo" + obj, rc_file])
+
+ except DistutilsExecError as msg:
+ raise CompileError(msg)
+ continue
+ else:
+ # how to handle this file?
+ raise CompileError("Don't know how to compile {} to {}"
+ .format(src, obj))
+
+ args = [self.cc] + compile_opts + pp_opts
+ if add_cpp_opts:
+ args.append('/EHsc')
+ args.append(input_opt)
+ args.append("/Fo" + obj)
+ args.extend(extra_postargs)
+
+ try:
+ self.spawn(args)
+ except DistutilsExecError as msg:
+ raise CompileError(msg)
+
+ return objects
+
+
+ def create_static_lib(self,
+ objects,
+ output_libname,
+ output_dir=None,
+ debug=0,
+ target_lang=None):
+
+ if not self.initialized:
+ self.initialize()
+ objects, output_dir = self._fix_object_args(objects, output_dir)
+ output_filename = self.library_filename(output_libname,
+ output_dir=output_dir)
+
+ if self._need_link(objects, output_filename):
+ lib_args = objects + ['/OUT:' + output_filename]
+ if debug:
+ pass # XXX what goes here?
+ try:
+ log.debug('Executing "%s" %s', self.lib, ' '.join(lib_args))
+ self.spawn([self.lib] + lib_args)
+ except DistutilsExecError as msg:
+ raise LibError(msg)
+ else:
+ log.debug("skipping %s (up-to-date)", output_filename)
+
+
+ def link(self,
+ target_desc,
+ objects,
+ output_filename,
+ output_dir=None,
+ libraries=None,
+ library_dirs=None,
+ runtime_library_dirs=None,
+ export_symbols=None,
+ debug=0,
+ extra_preargs=None,
+ extra_postargs=None,
+ build_temp=None,
+ target_lang=None):
+
+ if not self.initialized:
+ self.initialize()
+ objects, output_dir = self._fix_object_args(objects, output_dir)
+ fixed_args = self._fix_lib_args(libraries, library_dirs,
+ runtime_library_dirs)
+ libraries, library_dirs, runtime_library_dirs = fixed_args
+
+ if runtime_library_dirs:
+ self.warn("I don't know what to do with 'runtime_library_dirs': "
+ + str(runtime_library_dirs))
+
+ lib_opts = gen_lib_options(self,
+ library_dirs, runtime_library_dirs,
+ libraries)
+ if output_dir is not None:
+ output_filename = os.path.join(output_dir, output_filename)
+
+ if self._need_link(objects, output_filename):
+ ldflags = self._ldflags[target_desc, debug]
+
+ export_opts = ["/EXPORT:" + sym for sym in (export_symbols or [])]
+
+ ld_args = (ldflags + lib_opts + export_opts +
+ objects + ['/OUT:' + output_filename])
+
+ # The MSVC linker generates .lib and .exp files, which cannot be
+ # suppressed by any linker switches. The .lib files may even be
+ # needed! Make sure they are generated in the temporary build
+ # directory. Since they have different names for debug and release
+ # builds, they can go into the same directory.
+ build_temp = os.path.dirname(objects[0])
+ if export_symbols is not None:
+ (dll_name, dll_ext) = os.path.splitext(
+ os.path.basename(output_filename))
+ implib_file = os.path.join(
+ build_temp,
+ self.library_filename(dll_name))
+ ld_args.append ('/IMPLIB:' + implib_file)
+
+ if extra_preargs:
+ ld_args[:0] = extra_preargs
+ if extra_postargs:
+ ld_args.extend(extra_postargs)
+
+ self.mkpath(os.path.dirname(output_filename))
+ try:
+ log.debug('Executing "%s" %s', self.linker, ' '.join(ld_args))
+ self.spawn([self.linker] + ld_args)
+ except DistutilsExecError as msg:
+ raise LinkError(msg)
+ else:
+ log.debug("skipping %s (up-to-date)", output_filename)
+
+ def spawn(self, cmd):
+ old_path = os.getenv('path')
+ try:
+ os.environ['path'] = self._paths
+ return super().spawn(cmd)
+ finally:
+ os.environ['path'] = old_path
+
+ # -- Miscellaneous methods -----------------------------------------
+ # These are all used by the 'gen_lib_options() function, in
+ # ccompiler.py.
+
+ def library_dir_option(self, dir):
+ return "/LIBPATH:" + dir
+
+ def runtime_library_dir_option(self, dir):
+ raise DistutilsPlatformError(
+ "don't know how to set runtime library search path for MSVC")
+
+ def library_option(self, lib):
+ return self.library_filename(lib)
+
+ def find_library_file(self, dirs, lib, debug=0):
+ # Prefer a debugging library if found (and requested), but deal
+ # with it if we don't have one.
+ if debug:
+ try_names = [lib + "_d", lib]
+ else:
+ try_names = [lib]
+ for dir in dirs:
+ for name in try_names:
+ libfile = os.path.join(dir, self.library_filename(name))
+ if os.path.isfile(libfile):
+ return libfile
+ else:
+ # Oops, didn't find it in *any* of 'dirs'
+ return None
diff --git a/Lib/distutils/archive_util.py b/Lib/distutils/archive_util.py
index 4470bb0..bed1384 100644
--- a/Lib/distutils/archive_util.py
+++ b/Lib/distutils/archive_util.py
@@ -57,26 +57,28 @@ def make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0,
"""Create a (possibly compressed) tar file from all the files under
'base_dir'.
- 'compress' must be "gzip" (the default), "compress", "bzip2", or None.
- (compress will be deprecated in Python 3.2)
+ 'compress' must be "gzip" (the default), "bzip2", "xz", "compress", or
+ None. ("compress" will be deprecated in Python 3.2)
'owner' and 'group' can be used to define an owner and a group for the
archive that is being built. If not provided, the current owner and group
will be used.
The output tar file will be named 'base_dir' + ".tar", possibly plus
- the appropriate compression extension (".gz", ".bz2" or ".Z").
+ the appropriate compression extension (".gz", ".bz2", ".xz" or ".Z").
Returns the output filename.
"""
- tar_compression = {'gzip': 'gz', 'bzip2': 'bz2', None: '', 'compress': ''}
- compress_ext = {'gzip': '.gz', 'bzip2': '.bz2', 'compress': '.Z'}
+ tar_compression = {'gzip': 'gz', 'bzip2': 'bz2', 'xz': 'xz', None: '',
+ 'compress': ''}
+ compress_ext = {'gzip': '.gz', 'bzip2': '.bz2', 'xz': '.xz',
+ 'compress': '.Z'}
# flags for compression program, each element of list will be an argument
if compress is not None and compress not in compress_ext.keys():
raise ValueError(
- "bad value for 'compress': must be None, 'gzip', 'bzip2' "
- "or 'compress'")
+ "bad value for 'compress': must be None, 'gzip', 'bzip2', "
+ "'xz' or 'compress'")
archive_name = base_name + '.tar'
if compress != 'compress':
@@ -177,6 +179,7 @@ def make_zipfile(base_name, base_dir, verbose=0, dry_run=0):
ARCHIVE_FORMATS = {
'gztar': (make_tarball, [('compress', 'gzip')], "gzip'ed tar-file"),
'bztar': (make_tarball, [('compress', 'bzip2')], "bzip2'ed tar-file"),
+ 'xztar': (make_tarball, [('compress', 'xz')], "xz'ed tar-file"),
'ztar': (make_tarball, [('compress', 'compress')], "compressed tar file"),
'tar': (make_tarball, [('compress', None)], "uncompressed tar file"),
'zip': (make_zipfile, [],"ZIP file")
@@ -197,8 +200,8 @@ def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0,
"""Create an archive file (eg. zip or tar).
'base_name' is the name of the file to create, minus any format-specific
- extension; 'format' is the archive format: one of "zip", "tar", "ztar",
- or "gztar".
+ extension; 'format' is the archive format: one of "zip", "tar", "gztar",
+ "bztar", "xztar", or "ztar".
'root_dir' is a directory that will be the root directory of the
archive; ie. we typically chdir into 'root_dir' before creating the
diff --git a/Lib/distutils/ccompiler.py b/Lib/distutils/ccompiler.py
index 911e84d..b7df394 100644
--- a/Lib/distutils/ccompiler.py
+++ b/Lib/distutils/ccompiler.py
@@ -959,7 +959,7 @@ def get_default_compiler(osname=None, platform=None):
# is assumed to be in the 'distutils' package.)
compiler_class = { 'unix': ('unixccompiler', 'UnixCCompiler',
"standard UNIX-style compiler"),
- 'msvc': ('msvccompiler', 'MSVCCompiler',
+ 'msvc': ('_msvccompiler', 'MSVCCompiler',
"Microsoft Visual C++"),
'cygwin': ('cygwinccompiler', 'CygwinCCompiler',
"Cygwin port of GNU C Compiler for Win32"),
diff --git a/Lib/distutils/command/bdist.py b/Lib/distutils/command/bdist.py
index 6814a1c..014871d 100644
--- a/Lib/distutils/command/bdist.py
+++ b/Lib/distutils/command/bdist.py
@@ -61,13 +61,14 @@ class bdist(Command):
'nt': 'zip'}
# Establish the preferred order (for the --help-formats option).
- format_commands = ['rpm', 'gztar', 'bztar', 'ztar', 'tar',
+ format_commands = ['rpm', 'gztar', 'bztar', 'xztar', 'ztar', 'tar',
'wininst', 'zip', 'msi']
# And the real information.
format_command = {'rpm': ('bdist_rpm', "RPM distribution"),
'gztar': ('bdist_dumb', "gzip'ed tar file"),
'bztar': ('bdist_dumb', "bzip2'ed tar file"),
+ 'xztar': ('bdist_dumb', "xz'ed tar file"),
'ztar': ('bdist_dumb', "compressed tar file"),
'tar': ('bdist_dumb', "tar file"),
'wininst': ('bdist_wininst',
diff --git a/Lib/distutils/command/bdist_dumb.py b/Lib/distutils/command/bdist_dumb.py
index 4405d12..f1bfb24 100644
--- a/Lib/distutils/command/bdist_dumb.py
+++ b/Lib/distutils/command/bdist_dumb.py
@@ -22,7 +22,8 @@ class bdist_dumb(Command):
"platform name to embed in generated filenames "
"(default: %s)" % get_platform()),
('format=', 'f',
- "archive format to create (tar, ztar, gztar, zip)"),
+ "archive format to create (tar, gztar, bztar, xztar, "
+ "ztar, zip)"),
('keep-temp', 'k',
"keep the pseudo-installation tree around after " +
"creating the distribution archive"),
diff --git a/Lib/distutils/command/bdist_wininst.py b/Lib/distutils/command/bdist_wininst.py
index 959a8bf..0c0e2c1 100644
--- a/Lib/distutils/command/bdist_wininst.py
+++ b/Lib/distutils/command/bdist_wininst.py
@@ -303,7 +303,6 @@ class bdist_wininst(Command):
return installer_name
def get_exe_bytes(self):
- from distutils.msvccompiler import get_build_version
# If a target-version other than the current version has been
# specified, then using the MSVC version from *this* build is no good.
# Without actually finding and executing the target version and parsing
@@ -313,20 +312,33 @@ class bdist_wininst(Command):
# We can then execute this program to obtain any info we need, such
# as the real sys.version string for the build.
cur_version = get_python_version()
- if self.target_version and self.target_version != cur_version:
- # If the target version is *later* than us, then we assume they
- # use what we use
- # string compares seem wrong, but are what sysconfig.py itself uses
- if self.target_version > cur_version:
- bv = get_build_version()
+
+ # If the target version is *later* than us, then we assume they
+ # use what we use
+ # string compares seem wrong, but are what sysconfig.py itself uses
+ if self.target_version and self.target_version < cur_version:
+ if self.target_version < "2.4":
+ bv = 6.0
+ elif self.target_version == "2.4":
+ bv = 7.1
+ elif self.target_version == "2.5":
+ bv = 8.0
+ elif self.target_version <= "3.2":
+ bv = 9.0
+ elif self.target_version <= "3.4":
+ bv = 10.0
else:
- if self.target_version < "2.4":
- bv = 6.0
- else:
- bv = 7.1
+ bv = 14.0
else:
# for current version - use authoritative check.
- bv = get_build_version()
+ try:
+ from msvcrt import CRT_ASSEMBLY_VERSION
+ except ImportError:
+ # cross-building, so assume the latest version
+ bv = 14.0
+ else:
+ bv = float('.'.join(CRT_ASSEMBLY_VERSION.split('.', 2)[:2]))
+
# wininst-x.y.exe is in the same directory as this file
directory = os.path.dirname(__file__)
diff --git a/Lib/distutils/command/build.py b/Lib/distutils/command/build.py
index cfc15cf..337dd0b 100644
--- a/Lib/distutils/command/build.py
+++ b/Lib/distutils/command/build.py
@@ -36,6 +36,8 @@ class build(Command):
"(default: %s)" % get_platform()),
('compiler=', 'c',
"specify the compiler type"),
+ ('parallel=', 'j',
+ "number of parallel build jobs"),
('debug', 'g',
"compile extensions and libraries with debugging information"),
('force', 'f',
@@ -65,6 +67,7 @@ class build(Command):
self.debug = None
self.force = 0
self.executable = None
+ self.parallel = None
def finalize_options(self):
if self.plat_name is None:
@@ -116,6 +119,12 @@ class build(Command):
if self.executable is None:
self.executable = os.path.normpath(sys.executable)
+ if isinstance(self.parallel, str):
+ try:
+ self.parallel = int(self.parallel)
+ except ValueError:
+ raise DistutilsOptionError("parallel should be an integer")
+
def run(self):
# Run all relevant sub-commands. This will be some subset of:
# - build_py - pure Python modules
diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py
index acbe648..d4cb11e 100644
--- a/Lib/distutils/command/build_ext.py
+++ b/Lib/distutils/command/build_ext.py
@@ -4,7 +4,10 @@ Implements the Distutils 'build_ext' command, for building extension
modules (currently limited to C extensions, should accommodate C++
extensions ASAP)."""
-import sys, os, re
+import contextlib
+import os
+import re
+import sys
from distutils.core import Command
from distutils.errors import *
from distutils.sysconfig import customize_compiler, get_python_version
@@ -16,10 +19,6 @@ from distutils import log
from site import USER_BASE
-if os.name == 'nt':
- from distutils.msvccompiler import get_build_version
- MSVC_VERSION = int(get_build_version())
-
# An extension name is just a dot-separated list of Python NAMEs (ie.
# the same as a fully-qualified module name).
extension_name_re = re.compile \
@@ -85,6 +84,8 @@ class build_ext(Command):
"forcibly build everything (ignore file timestamps)"),
('compiler=', 'c',
"specify the compiler type"),
+ ('parallel=', 'j',
+ "number of parallel build jobs"),
('swig-cpp', None,
"make SWIG create C++ files (default is C)"),
('swig-opts=', None,
@@ -124,6 +125,7 @@ class build_ext(Command):
self.swig_cpp = None
self.swig_opts = None
self.user = None
+ self.parallel = None
def finalize_options(self):
from distutils import sysconfig
@@ -134,6 +136,7 @@ class build_ext(Command):
('compiler', 'compiler'),
('debug', 'debug'),
('force', 'force'),
+ ('parallel', 'parallel'),
('plat_name', 'plat_name'),
)
@@ -199,27 +202,17 @@ class build_ext(Command):
_sys_home = getattr(sys, '_home', None)
if _sys_home:
self.library_dirs.append(_sys_home)
- if MSVC_VERSION >= 9:
- # Use the .lib files for the correct architecture
- if self.plat_name == 'win32':
- suffix = ''
- else:
- # win-amd64 or win-ia64
- suffix = self.plat_name[4:]
- new_lib = os.path.join(sys.exec_prefix, 'PCbuild')
- if suffix:
- new_lib = os.path.join(new_lib, suffix)
- self.library_dirs.append(new_lib)
-
- elif MSVC_VERSION == 8:
- self.library_dirs.append(os.path.join(sys.exec_prefix,
- 'PC', 'VS8.0'))
- elif MSVC_VERSION == 7:
- self.library_dirs.append(os.path.join(sys.exec_prefix,
- 'PC', 'VS7.1'))
+
+ # Use the .lib files for the correct architecture
+ if self.plat_name == 'win32':
+ suffix = 'win32'
else:
- self.library_dirs.append(os.path.join(sys.exec_prefix,
- 'PC', 'VC6'))
+ # win-amd64 or win-ia64
+ suffix = self.plat_name[4:]
+ new_lib = os.path.join(sys.exec_prefix, 'PCbuild')
+ if suffix:
+ new_lib = os.path.join(new_lib, suffix)
+ self.library_dirs.append(new_lib)
# for extensions under Cygwin and AtheOS Python's library directory must be
# appended to library_dirs
@@ -274,6 +267,12 @@ class build_ext(Command):
self.library_dirs.append(user_lib)
self.rpath.append(user_lib)
+ if isinstance(self.parallel, str):
+ try:
+ self.parallel = int(self.parallel)
+ except ValueError:
+ raise DistutilsOptionError("parallel should be an integer")
+
def run(self):
from distutils.ccompiler import new_compiler
@@ -442,15 +441,45 @@ class build_ext(Command):
def build_extensions(self):
# First, sanity-check the 'extensions' list
self.check_extensions_list(self.extensions)
+ if self.parallel:
+ self._build_extensions_parallel()
+ else:
+ self._build_extensions_serial()
+
+ def _build_extensions_parallel(self):
+ workers = self.parallel
+ if self.parallel is True:
+ workers = os.cpu_count() # may return None
+ try:
+ from concurrent.futures import ThreadPoolExecutor
+ except ImportError:
+ workers = None
+
+ if workers is None:
+ self._build_extensions_serial()
+ return
+
+ with ThreadPoolExecutor(max_workers=workers) as executor:
+ futures = [executor.submit(self.build_extension, ext)
+ for ext in self.extensions]
+ for ext, fut in zip(self.extensions, futures):
+ with self._filter_build_errors(ext):
+ fut.result()
+ def _build_extensions_serial(self):
for ext in self.extensions:
- try:
+ with self._filter_build_errors(ext):
self.build_extension(ext)
- except (CCompilerError, DistutilsError, CompileError) as e:
- if not ext.optional:
- raise
- self.warn('building extension "%s" failed: %s' %
- (ext.name, e))
+
+ @contextlib.contextmanager
+ def _filter_build_errors(self, ext):
+ try:
+ yield
+ except (CCompilerError, DistutilsError, CompileError) as e:
+ if not ext.optional:
+ raise
+ self.warn('building extension "%s" failed: %s' %
+ (ext.name, e))
def build_extension(self, ext):
sources = ext.sources
@@ -502,15 +531,8 @@ class build_ext(Command):
extra_postargs=extra_args,
depends=ext.depends)
- # XXX -- this is a Vile HACK!
- #
- # The setup.py script for Python on Unix needs to be able to
- # get this list so it can perform all the clean up needed to
- # avoid keeping object files around when cleaning out a failed
- # build of an extension module. Since Distutils does not
- # track dependencies, we have to get rid of intermediates to
- # ensure all the intermediates will be properly re-built.
- #
+ # XXX outdated variable, kept here in case third-part code
+ # needs it.
self._built_objects = objects[:]
# Now link the object files together into a "shared object" --
@@ -655,10 +677,7 @@ class build_ext(Command):
"""
from distutils.sysconfig import get_config_var
ext_path = ext_name.split('.')
- # extensions in debug_mode are named 'module_d.pyd' under windows
ext_suffix = get_config_var('EXT_SUFFIX')
- if os.name == 'nt' and self.debug:
- return os.path.join(*ext_path) + '_d' + ext_suffix
return os.path.join(*ext_path) + ext_suffix
def get_export_symbols(self, ext):
@@ -683,7 +702,7 @@ class build_ext(Command):
# to need it mentioned explicitly, though, so that's what we do.
# Append '_d' to the python import library on debug builds.
if sys.platform == "win32":
- from distutils.msvccompiler import MSVCCompiler
+ from distutils._msvccompiler import MSVCCompiler
if not isinstance(self.compiler, MSVCCompiler):
template = "python%d%d"
if self.debug:
diff --git a/Lib/distutils/command/build_py.py b/Lib/distutils/command/build_py.py
index 9100b96..cf0ca57 100644
--- a/Lib/distutils/command/build_py.py
+++ b/Lib/distutils/command/build_py.py
@@ -314,10 +314,10 @@ class build_py (Command):
if include_bytecode:
if self.compile:
outputs.append(importlib.util.cache_from_source(
- filename, debug_override=True))
+ filename, optimization=''))
if self.optimize > 0:
outputs.append(importlib.util.cache_from_source(
- filename, debug_override=False))
+ filename, optimization=self.optimize))
outputs += [
os.path.join(build_dir, filename)
diff --git a/Lib/distutils/command/install.py b/Lib/distutils/command/install.py
index d768dc5..67db007 100644
--- a/Lib/distutils/command/install.py
+++ b/Lib/distutils/command/install.py
@@ -51,7 +51,7 @@ if HAS_USER_SITE:
'purelib': '$usersite',
'platlib': '$usersite',
'headers': '$userbase/Python$py_version_nodot/Include/$dist_name',
- 'scripts': '$userbase/Scripts',
+ 'scripts': '$userbase/Python$py_version_nodot/Scripts',
'data' : '$userbase',
}
diff --git a/Lib/distutils/command/install_lib.py b/Lib/distutils/command/install_lib.py
index 215813b..6154cf0 100644
--- a/Lib/distutils/command/install_lib.py
+++ b/Lib/distutils/command/install_lib.py
@@ -22,15 +22,15 @@ class install_lib(Command):
# possible scenarios:
# 1) no compilation at all (--no-compile --no-optimize)
# 2) compile .pyc only (--compile --no-optimize; default)
- # 3) compile .pyc and "level 1" .pyo (--compile --optimize)
- # 4) compile "level 1" .pyo only (--no-compile --optimize)
- # 5) compile .pyc and "level 2" .pyo (--compile --optimize-more)
- # 6) compile "level 2" .pyo only (--no-compile --optimize-more)
+ # 3) compile .pyc and "opt-1" .pyc (--compile --optimize)
+ # 4) compile "opt-1" .pyc only (--no-compile --optimize)
+ # 5) compile .pyc and "opt-2" .pyc (--compile --optimize-more)
+ # 6) compile "opt-2" .pyc only (--no-compile --optimize-more)
#
- # The UI for this is two option, 'compile' and 'optimize'.
+ # The UI for this is two options, 'compile' and 'optimize'.
# 'compile' is strictly boolean, and only decides whether to
# generate .pyc files. 'optimize' is three-way (0, 1, or 2), and
- # decides both whether to generate .pyo files and what level of
+ # decides both whether to generate .pyc files and what level of
# optimization to use.
user_options = [
@@ -166,10 +166,10 @@ class install_lib(Command):
continue
if self.compile:
bytecode_files.append(importlib.util.cache_from_source(
- py_file, debug_override=True))
+ py_file, optimization=''))
if self.optimize > 0:
bytecode_files.append(importlib.util.cache_from_source(
- py_file, debug_override=False))
+ py_file, optimization=self.optimize))
return bytecode_files
diff --git a/Lib/distutils/command/upload.py b/Lib/distutils/command/upload.py
index 1a96e22..1c4fc48 100644
--- a/Lib/distutils/command/upload.py
+++ b/Lib/distutils/command/upload.py
@@ -1,11 +1,14 @@
-"""distutils.command.upload
+"""
+distutils.command.upload
-Implements the Distutils 'upload' subcommand (upload package to PyPI)."""
+Implements the Distutils 'upload' subcommand (upload package to a package
+index).
+"""
-import sys
-import os, io
-import socket
+import os
+import io
import platform
+import hashlib
from base64 import standard_b64encode
from urllib.request import urlopen, Request, HTTPError
from urllib.parse import urlparse
@@ -14,12 +17,6 @@ from distutils.core import PyPIRCCommand
from distutils.spawn import spawn
from distutils import log
-# this keeps compatibility for 2.3 and 2.4
-if sys.version < "2.5":
- from md5 import md5
-else:
- from hashlib import md5
-
class upload(PyPIRCCommand):
description = "upload binary package to PyPI"
@@ -60,7 +57,8 @@ class upload(PyPIRCCommand):
def run(self):
if not self.distribution.dist_files:
- raise DistutilsOptionError("No dist file created in earlier command")
+ msg = "No dist file created in earlier command"
+ raise DistutilsOptionError(msg)
for command, pyversion, filename in self.distribution.dist_files:
self.upload_file(command, pyversion, filename)
@@ -103,10 +101,10 @@ class upload(PyPIRCCommand):
'content': (os.path.basename(filename),content),
'filetype': command,
'pyversion': pyversion,
- 'md5_digest': md5(content).hexdigest(),
+ 'md5_digest': hashlib.md5(content).hexdigest(),
# additional meta-data
- 'metadata_version' : '1.0',
+ 'metadata_version': '1.0',
'summary': meta.get_description(),
'home_page': meta.get_url(),
'author': meta.get_contact(),
@@ -149,7 +147,7 @@ class upload(PyPIRCCommand):
for key, value in data.items():
title = '\r\nContent-Disposition: form-data; name="%s"' % key
# handle multiple entries for the same name
- if type(value) != type([]):
+ if not isinstance(value, list):
value = [value]
for value in value:
if type(value) is tuple:
@@ -166,13 +164,15 @@ class upload(PyPIRCCommand):
body.write(end_boundary)
body = body.getvalue()
- self.announce("Submitting %s to %s" % (filename, self.repository), log.INFO)
+ msg = "Submitting %s to %s" % (filename, self.repository)
+ self.announce(msg, log.INFO)
# build the Request
- headers = {'Content-type':
- 'multipart/form-data; boundary=%s' % boundary,
- 'Content-length': str(len(body)),
- 'Authorization': auth}
+ headers = {
+ 'Content-type': 'multipart/form-data; boundary=%s' % boundary,
+ 'Content-length': str(len(body)),
+ 'Authorization': auth,
+ }
request = Request(self.repository, data=body,
headers=headers)
diff --git a/Lib/distutils/command/wininst-14.0-amd64.exe b/Lib/distutils/command/wininst-14.0-amd64.exe
new file mode 100644
index 0000000..7a5e78d
--- /dev/null
+++ b/Lib/distutils/command/wininst-14.0-amd64.exe
Binary files differ
diff --git a/Lib/distutils/command/wininst-14.0.exe b/Lib/distutils/command/wininst-14.0.exe
new file mode 100644
index 0000000..cc43296
--- /dev/null
+++ b/Lib/distutils/command/wininst-14.0.exe
Binary files differ
diff --git a/Lib/distutils/core.py b/Lib/distutils/core.py
index 2bfe66a..f05b34b 100644
--- a/Lib/distutils/core.py
+++ b/Lib/distutils/core.py
@@ -221,8 +221,6 @@ def run_setup (script_name, script_args=None, stop_after="run"):
# Hmm, should we do something if exiting with a non-zero code
# (ie. error)?
pass
- except:
- raise
if _setup_distribution is None:
raise RuntimeError(("'distutils.core.setup()' was never called -- "
diff --git a/Lib/distutils/dist.py b/Lib/distutils/dist.py
index 7eb04bc..ffb33ff6 100644
--- a/Lib/distutils/dist.py
+++ b/Lib/distutils/dist.py
@@ -4,7 +4,9 @@ Provides the Distribution class, which represents the module distribution
being built/installed/distributed.
"""
-import sys, os, re
+import sys
+import os
+import re
from email import message_from_file
try:
@@ -22,7 +24,7 @@ from distutils.debug import DEBUG
# the same as a Python NAME -- I don't allow leading underscores. The fact
# that they're very similar is no coincidence; the default naming scheme is
# to look for a Python module named after the command.
-command_re = re.compile (r'^[a-zA-Z]([a-zA-Z0-9_]*)$')
+command_re = re.compile(r'^[a-zA-Z]([a-zA-Z0-9_]*)$')
class Distribution:
@@ -39,7 +41,6 @@ class Distribution:
See the code for 'setup()', in core.py, for details.
"""
-
# 'global_options' describes the command-line options that may be
# supplied to the setup script prior to any actual commands.
# Eg. "./setup.py -n" or "./setup.py --quiet" both take advantage of
@@ -48,12 +49,13 @@ class Distribution:
# don't want to pollute the commands with too many options that they
# have minimal control over.
# The fourth entry for verbose means that it can be repeated.
- global_options = [('verbose', 'v', "run verbosely (default)", 1),
- ('quiet', 'q', "run quietly (turns verbosity off)"),
- ('dry-run', 'n', "don't actually do anything"),
- ('help', 'h', "show detailed help message"),
- ('no-user-cfg', None,
- 'ignore pydistutils.cfg in your home directory'),
+ global_options = [
+ ('verbose', 'v', "run verbosely (default)", 1),
+ ('quiet', 'q', "run quietly (turns verbosity off)"),
+ ('dry-run', 'n', "don't actually do anything"),
+ ('help', 'h', "show detailed help message"),
+ ('no-user-cfg', None,
+ 'ignore pydistutils.cfg in your home directory'),
]
# 'common_usage' is a short (2-3 line) string describing the common
@@ -115,10 +117,9 @@ Common commands: (see '--help-commands' for more)
# negative options are options that exclude other options
negative_opt = {'quiet': 'verbose'}
-
# -- Creation/initialization methods -------------------------------
- def __init__ (self, attrs=None):
+ def __init__(self, attrs=None):
"""Construct a new Distribution instance: initialize all the
attributes of a Distribution, and then use 'attrs' (a dictionary
mapping attribute names to values) to assign some of those
@@ -532,15 +533,15 @@ Common commands: (see '--help-commands' for more)
# to be sure that the basic "command" interface is implemented.
if not issubclass(cmd_class, Command):
raise DistutilsClassError(
- "command class %s must subclass Command" % cmd_class)
+ "command class %s must subclass Command" % cmd_class)
# Also make sure that the command object provides a list of its
# known options.
if not (hasattr(cmd_class, 'user_options') and
isinstance(cmd_class.user_options, list)):
- raise DistutilsClassError(("command class %s must provide " +
- "'user_options' attribute (a list of tuples)") % \
- cmd_class)
+ msg = ("command class %s must provide "
+ "'user_options' attribute (a list of tuples)")
+ raise DistutilsClassError(msg % cmd_class)
# If the command class has a list of negative alias options,
# merge it in with the global negative aliases.
@@ -552,12 +553,11 @@ Common commands: (see '--help-commands' for more)
# Check for help_options in command class. They have a different
# format (tuple of four) so we need to preprocess them here.
if (hasattr(cmd_class, 'help_options') and
- isinstance(cmd_class.help_options, list)):
+ isinstance(cmd_class.help_options, list)):
help_options = fix_help_options(cmd_class.help_options)
else:
help_options = []
-
# All commands support the global options too, just by adding
# in 'global_options'.
parser.set_option_table(self.global_options +
@@ -570,7 +570,7 @@ Common commands: (see '--help-commands' for more)
return
if (hasattr(cmd_class, 'help_options') and
- isinstance(cmd_class.help_options, list)):
+ isinstance(cmd_class.help_options, list)):
help_option_found=0
for (help_option, short, desc, func) in cmd_class.help_options:
if hasattr(opts, parser.get_attr_name(help_option)):
@@ -647,7 +647,7 @@ Common commands: (see '--help-commands' for more)
else:
klass = self.get_command_class(command)
if (hasattr(klass, 'help_options') and
- isinstance(klass.help_options, list)):
+ isinstance(klass.help_options, list)):
parser.set_option_table(klass.user_options +
fix_help_options(klass.help_options))
else:
@@ -814,7 +814,7 @@ Common commands: (see '--help-commands' for more)
klass_name = command
try:
- __import__ (module_name)
+ __import__(module_name)
module = sys.modules[module_name]
except ImportError:
continue
@@ -823,8 +823,8 @@ Common commands: (see '--help-commands' for more)
klass = getattr(module, klass_name)
except AttributeError:
raise DistutilsModuleError(
- "invalid command '%s' (no class '%s' in module '%s')"
- % (command, klass_name, module_name))
+ "invalid command '%s' (no class '%s' in module '%s')"
+ % (command, klass_name, module_name))
self.cmdclass[command] = klass
return klass
@@ -840,7 +840,7 @@ Common commands: (see '--help-commands' for more)
cmd_obj = self.command_obj.get(command)
if not cmd_obj and create:
if DEBUG:
- self.announce("Distribution.get_command_obj(): " \
+ self.announce("Distribution.get_command_obj(): "
"creating '%s' command object" % command)
klass = self.get_command_class(command)
@@ -897,8 +897,8 @@ Common commands: (see '--help-commands' for more)
setattr(command_obj, option, value)
else:
raise DistutilsOptionError(
- "error in %s: command '%s' has no such option '%s'"
- % (source, command_name, option))
+ "error in %s: command '%s' has no such option '%s'"
+ % (source, command_name, option))
except ValueError as msg:
raise DistutilsOptionError(msg)
@@ -974,7 +974,6 @@ Common commands: (see '--help-commands' for more)
cmd_obj.run()
self.have_run[command] = 1
-
# -- Distribution query methods ------------------------------------
def has_pure_modules(self):
@@ -1112,17 +1111,17 @@ class DistributionMetadata:
"""
version = '1.0'
if (self.provides or self.requires or self.obsoletes or
- self.classifiers or self.download_url):
+ self.classifiers or self.download_url):
version = '1.1'
file.write('Metadata-Version: %s\n' % version)
- file.write('Name: %s\n' % self.get_name() )
- file.write('Version: %s\n' % self.get_version() )
- file.write('Summary: %s\n' % self.get_description() )
- file.write('Home-page: %s\n' % self.get_url() )
- file.write('Author: %s\n' % self.get_contact() )
- file.write('Author-email: %s\n' % self.get_contact_email() )
- file.write('License: %s\n' % self.get_license() )
+ file.write('Name: %s\n' % self.get_name())
+ file.write('Version: %s\n' % self.get_version())
+ file.write('Summary: %s\n' % self.get_description())
+ file.write('Home-page: %s\n' % self.get_url())
+ file.write('Author: %s\n' % self.get_contact())
+ file.write('Author-email: %s\n' % self.get_contact_email())
+ file.write('License: %s\n' % self.get_license())
if self.download_url:
file.write('Download-URL: %s\n' % self.download_url)
@@ -1131,7 +1130,7 @@ class DistributionMetadata:
keywords = ','.join(self.get_keywords())
if keywords:
- file.write('Keywords: %s\n' % keywords )
+ file.write('Keywords: %s\n' % keywords)
self._write_list(file, 'Platform', self.get_platforms())
self._write_list(file, 'Classifier', self.get_classifiers())
diff --git a/Lib/distutils/extension.py b/Lib/distutils/extension.py
index a93655a..7efbb74 100644
--- a/Lib/distutils/extension.py
+++ b/Lib/distutils/extension.py
@@ -131,6 +131,14 @@ class Extension:
msg = "Unknown Extension options: %s" % options
warnings.warn(msg)
+ def __repr__(self):
+ return '<%s.%s(%r) at %#x>' % (
+ self.__class__.__module__,
+ self.__class__.__qualname__,
+ self.name,
+ id(self))
+
+
def read_setup_file(filename):
"""Reads a Setup file and returns Extension instances."""
from distutils.sysconfig import (parse_makefile, expand_makefile_vars,
diff --git a/Lib/distutils/msvc9compiler.py b/Lib/distutils/msvc9compiler.py
index a5a5010..da4b21d 100644
--- a/Lib/distutils/msvc9compiler.py
+++ b/Lib/distutils/msvc9compiler.py
@@ -179,6 +179,9 @@ def get_build_version():
i = i + len(prefix)
s, rest = sys.version[i:].split(" ", 1)
majorVersion = int(s[:-2]) - 6
+ if majorVersion >= 13:
+ # v13 was skipped and should be v14
+ majorVersion += 1
minorVersion = int(s[2:3]) / 10.0
# I don't think paths are affected by minor version in version 6
if majorVersion == 6:
diff --git a/Lib/distutils/msvccompiler.py b/Lib/distutils/msvccompiler.py
index 8116656..1048cd4 100644
--- a/Lib/distutils/msvccompiler.py
+++ b/Lib/distutils/msvccompiler.py
@@ -157,6 +157,9 @@ def get_build_version():
i = i + len(prefix)
s, rest = sys.version[i:].split(" ", 1)
majorVersion = int(s[:-2]) - 6
+ if majorVersion >= 13:
+ # v13 was skipped and should be v14
+ majorVersion += 1
minorVersion = int(s[2:3]) / 10.0
# I don't think paths are affected by minor version in version 6
if majorVersion == 6:
diff --git a/Lib/distutils/spawn.py b/Lib/distutils/spawn.py
index 22e87e8..5dd415a 100644
--- a/Lib/distutils/spawn.py
+++ b/Lib/distutils/spawn.py
@@ -137,9 +137,6 @@ def _spawn_posix(cmd, search_path=1, verbose=0, dry_run=0):
try:
pid, status = os.waitpid(pid, 0)
except OSError as exc:
- import errno
- if exc.errno == errno.EINTR:
- continue
if not DEBUG:
cmd = executable
raise DistutilsExecError(
diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py
index a1452fe..573724d 100644
--- a/Lib/distutils/sysconfig.py
+++ b/Lib/distutils/sysconfig.py
@@ -9,6 +9,7 @@ Written by: Fred L. Drake, Jr.
Email: <fdrake@acm.org>
"""
+import _imp
import os
import re
import sys
@@ -22,23 +23,15 @@ BASE_PREFIX = os.path.normpath(sys.base_prefix)
BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix)
# Path to the base directory of the project. On Windows the binary may
-# live in project/PCBuild9. If we're dealing with an x64 Windows build,
-# it'll live in project/PCbuild/amd64.
+# live in project/PCBuild/win32 or project/PCBuild/amd64.
# set for cross builds
if "_PYTHON_PROJECT_BASE" in os.environ:
project_base = os.path.abspath(os.environ["_PYTHON_PROJECT_BASE"])
else:
project_base = os.path.dirname(os.path.abspath(sys.executable))
-if os.name == "nt" and "pcbuild" in project_base[-8:].lower():
- project_base = os.path.abspath(os.path.join(project_base, os.path.pardir))
-# PC/VS7.1
-if os.name == "nt" and "\\pc\\v" in project_base[-10:].lower():
- project_base = os.path.abspath(os.path.join(project_base, os.path.pardir,
- os.path.pardir))
-# PC/AMD64
-if os.name == "nt" and "\\pcbuild\\amd64" in project_base[-14:].lower():
- project_base = os.path.abspath(os.path.join(project_base, os.path.pardir,
- os.path.pardir))
+if (os.name == 'nt' and
+ project_base.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))):
+ project_base = os.path.dirname(os.path.dirname(project_base))
# python_build: (Boolean) if true, we're either building Python or
# building an extension with an un-installed Python, so we use
@@ -51,11 +44,9 @@ def _is_python_source_dir(d):
return True
return False
_sys_home = getattr(sys, '_home', None)
-if _sys_home and os.name == 'nt' and \
- _sys_home.lower().endswith(('pcbuild', 'pcbuild\\amd64')):
- _sys_home = os.path.dirname(_sys_home)
- if _sys_home.endswith('pcbuild'): # must be amd64
- _sys_home = os.path.dirname(_sys_home)
+if (_sys_home and os.name == 'nt' and
+ _sys_home.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))):
+ _sys_home = os.path.dirname(os.path.dirname(_sys_home))
def _python_build():
if _sys_home:
return _is_python_source_dir(_sys_home)
@@ -468,7 +459,7 @@ def _init_nt():
# XXX hmmm.. a normal install puts include files here
g['INCLUDEPY'] = get_python_inc(plat_specific=0)
- g['EXT_SUFFIX'] = '.pyd'
+ g['EXT_SUFFIX'] = _imp.extension_suffixes()[0]
g['EXE'] = ".exe"
g['VERSION'] = get_python_version().replace(".", "")
g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable))
diff --git a/Lib/distutils/tests/test_archive_util.py b/Lib/distutils/tests/test_archive_util.py
index 2d72af4..02fa1e2 100644
--- a/Lib/distutils/tests/test_archive_util.py
+++ b/Lib/distutils/tests/test_archive_util.py
@@ -13,7 +13,7 @@ from distutils.archive_util import (check_archive_formats, make_tarball,
ARCHIVE_FORMATS)
from distutils.spawn import find_executable, spawn
from distutils.tests import support
-from test.support import check_warnings, run_unittest, patch
+from test.support import check_warnings, run_unittest, patch, change_cwd
try:
import grp
@@ -34,6 +34,16 @@ try:
except ImportError:
ZLIB_SUPPORT = False
+try:
+ import bz2
+except ImportError:
+ bz2 = None
+
+try:
+ import lzma
+except ImportError:
+ lzma = None
+
def can_fs_encode(filename):
"""
Return True if the filename can be saved in the file system.
@@ -52,19 +62,36 @@ class ArchiveUtilTestCase(support.TempdirManager,
unittest.TestCase):
@unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
- def test_make_tarball(self):
- self._make_tarball('archive')
+ def test_make_tarball(self, name='archive'):
+ # creating something to tar
+ tmpdir = self._create_files()
+ self._make_tarball(tmpdir, name, '.tar.gz')
+ # trying an uncompressed one
+ self._make_tarball(tmpdir, name, '.tar', compress=None)
@unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
+ def test_make_tarball_gzip(self):
+ tmpdir = self._create_files()
+ self._make_tarball(tmpdir, 'archive', '.tar.gz', compress='gzip')
+
+ @unittest.skipUnless(bz2, 'Need bz2 support to run')
+ def test_make_tarball_bzip2(self):
+ tmpdir = self._create_files()
+ self._make_tarball(tmpdir, 'archive', '.tar.bz2', compress='bzip2')
+
+ @unittest.skipUnless(lzma, 'Need lzma support to run')
+ def test_make_tarball_xz(self):
+ tmpdir = self._create_files()
+ self._make_tarball(tmpdir, 'archive', '.tar.xz', compress='xz')
+
@unittest.skipUnless(can_fs_encode('årchiv'),
'File system cannot handle this filename')
def test_make_tarball_latin1(self):
"""
Mirror test_make_tarball, except filename contains latin characters.
"""
- self._make_tarball('årchiv') # note this isn't a real word
+ self.test_make_tarball('årchiv') # note this isn't a real word
- @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
@unittest.skipUnless(can_fs_encode('のアーカイブ'),
'File system cannot handle this filename')
def test_make_tarball_extended(self):
@@ -72,16 +99,9 @@ class ArchiveUtilTestCase(support.TempdirManager,
Mirror test_make_tarball, except filename contains extended
characters outside the latin charset.
"""
- self._make_tarball('のアーカイブ') # japanese for archive
-
- def _make_tarball(self, target_name):
- # creating something to tar
- tmpdir = self.mkdtemp()
- self.write_file([tmpdir, 'file1'], 'xxx')
- self.write_file([tmpdir, 'file2'], 'xxx')
- os.mkdir(os.path.join(tmpdir, 'sub'))
- self.write_file([tmpdir, 'sub', 'file3'], 'xxx')
+ self.test_make_tarball('のアーカイブ') # japanese for archive
+ def _make_tarball(self, tmpdir, target_name, suffix, **kwargs):
tmpdir2 = self.mkdtemp()
unittest.skipUnless(splitdrive(tmpdir)[0] == splitdrive(tmpdir2)[0],
"source and target should be on same drive")
@@ -89,27 +109,13 @@ class ArchiveUtilTestCase(support.TempdirManager,
base_name = os.path.join(tmpdir2, target_name)
# working with relative paths to avoid tar warnings
- old_dir = os.getcwd()
- os.chdir(tmpdir)
- try:
- make_tarball(splitdrive(base_name)[1], '.')
- finally:
- os.chdir(old_dir)
+ with change_cwd(tmpdir):
+ make_tarball(splitdrive(base_name)[1], 'dist', **kwargs)
# check if the compressed tarball was created
- tarball = base_name + '.tar.gz'
- self.assertTrue(os.path.exists(tarball))
-
- # trying an uncompressed one
- base_name = os.path.join(tmpdir2, target_name)
- old_dir = os.getcwd()
- os.chdir(tmpdir)
- try:
- make_tarball(splitdrive(base_name)[1], '.', compress=None)
- finally:
- os.chdir(old_dir)
- tarball = base_name + '.tar'
+ tarball = base_name + suffix
self.assertTrue(os.path.exists(tarball))
+ self.assertEqual(self._tarinfo(tarball), self._created_files)
def _tarinfo(self, path):
tar = tarfile.open(path)
@@ -120,6 +126,9 @@ class ArchiveUtilTestCase(support.TempdirManager,
finally:
tar.close()
+ _created_files = ('dist', 'dist/file1', 'dist/file2',
+ 'dist/sub', 'dist/sub/file3', 'dist/sub2')
+
def _create_files(self):
# creating something to tar
tmpdir = self.mkdtemp()
@@ -130,15 +139,15 @@ class ArchiveUtilTestCase(support.TempdirManager,
os.mkdir(os.path.join(dist, 'sub'))
self.write_file([dist, 'sub', 'file3'], 'xxx')
os.mkdir(os.path.join(dist, 'sub2'))
- tmpdir2 = self.mkdtemp()
- base_name = os.path.join(tmpdir2, 'archive')
- return tmpdir, tmpdir2, base_name
+ return tmpdir
@unittest.skipUnless(find_executable('tar') and find_executable('gzip')
and ZLIB_SUPPORT,
'Need the tar, gzip and zlib command to run')
def test_tarfile_vs_tar(self):
- tmpdir, tmpdir2, base_name = self._create_files()
+ tmpdir = self._create_files()
+ tmpdir2 = self.mkdtemp()
+ base_name = os.path.join(tmpdir2, 'archive')
old_dir = os.getcwd()
os.chdir(tmpdir)
try:
@@ -164,7 +173,8 @@ class ArchiveUtilTestCase(support.TempdirManager,
self.assertTrue(os.path.exists(tarball2))
# let's compare both tarballs
- self.assertEqual(self._tarinfo(tarball), self._tarinfo(tarball2))
+ self.assertEqual(self._tarinfo(tarball), self._created_files)
+ self.assertEqual(self._tarinfo(tarball2), self._created_files)
# trying an uncompressed one
base_name = os.path.join(tmpdir2, 'archive')
@@ -191,7 +201,8 @@ class ArchiveUtilTestCase(support.TempdirManager,
@unittest.skipUnless(find_executable('compress'),
'The compress program is required')
def test_compress_deprecated(self):
- tmpdir, tmpdir2, base_name = self._create_files()
+ tmpdir = self._create_files()
+ base_name = os.path.join(self.mkdtemp(), 'archive')
# using compress and testing the PendingDeprecationWarning
old_dir = os.getcwd()
@@ -224,17 +235,17 @@ class ArchiveUtilTestCase(support.TempdirManager,
'Need zip and zlib support to run')
def test_make_zipfile(self):
# creating something to tar
- tmpdir = self.mkdtemp()
- self.write_file([tmpdir, 'file1'], 'xxx')
- self.write_file([tmpdir, 'file2'], 'xxx')
-
- tmpdir2 = self.mkdtemp()
- base_name = os.path.join(tmpdir2, 'archive')
- make_zipfile(base_name, tmpdir)
+ tmpdir = self._create_files()
+ base_name = os.path.join(self.mkdtemp(), 'archive')
+ with change_cwd(tmpdir):
+ make_zipfile(base_name, 'dist')
# check if the compressed tarball was created
tarball = base_name + '.zip'
self.assertTrue(os.path.exists(tarball))
+ with zipfile.ZipFile(tarball) as zf:
+ self.assertEqual(sorted(zf.namelist()),
+ ['dist/file1', 'dist/file2', 'dist/sub/file3'])
@unittest.skipUnless(ZIP_SUPPORT, 'Need zip support to run')
def test_make_zipfile_no_zlib(self):
@@ -250,18 +261,24 @@ class ArchiveUtilTestCase(support.TempdirManager,
patch(self, archive_util.zipfile, 'ZipFile', fake_zipfile)
# create something to tar and compress
- tmpdir, tmpdir2, base_name = self._create_files()
- make_zipfile(base_name, tmpdir)
+ tmpdir = self._create_files()
+ base_name = os.path.join(self.mkdtemp(), 'archive')
+ with change_cwd(tmpdir):
+ make_zipfile(base_name, 'dist')
tarball = base_name + '.zip'
self.assertEqual(called,
[((tarball, "w"), {'compression': zipfile.ZIP_STORED})])
self.assertTrue(os.path.exists(tarball))
+ with zipfile.ZipFile(tarball) as zf:
+ self.assertEqual(sorted(zf.namelist()),
+ ['dist/file1', 'dist/file2', 'dist/sub/file3'])
def test_check_archive_formats(self):
self.assertEqual(check_archive_formats(['gztar', 'xxx', 'zip']),
'xxx')
- self.assertEqual(check_archive_formats(['gztar', 'zip']), None)
+ self.assertIsNone(check_archive_formats(['gztar', 'bztar', 'xztar',
+ 'ztar', 'tar', 'zip']))
def test_make_archive(self):
tmpdir = self.mkdtemp()
@@ -282,6 +299,41 @@ class ArchiveUtilTestCase(support.TempdirManager,
finally:
del ARCHIVE_FORMATS['xxx']
+ def test_make_archive_tar(self):
+ base_dir = self._create_files()
+ base_name = os.path.join(self.mkdtemp() , 'archive')
+ res = make_archive(base_name, 'tar', base_dir, 'dist')
+ self.assertTrue(os.path.exists(res))
+ self.assertEqual(os.path.basename(res), 'archive.tar')
+ self.assertEqual(self._tarinfo(res), self._created_files)
+
+ @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
+ def test_make_archive_gztar(self):
+ base_dir = self._create_files()
+ base_name = os.path.join(self.mkdtemp() , 'archive')
+ res = make_archive(base_name, 'gztar', base_dir, 'dist')
+ self.assertTrue(os.path.exists(res))
+ self.assertEqual(os.path.basename(res), 'archive.tar.gz')
+ self.assertEqual(self._tarinfo(res), self._created_files)
+
+ @unittest.skipUnless(bz2, 'Need bz2 support to run')
+ def test_make_archive_bztar(self):
+ base_dir = self._create_files()
+ base_name = os.path.join(self.mkdtemp() , 'archive')
+ res = make_archive(base_name, 'bztar', base_dir, 'dist')
+ self.assertTrue(os.path.exists(res))
+ self.assertEqual(os.path.basename(res), 'archive.tar.bz2')
+ self.assertEqual(self._tarinfo(res), self._created_files)
+
+ @unittest.skipUnless(lzma, 'Need xz support to run')
+ def test_make_archive_xztar(self):
+ base_dir = self._create_files()
+ base_name = os.path.join(self.mkdtemp() , 'archive')
+ res = make_archive(base_name, 'xztar', base_dir, 'dist')
+ self.assertTrue(os.path.exists(res))
+ self.assertEqual(os.path.basename(res), 'archive.tar.xz')
+ self.assertEqual(self._tarinfo(res), self._created_files)
+
def test_make_archive_owner_group(self):
# testing make_archive with owner and group, with various combinations
# this works even if there's not gid/uid support
@@ -291,7 +343,8 @@ class ArchiveUtilTestCase(support.TempdirManager,
else:
group = owner = 'root'
- base_dir, root_dir, base_name = self._create_files()
+ base_dir = self._create_files()
+ root_dir = self.mkdtemp()
base_name = os.path.join(self.mkdtemp() , 'archive')
res = make_archive(base_name, 'zip', root_dir, base_dir, owner=owner,
group=group)
@@ -311,7 +364,8 @@ class ArchiveUtilTestCase(support.TempdirManager,
@unittest.skipUnless(ZLIB_SUPPORT, "Requires zlib")
@unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support")
def test_tarfile_root_owner(self):
- tmpdir, tmpdir2, base_name = self._create_files()
+ tmpdir = self._create_files()
+ base_name = os.path.join(self.mkdtemp(), 'archive')
old_dir = os.getcwd()
os.chdir(tmpdir)
group = grp.getgrgid(0)[0]
diff --git a/Lib/distutils/tests/test_bdist.py b/Lib/distutils/tests/test_bdist.py
index 503a6e8..f762f5d 100644
--- a/Lib/distutils/tests/test_bdist.py
+++ b/Lib/distutils/tests/test_bdist.py
@@ -21,7 +21,7 @@ class BuildTestCase(support.TempdirManager,
# what formats does bdist offer?
formats = ['bztar', 'gztar', 'msi', 'rpm', 'tar',
- 'wininst', 'zip', 'ztar']
+ 'wininst', 'xztar', 'zip', 'ztar']
found = sorted(cmd.format_command)
self.assertEqual(found, formats)
diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py
index b9f407f..366ffbe 100644
--- a/Lib/distutils/tests/test_build_ext.py
+++ b/Lib/distutils/tests/test_build_ext.py
@@ -37,6 +37,9 @@ class BuildExtTestCase(TempdirManager,
from distutils.command import build_ext
build_ext.USER_BASE = site.USER_BASE
+ def build_ext(self, *args, **kwargs):
+ return build_ext(*args, **kwargs)
+
def test_build_ext(self):
global ALREADY_TESTED
copy_xxmodule_c(self.tmp_dir)
@@ -44,7 +47,7 @@ class BuildExtTestCase(TempdirManager,
xx_ext = Extension('xx', [xx_c])
dist = Distribution({'name': 'xx', 'ext_modules': [xx_ext]})
dist.package_dir = self.tmp_dir
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
fixup_build_ext(cmd)
cmd.build_lib = self.tmp_dir
cmd.build_temp = self.tmp_dir
@@ -91,7 +94,7 @@ class BuildExtTestCase(TempdirManager,
def test_solaris_enable_shared(self):
dist = Distribution({'name': 'xx'})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
old = sys.platform
sys.platform = 'sunos' # fooling finalize_options
@@ -113,7 +116,7 @@ class BuildExtTestCase(TempdirManager,
def test_user_site(self):
import site
dist = Distribution({'name': 'xx'})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
# making sure the user option is there
options = [name for name, short, lable in
@@ -144,14 +147,14 @@ class BuildExtTestCase(TempdirManager,
# with the optional argument.
modules = [Extension('foo', ['xxx'], optional=False)]
dist = Distribution({'name': 'xx', 'ext_modules': modules})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.ensure_finalized()
self.assertRaises((UnknownFileError, CompileError),
cmd.run) # should raise an error
modules = [Extension('foo', ['xxx'], optional=True)]
dist = Distribution({'name': 'xx', 'ext_modules': modules})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.ensure_finalized()
cmd.run() # should pass
@@ -160,7 +163,7 @@ class BuildExtTestCase(TempdirManager,
# etc.) are in the include search path.
modules = [Extension('foo', ['xxx'], optional=False)]
dist = Distribution({'name': 'xx', 'ext_modules': modules})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.finalize_options()
from distutils import sysconfig
@@ -172,14 +175,14 @@ class BuildExtTestCase(TempdirManager,
# make sure cmd.libraries is turned into a list
# if it's a string
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.libraries = 'my_lib, other_lib lastlib'
cmd.finalize_options()
self.assertEqual(cmd.libraries, ['my_lib', 'other_lib', 'lastlib'])
# make sure cmd.library_dirs is turned into a list
# if it's a string
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.library_dirs = 'my_lib_dir%sother_lib_dir' % os.pathsep
cmd.finalize_options()
self.assertIn('my_lib_dir', cmd.library_dirs)
@@ -187,7 +190,7 @@ class BuildExtTestCase(TempdirManager,
# make sure rpath is turned into a list
# if it's a string
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.rpath = 'one%stwo' % os.pathsep
cmd.finalize_options()
self.assertEqual(cmd.rpath, ['one', 'two'])
@@ -196,32 +199,32 @@ class BuildExtTestCase(TempdirManager,
# make sure define is turned into 2-tuples
# strings if they are ','-separated strings
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.define = 'one,two'
cmd.finalize_options()
self.assertEqual(cmd.define, [('one', '1'), ('two', '1')])
# make sure undef is turned into a list of
# strings if they are ','-separated strings
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.undef = 'one,two'
cmd.finalize_options()
self.assertEqual(cmd.undef, ['one', 'two'])
# make sure swig_opts is turned into a list
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.swig_opts = None
cmd.finalize_options()
self.assertEqual(cmd.swig_opts, [])
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.swig_opts = '1 2'
cmd.finalize_options()
self.assertEqual(cmd.swig_opts, ['1', '2'])
def test_check_extensions_list(self):
dist = Distribution()
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.finalize_options()
#'extensions' option must be a list of Extension instances
@@ -270,7 +273,7 @@ class BuildExtTestCase(TempdirManager,
def test_get_source_files(self):
modules = [Extension('foo', ['xxx'], optional=False)]
dist = Distribution({'name': 'xx', 'ext_modules': modules})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.ensure_finalized()
self.assertEqual(cmd.get_source_files(), ['xxx'])
@@ -279,7 +282,7 @@ class BuildExtTestCase(TempdirManager,
# should not be overriden by a compiler instance
# when the command is run
dist = Distribution()
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.compiler = 'unix'
cmd.ensure_finalized()
cmd.run()
@@ -292,7 +295,7 @@ class BuildExtTestCase(TempdirManager,
ext = Extension('foo', [c_file], optional=False)
dist = Distribution({'name': 'xx',
'ext_modules': [ext]})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
fixup_build_ext(cmd)
cmd.ensure_finalized()
self.assertEqual(len(cmd.get_outputs()), 1)
@@ -355,7 +358,7 @@ class BuildExtTestCase(TempdirManager,
#etree_ext = Extension('lxml.etree', [etree_c])
#dist = Distribution({'name': 'lxml', 'ext_modules': [etree_ext]})
dist = Distribution()
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.inplace = 1
cmd.distribution.package_dir = {'': 'src'}
cmd.distribution.packages = ['lxml', 'lxml.html']
@@ -462,7 +465,7 @@ class BuildExtTestCase(TempdirManager,
'ext_modules': [deptarget_ext]
})
dist.package_dir = self.tmp_dir
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.build_lib = self.tmp_dir
cmd.build_temp = self.tmp_dir
@@ -481,8 +484,19 @@ class BuildExtTestCase(TempdirManager,
self.fail("Wrong deployment target during compilation")
+class ParallelBuildExtTestCase(BuildExtTestCase):
+
+ def build_ext(self, *args, **kwargs):
+ build_ext = super().build_ext(*args, **kwargs)
+ build_ext.parallel = True
+ return build_ext
+
+
def test_suite():
- return unittest.makeSuite(BuildExtTestCase)
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(BuildExtTestCase))
+ suite.addTest(unittest.makeSuite(ParallelBuildExtTestCase))
+ return suite
if __name__ == '__main__':
- support.run_unittest(test_suite())
+ support.run_unittest(__name__)
diff --git a/Lib/distutils/tests/test_build_py.py b/Lib/distutils/tests/test_build_py.py
index c8f6b89..18283dc 100644
--- a/Lib/distutils/tests/test_build_py.py
+++ b/Lib/distutils/tests/test_build_py.py
@@ -120,8 +120,8 @@ class BuildPyTestCase(support.TempdirManager,
found = os.listdir(cmd.build_lib)
self.assertEqual(sorted(found), ['__pycache__', 'boiledeggs.py'])
found = os.listdir(os.path.join(cmd.build_lib, '__pycache__'))
- self.assertEqual(sorted(found),
- ['boiledeggs.%s.pyo' % sys.implementation.cache_tag])
+ expect = 'boiledeggs.{}.opt-1.pyc'.format(sys.implementation.cache_tag)
+ self.assertEqual(sorted(found), [expect])
def test_dir_in_package_data(self):
"""
diff --git a/Lib/distutils/tests/test_install.py b/Lib/distutils/tests/test_install.py
index 18e1e57..9313330 100644
--- a/Lib/distutils/tests/test_install.py
+++ b/Lib/distutils/tests/test_install.py
@@ -20,8 +20,6 @@ from distutils.tests import support
def _make_ext_name(modname):
- if os.name == 'nt' and sys.executable.endswith('_d.exe'):
- modname += '_d'
return modname + sysconfig.get_config_var('EXT_SUFFIX')
diff --git a/Lib/distutils/tests/test_install_lib.py b/Lib/distutils/tests/test_install_lib.py
index 40dd1a9..5378aa8 100644
--- a/Lib/distutils/tests/test_install_lib.py
+++ b/Lib/distutils/tests/test_install_lib.py
@@ -44,12 +44,11 @@ class InstallLibTestCase(support.TempdirManager,
f = os.path.join(project_dir, 'foo.py')
self.write_file(f, '# python file')
cmd.byte_compile([f])
- pyc_file = importlib.util.cache_from_source('foo.py',
- debug_override=True)
- pyo_file = importlib.util.cache_from_source('foo.py',
- debug_override=False)
+ pyc_file = importlib.util.cache_from_source('foo.py', optimization='')
+ pyc_opt_file = importlib.util.cache_from_source('foo.py',
+ optimization=cmd.optimize)
self.assertTrue(os.path.exists(pyc_file))
- self.assertTrue(os.path.exists(pyo_file))
+ self.assertTrue(os.path.exists(pyc_opt_file))
def test_get_outputs(self):
project_dir, dist = self.create_dist()
@@ -66,8 +65,8 @@ class InstallLibTestCase(support.TempdirManager,
cmd.distribution.packages = ['spam']
cmd.distribution.script_name = 'setup.py'
- # get_outputs should return 4 elements: spam/__init__.py, .pyc and
- # .pyo, foo.import-tag-abiflags.so / foo.pyd
+ # get_outputs should return 4 elements: spam/__init__.py and .pyc,
+ # foo.import-tag-abiflags.so / foo.pyd
outputs = cmd.get_outputs()
self.assertEqual(len(outputs), 4, outputs)
diff --git a/Lib/distutils/tests/test_msvccompiler.py b/Lib/distutils/tests/test_msvccompiler.py
new file mode 100644
index 0000000..1f88907
--- /dev/null
+++ b/Lib/distutils/tests/test_msvccompiler.py
@@ -0,0 +1,39 @@
+"""Tests for distutils._msvccompiler."""
+import sys
+import unittest
+import os
+
+from distutils.errors import DistutilsPlatformError
+from distutils.tests import support
+from test.support import run_unittest
+
+
+SKIP_MESSAGE = (None if sys.platform == "win32" else
+ "These tests are only for win32")
+
+@unittest.skipUnless(SKIP_MESSAGE is None, SKIP_MESSAGE)
+class msvccompilerTestCase(support.TempdirManager,
+ unittest.TestCase):
+
+ def test_no_compiler(self):
+ # makes sure query_vcvarsall raises
+ # a DistutilsPlatformError if the compiler
+ # is not found
+ from distutils._msvccompiler import _get_vc_env
+ def _find_vcvarsall():
+ return None
+
+ import distutils._msvccompiler as _msvccompiler
+ old_find_vcvarsall = _msvccompiler._find_vcvarsall
+ _msvccompiler._find_vcvarsall = _find_vcvarsall
+ try:
+ self.assertRaises(DistutilsPlatformError, _get_vc_env,
+ 'wont find this version')
+ finally:
+ _msvccompiler._find_vcvarsall = old_find_vcvarsall
+
+def test_suite():
+ return unittest.makeSuite(msvccompilerTestCase)
+
+if __name__ == "__main__":
+ run_unittest(test_suite())
diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py
index 5adcac5..e423325 100644
--- a/Lib/distutils/util.py
+++ b/Lib/distutils/util.py
@@ -322,11 +322,11 @@ def byte_compile (py_files,
prefix=None, base_dir=None,
verbose=1, dry_run=0,
direct=None):
- """Byte-compile a collection of Python source files to either .pyc
- or .pyo files in a __pycache__ subdirectory. 'py_files' is a list
+ """Byte-compile a collection of Python source files to .pyc
+ files in a __pycache__ subdirectory. 'py_files' is a list
of files to compile; any files that don't end in ".py" are silently
skipped. 'optimize' must be one of the following:
- 0 - don't optimize (generate .pyc)
+ 0 - don't optimize
1 - normal optimization (like "python -O")
2 - extra optimization (like "python -OO")
If 'force' is true, all files are recompiled regardless of
@@ -438,8 +438,9 @@ byte_compile(files, optimize=%r, force=%r,
# cfile - byte-compiled file
# dfile - purported source filename (same as 'file' by default)
if optimize >= 0:
+ opt = '' if optimize == 0 else optimize
cfile = importlib.util.cache_from_source(
- file, debug_override=not optimize)
+ file, optimization=opt)
else:
cfile = importlib.util.cache_from_source(file)
dfile = file
diff --git a/Lib/distutils/version.py b/Lib/distutils/version.py
index ebcab84..af14cc1 100644
--- a/Lib/distutils/version.py
+++ b/Lib/distutils/version.py
@@ -48,12 +48,6 @@ class Version:
return c
return c == 0
- def __ne__(self, other):
- c = self._cmp(other)
- if c is NotImplemented:
- return c
- return c != 0
-
def __lt__(self, other):
c = self._cmp(other)
if c is NotImplemented:
diff --git a/Lib/doctest.py b/Lib/doctest.py
index 64e6d71..96ab0c4 100644
--- a/Lib/doctest.py
+++ b/Lib/doctest.py
@@ -530,8 +530,9 @@ class DocTest:
examples = '1 example'
else:
examples = '%d examples' % len(self.examples)
- return ('<DocTest %s from %s:%s (%s)>' %
- (self.name, self.filename, self.lineno, examples))
+ return ('<%s %s from %s:%s (%s)>' %
+ (self.__class__.__name__,
+ self.name, self.filename, self.lineno, examples))
def __eq__(self, other):
if type(self) is not type(other):
@@ -978,7 +979,8 @@ class DocTestFinder:
for valname, val in obj.__dict__.items():
valname = '%s.%s' % (name, valname)
# Recurse to functions & classes.
- if ((inspect.isroutine(val) or inspect.isclass(val)) and
+ if ((inspect.isroutine(inspect.unwrap(val))
+ or inspect.isclass(val)) and
self._from_module(module, val)):
self._find(tests, val, valname, module, source_lines,
globs, seen)
@@ -1049,7 +1051,7 @@ class DocTestFinder:
filename = None
else:
filename = getattr(module, '__file__', module.__name__)
- if filename[-4:] in (".pyc", ".pyo"):
+ if filename[-4:] == ".pyc":
filename = filename[:-1]
return self._parser.get_doctest(docstring, globs, name,
filename, lineno)
@@ -2367,15 +2369,6 @@ def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None,
suite = _DocTestSuite()
suite.addTest(SkipDocTestCase(module))
return suite
- elif not tests:
- # Why do we want to do this? Because it reveals a bug that might
- # otherwise be hidden.
- # It is probably a bug that this exception is not also raised if the
- # number of doctest examples in tests is zero (i.e. if no doctest
- # examples were found). However, we should probably not be raising
- # an exception at all here, though it is too late to make this change
- # for a maintenance release. See also issue #14649.
- raise ValueError(module, "has no docstrings")
tests.sort()
suite = _DocTestSuite()
@@ -2385,7 +2378,7 @@ def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None,
continue
if not test.filename:
filename = module.__file__
- if filename[-4:] in (".pyc", ".pyo"):
+ if filename[-4:] == ".pyc":
filename = filename[:-1]
test.filename = filename
suite.addTest(DocTestCase(test, **options))
diff --git a/Lib/email/__init__.py b/Lib/email/__init__.py
index ff16f6a..fae8724 100644
--- a/Lib/email/__init__.py
+++ b/Lib/email/__init__.py
@@ -4,8 +4,6 @@
"""A package for parsing, handling, and generating email messages."""
-__version__ = '5.1.0'
-
__all__ = [
'base64mime',
'charset',
diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py
index a9bdf44..f264191 100644
--- a/Lib/email/_header_value_parser.py
+++ b/Lib/email/_header_value_parser.py
@@ -320,17 +320,18 @@ class TokenList(list):
return ''.join(res)
def _fold(self, folded):
+ encoding = 'utf-8' if folded.policy.utf8 else 'ascii'
for part in self.parts:
tstr = str(part)
tlen = len(tstr)
try:
- str(part).encode('us-ascii')
+ str(part).encode(encoding)
except UnicodeEncodeError:
if any(isinstance(x, errors.UndecodableBytesDefect)
for x in part.all_defects):
charset = 'unknown-8bit'
else:
- # XXX: this should be a policy setting
+ # XXX: this should be a policy setting when utf8 is False.
charset = 'utf-8'
tstr = part.cte_encode(charset, folded.policy)
tlen = len(tstr)
@@ -394,11 +395,12 @@ class UnstructuredTokenList(TokenList):
def _fold(self, folded):
last_ew = None
+ encoding = 'utf-8' if folded.policy.utf8 else 'ascii'
for part in self.parts:
tstr = str(part)
is_ew = False
try:
- str(part).encode('us-ascii')
+ str(part).encode(encoding)
except UnicodeEncodeError:
if any(isinstance(x, errors.UndecodableBytesDefect)
for x in part.all_defects):
@@ -475,12 +477,13 @@ class Phrase(TokenList):
# comment that becomes a barrier across which we can't compose encoded
# words.
last_ew = None
+ encoding = 'utf-8' if folded.policy.utf8 else 'ascii'
for part in self.parts:
tstr = str(part)
tlen = len(tstr)
has_ew = False
try:
- str(part).encode('us-ascii')
+ str(part).encode(encoding)
except UnicodeEncodeError:
if any(isinstance(x, errors.UndecodableBytesDefect)
for x in part.all_defects):
diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py
index 8106114..c0d98a4 100644
--- a/Lib/email/_policybase.py
+++ b/Lib/email/_policybase.py
@@ -149,12 +149,18 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta):
during serialization. None or 0 means no line
wrapping is done. Default is 78.
+ mangle_from_ -- a flag that, when True escapes From_ lines in the
+ body of the message by putting a `>' in front of
+ them. This is used when the message is being
+ serialized by a generator. Default: True.
+
"""
raise_on_defect = False
linesep = '\n'
cte_type = '8bit'
max_line_length = 78
+ mangle_from_ = False
def handle_defect(self, obj, defect):
"""Based on policy, either raise defect or call register_defect.
@@ -266,6 +272,8 @@ class Compat32(Policy):
replicates the behavior of the email package version 5.1.
"""
+ mangle_from_ = True
+
def _sanitize_header(self, name, value):
# If the header value contains surrogates, return a Header using
# the unknown-8bit charset to encode the bytes as encoded words.
diff --git a/Lib/email/charset.py b/Lib/email/charset.py
index e999472..ee56404 100644
--- a/Lib/email/charset.py
+++ b/Lib/email/charset.py
@@ -249,9 +249,6 @@ class Charset:
def __eq__(self, other):
return str(self) == str(other).lower()
- def __ne__(self, other):
- return not self.__eq__(other)
-
def get_body_encoding(self):
"""Return the content-transfer-encoding used for body encoding.
diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py
index c95b27f..e2e3e96 100644
--- a/Lib/email/feedparser.py
+++ b/Lib/email/feedparser.py
@@ -26,6 +26,7 @@ import re
from email import errors
from email import message
from email._policybase import compat32
+from collections import deque
NLCRE = re.compile('\r\n|\r|\n')
NLCRE_bol = re.compile('(\r\n|\r|\n)')
@@ -52,8 +53,8 @@ class BufferedSubFile(object):
def __init__(self):
# Chunks of the last partial line pushed into this object.
self._partial = []
- # The list of full, pushed lines, in reverse order
- self._lines = []
+ # A deque of full, pushed lines
+ self._lines = deque()
# The stack of false-EOF checking predicates.
self._eofstack = []
# A flag indicating whether the file has been closed or not.
@@ -78,21 +79,21 @@ class BufferedSubFile(object):
return NeedMoreData
# Pop the line off the stack and see if it matches the current
# false-EOF predicate.
- line = self._lines.pop()
+ line = self._lines.popleft()
# RFC 2046, section 5.1.2 requires us to recognize outer level
# boundaries at any level of inner nesting. Do this, but be sure it's
# in the order of most to least nested.
- for ateof in self._eofstack[::-1]:
+ for ateof in reversed(self._eofstack):
if ateof(line):
# We're at the false EOF. But push the last line back first.
- self._lines.append(line)
+ self._lines.appendleft(line)
return ''
return line
def unreadline(self, line):
# Let the consumer push a line back into the buffer.
assert line is not NeedMoreData
- self._lines.append(line)
+ self._lines.appendleft(line)
def push(self, data):
"""Push some new data into this object."""
@@ -119,8 +120,7 @@ class BufferedSubFile(object):
self.pushlines(parts)
def pushlines(self, lines):
- # Reverse and insert at the front of the lines.
- self._lines[:0] = lines[::-1]
+ self._lines.extend(lines)
def __iter__(self):
return self
diff --git a/Lib/email/generator.py b/Lib/email/generator.py
index 4735721..11ff16d 100644
--- a/Lib/email/generator.py
+++ b/Lib/email/generator.py
@@ -32,16 +32,16 @@ class Generator:
# Public interface
#
- def __init__(self, outfp, mangle_from_=True, maxheaderlen=None, *,
+ def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, *,
policy=None):
"""Create the generator for message flattening.
outfp is the output file-like object for writing the message to. It
must have a write() method.
- Optional mangle_from_ is a flag that, when True (the default), escapes
- From_ lines in the body of the message by putting a `>' in front of
- them.
+ Optional mangle_from_ is a flag that, when True (the default if policy
+ is not set), escapes From_ lines in the body of the message by putting
+ a `>' in front of them.
Optional maxheaderlen specifies the longest length for a non-continued
header. When a header line is longer (in characters, with tabs
@@ -56,6 +56,9 @@ class Generator:
flatten method is used.
"""
+
+ if mangle_from_ is None:
+ mangle_from_ = True if policy is None else policy.mangle_from_
self._fp = outfp
self._mangle_from_ = mangle_from_
self.maxheaderlen = maxheaderlen
@@ -449,7 +452,7 @@ class DecodedGenerator(Generator):
Like the Generator base class, except that non-text parts are substituted
with a format string representing the part.
"""
- def __init__(self, outfp, mangle_from_=True, maxheaderlen=78, fmt=None):
+ def __init__(self, outfp, mangle_from_=None, maxheaderlen=78, fmt=None):
"""Like Generator.__init__() except that an additional optional
argument is allowed.
diff --git a/Lib/email/header.py b/Lib/email/header.py
index 9c89589..6820ea1 100644
--- a/Lib/email/header.py
+++ b/Lib/email/header.py
@@ -262,9 +262,6 @@ class Header:
# args and do another comparison.
return other == str(self)
- def __ne__(self, other):
- return not self == other
-
def append(self, s, charset=None, errors='strict'):
"""Append a string to the MIME header.
diff --git a/Lib/email/headerregistry.py b/Lib/email/headerregistry.py
index 911a2af..468ca9e 100644
--- a/Lib/email/headerregistry.py
+++ b/Lib/email/headerregistry.py
@@ -81,7 +81,8 @@ class Address:
return lp
def __repr__(self):
- return "Address(display_name={!r}, username={!r}, domain={!r})".format(
+ return "{}(display_name={!r}, username={!r}, domain={!r})".format(
+ self.__class__.__name__,
self.display_name, self.username, self.domain)
def __str__(self):
@@ -132,7 +133,8 @@ class Group:
return self._addresses
def __repr__(self):
- return "Group(display_name={!r}, addresses={!r}".format(
+ return "{}(display_name={!r}, addresses={!r}".format(
+ self.__class__.__name__,
self.display_name, self.addresses)
def __str__(self):
diff --git a/Lib/email/message.py b/Lib/email/message.py
index 2f37dbb..a892012 100644
--- a/Lib/email/message.py
+++ b/Lib/email/message.py
@@ -927,20 +927,21 @@ class Message:
"""
return [part.get_content_charset(failobj) for part in self.walk()]
+ def get_content_disposition(self):
+ """Return the message's content-disposition if it exists, or None.
+
+ The return values can be either 'inline', 'attachment' or None
+ according to the rfc2183.
+ """
+ value = self.get('content-disposition')
+ if value is None:
+ return None
+ c_d = _splitparam(value)[0].lower()
+ return c_d
+
# I.e. def walk(self): ...
from email.iterators import walk
-# XXX Support for temporary deprecation hack for is_attachment property.
-class _IsAttachment:
- def __init__(self, value):
- self.value = value
- def __call__(self):
- return self.value
- def __bool__(self):
- warnings.warn("is_attachment will be a method, not a property, in 3.5",
- DeprecationWarning,
- stacklevel=3)
- return self.value
class MIMEPart(Message):
@@ -950,12 +951,9 @@ class MIMEPart(Message):
policy = default
Message.__init__(self, policy)
- @property
def is_attachment(self):
c_d = self.get('content-disposition')
- result = False if c_d is None else c_d.content_disposition == 'attachment'
- # XXX transitional hack to raise deprecation if not called.
- return _IsAttachment(result)
+ return False if c_d is None else c_d.content_disposition == 'attachment'
def _find_body(self, part, preferencelist):
if part.is_attachment():
diff --git a/Lib/email/mime/text.py b/Lib/email/mime/text.py
index ec18b85..479928e 100644
--- a/Lib/email/mime/text.py
+++ b/Lib/email/mime/text.py
@@ -6,6 +6,7 @@
__all__ = ['MIMEText']
+from email.charset import Charset
from email.mime.nonmultipart import MIMENonMultipart
@@ -34,6 +35,8 @@ class MIMEText(MIMENonMultipart):
_charset = 'us-ascii'
except UnicodeEncodeError:
_charset = 'utf-8'
+ if isinstance(_charset, Charset):
+ _charset = str(_charset)
MIMENonMultipart.__init__(self, 'text', _subtype,
**{'charset': _charset})
diff --git a/Lib/email/policy.py b/Lib/email/policy.py
index f0b20f4..6ac64a5 100644
--- a/Lib/email/policy.py
+++ b/Lib/email/policy.py
@@ -35,6 +35,13 @@ class EmailPolicy(Policy):
In addition to the settable attributes listed above that apply to
all Policies, this policy adds the following additional attributes:
+ utf8 -- if False (the default) message headers will be
+ serialized as ASCII, using encoded words to encode
+ any non-ASCII characters in the source strings. If
+ True, the message headers will be serialized using
+ utf8 and will not contain encoded words (see RFC
+ 6532 for more on this serialization format).
+
refold_source -- if the value for a header in the Message object
came from the parsing of some source, this attribute
indicates whether or not a generator should refold
@@ -72,6 +79,7 @@ class EmailPolicy(Policy):
"""
+ utf8 = False
refold_source = 'long'
header_factory = HeaderRegistry()
content_manager = raw_data_manager
@@ -175,9 +183,13 @@ class EmailPolicy(Policy):
refold_header setting, since there is no way to know whether the binary
data consists of single byte characters or multibyte characters.
+ If utf8 is true, headers are encoded to utf8, otherwise to ascii with
+ non-ASCII unicode rendered as encoded words.
+
"""
folded = self._fold(name, value, refold_binary=self.cte_type=='7bit')
- return folded.encode('ascii', 'surrogateescape')
+ charset = 'utf8' if self.utf8 else 'ascii'
+ return folded.encode(charset, 'surrogateescape')
def _fold(self, name, value, refold_binary=False):
if hasattr(value, 'name'):
@@ -199,3 +211,4 @@ del default.header_factory
strict = default.clone(raise_on_defect=True)
SMTP = default.clone(linesep='\r\n')
HTTP = default.clone(linesep='\r\n', max_line_length=None)
+SMTPUTF8 = SMTP.clone(utf8=True)
diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py
index 4cbaade..67c828d 100644
--- a/Lib/encodings/aliases.py
+++ b/Lib/encodings/aliases.py
@@ -412,6 +412,11 @@ aliases = {
# koi8_r codec
'cskoi8r' : 'koi8_r',
+ # kz1048 codec
+ 'kz_1048' : 'kz1048',
+ 'rk1048' : 'kz1048',
+ 'strk1048_2002' : 'kz1048',
+
# latin_1 codec
#
# Note that the latin_1 codec is implemented internally in C and a
diff --git a/Lib/encodings/cp65001.py b/Lib/encodings/cp65001.py
index 287eb87..95cb2ae 100644
--- a/Lib/encodings/cp65001.py
+++ b/Lib/encodings/cp65001.py
@@ -11,20 +11,23 @@ if not hasattr(codecs, 'code_page_encode'):
### Codec APIs
encode = functools.partial(codecs.code_page_encode, 65001)
-decode = functools.partial(codecs.code_page_decode, 65001)
+_decode = functools.partial(codecs.code_page_decode, 65001)
+
+def decode(input, errors='strict'):
+ return codecs.code_page_decode(65001, input, errors, True)
class IncrementalEncoder(codecs.IncrementalEncoder):
def encode(self, input, final=False):
return encode(input, self.errors)[0]
class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
- _buffer_decode = decode
+ _buffer_decode = _decode
class StreamWriter(codecs.StreamWriter):
encode = encode
class StreamReader(codecs.StreamReader):
- decode = decode
+ decode = _decode
### encodings module API
diff --git a/Lib/encodings/koi8_t.py b/Lib/encodings/koi8_t.py
new file mode 100644
index 0000000..b5415ba
--- /dev/null
+++ b/Lib/encodings/koi8_t.py
@@ -0,0 +1,308 @@
+""" Python Character Mapping Codec koi8_t
+"""
+# http://ru.wikipedia.org/wiki/КОИ-8
+# http://www.opensource.apple.com/source/libiconv/libiconv-4/libiconv/tests/KOI8-T.TXT
+
+import codecs
+
+### Codec APIs
+
+class Codec(codecs.Codec):
+
+ def encode(self,input,errors='strict'):
+ return codecs.charmap_encode(input,errors,encoding_table)
+
+ def decode(self,input,errors='strict'):
+ return codecs.charmap_decode(input,errors,decoding_table)
+
+class IncrementalEncoder(codecs.IncrementalEncoder):
+ def encode(self, input, final=False):
+ return codecs.charmap_encode(input,self.errors,encoding_table)[0]
+
+class IncrementalDecoder(codecs.IncrementalDecoder):
+ def decode(self, input, final=False):
+ return codecs.charmap_decode(input,self.errors,decoding_table)[0]
+
+class StreamWriter(Codec,codecs.StreamWriter):
+ pass
+
+class StreamReader(Codec,codecs.StreamReader):
+ pass
+
+### encodings module API
+
+def getregentry():
+ return codecs.CodecInfo(
+ name='koi8-t',
+ encode=Codec().encode,
+ decode=Codec().decode,
+ incrementalencoder=IncrementalEncoder,
+ incrementaldecoder=IncrementalDecoder,
+ streamreader=StreamReader,
+ streamwriter=StreamWriter,
+ )
+
+
+### Decoding Table
+
+decoding_table = (
+ '\x00' # 0x00 -> NULL
+ '\x01' # 0x01 -> START OF HEADING
+ '\x02' # 0x02 -> START OF TEXT
+ '\x03' # 0x03 -> END OF TEXT
+ '\x04' # 0x04 -> END OF TRANSMISSION
+ '\x05' # 0x05 -> ENQUIRY
+ '\x06' # 0x06 -> ACKNOWLEDGE
+ '\x07' # 0x07 -> BELL
+ '\x08' # 0x08 -> BACKSPACE
+ '\t' # 0x09 -> HORIZONTAL TABULATION
+ '\n' # 0x0A -> LINE FEED
+ '\x0b' # 0x0B -> VERTICAL TABULATION
+ '\x0c' # 0x0C -> FORM FEED
+ '\r' # 0x0D -> CARRIAGE RETURN
+ '\x0e' # 0x0E -> SHIFT OUT
+ '\x0f' # 0x0F -> SHIFT IN
+ '\x10' # 0x10 -> DATA LINK ESCAPE
+ '\x11' # 0x11 -> DEVICE CONTROL ONE
+ '\x12' # 0x12 -> DEVICE CONTROL TWO
+ '\x13' # 0x13 -> DEVICE CONTROL THREE
+ '\x14' # 0x14 -> DEVICE CONTROL FOUR
+ '\x15' # 0x15 -> NEGATIVE ACKNOWLEDGE
+ '\x16' # 0x16 -> SYNCHRONOUS IDLE
+ '\x17' # 0x17 -> END OF TRANSMISSION BLOCK
+ '\x18' # 0x18 -> CANCEL
+ '\x19' # 0x19 -> END OF MEDIUM
+ '\x1a' # 0x1A -> SUBSTITUTE
+ '\x1b' # 0x1B -> ESCAPE
+ '\x1c' # 0x1C -> FILE SEPARATOR
+ '\x1d' # 0x1D -> GROUP SEPARATOR
+ '\x1e' # 0x1E -> RECORD SEPARATOR
+ '\x1f' # 0x1F -> UNIT SEPARATOR
+ ' ' # 0x20 -> SPACE
+ '!' # 0x21 -> EXCLAMATION MARK
+ '"' # 0x22 -> QUOTATION MARK
+ '#' # 0x23 -> NUMBER SIGN
+ '$' # 0x24 -> DOLLAR SIGN
+ '%' # 0x25 -> PERCENT SIGN
+ '&' # 0x26 -> AMPERSAND
+ "'" # 0x27 -> APOSTROPHE
+ '(' # 0x28 -> LEFT PARENTHESIS
+ ')' # 0x29 -> RIGHT PARENTHESIS
+ '*' # 0x2A -> ASTERISK
+ '+' # 0x2B -> PLUS SIGN
+ ',' # 0x2C -> COMMA
+ '-' # 0x2D -> HYPHEN-MINUS
+ '.' # 0x2E -> FULL STOP
+ '/' # 0x2F -> SOLIDUS
+ '0' # 0x30 -> DIGIT ZERO
+ '1' # 0x31 -> DIGIT ONE
+ '2' # 0x32 -> DIGIT TWO
+ '3' # 0x33 -> DIGIT THREE
+ '4' # 0x34 -> DIGIT FOUR
+ '5' # 0x35 -> DIGIT FIVE
+ '6' # 0x36 -> DIGIT SIX
+ '7' # 0x37 -> DIGIT SEVEN
+ '8' # 0x38 -> DIGIT EIGHT
+ '9' # 0x39 -> DIGIT NINE
+ ':' # 0x3A -> COLON
+ ';' # 0x3B -> SEMICOLON
+ '<' # 0x3C -> LESS-THAN SIGN
+ '=' # 0x3D -> EQUALS SIGN
+ '>' # 0x3E -> GREATER-THAN SIGN
+ '?' # 0x3F -> QUESTION MARK
+ '@' # 0x40 -> COMMERCIAL AT
+ 'A' # 0x41 -> LATIN CAPITAL LETTER A
+ 'B' # 0x42 -> LATIN CAPITAL LETTER B
+ 'C' # 0x43 -> LATIN CAPITAL LETTER C
+ 'D' # 0x44 -> LATIN CAPITAL LETTER D
+ 'E' # 0x45 -> LATIN CAPITAL LETTER E
+ 'F' # 0x46 -> LATIN CAPITAL LETTER F
+ 'G' # 0x47 -> LATIN CAPITAL LETTER G
+ 'H' # 0x48 -> LATIN CAPITAL LETTER H
+ 'I' # 0x49 -> LATIN CAPITAL LETTER I
+ 'J' # 0x4A -> LATIN CAPITAL LETTER J
+ 'K' # 0x4B -> LATIN CAPITAL LETTER K
+ 'L' # 0x4C -> LATIN CAPITAL LETTER L
+ 'M' # 0x4D -> LATIN CAPITAL LETTER M
+ 'N' # 0x4E -> LATIN CAPITAL LETTER N
+ 'O' # 0x4F -> LATIN CAPITAL LETTER O
+ 'P' # 0x50 -> LATIN CAPITAL LETTER P
+ 'Q' # 0x51 -> LATIN CAPITAL LETTER Q
+ 'R' # 0x52 -> LATIN CAPITAL LETTER R
+ 'S' # 0x53 -> LATIN CAPITAL LETTER S
+ 'T' # 0x54 -> LATIN CAPITAL LETTER T
+ 'U' # 0x55 -> LATIN CAPITAL LETTER U
+ 'V' # 0x56 -> LATIN CAPITAL LETTER V
+ 'W' # 0x57 -> LATIN CAPITAL LETTER W
+ 'X' # 0x58 -> LATIN CAPITAL LETTER X
+ 'Y' # 0x59 -> LATIN CAPITAL LETTER Y
+ 'Z' # 0x5A -> LATIN CAPITAL LETTER Z
+ '[' # 0x5B -> LEFT SQUARE BRACKET
+ '\\' # 0x5C -> REVERSE SOLIDUS
+ ']' # 0x5D -> RIGHT SQUARE BRACKET
+ '^' # 0x5E -> CIRCUMFLEX ACCENT
+ '_' # 0x5F -> LOW LINE
+ '`' # 0x60 -> GRAVE ACCENT
+ 'a' # 0x61 -> LATIN SMALL LETTER A
+ 'b' # 0x62 -> LATIN SMALL LETTER B
+ 'c' # 0x63 -> LATIN SMALL LETTER C
+ 'd' # 0x64 -> LATIN SMALL LETTER D
+ 'e' # 0x65 -> LATIN SMALL LETTER E
+ 'f' # 0x66 -> LATIN SMALL LETTER F
+ 'g' # 0x67 -> LATIN SMALL LETTER G
+ 'h' # 0x68 -> LATIN SMALL LETTER H
+ 'i' # 0x69 -> LATIN SMALL LETTER I
+ 'j' # 0x6A -> LATIN SMALL LETTER J
+ 'k' # 0x6B -> LATIN SMALL LETTER K
+ 'l' # 0x6C -> LATIN SMALL LETTER L
+ 'm' # 0x6D -> LATIN SMALL LETTER M
+ 'n' # 0x6E -> LATIN SMALL LETTER N
+ 'o' # 0x6F -> LATIN SMALL LETTER O
+ 'p' # 0x70 -> LATIN SMALL LETTER P
+ 'q' # 0x71 -> LATIN SMALL LETTER Q
+ 'r' # 0x72 -> LATIN SMALL LETTER R
+ 's' # 0x73 -> LATIN SMALL LETTER S
+ 't' # 0x74 -> LATIN SMALL LETTER T
+ 'u' # 0x75 -> LATIN SMALL LETTER U
+ 'v' # 0x76 -> LATIN SMALL LETTER V
+ 'w' # 0x77 -> LATIN SMALL LETTER W
+ 'x' # 0x78 -> LATIN SMALL LETTER X
+ 'y' # 0x79 -> LATIN SMALL LETTER Y
+ 'z' # 0x7A -> LATIN SMALL LETTER Z
+ '{' # 0x7B -> LEFT CURLY BRACKET
+ '|' # 0x7C -> VERTICAL LINE
+ '}' # 0x7D -> RIGHT CURLY BRACKET
+ '~' # 0x7E -> TILDE
+ '\x7f' # 0x7F -> DELETE
+ '\u049b' # 0x80 -> CYRILLIC SMALL LETTER KA WITH DESCENDER
+ '\u0493' # 0x81 -> CYRILLIC SMALL LETTER GHE WITH STROKE
+ '\u201a' # 0x82 -> SINGLE LOW-9 QUOTATION MARK
+ '\u0492' # 0x83 -> CYRILLIC CAPITAL LETTER GHE WITH STROKE
+ '\u201e' # 0x84 -> DOUBLE LOW-9 QUOTATION MARK
+ '\u2026' # 0x85 -> HORIZONTAL ELLIPSIS
+ '\u2020' # 0x86 -> DAGGER
+ '\u2021' # 0x87 -> DOUBLE DAGGER
+ '\ufffe' # 0x88 -> UNDEFINED
+ '\u2030' # 0x89 -> PER MILLE SIGN
+ '\u04b3' # 0x8A -> CYRILLIC SMALL LETTER HA WITH DESCENDER
+ '\u2039' # 0x8B -> SINGLE LEFT-POINTING ANGLE QUOTATION MARK
+ '\u04b2' # 0x8C -> CYRILLIC CAPITAL LETTER HA WITH DESCENDER
+ '\u04b7' # 0x8D -> CYRILLIC SMALL LETTER CHE WITH DESCENDER
+ '\u04b6' # 0x8E -> CYRILLIC CAPITAL LETTER CHE WITH DESCENDER
+ '\ufffe' # 0x8F -> UNDEFINED
+ '\u049a' # 0x90 -> CYRILLIC CAPITAL LETTER KA WITH DESCENDER
+ '\u2018' # 0x91 -> LEFT SINGLE QUOTATION MARK
+ '\u2019' # 0x92 -> RIGHT SINGLE QUOTATION MARK
+ '\u201c' # 0x93 -> LEFT DOUBLE QUOTATION MARK
+ '\u201d' # 0x94 -> RIGHT DOUBLE QUOTATION MARK
+ '\u2022' # 0x95 -> BULLET
+ '\u2013' # 0x96 -> EN DASH
+ '\u2014' # 0x97 -> EM DASH
+ '\ufffe' # 0x98 -> UNDEFINED
+ '\u2122' # 0x99 -> TRADE MARK SIGN
+ '\ufffe' # 0x9A -> UNDEFINED
+ '\u203a' # 0x9B -> SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
+ '\ufffe' # 0x9C -> UNDEFINED
+ '\ufffe' # 0x9D -> UNDEFINED
+ '\ufffe' # 0x9E -> UNDEFINED
+ '\ufffe' # 0x9F -> UNDEFINED
+ '\ufffe' # 0xA0 -> UNDEFINED
+ '\u04ef' # 0xA1 -> CYRILLIC SMALL LETTER U WITH MACRON
+ '\u04ee' # 0xA2 -> CYRILLIC CAPITAL LETTER U WITH MACRON
+ '\u0451' # 0xA3 -> CYRILLIC SMALL LETTER IO
+ '\xa4' # 0xA4 -> CURRENCY SIGN
+ '\u04e3' # 0xA5 -> CYRILLIC SMALL LETTER I WITH MACRON
+ '\xa6' # 0xA6 -> BROKEN BAR
+ '\xa7' # 0xA7 -> SECTION SIGN
+ '\ufffe' # 0xA8 -> UNDEFINED
+ '\ufffe' # 0xA9 -> UNDEFINED
+ '\ufffe' # 0xAA -> UNDEFINED
+ '\xab' # 0xAB -> LEFT-POINTING DOUBLE ANGLE QUOTATION MARK
+ '\xac' # 0xAC -> NOT SIGN
+ '\xad' # 0xAD -> SOFT HYPHEN
+ '\xae' # 0xAE -> REGISTERED SIGN
+ '\ufffe' # 0xAF -> UNDEFINED
+ '\xb0' # 0xB0 -> DEGREE SIGN
+ '\xb1' # 0xB1 -> PLUS-MINUS SIGN
+ '\xb2' # 0xB2 -> SUPERSCRIPT TWO
+ '\u0401' # 0xB3 -> CYRILLIC CAPITAL LETTER IO
+ '\ufffe' # 0xB4 -> UNDEFINED
+ '\u04e2' # 0xB5 -> CYRILLIC CAPITAL LETTER I WITH MACRON
+ '\xb6' # 0xB6 -> PILCROW SIGN
+ '\xb7' # 0xB7 -> MIDDLE DOT
+ '\ufffe' # 0xB8 -> UNDEFINED
+ '\u2116' # 0xB9 -> NUMERO SIGN
+ '\ufffe' # 0xBA -> UNDEFINED
+ '\xbb' # 0xBB -> RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK
+ '\ufffe' # 0xBC -> UNDEFINED
+ '\ufffe' # 0xBD -> UNDEFINED
+ '\ufffe' # 0xBE -> UNDEFINED
+ '\xa9' # 0xBF -> COPYRIGHT SIGN
+ '\u044e' # 0xC0 -> CYRILLIC SMALL LETTER YU
+ '\u0430' # 0xC1 -> CYRILLIC SMALL LETTER A
+ '\u0431' # 0xC2 -> CYRILLIC SMALL LETTER BE
+ '\u0446' # 0xC3 -> CYRILLIC SMALL LETTER TSE
+ '\u0434' # 0xC4 -> CYRILLIC SMALL LETTER DE
+ '\u0435' # 0xC5 -> CYRILLIC SMALL LETTER IE
+ '\u0444' # 0xC6 -> CYRILLIC SMALL LETTER EF
+ '\u0433' # 0xC7 -> CYRILLIC SMALL LETTER GHE
+ '\u0445' # 0xC8 -> CYRILLIC SMALL LETTER HA
+ '\u0438' # 0xC9 -> CYRILLIC SMALL LETTER I
+ '\u0439' # 0xCA -> CYRILLIC SMALL LETTER SHORT I
+ '\u043a' # 0xCB -> CYRILLIC SMALL LETTER KA
+ '\u043b' # 0xCC -> CYRILLIC SMALL LETTER EL
+ '\u043c' # 0xCD -> CYRILLIC SMALL LETTER EM
+ '\u043d' # 0xCE -> CYRILLIC SMALL LETTER EN
+ '\u043e' # 0xCF -> CYRILLIC SMALL LETTER O
+ '\u043f' # 0xD0 -> CYRILLIC SMALL LETTER PE
+ '\u044f' # 0xD1 -> CYRILLIC SMALL LETTER YA
+ '\u0440' # 0xD2 -> CYRILLIC SMALL LETTER ER
+ '\u0441' # 0xD3 -> CYRILLIC SMALL LETTER ES
+ '\u0442' # 0xD4 -> CYRILLIC SMALL LETTER TE
+ '\u0443' # 0xD5 -> CYRILLIC SMALL LETTER U
+ '\u0436' # 0xD6 -> CYRILLIC SMALL LETTER ZHE
+ '\u0432' # 0xD7 -> CYRILLIC SMALL LETTER VE
+ '\u044c' # 0xD8 -> CYRILLIC SMALL LETTER SOFT SIGN
+ '\u044b' # 0xD9 -> CYRILLIC SMALL LETTER YERU
+ '\u0437' # 0xDA -> CYRILLIC SMALL LETTER ZE
+ '\u0448' # 0xDB -> CYRILLIC SMALL LETTER SHA
+ '\u044d' # 0xDC -> CYRILLIC SMALL LETTER E
+ '\u0449' # 0xDD -> CYRILLIC SMALL LETTER SHCHA
+ '\u0447' # 0xDE -> CYRILLIC SMALL LETTER CHE
+ '\u044a' # 0xDF -> CYRILLIC SMALL LETTER HARD SIGN
+ '\u042e' # 0xE0 -> CYRILLIC CAPITAL LETTER YU
+ '\u0410' # 0xE1 -> CYRILLIC CAPITAL LETTER A
+ '\u0411' # 0xE2 -> CYRILLIC CAPITAL LETTER BE
+ '\u0426' # 0xE3 -> CYRILLIC CAPITAL LETTER TSE
+ '\u0414' # 0xE4 -> CYRILLIC CAPITAL LETTER DE
+ '\u0415' # 0xE5 -> CYRILLIC CAPITAL LETTER IE
+ '\u0424' # 0xE6 -> CYRILLIC CAPITAL LETTER EF
+ '\u0413' # 0xE7 -> CYRILLIC CAPITAL LETTER GHE
+ '\u0425' # 0xE8 -> CYRILLIC CAPITAL LETTER HA
+ '\u0418' # 0xE9 -> CYRILLIC CAPITAL LETTER I
+ '\u0419' # 0xEA -> CYRILLIC CAPITAL LETTER SHORT I
+ '\u041a' # 0xEB -> CYRILLIC CAPITAL LETTER KA
+ '\u041b' # 0xEC -> CYRILLIC CAPITAL LETTER EL
+ '\u041c' # 0xED -> CYRILLIC CAPITAL LETTER EM
+ '\u041d' # 0xEE -> CYRILLIC CAPITAL LETTER EN
+ '\u041e' # 0xEF -> CYRILLIC CAPITAL LETTER O
+ '\u041f' # 0xF0 -> CYRILLIC CAPITAL LETTER PE
+ '\u042f' # 0xF1 -> CYRILLIC CAPITAL LETTER YA
+ '\u0420' # 0xF2 -> CYRILLIC CAPITAL LETTER ER
+ '\u0421' # 0xF3 -> CYRILLIC CAPITAL LETTER ES
+ '\u0422' # 0xF4 -> CYRILLIC CAPITAL LETTER TE
+ '\u0423' # 0xF5 -> CYRILLIC CAPITAL LETTER U
+ '\u0416' # 0xF6 -> CYRILLIC CAPITAL LETTER ZHE
+ '\u0412' # 0xF7 -> CYRILLIC CAPITAL LETTER VE
+ '\u042c' # 0xF8 -> CYRILLIC CAPITAL LETTER SOFT SIGN
+ '\u042b' # 0xF9 -> CYRILLIC CAPITAL LETTER YERU
+ '\u0417' # 0xFA -> CYRILLIC CAPITAL LETTER ZE
+ '\u0428' # 0xFB -> CYRILLIC CAPITAL LETTER SHA
+ '\u042d' # 0xFC -> CYRILLIC CAPITAL LETTER E
+ '\u0429' # 0xFD -> CYRILLIC CAPITAL LETTER SHCHA
+ '\u0427' # 0xFE -> CYRILLIC CAPITAL LETTER CHE
+ '\u042a' # 0xFF -> CYRILLIC CAPITAL LETTER HARD SIGN
+)
+
+### Encoding table
+encoding_table=codecs.charmap_build(decoding_table)
diff --git a/Lib/encodings/kz1048.py b/Lib/encodings/kz1048.py
new file mode 100644
index 0000000..712aee6
--- /dev/null
+++ b/Lib/encodings/kz1048.py
@@ -0,0 +1,307 @@
+""" Python Character Mapping Codec kz1048 generated from 'MAPPINGS/VENDORS/MISC/KZ1048.TXT' with gencodec.py.
+
+"""#"
+
+import codecs
+
+### Codec APIs
+
+class Codec(codecs.Codec):
+
+ def encode(self, input, errors='strict'):
+ return codecs.charmap_encode(input, errors, encoding_table)
+
+ def decode(self, input, errors='strict'):
+ return codecs.charmap_decode(input, errors, decoding_table)
+
+class IncrementalEncoder(codecs.IncrementalEncoder):
+ def encode(self, input, final=False):
+ return codecs.charmap_encode(input, self.errors, encoding_table)[0]
+
+class IncrementalDecoder(codecs.IncrementalDecoder):
+ def decode(self, input, final=False):
+ return codecs.charmap_decode(input, self.errors, decoding_table)[0]
+
+class StreamWriter(Codec, codecs.StreamWriter):
+ pass
+
+class StreamReader(Codec, codecs.StreamReader):
+ pass
+
+### encodings module API
+
+def getregentry():
+ return codecs.CodecInfo(
+ name='kz1048',
+ encode=Codec().encode,
+ decode=Codec().decode,
+ incrementalencoder=IncrementalEncoder,
+ incrementaldecoder=IncrementalDecoder,
+ streamreader=StreamReader,
+ streamwriter=StreamWriter,
+ )
+
+
+### Decoding Table
+
+decoding_table = (
+ '\x00' # 0x00 -> NULL
+ '\x01' # 0x01 -> START OF HEADING
+ '\x02' # 0x02 -> START OF TEXT
+ '\x03' # 0x03 -> END OF TEXT
+ '\x04' # 0x04 -> END OF TRANSMISSION
+ '\x05' # 0x05 -> ENQUIRY
+ '\x06' # 0x06 -> ACKNOWLEDGE
+ '\x07' # 0x07 -> BELL
+ '\x08' # 0x08 -> BACKSPACE
+ '\t' # 0x09 -> HORIZONTAL TABULATION
+ '\n' # 0x0A -> LINE FEED
+ '\x0b' # 0x0B -> VERTICAL TABULATION
+ '\x0c' # 0x0C -> FORM FEED
+ '\r' # 0x0D -> CARRIAGE RETURN
+ '\x0e' # 0x0E -> SHIFT OUT
+ '\x0f' # 0x0F -> SHIFT IN
+ '\x10' # 0x10 -> DATA LINK ESCAPE
+ '\x11' # 0x11 -> DEVICE CONTROL ONE
+ '\x12' # 0x12 -> DEVICE CONTROL TWO
+ '\x13' # 0x13 -> DEVICE CONTROL THREE
+ '\x14' # 0x14 -> DEVICE CONTROL FOUR
+ '\x15' # 0x15 -> NEGATIVE ACKNOWLEDGE
+ '\x16' # 0x16 -> SYNCHRONOUS IDLE
+ '\x17' # 0x17 -> END OF TRANSMISSION BLOCK
+ '\x18' # 0x18 -> CANCEL
+ '\x19' # 0x19 -> END OF MEDIUM
+ '\x1a' # 0x1A -> SUBSTITUTE
+ '\x1b' # 0x1B -> ESCAPE
+ '\x1c' # 0x1C -> FILE SEPARATOR
+ '\x1d' # 0x1D -> GROUP SEPARATOR
+ '\x1e' # 0x1E -> RECORD SEPARATOR
+ '\x1f' # 0x1F -> UNIT SEPARATOR
+ ' ' # 0x20 -> SPACE
+ '!' # 0x21 -> EXCLAMATION MARK
+ '"' # 0x22 -> QUOTATION MARK
+ '#' # 0x23 -> NUMBER SIGN
+ '$' # 0x24 -> DOLLAR SIGN
+ '%' # 0x25 -> PERCENT SIGN
+ '&' # 0x26 -> AMPERSAND
+ "'" # 0x27 -> APOSTROPHE
+ '(' # 0x28 -> LEFT PARENTHESIS
+ ')' # 0x29 -> RIGHT PARENTHESIS
+ '*' # 0x2A -> ASTERISK
+ '+' # 0x2B -> PLUS SIGN
+ ',' # 0x2C -> COMMA
+ '-' # 0x2D -> HYPHEN-MINUS
+ '.' # 0x2E -> FULL STOP
+ '/' # 0x2F -> SOLIDUS
+ '0' # 0x30 -> DIGIT ZERO
+ '1' # 0x31 -> DIGIT ONE
+ '2' # 0x32 -> DIGIT TWO
+ '3' # 0x33 -> DIGIT THREE
+ '4' # 0x34 -> DIGIT FOUR
+ '5' # 0x35 -> DIGIT FIVE
+ '6' # 0x36 -> DIGIT SIX
+ '7' # 0x37 -> DIGIT SEVEN
+ '8' # 0x38 -> DIGIT EIGHT
+ '9' # 0x39 -> DIGIT NINE
+ ':' # 0x3A -> COLON
+ ';' # 0x3B -> SEMICOLON
+ '<' # 0x3C -> LESS-THAN SIGN
+ '=' # 0x3D -> EQUALS SIGN
+ '>' # 0x3E -> GREATER-THAN SIGN
+ '?' # 0x3F -> QUESTION MARK
+ '@' # 0x40 -> COMMERCIAL AT
+ 'A' # 0x41 -> LATIN CAPITAL LETTER A
+ 'B' # 0x42 -> LATIN CAPITAL LETTER B
+ 'C' # 0x43 -> LATIN CAPITAL LETTER C
+ 'D' # 0x44 -> LATIN CAPITAL LETTER D
+ 'E' # 0x45 -> LATIN CAPITAL LETTER E
+ 'F' # 0x46 -> LATIN CAPITAL LETTER F
+ 'G' # 0x47 -> LATIN CAPITAL LETTER G
+ 'H' # 0x48 -> LATIN CAPITAL LETTER H
+ 'I' # 0x49 -> LATIN CAPITAL LETTER I
+ 'J' # 0x4A -> LATIN CAPITAL LETTER J
+ 'K' # 0x4B -> LATIN CAPITAL LETTER K
+ 'L' # 0x4C -> LATIN CAPITAL LETTER L
+ 'M' # 0x4D -> LATIN CAPITAL LETTER M
+ 'N' # 0x4E -> LATIN CAPITAL LETTER N
+ 'O' # 0x4F -> LATIN CAPITAL LETTER O
+ 'P' # 0x50 -> LATIN CAPITAL LETTER P
+ 'Q' # 0x51 -> LATIN CAPITAL LETTER Q
+ 'R' # 0x52 -> LATIN CAPITAL LETTER R
+ 'S' # 0x53 -> LATIN CAPITAL LETTER S
+ 'T' # 0x54 -> LATIN CAPITAL LETTER T
+ 'U' # 0x55 -> LATIN CAPITAL LETTER U
+ 'V' # 0x56 -> LATIN CAPITAL LETTER V
+ 'W' # 0x57 -> LATIN CAPITAL LETTER W
+ 'X' # 0x58 -> LATIN CAPITAL LETTER X
+ 'Y' # 0x59 -> LATIN CAPITAL LETTER Y
+ 'Z' # 0x5A -> LATIN CAPITAL LETTER Z
+ '[' # 0x5B -> LEFT SQUARE BRACKET
+ '\\' # 0x5C -> REVERSE SOLIDUS
+ ']' # 0x5D -> RIGHT SQUARE BRACKET
+ '^' # 0x5E -> CIRCUMFLEX ACCENT
+ '_' # 0x5F -> LOW LINE
+ '`' # 0x60 -> GRAVE ACCENT
+ 'a' # 0x61 -> LATIN SMALL LETTER A
+ 'b' # 0x62 -> LATIN SMALL LETTER B
+ 'c' # 0x63 -> LATIN SMALL LETTER C
+ 'd' # 0x64 -> LATIN SMALL LETTER D
+ 'e' # 0x65 -> LATIN SMALL LETTER E
+ 'f' # 0x66 -> LATIN SMALL LETTER F
+ 'g' # 0x67 -> LATIN SMALL LETTER G
+ 'h' # 0x68 -> LATIN SMALL LETTER H
+ 'i' # 0x69 -> LATIN SMALL LETTER I
+ 'j' # 0x6A -> LATIN SMALL LETTER J
+ 'k' # 0x6B -> LATIN SMALL LETTER K
+ 'l' # 0x6C -> LATIN SMALL LETTER L
+ 'm' # 0x6D -> LATIN SMALL LETTER M
+ 'n' # 0x6E -> LATIN SMALL LETTER N
+ 'o' # 0x6F -> LATIN SMALL LETTER O
+ 'p' # 0x70 -> LATIN SMALL LETTER P
+ 'q' # 0x71 -> LATIN SMALL LETTER Q
+ 'r' # 0x72 -> LATIN SMALL LETTER R
+ 's' # 0x73 -> LATIN SMALL LETTER S
+ 't' # 0x74 -> LATIN SMALL LETTER T
+ 'u' # 0x75 -> LATIN SMALL LETTER U
+ 'v' # 0x76 -> LATIN SMALL LETTER V
+ 'w' # 0x77 -> LATIN SMALL LETTER W
+ 'x' # 0x78 -> LATIN SMALL LETTER X
+ 'y' # 0x79 -> LATIN SMALL LETTER Y
+ 'z' # 0x7A -> LATIN SMALL LETTER Z
+ '{' # 0x7B -> LEFT CURLY BRACKET
+ '|' # 0x7C -> VERTICAL LINE
+ '}' # 0x7D -> RIGHT CURLY BRACKET
+ '~' # 0x7E -> TILDE
+ '\x7f' # 0x7F -> DELETE
+ '\u0402' # 0x80 -> CYRILLIC CAPITAL LETTER DJE
+ '\u0403' # 0x81 -> CYRILLIC CAPITAL LETTER GJE
+ '\u201a' # 0x82 -> SINGLE LOW-9 QUOTATION MARK
+ '\u0453' # 0x83 -> CYRILLIC SMALL LETTER GJE
+ '\u201e' # 0x84 -> DOUBLE LOW-9 QUOTATION MARK
+ '\u2026' # 0x85 -> HORIZONTAL ELLIPSIS
+ '\u2020' # 0x86 -> DAGGER
+ '\u2021' # 0x87 -> DOUBLE DAGGER
+ '\u20ac' # 0x88 -> EURO SIGN
+ '\u2030' # 0x89 -> PER MILLE SIGN
+ '\u0409' # 0x8A -> CYRILLIC CAPITAL LETTER LJE
+ '\u2039' # 0x8B -> SINGLE LEFT-POINTING ANGLE QUOTATION MARK
+ '\u040a' # 0x8C -> CYRILLIC CAPITAL LETTER NJE
+ '\u049a' # 0x8D -> CYRILLIC CAPITAL LETTER KA WITH DESCENDER
+ '\u04ba' # 0x8E -> CYRILLIC CAPITAL LETTER SHHA
+ '\u040f' # 0x8F -> CYRILLIC CAPITAL LETTER DZHE
+ '\u0452' # 0x90 -> CYRILLIC SMALL LETTER DJE
+ '\u2018' # 0x91 -> LEFT SINGLE QUOTATION MARK
+ '\u2019' # 0x92 -> RIGHT SINGLE QUOTATION MARK
+ '\u201c' # 0x93 -> LEFT DOUBLE QUOTATION MARK
+ '\u201d' # 0x94 -> RIGHT DOUBLE QUOTATION MARK
+ '\u2022' # 0x95 -> BULLET
+ '\u2013' # 0x96 -> EN DASH
+ '\u2014' # 0x97 -> EM DASH
+ '\ufffe' # 0x98 -> UNDEFINED
+ '\u2122' # 0x99 -> TRADE MARK SIGN
+ '\u0459' # 0x9A -> CYRILLIC SMALL LETTER LJE
+ '\u203a' # 0x9B -> SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
+ '\u045a' # 0x9C -> CYRILLIC SMALL LETTER NJE
+ '\u049b' # 0x9D -> CYRILLIC SMALL LETTER KA WITH DESCENDER
+ '\u04bb' # 0x9E -> CYRILLIC SMALL LETTER SHHA
+ '\u045f' # 0x9F -> CYRILLIC SMALL LETTER DZHE
+ '\xa0' # 0xA0 -> NO-BREAK SPACE
+ '\u04b0' # 0xA1 -> CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE
+ '\u04b1' # 0xA2 -> CYRILLIC SMALL LETTER STRAIGHT U WITH STROKE
+ '\u04d8' # 0xA3 -> CYRILLIC CAPITAL LETTER SCHWA
+ '\xa4' # 0xA4 -> CURRENCY SIGN
+ '\u04e8' # 0xA5 -> CYRILLIC CAPITAL LETTER BARRED O
+ '\xa6' # 0xA6 -> BROKEN BAR
+ '\xa7' # 0xA7 -> SECTION SIGN
+ '\u0401' # 0xA8 -> CYRILLIC CAPITAL LETTER IO
+ '\xa9' # 0xA9 -> COPYRIGHT SIGN
+ '\u0492' # 0xAA -> CYRILLIC CAPITAL LETTER GHE WITH STROKE
+ '\xab' # 0xAB -> LEFT-POINTING DOUBLE ANGLE QUOTATION MARK
+ '\xac' # 0xAC -> NOT SIGN
+ '\xad' # 0xAD -> SOFT HYPHEN
+ '\xae' # 0xAE -> REGISTERED SIGN
+ '\u04ae' # 0xAF -> CYRILLIC CAPITAL LETTER STRAIGHT U
+ '\xb0' # 0xB0 -> DEGREE SIGN
+ '\xb1' # 0xB1 -> PLUS-MINUS SIGN
+ '\u0406' # 0xB2 -> CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I
+ '\u0456' # 0xB3 -> CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I
+ '\u04e9' # 0xB4 -> CYRILLIC SMALL LETTER BARRED O
+ '\xb5' # 0xB5 -> MICRO SIGN
+ '\xb6' # 0xB6 -> PILCROW SIGN
+ '\xb7' # 0xB7 -> MIDDLE DOT
+ '\u0451' # 0xB8 -> CYRILLIC SMALL LETTER IO
+ '\u2116' # 0xB9 -> NUMERO SIGN
+ '\u0493' # 0xBA -> CYRILLIC SMALL LETTER GHE WITH STROKE
+ '\xbb' # 0xBB -> RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK
+ '\u04d9' # 0xBC -> CYRILLIC SMALL LETTER SCHWA
+ '\u04a2' # 0xBD -> CYRILLIC CAPITAL LETTER EN WITH DESCENDER
+ '\u04a3' # 0xBE -> CYRILLIC SMALL LETTER EN WITH DESCENDER
+ '\u04af' # 0xBF -> CYRILLIC SMALL LETTER STRAIGHT U
+ '\u0410' # 0xC0 -> CYRILLIC CAPITAL LETTER A
+ '\u0411' # 0xC1 -> CYRILLIC CAPITAL LETTER BE
+ '\u0412' # 0xC2 -> CYRILLIC CAPITAL LETTER VE
+ '\u0413' # 0xC3 -> CYRILLIC CAPITAL LETTER GHE
+ '\u0414' # 0xC4 -> CYRILLIC CAPITAL LETTER DE
+ '\u0415' # 0xC5 -> CYRILLIC CAPITAL LETTER IE
+ '\u0416' # 0xC6 -> CYRILLIC CAPITAL LETTER ZHE
+ '\u0417' # 0xC7 -> CYRILLIC CAPITAL LETTER ZE
+ '\u0418' # 0xC8 -> CYRILLIC CAPITAL LETTER I
+ '\u0419' # 0xC9 -> CYRILLIC CAPITAL LETTER SHORT I
+ '\u041a' # 0xCA -> CYRILLIC CAPITAL LETTER KA
+ '\u041b' # 0xCB -> CYRILLIC CAPITAL LETTER EL
+ '\u041c' # 0xCC -> CYRILLIC CAPITAL LETTER EM
+ '\u041d' # 0xCD -> CYRILLIC CAPITAL LETTER EN
+ '\u041e' # 0xCE -> CYRILLIC CAPITAL LETTER O
+ '\u041f' # 0xCF -> CYRILLIC CAPITAL LETTER PE
+ '\u0420' # 0xD0 -> CYRILLIC CAPITAL LETTER ER
+ '\u0421' # 0xD1 -> CYRILLIC CAPITAL LETTER ES
+ '\u0422' # 0xD2 -> CYRILLIC CAPITAL LETTER TE
+ '\u0423' # 0xD3 -> CYRILLIC CAPITAL LETTER U
+ '\u0424' # 0xD4 -> CYRILLIC CAPITAL LETTER EF
+ '\u0425' # 0xD5 -> CYRILLIC CAPITAL LETTER HA
+ '\u0426' # 0xD6 -> CYRILLIC CAPITAL LETTER TSE
+ '\u0427' # 0xD7 -> CYRILLIC CAPITAL LETTER CHE
+ '\u0428' # 0xD8 -> CYRILLIC CAPITAL LETTER SHA
+ '\u0429' # 0xD9 -> CYRILLIC CAPITAL LETTER SHCHA
+ '\u042a' # 0xDA -> CYRILLIC CAPITAL LETTER HARD SIGN
+ '\u042b' # 0xDB -> CYRILLIC CAPITAL LETTER YERU
+ '\u042c' # 0xDC -> CYRILLIC CAPITAL LETTER SOFT SIGN
+ '\u042d' # 0xDD -> CYRILLIC CAPITAL LETTER E
+ '\u042e' # 0xDE -> CYRILLIC CAPITAL LETTER YU
+ '\u042f' # 0xDF -> CYRILLIC CAPITAL LETTER YA
+ '\u0430' # 0xE0 -> CYRILLIC SMALL LETTER A
+ '\u0431' # 0xE1 -> CYRILLIC SMALL LETTER BE
+ '\u0432' # 0xE2 -> CYRILLIC SMALL LETTER VE
+ '\u0433' # 0xE3 -> CYRILLIC SMALL LETTER GHE
+ '\u0434' # 0xE4 -> CYRILLIC SMALL LETTER DE
+ '\u0435' # 0xE5 -> CYRILLIC SMALL LETTER IE
+ '\u0436' # 0xE6 -> CYRILLIC SMALL LETTER ZHE
+ '\u0437' # 0xE7 -> CYRILLIC SMALL LETTER ZE
+ '\u0438' # 0xE8 -> CYRILLIC SMALL LETTER I
+ '\u0439' # 0xE9 -> CYRILLIC SMALL LETTER SHORT I
+ '\u043a' # 0xEA -> CYRILLIC SMALL LETTER KA
+ '\u043b' # 0xEB -> CYRILLIC SMALL LETTER EL
+ '\u043c' # 0xEC -> CYRILLIC SMALL LETTER EM
+ '\u043d' # 0xED -> CYRILLIC SMALL LETTER EN
+ '\u043e' # 0xEE -> CYRILLIC SMALL LETTER O
+ '\u043f' # 0xEF -> CYRILLIC SMALL LETTER PE
+ '\u0440' # 0xF0 -> CYRILLIC SMALL LETTER ER
+ '\u0441' # 0xF1 -> CYRILLIC SMALL LETTER ES
+ '\u0442' # 0xF2 -> CYRILLIC SMALL LETTER TE
+ '\u0443' # 0xF3 -> CYRILLIC SMALL LETTER U
+ '\u0444' # 0xF4 -> CYRILLIC SMALL LETTER EF
+ '\u0445' # 0xF5 -> CYRILLIC SMALL LETTER HA
+ '\u0446' # 0xF6 -> CYRILLIC SMALL LETTER TSE
+ '\u0447' # 0xF7 -> CYRILLIC SMALL LETTER CHE
+ '\u0448' # 0xF8 -> CYRILLIC SMALL LETTER SHA
+ '\u0449' # 0xF9 -> CYRILLIC SMALL LETTER SHCHA
+ '\u044a' # 0xFA -> CYRILLIC SMALL LETTER HARD SIGN
+ '\u044b' # 0xFB -> CYRILLIC SMALL LETTER YERU
+ '\u044c' # 0xFC -> CYRILLIC SMALL LETTER SOFT SIGN
+ '\u044d' # 0xFD -> CYRILLIC SMALL LETTER E
+ '\u044e' # 0xFE -> CYRILLIC SMALL LETTER YU
+ '\u044f' # 0xFF -> CYRILLIC SMALL LETTER YA
+)
+
+### Encoding table
+encoding_table = codecs.charmap_build(decoding_table)
diff --git a/Lib/enum.py b/Lib/enum.py
index 3cd3df8..c28f345 100644
--- a/Lib/enum.py
+++ b/Lib/enum.py
@@ -106,12 +106,20 @@ class EnumMeta(type):
raise ValueError('Invalid enum member name: {0}'.format(
','.join(invalid_names)))
+ # create a default docstring if one has not been provided
+ if '__doc__' not in classdict:
+ classdict['__doc__'] = 'An enumeration.'
+
# create our new Enum type
enum_class = super().__new__(metacls, cls, bases, classdict)
enum_class._member_names_ = [] # names in definition order
enum_class._member_map_ = OrderedDict() # name->value map
enum_class._member_type_ = member_type
+ # save attributes from super classes so we know if we can take
+ # the shortcut of storing members in the class dict
+ base_attributes = {a for b in bases for a in b.__dict__}
+
# Reverse value->name map for hashable values.
enum_class._value2member_map_ = {}
@@ -165,6 +173,11 @@ class EnumMeta(type):
else:
# Aliases don't appear in member names (only in __members__).
enum_class._member_names_.append(member_name)
+ # performance boost for any member that would not shadow
+ # a DynamicClassAttribute
+ if member_name not in base_attributes:
+ setattr(enum_class, member_name, enum_member)
+ # now add to _member_map_
enum_class._member_map_[member_name] = enum_member
try:
# This may fail if value is not hashable. We can't add the value
@@ -193,7 +206,7 @@ class EnumMeta(type):
enum_class.__new__ = Enum.__new__
return enum_class
- def __call__(cls, value, names=None, *, module=None, qualname=None, type=None):
+ def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1):
"""Either returns an existing member, or creates a new enum class.
This method is used both when an enum class is given a value to match
@@ -205,7 +218,7 @@ class EnumMeta(type):
`value` will be the name of the new class.
`names` should be either a string of white-space/comma delimited names
- (values will start at 1), or an iterator/mapping of name, value pairs.
+ (values will start at `start`), or an iterator/mapping of name, value pairs.
`module` should be set to the module this class is being created in;
if it is not set, an attempt to find that module will be made, but if
@@ -221,7 +234,7 @@ class EnumMeta(type):
if names is None: # simple value lookup
return cls.__new__(cls, value)
# otherwise, functional API: we're creating a new Enum type
- return cls._create_(value, names, module=module, qualname=qualname, type=type)
+ return cls._create_(value, names, module=module, qualname=qualname, type=type, start=start)
def __contains__(cls, member):
return isinstance(member, cls) and member._name_ in cls._member_map_
@@ -292,16 +305,16 @@ class EnumMeta(type):
raise AttributeError('Cannot reassign members.')
super().__setattr__(name, value)
- def _create_(cls, class_name, names=None, *, module=None, qualname=None, type=None):
+ def _create_(cls, class_name, names=None, *, module=None, qualname=None, type=None, start=1):
"""Convenience method to create a new Enum class.
`names` can be:
* A string containing member names, separated either with spaces or
- commas. Values are auto-numbered from 1.
- * An iterable of member names. Values are auto-numbered from 1.
+ commas. Values are incremented by 1 from `start`.
+ * An iterable of member names. Values are incremented by 1 from `start`.
* An iterable of (member name, value) pairs.
- * A mapping of member name -> value.
+ * A mapping of member name -> value pairs.
"""
metacls = cls.__class__
@@ -312,7 +325,7 @@ class EnumMeta(type):
if isinstance(names, str):
names = names.replace(',', ' ').split()
if isinstance(names, (tuple, list)) and isinstance(names[0], str):
- names = [(e, i) for (i, e) in enumerate(names, 1)]
+ names = [(e, i) for (i, e) in enumerate(names, start)]
# Here, names is either an iterable of (name, value) or a mapping.
for item in names:
@@ -468,10 +481,9 @@ class Enum(metaclass=EnumMeta):
m
for cls in self.__class__.mro()
for m in cls.__dict__
- if m[0] != '_'
+ if m[0] != '_' and m not in self._member_map_
]
- return (['__class__', '__doc__', '__module__', 'name', 'value'] +
- added_behavior)
+ return (['__class__', '__doc__', '__module__'] + added_behavior)
def __format__(self, format_spec):
# mixed-in Enums should use the mixed-in type's __format__, otherwise
diff --git a/Lib/fileinput.py b/Lib/fileinput.py
index 8af4a57..af810d1 100644
--- a/Lib/fileinput.py
+++ b/Lib/fileinput.py
@@ -277,24 +277,24 @@ class FileInput:
def nextfile(self):
savestdout = self._savestdout
- self._savestdout = 0
+ self._savestdout = None
if savestdout:
sys.stdout = savestdout
output = self._output
- self._output = 0
+ self._output = None
try:
if output:
output.close()
finally:
file = self._file
- self._file = 0
+ self._file = None
try:
if file and not self._isstdin:
file.close()
finally:
backupfilename = self._backupfilename
- self._backupfilename = 0
+ self._backupfilename = None
if backupfilename and not self._backup:
try: os.unlink(backupfilename)
except OSError: pass
diff --git a/Lib/formatter.py b/Lib/formatter.py
index 9338261..5e8e2ff 100644
--- a/Lib/formatter.py
+++ b/Lib/formatter.py
@@ -21,7 +21,7 @@ manage and inserting data into the output.
import sys
import warnings
warnings.warn('the formatter module is deprecated and will be removed in '
- 'Python 3.6', PendingDeprecationWarning)
+ 'Python 3.6', DeprecationWarning, stacklevel=2)
AS_IS = None
diff --git a/Lib/fractions.py b/Lib/fractions.py
index 79e83ff..60b0728 100644
--- a/Lib/fractions.py
+++ b/Lib/fractions.py
@@ -20,6 +20,17 @@ def gcd(a, b):
Unless b==0, the result will have the same sign as b (so that when
b is divided by it, the result comes out positive).
"""
+ import warnings
+ warnings.warn('fractions.gcd() is deprecated. Use math.gcd() instead.',
+ DeprecationWarning, 2)
+ if type(a) is int is type(b):
+ if (b or a) < 0:
+ return -math.gcd(a, b)
+ return math.gcd(a, b)
+ return _gcd(a, b)
+
+def _gcd(a, b):
+ # Supports non-integers for backward compatibility.
while b:
a, b = b, a%b
return a
@@ -70,7 +81,7 @@ class Fraction(numbers.Rational):
__slots__ = ('_numerator', '_denominator')
# We're immutable, so use __new__ not __init__
- def __new__(cls, numerator=0, denominator=None):
+ def __new__(cls, numerator=0, denominator=None, _normalize=True):
"""Constructs a Rational.
Takes a string like '3/2' or '1.5', another Rational instance, a
@@ -104,7 +115,12 @@ class Fraction(numbers.Rational):
self = super(Fraction, cls).__new__(cls)
if denominator is None:
- if isinstance(numerator, numbers.Rational):
+ if type(numerator) is int:
+ self._numerator = numerator
+ self._denominator = 1
+ return self
+
+ elif isinstance(numerator, numbers.Rational):
self._numerator = numerator.numerator
self._denominator = numerator.denominator
return self
@@ -153,6 +169,9 @@ class Fraction(numbers.Rational):
raise TypeError("argument should be a string "
"or a Rational instance")
+ elif type(numerator) is int is type(denominator):
+ pass # *very* normal case
+
elif (isinstance(numerator, numbers.Rational) and
isinstance(denominator, numbers.Rational)):
numerator, denominator = (
@@ -165,9 +184,18 @@ class Fraction(numbers.Rational):
if denominator == 0:
raise ZeroDivisionError('Fraction(%s, 0)' % numerator)
- g = gcd(numerator, denominator)
- self._numerator = numerator // g
- self._denominator = denominator // g
+ if _normalize:
+ if type(numerator) is int is type(denominator):
+ # *very* normal case
+ g = math.gcd(numerator, denominator)
+ if denominator < 0:
+ g = -g
+ else:
+ g = _gcd(numerator, denominator)
+ numerator //= g
+ denominator //= g
+ self._numerator = numerator
+ self._denominator = denominator
return self
@classmethod
@@ -277,7 +305,8 @@ class Fraction(numbers.Rational):
def __repr__(self):
"""repr(self)"""
- return ('Fraction(%s, %s)' % (self._numerator, self._denominator))
+ return '%s(%s, %s)' % (self.__class__.__name__,
+ self._numerator, self._denominator)
def __str__(self):
"""str(self)"""
@@ -395,17 +424,17 @@ class Fraction(numbers.Rational):
def _add(a, b):
"""a + b"""
- return Fraction(a.numerator * b.denominator +
- b.numerator * a.denominator,
- a.denominator * b.denominator)
+ da, db = a.denominator, b.denominator
+ return Fraction(a.numerator * db + b.numerator * da,
+ da * db)
__add__, __radd__ = _operator_fallbacks(_add, operator.add)
def _sub(a, b):
"""a - b"""
- return Fraction(a.numerator * b.denominator -
- b.numerator * a.denominator,
- a.denominator * b.denominator)
+ da, db = a.denominator, b.denominator
+ return Fraction(a.numerator * db - b.numerator * da,
+ da * db)
__sub__, __rsub__ = _operator_fallbacks(_sub, operator.sub)
@@ -453,10 +482,12 @@ class Fraction(numbers.Rational):
power = b.numerator
if power >= 0:
return Fraction(a._numerator ** power,
- a._denominator ** power)
+ a._denominator ** power,
+ _normalize=False)
else:
return Fraction(a._denominator ** -power,
- a._numerator ** -power)
+ a._numerator ** -power,
+ _normalize=False)
else:
# A fractional power will generally produce an
# irrational number.
@@ -480,15 +511,15 @@ class Fraction(numbers.Rational):
def __pos__(a):
"""+a: Coerces a subclass instance to Fraction"""
- return Fraction(a._numerator, a._denominator)
+ return Fraction(a._numerator, a._denominator, _normalize=False)
def __neg__(a):
"""-a"""
- return Fraction(-a._numerator, a._denominator)
+ return Fraction(-a._numerator, a._denominator, _normalize=False)
def __abs__(a):
"""abs(a)"""
- return Fraction(abs(a._numerator), a._denominator)
+ return Fraction(abs(a._numerator), a._denominator, _normalize=False)
def __trunc__(a):
"""trunc(a)"""
@@ -555,6 +586,8 @@ class Fraction(numbers.Rational):
def __eq__(a, b):
"""a == b"""
+ if type(b) is int:
+ return a._numerator == b and a._denominator == 1
if isinstance(b, numbers.Rational):
return (a._numerator == b.numerator and
a._denominator == b.denominator)
diff --git a/Lib/ftplib.py b/Lib/ftplib.py
index 4d92b86..54b0e2c 100644
--- a/Lib/ftplib.py
+++ b/Lib/ftplib.py
@@ -42,7 +42,7 @@ import socket
import warnings
from socket import _GLOBAL_DEFAULT_TIMEOUT
-__all__ = ["FTP", "Netrc"]
+__all__ = ["FTP"]
# Magic number from <socket.h>
MSG_OOB = 0x1 # Process data out of band
@@ -923,115 +923,6 @@ def ftpcp(source, sourcename, target, targetname = '', type = 'I'):
target.voidresp()
-class Netrc:
- """Class to parse & provide access to 'netrc' format files.
-
- See the netrc(4) man page for information on the file format.
-
- WARNING: This class is obsolete -- use module netrc instead.
-
- """
- __defuser = None
- __defpasswd = None
- __defacct = None
-
- def __init__(self, filename=None):
- warnings.warn("This class is deprecated, use the netrc module instead",
- DeprecationWarning, 2)
- if filename is None:
- if "HOME" in os.environ:
- filename = os.path.join(os.environ["HOME"],
- ".netrc")
- else:
- raise OSError("specify file to load or set $HOME")
- self.__hosts = {}
- self.__macros = {}
- fp = open(filename, "r")
- in_macro = 0
- while 1:
- line = fp.readline()
- if not line:
- break
- if in_macro and line.strip():
- macro_lines.append(line)
- continue
- elif in_macro:
- self.__macros[macro_name] = tuple(macro_lines)
- in_macro = 0
- words = line.split()
- host = user = passwd = acct = None
- default = 0
- i = 0
- while i < len(words):
- w1 = words[i]
- if i+1 < len(words):
- w2 = words[i + 1]
- else:
- w2 = None
- if w1 == 'default':
- default = 1
- elif w1 == 'machine' and w2:
- host = w2.lower()
- i = i + 1
- elif w1 == 'login' and w2:
- user = w2
- i = i + 1
- elif w1 == 'password' and w2:
- passwd = w2
- i = i + 1
- elif w1 == 'account' and w2:
- acct = w2
- i = i + 1
- elif w1 == 'macdef' and w2:
- macro_name = w2
- macro_lines = []
- in_macro = 1
- break
- i = i + 1
- if default:
- self.__defuser = user or self.__defuser
- self.__defpasswd = passwd or self.__defpasswd
- self.__defacct = acct or self.__defacct
- if host:
- if host in self.__hosts:
- ouser, opasswd, oacct = \
- self.__hosts[host]
- user = user or ouser
- passwd = passwd or opasswd
- acct = acct or oacct
- self.__hosts[host] = user, passwd, acct
- fp.close()
-
- def get_hosts(self):
- """Return a list of hosts mentioned in the .netrc file."""
- return self.__hosts.keys()
-
- def get_account(self, host):
- """Returns login information for the named host.
-
- The return value is a triple containing userid,
- password, and the accounting field.
-
- """
- host = host.lower()
- user = passwd = acct = None
- if host in self.__hosts:
- user, passwd, acct = self.__hosts[host]
- user = user or self.__defuser
- passwd = passwd or self.__defpasswd
- acct = acct or self.__defacct
- return user, passwd, acct
-
- def get_macros(self):
- """Return a list of all defined macro names."""
- return self.__macros.keys()
-
- def get_macro(self, macro):
- """Return a sequence of lines which define a named macro."""
- return self.__macros[macro]
-
-
-
def test():
'''Test program.
Usage: ftp [-d] [-r[file]] host [-l[dir]] [-d[dir]] [-p] [file] ...
@@ -1045,6 +936,8 @@ def test():
print(test.__doc__)
sys.exit(0)
+ import netrc
+
debugging = 0
rcfile = None
while sys.argv[1] == '-d':
@@ -1059,14 +952,14 @@ def test():
ftp.set_debuglevel(debugging)
userid = passwd = acct = ''
try:
- netrc = Netrc(rcfile)
+ netrcobj = netrc.netrc(rcfile)
except OSError:
if rcfile is not None:
sys.stderr.write("Could not open account file"
" -- using anonymous login.")
else:
try:
- userid, passwd, acct = netrc.get_account(host)
+ userid, acct, passwd = netrcobj.authenticators(host)
except KeyError:
# no account for host
sys.stderr.write(
diff --git a/Lib/functools.py b/Lib/functools.py
index 2c299d7..06a4ff1 100644
--- a/Lib/functools.py
+++ b/Lib/functools.py
@@ -23,7 +23,7 @@ from types import MappingProxyType
from weakref import WeakKeyDictionary
try:
from _thread import RLock
-except:
+except ImportError:
class RLock:
'Dummy reentrant lock for builds without threads'
def __enter__(self): pass
@@ -94,108 +94,109 @@ def wraps(wrapped,
# infinite recursion that could occur when the operator dispatch logic
# detects a NotImplemented result and then calls a reflected method.
-def _gt_from_lt(self, other):
+def _gt_from_lt(self, other, NotImplemented=NotImplemented):
'Return a > b. Computed by @total_ordering from (not a < b) and (a != b).'
op_result = self.__lt__(other)
if op_result is NotImplemented:
- return NotImplemented
+ return op_result
return not op_result and self != other
-def _le_from_lt(self, other):
+def _le_from_lt(self, other, NotImplemented=NotImplemented):
'Return a <= b. Computed by @total_ordering from (a < b) or (a == b).'
op_result = self.__lt__(other)
return op_result or self == other
-def _ge_from_lt(self, other):
+def _ge_from_lt(self, other, NotImplemented=NotImplemented):
'Return a >= b. Computed by @total_ordering from (not a < b).'
op_result = self.__lt__(other)
if op_result is NotImplemented:
- return NotImplemented
+ return op_result
return not op_result
-def _ge_from_le(self, other):
+def _ge_from_le(self, other, NotImplemented=NotImplemented):
'Return a >= b. Computed by @total_ordering from (not a <= b) or (a == b).'
op_result = self.__le__(other)
if op_result is NotImplemented:
- return NotImplemented
+ return op_result
return not op_result or self == other
-def _lt_from_le(self, other):
+def _lt_from_le(self, other, NotImplemented=NotImplemented):
'Return a < b. Computed by @total_ordering from (a <= b) and (a != b).'
op_result = self.__le__(other)
if op_result is NotImplemented:
- return NotImplemented
+ return op_result
return op_result and self != other
-def _gt_from_le(self, other):
+def _gt_from_le(self, other, NotImplemented=NotImplemented):
'Return a > b. Computed by @total_ordering from (not a <= b).'
op_result = self.__le__(other)
if op_result is NotImplemented:
- return NotImplemented
+ return op_result
return not op_result
-def _lt_from_gt(self, other):
+def _lt_from_gt(self, other, NotImplemented=NotImplemented):
'Return a < b. Computed by @total_ordering from (not a > b) and (a != b).'
op_result = self.__gt__(other)
if op_result is NotImplemented:
- return NotImplemented
+ return op_result
return not op_result and self != other
-def _ge_from_gt(self, other):
+def _ge_from_gt(self, other, NotImplemented=NotImplemented):
'Return a >= b. Computed by @total_ordering from (a > b) or (a == b).'
op_result = self.__gt__(other)
return op_result or self == other
-def _le_from_gt(self, other):
+def _le_from_gt(self, other, NotImplemented=NotImplemented):
'Return a <= b. Computed by @total_ordering from (not a > b).'
op_result = self.__gt__(other)
if op_result is NotImplemented:
- return NotImplemented
+ return op_result
return not op_result
-def _le_from_ge(self, other):
+def _le_from_ge(self, other, NotImplemented=NotImplemented):
'Return a <= b. Computed by @total_ordering from (not a >= b) or (a == b).'
op_result = self.__ge__(other)
if op_result is NotImplemented:
- return NotImplemented
+ return op_result
return not op_result or self == other
-def _gt_from_ge(self, other):
+def _gt_from_ge(self, other, NotImplemented=NotImplemented):
'Return a > b. Computed by @total_ordering from (a >= b) and (a != b).'
op_result = self.__ge__(other)
if op_result is NotImplemented:
- return NotImplemented
+ return op_result
return op_result and self != other
-def _lt_from_ge(self, other):
+def _lt_from_ge(self, other, NotImplemented=NotImplemented):
'Return a < b. Computed by @total_ordering from (not a >= b).'
op_result = self.__ge__(other)
if op_result is NotImplemented:
- return NotImplemented
+ return op_result
return not op_result
+_convert = {
+ '__lt__': [('__gt__', _gt_from_lt),
+ ('__le__', _le_from_lt),
+ ('__ge__', _ge_from_lt)],
+ '__le__': [('__ge__', _ge_from_le),
+ ('__lt__', _lt_from_le),
+ ('__gt__', _gt_from_le)],
+ '__gt__': [('__lt__', _lt_from_gt),
+ ('__ge__', _ge_from_gt),
+ ('__le__', _le_from_gt)],
+ '__ge__': [('__le__', _le_from_ge),
+ ('__gt__', _gt_from_ge),
+ ('__lt__', _lt_from_ge)]
+}
+
def total_ordering(cls):
"""Class decorator that fills in missing ordering methods"""
- convert = {
- '__lt__': [('__gt__', _gt_from_lt),
- ('__le__', _le_from_lt),
- ('__ge__', _ge_from_lt)],
- '__le__': [('__ge__', _ge_from_le),
- ('__lt__', _lt_from_le),
- ('__gt__', _gt_from_le)],
- '__gt__': [('__lt__', _lt_from_gt),
- ('__ge__', _ge_from_gt),
- ('__le__', _le_from_gt)],
- '__ge__': [('__le__', _le_from_ge),
- ('__gt__', _gt_from_ge),
- ('__lt__', _lt_from_ge)]
- }
# Find user-defined comparisons (not those inherited from object).
- roots = [op for op in convert if getattr(cls, op, None) is not getattr(object, op, None)]
+ roots = [op for op in _convert if getattr(cls, op, None) is not getattr(object, op, None)]
if not roots:
raise ValueError('must define at least one ordering operation: < > <= >=')
root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__
- for opname, opfunc in convert[root]:
+ for opname, opfunc in _convert[root]:
if opname not in roots:
opfunc.__name__ = opname
setattr(cls, opname, opfunc)
@@ -222,8 +223,6 @@ def cmp_to_key(mycmp):
return mycmp(self.obj, other.obj) <= 0
def __ge__(self, other):
return mycmp(self.obj, other.obj) >= 0
- def __ne__(self, other):
- return mycmp(self.obj, other.obj) != 0
__hash__ = None
return K
@@ -242,6 +241,14 @@ def partial(func, *args, **keywords):
"""New function with partial application of the given arguments
and keywords.
"""
+ if hasattr(func, 'func'):
+ args = func.args + args
+ tmpkw = func.keywords.copy()
+ tmpkw.update(keywords)
+ keywords = tmpkw
+ del tmpkw
+ func = func.func
+
def newfunc(*fargs, **fkeywords):
newkeywords = keywords.copy()
newkeywords.update(fkeywords)
@@ -291,7 +298,7 @@ class partialmethod(object):
for k, v in self.keywords.items())
format_string = "{module}.{cls}({func}, {args}, {keywords})"
return format_string.format(module=self.__class__.__module__,
- cls=self.__class__.__name__,
+ cls=self.__class__.__qualname__,
func=self.func,
args=args,
keywords=keywords)
@@ -412,120 +419,129 @@ def lru_cache(maxsize=128, typed=False):
if maxsize is not None and not isinstance(maxsize, int):
raise TypeError('Expected maxsize to be an integer or None')
+ def decorating_function(user_function):
+ wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo)
+ return update_wrapper(wrapper, user_function)
+
+ return decorating_function
+
+def _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo):
# Constants shared by all lru cache instances:
sentinel = object() # unique object used to signal cache misses
make_key = _make_key # build a key from the function arguments
PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields
- def decorating_function(user_function):
- cache = {}
- hits = misses = 0
- full = False
- cache_get = cache.get # bound method to lookup a key or return None
- lock = RLock() # because linkedlist updates aren't threadsafe
- root = [] # root of the circular doubly linked list
- root[:] = [root, root, None, None] # initialize by pointing to self
-
- if maxsize == 0:
-
- def wrapper(*args, **kwds):
- # No caching -- just a statistics update after a successful call
- nonlocal misses
- result = user_function(*args, **kwds)
- misses += 1
+ cache = {}
+ hits = misses = 0
+ full = False
+ cache_get = cache.get # bound method to lookup a key or return None
+ lock = RLock() # because linkedlist updates aren't threadsafe
+ root = [] # root of the circular doubly linked list
+ root[:] = [root, root, None, None] # initialize by pointing to self
+
+ if maxsize == 0:
+
+ def wrapper(*args, **kwds):
+ # No caching -- just a statistics update after a successful call
+ nonlocal misses
+ result = user_function(*args, **kwds)
+ misses += 1
+ return result
+
+ elif maxsize is None:
+
+ def wrapper(*args, **kwds):
+ # Simple caching without ordering or size limit
+ nonlocal hits, misses
+ key = make_key(args, kwds, typed)
+ result = cache_get(key, sentinel)
+ if result is not sentinel:
+ hits += 1
return result
+ result = user_function(*args, **kwds)
+ cache[key] = result
+ misses += 1
+ return result
- elif maxsize is None:
+ else:
- def wrapper(*args, **kwds):
- # Simple caching without ordering or size limit
- nonlocal hits, misses
- key = make_key(args, kwds, typed)
- result = cache_get(key, sentinel)
- if result is not sentinel:
+ def wrapper(*args, **kwds):
+ # Size limited caching that tracks accesses by recency
+ nonlocal root, hits, misses, full
+ key = make_key(args, kwds, typed)
+ with lock:
+ link = cache_get(key)
+ if link is not None:
+ # Move the link to the front of the circular queue
+ link_prev, link_next, _key, result = link
+ link_prev[NEXT] = link_next
+ link_next[PREV] = link_prev
+ last = root[PREV]
+ last[NEXT] = root[PREV] = link
+ link[PREV] = last
+ link[NEXT] = root
hits += 1
return result
- result = user_function(*args, **kwds)
- cache[key] = result
+ result = user_function(*args, **kwds)
+ with lock:
+ if key in cache:
+ # Getting here means that this same key was added to the
+ # cache while the lock was released. Since the link
+ # update is already done, we need only return the
+ # computed result and update the count of misses.
+ pass
+ elif full:
+ # Use the old root to store the new key and result.
+ oldroot = root
+ oldroot[KEY] = key
+ oldroot[RESULT] = result
+ # Empty the oldest link and make it the new root.
+ # Keep a reference to the old key and old result to
+ # prevent their ref counts from going to zero during the
+ # update. That will prevent potentially arbitrary object
+ # clean-up code (i.e. __del__) from running while we're
+ # still adjusting the links.
+ root = oldroot[NEXT]
+ oldkey = root[KEY]
+ oldresult = root[RESULT]
+ root[KEY] = root[RESULT] = None
+ # Now update the cache dictionary.
+ del cache[oldkey]
+ # Save the potentially reentrant cache[key] assignment
+ # for last, after the root and links have been put in
+ # a consistent state.
+ cache[key] = oldroot
+ else:
+ # Put result in a new link at the front of the queue.
+ last = root[PREV]
+ link = [last, root, key, result]
+ last[NEXT] = root[PREV] = cache[key] = link
+ full = (len(cache) >= maxsize)
misses += 1
- return result
+ return result
- else:
+ def cache_info():
+ """Report cache statistics"""
+ with lock:
+ return _CacheInfo(hits, misses, maxsize, len(cache))
- def wrapper(*args, **kwds):
- # Size limited caching that tracks accesses by recency
- nonlocal root, hits, misses, full
- key = make_key(args, kwds, typed)
- with lock:
- link = cache_get(key)
- if link is not None:
- # Move the link to the front of the circular queue
- link_prev, link_next, _key, result = link
- link_prev[NEXT] = link_next
- link_next[PREV] = link_prev
- last = root[PREV]
- last[NEXT] = root[PREV] = link
- link[PREV] = last
- link[NEXT] = root
- hits += 1
- return result
- result = user_function(*args, **kwds)
- with lock:
- if key in cache:
- # Getting here means that this same key was added to the
- # cache while the lock was released. Since the link
- # update is already done, we need only return the
- # computed result and update the count of misses.
- pass
- elif full:
- # Use the old root to store the new key and result.
- oldroot = root
- oldroot[KEY] = key
- oldroot[RESULT] = result
- # Empty the oldest link and make it the new root.
- # Keep a reference to the old key and old result to
- # prevent their ref counts from going to zero during the
- # update. That will prevent potentially arbitrary object
- # clean-up code (i.e. __del__) from running while we're
- # still adjusting the links.
- root = oldroot[NEXT]
- oldkey = root[KEY]
- oldresult = root[RESULT]
- root[KEY] = root[RESULT] = None
- # Now update the cache dictionary.
- del cache[oldkey]
- # Save the potentially reentrant cache[key] assignment
- # for last, after the root and links have been put in
- # a consistent state.
- cache[key] = oldroot
- else:
- # Put result in a new link at the front of the queue.
- last = root[PREV]
- link = [last, root, key, result]
- last[NEXT] = root[PREV] = cache[key] = link
- full = (len(cache) >= maxsize)
- misses += 1
- return result
+ def cache_clear():
+ """Clear the cache and cache statistics"""
+ nonlocal hits, misses, full
+ with lock:
+ cache.clear()
+ root[:] = [root, root, None, None]
+ hits = misses = 0
+ full = False
- def cache_info():
- """Report cache statistics"""
- with lock:
- return _CacheInfo(hits, misses, maxsize, len(cache))
+ wrapper.cache_info = cache_info
+ wrapper.cache_clear = cache_clear
+ return update_wrapper(wrapper, user_function)
- def cache_clear():
- """Clear the cache and cache statistics"""
- nonlocal hits, misses, full
- with lock:
- cache.clear()
- root[:] = [root, root, None, None]
- hits = misses = 0
- full = False
-
- wrapper.cache_info = cache_info
- wrapper.cache_clear = cache_clear
- return update_wrapper(wrapper, user_function)
-
- return decorating_function
+try:
+ from _functools import _lru_cache_wrapper
+except ImportError:
+ pass
################################################################################
diff --git a/Lib/genericpath.py b/Lib/genericpath.py
index ca4a510..6714061 100644
--- a/Lib/genericpath.py
+++ b/Lib/genericpath.py
@@ -130,3 +130,16 @@ def _splitext(p, sep, altsep, extsep):
filenameIndex += 1
return p, p[:0]
+
+def _check_arg_types(funcname, *args):
+ hasstr = hasbytes = False
+ for s in args:
+ if isinstance(s, str):
+ hasstr = True
+ elif isinstance(s, bytes):
+ hasbytes = True
+ else:
+ raise TypeError('%s() argument must be str or bytes, not %r' %
+ (funcname, s.__class__.__name__)) from None
+ if hasstr and hasbytes:
+ raise TypeError("Can't mix strings and bytes in path components") from None
diff --git a/Lib/gettext.py b/Lib/gettext.py
index 8caf1d1..101378f 100644
--- a/Lib/gettext.py
+++ b/Lib/gettext.py
@@ -227,6 +227,13 @@ class GNUTranslations(NullTranslations):
LE_MAGIC = 0x950412de
BE_MAGIC = 0xde120495
+ # Acceptable .mo versions
+ VERSIONS = (0, 1)
+
+ def _get_versions(self, version):
+ """Returns a tuple of major version, minor version"""
+ return (version >> 16, version & 0xffff)
+
def _parse(self, fp):
"""Override this method to support alternative .mo formats."""
unpack = struct.unpack
@@ -247,6 +254,12 @@ class GNUTranslations(NullTranslations):
ii = '>II'
else:
raise OSError(0, 'Bad magic number', filename)
+
+ major_version, minor_version = self._get_versions(version)
+
+ if major_version not in self.VERSIONS:
+ raise OSError(0, 'Bad version number ' + str(major_version), filename)
+
# Now put all messages from the .mo file buffer into the catalog
# dictionary.
for i in range(0, msgcount):
diff --git a/Lib/glob.py b/Lib/glob.py
index d6eca24..56d6704 100644
--- a/Lib/glob.py
+++ b/Lib/glob.py
@@ -6,7 +6,7 @@ import fnmatch
__all__ = ["glob", "iglob"]
-def glob(pathname):
+def glob(pathname, *, recursive=False):
"""Return a list of paths matching a pathname pattern.
The pattern may contain simple shell-style wildcards a la
@@ -14,10 +14,12 @@ def glob(pathname):
dot are special cases that are not matched by '*' and '?'
patterns.
+ If recursive is true, the pattern '**' will match any files and
+ zero or more directories and subdirectories.
"""
- return list(iglob(pathname))
+ return list(iglob(pathname, recursive=recursive))
-def iglob(pathname):
+def iglob(pathname, *, recursive=False):
"""Return an iterator which yields the paths matching a pathname pattern.
The pattern may contain simple shell-style wildcards a la
@@ -25,6 +27,8 @@ def iglob(pathname):
dot are special cases that are not matched by '*' and '?'
patterns.
+ If recursive is true, the pattern '**' will match any files and
+ zero or more directories and subdirectories.
"""
dirname, basename = os.path.split(pathname)
if not has_magic(pathname):
@@ -37,17 +41,23 @@ def iglob(pathname):
yield pathname
return
if not dirname:
- yield from glob1(None, basename)
+ if recursive and _isrecursive(basename):
+ yield from glob2(dirname, basename)
+ else:
+ yield from glob1(dirname, basename)
return
# `os.path.split()` returns the argument itself as a dirname if it is a
# drive or UNC path. Prevent an infinite recursion if a drive or UNC path
# contains magic characters (i.e. r'\\?\C:').
if dirname != pathname and has_magic(dirname):
- dirs = iglob(dirname)
+ dirs = iglob(dirname, recursive=recursive)
else:
dirs = [dirname]
if has_magic(basename):
- glob_in_dir = glob1
+ if recursive and _isrecursive(basename):
+ glob_in_dir = glob2
+ else:
+ glob_in_dir = glob1
else:
glob_in_dir = glob0
for dirname in dirs:
@@ -83,6 +93,34 @@ def glob0(dirname, basename):
return [basename]
return []
+# This helper function recursively yields relative pathnames inside a literal
+# directory.
+
+def glob2(dirname, pattern):
+ assert _isrecursive(pattern)
+ if dirname:
+ yield pattern[:0]
+ yield from _rlistdir(dirname)
+
+# Recursively yields relative pathnames inside a literal directory.
+
+def _rlistdir(dirname):
+ if not dirname:
+ if isinstance(dirname, bytes):
+ dirname = bytes(os.curdir, 'ASCII')
+ else:
+ dirname = os.curdir
+ try:
+ names = os.listdir(dirname)
+ except os.error:
+ return
+ for x in names:
+ if not _ishidden(x):
+ yield x
+ path = os.path.join(dirname, x) if dirname else x
+ for y in _rlistdir(path):
+ yield os.path.join(x, y)
+
magic_check = re.compile('([*?[])')
magic_check_bytes = re.compile(b'([*?[])')
@@ -97,6 +135,12 @@ def has_magic(s):
def _ishidden(path):
return path[0] in ('.', b'.'[0])
+def _isrecursive(pattern):
+ if isinstance(pattern, bytes):
+ return pattern == b'**'
+ else:
+ return pattern == '**'
+
def escape(pathname):
"""Escape all special characters.
"""
diff --git a/Lib/gzip.py b/Lib/gzip.py
index 7ad00e1..45152e4 100644
--- a/Lib/gzip.py
+++ b/Lib/gzip.py
@@ -9,6 +9,7 @@ import struct, sys, time, os
import zlib
import builtins
import io
+import _compression
__all__ = ["GzipFile", "open", "compress", "decompress"]
@@ -89,49 +90,35 @@ class _PaddedFile:
return self._buffer[read:] + \
self.file.read(size-self._length+read)
- def prepend(self, prepend=b'', readprevious=False):
+ def prepend(self, prepend=b''):
if self._read is None:
self._buffer = prepend
- elif readprevious and len(prepend) <= self._read:
+ else: # Assume data was read since the last prepend() call
self._read -= len(prepend)
return
- else:
- self._buffer = self._buffer[self._read:] + prepend
self._length = len(self._buffer)
self._read = 0
- def unused(self):
- if self._read is None:
- return b''
- return self._buffer[self._read:]
-
- def seek(self, offset, whence=0):
- # This is only ever called with offset=whence=0
- if whence == 1 and self._read is not None:
- if 0 <= offset + self._read <= self._length:
- self._read += offset
- return
- else:
- offset += self._length - self._read
+ def seek(self, off):
self._read = None
self._buffer = None
- return self.file.seek(offset, whence)
-
- def __getattr__(self, name):
- return getattr(self.file, name)
+ return self.file.seek(off)
+ def seekable(self):
+ return True # Allows fast-forwarding even in unseekable streams
-class GzipFile(io.BufferedIOBase):
+class GzipFile(_compression.BaseStream):
"""The GzipFile class simulates most of the methods of a file object with
- the exception of the readinto() and truncate() methods.
+ the exception of the truncate() method.
This class only supports opening files in binary mode. If you need to open a
compressed file in text mode, use the gzip.open() function.
"""
+ # Overridden with internal file object to be closed, if only a filename
+ # is passed in
myfileobj = None
- max_read_chunk = 10 * 1024 * 1024 # 10Mb
def __init__(self, filename=None, mode=None,
compresslevel=9, fileobj=None, mtime=None):
@@ -163,13 +150,8 @@ class GzipFile(io.BufferedIOBase):
at all. The default is 9.
The mtime argument is an optional numeric timestamp to be written
- to the stream when compressing. All gzip compressed streams
- are required to contain a timestamp. If omitted or None, the
- current time is used. This module ignores the timestamp when
- decompressing; however, some programs, such as gunzip, make use
- of it. The format of the timestamp is the same as that of the
- return value of time.time() and of the st_mtime member of the
- object returned by os.stat().
+ to the last modification time field in the stream when compressing.
+ If omitted or None, the current time is used.
"""
@@ -188,18 +170,9 @@ class GzipFile(io.BufferedIOBase):
if mode.startswith('r'):
self.mode = READ
- # Set flag indicating start of a new member
- self._new_member = True
- # Buffer data read from gzip file. extrastart is offset in
- # stream where buffer starts. extrasize is number of
- # bytes remaining in buffer from current stream position.
- self.extrabuf = b""
- self.extrasize = 0
- self.extrastart = 0
+ raw = _GzipReader(fileobj)
+ self._buffer = io.BufferedReader(raw)
self.name = filename
- # Starts small, scales exponentially
- self.min_readsize = 100
- fileobj = _PaddedFile(fileobj)
elif mode.startswith(('w', 'a', 'x')):
self.mode = WRITE
@@ -209,12 +182,11 @@ class GzipFile(io.BufferedIOBase):
-zlib.MAX_WBITS,
zlib.DEF_MEM_LEVEL,
0)
+ self._write_mtime = mtime
else:
raise ValueError("Invalid mode: {!r}".format(mode))
self.fileobj = fileobj
- self.offset = 0
- self.mtime = mtime
if self.mode == WRITE:
self._write_gzip_header()
@@ -227,26 +199,22 @@ class GzipFile(io.BufferedIOBase):
return self.name + ".gz"
return self.name
+ @property
+ def mtime(self):
+ """Last modification time read from stream, or None"""
+ return self._buffer.raw._last_mtime
+
def __repr__(self):
- fileobj = self.fileobj
- if isinstance(fileobj, _PaddedFile):
- fileobj = fileobj.file
- s = repr(fileobj)
+ s = repr(self.fileobj)
return '<gzip ' + s[1:-1] + ' ' + hex(id(self)) + '>'
- def _check_closed(self):
- """Raises a ValueError if the underlying file object has been closed.
-
- """
- if self.closed:
- raise ValueError('I/O operation on closed file.')
-
def _init_write(self, filename):
self.name = filename
self.crc = zlib.crc32(b"") & 0xffffffff
self.size = 0
self.writebuf = []
self.bufsize = 0
+ self.offset = 0 # Current file offset for seek(), tell(), etc
def _write_gzip_header(self):
self.fileobj.write(b'\037\213') # magic header
@@ -265,7 +233,7 @@ class GzipFile(io.BufferedIOBase):
if fname:
flags = FNAME
self.fileobj.write(chr(flags).encode('latin-1'))
- mtime = self.mtime
+ mtime = self._write_mtime
if mtime is None:
mtime = time.time()
write32u(self.fileobj, int(mtime))
@@ -274,59 +242,8 @@ class GzipFile(io.BufferedIOBase):
if fname:
self.fileobj.write(fname + b'\000')
- def _init_read(self):
- self.crc = zlib.crc32(b"") & 0xffffffff
- self.size = 0
-
- def _read_exact(self, n):
- data = self.fileobj.read(n)
- while len(data) < n:
- b = self.fileobj.read(n - len(data))
- if not b:
- raise EOFError("Compressed file ended before the "
- "end-of-stream marker was reached")
- data += b
- return data
-
- def _read_gzip_header(self):
- magic = self.fileobj.read(2)
- if magic == b'':
- return False
-
- if magic != b'\037\213':
- raise OSError('Not a gzipped file')
-
- method, flag, self.mtime = struct.unpack("<BBIxx", self._read_exact(8))
- if method != 8:
- raise OSError('Unknown compression method')
-
- if flag & FEXTRA:
- # Read & discard the extra field, if present
- extra_len, = struct.unpack("<H", self._read_exact(2))
- self._read_exact(extra_len)
- if flag & FNAME:
- # Read and discard a null-terminated string containing the filename
- while True:
- s = self.fileobj.read(1)
- if not s or s==b'\000':
- break
- if flag & FCOMMENT:
- # Read and discard a null-terminated string containing a comment
- while True:
- s = self.fileobj.read(1)
- if not s or s==b'\000':
- break
- if flag & FHCRC:
- self._read_exact(2) # Read & discard the 16-bit header CRC
-
- unused = self.fileobj.unused()
- if unused:
- uncompress = self.decompress.decompress(unused)
- self._add_read_data(uncompress)
- return True
-
def write(self,data):
- self._check_closed()
+ self._check_not_closed()
if self.mode != WRITE:
import errno
raise OSError(errno.EBADF, "write() on read-only GzipFile object")
@@ -334,166 +251,47 @@ class GzipFile(io.BufferedIOBase):
if self.fileobj is None:
raise ValueError("write() on closed GzipFile object")
- # Convert data type if called by io.BufferedWriter.
- if isinstance(data, memoryview):
- data = data.tobytes()
+ if isinstance(data, bytes):
+ length = len(data)
+ else:
+ # accept any data that supports the buffer protocol
+ data = memoryview(data)
+ length = data.nbytes
- if len(data) > 0:
+ if length > 0:
self.fileobj.write(self.compress.compress(data))
- self.size += len(data)
+ self.size += length
self.crc = zlib.crc32(data, self.crc) & 0xffffffff
- self.offset += len(data)
+ self.offset += length
- return len(data)
+ return length
def read(self, size=-1):
- self._check_closed()
+ self._check_not_closed()
if self.mode != READ:
import errno
raise OSError(errno.EBADF, "read() on write-only GzipFile object")
-
- if self.extrasize <= 0 and self.fileobj is None:
- return b''
-
- readsize = 1024
- if size < 0: # get the whole thing
- while self._read(readsize):
- readsize = min(self.max_read_chunk, readsize * 2)
- size = self.extrasize
- else: # just get some more of it
- while size > self.extrasize:
- if not self._read(readsize):
- if size > self.extrasize:
- size = self.extrasize
- break
- readsize = min(self.max_read_chunk, readsize * 2)
-
- offset = self.offset - self.extrastart
- chunk = self.extrabuf[offset: offset + size]
- self.extrasize = self.extrasize - size
-
- self.offset += size
- return chunk
+ return self._buffer.read(size)
def read1(self, size=-1):
- self._check_closed()
+ """Implements BufferedIOBase.read1()
+
+ Reads up to a buffer's worth of data is size is negative."""
+ self._check_not_closed()
if self.mode != READ:
import errno
raise OSError(errno.EBADF, "read1() on write-only GzipFile object")
- if self.extrasize <= 0 and self.fileobj is None:
- return b''
-
- # For certain input data, a single call to _read() may not return
- # any data. In this case, retry until we get some data or reach EOF.
- while self.extrasize <= 0 and self._read():
- pass
- if size < 0 or size > self.extrasize:
- size = self.extrasize
-
- offset = self.offset - self.extrastart
- chunk = self.extrabuf[offset: offset + size]
- self.extrasize -= size
- self.offset += size
- return chunk
+ if size < 0:
+ size = io.DEFAULT_BUFFER_SIZE
+ return self._buffer.read1(size)
def peek(self, n):
+ self._check_not_closed()
if self.mode != READ:
import errno
raise OSError(errno.EBADF, "peek() on write-only GzipFile object")
-
- # Do not return ridiculously small buffers, for one common idiom
- # is to call peek(1) and expect more bytes in return.
- if n < 100:
- n = 100
- if self.extrasize == 0:
- if self.fileobj is None:
- return b''
- # Ensure that we don't return b"" if we haven't reached EOF.
- # 1024 is the same buffering heuristic used in read()
- while self.extrasize == 0 and self._read(max(n, 1024)):
- pass
- offset = self.offset - self.extrastart
- remaining = self.extrasize
- assert remaining == len(self.extrabuf) - offset
- return self.extrabuf[offset:offset + n]
-
- def _unread(self, buf):
- self.extrasize = len(buf) + self.extrasize
- self.offset -= len(buf)
-
- def _read(self, size=1024):
- if self.fileobj is None:
- return False
-
- if self._new_member:
- # If the _new_member flag is set, we have to
- # jump to the next member, if there is one.
- self._init_read()
- if not self._read_gzip_header():
- return False
- self.decompress = zlib.decompressobj(-zlib.MAX_WBITS)
- self._new_member = False
-
- # Read a chunk of data from the file
- buf = self.fileobj.read(size)
-
- # If the EOF has been reached, flush the decompression object
- # and mark this object as finished.
-
- if buf == b"":
- uncompress = self.decompress.flush()
- # Prepend the already read bytes to the fileobj to they can be
- # seen by _read_eof()
- self.fileobj.prepend(self.decompress.unused_data, True)
- self._read_eof()
- self._add_read_data( uncompress )
- return False
-
- uncompress = self.decompress.decompress(buf)
- self._add_read_data( uncompress )
-
- if self.decompress.unused_data != b"":
- # Ending case: we've come to the end of a member in the file,
- # so seek back to the start of the unused data, finish up
- # this member, and read a new gzip header.
- # Prepend the already read bytes to the fileobj to they can be
- # seen by _read_eof() and _read_gzip_header()
- self.fileobj.prepend(self.decompress.unused_data, True)
- # Check the CRC and file size, and set the flag so we read
- # a new member on the next call
- self._read_eof()
- self._new_member = True
- return True
-
- def _add_read_data(self, data):
- self.crc = zlib.crc32(data, self.crc) & 0xffffffff
- offset = self.offset - self.extrastart
- self.extrabuf = self.extrabuf[offset:] + data
- self.extrasize = self.extrasize + len(data)
- self.extrastart = self.offset
- self.size = self.size + len(data)
-
- def _read_eof(self):
- # We've read to the end of the file
- # We check the that the computed CRC and size of the
- # uncompressed data matches the stored values. Note that the size
- # stored is the true file size mod 2**32.
- crc32, isize = struct.unpack("<II", self._read_exact(8))
- if crc32 != self.crc:
- raise OSError("CRC check failed %s != %s" % (hex(crc32),
- hex(self.crc)))
- elif isize != (self.size & 0xffffffff):
- raise OSError("Incorrect length of data produced")
-
- # Gzip files can be padded with zeroes and still have archives.
- # Consume all zero bytes and set the file position to the first
- # non-zero byte. See http://www.gzip.org/#faq8
- c = b"\x00"
- while c == b"\x00":
- c = self.fileobj.read(1)
- if c:
- self.fileobj.prepend(c, True)
+ return self._buffer.peek(n)
@property
def closed(self):
@@ -510,6 +308,8 @@ class GzipFile(io.BufferedIOBase):
write32u(fileobj, self.crc)
# self.size may exceed 2GB, or even 4GB
write32u(fileobj, self.size & 0xffffffff)
+ elif self.mode == READ:
+ self._buffer.close()
finally:
myfileobj = self.myfileobj
if myfileobj:
@@ -517,7 +317,7 @@ class GzipFile(io.BufferedIOBase):
myfileobj.close()
def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
- self._check_closed()
+ self._check_not_closed()
if self.mode == WRITE:
# Ensure the compressor's buffer is flushed
self.fileobj.write(self.compress.flush(zlib_mode))
@@ -536,12 +336,7 @@ class GzipFile(io.BufferedIOBase):
beginning of the file'''
if self.mode != READ:
raise OSError("Can't rewind in write mode")
- self.fileobj.seek(0)
- self._new_member = True
- self.extrabuf = b""
- self.extrasize = 0
- self.extrastart = 0
- self.offset = 0
+ self._buffer.seek(0)
def readable(self):
return self.mode == READ
@@ -552,13 +347,13 @@ class GzipFile(io.BufferedIOBase):
def seekable(self):
return True
- def seek(self, offset, whence=0):
- if whence:
- if whence == 1:
- offset = self.offset + offset
- else:
- raise ValueError('Seek from end not supported')
+ def seek(self, offset, whence=io.SEEK_SET):
if self.mode == WRITE:
+ if whence != io.SEEK_SET:
+ if whence == io.SEEK_CUR:
+ offset = self.offset + offset
+ else:
+ raise ValueError('Seek from end not supported')
if offset < self.offset:
raise OSError('Negative seek in write mode')
count = offset - self.offset
@@ -567,55 +362,156 @@ class GzipFile(io.BufferedIOBase):
self.write(chunk)
self.write(bytes(count % 1024))
elif self.mode == READ:
- if offset < self.offset:
- # for negative seek, rewind and do positive seek
- self.rewind()
- count = offset - self.offset
- for i in range(count // 1024):
- self.read(1024)
- self.read(count % 1024)
+ self._check_not_closed()
+ return self._buffer.seek(offset, whence)
return self.offset
def readline(self, size=-1):
+ self._check_not_closed()
+ return self._buffer.readline(size)
+
+
+class _GzipReader(_compression.DecompressReader):
+ def __init__(self, fp):
+ super().__init__(_PaddedFile(fp), zlib.decompressobj,
+ wbits=-zlib.MAX_WBITS)
+ # Set flag indicating start of a new member
+ self._new_member = True
+ self._last_mtime = None
+
+ def _init_read(self):
+ self._crc = zlib.crc32(b"") & 0xffffffff
+ self._stream_size = 0 # Decompressed size of unconcatenated stream
+
+ def _read_exact(self, n):
+ '''Read exactly *n* bytes from `self._fp`
+
+ This method is required because self._fp may be unbuffered,
+ i.e. return short reads.
+ '''
+
+ data = self._fp.read(n)
+ while len(data) < n:
+ b = self._fp.read(n - len(data))
+ if not b:
+ raise EOFError("Compressed file ended before the "
+ "end-of-stream marker was reached")
+ data += b
+ return data
+
+ def _read_gzip_header(self):
+ magic = self._fp.read(2)
+ if magic == b'':
+ return False
+
+ if magic != b'\037\213':
+ raise OSError('Not a gzipped file (%r)' % magic)
+
+ (method, flag,
+ self._last_mtime) = struct.unpack("<BBIxx", self._read_exact(8))
+ if method != 8:
+ raise OSError('Unknown compression method')
+
+ if flag & FEXTRA:
+ # Read & discard the extra field, if present
+ extra_len, = struct.unpack("<H", self._read_exact(2))
+ self._read_exact(extra_len)
+ if flag & FNAME:
+ # Read and discard a null-terminated string containing the filename
+ while True:
+ s = self._fp.read(1)
+ if not s or s==b'\000':
+ break
+ if flag & FCOMMENT:
+ # Read and discard a null-terminated string containing a comment
+ while True:
+ s = self._fp.read(1)
+ if not s or s==b'\000':
+ break
+ if flag & FHCRC:
+ self._read_exact(2) # Read & discard the 16-bit header CRC
+ return True
+
+ def read(self, size=-1):
if size < 0:
- # Shortcut common case - newline found in buffer.
- offset = self.offset - self.extrastart
- i = self.extrabuf.find(b'\n', offset) + 1
- if i > 0:
- self.extrasize -= i - offset
- self.offset += i - offset
- return self.extrabuf[offset: i]
-
- size = sys.maxsize
- readsize = self.min_readsize
- else:
- readsize = size
- bufs = []
- while size != 0:
- c = self.read(readsize)
- i = c.find(b'\n')
-
- # We set i=size to break out of the loop under two
- # conditions: 1) there's no newline, and the chunk is
- # larger than size, or 2) there is a newline, but the
- # resulting line would be longer than 'size'.
- if (size <= i) or (i == -1 and len(c) > size):
- i = size - 1
-
- if i >= 0 or c == b'':
- bufs.append(c[:i + 1]) # Add portion of last chunk
- self._unread(c[i + 1:]) # Push back rest of chunk
+ return self.readall()
+ # size=0 is special because decompress(max_length=0) is not supported
+ if not size:
+ return b""
+
+ # For certain input data, a single
+ # call to decompress() may not return
+ # any data. In this case, retry until we get some data or reach EOF.
+ while True:
+ if self._decompressor.eof:
+ # Ending case: we've come to the end of a member in the file,
+ # so finish up this member, and read a new gzip header.
+ # Check the CRC and file size, and set the flag so we read
+ # a new member
+ self._read_eof()
+ self._new_member = True
+ self._decompressor = self._decomp_factory(
+ **self._decomp_args)
+
+ if self._new_member:
+ # If the _new_member flag is set, we have to
+ # jump to the next member, if there is one.
+ self._init_read()
+ if not self._read_gzip_header():
+ self._size = self._pos
+ return b""
+ self._new_member = False
+
+ # Read a chunk of data from the file
+ buf = self._fp.read(io.DEFAULT_BUFFER_SIZE)
+
+ uncompress = self._decompressor.decompress(buf, size)
+ if self._decompressor.unconsumed_tail != b"":
+ self._fp.prepend(self._decompressor.unconsumed_tail)
+ elif self._decompressor.unused_data != b"":
+ # Prepend the already read bytes to the fileobj so they can
+ # be seen by _read_eof() and _read_gzip_header()
+ self._fp.prepend(self._decompressor.unused_data)
+
+ if uncompress != b"":
break
+ if buf == b"":
+ raise EOFError("Compressed file ended before the "
+ "end-of-stream marker was reached")
- # Append chunk to list, decrease 'size',
- bufs.append(c)
- size = size - len(c)
- readsize = min(size, readsize * 2)
- if readsize > self.min_readsize:
- self.min_readsize = min(readsize, self.min_readsize * 2, 512)
- return b''.join(bufs) # Return resulting line
+ self._add_read_data( uncompress )
+ self._pos += len(uncompress)
+ return uncompress
+
+ def _add_read_data(self, data):
+ self._crc = zlib.crc32(data, self._crc) & 0xffffffff
+ self._stream_size = self._stream_size + len(data)
+
+ def _read_eof(self):
+ # We've read to the end of the file
+ # We check the that the computed CRC and size of the
+ # uncompressed data matches the stored values. Note that the size
+ # stored is the true file size mod 2**32.
+ crc32, isize = struct.unpack("<II", self._read_exact(8))
+ if crc32 != self._crc:
+ raise OSError("CRC check failed %s != %s" % (hex(crc32),
+ hex(self._crc)))
+ elif isize != (self._stream_size & 0xffffffff):
+ raise OSError("Incorrect length of data produced")
+
+ # Gzip files can be padded with zeroes and still have archives.
+ # Consume all zero bytes and set the file position to the first
+ # non-zero byte. See http://www.gzip.org/#faq8
+ c = b"\x00"
+ while c == b"\x00":
+ c = self._fp.read(1)
+ if c:
+ self._fp.prepend(c)
+ def _rewind(self):
+ super()._rewind()
+ self._new_member = True
def compress(data, compresslevel=9):
"""Compress data in one shot and return the compressed string.
diff --git a/Lib/heapq.py b/Lib/heapq.py
index d615239..07af37e 100644
--- a/Lib/heapq.py
+++ b/Lib/heapq.py
@@ -127,8 +127,6 @@ From all times, sorting has always been a Great Art! :-)
__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge',
'nlargest', 'nsmallest', 'heappushpop']
-from itertools import islice, count, tee, chain
-
def heappush(heap, item):
"""Push item onto heap, maintaining the heap invariant."""
heap.append(item)
@@ -141,9 +139,8 @@ def heappop(heap):
returnitem = heap[0]
heap[0] = lastelt
_siftup(heap, 0)
- else:
- returnitem = lastelt
- return returnitem
+ return returnitem
+ return lastelt
def heapreplace(heap, item):
"""Pop and return the current smallest value, and add the new item.
@@ -179,12 +176,22 @@ def heapify(x):
for i in reversed(range(n//2)):
_siftup(x, i)
-def _heappushpop_max(heap, item):
- """Maxheap version of a heappush followed by a heappop."""
- if heap and item < heap[0]:
- item, heap[0] = heap[0], item
+def _heappop_max(heap):
+ """Maxheap version of a heappop."""
+ lastelt = heap.pop() # raises appropriate IndexError if heap is empty
+ if heap:
+ returnitem = heap[0]
+ heap[0] = lastelt
_siftup_max(heap, 0)
- return item
+ return returnitem
+ return lastelt
+
+def _heapreplace_max(heap, item):
+ """Maxheap version of a heappop followed by a heappush."""
+ returnitem = heap[0] # raises appropriate IndexError if heap is empty
+ heap[0] = item
+ _siftup_max(heap, 0)
+ return returnitem
def _heapify_max(x):
"""Transform list into a maxheap, in-place, in O(len(x)) time."""
@@ -192,42 +199,6 @@ def _heapify_max(x):
for i in reversed(range(n//2)):
_siftup_max(x, i)
-def nlargest(n, iterable):
- """Find the n largest elements in a dataset.
-
- Equivalent to: sorted(iterable, reverse=True)[:n]
- """
- if n < 0:
- return []
- it = iter(iterable)
- result = list(islice(it, n))
- if not result:
- return result
- heapify(result)
- _heappushpop = heappushpop
- for elem in it:
- _heappushpop(result, elem)
- result.sort(reverse=True)
- return result
-
-def nsmallest(n, iterable):
- """Find the n smallest elements in a dataset.
-
- Equivalent to: sorted(iterable)[:n]
- """
- if n < 0:
- return []
- it = iter(iterable)
- result = list(islice(it, n))
- if not result:
- return result
- _heapify_max(result)
- _heappushpop = _heappushpop_max
- for elem in it:
- _heappushpop(result, elem)
- result.sort()
- return result
-
# 'heap' is a heap at all indices >= startpos, except possibly for pos. pos
# is the index of a leaf with a possibly out-of-order value. Restore the
# heap invariant.
@@ -340,13 +311,7 @@ def _siftup_max(heap, pos):
heap[pos] = newitem
_siftdown_max(heap, startpos, pos)
-# If available, use C implementation
-try:
- from _heapq import *
-except ImportError:
- pass
-
-def merge(*iterables):
+def merge(*iterables, key=None, reverse=False):
'''Merge multiple sorted inputs into a single sorted output.
Similar to sorted(itertools.chain(*iterables)) but returns a generator,
@@ -356,51 +321,158 @@ def merge(*iterables):
>>> list(merge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25]))
[0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25]
+ If *key* is not None, applies a key function to each element to determine
+ its sort order.
+
+ >>> list(merge(['dog', 'horse'], ['cat', 'fish', 'kangaroo'], key=len))
+ ['dog', 'cat', 'fish', 'horse', 'kangaroo']
+
'''
- _heappop, _heapreplace, _StopIteration = heappop, heapreplace, StopIteration
- _len = len
h = []
h_append = h.append
- for itnum, it in enumerate(map(iter, iterables)):
+
+ if reverse:
+ _heapify = _heapify_max
+ _heappop = _heappop_max
+ _heapreplace = _heapreplace_max
+ direction = -1
+ else:
+ _heapify = heapify
+ _heappop = heappop
+ _heapreplace = heapreplace
+ direction = 1
+
+ if key is None:
+ for order, it in enumerate(map(iter, iterables)):
+ try:
+ next = it.__next__
+ h_append([next(), order * direction, next])
+ except StopIteration:
+ pass
+ _heapify(h)
+ while len(h) > 1:
+ try:
+ while True:
+ value, order, next = s = h[0]
+ yield value
+ s[0] = next() # raises StopIteration when exhausted
+ _heapreplace(h, s) # restore heap condition
+ except StopIteration:
+ _heappop(h) # remove empty iterator
+ if h:
+ # fast case when only a single iterator remains
+ value, order, next = h[0]
+ yield value
+ yield from next.__self__
+ return
+
+ for order, it in enumerate(map(iter, iterables)):
try:
next = it.__next__
- h_append([next(), itnum, next])
- except _StopIteration:
+ value = next()
+ h_append([key(value), order * direction, value, next])
+ except StopIteration:
pass
- heapify(h)
-
- while _len(h) > 1:
+ _heapify(h)
+ while len(h) > 1:
try:
while True:
- v, itnum, next = s = h[0]
- yield v
- s[0] = next() # raises StopIteration when exhausted
- _heapreplace(h, s) # restore heap condition
- except _StopIteration:
- _heappop(h) # remove empty iterator
+ key_value, order, value, next = s = h[0]
+ yield value
+ value = next()
+ s[0] = key(value)
+ s[2] = value
+ _heapreplace(h, s)
+ except StopIteration:
+ _heappop(h)
if h:
- # fast case when only a single iterator remains
- v, itnum, next = h[0]
- yield v
+ key_value, order, value, next = h[0]
+ yield value
yield from next.__self__
-# Extend the implementations of nsmallest and nlargest to use a key= argument
-_nsmallest = nsmallest
+
+# Algorithm notes for nlargest() and nsmallest()
+# ==============================================
+#
+# Make a single pass over the data while keeping the k most extreme values
+# in a heap. Memory consumption is limited to keeping k values in a list.
+#
+# Measured performance for random inputs:
+#
+# number of comparisons
+# n inputs k-extreme values (average of 5 trials) % more than min()
+# ------------- ---------------- --------------------- -----------------
+# 1,000 100 3,317 231.7%
+# 10,000 100 14,046 40.5%
+# 100,000 100 105,749 5.7%
+# 1,000,000 100 1,007,751 0.8%
+# 10,000,000 100 10,009,401 0.1%
+#
+# Theoretical number of comparisons for k smallest of n random inputs:
+#
+# Step Comparisons Action
+# ---- -------------------------- ---------------------------
+# 1 1.66 * k heapify the first k-inputs
+# 2 n - k compare remaining elements to top of heap
+# 3 k * (1 + lg2(k)) * ln(n/k) replace the topmost value on the heap
+# 4 k * lg2(k) - (k/2) final sort of the k most extreme values
+#
+# Combining and simplifying for a rough estimate gives:
+#
+# comparisons = n + k * (log(k, 2) * log(n/k) + log(k, 2) + log(n/k))
+#
+# Computing the number of comparisons for step 3:
+# -----------------------------------------------
+# * For the i-th new value from the iterable, the probability of being in the
+# k most extreme values is k/i. For example, the probability of the 101st
+# value seen being in the 100 most extreme values is 100/101.
+# * If the value is a new extreme value, the cost of inserting it into the
+# heap is 1 + log(k, 2).
+# * The probability times the cost gives:
+# (k/i) * (1 + log(k, 2))
+# * Summing across the remaining n-k elements gives:
+# sum((k/i) * (1 + log(k, 2)) for i in range(k+1, n+1))
+# * This reduces to:
+# (H(n) - H(k)) * k * (1 + log(k, 2))
+# * Where H(n) is the n-th harmonic number estimated by:
+# gamma = 0.5772156649
+# H(n) = log(n, e) + gamma + 1 / (2 * n)
+# http://en.wikipedia.org/wiki/Harmonic_series_(mathematics)#Rate_of_divergence
+# * Substituting the H(n) formula:
+# comparisons = k * (1 + log(k, 2)) * (log(n/k, e) + (1/n - 1/k) / 2)
+#
+# Worst-case for step 3:
+# ----------------------
+# In the worst case, the input data is reversed sorted so that every new element
+# must be inserted in the heap:
+#
+# comparisons = 1.66 * k + log(k, 2) * (n - k)
+#
+# Alternative Algorithms
+# ----------------------
+# Other algorithms were not used because they:
+# 1) Took much more auxiliary memory,
+# 2) Made multiple passes over the data.
+# 3) Made more comparisons in common cases (small k, large n, semi-random input).
+# See the more detailed comparison of approach at:
+# http://code.activestate.com/recipes/577573-compare-algorithms-for-heapqsmallest
+
def nsmallest(n, iterable, key=None):
"""Find the n smallest elements in a dataset.
Equivalent to: sorted(iterable, key=key)[:n]
"""
- # Short-cut for n==1 is to use min() when len(iterable)>0
+
+ # Short-cut for n==1 is to use min()
if n == 1:
it = iter(iterable)
- head = list(islice(it, 1))
- if not head:
- return []
+ sentinel = object()
if key is None:
- return [min(chain(head, it))]
- return [min(chain(head, it), key=key)]
+ result = min(it, default=sentinel)
+ else:
+ result = min(it, default=sentinel, key=key)
+ return [] if result is sentinel else [result]
# When n>=size, it's faster to use sorted()
try:
@@ -413,32 +485,57 @@ def nsmallest(n, iterable, key=None):
# When key is none, use simpler decoration
if key is None:
- it = zip(iterable, count()) # decorate
- result = _nsmallest(n, it)
- return [r[0] for r in result] # undecorate
+ it = iter(iterable)
+ # put the range(n) first so that zip() doesn't
+ # consume one too many elements from the iterator
+ result = [(elem, i) for i, elem in zip(range(n), it)]
+ if not result:
+ return result
+ _heapify_max(result)
+ top = result[0][0]
+ order = n
+ _heapreplace = _heapreplace_max
+ for elem in it:
+ if elem < top:
+ _heapreplace(result, (elem, order))
+ top = result[0][0]
+ order += 1
+ result.sort()
+ return [r[0] for r in result]
# General case, slowest method
- in1, in2 = tee(iterable)
- it = zip(map(key, in1), count(), in2) # decorate
- result = _nsmallest(n, it)
- return [r[2] for r in result] # undecorate
+ it = iter(iterable)
+ result = [(key(elem), i, elem) for i, elem in zip(range(n), it)]
+ if not result:
+ return result
+ _heapify_max(result)
+ top = result[0][0]
+ order = n
+ _heapreplace = _heapreplace_max
+ for elem in it:
+ k = key(elem)
+ if k < top:
+ _heapreplace(result, (k, order, elem))
+ top = result[0][0]
+ order += 1
+ result.sort()
+ return [r[2] for r in result]
-_nlargest = nlargest
def nlargest(n, iterable, key=None):
"""Find the n largest elements in a dataset.
Equivalent to: sorted(iterable, key=key, reverse=True)[:n]
"""
- # Short-cut for n==1 is to use max() when len(iterable)>0
+ # Short-cut for n==1 is to use max()
if n == 1:
it = iter(iterable)
- head = list(islice(it, 1))
- if not head:
- return []
+ sentinel = object()
if key is None:
- return [max(chain(head, it))]
- return [max(chain(head, it), key=key)]
+ result = max(it, default=sentinel)
+ else:
+ result = max(it, default=sentinel, key=key)
+ return [] if result is sentinel else [result]
# When n>=size, it's faster to use sorted()
try:
@@ -451,26 +548,60 @@ def nlargest(n, iterable, key=None):
# When key is none, use simpler decoration
if key is None:
- it = zip(iterable, count(0,-1)) # decorate
- result = _nlargest(n, it)
- return [r[0] for r in result] # undecorate
+ it = iter(iterable)
+ result = [(elem, i) for i, elem in zip(range(0, -n, -1), it)]
+ if not result:
+ return result
+ heapify(result)
+ top = result[0][0]
+ order = -n
+ _heapreplace = heapreplace
+ for elem in it:
+ if top < elem:
+ _heapreplace(result, (elem, order))
+ top = result[0][0]
+ order -= 1
+ result.sort(reverse=True)
+ return [r[0] for r in result]
# General case, slowest method
- in1, in2 = tee(iterable)
- it = zip(map(key, in1), count(0,-1), in2) # decorate
- result = _nlargest(n, it)
- return [r[2] for r in result] # undecorate
+ it = iter(iterable)
+ result = [(key(elem), i, elem) for i, elem in zip(range(0, -n, -1), it)]
+ if not result:
+ return result
+ heapify(result)
+ top = result[0][0]
+ order = -n
+ _heapreplace = heapreplace
+ for elem in it:
+ k = key(elem)
+ if top < k:
+ _heapreplace(result, (k, order, elem))
+ top = result[0][0]
+ order -= 1
+ result.sort(reverse=True)
+ return [r[2] for r in result]
+
+# If available, use C implementation
+try:
+ from _heapq import *
+except ImportError:
+ pass
+try:
+ from _heapq import _heapreplace_max
+except ImportError:
+ pass
+try:
+ from _heapq import _heapify_max
+except ImportError:
+ pass
+try:
+ from _heapq import _heappop_max
+except ImportError:
+ pass
+
if __name__ == "__main__":
- # Simple sanity test
- heap = []
- data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0]
- for item in data:
- heappush(heap, item)
- sort = []
- while heap:
- sort.append(heappop(heap))
- print(sort)
import doctest
- doctest.testmod()
+ print(doctest.testmod())
diff --git a/Lib/html/entities.py b/Lib/html/entities.py
index f7deae6..3e1778b 100644
--- a/Lib/html/entities.py
+++ b/Lib/html/entities.py
@@ -1,5 +1,8 @@
"""HTML character entity references."""
+__all__ = ['html5', 'name2codepoint', 'codepoint2name', 'entitydefs']
+
+
# maps the HTML entity name to the Unicode code point
name2codepoint = {
'AElig': 0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1
diff --git a/Lib/html/parser.py b/Lib/html/parser.py
index 9ae31b9..43e6411 100644
--- a/Lib/html/parser.py
+++ b/Lib/html/parser.py
@@ -29,35 +29,15 @@ starttagopen = re.compile('<[a-zA-Z]')
piclose = re.compile('>')
commentclose = re.compile(r'--\s*>')
# Note:
-# 1) the strict attrfind isn't really strict, but we can't make it
-# correctly strict without breaking backward compatibility;
-# 2) if you change tagfind/attrfind remember to update locatestarttagend too;
-# 3) if you change tagfind/attrfind and/or locatestarttagend the parser will
+# 1) if you change tagfind/attrfind remember to update locatestarttagend too;
+# 2) if you change tagfind/attrfind and/or locatestarttagend the parser will
# explode, so don't do it.
-tagfind = re.compile('([a-zA-Z][-.a-zA-Z0-9:_]*)(?:\s|/(?!>))*')
# see http://www.w3.org/TR/html5/tokenization.html#tag-open-state
# and http://www.w3.org/TR/html5/tokenization.html#tag-name-state
tagfind_tolerant = re.compile('([a-zA-Z][^\t\n\r\f />\x00]*)(?:\s|/(?!>))*')
-attrfind = re.compile(
- r'\s*([a-zA-Z_][-.:a-zA-Z_0-9]*)(\s*=\s*'
- r'(\'[^\']*\'|"[^"]*"|[^\s"\'=<>`]*))?')
attrfind_tolerant = re.compile(
r'((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*'
r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*')
-locatestarttagend = re.compile(r"""
- <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name
- (?:\s+ # whitespace before attribute name
- (?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name
- (?:\s*=\s* # value indicator
- (?:'[^']*' # LITA-enclosed value
- |\"[^\"]*\" # LIT-enclosed value
- |[^'\">\s]+ # bare value
- )
- )?
- )
- )*
- \s* # trailing whitespace
-""", re.VERBOSE)
locatestarttagend_tolerant = re.compile(r"""
<[a-zA-Z][^\t\n\r\f />\x00]* # tag name
(?:[\s/]* # optional whitespace before attribute name
@@ -79,25 +59,6 @@ endendtag = re.compile('>')
endtagfind = re.compile('</\s*([a-zA-Z][-.a-zA-Z0-9:_]*)\s*>')
-class HTMLParseError(Exception):
- """Exception raised for all parse errors."""
-
- def __init__(self, msg, position=(None, None)):
- assert msg
- self.msg = msg
- self.lineno = position[0]
- self.offset = position[1]
-
- def __str__(self):
- result = self.msg
- if self.lineno is not None:
- result = result + ", at line %d" % self.lineno
- if self.offset is not None:
- result = result + ", column %d" % (self.offset + 1)
- return result
-
-
-_default_sentinel = object()
class HTMLParser(_markupbase.ParserBase):
"""Find tags and other markup and call handler functions.
@@ -123,27 +84,12 @@ class HTMLParser(_markupbase.ParserBase):
CDATA_CONTENT_ELEMENTS = ("script", "style")
- def __init__(self, strict=_default_sentinel, *,
- convert_charrefs=_default_sentinel):
+ def __init__(self, *, convert_charrefs=True):
"""Initialize and reset this instance.
- If convert_charrefs is True (default: False), all character references
+ If convert_charrefs is True (the default), all character references
are automatically converted to the corresponding Unicode characters.
- If strict is set to False (the default) the parser will parse invalid
- markup, otherwise it will raise an error. Note that the strict mode
- and argument are deprecated.
"""
- if strict is not _default_sentinel:
- warnings.warn("The strict argument and mode are deprecated.",
- DeprecationWarning, stacklevel=2)
- else:
- strict = False # default
- self.strict = strict
- if convert_charrefs is _default_sentinel:
- convert_charrefs = False # default
- warnings.warn("The value of convert_charrefs will become True in "
- "3.5. You are encouraged to set the value explicitly.",
- DeprecationWarning, stacklevel=2)
self.convert_charrefs = convert_charrefs
self.reset()
@@ -168,11 +114,6 @@ class HTMLParser(_markupbase.ParserBase):
"""Handle any buffered data."""
self.goahead(1)
- def error(self, message):
- warnings.warn("The 'error' method is deprecated.",
- DeprecationWarning, stacklevel=2)
- raise HTMLParseError(message, self.getpos())
-
__starttag_text = None
def get_starttag_text(self):
@@ -235,10 +176,7 @@ class HTMLParser(_markupbase.ParserBase):
elif startswith("<?", i):
k = self.parse_pi(i)
elif startswith("<!", i):
- if self.strict:
- k = self.parse_declaration(i)
- else:
- k = self.parse_html_declaration(i)
+ k = self.parse_html_declaration(i)
elif (i + 1) < n:
self.handle_data("<")
k = i + 1
@@ -247,8 +185,6 @@ class HTMLParser(_markupbase.ParserBase):
if k < 0:
if not end:
break
- if self.strict:
- self.error("EOF in middle of construct")
k = rawdata.find('>', i + 1)
if k < 0:
k = rawdata.find('<', i + 1)
@@ -290,13 +226,10 @@ class HTMLParser(_markupbase.ParserBase):
if match:
# match.group() will contain at least 2 chars
if end and match.group() == rawdata[i:]:
- if self.strict:
- self.error("EOF in middle of entity or char ref")
- else:
- k = match.end()
- if k <= i:
- k = n
- i = self.updatepos(i, i + 1)
+ k = match.end()
+ if k <= i:
+ k = n
+ i = self.updatepos(i, i + 1)
# incomplete
break
elif (i + 1) < n:
@@ -375,18 +308,12 @@ class HTMLParser(_markupbase.ParserBase):
# Now parse the data between i+1 and j into a tag and attrs
attrs = []
- if self.strict:
- match = tagfind.match(rawdata, i+1)
- else:
- match = tagfind_tolerant.match(rawdata, i+1)
+ match = tagfind_tolerant.match(rawdata, i+1)
assert match, 'unexpected call to parse_starttag()'
k = match.end()
self.lasttag = tag = match.group(1).lower()
while k < endpos:
- if self.strict:
- m = attrfind.match(rawdata, k)
- else:
- m = attrfind_tolerant.match(rawdata, k)
+ m = attrfind_tolerant.match(rawdata, k)
if not m:
break
attrname, rest, attrvalue = m.group(1, 2, 3)
@@ -409,9 +336,6 @@ class HTMLParser(_markupbase.ParserBase):
- self.__starttag_text.rfind("\n")
else:
offset = offset + len(self.__starttag_text)
- if self.strict:
- self.error("junk characters in start tag: %r"
- % (rawdata[k:endpos][:20],))
self.handle_data(rawdata[i:endpos])
return endpos
if end.endswith('/>'):
@@ -427,10 +351,7 @@ class HTMLParser(_markupbase.ParserBase):
# or -1 if incomplete.
def check_for_whole_start_tag(self, i):
rawdata = self.rawdata
- if self.strict:
- m = locatestarttagend.match(rawdata, i)
- else:
- m = locatestarttagend_tolerant.match(rawdata, i)
+ m = locatestarttagend_tolerant.match(rawdata, i)
if m:
j = m.end()
next = rawdata[j:j+1]
@@ -443,9 +364,6 @@ class HTMLParser(_markupbase.ParserBase):
# buffer boundary
return -1
# else bogus input
- if self.strict:
- self.updatepos(i, j + 1)
- self.error("malformed empty start tag")
if j > i:
return j
else:
@@ -458,9 +376,6 @@ class HTMLParser(_markupbase.ParserBase):
# end of input in or before attribute value, or we have the
# '/' from a '/>' ending
return -1
- if self.strict:
- self.updatepos(i, j)
- self.error("malformed start tag")
if j > i:
return j
else:
@@ -480,8 +395,6 @@ class HTMLParser(_markupbase.ParserBase):
if self.cdata_elem is not None:
self.handle_data(rawdata[i:gtpos])
return gtpos
- if self.strict:
- self.error("bad end tag: %r" % (rawdata[i:gtpos],))
# find the name: w3.org/TR/html5/tokenization.html#tag-name-state
namematch = tagfind_tolerant.match(rawdata, i+2)
if not namematch:
@@ -547,8 +460,7 @@ class HTMLParser(_markupbase.ParserBase):
pass
def unknown_decl(self, data):
- if self.strict:
- self.error("unknown declaration: %r" % (data,))
+ pass
# Internal -- helper to remove special character quoting
def unescape(self, s):
diff --git a/Lib/http/__init__.py b/Lib/http/__init__.py
index 196d378..d4334cc 100644
--- a/Lib/http/__init__.py
+++ b/Lib/http/__init__.py
@@ -1 +1,134 @@
-# This directory is a Python package.
+from enum import IntEnum
+
+__all__ = ['HTTPStatus']
+
+class HTTPStatus(IntEnum):
+ """HTTP status codes and reason phrases
+
+ Status codes from the following RFCs are all observed:
+
+ * RFC 7231: Hypertext Transfer Protocol (HTTP/1.1), obsoletes 2616
+ * RFC 6585: Additional HTTP Status Codes
+ * RFC 3229: Delta encoding in HTTP
+ * RFC 4918: HTTP Extensions for WebDAV, obsoletes 2518
+ * RFC 5842: Binding Extensions to WebDAV
+ * RFC 7238: Permanent Redirect
+ * RFC 2295: Transparent Content Negotiation in HTTP
+ * RFC 2774: An HTTP Extension Framework
+ """
+ def __new__(cls, value, phrase, description=''):
+ obj = int.__new__(cls, value)
+ obj._value_ = value
+
+ obj.phrase = phrase
+ obj.description = description
+ return obj
+
+ # informational
+ CONTINUE = 100, 'Continue', 'Request received, please continue'
+ SWITCHING_PROTOCOLS = (101, 'Switching Protocols',
+ 'Switching to new protocol; obey Upgrade header')
+ PROCESSING = 102, 'Processing'
+
+ # success
+ OK = 200, 'OK', 'Request fulfilled, document follows'
+ CREATED = 201, 'Created', 'Document created, URL follows'
+ ACCEPTED = (202, 'Accepted',
+ 'Request accepted, processing continues off-line')
+ NON_AUTHORITATIVE_INFORMATION = (203,
+ 'Non-Authoritative Information', 'Request fulfilled from cache')
+ NO_CONTENT = 204, 'No Content', 'Request fulfilled, nothing follows'
+ RESET_CONTENT = 205, 'Reset Content', 'Clear input form for further input'
+ PARTIAL_CONTENT = 206, 'Partial Content', 'Partial content follows'
+ MULTI_STATUS = 207, 'Multi-Status'
+ ALREADY_REPORTED = 208, 'Already Reported'
+ IM_USED = 226, 'IM Used'
+
+ # redirection
+ MULTIPLE_CHOICES = (300, 'Multiple Choices',
+ 'Object has several resources -- see URI list')
+ MOVED_PERMANENTLY = (301, 'Moved Permanently',
+ 'Object moved permanently -- see URI list')
+ FOUND = 302, 'Found', 'Object moved temporarily -- see URI list'
+ SEE_OTHER = 303, 'See Other', 'Object moved -- see Method and URL list'
+ NOT_MODIFIED = (304, 'Not Modified',
+ 'Document has not changed since given time')
+ USE_PROXY = (305, 'Use Proxy',
+ 'You must use proxy specified in Location to access this resource')
+ TEMPORARY_REDIRECT = (307, 'Temporary Redirect',
+ 'Object moved temporarily -- see URI list')
+ PERMANENT_REDIRECT = (308, 'Permanent Redirect',
+ 'Object moved temporarily -- see URI list')
+
+ # client error
+ BAD_REQUEST = (400, 'Bad Request',
+ 'Bad request syntax or unsupported method')
+ UNAUTHORIZED = (401, 'Unauthorized',
+ 'No permission -- see authorization schemes')
+ PAYMENT_REQUIRED = (402, 'Payment Required',
+ 'No payment -- see charging schemes')
+ FORBIDDEN = (403, 'Forbidden',
+ 'Request forbidden -- authorization will not help')
+ NOT_FOUND = (404, 'Not Found',
+ 'Nothing matches the given URI')
+ METHOD_NOT_ALLOWED = (405, 'Method Not Allowed',
+ 'Specified method is invalid for this resource')
+ NOT_ACCEPTABLE = (406, 'Not Acceptable',
+ 'URI not available in preferred format')
+ PROXY_AUTHENTICATION_REQUIRED = (407,
+ 'Proxy Authentication Required',
+ 'You must authenticate with this proxy before proceeding')
+ REQUEST_TIMEOUT = (408, 'Request Timeout',
+ 'Request timed out; try again later')
+ CONFLICT = 409, 'Conflict', 'Request conflict'
+ GONE = (410, 'Gone',
+ 'URI no longer exists and has been permanently removed')
+ LENGTH_REQUIRED = (411, 'Length Required',
+ 'Client must specify Content-Length')
+ PRECONDITION_FAILED = (412, 'Precondition Failed',
+ 'Precondition in headers is false')
+ REQUEST_ENTITY_TOO_LARGE = (413, 'Request Entity Too Large',
+ 'Entity is too large')
+ REQUEST_URI_TOO_LONG = (414, 'Request-URI Too Long',
+ 'URI is too long')
+ UNSUPPORTED_MEDIA_TYPE = (415, 'Unsupported Media Type',
+ 'Entity body in unsupported format')
+ REQUESTED_RANGE_NOT_SATISFIABLE = (416,
+ 'Requested Range Not Satisfiable',
+ 'Cannot satisfy request range')
+ EXPECTATION_FAILED = (417, 'Expectation Failed',
+ 'Expect condition could not be satisfied')
+ UNPROCESSABLE_ENTITY = 422, 'Unprocessable Entity'
+ LOCKED = 423, 'Locked'
+ FAILED_DEPENDENCY = 424, 'Failed Dependency'
+ UPGRADE_REQUIRED = 426, 'Upgrade Required'
+ PRECONDITION_REQUIRED = (428, 'Precondition Required',
+ 'The origin server requires the request to be conditional')
+ TOO_MANY_REQUESTS = (429, 'Too Many Requests',
+ 'The user has sent too many requests in '
+ 'a given amount of time ("rate limiting")')
+ REQUEST_HEADER_FIELDS_TOO_LARGE = (431,
+ 'Request Header Fields Too Large',
+ 'The server is unwilling to process the request because its header '
+ 'fields are too large')
+
+ # server errors
+ INTERNAL_SERVER_ERROR = (500, 'Internal Server Error',
+ 'Server got itself in trouble')
+ NOT_IMPLEMENTED = (501, 'Not Implemented',
+ 'Server does not support this operation')
+ BAD_GATEWAY = (502, 'Bad Gateway',
+ 'Invalid responses from another server/proxy')
+ SERVICE_UNAVAILABLE = (503, 'Service Unavailable',
+ 'The server cannot process the request due to a high load')
+ GATEWAY_TIMEOUT = (504, 'Gateway Timeout',
+ 'The gateway server did not receive a timely response')
+ HTTP_VERSION_NOT_SUPPORTED = (505, 'HTTP Version Not Supported',
+ 'Cannot fulfill request')
+ VARIANT_ALSO_NEGOTIATES = 506, 'Variant Also Negotiates'
+ INSUFFICIENT_STORAGE = 507, 'Insufficient Storage'
+ LOOP_DETECTED = 508, 'Loop Detected'
+ NOT_EXTENDED = 510, 'Not Extended'
+ NETWORK_AUTHENTICATION_REQUIRED = (511,
+ 'Network Authentication Required',
+ 'The client needs to authenticate to gain network access')
diff --git a/Lib/http/client.py b/Lib/http/client.py
index 1c69dcb..80c80cf 100644
--- a/Lib/http/client.py
+++ b/Lib/http/client.py
@@ -20,10 +20,12 @@ request. This diagram details these state transitions:
| ( putheader() )* endheaders()
v
Request-sent
- |
- | response = getresponse()
- v
- Unread-response [Response-headers-read]
+ |\_____________________________
+ | | getresponse() raises
+ | response = getresponse() | ConnectionError
+ v v
+ Unread-response Idle
+ [Response-headers-read]
|\____________________
| |
| response.read() | putrequest()
@@ -68,6 +70,7 @@ Req-sent-unread-response _CS_REQ_SENT <response_class>
import email.parser
import email.message
+import http
import io
import os
import re
@@ -82,7 +85,8 @@ __all__ = ["HTTPResponse", "HTTPConnection",
"UnknownTransferEncoding", "UnimplementedFileMode",
"IncompleteRead", "InvalidURL", "ImproperConnectionState",
"CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
- "BadStatusLine", "LineTooLong", "error", "responses"]
+ "BadStatusLine", "LineTooLong", "RemoteDisconnected", "error",
+ "responses"]
HTTP_PORT = 80
HTTPS_PORT = 443
@@ -94,122 +98,13 @@ _CS_IDLE = 'Idle'
_CS_REQ_STARTED = 'Request-started'
_CS_REQ_SENT = 'Request-sent'
-# status codes
-# informational
-CONTINUE = 100
-SWITCHING_PROTOCOLS = 101
-PROCESSING = 102
-
-# successful
-OK = 200
-CREATED = 201
-ACCEPTED = 202
-NON_AUTHORITATIVE_INFORMATION = 203
-NO_CONTENT = 204
-RESET_CONTENT = 205
-PARTIAL_CONTENT = 206
-MULTI_STATUS = 207
-IM_USED = 226
-
-# redirection
-MULTIPLE_CHOICES = 300
-MOVED_PERMANENTLY = 301
-FOUND = 302
-SEE_OTHER = 303
-NOT_MODIFIED = 304
-USE_PROXY = 305
-TEMPORARY_REDIRECT = 307
-
-# client error
-BAD_REQUEST = 400
-UNAUTHORIZED = 401
-PAYMENT_REQUIRED = 402
-FORBIDDEN = 403
-NOT_FOUND = 404
-METHOD_NOT_ALLOWED = 405
-NOT_ACCEPTABLE = 406
-PROXY_AUTHENTICATION_REQUIRED = 407
-REQUEST_TIMEOUT = 408
-CONFLICT = 409
-GONE = 410
-LENGTH_REQUIRED = 411
-PRECONDITION_FAILED = 412
-REQUEST_ENTITY_TOO_LARGE = 413
-REQUEST_URI_TOO_LONG = 414
-UNSUPPORTED_MEDIA_TYPE = 415
-REQUESTED_RANGE_NOT_SATISFIABLE = 416
-EXPECTATION_FAILED = 417
-UNPROCESSABLE_ENTITY = 422
-LOCKED = 423
-FAILED_DEPENDENCY = 424
-UPGRADE_REQUIRED = 426
-PRECONDITION_REQUIRED = 428
-TOO_MANY_REQUESTS = 429
-REQUEST_HEADER_FIELDS_TOO_LARGE = 431
-
-# server error
-INTERNAL_SERVER_ERROR = 500
-NOT_IMPLEMENTED = 501
-BAD_GATEWAY = 502
-SERVICE_UNAVAILABLE = 503
-GATEWAY_TIMEOUT = 504
-HTTP_VERSION_NOT_SUPPORTED = 505
-INSUFFICIENT_STORAGE = 507
-NOT_EXTENDED = 510
-NETWORK_AUTHENTICATION_REQUIRED = 511
+# hack to maintain backwards compatibility
+globals().update(http.HTTPStatus.__members__)
+
+# another hack to maintain backwards compatibility
# Mapping status codes to official W3C names
-responses = {
- 100: 'Continue',
- 101: 'Switching Protocols',
-
- 200: 'OK',
- 201: 'Created',
- 202: 'Accepted',
- 203: 'Non-Authoritative Information',
- 204: 'No Content',
- 205: 'Reset Content',
- 206: 'Partial Content',
-
- 300: 'Multiple Choices',
- 301: 'Moved Permanently',
- 302: 'Found',
- 303: 'See Other',
- 304: 'Not Modified',
- 305: 'Use Proxy',
- 306: '(Unused)',
- 307: 'Temporary Redirect',
-
- 400: 'Bad Request',
- 401: 'Unauthorized',
- 402: 'Payment Required',
- 403: 'Forbidden',
- 404: 'Not Found',
- 405: 'Method Not Allowed',
- 406: 'Not Acceptable',
- 407: 'Proxy Authentication Required',
- 408: 'Request Timeout',
- 409: 'Conflict',
- 410: 'Gone',
- 411: 'Length Required',
- 412: 'Precondition Failed',
- 413: 'Request Entity Too Large',
- 414: 'Request-URI Too Long',
- 415: 'Unsupported Media Type',
- 416: 'Requested Range Not Satisfiable',
- 417: 'Expectation Failed',
- 428: 'Precondition Required',
- 429: 'Too Many Requests',
- 431: 'Request Header Fields Too Large',
-
- 500: 'Internal Server Error',
- 501: 'Not Implemented',
- 502: 'Bad Gateway',
- 503: 'Service Unavailable',
- 504: 'Gateway Timeout',
- 505: 'HTTP Version Not Supported',
- 511: 'Network Authentication Required',
-}
+responses = {v: v.phrase for v in http.HTTPStatus.__members__.values()}
# maximal amount of data to read at one time in _safe_read
MAXAMOUNT = 1048576
@@ -305,7 +200,7 @@ def parse_headers(fp, _class=HTTPMessage):
return email.parser.Parser(_class=_class).parsestr(hstring)
-class HTTPResponse(io.RawIOBase):
+class HTTPResponse(io.BufferedIOBase):
# See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
@@ -353,7 +248,8 @@ class HTTPResponse(io.RawIOBase):
if not line:
# Presumably, the server closed the connection before
# sending a valid response.
- raise BadStatusLine(line)
+ raise RemoteDisconnected("Remote end closed connection without"
+ " response")
try:
version, status, reason = line.split(None, 2)
except ValueError:
@@ -532,9 +428,10 @@ class HTTPResponse(io.RawIOBase):
return b""
if amt is not None:
- # Amount is given, so call base class version
- # (which is implemented in terms of self.readinto)
- return super(HTTPResponse, self).read(amt)
+ # Amount is given, implement using readinto
+ b = bytearray(amt)
+ n = self.readinto(b)
+ return memoryview(b)[:n].tobytes()
else:
# Amount is not given (unbounded read) so we must check self.length
# and self.chunked
@@ -614,71 +511,67 @@ class HTTPResponse(io.RawIOBase):
if line in (b'\r\n', b'\n', b''):
break
+ def _get_chunk_left(self):
+ # return self.chunk_left, reading a new chunk if necessary.
+ # chunk_left == 0: at the end of the current chunk, need to close it
+ # chunk_left == None: No current chunk, should read next.
+ # This function returns non-zero or None if the last chunk has
+ # been read.
+ chunk_left = self.chunk_left
+ if not chunk_left: # Can be 0 or None
+ if chunk_left is not None:
+ # We are at the end of chunk. dicard chunk end
+ self._safe_read(2) # toss the CRLF at the end of the chunk
+ try:
+ chunk_left = self._read_next_chunk_size()
+ except ValueError:
+ raise IncompleteRead(b'')
+ if chunk_left == 0:
+ # last chunk: 1*("0") [ chunk-extension ] CRLF
+ self._read_and_discard_trailer()
+ # we read everything; close the "file"
+ self._close_conn()
+ chunk_left = None
+ self.chunk_left = chunk_left
+ return chunk_left
+
def _readall_chunked(self):
assert self.chunked != _UNKNOWN
- chunk_left = self.chunk_left
value = []
- while True:
- if chunk_left is None:
- try:
- chunk_left = self._read_next_chunk_size()
- if chunk_left == 0:
- break
- except ValueError:
- raise IncompleteRead(b''.join(value))
- value.append(self._safe_read(chunk_left))
-
- # we read the whole chunk, get another
- self._safe_read(2) # toss the CRLF at the end of the chunk
- chunk_left = None
-
- self._read_and_discard_trailer()
-
- # we read everything; close the "file"
- self._close_conn()
-
- return b''.join(value)
+ try:
+ while True:
+ chunk_left = self._get_chunk_left()
+ if chunk_left is None:
+ break
+ value.append(self._safe_read(chunk_left))
+ self.chunk_left = 0
+ return b''.join(value)
+ except IncompleteRead:
+ raise IncompleteRead(b''.join(value))
def _readinto_chunked(self, b):
assert self.chunked != _UNKNOWN
- chunk_left = self.chunk_left
-
total_bytes = 0
mvb = memoryview(b)
- while True:
- if chunk_left is None:
- try:
- chunk_left = self._read_next_chunk_size()
- if chunk_left == 0:
- break
- except ValueError:
- raise IncompleteRead(bytes(b[0:total_bytes]))
-
- if len(mvb) < chunk_left:
- n = self._safe_readinto(mvb)
- self.chunk_left = chunk_left - n
- return total_bytes + n
- elif len(mvb) == chunk_left:
- n = self._safe_readinto(mvb)
- self._safe_read(2) # toss the CRLF at the end of the chunk
- self.chunk_left = None
- return total_bytes + n
- else:
- temp_mvb = mvb[0:chunk_left]
+ try:
+ while True:
+ chunk_left = self._get_chunk_left()
+ if chunk_left is None:
+ return total_bytes
+
+ if len(mvb) <= chunk_left:
+ n = self._safe_readinto(mvb)
+ self.chunk_left = chunk_left - n
+ return total_bytes + n
+
+ temp_mvb = mvb[:chunk_left]
n = self._safe_readinto(temp_mvb)
mvb = mvb[n:]
total_bytes += n
+ self.chunk_left = 0
- # we read the whole chunk, get another
- self._safe_read(2) # toss the CRLF at the end of the chunk
- chunk_left = None
-
- self._read_and_discard_trailer()
-
- # we read everything; close the "file"
- self._close_conn()
-
- return total_bytes
+ except IncompleteRead:
+ raise IncompleteRead(bytes(b[0:total_bytes]))
def _safe_read(self, amt):
"""Read the number of bytes requested, compensating for partial reads.
@@ -719,6 +612,73 @@ class HTTPResponse(io.RawIOBase):
total_bytes += n
return total_bytes
+ def read1(self, n=-1):
+ """Read with at most one underlying system call. If at least one
+ byte is buffered, return that instead.
+ """
+ if self.fp is None or self._method == "HEAD":
+ return b""
+ if self.chunked:
+ return self._read1_chunked(n)
+ try:
+ result = self.fp.read1(n)
+ except ValueError:
+ if n >= 0:
+ raise
+ # some implementations, like BufferedReader, don't support -1
+ # Read an arbitrarily selected largeish chunk.
+ result = self.fp.read1(16*1024)
+ if not result and n:
+ self._close_conn()
+ return result
+
+ def peek(self, n=-1):
+ # Having this enables IOBase.readline() to read more than one
+ # byte at a time
+ if self.fp is None or self._method == "HEAD":
+ return b""
+ if self.chunked:
+ return self._peek_chunked(n)
+ return self.fp.peek(n)
+
+ def readline(self, limit=-1):
+ if self.fp is None or self._method == "HEAD":
+ return b""
+ if self.chunked:
+ # Fallback to IOBase readline which uses peek() and read()
+ return super().readline(limit)
+ result = self.fp.readline(limit)
+ if not result and limit:
+ self._close_conn()
+ return result
+
+ def _read1_chunked(self, n):
+ # Strictly speaking, _get_chunk_left() may cause more than one read,
+ # but that is ok, since that is to satisfy the chunked protocol.
+ chunk_left = self._get_chunk_left()
+ if chunk_left is None or n == 0:
+ return b''
+ if not (0 <= n <= chunk_left):
+ n = chunk_left # if n is negative or larger than chunk_left
+ read = self.fp.read1(n)
+ self.chunk_left -= len(read)
+ if not read:
+ raise IncompleteRead(b"")
+ return read
+
+ def _peek_chunked(self, n):
+ # Strictly speaking, _get_chunk_left() may cause more than one read,
+ # but that is ok, since that is to satisfy the chunked protocol.
+ try:
+ chunk_left = self._get_chunk_left()
+ except IncompleteRead:
+ return b'' # peek doesn't worry about protocol
+ if chunk_left is None:
+ return b'' # eof
+ # peek is allowed to return more than requested. Just request the
+ # entire chunk, and truncate what we get.
+ return self.fp.peek(chunk_left)[:chunk_left]
+
def fileno(self):
return self.fp.fileno()
@@ -762,14 +722,6 @@ class HTTPConnection:
default_port = HTTP_PORT
auto_open = 1
debuglevel = 0
- # TCP Maximum Segment Size (MSS) is determined by the TCP stack on
- # a per-connection basis. There is no simple and efficient
- # platform independent mechanism for determining the MSS, so
- # instead a reasonable estimate is chosen. The getsockopt()
- # interface using the TCP_MAXSEG parameter may be a suitable
- # approach on some operating systems. A value of 16KiB is chosen
- # as a reasonable estimate of the maximum MSS.
- mss = 16384
def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
source_address=None):
@@ -851,7 +803,7 @@ class HTTPConnection:
response = self.response_class(self.sock, method=self._method)
(version, code, message) = response._read_status()
- if code != 200:
+ if code != http.HTTPStatus.OK:
self.close()
raise OSError("Tunnel connection failed: %d %s" % (code,
message.strip()))
@@ -865,10 +817,14 @@ class HTTPConnection:
if line in (b'\r\n', b'\n', b''):
break
+ if self.debuglevel > 0:
+ print('header:', line.decode())
+
def connect(self):
"""Connect to the host and port specified in __init__."""
- self.sock = self._create_connection((self.host,self.port),
- self.timeout, self.source_address)
+ self.sock = self._create_connection(
+ (self.host,self.port), self.timeout, self.source_address)
+ self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
if self._tunnel_host:
self._tunnel()
@@ -951,19 +907,9 @@ class HTTPConnection:
self._buffer.extend((b"", b""))
msg = b"\r\n".join(self._buffer)
del self._buffer[:]
- # If msg and message_body are sent in a single send() call,
- # it will avoid performance problems caused by the interaction
- # between delayed ack and the Nagle algorithm. However,
- # there is no performance gain if the message is larger
- # than MSS (and there is a memory penalty for the message
- # copy).
- if isinstance(message_body, bytes) and len(message_body) < self.mss:
- msg += message_body
- message_body = None
+
self.send(msg)
if message_body is not None:
- # message_body was not a string (i.e. it is a file), and
- # we must run the risk of Nagle.
self.send(message_body)
def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
@@ -1224,7 +1170,11 @@ class HTTPConnection:
response = self.response_class(self.sock, method=self._method)
try:
- response.begin()
+ try:
+ response.begin()
+ except ConnectionError:
+ self.close()
+ raise
assert response.will_close != _UNKNOWN
self.__state = _CS_IDLE
@@ -1327,7 +1277,8 @@ class IncompleteRead(HTTPException):
e = ', %i more expected' % self.expected
else:
e = ''
- return 'IncompleteRead(%i bytes read%s)' % (len(self.partial), e)
+ return '%s(%i bytes read%s)' % (self.__class__.__name__,
+ len(self.partial), e)
def __str__(self):
return repr(self)
@@ -1355,5 +1306,10 @@ class LineTooLong(HTTPException):
HTTPException.__init__(self, "got more than %d bytes when reading %s"
% (_MAXLINE, line_type))
+class RemoteDisconnected(ConnectionResetError, BadStatusLine):
+ def __init__(self, *pos, **kw):
+ BadStatusLine.__init__(self, "")
+ ConnectionResetError.__init__(self, *pos, **kw)
+
# for backwards compatibility
error = HTTPException
diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py
index 92ead53..b1ba72e 100644
--- a/Lib/http/cookiejar.py
+++ b/Lib/http/cookiejar.py
@@ -821,7 +821,7 @@ class Cookie:
args.append("%s=%s" % (name, repr(attr)))
args.append("rest=%s" % repr(self._rest))
args.append("rfc2109=%s" % repr(self.rfc2109))
- return "Cookie(%s)" % ", ".join(args)
+ return "%s(%s)" % (self.__class__.__name__, ", ".join(args))
class CookiePolicy:
@@ -1999,7 +1999,6 @@ class MozillaCookieJar(FileCookieJar):
magic = f.readline()
if not self.magic_re.search(magic):
- f.close()
raise LoadError(
"%r does not look like a Netscape format cookies file" %
filename)
diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py
index 482e601..fda02b7 100644
--- a/Lib/http/cookies.py
+++ b/Lib/http/cookies.py
@@ -138,6 +138,12 @@ _nulljoin = ''.join
_semispacejoin = '; '.join
_spacejoin = ' '.join
+def _warn_deprecated_setter(setter):
+ import warnings
+ msg = ('The .%s setter is deprecated. The attribute will be read-only in '
+ 'future releases. Please use the set() method instead.' % setter)
+ warnings.warn(msg, DeprecationWarning, stacklevel=3)
+
#
# Define an exception visible to External modules
#
@@ -151,88 +157,36 @@ class CookieError(Exception):
# into a 4 character sequence: a forward-slash followed by the
# three-digit octal equivalent of the character. Any '\' or '"' is
# quoted with a preceeding '\' slash.
+# Because of the way browsers really handle cookies (as opposed to what
+# the RFC says) we also encode "," and ";".
#
# These are taken from RFC2068 and RFC2109.
# _LegalChars is the list of chars which don't require "'s
# _Translator hash-table for fast quoting
#
-_LegalChars = string.ascii_letters + string.digits + "!#$%&'*+-.^_`|~:"
-_Translator = {
- '\000' : '\\000', '\001' : '\\001', '\002' : '\\002',
- '\003' : '\\003', '\004' : '\\004', '\005' : '\\005',
- '\006' : '\\006', '\007' : '\\007', '\010' : '\\010',
- '\011' : '\\011', '\012' : '\\012', '\013' : '\\013',
- '\014' : '\\014', '\015' : '\\015', '\016' : '\\016',
- '\017' : '\\017', '\020' : '\\020', '\021' : '\\021',
- '\022' : '\\022', '\023' : '\\023', '\024' : '\\024',
- '\025' : '\\025', '\026' : '\\026', '\027' : '\\027',
- '\030' : '\\030', '\031' : '\\031', '\032' : '\\032',
- '\033' : '\\033', '\034' : '\\034', '\035' : '\\035',
- '\036' : '\\036', '\037' : '\\037',
-
- # Because of the way browsers really handle cookies (as opposed
- # to what the RFC says) we also encode , and ;
-
- ',' : '\\054', ';' : '\\073',
-
- '"' : '\\"', '\\' : '\\\\',
-
- '\177' : '\\177', '\200' : '\\200', '\201' : '\\201',
- '\202' : '\\202', '\203' : '\\203', '\204' : '\\204',
- '\205' : '\\205', '\206' : '\\206', '\207' : '\\207',
- '\210' : '\\210', '\211' : '\\211', '\212' : '\\212',
- '\213' : '\\213', '\214' : '\\214', '\215' : '\\215',
- '\216' : '\\216', '\217' : '\\217', '\220' : '\\220',
- '\221' : '\\221', '\222' : '\\222', '\223' : '\\223',
- '\224' : '\\224', '\225' : '\\225', '\226' : '\\226',
- '\227' : '\\227', '\230' : '\\230', '\231' : '\\231',
- '\232' : '\\232', '\233' : '\\233', '\234' : '\\234',
- '\235' : '\\235', '\236' : '\\236', '\237' : '\\237',
- '\240' : '\\240', '\241' : '\\241', '\242' : '\\242',
- '\243' : '\\243', '\244' : '\\244', '\245' : '\\245',
- '\246' : '\\246', '\247' : '\\247', '\250' : '\\250',
- '\251' : '\\251', '\252' : '\\252', '\253' : '\\253',
- '\254' : '\\254', '\255' : '\\255', '\256' : '\\256',
- '\257' : '\\257', '\260' : '\\260', '\261' : '\\261',
- '\262' : '\\262', '\263' : '\\263', '\264' : '\\264',
- '\265' : '\\265', '\266' : '\\266', '\267' : '\\267',
- '\270' : '\\270', '\271' : '\\271', '\272' : '\\272',
- '\273' : '\\273', '\274' : '\\274', '\275' : '\\275',
- '\276' : '\\276', '\277' : '\\277', '\300' : '\\300',
- '\301' : '\\301', '\302' : '\\302', '\303' : '\\303',
- '\304' : '\\304', '\305' : '\\305', '\306' : '\\306',
- '\307' : '\\307', '\310' : '\\310', '\311' : '\\311',
- '\312' : '\\312', '\313' : '\\313', '\314' : '\\314',
- '\315' : '\\315', '\316' : '\\316', '\317' : '\\317',
- '\320' : '\\320', '\321' : '\\321', '\322' : '\\322',
- '\323' : '\\323', '\324' : '\\324', '\325' : '\\325',
- '\326' : '\\326', '\327' : '\\327', '\330' : '\\330',
- '\331' : '\\331', '\332' : '\\332', '\333' : '\\333',
- '\334' : '\\334', '\335' : '\\335', '\336' : '\\336',
- '\337' : '\\337', '\340' : '\\340', '\341' : '\\341',
- '\342' : '\\342', '\343' : '\\343', '\344' : '\\344',
- '\345' : '\\345', '\346' : '\\346', '\347' : '\\347',
- '\350' : '\\350', '\351' : '\\351', '\352' : '\\352',
- '\353' : '\\353', '\354' : '\\354', '\355' : '\\355',
- '\356' : '\\356', '\357' : '\\357', '\360' : '\\360',
- '\361' : '\\361', '\362' : '\\362', '\363' : '\\363',
- '\364' : '\\364', '\365' : '\\365', '\366' : '\\366',
- '\367' : '\\367', '\370' : '\\370', '\371' : '\\371',
- '\372' : '\\372', '\373' : '\\373', '\374' : '\\374',
- '\375' : '\\375', '\376' : '\\376', '\377' : '\\377'
- }
+_LegalChars = string.ascii_letters + string.digits + "!#$%&'*+-.^_`|~:"
+_UnescapedChars = _LegalChars + ' ()/<=>?@[]{}'
+
+_Translator = {n: '\\%03o' % n
+ for n in set(range(256)) - set(map(ord, _UnescapedChars))}
+_Translator.update({
+ ord('"'): '\\"',
+ ord('\\'): '\\\\',
+})
-def _quote(str, LegalChars=_LegalChars):
+_is_legal_key = re.compile('[%s]+' % _LegalChars).fullmatch
+
+def _quote(str):
r"""Quote a string for use in a cookie header.
If the string does not need to be double-quoted, then just return the
string. Otherwise, surround the string in doublequotes and quote
(with a \) special characters.
"""
- if all(c in LegalChars for c in str):
+ if str is None or _is_legal_key(str):
return str
else:
- return '"' + _nulljoin(_Translator.get(s, s) for s in str) + '"'
+ return '"' + str.translate(_Translator) + '"'
_OctalPatt = re.compile(r"\\[0-3][0-7][0-7]")
@@ -241,7 +195,7 @@ _QuotePatt = re.compile(r"[\\].")
def _unquote(str):
# If there aren't any doublequotes,
# then there can't be any special characters. See RFC 2109.
- if len(str) < 2:
+ if str is None or len(str) < 2:
return str
if str[0] != '"' or str[-1] != '"':
return str
@@ -339,33 +293,108 @@ class Morsel(dict):
def __init__(self):
# Set defaults
- self.key = self.value = self.coded_value = None
+ self._key = self._value = self._coded_value = None
# Set default attributes
for key in self._reserved:
dict.__setitem__(self, key, "")
+ @property
+ def key(self):
+ return self._key
+
+ @key.setter
+ def key(self, key):
+ _warn_deprecated_setter('key')
+ self._key = key
+
+ @property
+ def value(self):
+ return self._value
+
+ @value.setter
+ def value(self, value):
+ _warn_deprecated_setter('value')
+ self._value = value
+
+ @property
+ def coded_value(self):
+ return self._coded_value
+
+ @coded_value.setter
+ def coded_value(self, coded_value):
+ _warn_deprecated_setter('coded_value')
+ self._coded_value = coded_value
+
def __setitem__(self, K, V):
K = K.lower()
if not K in self._reserved:
- raise CookieError("Invalid Attribute %s" % K)
+ raise CookieError("Invalid attribute %r" % (K,))
dict.__setitem__(self, K, V)
+ def setdefault(self, key, val=None):
+ key = key.lower()
+ if key not in self._reserved:
+ raise CookieError("Invalid attribute %r" % (key,))
+ return dict.setdefault(self, key, val)
+
+ def __eq__(self, morsel):
+ if not isinstance(morsel, Morsel):
+ return NotImplemented
+ return (dict.__eq__(self, morsel) and
+ self._value == morsel._value and
+ self._key == morsel._key and
+ self._coded_value == morsel._coded_value)
+
+ __ne__ = object.__ne__
+
+ def copy(self):
+ morsel = Morsel()
+ dict.update(morsel, self)
+ morsel.__dict__.update(self.__dict__)
+ return morsel
+
+ def update(self, values):
+ data = {}
+ for key, val in dict(values).items():
+ key = key.lower()
+ if key not in self._reserved:
+ raise CookieError("Invalid attribute %r" % (key,))
+ data[key] = val
+ dict.update(self, data)
+
def isReservedKey(self, K):
return K.lower() in self._reserved
def set(self, key, val, coded_val, LegalChars=_LegalChars):
- # First we verify that the key isn't a reserved word
- # Second we make sure it only contains legal characters
+ if LegalChars != _LegalChars:
+ import warnings
+ warnings.warn(
+ 'LegalChars parameter is deprecated, ignored and will '
+ 'be removed in future versions.', DeprecationWarning,
+ stacklevel=2)
+
if key.lower() in self._reserved:
- raise CookieError("Attempt to set a reserved key: %s" % key)
- if any(c not in LegalChars for c in key):
- raise CookieError("Illegal key value: %s" % key)
+ raise CookieError('Attempt to set a reserved key %r' % (key,))
+ if not _is_legal_key(key):
+ raise CookieError('Illegal key %r' % (key,))
# It's a good key, so save it.
- self.key = key
- self.value = val
- self.coded_value = coded_val
+ self._key = key
+ self._value = val
+ self._coded_value = coded_val
+
+ def __getstate__(self):
+ return {
+ 'key': self._key,
+ 'value': self._value,
+ 'coded_value': self._coded_value,
+ }
+
+ def __setstate__(self, state):
+ self._key = state['key']
+ self._value = state['value']
+ self._coded_value = state['coded_value']
def output(self, attrs=None, header="Set-Cookie:"):
return "%s %s" % (header, self.OutputString(attrs))
@@ -373,8 +402,7 @@ class Morsel(dict):
__str__ = output
def __repr__(self):
- return '<%s: %s=%s>' % (self.__class__.__name__,
- self.key, repr(self.value))
+ return '<%s: %s>' % (self.__class__.__name__, self.OutputString())
def js_output(self, attrs=None):
# Print javascript
@@ -408,10 +436,9 @@ class Morsel(dict):
append("%s=%s" % (self._reserved[key], _getdate(value)))
elif key == "max-age" and isinstance(value, int):
append("%s=%d" % (self._reserved[key], value))
- elif key == "secure":
- append(str(self._reserved[key]))
- elif key == "httponly":
- append(str(self._reserved[key]))
+ elif key in self._flags:
+ if value:
+ append(str(self._reserved[key]))
else:
append("%s=%s" % (self._reserved[key], value))
@@ -534,10 +561,17 @@ class BaseCookie(dict):
return
def __parse_string(self, str, patt=_CookiePattern):
- i = 0 # Our starting point
- n = len(str) # Length of string
- M = None # current morsel
+ i = 0 # Our starting point
+ n = len(str) # Length of string
+ parsed_items = [] # Parsed (type, key, value) triples
+ morsel_seen = False # A key=value pair was previously encountered
+
+ TYPE_ATTRIBUTE = 1
+ TYPE_KEYVALUE = 2
+ # We first parse the whole cookie string and reject it if it's
+ # syntactically invalid (this helps avoid some classes of injection
+ # attacks).
while 0 <= i < n:
# Start looking for a cookie
match = patt.match(str, i)
@@ -548,22 +582,41 @@ class BaseCookie(dict):
key, value = match.group("key"), match.group("val")
i = match.end(0)
- # Parse the key, value in case it's metainfo
if key[0] == "$":
- # We ignore attributes which pertain to the cookie
- # mechanism as a whole. See RFC 2109.
- # (Does anyone care?)
- if M:
- M[key[1:]] = value
+ if not morsel_seen:
+ # We ignore attributes which pertain to the cookie
+ # mechanism as a whole, such as "$Version".
+ # See RFC 2965. (Does anyone care?)
+ continue
+ parsed_items.append((TYPE_ATTRIBUTE, key[1:], value))
elif key.lower() in Morsel._reserved:
- if M:
- if value is None:
- if key.lower() in Morsel._flags:
- M[key] = True
+ if not morsel_seen:
+ # Invalid cookie string
+ return
+ if value is None:
+ if key.lower() in Morsel._flags:
+ parsed_items.append((TYPE_ATTRIBUTE, key, True))
else:
- M[key] = _unquote(value)
+ # Invalid cookie string
+ return
+ else:
+ parsed_items.append((TYPE_ATTRIBUTE, key, _unquote(value)))
elif value is not None:
- rval, cval = self.value_decode(value)
+ parsed_items.append((TYPE_KEYVALUE, key, self.value_decode(value)))
+ morsel_seen = True
+ else:
+ # Invalid cookie string
+ return
+
+ # The cookie string is valid, apply it.
+ M = None # current morsel
+ for tp, key, value in parsed_items:
+ if tp == TYPE_ATTRIBUTE:
+ assert M is not None
+ M[key] = value
+ else:
+ assert tp == TYPE_KEYVALUE
+ rval, cval = value
self.__set(key, rval, cval)
M = self[key]
diff --git a/Lib/http/server.py b/Lib/http/server.py
index ce0f6cf..6a0e6fe 100644
--- a/Lib/http/server.py
+++ b/Lib/http/server.py
@@ -103,6 +103,8 @@ import urllib.parse
import copy
import argparse
+from http import HTTPStatus
+
# Default error message template
DEFAULT_ERROR_MESSAGE = """\
@@ -281,7 +283,9 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
if len(words) == 3:
command, path, version = words
if version[:5] != 'HTTP/':
- self.send_error(400, "Bad request version (%r)" % version)
+ self.send_error(
+ HTTPStatus.BAD_REQUEST,
+ "Bad request version (%r)" % version)
return False
try:
base_version_number = version.split('/', 1)[1]
@@ -296,25 +300,31 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
raise ValueError
version_number = int(version_number[0]), int(version_number[1])
except (ValueError, IndexError):
- self.send_error(400, "Bad request version (%r)" % version)
+ self.send_error(
+ HTTPStatus.BAD_REQUEST,
+ "Bad request version (%r)" % version)
return False
if version_number >= (1, 1) and self.protocol_version >= "HTTP/1.1":
self.close_connection = False
if version_number >= (2, 0):
- self.send_error(505,
- "Invalid HTTP Version (%s)" % base_version_number)
+ self.send_error(
+ HTTPStatus.HTTP_VERSION_NOT_SUPPORTED,
+ "Invalid HTTP Version (%s)" % base_version_number)
return False
elif len(words) == 2:
command, path = words
self.close_connection = True
if command != 'GET':
- self.send_error(400,
- "Bad HTTP/0.9 request type (%r)" % command)
+ self.send_error(
+ HTTPStatus.BAD_REQUEST,
+ "Bad HTTP/0.9 request type (%r)" % command)
return False
elif not words:
return False
else:
- self.send_error(400, "Bad request syntax (%r)" % requestline)
+ self.send_error(
+ HTTPStatus.BAD_REQUEST,
+ "Bad request syntax (%r)" % requestline)
return False
self.command, self.path, self.request_version = command, path, version
@@ -323,7 +333,9 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
self.headers = http.client.parse_headers(self.rfile,
_class=self.MessageClass)
except http.client.LineTooLong:
- self.send_error(400, "Line too long")
+ self.send_error(
+ HTTPStatus.BAD_REQUEST,
+ "Line too long")
return False
conntype = self.headers.get('Connection', "")
@@ -355,7 +367,7 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
False.
"""
- self.send_response_only(100)
+ self.send_response_only(HTTPStatus.CONTINUE)
self.end_headers()
return True
@@ -373,7 +385,7 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
self.requestline = ''
self.request_version = ''
self.command = ''
- self.send_error(414)
+ self.send_error(HTTPStatus.REQUEST_URI_TOO_LONG)
return
if not self.raw_requestline:
self.close_connection = True
@@ -383,7 +395,9 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
return
mname = 'do_' + self.command
if not hasattr(self, mname):
- self.send_error(501, "Unsupported method (%r)" % self.command)
+ self.send_error(
+ HTTPStatus.NOT_IMPLEMENTED,
+ "Unsupported method (%r)" % self.command)
return
method = getattr(self, mname)
method()
@@ -438,7 +452,11 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
self.send_header('Connection', 'close')
self.send_header('Content-Length', int(len(body)))
self.end_headers()
- if self.command != 'HEAD' and code >= 200 and code not in (204, 304):
+
+ if (self.command != 'HEAD' and
+ code >= 200 and
+ code not in (
+ HTTPStatus.NO_CONTENT, HTTPStatus.NOT_MODIFIED)):
self.wfile.write(body)
def send_response(self, code, message=None):
@@ -499,7 +517,8 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
This is called by send_response().
"""
-
+ if isinstance(code, HTTPStatus):
+ code = code.value
self.log_message('"%s" %s %s',
self.requestline, str(code), str(size))
@@ -582,82 +601,11 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
# MessageClass used to parse headers
MessageClass = http.client.HTTPMessage
- # Table mapping response codes to messages; entries have the
- # form {code: (shortmessage, longmessage)}.
- # See RFC 2616 and 6585.
+ # hack to maintain backwards compatibility
responses = {
- 100: ('Continue', 'Request received, please continue'),
- 101: ('Switching Protocols',
- 'Switching to new protocol; obey Upgrade header'),
-
- 200: ('OK', 'Request fulfilled, document follows'),
- 201: ('Created', 'Document created, URL follows'),
- 202: ('Accepted',
- 'Request accepted, processing continues off-line'),
- 203: ('Non-Authoritative Information', 'Request fulfilled from cache'),
- 204: ('No Content', 'Request fulfilled, nothing follows'),
- 205: ('Reset Content', 'Clear input form for further input.'),
- 206: ('Partial Content', 'Partial content follows.'),
-
- 300: ('Multiple Choices',
- 'Object has several resources -- see URI list'),
- 301: ('Moved Permanently', 'Object moved permanently -- see URI list'),
- 302: ('Found', 'Object moved temporarily -- see URI list'),
- 303: ('See Other', 'Object moved -- see Method and URL list'),
- 304: ('Not Modified',
- 'Document has not changed since given time'),
- 305: ('Use Proxy',
- 'You must use proxy specified in Location to access this '
- 'resource.'),
- 307: ('Temporary Redirect',
- 'Object moved temporarily -- see URI list'),
-
- 400: ('Bad Request',
- 'Bad request syntax or unsupported method'),
- 401: ('Unauthorized',
- 'No permission -- see authorization schemes'),
- 402: ('Payment Required',
- 'No payment -- see charging schemes'),
- 403: ('Forbidden',
- 'Request forbidden -- authorization will not help'),
- 404: ('Not Found', 'Nothing matches the given URI'),
- 405: ('Method Not Allowed',
- 'Specified method is invalid for this resource.'),
- 406: ('Not Acceptable', 'URI not available in preferred format.'),
- 407: ('Proxy Authentication Required', 'You must authenticate with '
- 'this proxy before proceeding.'),
- 408: ('Request Timeout', 'Request timed out; try again later.'),
- 409: ('Conflict', 'Request conflict.'),
- 410: ('Gone',
- 'URI no longer exists and has been permanently removed.'),
- 411: ('Length Required', 'Client must specify Content-Length.'),
- 412: ('Precondition Failed', 'Precondition in headers is false.'),
- 413: ('Request Entity Too Large', 'Entity is too large.'),
- 414: ('Request-URI Too Long', 'URI is too long.'),
- 415: ('Unsupported Media Type', 'Entity body in unsupported format.'),
- 416: ('Requested Range Not Satisfiable',
- 'Cannot satisfy request range.'),
- 417: ('Expectation Failed',
- 'Expect condition could not be satisfied.'),
- 428: ('Precondition Required',
- 'The origin server requires the request to be conditional.'),
- 429: ('Too Many Requests', 'The user has sent too many requests '
- 'in a given amount of time ("rate limiting").'),
- 431: ('Request Header Fields Too Large', 'The server is unwilling to '
- 'process the request because its header fields are too large.'),
-
- 500: ('Internal Server Error', 'Server got itself in trouble'),
- 501: ('Not Implemented',
- 'Server does not support this operation'),
- 502: ('Bad Gateway', 'Invalid responses from another server/proxy.'),
- 503: ('Service Unavailable',
- 'The server cannot process the request due to a high load'),
- 504: ('Gateway Timeout',
- 'The gateway server did not receive a timely response'),
- 505: ('HTTP Version Not Supported', 'Cannot fulfill request.'),
- 511: ('Network Authentication Required',
- 'The client needs to authenticate to gain network access.'),
- }
+ v: (v.phrase, v.description)
+ for v in HTTPStatus.__members__.values()
+ }
class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
@@ -707,7 +655,7 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
parts = urllib.parse.urlsplit(self.path)
if not parts.path.endswith('/'):
# redirect browser - doing basically what apache does
- self.send_response(301)
+ self.send_response(HTTPStatus.MOVED_PERMANENTLY)
new_parts = (parts[0], parts[1], parts[2] + '/',
parts[3], parts[4])
new_url = urllib.parse.urlunsplit(new_parts)
@@ -725,10 +673,10 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
try:
f = open(path, 'rb')
except OSError:
- self.send_error(404, "File not found")
+ self.send_error(HTTPStatus.NOT_FOUND, "File not found")
return None
try:
- self.send_response(200)
+ self.send_response(HTTPStatus.OK)
self.send_header("Content-type", ctype)
fs = os.fstat(f.fileno())
self.send_header("Content-Length", str(fs[6]))
@@ -750,7 +698,9 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
try:
list = os.listdir(path)
except OSError:
- self.send_error(404, "No permission to list directory")
+ self.send_error(
+ HTTPStatus.NOT_FOUND,
+ "No permission to list directory")
return None
list.sort(key=lambda a: a.lower())
r = []
@@ -789,7 +739,7 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
f = io.BytesIO()
f.write(encoded)
f.seek(0)
- self.send_response(200)
+ self.send_response(HTTPStatus.OK)
self.send_header("Content-type", "text/html; charset=%s" % enc)
self.send_header("Content-Length", str(len(encoded)))
self.end_headers()
@@ -971,7 +921,9 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler):
if self.is_cgi():
self.run_cgi()
else:
- self.send_error(501, "Can only POST to CGI scripts")
+ self.send_error(
+ HTTPStatus.NOT_IMPLEMENTED,
+ "Can only POST to CGI scripts")
def send_head(self):
"""Version of send_head that support CGI scripts"""
@@ -1049,17 +1001,21 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler):
scriptname = dir + '/' + script
scriptfile = self.translate_path(scriptname)
if not os.path.exists(scriptfile):
- self.send_error(404, "No such CGI script (%r)" % scriptname)
+ self.send_error(
+ HTTPStatus.NOT_FOUND,
+ "No such CGI script (%r)" % scriptname)
return
if not os.path.isfile(scriptfile):
- self.send_error(403, "CGI script is not a plain file (%r)" %
- scriptname)
+ self.send_error(
+ HTTPStatus.FORBIDDEN,
+ "CGI script is not a plain file (%r)" % scriptname)
return
ispy = self.is_python(scriptname)
if self.have_fork or not ispy:
if not self.is_executable(scriptfile):
- self.send_error(403, "CGI script is not executable (%r)" %
- scriptname)
+ self.send_error(
+ HTTPStatus.FORBIDDEN,
+ "CGI script is not executable (%r)" % scriptname)
return
# Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
@@ -1127,7 +1083,7 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler):
'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
env.setdefault(k, "")
- self.send_response(200, "Script output follows")
+ self.send_response(HTTPStatus.OK, "Script output follows")
self.flush_headers()
decoded_query = query.replace('+', ' ')
diff --git a/Lib/idlelib/ChangeLog b/Lib/idlelib/ChangeLog
index 985871b..90e02f6 100644
--- a/Lib/idlelib/ChangeLog
+++ b/Lib/idlelib/ChangeLog
@@ -20,7 +20,7 @@ IDLEfork ChangeLog
2001-07-19 14:49 elguavas
* ChangeLog, EditorWindow.py, INSTALLATION, NEWS.txt, README.txt,
- TODO.txt, idlever.py:
+ TODO.txt, idlever.py:
minor tidy-ups ready for 0.8.1 alpha tarball release
2001-07-17 15:12 kbk
@@ -172,7 +172,7 @@ IDLEfork ChangeLog
all this work w/ a future-stmt just looks harder and harder."
--tim_one
- (From Rel 1.8: "Hack to make this still work with Python 1.5.2.
+ (From Rel 1.8: "Hack to make this still work with Python 1.5.2.
;-( " --fdrake)
2001-07-14 14:51 kbk
@@ -193,7 +193,7 @@ IDLEfork ChangeLog
test() to _test()." --GvR
This was an interesting merge. The join completely missed removing
- goodname(), which was adjacent, but outside of, a small conflict.
+ goodname(), which was adjacent, but outside of, a small conflict.
I only caught it by comparing the 1.1.3.2/1.1.3.3 diff. CVS ain't
infallible.
@@ -516,12 +516,12 @@ IDLEfork ChangeLog
2000-08-15 22:51 nowonder
- * IDLEFORK.html:
+ * IDLEFORK.html:
corrected email address
2000-08-15 22:47 nowonder
- * IDLEFORK.html:
+ * IDLEFORK.html:
added .html file for http://idlefork.sourceforge.net
2000-08-15 11:13 dscherer
diff --git a/Lib/idlelib/CodeContext.py b/Lib/idlelib/CodeContext.py
index 44783b6..7d25ada 100644
--- a/Lib/idlelib/CodeContext.py
+++ b/Lib/idlelib/CodeContext.py
@@ -57,18 +57,18 @@ class CodeContext:
# Calculate the border width and horizontal padding required to
# align the context with the text in the main Text widget.
#
- # All values are passed through int(str(<value>)), since some
+ # All values are passed through getint(), since some
# values may be pixel objects, which can't simply be added to ints.
widgets = self.editwin.text, self.editwin.text_frame
# Calculate the required vertical padding
padx = 0
for widget in widgets:
- padx += int(str( widget.pack_info()['padx'] ))
- padx += int(str( widget.cget('padx') ))
+ padx += widget.tk.getint(widget.pack_info()['padx'])
+ padx += widget.tk.getint(widget.cget('padx'))
# Calculate the required border width
border = 0
for widget in widgets:
- border += int(str( widget.cget('border') ))
+ border += widget.tk.getint(widget.cget('border'))
self.label = tkinter.Label(self.editwin.top,
text="\n" * (self.context_depth - 1),
anchor=W, justify=LEFT,
diff --git a/Lib/idlelib/HISTORY.txt b/Lib/idlelib/HISTORY.txt
index 01d73ed..731fabd 100644
--- a/Lib/idlelib/HISTORY.txt
+++ b/Lib/idlelib/HISTORY.txt
@@ -11,7 +11,7 @@ What's New in IDLEfork 0.8.1?
*Release date: 22-Jul-2001*
- New tarball released as a result of the 'revitalisation' of the IDLEfork
- project.
+ project.
- This release requires python 2.1 or better. Compatibility with earlier
versions of python (especially ancient ones like 1.5x) is no longer a
@@ -26,8 +26,8 @@ What's New in IDLEfork 0.8.1?
not working, but I believe this was the case with the previous IDLE fork
release (0.7.1) as well.
-- This release is being made now to mark the point at which IDLEfork is
- launching into a new stage of development.
+- This release is being made now to mark the point at which IDLEfork is
+ launching into a new stage of development.
- IDLEfork CVS will now be branched to enable further development and
exploration of the two "execution in a remote process" patches submitted by
@@ -96,7 +96,7 @@ IDLEfork 0.7.1 - 29 May 2000
instead of the IDLE help; shift-TAB is now a synonym for unindent.
- New modules:
-
+
ExecBinding.py Executes program through loader
loader.py Bootstraps user program
protocol.py RPC protocol
diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index 262ee06..be063c4 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -1,6 +1,6 @@
-What's New in Idle 3.4.4?
+What's New in IDLE 3.5.0?
=========================
-*Release date: 2015-??-??*
+*Release date: 2015-09-13* ??
- Issue #23672: Allow Idle to edit and run files with astral chars in name.
Patch by Mohd Sanad Zaki Rizvi.
@@ -18,11 +18,6 @@ What's New in Idle 3.4.4?
- Issue #23184: remove unused names and imports in idlelib.
Initial patch by Al Sweigart.
-
-What's New in Idle 3.4.3?
-=========================
-*Release date: 2015-02-25*
-
- Issue #20577: Configuration of the max line length for the FormatParagraph
extension has been moved from the General tab of the Idle preferences dialog
to the FormatParagraph tab of the Config Extensions dialog.
@@ -47,15 +42,10 @@ What's New in Idle 3.4.3?
- Issue #21986: Code objects are not normally pickled by the pickle module.
To match this, they are no longer pickled when running under Idle.
-
+
- Issue #23180: Rename IDLE "Windows" menu item to "Window".
Patch by Al Sweigart.
-
-What's New in IDLE 3.4.2?
-=========================
-*Release date: 2014-10-06*
-
- Issue #17390: Adjust Editor window title; remove 'Python',
move version to end.
@@ -90,14 +80,8 @@ What's New in IDLE 3.4.2?
- Issue #18409: Add unittest for AutoComplete. Patch by Phil Webster.
-- Issue #18104: Add idlelib/idle_test/htest.py with a few sample tests to begin
- consolidating and improving human-validated tests of Idle. Change other files
- as needed to work with htest. Running the module as __main__ runs all tests.
-
-
-What's New in IDLE 3.4.1?
-=========================
-*Release date: 2014-05-18*
+- Issue #21477: htest.py - Improve framework, complete set of tests.
+ Patches by Saimadhav Heblikar
- Issue #18104: Add idlelib/idle_test/htest.py with a few sample tests to begin
consolidating and improving human-validated tests of Idle. Change other files
@@ -174,7 +158,6 @@ What's New in IDLE 3.3.0?
What's New in IDLE 3.2.1?
=========================
-
*Release date: 15-May-11*
- Issue #6378: Further adjust idle.bat to start associated Python
@@ -192,7 +175,6 @@ What's New in IDLE 3.2.1?
What's New in IDLE 3.1b1?
=========================
-
*Release date: 06-May-09*
- Use of 'filter' in keybindingDialog.py was causing custom key assignment to
@@ -201,7 +183,6 @@ What's New in IDLE 3.1b1?
What's New in IDLE 3.1a1?
=========================
-
*Release date: 07-Mar-09*
- Issue #4815: Offer conversion to UTF-8 if source files have
@@ -219,7 +200,6 @@ What's New in IDLE 3.1a1?
What's New in IDLE 2.7? (UNRELEASED, but merged into 3.1 releases above.)
=======================
-
*Release date: XX-XXX-2010*
- idle.py modified and simplified to better support developing experimental
diff --git a/Lib/idlelib/README.txt b/Lib/idlelib/README.txt
index b2bb73b..7f4a66d 100644
--- a/Lib/idlelib/README.txt
+++ b/Lib/idlelib/README.txt
@@ -14,7 +14,7 @@ code objects from a top level viewpoint without dealing with code folding.
There is a Python Shell window which features colorizing and command recall.
IDLE executes Python code in a separate process, which is restarted for each
-Run (F5) initiated from an editor window. The environment can also be
+Run (F5) initiated from an editor window. The environment can also be
restarted from the Shell window without restarting IDLE.
This enhancement has often been requested, and is now finally available. The
diff --git a/Lib/idlelib/WidgetRedirector.py b/Lib/idlelib/WidgetRedirector.py
index b3d7bfa..67d7f61 100644
--- a/Lib/idlelib/WidgetRedirector.py
+++ b/Lib/idlelib/WidgetRedirector.py
@@ -47,8 +47,9 @@ class WidgetRedirector:
tk.createcommand(w, self.dispatch)
def __repr__(self):
- return "WidgetRedirector(%s<%s>)" % (self.widget.__class__.__name__,
- self.widget._w)
+ return "%s(%s<%s>)" % (self.__class__.__name__,
+ self.widget.__class__.__name__,
+ self.widget._w)
def close(self):
"Unregister operations and revert redirection created by .__init__."
@@ -142,7 +143,8 @@ class OriginalCommand:
self.orig_and_operation = (redir.orig, operation)
def __repr__(self):
- return "OriginalCommand(%r, %r)" % (self.redir, self.operation)
+ return "%s(%r, %r)" % (self.__class__.__name__,
+ self.redir, self.operation)
def __call__(self, *args):
return self.tk_call(self.orig_and_operation + args)
diff --git a/Lib/idlelib/idle_test/test_searchengine.py b/Lib/idlelib/idle_test/test_searchengine.py
index c7792fb..edbd558 100644
--- a/Lib/idlelib/idle_test/test_searchengine.py
+++ b/Lib/idlelib/idle_test/test_searchengine.py
@@ -178,7 +178,7 @@ class SearchEngineTest(unittest.TestCase):
engine.revar.set(1)
Equal(engine.getprog(), None)
self.assertEqual(Mbox.showerror.message,
- 'Error: nothing to repeat\nPattern: +')
+ 'Error: nothing to repeat at position 0\nPattern: +')
def test_report_error(self):
showerror = Mbox.showerror
diff --git a/Lib/idlelib/idlever.py b/Lib/idlelib/idlever.py
index 37b7712..563d933 100644
--- a/Lib/idlelib/idlever.py
+++ b/Lib/idlelib/idlever.py
@@ -2,4 +2,3 @@
Kept only for possible existing extension use."""
from sys import version
IDLE_VERSION = version[:version.index(' ')]
-
diff --git a/Lib/imaplib.py b/Lib/imaplib.py
index eb05dcb..4e8a4bb 100644
--- a/Lib/imaplib.py
+++ b/Lib/imaplib.py
@@ -66,6 +66,7 @@ Commands = {
'CREATE': ('AUTH', 'SELECTED'),
'DELETE': ('AUTH', 'SELECTED'),
'DELETEACL': ('AUTH', 'SELECTED'),
+ 'ENABLE': ('AUTH', ),
'EXAMINE': ('AUTH', 'SELECTED'),
'EXPUNGE': ('SELECTED',),
'FETCH': ('SELECTED',),
@@ -107,12 +108,17 @@ InternalDate = re.compile(br'.*INTERNALDATE "'
br' (?P<hour>[0-9][0-9]):(?P<min>[0-9][0-9]):(?P<sec>[0-9][0-9])'
br' (?P<zonen>[-+])(?P<zoneh>[0-9][0-9])(?P<zonem>[0-9][0-9])'
br'"')
+# Literal is no longer used; kept for backward compatibility.
Literal = re.compile(br'.*{(?P<size>\d+)}$', re.ASCII)
MapCRLF = re.compile(br'\r\n|\r|\n')
Response_code = re.compile(br'\[(?P<type>[A-Z-]+)( (?P<data>[^\]]*))?\]')
Untagged_response = re.compile(br'\* (?P<type>[A-Z-]+)( (?P<data>.*))?')
+# Untagged_status is no longer used; kept for backward compatibility
Untagged_status = re.compile(
br'\* (?P<data>\d+) (?P<type>[A-Z-]+)( (?P<data2>.*))?', re.ASCII)
+# We compile these in _mode_xxx.
+_Literal = br'.*{(?P<size>\d+)}$'
+_Untagged_status = br'\* (?P<data>\d+) (?P<type>[A-Z-]+)( (?P<data2>.*))?'
@@ -166,7 +172,7 @@ class IMAP4:
class abort(error): pass # Service errors - close and retry
class readonly(abort): pass # Mailbox status changed to READ-ONLY
- def __init__(self, host = '', port = IMAP4_PORT):
+ def __init__(self, host='', port=IMAP4_PORT):
self.debug = Debug
self.state = 'LOGOUT'
self.literal = None # A literal argument to a command
@@ -176,6 +182,7 @@ class IMAP4:
self.is_readonly = False # READ-ONLY desired state
self.tagnum = 0
self._tls_established = False
+ self._mode_ascii()
# Open socket to server.
@@ -190,6 +197,19 @@ class IMAP4:
pass
raise
+ def _mode_ascii(self):
+ self.utf8_enabled = False
+ self._encoding = 'ascii'
+ self.Literal = re.compile(_Literal, re.ASCII)
+ self.Untagged_status = re.compile(_Untagged_status, re.ASCII)
+
+
+ def _mode_utf8(self):
+ self.utf8_enabled = True
+ self._encoding = 'utf-8'
+ self.Literal = re.compile(_Literal)
+ self.Untagged_status = re.compile(_Untagged_status)
+
def _connect(self):
# Create unique tag for this session,
@@ -239,6 +259,14 @@ class IMAP4:
return getattr(self, attr.lower())
raise AttributeError("Unknown IMAP4 command: '%s'" % attr)
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ try:
+ self.logout()
+ except OSError:
+ pass
# Overridable methods
@@ -352,7 +380,10 @@ class IMAP4:
date_time = Time2Internaldate(date_time)
else:
date_time = None
- self.literal = MapCRLF.sub(CRLF, message)
+ literal = MapCRLF.sub(CRLF, message)
+ if self.utf8_enabled:
+ literal = b'UTF8 (' + literal + b')'
+ self.literal = literal
return self._simple_command(name, mailbox, flags, date_time)
@@ -447,6 +478,18 @@ class IMAP4:
"""
return self._simple_command('DELETEACL', mailbox, who)
+ def enable(self, capability):
+ """Send an RFC5161 enable string to the server.
+
+ (typ, [data]) = <intance>.enable(capability)
+ """
+ if 'ENABLE' not in self.capabilities:
+ raise IMAP4.error("Server does not support ENABLE")
+ typ, data = self._simple_command('ENABLE', capability)
+ if typ == 'OK' and 'UTF8=ACCEPT' in capability.upper():
+ self._mode_utf8()
+ return typ, data
+
def expunge(self):
"""Permanently remove deleted items from selected mailbox.
@@ -553,7 +596,7 @@ class IMAP4:
def _CRAM_MD5_AUTH(self, challenge):
""" Authobject to use with CRAM-MD5 authentication. """
import hmac
- pwd = (self.password.encode('ASCII') if isinstance(self.password, str)
+ pwd = (self.password.encode('utf-8') if isinstance(self.password, str)
else self.password)
return self.user + " " + hmac.HMAC(pwd, challenge, 'md5').hexdigest()
@@ -653,9 +696,12 @@ class IMAP4:
(typ, [data]) = <instance>.search(charset, criterion, ...)
'data' is space separated list of matching message numbers.
+ If UTF8 is enabled, charset MUST be None.
"""
name = 'SEARCH'
if charset:
+ if self.utf8_enabled:
+ raise IMAP4.error("Non-None charset not valid in UTF8 mode")
typ, dat = self._simple_command(name, 'CHARSET', charset, *criteria)
else:
typ, dat = self._simple_command(name, *criteria)
@@ -869,7 +915,7 @@ class IMAP4:
def _check_bye(self):
bye = self.untagged_responses.get('BYE')
if bye:
- raise self.abort(bye[-1].decode('ascii', 'replace'))
+ raise self.abort(bye[-1].decode(self._encoding, 'replace'))
def _command(self, name, *args):
@@ -890,12 +936,12 @@ class IMAP4:
raise self.readonly('mailbox status changed to READ-ONLY')
tag = self._new_tag()
- name = bytes(name, 'ASCII')
+ name = bytes(name, self._encoding)
data = tag + b' ' + name
for arg in args:
if arg is None: continue
if isinstance(arg, str):
- arg = bytes(arg, "ASCII")
+ arg = bytes(arg, self._encoding)
data = data + b' ' + arg
literal = self.literal
@@ -905,7 +951,7 @@ class IMAP4:
literator = literal
else:
literator = None
- data = data + bytes(' {%s}' % len(literal), 'ASCII')
+ data = data + bytes(' {%s}' % len(literal), self._encoding)
if __debug__:
if self.debug >= 4:
@@ -970,7 +1016,7 @@ class IMAP4:
typ, dat = self.capability()
if dat == [None]:
raise self.error('no CAPABILITY response from server')
- dat = str(dat[-1], "ASCII")
+ dat = str(dat[-1], self._encoding)
dat = dat.upper()
self.capabilities = tuple(dat.split())
@@ -989,10 +1035,10 @@ class IMAP4:
if self._match(self.tagre, resp):
tag = self.mo.group('tag')
if not tag in self.tagged_commands:
- raise self.abort('unexpected tagged response: %s' % resp)
+ raise self.abort('unexpected tagged response: %r' % resp)
typ = self.mo.group('type')
- typ = str(typ, 'ASCII')
+ typ = str(typ, self._encoding)
dat = self.mo.group('data')
self.tagged_commands[tag] = (typ, [dat])
else:
@@ -1001,7 +1047,7 @@ class IMAP4:
# '*' (untagged) responses?
if not self._match(Untagged_response, resp):
- if self._match(Untagged_status, resp):
+ if self._match(self.Untagged_status, resp):
dat2 = self.mo.group('data2')
if self.mo is None:
@@ -1011,17 +1057,17 @@ class IMAP4:
self.continuation_response = self.mo.group('data')
return None # NB: indicates continuation
- raise self.abort("unexpected response: '%s'" % resp)
+ raise self.abort("unexpected response: %r" % resp)
typ = self.mo.group('type')
- typ = str(typ, 'ascii')
+ typ = str(typ, self._encoding)
dat = self.mo.group('data')
if dat is None: dat = b'' # Null untagged response
if dat2: dat = dat + b' ' + dat2
# Is there a literal to come?
- while self._match(Literal, dat):
+ while self._match(self.Literal, dat):
# Read literal direct from connection.
@@ -1045,7 +1091,7 @@ class IMAP4:
if typ in ('OK', 'NO', 'BAD') and self._match(Response_code, dat):
typ = self.mo.group('type')
- typ = str(typ, "ASCII")
+ typ = str(typ, self._encoding)
self._append_untagged(typ, self.mo.group('data'))
if __debug__:
@@ -1115,7 +1161,7 @@ class IMAP4:
def _new_tag(self):
- tag = self.tagpre + bytes(str(self.tagnum), 'ASCII')
+ tag = self.tagpre + bytes(str(self.tagnum), self._encoding)
self.tagnum = self.tagnum + 1
self.tagged_commands[tag] = None
return tag
@@ -1205,7 +1251,8 @@ if HAVE_SSL:
"""
- def __init__(self, host='', port=IMAP4_SSL_PORT, keyfile=None, certfile=None, ssl_context=None):
+ def __init__(self, host='', port=IMAP4_SSL_PORT, keyfile=None,
+ certfile=None, ssl_context=None):
if ssl_context is not None and keyfile is not None:
raise ValueError("ssl_context and keyfile arguments are mutually "
"exclusive")
@@ -1243,7 +1290,7 @@ class IMAP4_stream(IMAP4):
Instantiate with: IMAP4_stream(command)
- where "command" is a string that can be passed to subprocess.Popen()
+ "command" - a string that can be passed to subprocess.Popen()
for more documentation see the docstring of the parent class IMAP4.
"""
@@ -1320,7 +1367,7 @@ class _Authenticator:
#
oup = b''
if isinstance(inp, str):
- inp = inp.encode('ASCII')
+ inp = inp.encode('utf-8')
while inp:
if len(inp) > 48:
t = inp[:48]
diff --git a/Lib/imghdr.py b/Lib/imghdr.py
index add2ea8..b267925 100644
--- a/Lib/imghdr.py
+++ b/Lib/imghdr.py
@@ -110,6 +110,18 @@ def test_bmp(h, f):
tests.append(test_bmp)
+def test_webp(h, f):
+ if h.startswith(b'RIFF') and h[8:12] == b'WEBP':
+ return 'webp'
+
+tests.append(test_webp)
+
+def test_exr(h, f):
+ if h.startswith(b'\x76\x2f\x31\x01'):
+ return 'exr'
+
+tests.append(test_exr)
+
#--------------------#
# Small test program #
#--------------------#
diff --git a/Lib/imp.py b/Lib/imp.py
index c922e92..2cd6440 100644
--- a/Lib/imp.py
+++ b/Lib/imp.py
@@ -8,15 +8,16 @@ functionality over this module.
# (Probably) need to stay in _imp
from _imp import (lock_held, acquire_lock, release_lock,
get_frozen_object, is_frozen_package,
- init_builtin, init_frozen, is_builtin, is_frozen,
+ init_frozen, is_builtin, is_frozen,
_fix_co_filename)
try:
- from _imp import load_dynamic
+ from _imp import create_dynamic
except ImportError:
# Platform doesn't support dynamic loading.
- load_dynamic = None
+ create_dynamic = None
-from importlib._bootstrap import SourcelessFileLoader, _ERR_MSG, _SpecMethods
+from importlib._bootstrap import _ERR_MSG, _exec, _load, _builtin_from_name
+from importlib._bootstrap_external import SourcelessFileLoader
from importlib import machinery
from importlib import util
@@ -29,7 +30,7 @@ import warnings
warnings.warn("the imp module is deprecated in favour of importlib; "
"see the module's documentation for alternative uses",
- PendingDeprecationWarning)
+ PendingDeprecationWarning, stacklevel=2)
# DEPRECATED
SEARCH_ERROR = 0
@@ -58,24 +59,23 @@ def new_module(name):
def get_magic():
"""**DEPRECATED**
- Return the magic number for .pyc or .pyo files.
+ Return the magic number for .pyc files.
"""
return util.MAGIC_NUMBER
def get_tag():
- """Return the magic tag for .pyc or .pyo files."""
+ """Return the magic tag for .pyc files."""
return sys.implementation.cache_tag
def cache_from_source(path, debug_override=None):
"""**DEPRECATED**
- Given the path to a .py file, return the path to its .pyc/.pyo file.
+ Given the path to a .py file, return the path to its .pyc file.
The .py file does not need to exist; this simply returns the path to the
- .pyc/.pyo file calculated as if the .py file were imported. The extension
- will be .pyc unless sys.flags.optimize is non-zero, then it will be .pyo.
+ .pyc file calculated as if the .py file were imported.
If debug_override is not None, then it must be a boolean and is used in
place of sys.flags.optimize.
@@ -83,16 +83,18 @@ def cache_from_source(path, debug_override=None):
If sys.implementation.cache_tag is None then NotImplementedError is raised.
"""
- return util.cache_from_source(path, debug_override)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore')
+ return util.cache_from_source(path, debug_override)
def source_from_cache(path):
"""**DEPRECATED**
- Given the path to a .pyc./.pyo file, return the path to its .py file.
+ Given the path to a .pyc. file, return the path to its .py file.
- The .pyc/.pyo file does not need to exist; this simply returns the path to
- the .py file calculated to correspond to the .pyc/.pyo file. If path does
+ The .pyc file does not need to exist; this simply returns the path to
+ the .py file calculated to correspond to the .pyc file. If path does
not conform to PEP 3147 format, ValueError will be raised. If
sys.implementation.cache_tag is None then NotImplementedError is raised.
@@ -164,11 +166,10 @@ class _LoadSourceCompatibility(_HackedGetData, machinery.SourceFileLoader):
def load_source(name, pathname, file=None):
loader = _LoadSourceCompatibility(name, pathname, file)
spec = util.spec_from_file_location(name, pathname, loader=loader)
- methods = _SpecMethods(spec)
if name in sys.modules:
- module = methods.exec(sys.modules[name])
+ module = _exec(spec, sys.modules[name])
else:
- module = methods.load()
+ module = _load(spec)
# To allow reloading to potentially work, use a non-hacked loader which
# won't rely on a now-closed file object.
module.__loader__ = machinery.SourceFileLoader(name, pathname)
@@ -185,11 +186,10 @@ def load_compiled(name, pathname, file=None):
"""**DEPRECATED**"""
loader = _LoadCompiledCompatibility(name, pathname, file)
spec = util.spec_from_file_location(name, pathname, loader=loader)
- methods = _SpecMethods(spec)
if name in sys.modules:
- module = methods.exec(sys.modules[name])
+ module = _exec(spec, sys.modules[name])
else:
- module = methods.load()
+ module = _load(spec)
# To allow reloading to potentially work, use a non-hacked loader which
# won't rely on a now-closed file object.
module.__loader__ = SourcelessFileLoader(name, pathname)
@@ -210,11 +210,10 @@ def load_package(name, path):
raise ValueError('{!r} is not a package'.format(path))
spec = util.spec_from_file_location(name, path,
submodule_search_locations=[])
- methods = _SpecMethods(spec)
if name in sys.modules:
- return methods.exec(sys.modules[name])
+ return _exec(spec, sys.modules[name])
else:
- return methods.load()
+ return _load(spec)
def load_module(name, file, filename, details):
@@ -313,3 +312,28 @@ def reload(module):
"""
return importlib.reload(module)
+
+
+def init_builtin(name):
+ """**DEPRECATED**
+
+ Load and return a built-in module by name, or None is such module doesn't
+ exist
+ """
+ try:
+ return _builtin_from_name(name)
+ except ImportError:
+ return None
+
+
+if create_dynamic:
+ def load_dynamic(name, path, file=None):
+ """**DEPRECATED**
+
+ Load an extension module.
+ """
+ import importlib.machinery
+ loader = importlib.machinery.ExtensionFileLoader(name, path)
+ return loader.load_module()
+else:
+ load_dynamic = None
diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py
index 1bc9947..b6a9f82 100644
--- a/Lib/importlib/__init__.py
+++ b/Lib/importlib/__init__.py
@@ -30,9 +30,26 @@ else:
pass
sys.modules['importlib._bootstrap'] = _bootstrap
+try:
+ import _frozen_importlib_external as _bootstrap_external
+except ImportError:
+ from . import _bootstrap_external
+ _bootstrap_external._setup(_bootstrap)
+ _bootstrap._bootstrap_external = _bootstrap_external
+else:
+ _bootstrap_external.__name__ = 'importlib._bootstrap_external'
+ _bootstrap_external.__package__ = 'importlib'
+ try:
+ _bootstrap_external.__file__ = __file__.replace('__init__.py', '_bootstrap_external.py')
+ except NameError:
+ # __file__ is not guaranteed to be defined, e.g. if this code gets
+ # frozen by a tool like cx_Freeze.
+ pass
+ sys.modules['importlib._bootstrap_external'] = _bootstrap_external
+
# To simplify imports in test code
-_w_long = _bootstrap._w_long
-_r_long = _bootstrap._r_long
+_w_long = _bootstrap_external._w_long
+_r_long = _bootstrap_external._r_long
# Fully bootstrapped at this point, import whatever you like, circular
# dependencies and startup overhead minimisation permitting :)
@@ -73,7 +90,7 @@ def find_loader(name, path=None):
except KeyError:
pass
except AttributeError:
- raise ValueError('{}.__loader__ is not set'.format(name))
+ raise ValueError('{}.__loader__ is not set'.format(name)) from None
spec = _bootstrap._find_spec(name, path)
# We won't worry about malformed specs (missing attributes).
@@ -138,15 +155,15 @@ def reload(module):
parent = sys.modules[parent_name]
except KeyError:
msg = "parent {!r} not in sys.modules"
- raise ImportError(msg.format(parent_name), name=parent_name)
+ raise ImportError(msg.format(parent_name),
+ name=parent_name) from None
else:
pkgpath = parent.__path__
else:
pkgpath = None
target = module
spec = module.__spec__ = _bootstrap._find_spec(name, pkgpath, target)
- methods = _bootstrap._SpecMethods(spec)
- methods.exec(module)
+ _bootstrap._exec(spec, module)
# The module may have replaced itself in sys.modules!
return sys.modules[name]
finally:
diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
index 5b91c05..6f62bb3 100644
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -9,7 +9,7 @@ work. One should use importlib as the public-facing version of this module.
#
# IMPORTANT: Whenever making changes to this module, be sure to run
# a top-level make in order to get the frozen version of the module
-# update. Not doing so will result in the Makefile to fail for
+# updated. Not doing so will result in the Makefile to fail for
# all others who don't have a ./python around to freeze the module
# in the early stages of compilation.
#
@@ -22,101 +22,7 @@ work. One should use importlib as the public-facing version of this module.
# Bootstrap-related code ######################################################
-_CASE_INSENSITIVE_PLATFORMS = 'win', 'cygwin', 'darwin'
-
-
-def _make_relax_case():
- if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
- def _relax_case():
- """True if filenames must be checked case-insensitively."""
- return b'PYTHONCASEOK' in _os.environ
- else:
- def _relax_case():
- """True if filenames must be checked case-insensitively."""
- return False
- return _relax_case
-
-
-def _w_long(x):
- """Convert a 32-bit integer to little-endian."""
- return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little')
-
-
-def _r_long(int_bytes):
- """Convert 4 bytes in little-endian to an integer."""
- return int.from_bytes(int_bytes, 'little')
-
-
-def _path_join(*path_parts):
- """Replacement for os.path.join()."""
- return path_sep.join([part.rstrip(path_separators)
- for part in path_parts if part])
-
-
-def _path_split(path):
- """Replacement for os.path.split()."""
- if len(path_separators) == 1:
- front, _, tail = path.rpartition(path_sep)
- return front, tail
- for x in reversed(path):
- if x in path_separators:
- front, tail = path.rsplit(x, maxsplit=1)
- return front, tail
- return '', path
-
-
-def _path_stat(path):
- """Stat the path.
-
- Made a separate function to make it easier to override in experiments
- (e.g. cache stat results).
-
- """
- return _os.stat(path)
-
-
-def _path_is_mode_type(path, mode):
- """Test whether the path is the specified mode type."""
- try:
- stat_info = _path_stat(path)
- except OSError:
- return False
- return (stat_info.st_mode & 0o170000) == mode
-
-
-def _path_isfile(path):
- """Replacement for os.path.isfile."""
- return _path_is_mode_type(path, 0o100000)
-
-
-def _path_isdir(path):
- """Replacement for os.path.isdir."""
- if not path:
- path = _os.getcwd()
- return _path_is_mode_type(path, 0o040000)
-
-
-def _write_atomic(path, data, mode=0o666):
- """Best-effort function to write data to a path atomically.
- Be prepared to handle a FileExistsError if concurrent writing of the
- temporary file is attempted."""
- # id() is used to generate a pseudo-random filename.
- path_tmp = '{}.{}'.format(path, id(path))
- fd = _os.open(path_tmp,
- _os.O_EXCL | _os.O_CREAT | _os.O_WRONLY, mode & 0o666)
- try:
- # We first write data to a temporary file, and then use os.replace() to
- # perform an atomic rename.
- with _io.FileIO(fd, 'wb') as file:
- file.write(data)
- _os.replace(path_tmp, path)
- except OSError:
- try:
- _os.unlink(path_tmp)
- except OSError:
- pass
- raise
-
+_bootstrap_external = None
def _wrap(new, old):
"""Simple substitute for functools.update_wrapper."""
@@ -130,10 +36,6 @@ def _new_module(name):
return type(sys)(name)
-_code_type = type(_wrap.__code__)
-
-
-
class _ManageReload:
"""Manages the possible clean-up of sys.modules for load_module()."""
@@ -309,7 +211,6 @@ def _lock_unlock_module(name):
lock.release()
# Frame stripping magic ###############################################
-
def _call_with_frames_removed(f, *args, **kwds):
"""remove_importlib_frames in import.c will always remove sequences
of importlib frames that end with a call to this function
@@ -321,200 +222,6 @@ def _call_with_frames_removed(f, *args, **kwds):
return f(*args, **kwds)
-# Finder/loader utility code ###############################################
-
-# Magic word to reject .pyc files generated by other Python versions.
-# It should change for each incompatible change to the bytecode.
-#
-# The value of CR and LF is incorporated so if you ever read or write
-# a .pyc file in text mode the magic number will be wrong; also, the
-# Apple MPW compiler swaps their values, botching string constants.
-#
-# The magic numbers must be spaced apart at least 2 values, as the
-# -U interpeter flag will cause MAGIC+1 being used. They have been
-# odd numbers for some time now.
-#
-# There were a variety of old schemes for setting the magic number.
-# The current working scheme is to increment the previous value by
-# 10.
-#
-# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic
-# number also includes a new "magic tag", i.e. a human readable string used
-# to represent the magic number in __pycache__ directories. When you change
-# the magic number, you must also set a new unique magic tag. Generally this
-# can be named after the Python major version of the magic number bump, but
-# it can really be anything, as long as it's different than anything else
-# that's come before. The tags are included in the following table, starting
-# with Python 3.2a0.
-#
-# Known values:
-# Python 1.5: 20121
-# Python 1.5.1: 20121
-# Python 1.5.2: 20121
-# Python 1.6: 50428
-# Python 2.0: 50823
-# Python 2.0.1: 50823
-# Python 2.1: 60202
-# Python 2.1.1: 60202
-# Python 2.1.2: 60202
-# Python 2.2: 60717
-# Python 2.3a0: 62011
-# Python 2.3a0: 62021
-# Python 2.3a0: 62011 (!)
-# Python 2.4a0: 62041
-# Python 2.4a3: 62051
-# Python 2.4b1: 62061
-# Python 2.5a0: 62071
-# Python 2.5a0: 62081 (ast-branch)
-# Python 2.5a0: 62091 (with)
-# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode)
-# Python 2.5b3: 62101 (fix wrong code: for x, in ...)
-# Python 2.5b3: 62111 (fix wrong code: x += yield)
-# Python 2.5c1: 62121 (fix wrong lnotab with for loops and
-# storing constants that should have been removed)
-# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp)
-# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode)
-# Python 2.6a1: 62161 (WITH_CLEANUP optimization)
-# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND)
-# Python 2.7a0: 62181 (optimize conditional branches:
-# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
-# Python 2.7a0 62191 (introduce SETUP_WITH)
-# Python 2.7a0 62201 (introduce BUILD_SET)
-# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD)
-# Python 3000: 3000
-# 3010 (removed UNARY_CONVERT)
-# 3020 (added BUILD_SET)
-# 3030 (added keyword-only parameters)
-# 3040 (added signature annotations)
-# 3050 (print becomes a function)
-# 3060 (PEP 3115 metaclass syntax)
-# 3061 (string literals become unicode)
-# 3071 (PEP 3109 raise changes)
-# 3081 (PEP 3137 make __file__ and __name__ unicode)
-# 3091 (kill str8 interning)
-# 3101 (merge from 2.6a0, see 62151)
-# 3103 (__file__ points to source file)
-# Python 3.0a4: 3111 (WITH_CLEANUP optimization).
-# Python 3.0a5: 3131 (lexical exception stacking, including POP_EXCEPT)
-# Python 3.1a0: 3141 (optimize list, set and dict comprehensions:
-# change LIST_APPEND and SET_ADD, add MAP_ADD)
-# Python 3.1a0: 3151 (optimize conditional branches:
-# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
-# Python 3.2a0: 3160 (add SETUP_WITH)
-# tag: cpython-32
-# Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR)
-# tag: cpython-32
-# Python 3.2a2 3180 (add DELETE_DEREF)
-# Python 3.3a0 3190 __class__ super closure changed
-# Python 3.3a0 3200 (__qualname__ added)
-# 3210 (added size modulo 2**32 to the pyc header)
-# Python 3.3a1 3220 (changed PEP 380 implementation)
-# Python 3.3a4 3230 (revert changes to implicit __class__ closure)
-# Python 3.4a1 3250 (evaluate positional default arguments before
-# keyword-only defaults)
-# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override
-# free vars)
-# Python 3.4a1 3270 (various tweaks to the __class__ closure)
-# Python 3.4a1 3280 (remove implicit class argument)
-# Python 3.4a4 3290 (changes to __qualname__ computation)
-# Python 3.4a4 3300 (more changes to __qualname__ computation)
-# Python 3.4rc2 3310 (alter __qualname__ computation)
-#
-# MAGIC must change whenever the bytecode emitted by the compiler may no
-# longer be understood by older implementations of the eval loop (usually
-# due to the addition of new opcodes).
-
-MAGIC_NUMBER = (3310).to_bytes(2, 'little') + b'\r\n'
-_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
-
-_PYCACHE = '__pycache__'
-
-SOURCE_SUFFIXES = ['.py'] # _setup() adds .pyw as needed.
-
-DEBUG_BYTECODE_SUFFIXES = ['.pyc']
-OPTIMIZED_BYTECODE_SUFFIXES = ['.pyo']
-
-def cache_from_source(path, debug_override=None):
- """Given the path to a .py file, return the path to its .pyc/.pyo file.
-
- The .py file does not need to exist; this simply returns the path to the
- .pyc/.pyo file calculated as if the .py file were imported. The extension
- will be .pyc unless sys.flags.optimize is non-zero, then it will be .pyo.
-
- If debug_override is not None, then it must be a boolean and is used in
- place of sys.flags.optimize.
-
- If sys.implementation.cache_tag is None then NotImplementedError is raised.
-
- """
- debug = not sys.flags.optimize if debug_override is None else debug_override
- if debug:
- suffixes = DEBUG_BYTECODE_SUFFIXES
- else:
- suffixes = OPTIMIZED_BYTECODE_SUFFIXES
- head, tail = _path_split(path)
- base, sep, rest = tail.rpartition('.')
- tag = sys.implementation.cache_tag
- if tag is None:
- raise NotImplementedError('sys.implementation.cache_tag is None')
- filename = ''.join([(base if base else rest), sep, tag, suffixes[0]])
- return _path_join(head, _PYCACHE, filename)
-
-
-def source_from_cache(path):
- """Given the path to a .pyc./.pyo file, return the path to its .py file.
-
- The .pyc/.pyo file does not need to exist; this simply returns the path to
- the .py file calculated to correspond to the .pyc/.pyo file. If path does
- not conform to PEP 3147 format, ValueError will be raised. If
- sys.implementation.cache_tag is None then NotImplementedError is raised.
-
- """
- if sys.implementation.cache_tag is None:
- raise NotImplementedError('sys.implementation.cache_tag is None')
- head, pycache_filename = _path_split(path)
- head, pycache = _path_split(head)
- if pycache != _PYCACHE:
- raise ValueError('{} not bottom-level directory in '
- '{!r}'.format(_PYCACHE, path))
- if pycache_filename.count('.') != 2:
- raise ValueError('expected only 2 dots in '
- '{!r}'.format(pycache_filename))
- base_filename = pycache_filename.partition('.')[0]
- return _path_join(head, base_filename + SOURCE_SUFFIXES[0])
-
-
-def _get_sourcefile(bytecode_path):
- """Convert a bytecode file path to a source path (if possible).
-
- This function exists purely for backwards-compatibility for
- PyImport_ExecCodeModuleWithFilenames() in the C API.
-
- """
- if len(bytecode_path) == 0:
- return None
- rest, _, extension = bytecode_path.rpartition('.')
- if not rest or extension.lower()[-3:-1] != 'py':
- return bytecode_path
- try:
- source_path = source_from_cache(bytecode_path)
- except (NotImplementedError, ValueError):
- source_path = bytecode_path[:-1]
- return source_path if _path_isfile(source_path) else bytecode_path
-
-
-def _calc_mode(path):
- """Calculate the mode permissions for a bytecode file."""
- try:
- mode = _path_stat(path).st_mode
- except OSError:
- mode = 0o666
- # We always ensure write access so we can update cached files
- # later even when the source files are read-only on Windows (#6074)
- mode |= 0o200
- return mode
-
-
def _verbose_message(message, *args, verbosity=1):
"""Print the message to stderr if -v/PYTHONVERBOSE is turned on."""
if sys.flags.verbose >= verbosity:
@@ -523,24 +230,6 @@ def _verbose_message(message, *args, verbosity=1):
print(message.format(*args), file=sys.stderr)
-def _check_name(method):
- """Decorator to verify that the module being requested matches the one the
- loader can handle.
-
- The first argument (self) must define _name which the second argument is
- compared against. If the comparison fails then ImportError is raised.
-
- """
- def _check_name_wrapper(self, name=None, *args, **kwargs):
- if name is None:
- name = self.name
- elif self.name != name:
- raise ImportError('loader cannot handle %s' % name, name=name)
- return method(self, name, *args, **kwargs)
- _wrap(_check_name_wrapper, method)
- return _check_name_wrapper
-
-
def _requires_builtin(fxn):
"""Decorator to verify the named module is built-in."""
def _requires_builtin_wrapper(self, fullname):
@@ -563,23 +252,7 @@ def _requires_frozen(fxn):
return _requires_frozen_wrapper
-def _find_module_shim(self, fullname):
- """Try to find a loader for the specified module by delegating to
- self.find_loader().
-
- This method is deprecated in favor of finder.find_spec().
-
- """
- # Call find_loader(). If it returns a string (indicating this
- # is a namespace package portion), generate a warning and
- # return None.
- loader, portions = self.find_loader(fullname)
- if loader is None and len(portions):
- msg = 'Not importing directory {}: missing __init__'
- _warnings.warn(msg.format(portions[0]), ImportWarning)
- return loader
-
-
+# Typically used by loader classes as a method replacement.
def _load_module_shim(self, fullname):
"""Load the specified module into sys.modules and return it.
@@ -587,103 +260,12 @@ def _load_module_shim(self, fullname):
"""
spec = spec_from_loader(fullname, self)
- methods = _SpecMethods(spec)
if fullname in sys.modules:
module = sys.modules[fullname]
- methods.exec(module)
+ _exec(spec, module)
return sys.modules[fullname]
else:
- return methods.load()
-
-
-def _validate_bytecode_header(data, source_stats=None, name=None, path=None):
- """Validate the header of the passed-in bytecode against source_stats (if
- given) and returning the bytecode that can be compiled by compile().
-
- All other arguments are used to enhance error reporting.
-
- ImportError is raised when the magic number is incorrect or the bytecode is
- found to be stale. EOFError is raised when the data is found to be
- truncated.
-
- """
- exc_details = {}
- if name is not None:
- exc_details['name'] = name
- else:
- # To prevent having to make all messages have a conditional name.
- name = '<bytecode>'
- if path is not None:
- exc_details['path'] = path
- magic = data[:4]
- raw_timestamp = data[4:8]
- raw_size = data[8:12]
- if magic != MAGIC_NUMBER:
- message = 'bad magic number in {!r}: {!r}'.format(name, magic)
- _verbose_message(message)
- raise ImportError(message, **exc_details)
- elif len(raw_timestamp) != 4:
- message = 'reached EOF while reading timestamp in {!r}'.format(name)
- _verbose_message(message)
- raise EOFError(message)
- elif len(raw_size) != 4:
- message = 'reached EOF while reading size of source in {!r}'.format(name)
- _verbose_message(message)
- raise EOFError(message)
- if source_stats is not None:
- try:
- source_mtime = int(source_stats['mtime'])
- except KeyError:
- pass
- else:
- if _r_long(raw_timestamp) != source_mtime:
- message = 'bytecode is stale for {!r}'.format(name)
- _verbose_message(message)
- raise ImportError(message, **exc_details)
- try:
- source_size = source_stats['size'] & 0xFFFFFFFF
- except KeyError:
- pass
- else:
- if _r_long(raw_size) != source_size:
- raise ImportError('bytecode is stale for {!r}'.format(name),
- **exc_details)
- return data[12:]
-
-
-def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None):
- """Compile bytecode as returned by _validate_bytecode_header()."""
- code = marshal.loads(data)
- if isinstance(code, _code_type):
- _verbose_message('code object from {!r}', bytecode_path)
- if source_path is not None:
- _imp._fix_co_filename(code, source_path)
- return code
- else:
- raise ImportError('Non-code object in {!r}'.format(bytecode_path),
- name=name, path=bytecode_path)
-
-def _code_to_bytecode(code, mtime=0, source_size=0):
- """Compile a code object into bytecode for writing out to a byte-compiled
- file."""
- data = bytearray(MAGIC_NUMBER)
- data.extend(_w_long(mtime))
- data.extend(_w_long(source_size))
- data.extend(marshal.dumps(code))
- return data
-
-
-def decode_source(source_bytes):
- """Decode bytes representing source code and return the string.
-
- Universal newline support is used in the decoding.
- """
- import tokenize # To avoid bootstrap issues.
- source_bytes_readline = _io.BytesIO(source_bytes).readline
- encoding = tokenize.detect_encoding(source_bytes_readline)
- newline_decoder = _io.IncrementalNewlineDecoder(None, True)
- return newline_decoder.decode(source_bytes.decode(encoding[0]))
-
+ return _load(spec)
# Module specifications #######################################################
@@ -704,7 +286,7 @@ def _module_repr(module):
pass
else:
if spec is not None:
- return _SpecMethods(spec).module_repr()
+ return _module_repr_from_spec(spec)
# We could use module.__class__.__name__ instead of 'module' in the
# various repr permutations.
@@ -825,14 +407,9 @@ class ModuleSpec:
def cached(self):
if self._cached is None:
if self.origin is not None and self._set_fileattr:
- filename = self.origin
- if filename.endswith(tuple(SOURCE_SUFFIXES)):
- try:
- self._cached = cache_from_source(filename)
- except NotImplementedError:
- pass
- elif filename.endswith(tuple(BYTECODE_SUFFIXES)):
- self._cached = filename
+ if _bootstrap_external is None:
+ raise NotImplementedError
+ self._cached = _bootstrap_external._get_cached(self.origin)
return self._cached
@cached.setter
@@ -859,6 +436,10 @@ class ModuleSpec:
def spec_from_loader(name, loader, *, origin=None, is_package=None):
"""Return a module spec based on various loader methods."""
if hasattr(loader, 'get_filename'):
+ if _bootstrap_external is None:
+ raise NotImplementedError
+ spec_from_file_location = _bootstrap_external.spec_from_file_location
+
if is_package is None:
return spec_from_file_location(name, loader=loader)
search = [] if is_package else None
@@ -881,70 +462,6 @@ def spec_from_loader(name, loader, *, origin=None, is_package=None):
_POPULATE = object()
-def spec_from_file_location(name, location=None, *, loader=None,
- submodule_search_locations=_POPULATE):
- """Return a module spec based on a file location.
-
- To indicate that the module is a package, set
- submodule_search_locations to a list of directory paths. An
- empty list is sufficient, though its not otherwise useful to the
- import system.
-
- The loader must take a spec as its only __init__() arg.
-
- """
- if location is None:
- # The caller may simply want a partially populated location-
- # oriented spec. So we set the location to a bogus value and
- # fill in as much as we can.
- location = '<unknown>'
- if hasattr(loader, 'get_filename'):
- # ExecutionLoader
- try:
- location = loader.get_filename(name)
- except ImportError:
- pass
-
- # If the location is on the filesystem, but doesn't actually exist,
- # we could return None here, indicating that the location is not
- # valid. However, we don't have a good way of testing since an
- # indirect location (e.g. a zip file or URL) will look like a
- # non-existent file relative to the filesystem.
-
- spec = ModuleSpec(name, loader, origin=location)
- spec._set_fileattr = True
-
- # Pick a loader if one wasn't provided.
- if loader is None:
- for loader_class, suffixes in _get_supported_file_loaders():
- if location.endswith(tuple(suffixes)):
- loader = loader_class(name, location)
- spec.loader = loader
- break
- else:
- return None
-
- # Set submodule_search_paths appropriately.
- if submodule_search_locations is _POPULATE:
- # Check the loader.
- if hasattr(loader, 'is_package'):
- try:
- is_package = loader.is_package(name)
- except ImportError:
- pass
- else:
- if is_package:
- spec.submodule_search_locations = []
- else:
- spec.submodule_search_locations = submodule_search_locations
- if spec.submodule_search_locations == []:
- if location:
- dirname = _path_split(location)[0]
- spec.submodule_search_locations.append(dirname)
-
- return spec
-
-
def _spec_from_module(module, loader=None, origin=None):
# This function is meant for use in _setup().
try:
@@ -990,257 +507,190 @@ def _spec_from_module(module, loader=None, origin=None):
return spec
-class _SpecMethods:
-
- """Convenience wrapper around spec objects to provide spec-specific
- methods."""
-
- # The various spec_from_* functions could be made factory methods here.
-
- def __init__(self, spec):
- self.spec = spec
-
- def module_repr(self):
- """Return the repr to use for the module."""
- # We mostly replicate _module_repr() using the spec attributes.
- spec = self.spec
- name = '?' if spec.name is None else spec.name
- if spec.origin is None:
- if spec.loader is None:
- return '<module {!r}>'.format(name)
- else:
- return '<module {!r} ({!r})>'.format(name, spec.loader)
- else:
- if spec.has_location:
- return '<module {!r} from {!r}>'.format(name, spec.origin)
- else:
- return '<module {!r} ({})>'.format(spec.name, spec.origin)
-
- def init_module_attrs(self, module, *, _override=False, _force_name=True):
- """Set the module's attributes.
-
- All missing import-related module attributes will be set. Here
- is how the spec attributes map onto the module:
-
- spec.name -> module.__name__
- spec.loader -> module.__loader__
- spec.parent -> module.__package__
- spec -> module.__spec__
-
- Optional:
- spec.origin -> module.__file__ (if spec.set_fileattr is true)
- spec.cached -> module.__cached__ (if __file__ also set)
- spec.submodule_search_locations -> module.__path__ (if set)
-
- """
- spec = self.spec
-
- # The passed in module may be not support attribute assignment,
- # in which case we simply don't set the attributes.
-
- # __name__
- if (_override or _force_name or
- getattr(module, '__name__', None) is None):
- try:
- module.__name__ = spec.name
- except AttributeError:
- pass
+def _init_module_attrs(spec, module, *, override=False):
+ # The passed-in module may be not support attribute assignment,
+ # in which case we simply don't set the attributes.
+ # __name__
+ if (override or getattr(module, '__name__', None) is None):
+ try:
+ module.__name__ = spec.name
+ except AttributeError:
+ pass
+ # __loader__
+ if override or getattr(module, '__loader__', None) is None:
+ loader = spec.loader
+ if loader is None:
+ # A backward compatibility hack.
+ if spec.submodule_search_locations is not None:
+ if _bootstrap_external is None:
+ raise NotImplementedError
+ _NamespaceLoader = _bootstrap_external._NamespaceLoader
- # __loader__
- if _override or getattr(module, '__loader__', None) is None:
- loader = spec.loader
- if loader is None:
- # A backward compatibility hack.
- if spec.submodule_search_locations is not None:
- loader = _NamespaceLoader.__new__(_NamespaceLoader)
- loader._path = spec.submodule_search_locations
+ loader = _NamespaceLoader.__new__(_NamespaceLoader)
+ loader._path = spec.submodule_search_locations
+ try:
+ module.__loader__ = loader
+ except AttributeError:
+ pass
+ # __package__
+ if override or getattr(module, '__package__', None) is None:
+ try:
+ module.__package__ = spec.parent
+ except AttributeError:
+ pass
+ # __spec__
+ try:
+ module.__spec__ = spec
+ except AttributeError:
+ pass
+ # __path__
+ if override or getattr(module, '__path__', None) is None:
+ if spec.submodule_search_locations is not None:
try:
- module.__loader__ = loader
+ module.__path__ = spec.submodule_search_locations
except AttributeError:
pass
-
- # __package__
- if _override or getattr(module, '__package__', None) is None:
+ # __file__/__cached__
+ if spec.has_location:
+ if override or getattr(module, '__file__', None) is None:
try:
- module.__package__ = spec.parent
+ module.__file__ = spec.origin
except AttributeError:
pass
- # __spec__
- try:
- module.__spec__ = spec
- except AttributeError:
- pass
-
- # __path__
- if _override or getattr(module, '__path__', None) is None:
- if spec.submodule_search_locations is not None:
- try:
- module.__path__ = spec.submodule_search_locations
- except AttributeError:
- pass
-
- if spec.has_location:
- # __file__
- if _override or getattr(module, '__file__', None) is None:
+ if override or getattr(module, '__cached__', None) is None:
+ if spec.cached is not None:
try:
- module.__file__ = spec.origin
+ module.__cached__ = spec.cached
except AttributeError:
pass
+ return module
- # __cached__
- if _override or getattr(module, '__cached__', None) is None:
- if spec.cached is not None:
- try:
- module.__cached__ = spec.cached
- except AttributeError:
- pass
- def create(self):
- """Return a new module to be loaded.
+def module_from_spec(spec):
+ """Create a module based on the provided spec."""
+ # Typically loaders will not implement create_module().
+ module = None
+ if hasattr(spec.loader, 'create_module'):
+ # If create_module() returns `None` then it means default
+ # module creation should be used.
+ module = spec.loader.create_module(spec)
+ elif hasattr(spec.loader, 'exec_module'):
+ _warnings.warn('starting in Python 3.6, loaders defining exec_module() '
+ 'must also define create_module()',
+ DeprecationWarning, stacklevel=2)
+ if module is None:
+ module = _new_module(spec.name)
+ _init_module_attrs(spec, module)
+ return module
- The import-related module attributes are also set with the
- appropriate values from the spec.
- """
- spec = self.spec
- # Typically loaders will not implement create_module().
- if hasattr(spec.loader, 'create_module'):
- # If create_module() returns `None` it means the default
- # module creation should be used.
- module = spec.loader.create_module(spec)
+def _module_repr_from_spec(spec):
+ """Return the repr to use for the module."""
+ # We mostly replicate _module_repr() using the spec attributes.
+ name = '?' if spec.name is None else spec.name
+ if spec.origin is None:
+ if spec.loader is None:
+ return '<module {!r}>'.format(name)
else:
- module = None
- if module is None:
- # This must be done before open() is ever called as the 'io'
- # module implicitly imports 'locale' and would otherwise
- # trigger an infinite loop.
- module = _new_module(spec.name)
- self.init_module_attrs(module)
- return module
-
- def _exec(self, module):
- """Do everything necessary to execute the module.
+ return '<module {!r} ({!r})>'.format(name, spec.loader)
+ else:
+ if spec.has_location:
+ return '<module {!r} from {!r}>'.format(name, spec.origin)
+ else:
+ return '<module {!r} ({})>'.format(spec.name, spec.origin)
- The namespace of `module` is used as the target of execution.
- This method uses the loader's `exec_module()` method.
- """
- self.spec.loader.exec_module(module)
+# Used by importlib.reload() and _load_module_shim().
+def _exec(spec, module):
+ """Execute the spec in an existing module's namespace."""
+ name = spec.name
+ _imp.acquire_lock()
+ with _ModuleLockManager(name):
+ if sys.modules.get(name) is not module:
+ msg = 'module {!r} not in sys.modules'.format(name)
+ raise ImportError(msg, name=name)
+ if spec.loader is None:
+ if spec.submodule_search_locations is None:
+ raise ImportError('missing loader', name=spec.name)
+ # namespace package
+ _init_module_attrs(spec, module, override=True)
+ return module
+ _init_module_attrs(spec, module, override=True)
+ if not hasattr(spec.loader, 'exec_module'):
+ # (issue19713) Once BuiltinImporter and ExtensionFileLoader
+ # have exec_module() implemented, we can add a deprecation
+ # warning here.
+ spec.loader.load_module(name)
+ else:
+ spec.loader.exec_module(module)
+ return sys.modules[name]
+
+
+def _load_backward_compatible(spec):
+ # (issue19713) Once BuiltinImporter and ExtensionFileLoader
+ # have exec_module() implemented, we can add a deprecation
+ # warning here.
+ spec.loader.load_module(spec.name)
+ # The module must be in sys.modules at this point!
+ module = sys.modules[spec.name]
+ if getattr(module, '__loader__', None) is None:
+ try:
+ module.__loader__ = spec.loader
+ except AttributeError:
+ pass
+ if getattr(module, '__package__', None) is None:
+ try:
+ # Since module.__path__ may not line up with
+ # spec.submodule_search_paths, we can't necessarily rely
+ # on spec.parent here.
+ module.__package__ = module.__name__
+ if not hasattr(module, '__path__'):
+ module.__package__ = spec.name.rpartition('.')[0]
+ except AttributeError:
+ pass
+ if getattr(module, '__spec__', None) is None:
+ try:
+ module.__spec__ = spec
+ except AttributeError:
+ pass
+ return module
- # Used by importlib.reload() and _load_module_shim().
- def exec(self, module):
- """Execute the spec in an existing module's namespace."""
- name = self.spec.name
- _imp.acquire_lock()
- with _ModuleLockManager(name):
- if sys.modules.get(name) is not module:
- msg = 'module {!r} not in sys.modules'.format(name)
- raise ImportError(msg, name=name)
- if self.spec.loader is None:
- if self.spec.submodule_search_locations is None:
- raise ImportError('missing loader', name=self.spec.name)
- # namespace package
- self.init_module_attrs(module, _override=True)
- return module
- self.init_module_attrs(module, _override=True)
- if not hasattr(self.spec.loader, 'exec_module'):
- # (issue19713) Once BuiltinImporter and ExtensionFileLoader
- # have exec_module() implemented, we can add a deprecation
- # warning here.
- self.spec.loader.load_module(name)
- else:
- self._exec(module)
- return sys.modules[name]
-
- def _load_backward_compatible(self):
- # (issue19713) Once BuiltinImporter and ExtensionFileLoader
- # have exec_module() implemented, we can add a deprecation
- # warning here.
- spec = self.spec
- spec.loader.load_module(spec.name)
- # The module must be in sys.modules at this point!
- module = sys.modules[spec.name]
- if getattr(module, '__loader__', None) is None:
- try:
- module.__loader__ = spec.loader
- except AttributeError:
- pass
- if getattr(module, '__package__', None) is None:
- try:
- # Since module.__path__ may not line up with
- # spec.submodule_search_paths, we can't necessarily rely
- # on spec.parent here.
- module.__package__ = module.__name__
- if not hasattr(module, '__path__'):
- module.__package__ = spec.name.rpartition('.')[0]
- except AttributeError:
- pass
- if getattr(module, '__spec__', None) is None:
- try:
- module.__spec__ = spec
- except AttributeError:
- pass
- return module
-
- def _load_unlocked(self):
- # A helper for direct use by the import system.
- if self.spec.loader is not None:
- # not a namespace package
- if not hasattr(self.spec.loader, 'exec_module'):
- return self._load_backward_compatible()
-
- module = self.create()
- with _installed_safely(module):
- if self.spec.loader is None:
- if self.spec.submodule_search_locations is None:
- raise ImportError('missing loader', name=self.spec.name)
- # A namespace package so do nothing.
- else:
- self._exec(module)
+def _load_unlocked(spec):
+ # A helper for direct use by the import system.
+ if spec.loader is not None:
+ # not a namespace package
+ if not hasattr(spec.loader, 'exec_module'):
+ return _load_backward_compatible(spec)
+
+ module = module_from_spec(spec)
+ with _installed_safely(module):
+ if spec.loader is None:
+ if spec.submodule_search_locations is None:
+ raise ImportError('missing loader', name=spec.name)
+ # A namespace package so do nothing.
+ else:
+ spec.loader.exec_module(module)
- # We don't ensure that the import-related module attributes get
- # set in the sys.modules replacement case. Such modules are on
- # their own.
- return sys.modules[self.spec.name]
+ # We don't ensure that the import-related module attributes get
+ # set in the sys.modules replacement case. Such modules are on
+ # their own.
+ return sys.modules[spec.name]
- # A method used during testing of _load_unlocked() and by
- # _load_module_shim().
- def load(self):
- """Return a new module object, loaded by the spec's loader.
+# A method used during testing of _load_unlocked() and by
+# _load_module_shim().
+def _load(spec):
+ """Return a new module object, loaded by the spec's loader.
- The module is not added to its parent.
+ The module is not added to its parent.
- If a module is already in sys.modules, that existing module gets
- clobbered.
+ If a module is already in sys.modules, that existing module gets
+ clobbered.
- """
- _imp.acquire_lock()
- with _ModuleLockManager(self.spec.name):
- return self._load_unlocked()
-
-
-def _fix_up_module(ns, name, pathname, cpathname=None):
- # This function is used by PyImport_ExecCodeModuleObject().
- loader = ns.get('__loader__')
- spec = ns.get('__spec__')
- if not loader:
- if spec:
- loader = spec.loader
- elif pathname == cpathname:
- loader = SourcelessFileLoader(name, pathname)
- else:
- loader = SourceFileLoader(name, pathname)
- if not spec:
- spec = spec_from_file_location(name, pathname, loader=loader)
- try:
- ns['__spec__'] = spec
- ns['__loader__'] = loader
- ns['__file__'] = pathname
- ns['__cached__'] = cpathname
- except Exception:
- # Not important enough to report.
- pass
+ """
+ _imp.acquire_lock()
+ with _ModuleLockManager(spec.name):
+ return _load_unlocked(spec)
# Loaders #####################################################################
@@ -1285,16 +735,17 @@ class BuiltinImporter:
return spec.loader if spec is not None else None
@classmethod
- @_requires_builtin
- def load_module(cls, fullname):
- """Load a built-in module."""
- # Once an exec_module() implementation is added we can also
- # add a deprecation warning here.
- with _ManageReload(fullname):
- module = _call_with_frames_removed(_imp.init_builtin, fullname)
- module.__loader__ = cls
- module.__package__ = ''
- return module
+ def create_module(self, spec):
+ """Create a built-in module"""
+ if spec.name not in sys.builtin_module_names:
+ raise ImportError('{!r} is not a built-in module'.format(spec.name),
+ name=spec.name)
+ return _call_with_frames_removed(_imp.create_builtin, spec)
+
+ @classmethod
+ def exec_module(self, module):
+ """Exec a built-in module"""
+ _call_with_frames_removed(_imp.exec_builtin, module)
@classmethod
@_requires_builtin
@@ -1314,6 +765,8 @@ class BuiltinImporter:
"""Return False as built-in modules are never packages."""
return False
+ load_module = classmethod(_load_module_shim)
+
class FrozenImporter:
@@ -1349,6 +802,10 @@ class FrozenImporter:
"""
return cls if _imp.is_frozen(fullname) else None
+ @classmethod
+ def create_module(cls, spec):
+ """Use default semantics for module creation."""
+
@staticmethod
def exec_module(module):
name = module.__spec__.name
@@ -1386,731 +843,6 @@ class FrozenImporter:
return _imp.is_frozen_package(fullname)
-class WindowsRegistryFinder:
-
- """Meta path finder for modules declared in the Windows registry."""
-
- REGISTRY_KEY = (
- 'Software\\Python\\PythonCore\\{sys_version}'
- '\\Modules\\{fullname}')
- REGISTRY_KEY_DEBUG = (
- 'Software\\Python\\PythonCore\\{sys_version}'
- '\\Modules\\{fullname}\\Debug')
- DEBUG_BUILD = False # Changed in _setup()
-
- @classmethod
- def _open_registry(cls, key):
- try:
- return _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, key)
- except OSError:
- return _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, key)
-
- @classmethod
- def _search_registry(cls, fullname):
- if cls.DEBUG_BUILD:
- registry_key = cls.REGISTRY_KEY_DEBUG
- else:
- registry_key = cls.REGISTRY_KEY
- key = registry_key.format(fullname=fullname,
- sys_version=sys.version[:3])
- try:
- with cls._open_registry(key) as hkey:
- filepath = _winreg.QueryValue(hkey, '')
- except OSError:
- return None
- return filepath
-
- @classmethod
- def find_spec(cls, fullname, path=None, target=None):
- filepath = cls._search_registry(fullname)
- if filepath is None:
- return None
- try:
- _path_stat(filepath)
- except OSError:
- return None
- for loader, suffixes in _get_supported_file_loaders():
- if filepath.endswith(tuple(suffixes)):
- spec = spec_from_loader(fullname, loader(fullname, filepath),
- origin=filepath)
- return spec
-
- @classmethod
- def find_module(cls, fullname, path=None):
- """Find module named in the registry.
-
- This method is deprecated. Use exec_module() instead.
-
- """
- spec = cls.find_spec(fullname, path)
- if spec is not None:
- return spec.loader
- else:
- return None
-
-
-class _LoaderBasics:
-
- """Base class of common code needed by both SourceLoader and
- SourcelessFileLoader."""
-
- def is_package(self, fullname):
- """Concrete implementation of InspectLoader.is_package by checking if
- the path returned by get_filename has a filename of '__init__.py'."""
- filename = _path_split(self.get_filename(fullname))[1]
- filename_base = filename.rsplit('.', 1)[0]
- tail_name = fullname.rpartition('.')[2]
- return filename_base == '__init__' and tail_name != '__init__'
-
- def exec_module(self, module):
- """Execute the module."""
- code = self.get_code(module.__name__)
- if code is None:
- raise ImportError('cannot load module {!r} when get_code() '
- 'returns None'.format(module.__name__))
- _call_with_frames_removed(exec, code, module.__dict__)
-
- load_module = _load_module_shim
-
-
-class SourceLoader(_LoaderBasics):
-
- def path_mtime(self, path):
- """Optional method that returns the modification time (an int) for the
- specified path, where path is a str.
-
- Raises IOError when the path cannot be handled.
- """
- raise IOError
-
- def path_stats(self, path):
- """Optional method returning a metadata dict for the specified path
- to by the path (str).
- Possible keys:
- - 'mtime' (mandatory) is the numeric timestamp of last source
- code modification;
- - 'size' (optional) is the size in bytes of the source code.
-
- Implementing this method allows the loader to read bytecode files.
- Raises IOError when the path cannot be handled.
- """
- return {'mtime': self.path_mtime(path)}
-
- def _cache_bytecode(self, source_path, cache_path, data):
- """Optional method which writes data (bytes) to a file path (a str).
-
- Implementing this method allows for the writing of bytecode files.
-
- The source path is needed in order to correctly transfer permissions
- """
- # For backwards compatibility, we delegate to set_data()
- return self.set_data(cache_path, data)
-
- def set_data(self, path, data):
- """Optional method which writes data (bytes) to a file path (a str).
-
- Implementing this method allows for the writing of bytecode files.
- """
-
-
- def get_source(self, fullname):
- """Concrete implementation of InspectLoader.get_source."""
- path = self.get_filename(fullname)
- try:
- source_bytes = self.get_data(path)
- except OSError as exc:
- raise ImportError('source not available through get_data()',
- name=fullname) from exc
- return decode_source(source_bytes)
-
- def source_to_code(self, data, path, *, _optimize=-1):
- """Return the code object compiled from source.
-
- The 'data' argument can be any object type that compile() supports.
- """
- return _call_with_frames_removed(compile, data, path, 'exec',
- dont_inherit=True, optimize=_optimize)
-
- def get_code(self, fullname):
- """Concrete implementation of InspectLoader.get_code.
-
- Reading of bytecode requires path_stats to be implemented. To write
- bytecode, set_data must also be implemented.
-
- """
- source_path = self.get_filename(fullname)
- source_mtime = None
- try:
- bytecode_path = cache_from_source(source_path)
- except NotImplementedError:
- bytecode_path = None
- else:
- try:
- st = self.path_stats(source_path)
- except IOError:
- pass
- else:
- source_mtime = int(st['mtime'])
- try:
- data = self.get_data(bytecode_path)
- except OSError:
- pass
- else:
- try:
- bytes_data = _validate_bytecode_header(data,
- source_stats=st, name=fullname,
- path=bytecode_path)
- except (ImportError, EOFError):
- pass
- else:
- _verbose_message('{} matches {}', bytecode_path,
- source_path)
- return _compile_bytecode(bytes_data, name=fullname,
- bytecode_path=bytecode_path,
- source_path=source_path)
- source_bytes = self.get_data(source_path)
- code_object = self.source_to_code(source_bytes, source_path)
- _verbose_message('code object from {}', source_path)
- if (not sys.dont_write_bytecode and bytecode_path is not None and
- source_mtime is not None):
- data = _code_to_bytecode(code_object, source_mtime,
- len(source_bytes))
- try:
- self._cache_bytecode(source_path, bytecode_path, data)
- _verbose_message('wrote {!r}', bytecode_path)
- except NotImplementedError:
- pass
- return code_object
-
-
-class FileLoader:
-
- """Base file loader class which implements the loader protocol methods that
- require file system usage."""
-
- def __init__(self, fullname, path):
- """Cache the module name and the path to the file found by the
- finder."""
- self.name = fullname
- self.path = path
-
- def __eq__(self, other):
- return (self.__class__ == other.__class__ and
- self.__dict__ == other.__dict__)
-
- def __hash__(self):
- return hash(self.name) ^ hash(self.path)
-
- @_check_name
- def load_module(self, fullname):
- """Load a module from a file.
-
- This method is deprecated. Use exec_module() instead.
-
- """
- # The only reason for this method is for the name check.
- # Issue #14857: Avoid the zero-argument form of super so the implementation
- # of that form can be updated without breaking the frozen module
- return super(FileLoader, self).load_module(fullname)
-
- @_check_name
- def get_filename(self, fullname):
- """Return the path to the source file as found by the finder."""
- return self.path
-
- def get_data(self, path):
- """Return the data from path as raw bytes."""
- with _io.FileIO(path, 'r') as file:
- return file.read()
-
-
-class SourceFileLoader(FileLoader, SourceLoader):
-
- """Concrete implementation of SourceLoader using the file system."""
-
- def path_stats(self, path):
- """Return the metadata for the path."""
- st = _path_stat(path)
- return {'mtime': st.st_mtime, 'size': st.st_size}
-
- def _cache_bytecode(self, source_path, bytecode_path, data):
- # Adapt between the two APIs
- mode = _calc_mode(source_path)
- return self.set_data(bytecode_path, data, _mode=mode)
-
- def set_data(self, path, data, *, _mode=0o666):
- """Write bytes data to a file."""
- parent, filename = _path_split(path)
- path_parts = []
- # Figure out what directories are missing.
- while parent and not _path_isdir(parent):
- parent, part = _path_split(parent)
- path_parts.append(part)
- # Create needed directories.
- for part in reversed(path_parts):
- parent = _path_join(parent, part)
- try:
- _os.mkdir(parent)
- except FileExistsError:
- # Probably another Python process already created the dir.
- continue
- except OSError as exc:
- # Could be a permission error, read-only filesystem: just forget
- # about writing the data.
- _verbose_message('could not create {!r}: {!r}', parent, exc)
- return
- try:
- _write_atomic(path, data, _mode)
- _verbose_message('created {!r}', path)
- except OSError as exc:
- # Same as above: just don't write the bytecode.
- _verbose_message('could not create {!r}: {!r}', path, exc)
-
-
-class SourcelessFileLoader(FileLoader, _LoaderBasics):
-
- """Loader which handles sourceless file imports."""
-
- def get_code(self, fullname):
- path = self.get_filename(fullname)
- data = self.get_data(path)
- bytes_data = _validate_bytecode_header(data, name=fullname, path=path)
- return _compile_bytecode(bytes_data, name=fullname, bytecode_path=path)
-
- def get_source(self, fullname):
- """Return None as there is no source code."""
- return None
-
-
-# Filled in by _setup().
-EXTENSION_SUFFIXES = []
-
-
-class ExtensionFileLoader:
-
- """Loader for extension modules.
-
- The constructor is designed to work with FileFinder.
-
- """
-
- def __init__(self, name, path):
- self.name = name
- self.path = path
-
- def __eq__(self, other):
- return (self.__class__ == other.__class__ and
- self.__dict__ == other.__dict__)
-
- def __hash__(self):
- return hash(self.name) ^ hash(self.path)
-
- @_check_name
- def load_module(self, fullname):
- """Load an extension module."""
- # Once an exec_module() implementation is added we can also
- # add a deprecation warning here.
- with _ManageReload(fullname):
- module = _call_with_frames_removed(_imp.load_dynamic,
- fullname, self.path)
- _verbose_message('extension module loaded from {!r}', self.path)
- is_package = self.is_package(fullname)
- if is_package and not hasattr(module, '__path__'):
- module.__path__ = [_path_split(self.path)[0]]
- module.__loader__ = self
- module.__package__ = module.__name__
- if not is_package:
- module.__package__ = module.__package__.rpartition('.')[0]
- return module
-
- def is_package(self, fullname):
- """Return True if the extension module is a package."""
- file_name = _path_split(self.path)[1]
- return any(file_name == '__init__' + suffix
- for suffix in EXTENSION_SUFFIXES)
-
- def get_code(self, fullname):
- """Return None as an extension module cannot create a code object."""
- return None
-
- def get_source(self, fullname):
- """Return None as extension modules have no source code."""
- return None
-
- @_check_name
- def get_filename(self, fullname):
- """Return the path to the source file as found by the finder."""
- return self.path
-
-
-class _NamespacePath:
- """Represents a namespace package's path. It uses the module name
- to find its parent module, and from there it looks up the parent's
- __path__. When this changes, the module's own path is recomputed,
- using path_finder. For top-level modules, the parent module's path
- is sys.path."""
-
- def __init__(self, name, path, path_finder):
- self._name = name
- self._path = path
- self._last_parent_path = tuple(self._get_parent_path())
- self._path_finder = path_finder
-
- def _find_parent_path_names(self):
- """Returns a tuple of (parent-module-name, parent-path-attr-name)"""
- parent, dot, me = self._name.rpartition('.')
- if dot == '':
- # This is a top-level module. sys.path contains the parent path.
- return 'sys', 'path'
- # Not a top-level module. parent-module.__path__ contains the
- # parent path.
- return parent, '__path__'
-
- def _get_parent_path(self):
- parent_module_name, path_attr_name = self._find_parent_path_names()
- return getattr(sys.modules[parent_module_name], path_attr_name)
-
- def _recalculate(self):
- # If the parent's path has changed, recalculate _path
- parent_path = tuple(self._get_parent_path()) # Make a copy
- if parent_path != self._last_parent_path:
- spec = self._path_finder(self._name, parent_path)
- # Note that no changes are made if a loader is returned, but we
- # do remember the new parent path
- if spec is not None and spec.loader is None:
- if spec.submodule_search_locations:
- self._path = spec.submodule_search_locations
- self._last_parent_path = parent_path # Save the copy
- return self._path
-
- def __iter__(self):
- return iter(self._recalculate())
-
- def __len__(self):
- return len(self._recalculate())
-
- def __repr__(self):
- return '_NamespacePath({!r})'.format(self._path)
-
- def __contains__(self, item):
- return item in self._recalculate()
-
- def append(self, item):
- self._path.append(item)
-
-
-# We use this exclusively in init_module_attrs() for backward-compatibility.
-class _NamespaceLoader:
- def __init__(self, name, path, path_finder):
- self._path = _NamespacePath(name, path, path_finder)
-
- @classmethod
- def module_repr(cls, module):
- """Return repr for the module.
-
- The method is deprecated. The import machinery does the job itself.
-
- """
- return '<module {!r} (namespace)>'.format(module.__name__)
-
- def is_package(self, fullname):
- return True
-
- def get_source(self, fullname):
- return ''
-
- def get_code(self, fullname):
- return compile('', '<string>', 'exec', dont_inherit=True)
-
- def exec_module(self, module):
- pass
-
- def load_module(self, fullname):
- """Load a namespace module.
-
- This method is deprecated. Use exec_module() instead.
-
- """
- # The import system never calls this method.
- _verbose_message('namespace module loaded with path {!r}', self._path)
- return _load_module_shim(self, fullname)
-
-
-# Finders #####################################################################
-
-class PathFinder:
-
- """Meta path finder for sys.path and package __path__ attributes."""
-
- @classmethod
- def invalidate_caches(cls):
- """Call the invalidate_caches() method on all path entry finders
- stored in sys.path_importer_caches (where implemented)."""
- for finder in sys.path_importer_cache.values():
- if hasattr(finder, 'invalidate_caches'):
- finder.invalidate_caches()
-
- @classmethod
- def _path_hooks(cls, path):
- """Search sequence of hooks for a finder for 'path'.
-
- If 'hooks' is false then use sys.path_hooks.
-
- """
- if not sys.path_hooks:
- _warnings.warn('sys.path_hooks is empty', ImportWarning)
- for hook in sys.path_hooks:
- try:
- return hook(path)
- except ImportError:
- continue
- else:
- return None
-
- @classmethod
- def _path_importer_cache(cls, path):
- """Get the finder for the path entry from sys.path_importer_cache.
-
- If the path entry is not in the cache, find the appropriate finder
- and cache it. If no finder is available, store None.
-
- """
- if path == '':
- path = _os.getcwd()
- try:
- finder = sys.path_importer_cache[path]
- except KeyError:
- finder = cls._path_hooks(path)
- sys.path_importer_cache[path] = finder
- return finder
-
- @classmethod
- def _legacy_get_spec(cls, fullname, finder):
- # This would be a good place for a DeprecationWarning if
- # we ended up going that route.
- if hasattr(finder, 'find_loader'):
- loader, portions = finder.find_loader(fullname)
- else:
- loader = finder.find_module(fullname)
- portions = []
- if loader is not None:
- return spec_from_loader(fullname, loader)
- spec = ModuleSpec(fullname, None)
- spec.submodule_search_locations = portions
- return spec
-
- @classmethod
- def _get_spec(cls, fullname, path, target=None):
- """Find the loader or namespace_path for this module/package name."""
- # If this ends up being a namespace package, namespace_path is
- # the list of paths that will become its __path__
- namespace_path = []
- for entry in path:
- if not isinstance(entry, (str, bytes)):
- continue
- finder = cls._path_importer_cache(entry)
- if finder is not None:
- if hasattr(finder, 'find_spec'):
- spec = finder.find_spec(fullname, target)
- else:
- spec = cls._legacy_get_spec(fullname, finder)
- if spec is None:
- continue
- if spec.loader is not None:
- return spec
- portions = spec.submodule_search_locations
- if portions is None:
- raise ImportError('spec missing loader')
- # This is possibly part of a namespace package.
- # Remember these path entries (if any) for when we
- # create a namespace package, and continue iterating
- # on path.
- namespace_path.extend(portions)
- else:
- spec = ModuleSpec(fullname, None)
- spec.submodule_search_locations = namespace_path
- return spec
-
- @classmethod
- def find_spec(cls, fullname, path=None, target=None):
- """find the module on sys.path or 'path' based on sys.path_hooks and
- sys.path_importer_cache."""
- if path is None:
- path = sys.path
- spec = cls._get_spec(fullname, path, target)
- if spec is None:
- return None
- elif spec.loader is None:
- namespace_path = spec.submodule_search_locations
- if namespace_path:
- # We found at least one namespace path. Return a
- # spec which can create the namespace package.
- spec.origin = 'namespace'
- spec.submodule_search_locations = _NamespacePath(fullname, namespace_path, cls._get_spec)
- return spec
- else:
- return None
- else:
- return spec
-
- @classmethod
- def find_module(cls, fullname, path=None):
- """find the module on sys.path or 'path' based on sys.path_hooks and
- sys.path_importer_cache.
-
- This method is deprecated. Use find_spec() instead.
-
- """
- spec = cls.find_spec(fullname, path)
- if spec is None:
- return None
- return spec.loader
-
-
-class FileFinder:
-
- """File-based finder.
-
- Interactions with the file system are cached for performance, being
- refreshed when the directory the finder is handling has been modified.
-
- """
-
- def __init__(self, path, *loader_details):
- """Initialize with the path to search on and a variable number of
- 2-tuples containing the loader and the file suffixes the loader
- recognizes."""
- loaders = []
- for loader, suffixes in loader_details:
- loaders.extend((suffix, loader) for suffix in suffixes)
- self._loaders = loaders
- # Base (directory) path
- self.path = path or '.'
- self._path_mtime = -1
- self._path_cache = set()
- self._relaxed_path_cache = set()
-
- def invalidate_caches(self):
- """Invalidate the directory mtime."""
- self._path_mtime = -1
-
- find_module = _find_module_shim
-
- def find_loader(self, fullname):
- """Try to find a loader for the specified module, or the namespace
- package portions. Returns (loader, list-of-portions).
-
- This method is deprecated. Use find_spec() instead.
-
- """
- spec = self.find_spec(fullname)
- if spec is None:
- return None, []
- return spec.loader, spec.submodule_search_locations or []
-
- def _get_spec(self, loader_class, fullname, path, smsl, target):
- loader = loader_class(fullname, path)
- return spec_from_file_location(fullname, path, loader=loader,
- submodule_search_locations=smsl)
-
- def find_spec(self, fullname, target=None):
- """Try to find a loader for the specified module, or the namespace
- package portions. Returns (loader, list-of-portions)."""
- is_namespace = False
- tail_module = fullname.rpartition('.')[2]
- try:
- mtime = _path_stat(self.path or _os.getcwd()).st_mtime
- except OSError:
- mtime = -1
- if mtime != self._path_mtime:
- self._fill_cache()
- self._path_mtime = mtime
- # tail_module keeps the original casing, for __file__ and friends
- if _relax_case():
- cache = self._relaxed_path_cache
- cache_module = tail_module.lower()
- else:
- cache = self._path_cache
- cache_module = tail_module
- # Check if the module is the name of a directory (and thus a package).
- if cache_module in cache:
- base_path = _path_join(self.path, tail_module)
- for suffix, loader_class in self._loaders:
- init_filename = '__init__' + suffix
- full_path = _path_join(base_path, init_filename)
- if _path_isfile(full_path):
- return self._get_spec(loader_class, fullname, full_path, [base_path], target)
- else:
- # If a namespace package, return the path if we don't
- # find a module in the next section.
- is_namespace = _path_isdir(base_path)
- # Check for a file w/ a proper suffix exists.
- for suffix, loader_class in self._loaders:
- full_path = _path_join(self.path, tail_module + suffix)
- _verbose_message('trying {}'.format(full_path), verbosity=2)
- if cache_module + suffix in cache:
- if _path_isfile(full_path):
- return self._get_spec(loader_class, fullname, full_path, None, target)
- if is_namespace:
- _verbose_message('possible namespace for {}'.format(base_path))
- spec = ModuleSpec(fullname, None)
- spec.submodule_search_locations = [base_path]
- return spec
- return None
-
- def _fill_cache(self):
- """Fill the cache of potential modules and packages for this directory."""
- path = self.path
- try:
- contents = _os.listdir(path or _os.getcwd())
- except (FileNotFoundError, PermissionError, NotADirectoryError):
- # Directory has either been removed, turned into a file, or made
- # unreadable.
- contents = []
- # We store two cached versions, to handle runtime changes of the
- # PYTHONCASEOK environment variable.
- if not sys.platform.startswith('win'):
- self._path_cache = set(contents)
- else:
- # Windows users can import modules with case-insensitive file
- # suffixes (for legacy reasons). Make the suffix lowercase here
- # so it's done once instead of for every import. This is safe as
- # the specified suffixes to check against are always specified in a
- # case-sensitive manner.
- lower_suffix_contents = set()
- for item in contents:
- name, dot, suffix = item.partition('.')
- if dot:
- new_name = '{}.{}'.format(name, suffix.lower())
- else:
- new_name = name
- lower_suffix_contents.add(new_name)
- self._path_cache = lower_suffix_contents
- if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
- self._relaxed_path_cache = {fn.lower() for fn in contents}
-
- @classmethod
- def path_hook(cls, *loader_details):
- """A class method which returns a closure to use on sys.path_hook
- which will return an instance using the specified loaders and the path
- called on the closure.
-
- If the path called on the closure is not a directory, ImportError is
- raised.
-
- """
- def path_hook_for_FileFinder(path):
- """Path hook for importlib.machinery.FileFinder."""
- if not _path_isdir(path):
- raise ImportError('only directories are supported', path=path)
- return cls(path, *loader_details)
-
- return path_hook_for_FileFinder
-
- def __repr__(self):
- return 'FileFinder({!r})'.format(self.path)
-
-
# Import itself ###############################################################
class _ImportLockContext:
@@ -2146,7 +878,7 @@ def _find_spec_legacy(finder, name, path):
def _find_spec(name, path, target=None):
"""Find a module's loader."""
- if not sys.meta_path:
+ if sys.meta_path is not None and not sys.meta_path:
_warnings.warn('sys.meta_path is empty', ImportWarning)
# We check sys.modules here for the reload case. While a passed-in
# target will usually indicate a reload there is no guarantee, whereas
@@ -2218,12 +950,12 @@ def _find_and_load_unlocked(name, import_):
path = parent_module.__path__
except AttributeError:
msg = (_ERR_MSG + '; {!r} is not a package').format(name, parent)
- raise ImportError(msg, name=name)
+ raise ImportError(msg, name=name) from None
spec = _find_spec(name, path)
if spec is None:
raise ImportError(_ERR_MSG.format(name), name=name)
else:
- module = _SpecMethods(spec)._load_unlocked()
+ module = _load_unlocked(spec)
if parent:
# Set the module as an attribute on its parent.
parent_module = sys.modules[parent]
@@ -2308,17 +1040,6 @@ def _calc___package__(globals):
return package
-def _get_supported_file_loaders():
- """Returns a list of file-based module loaders.
-
- Each item is a tuple (loader, suffixes).
- """
- extensions = ExtensionFileLoader, _imp.extension_suffixes()
- source = SourceFileLoader, SOURCE_SUFFIXES
- bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES
- return [extensions, source, bytecode]
-
-
def __import__(name, globals=None, locals=None, fromlist=(), level=0):
"""Import a module.
@@ -2358,8 +1079,7 @@ def _builtin_from_name(name):
spec = BuiltinImporter.find_spec(name)
if spec is None:
raise ImportError('no built-in module named ' + name)
- methods = _SpecMethods(spec)
- return methods._load_unlocked()
+ return _load_unlocked(spec)
def _setup(sys_module, _imp_module):
@@ -2370,15 +1090,10 @@ def _setup(sys_module, _imp_module):
modules, those two modules must be explicitly passed in.
"""
- global _imp, sys, BYTECODE_SUFFIXES
+ global _imp, sys
_imp = _imp_module
sys = sys_module
- if sys.flags.optimize:
- BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES
- else:
- BYTECODE_SUFFIXES = DEBUG_BYTECODE_SUFFIXES
-
# Set up the spec for existing builtin/frozen modules.
module_type = type(sys)
for name, module in sys.modules.items():
@@ -2390,39 +1105,17 @@ def _setup(sys_module, _imp_module):
else:
continue
spec = _spec_from_module(module, loader)
- methods = _SpecMethods(spec)
- methods.init_module_attrs(module)
+ _init_module_attrs(spec, module)
# Directly load built-in modules needed during bootstrap.
self_module = sys.modules[__name__]
- for builtin_name in ('_io', '_warnings', 'builtins', 'marshal'):
+ for builtin_name in ('_warnings',):
if builtin_name not in sys.modules:
builtin_module = _builtin_from_name(builtin_name)
else:
builtin_module = sys.modules[builtin_name]
setattr(self_module, builtin_name, builtin_module)
- # Directly load the os module (needed during bootstrap).
- os_details = ('posix', ['/']), ('nt', ['\\', '/'])
- for builtin_os, path_separators in os_details:
- # Assumption made in _path_join()
- assert all(len(sep) == 1 for sep in path_separators)
- path_sep = path_separators[0]
- if builtin_os in sys.modules:
- os_module = sys.modules[builtin_os]
- break
- else:
- try:
- os_module = _builtin_from_name(builtin_os)
- break
- except ImportError:
- continue
- else:
- raise ImportError('importlib requires posix or nt')
- setattr(self_module, '_os', os_module)
- setattr(self_module, 'path_sep', path_sep)
- setattr(self_module, 'path_separators', ''.join(path_separators))
-
# Directly load the _thread module (needed during bootstrap).
try:
thread_module = _builtin_from_name('_thread')
@@ -2435,27 +1128,15 @@ def _setup(sys_module, _imp_module):
weakref_module = _builtin_from_name('_weakref')
setattr(self_module, '_weakref', weakref_module)
- # Directly load the winreg module (needed during bootstrap).
- if builtin_os == 'nt':
- winreg_module = _builtin_from_name('winreg')
- setattr(self_module, '_winreg', winreg_module)
-
- # Constants
- setattr(self_module, '_relax_case', _make_relax_case())
- EXTENSION_SUFFIXES.extend(_imp.extension_suffixes())
- if builtin_os == 'nt':
- SOURCE_SUFFIXES.append('.pyw')
- if '_d.pyd' in EXTENSION_SUFFIXES:
- WindowsRegistryFinder.DEBUG_BUILD = True
-
def _install(sys_module, _imp_module):
"""Install importlib as the implementation of import."""
_setup(sys_module, _imp_module)
- supported_loaders = _get_supported_file_loaders()
- sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)])
+
sys.meta_path.append(BuiltinImporter)
sys.meta_path.append(FrozenImporter)
- if _os.__name__ == 'nt':
- sys.meta_path.append(WindowsRegistryFinder)
- sys.meta_path.append(PathFinder)
+
+ global _bootstrap_external
+ import _frozen_importlib_external
+ _bootstrap_external = _frozen_importlib_external
+ _frozen_importlib_external._install(sys.modules[__name__])
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
new file mode 100644
index 0000000..3508ce9
--- /dev/null
+++ b/Lib/importlib/_bootstrap_external.py
@@ -0,0 +1,1426 @@
+"""Core implementation of path-based import.
+
+This module is NOT meant to be directly imported! It has been designed such
+that it can be bootstrapped into Python as the implementation of import. As
+such it requires the injection of specific modules and attributes in order to
+work. One should use importlib as the public-facing version of this module.
+
+"""
+#
+# IMPORTANT: Whenever making changes to this module, be sure to run
+# a top-level make in order to get the frozen version of the module
+# updated. Not doing so will result in the Makefile to fail for
+# all others who don't have a ./python around to freeze the module
+# in the early stages of compilation.
+#
+
+# See importlib._setup() for what is injected into the global namespace.
+
+# When editing this code be aware that code executed at import time CANNOT
+# reference any injected objects! This includes not only global code but also
+# anything specified at the class level.
+
+# Bootstrap-related code ######################################################
+
+_CASE_INSENSITIVE_PLATFORMS = 'win', 'cygwin', 'darwin'
+
+
+def _make_relax_case():
+ if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
+ def _relax_case():
+ """True if filenames must be checked case-insensitively."""
+ return b'PYTHONCASEOK' in _os.environ
+ else:
+ def _relax_case():
+ """True if filenames must be checked case-insensitively."""
+ return False
+ return _relax_case
+
+
+def _w_long(x):
+ """Convert a 32-bit integer to little-endian."""
+ return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little')
+
+
+def _r_long(int_bytes):
+ """Convert 4 bytes in little-endian to an integer."""
+ return int.from_bytes(int_bytes, 'little')
+
+
+def _path_join(*path_parts):
+ """Replacement for os.path.join()."""
+ return path_sep.join([part.rstrip(path_separators)
+ for part in path_parts if part])
+
+
+def _path_split(path):
+ """Replacement for os.path.split()."""
+ if len(path_separators) == 1:
+ front, _, tail = path.rpartition(path_sep)
+ return front, tail
+ for x in reversed(path):
+ if x in path_separators:
+ front, tail = path.rsplit(x, maxsplit=1)
+ return front, tail
+ return '', path
+
+
+def _path_stat(path):
+ """Stat the path.
+
+ Made a separate function to make it easier to override in experiments
+ (e.g. cache stat results).
+
+ """
+ return _os.stat(path)
+
+
+def _path_is_mode_type(path, mode):
+ """Test whether the path is the specified mode type."""
+ try:
+ stat_info = _path_stat(path)
+ except OSError:
+ return False
+ return (stat_info.st_mode & 0o170000) == mode
+
+
+def _path_isfile(path):
+ """Replacement for os.path.isfile."""
+ return _path_is_mode_type(path, 0o100000)
+
+
+def _path_isdir(path):
+ """Replacement for os.path.isdir."""
+ if not path:
+ path = _os.getcwd()
+ return _path_is_mode_type(path, 0o040000)
+
+
+def _write_atomic(path, data, mode=0o666):
+ """Best-effort function to write data to a path atomically.
+ Be prepared to handle a FileExistsError if concurrent writing of the
+ temporary file is attempted."""
+ # id() is used to generate a pseudo-random filename.
+ path_tmp = '{}.{}'.format(path, id(path))
+ fd = _os.open(path_tmp,
+ _os.O_EXCL | _os.O_CREAT | _os.O_WRONLY, mode & 0o666)
+ try:
+ # We first write data to a temporary file, and then use os.replace() to
+ # perform an atomic rename.
+ with _io.FileIO(fd, 'wb') as file:
+ file.write(data)
+ _os.replace(path_tmp, path)
+ except OSError:
+ try:
+ _os.unlink(path_tmp)
+ except OSError:
+ pass
+ raise
+
+
+_code_type = type(_write_atomic.__code__)
+
+
+# Finder/loader utility code ###############################################
+
+# Magic word to reject .pyc files generated by other Python versions.
+# It should change for each incompatible change to the bytecode.
+#
+# The value of CR and LF is incorporated so if you ever read or write
+# a .pyc file in text mode the magic number will be wrong; also, the
+# Apple MPW compiler swaps their values, botching string constants.
+#
+# The magic numbers must be spaced apart at least 2 values, as the
+# -U interpeter flag will cause MAGIC+1 being used. They have been
+# odd numbers for some time now.
+#
+# There were a variety of old schemes for setting the magic number.
+# The current working scheme is to increment the previous value by
+# 10.
+#
+# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic
+# number also includes a new "magic tag", i.e. a human readable string used
+# to represent the magic number in __pycache__ directories. When you change
+# the magic number, you must also set a new unique magic tag. Generally this
+# can be named after the Python major version of the magic number bump, but
+# it can really be anything, as long as it's different than anything else
+# that's come before. The tags are included in the following table, starting
+# with Python 3.2a0.
+#
+# Known values:
+# Python 1.5: 20121
+# Python 1.5.1: 20121
+# Python 1.5.2: 20121
+# Python 1.6: 50428
+# Python 2.0: 50823
+# Python 2.0.1: 50823
+# Python 2.1: 60202
+# Python 2.1.1: 60202
+# Python 2.1.2: 60202
+# Python 2.2: 60717
+# Python 2.3a0: 62011
+# Python 2.3a0: 62021
+# Python 2.3a0: 62011 (!)
+# Python 2.4a0: 62041
+# Python 2.4a3: 62051
+# Python 2.4b1: 62061
+# Python 2.5a0: 62071
+# Python 2.5a0: 62081 (ast-branch)
+# Python 2.5a0: 62091 (with)
+# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode)
+# Python 2.5b3: 62101 (fix wrong code: for x, in ...)
+# Python 2.5b3: 62111 (fix wrong code: x += yield)
+# Python 2.5c1: 62121 (fix wrong lnotab with for loops and
+# storing constants that should have been removed)
+# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp)
+# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode)
+# Python 2.6a1: 62161 (WITH_CLEANUP optimization)
+# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND)
+# Python 2.7a0: 62181 (optimize conditional branches:
+# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
+# Python 2.7a0 62191 (introduce SETUP_WITH)
+# Python 2.7a0 62201 (introduce BUILD_SET)
+# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD)
+# Python 3000: 3000
+# 3010 (removed UNARY_CONVERT)
+# 3020 (added BUILD_SET)
+# 3030 (added keyword-only parameters)
+# 3040 (added signature annotations)
+# 3050 (print becomes a function)
+# 3060 (PEP 3115 metaclass syntax)
+# 3061 (string literals become unicode)
+# 3071 (PEP 3109 raise changes)
+# 3081 (PEP 3137 make __file__ and __name__ unicode)
+# 3091 (kill str8 interning)
+# 3101 (merge from 2.6a0, see 62151)
+# 3103 (__file__ points to source file)
+# Python 3.0a4: 3111 (WITH_CLEANUP optimization).
+# Python 3.0a5: 3131 (lexical exception stacking, including POP_EXCEPT)
+# Python 3.1a0: 3141 (optimize list, set and dict comprehensions:
+# change LIST_APPEND and SET_ADD, add MAP_ADD)
+# Python 3.1a0: 3151 (optimize conditional branches:
+# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
+# Python 3.2a0: 3160 (add SETUP_WITH)
+# tag: cpython-32
+# Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR)
+# tag: cpython-32
+# Python 3.2a2 3180 (add DELETE_DEREF)
+# Python 3.3a0 3190 __class__ super closure changed
+# Python 3.3a0 3200 (__qualname__ added)
+# 3210 (added size modulo 2**32 to the pyc header)
+# Python 3.3a1 3220 (changed PEP 380 implementation)
+# Python 3.3a4 3230 (revert changes to implicit __class__ closure)
+# Python 3.4a1 3250 (evaluate positional default arguments before
+# keyword-only defaults)
+# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override
+# free vars)
+# Python 3.4a1 3270 (various tweaks to the __class__ closure)
+# Python 3.4a1 3280 (remove implicit class argument)
+# Python 3.4a4 3290 (changes to __qualname__ computation)
+# Python 3.4a4 3300 (more changes to __qualname__ computation)
+# Python 3.4rc2 3310 (alter __qualname__ computation)
+# Python 3.5a0 3320 (matrix multiplication operator)
+# Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations)
+# Python 3.5b2 3340 (fix dictionary display evaluation order #11205)
+# Python 3.5b2 3350 (add GET_YIELD_FROM_ITER opcode #24400)
+#
+# MAGIC must change whenever the bytecode emitted by the compiler may no
+# longer be understood by older implementations of the eval loop (usually
+# due to the addition of new opcodes).
+
+MAGIC_NUMBER = (3350).to_bytes(2, 'little') + b'\r\n'
+_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
+
+_PYCACHE = '__pycache__'
+_OPT = 'opt-'
+
+SOURCE_SUFFIXES = ['.py'] # _setup() adds .pyw as needed.
+
+BYTECODE_SUFFIXES = ['.pyc']
+# Deprecated.
+DEBUG_BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES = BYTECODE_SUFFIXES
+
+def cache_from_source(path, debug_override=None, *, optimization=None):
+ """Given the path to a .py file, return the path to its .pyc file.
+
+ The .py file does not need to exist; this simply returns the path to the
+ .pyc file calculated as if the .py file were imported.
+
+ The 'optimization' parameter controls the presumed optimization level of
+ the bytecode file. If 'optimization' is not None, the string representation
+ of the argument is taken and verified to be alphanumeric (else ValueError
+ is raised).
+
+ The debug_override parameter is deprecated. If debug_override is not None,
+ a True value is the same as setting 'optimization' to the empty string
+ while a False value is equivalent to setting 'optimization' to '1'.
+
+ If sys.implementation.cache_tag is None then NotImplementedError is raised.
+
+ """
+ if debug_override is not None:
+ _warnings.warn('the debug_override parameter is deprecated; use '
+ "'optimization' instead", DeprecationWarning)
+ if optimization is not None:
+ message = 'debug_override or optimization must be set to None'
+ raise TypeError(message)
+ optimization = '' if debug_override else 1
+ head, tail = _path_split(path)
+ base, sep, rest = tail.rpartition('.')
+ tag = sys.implementation.cache_tag
+ if tag is None:
+ raise NotImplementedError('sys.implementation.cache_tag is None')
+ almost_filename = ''.join([(base if base else rest), sep, tag])
+ if optimization is None:
+ if sys.flags.optimize == 0:
+ optimization = ''
+ else:
+ optimization = sys.flags.optimize
+ optimization = str(optimization)
+ if optimization != '':
+ if not optimization.isalnum():
+ raise ValueError('{!r} is not alphanumeric'.format(optimization))
+ almost_filename = '{}.{}{}'.format(almost_filename, _OPT, optimization)
+ return _path_join(head, _PYCACHE, almost_filename + BYTECODE_SUFFIXES[0])
+
+
+def source_from_cache(path):
+ """Given the path to a .pyc. file, return the path to its .py file.
+
+ The .pyc file does not need to exist; this simply returns the path to
+ the .py file calculated to correspond to the .pyc file. If path does
+ not conform to PEP 3147/488 format, ValueError will be raised. If
+ sys.implementation.cache_tag is None then NotImplementedError is raised.
+
+ """
+ if sys.implementation.cache_tag is None:
+ raise NotImplementedError('sys.implementation.cache_tag is None')
+ head, pycache_filename = _path_split(path)
+ head, pycache = _path_split(head)
+ if pycache != _PYCACHE:
+ raise ValueError('{} not bottom-level directory in '
+ '{!r}'.format(_PYCACHE, path))
+ dot_count = pycache_filename.count('.')
+ if dot_count not in {2, 3}:
+ raise ValueError('expected only 2 or 3 dots in '
+ '{!r}'.format(pycache_filename))
+ elif dot_count == 3:
+ optimization = pycache_filename.rsplit('.', 2)[-2]
+ if not optimization.startswith(_OPT):
+ raise ValueError("optimization portion of filename does not start "
+ "with {!r}".format(_OPT))
+ opt_level = optimization[len(_OPT):]
+ if not opt_level.isalnum():
+ raise ValueError("optimization level {!r} is not an alphanumeric "
+ "value".format(optimization))
+ base_filename = pycache_filename.partition('.')[0]
+ return _path_join(head, base_filename + SOURCE_SUFFIXES[0])
+
+
+def _get_sourcefile(bytecode_path):
+ """Convert a bytecode file path to a source path (if possible).
+
+ This function exists purely for backwards-compatibility for
+ PyImport_ExecCodeModuleWithFilenames() in the C API.
+
+ """
+ if len(bytecode_path) == 0:
+ return None
+ rest, _, extension = bytecode_path.rpartition('.')
+ if not rest or extension.lower()[-3:-1] != 'py':
+ return bytecode_path
+ try:
+ source_path = source_from_cache(bytecode_path)
+ except (NotImplementedError, ValueError):
+ source_path = bytecode_path[:-1]
+ return source_path if _path_isfile(source_path) else bytecode_path
+
+
+def _get_cached(filename):
+ if filename.endswith(tuple(SOURCE_SUFFIXES)):
+ try:
+ return cache_from_source(filename)
+ except NotImplementedError:
+ pass
+ elif filename.endswith(tuple(BYTECODE_SUFFIXES)):
+ return filename
+ else:
+ return None
+
+
+def _calc_mode(path):
+ """Calculate the mode permissions for a bytecode file."""
+ try:
+ mode = _path_stat(path).st_mode
+ except OSError:
+ mode = 0o666
+ # We always ensure write access so we can update cached files
+ # later even when the source files are read-only on Windows (#6074)
+ mode |= 0o200
+ return mode
+
+
+def _verbose_message(message, *args, verbosity=1):
+ """Print the message to stderr if -v/PYTHONVERBOSE is turned on."""
+ if sys.flags.verbose >= verbosity:
+ if not message.startswith(('#', 'import ')):
+ message = '# ' + message
+ print(message.format(*args), file=sys.stderr)
+
+
+def _check_name(method):
+ """Decorator to verify that the module being requested matches the one the
+ loader can handle.
+
+ The first argument (self) must define _name which the second argument is
+ compared against. If the comparison fails then ImportError is raised.
+
+ """
+ def _check_name_wrapper(self, name=None, *args, **kwargs):
+ if name is None:
+ name = self.name
+ elif self.name != name:
+ raise ImportError('loader for %s cannot handle %s' %
+ (self.name, name), name=name)
+ return method(self, name, *args, **kwargs)
+ try:
+ _wrap = _bootstrap._wrap
+ except NameError:
+ # XXX yuck
+ def _wrap(new, old):
+ for replace in ['__module__', '__name__', '__qualname__', '__doc__']:
+ if hasattr(old, replace):
+ setattr(new, replace, getattr(old, replace))
+ new.__dict__.update(old.__dict__)
+ _wrap(_check_name_wrapper, method)
+ return _check_name_wrapper
+
+
+def _find_module_shim(self, fullname):
+ """Try to find a loader for the specified module by delegating to
+ self.find_loader().
+
+ This method is deprecated in favor of finder.find_spec().
+
+ """
+ # Call find_loader(). If it returns a string (indicating this
+ # is a namespace package portion), generate a warning and
+ # return None.
+ loader, portions = self.find_loader(fullname)
+ if loader is None and len(portions):
+ msg = 'Not importing directory {}: missing __init__'
+ _warnings.warn(msg.format(portions[0]), ImportWarning)
+ return loader
+
+
+def _validate_bytecode_header(data, source_stats=None, name=None, path=None):
+ """Validate the header of the passed-in bytecode against source_stats (if
+ given) and returning the bytecode that can be compiled by compile().
+
+ All other arguments are used to enhance error reporting.
+
+ ImportError is raised when the magic number is incorrect or the bytecode is
+ found to be stale. EOFError is raised when the data is found to be
+ truncated.
+
+ """
+ exc_details = {}
+ if name is not None:
+ exc_details['name'] = name
+ else:
+ # To prevent having to make all messages have a conditional name.
+ name = '<bytecode>'
+ if path is not None:
+ exc_details['path'] = path
+ magic = data[:4]
+ raw_timestamp = data[4:8]
+ raw_size = data[8:12]
+ if magic != MAGIC_NUMBER:
+ message = 'bad magic number in {!r}: {!r}'.format(name, magic)
+ _verbose_message(message)
+ raise ImportError(message, **exc_details)
+ elif len(raw_timestamp) != 4:
+ message = 'reached EOF while reading timestamp in {!r}'.format(name)
+ _verbose_message(message)
+ raise EOFError(message)
+ elif len(raw_size) != 4:
+ message = 'reached EOF while reading size of source in {!r}'.format(name)
+ _verbose_message(message)
+ raise EOFError(message)
+ if source_stats is not None:
+ try:
+ source_mtime = int(source_stats['mtime'])
+ except KeyError:
+ pass
+ else:
+ if _r_long(raw_timestamp) != source_mtime:
+ message = 'bytecode is stale for {!r}'.format(name)
+ _verbose_message(message)
+ raise ImportError(message, **exc_details)
+ try:
+ source_size = source_stats['size'] & 0xFFFFFFFF
+ except KeyError:
+ pass
+ else:
+ if _r_long(raw_size) != source_size:
+ raise ImportError('bytecode is stale for {!r}'.format(name),
+ **exc_details)
+ return data[12:]
+
+
+def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None):
+ """Compile bytecode as returned by _validate_bytecode_header()."""
+ code = marshal.loads(data)
+ if isinstance(code, _code_type):
+ _verbose_message('code object from {!r}', bytecode_path)
+ if source_path is not None:
+ _imp._fix_co_filename(code, source_path)
+ return code
+ else:
+ raise ImportError('Non-code object in {!r}'.format(bytecode_path),
+ name=name, path=bytecode_path)
+
+def _code_to_bytecode(code, mtime=0, source_size=0):
+ """Compile a code object into bytecode for writing out to a byte-compiled
+ file."""
+ data = bytearray(MAGIC_NUMBER)
+ data.extend(_w_long(mtime))
+ data.extend(_w_long(source_size))
+ data.extend(marshal.dumps(code))
+ return data
+
+
+def decode_source(source_bytes):
+ """Decode bytes representing source code and return the string.
+
+ Universal newline support is used in the decoding.
+ """
+ import tokenize # To avoid bootstrap issues.
+ source_bytes_readline = _io.BytesIO(source_bytes).readline
+ encoding = tokenize.detect_encoding(source_bytes_readline)
+ newline_decoder = _io.IncrementalNewlineDecoder(None, True)
+ return newline_decoder.decode(source_bytes.decode(encoding[0]))
+
+
+# Module specifications #######################################################
+
+_POPULATE = object()
+
+
+def spec_from_file_location(name, location=None, *, loader=None,
+ submodule_search_locations=_POPULATE):
+ """Return a module spec based on a file location.
+
+ To indicate that the module is a package, set
+ submodule_search_locations to a list of directory paths. An
+ empty list is sufficient, though its not otherwise useful to the
+ import system.
+
+ The loader must take a spec as its only __init__() arg.
+
+ """
+ if location is None:
+ # The caller may simply want a partially populated location-
+ # oriented spec. So we set the location to a bogus value and
+ # fill in as much as we can.
+ location = '<unknown>'
+ if hasattr(loader, 'get_filename'):
+ # ExecutionLoader
+ try:
+ location = loader.get_filename(name)
+ except ImportError:
+ pass
+
+ # If the location is on the filesystem, but doesn't actually exist,
+ # we could return None here, indicating that the location is not
+ # valid. However, we don't have a good way of testing since an
+ # indirect location (e.g. a zip file or URL) will look like a
+ # non-existent file relative to the filesystem.
+
+ spec = _bootstrap.ModuleSpec(name, loader, origin=location)
+ spec._set_fileattr = True
+
+ # Pick a loader if one wasn't provided.
+ if loader is None:
+ for loader_class, suffixes in _get_supported_file_loaders():
+ if location.endswith(tuple(suffixes)):
+ loader = loader_class(name, location)
+ spec.loader = loader
+ break
+ else:
+ return None
+
+ # Set submodule_search_paths appropriately.
+ if submodule_search_locations is _POPULATE:
+ # Check the loader.
+ if hasattr(loader, 'is_package'):
+ try:
+ is_package = loader.is_package(name)
+ except ImportError:
+ pass
+ else:
+ if is_package:
+ spec.submodule_search_locations = []
+ else:
+ spec.submodule_search_locations = submodule_search_locations
+ if spec.submodule_search_locations == []:
+ if location:
+ dirname = _path_split(location)[0]
+ spec.submodule_search_locations.append(dirname)
+
+ return spec
+
+
+# Loaders #####################################################################
+
+class WindowsRegistryFinder:
+
+ """Meta path finder for modules declared in the Windows registry."""
+
+ REGISTRY_KEY = (
+ 'Software\\Python\\PythonCore\\{sys_version}'
+ '\\Modules\\{fullname}')
+ REGISTRY_KEY_DEBUG = (
+ 'Software\\Python\\PythonCore\\{sys_version}'
+ '\\Modules\\{fullname}\\Debug')
+ DEBUG_BUILD = False # Changed in _setup()
+
+ @classmethod
+ def _open_registry(cls, key):
+ try:
+ return _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, key)
+ except OSError:
+ return _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, key)
+
+ @classmethod
+ def _search_registry(cls, fullname):
+ if cls.DEBUG_BUILD:
+ registry_key = cls.REGISTRY_KEY_DEBUG
+ else:
+ registry_key = cls.REGISTRY_KEY
+ key = registry_key.format(fullname=fullname,
+ sys_version=sys.version[:3])
+ try:
+ with cls._open_registry(key) as hkey:
+ filepath = _winreg.QueryValue(hkey, '')
+ except OSError:
+ return None
+ return filepath
+
+ @classmethod
+ def find_spec(cls, fullname, path=None, target=None):
+ filepath = cls._search_registry(fullname)
+ if filepath is None:
+ return None
+ try:
+ _path_stat(filepath)
+ except OSError:
+ return None
+ for loader, suffixes in _get_supported_file_loaders():
+ if filepath.endswith(tuple(suffixes)):
+ spec = _bootstrap.spec_from_loader(fullname,
+ loader(fullname, filepath),
+ origin=filepath)
+ return spec
+
+ @classmethod
+ def find_module(cls, fullname, path=None):
+ """Find module named in the registry.
+
+ This method is deprecated. Use exec_module() instead.
+
+ """
+ spec = cls.find_spec(fullname, path)
+ if spec is not None:
+ return spec.loader
+ else:
+ return None
+
+
+class _LoaderBasics:
+
+ """Base class of common code needed by both SourceLoader and
+ SourcelessFileLoader."""
+
+ def is_package(self, fullname):
+ """Concrete implementation of InspectLoader.is_package by checking if
+ the path returned by get_filename has a filename of '__init__.py'."""
+ filename = _path_split(self.get_filename(fullname))[1]
+ filename_base = filename.rsplit('.', 1)[0]
+ tail_name = fullname.rpartition('.')[2]
+ return filename_base == '__init__' and tail_name != '__init__'
+
+ def create_module(self, spec):
+ """Use default semantics for module creation."""
+
+ def exec_module(self, module):
+ """Execute the module."""
+ code = self.get_code(module.__name__)
+ if code is None:
+ raise ImportError('cannot load module {!r} when get_code() '
+ 'returns None'.format(module.__name__))
+ _bootstrap._call_with_frames_removed(exec, code, module.__dict__)
+
+ def load_module(self, fullname):
+ return _bootstrap._load_module_shim(self, fullname)
+
+
+class SourceLoader(_LoaderBasics):
+
+ def path_mtime(self, path):
+ """Optional method that returns the modification time (an int) for the
+ specified path, where path is a str.
+
+ Raises IOError when the path cannot be handled.
+ """
+ raise IOError
+
+ def path_stats(self, path):
+ """Optional method returning a metadata dict for the specified path
+ to by the path (str).
+ Possible keys:
+ - 'mtime' (mandatory) is the numeric timestamp of last source
+ code modification;
+ - 'size' (optional) is the size in bytes of the source code.
+
+ Implementing this method allows the loader to read bytecode files.
+ Raises IOError when the path cannot be handled.
+ """
+ return {'mtime': self.path_mtime(path)}
+
+ def _cache_bytecode(self, source_path, cache_path, data):
+ """Optional method which writes data (bytes) to a file path (a str).
+
+ Implementing this method allows for the writing of bytecode files.
+
+ The source path is needed in order to correctly transfer permissions
+ """
+ # For backwards compatibility, we delegate to set_data()
+ return self.set_data(cache_path, data)
+
+ def set_data(self, path, data):
+ """Optional method which writes data (bytes) to a file path (a str).
+
+ Implementing this method allows for the writing of bytecode files.
+ """
+
+
+ def get_source(self, fullname):
+ """Concrete implementation of InspectLoader.get_source."""
+ path = self.get_filename(fullname)
+ try:
+ source_bytes = self.get_data(path)
+ except OSError as exc:
+ raise ImportError('source not available through get_data()',
+ name=fullname) from exc
+ return decode_source(source_bytes)
+
+ def source_to_code(self, data, path, *, _optimize=-1):
+ """Return the code object compiled from source.
+
+ The 'data' argument can be any object type that compile() supports.
+ """
+ return _bootstrap._call_with_frames_removed(compile, data, path, 'exec',
+ dont_inherit=True, optimize=_optimize)
+
+ def get_code(self, fullname):
+ """Concrete implementation of InspectLoader.get_code.
+
+ Reading of bytecode requires path_stats to be implemented. To write
+ bytecode, set_data must also be implemented.
+
+ """
+ source_path = self.get_filename(fullname)
+ source_mtime = None
+ try:
+ bytecode_path = cache_from_source(source_path)
+ except NotImplementedError:
+ bytecode_path = None
+ else:
+ try:
+ st = self.path_stats(source_path)
+ except IOError:
+ pass
+ else:
+ source_mtime = int(st['mtime'])
+ try:
+ data = self.get_data(bytecode_path)
+ except OSError:
+ pass
+ else:
+ try:
+ bytes_data = _validate_bytecode_header(data,
+ source_stats=st, name=fullname,
+ path=bytecode_path)
+ except (ImportError, EOFError):
+ pass
+ else:
+ _verbose_message('{} matches {}', bytecode_path,
+ source_path)
+ return _compile_bytecode(bytes_data, name=fullname,
+ bytecode_path=bytecode_path,
+ source_path=source_path)
+ source_bytes = self.get_data(source_path)
+ code_object = self.source_to_code(source_bytes, source_path)
+ _verbose_message('code object from {}', source_path)
+ if (not sys.dont_write_bytecode and bytecode_path is not None and
+ source_mtime is not None):
+ data = _code_to_bytecode(code_object, source_mtime,
+ len(source_bytes))
+ try:
+ self._cache_bytecode(source_path, bytecode_path, data)
+ _verbose_message('wrote {!r}', bytecode_path)
+ except NotImplementedError:
+ pass
+ return code_object
+
+
+class FileLoader:
+
+ """Base file loader class which implements the loader protocol methods that
+ require file system usage."""
+
+ def __init__(self, fullname, path):
+ """Cache the module name and the path to the file found by the
+ finder."""
+ self.name = fullname
+ self.path = path
+
+ def __eq__(self, other):
+ return (self.__class__ == other.__class__ and
+ self.__dict__ == other.__dict__)
+
+ def __hash__(self):
+ return hash(self.name) ^ hash(self.path)
+
+ @_check_name
+ def load_module(self, fullname):
+ """Load a module from a file.
+
+ This method is deprecated. Use exec_module() instead.
+
+ """
+ # The only reason for this method is for the name check.
+ # Issue #14857: Avoid the zero-argument form of super so the implementation
+ # of that form can be updated without breaking the frozen module
+ return super(FileLoader, self).load_module(fullname)
+
+ @_check_name
+ def get_filename(self, fullname):
+ """Return the path to the source file as found by the finder."""
+ return self.path
+
+ def get_data(self, path):
+ """Return the data from path as raw bytes."""
+ with _io.FileIO(path, 'r') as file:
+ return file.read()
+
+
+class SourceFileLoader(FileLoader, SourceLoader):
+
+ """Concrete implementation of SourceLoader using the file system."""
+
+ def path_stats(self, path):
+ """Return the metadata for the path."""
+ st = _path_stat(path)
+ return {'mtime': st.st_mtime, 'size': st.st_size}
+
+ def _cache_bytecode(self, source_path, bytecode_path, data):
+ # Adapt between the two APIs
+ mode = _calc_mode(source_path)
+ return self.set_data(bytecode_path, data, _mode=mode)
+
+ def set_data(self, path, data, *, _mode=0o666):
+ """Write bytes data to a file."""
+ parent, filename = _path_split(path)
+ path_parts = []
+ # Figure out what directories are missing.
+ while parent and not _path_isdir(parent):
+ parent, part = _path_split(parent)
+ path_parts.append(part)
+ # Create needed directories.
+ for part in reversed(path_parts):
+ parent = _path_join(parent, part)
+ try:
+ _os.mkdir(parent)
+ except FileExistsError:
+ # Probably another Python process already created the dir.
+ continue
+ except OSError as exc:
+ # Could be a permission error, read-only filesystem: just forget
+ # about writing the data.
+ _verbose_message('could not create {!r}: {!r}', parent, exc)
+ return
+ try:
+ _write_atomic(path, data, _mode)
+ _verbose_message('created {!r}', path)
+ except OSError as exc:
+ # Same as above: just don't write the bytecode.
+ _verbose_message('could not create {!r}: {!r}', path, exc)
+
+
+class SourcelessFileLoader(FileLoader, _LoaderBasics):
+
+ """Loader which handles sourceless file imports."""
+
+ def get_code(self, fullname):
+ path = self.get_filename(fullname)
+ data = self.get_data(path)
+ bytes_data = _validate_bytecode_header(data, name=fullname, path=path)
+ return _compile_bytecode(bytes_data, name=fullname, bytecode_path=path)
+
+ def get_source(self, fullname):
+ """Return None as there is no source code."""
+ return None
+
+
+# Filled in by _setup().
+EXTENSION_SUFFIXES = []
+
+
+class ExtensionFileLoader(FileLoader, _LoaderBasics):
+
+ """Loader for extension modules.
+
+ The constructor is designed to work with FileFinder.
+
+ """
+
+ def __init__(self, name, path):
+ self.name = name
+ self.path = path
+
+ def __eq__(self, other):
+ return (self.__class__ == other.__class__ and
+ self.__dict__ == other.__dict__)
+
+ def __hash__(self):
+ return hash(self.name) ^ hash(self.path)
+
+ def create_module(self, spec):
+ """Create an unitialized extension module"""
+ module = _bootstrap._call_with_frames_removed(
+ _imp.create_dynamic, spec)
+ _verbose_message('extension module {!r} loaded from {!r}',
+ spec.name, self.path)
+ return module
+
+ def exec_module(self, module):
+ """Initialize an extension module"""
+ _bootstrap._call_with_frames_removed(_imp.exec_dynamic, module)
+ _verbose_message('extension module {!r} executed from {!r}',
+ self.name, self.path)
+
+ def is_package(self, fullname):
+ """Return True if the extension module is a package."""
+ file_name = _path_split(self.path)[1]
+ return any(file_name == '__init__' + suffix
+ for suffix in EXTENSION_SUFFIXES)
+
+ def get_code(self, fullname):
+ """Return None as an extension module cannot create a code object."""
+ return None
+
+ def get_source(self, fullname):
+ """Return None as extension modules have no source code."""
+ return None
+
+ @_check_name
+ def get_filename(self, fullname):
+ """Return the path to the source file as found by the finder."""
+ return self.path
+
+
+class _NamespacePath:
+ """Represents a namespace package's path. It uses the module name
+ to find its parent module, and from there it looks up the parent's
+ __path__. When this changes, the module's own path is recomputed,
+ using path_finder. For top-level modules, the parent module's path
+ is sys.path."""
+
+ def __init__(self, name, path, path_finder):
+ self._name = name
+ self._path = path
+ self._last_parent_path = tuple(self._get_parent_path())
+ self._path_finder = path_finder
+
+ def _find_parent_path_names(self):
+ """Returns a tuple of (parent-module-name, parent-path-attr-name)"""
+ parent, dot, me = self._name.rpartition('.')
+ if dot == '':
+ # This is a top-level module. sys.path contains the parent path.
+ return 'sys', 'path'
+ # Not a top-level module. parent-module.__path__ contains the
+ # parent path.
+ return parent, '__path__'
+
+ def _get_parent_path(self):
+ parent_module_name, path_attr_name = self._find_parent_path_names()
+ return getattr(sys.modules[parent_module_name], path_attr_name)
+
+ def _recalculate(self):
+ # If the parent's path has changed, recalculate _path
+ parent_path = tuple(self._get_parent_path()) # Make a copy
+ if parent_path != self._last_parent_path:
+ spec = self._path_finder(self._name, parent_path)
+ # Note that no changes are made if a loader is returned, but we
+ # do remember the new parent path
+ if spec is not None and spec.loader is None:
+ if spec.submodule_search_locations:
+ self._path = spec.submodule_search_locations
+ self._last_parent_path = parent_path # Save the copy
+ return self._path
+
+ def __iter__(self):
+ return iter(self._recalculate())
+
+ def __len__(self):
+ return len(self._recalculate())
+
+ def __repr__(self):
+ return '_NamespacePath({!r})'.format(self._path)
+
+ def __contains__(self, item):
+ return item in self._recalculate()
+
+ def append(self, item):
+ self._path.append(item)
+
+
+# We use this exclusively in module_from_spec() for backward-compatibility.
+class _NamespaceLoader:
+ def __init__(self, name, path, path_finder):
+ self._path = _NamespacePath(name, path, path_finder)
+
+ @classmethod
+ def module_repr(cls, module):
+ """Return repr for the module.
+
+ The method is deprecated. The import machinery does the job itself.
+
+ """
+ return '<module {!r} (namespace)>'.format(module.__name__)
+
+ def is_package(self, fullname):
+ return True
+
+ def get_source(self, fullname):
+ return ''
+
+ def get_code(self, fullname):
+ return compile('', '<string>', 'exec', dont_inherit=True)
+
+ def create_module(self, spec):
+ """Use default semantics for module creation."""
+
+ def exec_module(self, module):
+ pass
+
+ def load_module(self, fullname):
+ """Load a namespace module.
+
+ This method is deprecated. Use exec_module() instead.
+
+ """
+ # The import system never calls this method.
+ _verbose_message('namespace module loaded with path {!r}', self._path)
+ return _bootstrap._load_module_shim(self, fullname)
+
+
+# Finders #####################################################################
+
+class PathFinder:
+
+ """Meta path finder for sys.path and package __path__ attributes."""
+
+ @classmethod
+ def invalidate_caches(cls):
+ """Call the invalidate_caches() method on all path entry finders
+ stored in sys.path_importer_caches (where implemented)."""
+ for finder in sys.path_importer_cache.values():
+ if hasattr(finder, 'invalidate_caches'):
+ finder.invalidate_caches()
+
+ @classmethod
+ def _path_hooks(cls, path):
+ """Search sequence of hooks for a finder for 'path'.
+
+ If 'hooks' is false then use sys.path_hooks.
+
+ """
+ if sys.path_hooks is not None and not sys.path_hooks:
+ _warnings.warn('sys.path_hooks is empty', ImportWarning)
+ for hook in sys.path_hooks:
+ try:
+ return hook(path)
+ except ImportError:
+ continue
+ else:
+ return None
+
+ @classmethod
+ def _path_importer_cache(cls, path):
+ """Get the finder for the path entry from sys.path_importer_cache.
+
+ If the path entry is not in the cache, find the appropriate finder
+ and cache it. If no finder is available, store None.
+
+ """
+ if path == '':
+ try:
+ path = _os.getcwd()
+ except FileNotFoundError:
+ # Don't cache the failure as the cwd can easily change to
+ # a valid directory later on.
+ return None
+ try:
+ finder = sys.path_importer_cache[path]
+ except KeyError:
+ finder = cls._path_hooks(path)
+ sys.path_importer_cache[path] = finder
+ return finder
+
+ @classmethod
+ def _legacy_get_spec(cls, fullname, finder):
+ # This would be a good place for a DeprecationWarning if
+ # we ended up going that route.
+ if hasattr(finder, 'find_loader'):
+ loader, portions = finder.find_loader(fullname)
+ else:
+ loader = finder.find_module(fullname)
+ portions = []
+ if loader is not None:
+ return _bootstrap.spec_from_loader(fullname, loader)
+ spec = _bootstrap.ModuleSpec(fullname, None)
+ spec.submodule_search_locations = portions
+ return spec
+
+ @classmethod
+ def _get_spec(cls, fullname, path, target=None):
+ """Find the loader or namespace_path for this module/package name."""
+ # If this ends up being a namespace package, namespace_path is
+ # the list of paths that will become its __path__
+ namespace_path = []
+ for entry in path:
+ if not isinstance(entry, (str, bytes)):
+ continue
+ finder = cls._path_importer_cache(entry)
+ if finder is not None:
+ if hasattr(finder, 'find_spec'):
+ spec = finder.find_spec(fullname, target)
+ else:
+ spec = cls._legacy_get_spec(fullname, finder)
+ if spec is None:
+ continue
+ if spec.loader is not None:
+ return spec
+ portions = spec.submodule_search_locations
+ if portions is None:
+ raise ImportError('spec missing loader')
+ # This is possibly part of a namespace package.
+ # Remember these path entries (if any) for when we
+ # create a namespace package, and continue iterating
+ # on path.
+ namespace_path.extend(portions)
+ else:
+ spec = _bootstrap.ModuleSpec(fullname, None)
+ spec.submodule_search_locations = namespace_path
+ return spec
+
+ @classmethod
+ def find_spec(cls, fullname, path=None, target=None):
+ """find the module on sys.path or 'path' based on sys.path_hooks and
+ sys.path_importer_cache."""
+ if path is None:
+ path = sys.path
+ spec = cls._get_spec(fullname, path, target)
+ if spec is None:
+ return None
+ elif spec.loader is None:
+ namespace_path = spec.submodule_search_locations
+ if namespace_path:
+ # We found at least one namespace path. Return a
+ # spec which can create the namespace package.
+ spec.origin = 'namespace'
+ spec.submodule_search_locations = _NamespacePath(fullname, namespace_path, cls._get_spec)
+ return spec
+ else:
+ return None
+ else:
+ return spec
+
+ @classmethod
+ def find_module(cls, fullname, path=None):
+ """find the module on sys.path or 'path' based on sys.path_hooks and
+ sys.path_importer_cache.
+
+ This method is deprecated. Use find_spec() instead.
+
+ """
+ spec = cls.find_spec(fullname, path)
+ if spec is None:
+ return None
+ return spec.loader
+
+
+class FileFinder:
+
+ """File-based finder.
+
+ Interactions with the file system are cached for performance, being
+ refreshed when the directory the finder is handling has been modified.
+
+ """
+
+ def __init__(self, path, *loader_details):
+ """Initialize with the path to search on and a variable number of
+ 2-tuples containing the loader and the file suffixes the loader
+ recognizes."""
+ loaders = []
+ for loader, suffixes in loader_details:
+ loaders.extend((suffix, loader) for suffix in suffixes)
+ self._loaders = loaders
+ # Base (directory) path
+ self.path = path or '.'
+ self._path_mtime = -1
+ self._path_cache = set()
+ self._relaxed_path_cache = set()
+
+ def invalidate_caches(self):
+ """Invalidate the directory mtime."""
+ self._path_mtime = -1
+
+ find_module = _find_module_shim
+
+ def find_loader(self, fullname):
+ """Try to find a loader for the specified module, or the namespace
+ package portions. Returns (loader, list-of-portions).
+
+ This method is deprecated. Use find_spec() instead.
+
+ """
+ spec = self.find_spec(fullname)
+ if spec is None:
+ return None, []
+ return spec.loader, spec.submodule_search_locations or []
+
+ def _get_spec(self, loader_class, fullname, path, smsl, target):
+ loader = loader_class(fullname, path)
+ return spec_from_file_location(fullname, path, loader=loader,
+ submodule_search_locations=smsl)
+
+ def find_spec(self, fullname, target=None):
+ """Try to find a loader for the specified module, or the namespace
+ package portions. Returns (loader, list-of-portions)."""
+ is_namespace = False
+ tail_module = fullname.rpartition('.')[2]
+ try:
+ mtime = _path_stat(self.path or _os.getcwd()).st_mtime
+ except OSError:
+ mtime = -1
+ if mtime != self._path_mtime:
+ self._fill_cache()
+ self._path_mtime = mtime
+ # tail_module keeps the original casing, for __file__ and friends
+ if _relax_case():
+ cache = self._relaxed_path_cache
+ cache_module = tail_module.lower()
+ else:
+ cache = self._path_cache
+ cache_module = tail_module
+ # Check if the module is the name of a directory (and thus a package).
+ if cache_module in cache:
+ base_path = _path_join(self.path, tail_module)
+ for suffix, loader_class in self._loaders:
+ init_filename = '__init__' + suffix
+ full_path = _path_join(base_path, init_filename)
+ if _path_isfile(full_path):
+ return self._get_spec(loader_class, fullname, full_path, [base_path], target)
+ else:
+ # If a namespace package, return the path if we don't
+ # find a module in the next section.
+ is_namespace = _path_isdir(base_path)
+ # Check for a file w/ a proper suffix exists.
+ for suffix, loader_class in self._loaders:
+ full_path = _path_join(self.path, tail_module + suffix)
+ _verbose_message('trying {}'.format(full_path), verbosity=2)
+ if cache_module + suffix in cache:
+ if _path_isfile(full_path):
+ return self._get_spec(loader_class, fullname, full_path, None, target)
+ if is_namespace:
+ _verbose_message('possible namespace for {}'.format(base_path))
+ spec = _bootstrap.ModuleSpec(fullname, None)
+ spec.submodule_search_locations = [base_path]
+ return spec
+ return None
+
+ def _fill_cache(self):
+ """Fill the cache of potential modules and packages for this directory."""
+ path = self.path
+ try:
+ contents = _os.listdir(path or _os.getcwd())
+ except (FileNotFoundError, PermissionError, NotADirectoryError):
+ # Directory has either been removed, turned into a file, or made
+ # unreadable.
+ contents = []
+ # We store two cached versions, to handle runtime changes of the
+ # PYTHONCASEOK environment variable.
+ if not sys.platform.startswith('win'):
+ self._path_cache = set(contents)
+ else:
+ # Windows users can import modules with case-insensitive file
+ # suffixes (for legacy reasons). Make the suffix lowercase here
+ # so it's done once instead of for every import. This is safe as
+ # the specified suffixes to check against are always specified in a
+ # case-sensitive manner.
+ lower_suffix_contents = set()
+ for item in contents:
+ name, dot, suffix = item.partition('.')
+ if dot:
+ new_name = '{}.{}'.format(name, suffix.lower())
+ else:
+ new_name = name
+ lower_suffix_contents.add(new_name)
+ self._path_cache = lower_suffix_contents
+ if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
+ self._relaxed_path_cache = {fn.lower() for fn in contents}
+
+ @classmethod
+ def path_hook(cls, *loader_details):
+ """A class method which returns a closure to use on sys.path_hook
+ which will return an instance using the specified loaders and the path
+ called on the closure.
+
+ If the path called on the closure is not a directory, ImportError is
+ raised.
+
+ """
+ def path_hook_for_FileFinder(path):
+ """Path hook for importlib.machinery.FileFinder."""
+ if not _path_isdir(path):
+ raise ImportError('only directories are supported', path=path)
+ return cls(path, *loader_details)
+
+ return path_hook_for_FileFinder
+
+ def __repr__(self):
+ return 'FileFinder({!r})'.format(self.path)
+
+
+# Import setup ###############################################################
+
+def _fix_up_module(ns, name, pathname, cpathname=None):
+ # This function is used by PyImport_ExecCodeModuleObject().
+ loader = ns.get('__loader__')
+ spec = ns.get('__spec__')
+ if not loader:
+ if spec:
+ loader = spec.loader
+ elif pathname == cpathname:
+ loader = SourcelessFileLoader(name, pathname)
+ else:
+ loader = SourceFileLoader(name, pathname)
+ if not spec:
+ spec = spec_from_file_location(name, pathname, loader=loader)
+ try:
+ ns['__spec__'] = spec
+ ns['__loader__'] = loader
+ ns['__file__'] = pathname
+ ns['__cached__'] = cpathname
+ except Exception:
+ # Not important enough to report.
+ pass
+
+
+def _get_supported_file_loaders():
+ """Returns a list of file-based module loaders.
+
+ Each item is a tuple (loader, suffixes).
+ """
+ extensions = ExtensionFileLoader, _imp.extension_suffixes()
+ source = SourceFileLoader, SOURCE_SUFFIXES
+ bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES
+ return [extensions, source, bytecode]
+
+
+def _setup(_bootstrap_module):
+ """Setup the path-based importers for importlib by importing needed
+ built-in modules and injecting them into the global namespace.
+
+ Other components are extracted from the core bootstrap module.
+
+ """
+ global sys, _imp, _bootstrap
+ _bootstrap = _bootstrap_module
+ sys = _bootstrap.sys
+ _imp = _bootstrap._imp
+
+ # Directly load built-in modules needed during bootstrap.
+ self_module = sys.modules[__name__]
+ for builtin_name in ('_io', '_warnings', 'builtins', 'marshal'):
+ if builtin_name not in sys.modules:
+ builtin_module = _bootstrap._builtin_from_name(builtin_name)
+ else:
+ builtin_module = sys.modules[builtin_name]
+ setattr(self_module, builtin_name, builtin_module)
+
+ # Directly load the os module (needed during bootstrap).
+ os_details = ('posix', ['/']), ('nt', ['\\', '/'])
+ for builtin_os, path_separators in os_details:
+ # Assumption made in _path_join()
+ assert all(len(sep) == 1 for sep in path_separators)
+ path_sep = path_separators[0]
+ if builtin_os in sys.modules:
+ os_module = sys.modules[builtin_os]
+ break
+ else:
+ try:
+ os_module = _bootstrap._builtin_from_name(builtin_os)
+ break
+ except ImportError:
+ continue
+ else:
+ raise ImportError('importlib requires posix or nt')
+ setattr(self_module, '_os', os_module)
+ setattr(self_module, 'path_sep', path_sep)
+ setattr(self_module, 'path_separators', ''.join(path_separators))
+
+ # Directly load the _thread module (needed during bootstrap).
+ try:
+ thread_module = _bootstrap._builtin_from_name('_thread')
+ except ImportError:
+ # Python was built without threads
+ thread_module = None
+ setattr(self_module, '_thread', thread_module)
+
+ # Directly load the _weakref module (needed during bootstrap).
+ weakref_module = _bootstrap._builtin_from_name('_weakref')
+ setattr(self_module, '_weakref', weakref_module)
+
+ # Directly load the winreg module (needed during bootstrap).
+ if builtin_os == 'nt':
+ winreg_module = _bootstrap._builtin_from_name('winreg')
+ setattr(self_module, '_winreg', winreg_module)
+
+ # Constants
+ setattr(self_module, '_relax_case', _make_relax_case())
+ EXTENSION_SUFFIXES.extend(_imp.extension_suffixes())
+ if builtin_os == 'nt':
+ SOURCE_SUFFIXES.append('.pyw')
+ if '_d.pyd' in EXTENSION_SUFFIXES:
+ WindowsRegistryFinder.DEBUG_BUILD = True
+
+
+def _install(_bootstrap_module):
+ """Install the path-based import components."""
+ _setup(_bootstrap_module)
+ supported_loaders = _get_supported_file_loaders()
+ sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)])
+ if _os.__name__ == 'nt':
+ sys.meta_path.append(WindowsRegistryFinder)
+ sys.meta_path.append(PathFinder)
+
+ # XXX We expose a couple of classes in _bootstrap for the sake of
+ # a setuptools bug (https://bitbucket.org/pypa/setuptools/issue/378).
+ _bootstrap_module.FileFinder = FileFinder
+ _bootstrap_module.SourceFileLoader = SourceFileLoader
diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py
index 558abd3..11af22d 100644
--- a/Lib/importlib/abc.py
+++ b/Lib/importlib/abc.py
@@ -1,12 +1,18 @@
"""Abstract base classes related to import."""
from . import _bootstrap
+from . import _bootstrap_external
from . import machinery
try:
import _frozen_importlib
+# import _frozen_importlib_external
except ImportError as exc:
if exc.name != '_frozen_importlib':
raise
_frozen_importlib = None
+try:
+ import _frozen_importlib_external
+except ImportError as exc:
+ _frozen_importlib_external = _bootstrap_external
import abc
@@ -14,7 +20,10 @@ def _register(abstract_cls, *classes):
for cls in classes:
abstract_cls.register(cls)
if _frozen_importlib is not None:
- frozen_cls = getattr(_frozen_importlib, cls.__name__)
+ try:
+ frozen_cls = getattr(_frozen_importlib, cls.__name__)
+ except AttributeError:
+ frozen_cls = getattr(_frozen_importlib_external, cls.__name__)
abstract_cls.register(frozen_cls)
@@ -102,7 +111,7 @@ class PathEntryFinder(Finder):
else:
return None, []
- find_module = _bootstrap._find_module_shim
+ find_module = _bootstrap_external._find_module_shim
def invalidate_caches(self):
"""An optional method for clearing the finder's cache, if any.
@@ -122,11 +131,8 @@ class Loader(metaclass=abc.ABCMeta):
This method should raise ImportError if anything prevents it
from creating a new module. It may return None to indicate
that the spec should create the new module.
-
- create_module() is optional.
-
"""
- # By default, defer to _SpecMethods.create() for the new module.
+ # By default, defer to default semantics for the new module.
return None
# We don't define exec_module() here since that would break
@@ -217,15 +223,16 @@ class InspectLoader(Loader):
"""
raise ImportError
- def source_to_code(self, data, path='<string>'):
+ @staticmethod
+ def source_to_code(data, path='<string>'):
"""Compile 'data' into a code object.
The 'data' argument can be anything that compile() can handle. The'path'
argument should be where the data was retrieved (when applicable)."""
return compile(data, path, 'exec', dont_inherit=True)
- exec_module = _bootstrap._LoaderBasics.exec_module
- load_module = _bootstrap._LoaderBasics.load_module
+ exec_module = _bootstrap_external._LoaderBasics.exec_module
+ load_module = _bootstrap_external._LoaderBasics.load_module
_register(InspectLoader, machinery.BuiltinImporter, machinery.FrozenImporter)
@@ -267,7 +274,7 @@ class ExecutionLoader(InspectLoader):
_register(ExecutionLoader, machinery.ExtensionFileLoader)
-class FileLoader(_bootstrap.FileLoader, ResourceLoader, ExecutionLoader):
+class FileLoader(_bootstrap_external.FileLoader, ResourceLoader, ExecutionLoader):
"""Abstract base class partially implementing the ResourceLoader and
ExecutionLoader ABCs."""
@@ -276,7 +283,7 @@ _register(FileLoader, machinery.SourceFileLoader,
machinery.SourcelessFileLoader)
-class SourceLoader(_bootstrap.SourceLoader, ResourceLoader, ExecutionLoader):
+class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLoader):
"""Abstract base class for loading source code (and optionally any
corresponding bytecode).
diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py
index 2e1b2d7..1b2b5c9 100644
--- a/Lib/importlib/machinery.py
+++ b/Lib/importlib/machinery.py
@@ -2,18 +2,18 @@
import _imp
-from ._bootstrap import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES,
- OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES,
- EXTENSION_SUFFIXES)
from ._bootstrap import ModuleSpec
from ._bootstrap import BuiltinImporter
from ._bootstrap import FrozenImporter
-from ._bootstrap import WindowsRegistryFinder
-from ._bootstrap import PathFinder
-from ._bootstrap import FileFinder
-from ._bootstrap import SourceFileLoader
-from ._bootstrap import SourcelessFileLoader
-from ._bootstrap import ExtensionFileLoader
+from ._bootstrap_external import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES,
+ OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES,
+ EXTENSION_SUFFIXES)
+from ._bootstrap_external import WindowsRegistryFinder
+from ._bootstrap_external import PathFinder
+from ._bootstrap_external import FileFinder
+from ._bootstrap_external import SourceFileLoader
+from ._bootstrap_external import SourcelessFileLoader
+from ._bootstrap_external import ExtensionFileLoader
def all_suffixes():
diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py
index 6d73b1d..1dbff26 100644
--- a/Lib/importlib/util.py
+++ b/Lib/importlib/util.py
@@ -1,17 +1,19 @@
"""Utility code for constructing importers, etc."""
-
-from ._bootstrap import MAGIC_NUMBER
-from ._bootstrap import cache_from_source
-from ._bootstrap import decode_source
-from ._bootstrap import source_from_cache
-from ._bootstrap import spec_from_loader
-from ._bootstrap import spec_from_file_location
+from . import abc
+from ._bootstrap import module_from_spec
from ._bootstrap import _resolve_name
+from ._bootstrap import spec_from_loader
from ._bootstrap import _find_spec
+from ._bootstrap_external import MAGIC_NUMBER
+from ._bootstrap_external import cache_from_source
+from ._bootstrap_external import decode_source
+from ._bootstrap_external import source_from_cache
+from ._bootstrap_external import spec_from_file_location
from contextlib import contextmanager
import functools
import sys
+import types
import warnings
@@ -54,7 +56,7 @@ def _find_spec_from_path(name, path=None):
try:
spec = module.__spec__
except AttributeError:
- raise ValueError('{}.__spec__ is not set'.format(name))
+ raise ValueError('{}.__spec__ is not set'.format(name)) from None
else:
if spec is None:
raise ValueError('{}.__spec__ is None'.format(name))
@@ -94,7 +96,7 @@ def find_spec(name, package=None):
try:
spec = module.__spec__
except AttributeError:
- raise ValueError('{}.__spec__ is not set'.format(name))
+ raise ValueError('{}.__spec__ is not set'.format(name)) from None
else:
if spec is None:
raise ValueError('{}.__spec__ is None'.format(name))
@@ -200,3 +202,94 @@ def module_for_loader(fxn):
return fxn(self, module, *args, **kwargs)
return module_for_loader_wrapper
+
+
+class _Module(types.ModuleType):
+
+ """A subclass of the module type to allow __class__ manipulation."""
+
+
+class _LazyModule(types.ModuleType):
+
+ """A subclass of the module type which triggers loading upon attribute access."""
+
+ def __getattribute__(self, attr):
+ """Trigger the load of the module and return the attribute."""
+ # All module metadata must be garnered from __spec__ in order to avoid
+ # using mutated values.
+ # Stop triggering this method.
+ self.__class__ = _Module
+ # Get the original name to make sure no object substitution occurred
+ # in sys.modules.
+ original_name = self.__spec__.name
+ # Figure out exactly what attributes were mutated between the creation
+ # of the module and now.
+ attrs_then = self.__spec__.loader_state
+ attrs_now = self.__dict__
+ attrs_updated = {}
+ for key, value in attrs_now.items():
+ # Code that set the attribute may have kept a reference to the
+ # assigned object, making identity more important than equality.
+ if key not in attrs_then:
+ attrs_updated[key] = value
+ elif id(attrs_now[key]) != id(attrs_then[key]):
+ attrs_updated[key] = value
+ self.__spec__.loader.exec_module(self)
+ # If exec_module() was used directly there is no guarantee the module
+ # object was put into sys.modules.
+ if original_name in sys.modules:
+ if id(self) != id(sys.modules[original_name]):
+ msg = ('module object for {!r} substituted in sys.modules '
+ 'during a lazy load')
+ raise ValueError(msg.format(original_name))
+ # Update after loading since that's what would happen in an eager
+ # loading situation.
+ self.__dict__.update(attrs_updated)
+ return getattr(self, attr)
+
+ def __delattr__(self, attr):
+ """Trigger the load and then perform the deletion."""
+ # To trigger the load and raise an exception if the attribute
+ # doesn't exist.
+ self.__getattribute__(attr)
+ delattr(self, attr)
+
+
+class LazyLoader(abc.Loader):
+
+ """A loader that creates a module which defers loading until attribute access."""
+
+ @staticmethod
+ def __check_eager_loader(loader):
+ if not hasattr(loader, 'exec_module'):
+ raise TypeError('loader must define exec_module()')
+ elif hasattr(loader.__class__, 'create_module'):
+ if abc.Loader.create_module != loader.__class__.create_module:
+ # Only care if create_module() is overridden in a subclass of
+ # importlib.abc.Loader.
+ raise TypeError('loader cannot define create_module()')
+
+ @classmethod
+ def factory(cls, loader):
+ """Construct a callable which returns the eager loader made lazy."""
+ cls.__check_eager_loader(loader)
+ return lambda *args, **kwargs: cls(loader(*args, **kwargs))
+
+ def __init__(self, loader):
+ self.__check_eager_loader(loader)
+ self.loader = loader
+
+ def create_module(self, spec):
+ """Create a module which can have its __class__ manipulated."""
+ return _Module(spec.name)
+
+ def exec_module(self, module):
+ """Make the module load lazily."""
+ module.__spec__.loader = self.loader
+ module.__loader__ = self.loader
+ # Don't need to worry about deep-copying as trying to set an attribute
+ # on an object would have triggered the load,
+ # e.g. ``module.__spec__.loader = None`` would trigger a load from
+ # trying to access module.__spec__.
+ module.__spec__.loader_state = module.__dict__.copy()
+ module.__class__ = _LazyModule
diff --git a/Lib/inspect.py b/Lib/inspect.py
index 4298de6..bf4f87d 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -17,7 +17,7 @@ Here are some of the useful functions provided by this module:
getclasstree() - arrange classes so as to represent their hierarchy
getargspec(), getargvalues(), getcallargs() - get info about function arguments
- getfullargspec() - same, with support for Python-3000 features
+ getfullargspec() - same, with support for Python 3 features
formatargspec(), formatargvalues() - format an argument spec
getouterframes(), getinnerframes() - get info about frames
currentframe() - get the current stack frame
@@ -32,6 +32,9 @@ __author__ = ('Ka-Ping Yee <ping@lfw.org>',
'Yury Selivanov <yselivanov@sprymix.com>')
import ast
+import dis
+import collections.abc
+import enum
import importlib.machinery
import itertools
import linecache
@@ -48,18 +51,10 @@ from operator import attrgetter
from collections import namedtuple, OrderedDict
# Create constants for the compiler flags in Include/code.h
-# We try to get them from dis to avoid duplication, but fall
-# back to hard-coding so the dependency is optional
-try:
- from dis import COMPILER_FLAG_NAMES as _flag_names
-except ImportError:
- CO_OPTIMIZED, CO_NEWLOCALS = 0x1, 0x2
- CO_VARARGS, CO_VARKEYWORDS = 0x4, 0x8
- CO_NESTED, CO_GENERATOR, CO_NOFREE = 0x10, 0x20, 0x40
-else:
- mod_dict = globals()
- for k, v in _flag_names.items():
- mod_dict["CO_" + v] = k
+# We try to get them from dis to avoid duplication
+mod_dict = globals()
+for k, v in dis.COMPILER_FLAG_NAMES.items():
+ mod_dict["CO_" + v] = k
# See Include/object.h
TPFLAGS_IS_ABSTRACT = 1 << 20
@@ -182,6 +177,15 @@ def isgeneratorfunction(object):
return bool((isfunction(object) or ismethod(object)) and
object.__code__.co_flags & CO_GENERATOR)
+def iscoroutinefunction(object):
+ """Return true if the object is a coroutine function.
+
+ Coroutine functions are defined with "async def" syntax,
+ or generators decorated with "types.coroutine".
+ """
+ return bool((isfunction(object) or ismethod(object)) and
+ object.__code__.co_flags & CO_COROUTINE)
+
def isgenerator(object):
"""Return true if the object is a generator.
@@ -199,6 +203,17 @@ def isgenerator(object):
throw used to raise an exception inside the generator"""
return isinstance(object, types.GeneratorType)
+def iscoroutine(object):
+ """Return true if the object is a coroutine."""
+ return isinstance(object, types.CoroutineType)
+
+def isawaitable(object):
+ """Return true is object can be passed to an ``await`` expression."""
+ return (isinstance(object, types.CoroutineType) or
+ isinstance(object, types.GeneratorType) and
+ object.gi_code.co_flags & CO_ITERABLE_COROUTINE or
+ isinstance(object, collections.abc.Awaitable))
+
def istraceback(object):
"""Return true if the object is a traceback.
@@ -467,6 +482,74 @@ def indentsize(line):
expline = line.expandtabs()
return len(expline) - len(expline.lstrip())
+def _findclass(func):
+ cls = sys.modules.get(func.__module__)
+ if cls is None:
+ return None
+ for name in func.__qualname__.split('.')[:-1]:
+ cls = getattr(cls, name)
+ if not isclass(cls):
+ return None
+ return cls
+
+def _finddoc(obj):
+ if isclass(obj):
+ for base in obj.__mro__:
+ if base is not object:
+ try:
+ doc = base.__doc__
+ except AttributeError:
+ continue
+ if doc is not None:
+ return doc
+ return None
+
+ if ismethod(obj):
+ name = obj.__func__.__name__
+ self = obj.__self__
+ if (isclass(self) and
+ getattr(getattr(self, name, None), '__func__') is obj.__func__):
+ # classmethod
+ cls = self
+ else:
+ cls = self.__class__
+ elif isfunction(obj):
+ name = obj.__name__
+ cls = _findclass(obj)
+ if cls is None or getattr(cls, name) is not obj:
+ return None
+ elif isbuiltin(obj):
+ name = obj.__name__
+ self = obj.__self__
+ if (isclass(self) and
+ self.__qualname__ + '.' + name == obj.__qualname__):
+ # classmethod
+ cls = self
+ else:
+ cls = self.__class__
+ elif ismethoddescriptor(obj) or isdatadescriptor(obj):
+ name = obj.__name__
+ cls = obj.__objclass__
+ if getattr(cls, name) is not obj:
+ return None
+ elif isinstance(obj, property):
+ func = f.fget
+ name = func.__name__
+ cls = _findclass(func)
+ if cls is None or getattr(cls, name) is not obj:
+ return None
+ else:
+ return None
+
+ for base in cls.__mro__:
+ try:
+ doc = getattr(base, name).__doc__
+ except AttributeError:
+ continue
+ if doc is not None:
+ return doc
+ return None
+
def getdoc(object):
"""Get the documentation string for an object.
@@ -477,6 +560,11 @@ def getdoc(object):
doc = object.__doc__
except AttributeError:
return None
+ if doc is None:
+ try:
+ doc = _finddoc(object)
+ except (AttributeError, TypeError):
+ return None
if not isinstance(doc, str):
return None
return cleandoc(doc)
@@ -710,7 +798,7 @@ def findsource(object):
if not hasattr(object, 'co_firstlineno'):
raise OSError('could not find function definition')
lnum = object.co_firstlineno - 1
- pat = re.compile(r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)')
+ pat = re.compile(r'^(\s*def\s)|(\s*async\s+def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)')
while lnum > 0:
if pat.match(lines[lnum]): break
lnum = lnum - 1
@@ -771,21 +859,37 @@ class BlockFinder:
self.islambda = False
self.started = False
self.passline = False
+ self.indecorator = False
+ self.decoratorhasargs = False
self.last = 1
def tokeneater(self, type, token, srowcol, erowcol, line):
- if not self.started:
+ if not self.started and not self.indecorator:
+ # skip any decorators
+ if token == "@":
+ self.indecorator = True
# look for the first "def", "class" or "lambda"
- if token in ("def", "class", "lambda"):
+ elif token in ("def", "class", "lambda"):
if token == "lambda":
self.islambda = True
self.started = True
self.passline = True # skip to the end of the line
+ elif token == "(":
+ if self.indecorator:
+ self.decoratorhasargs = True
+ elif token == ")":
+ if self.indecorator:
+ self.indecorator = False
+ self.decoratorhasargs = False
elif type == tokenize.NEWLINE:
self.passline = False # stop skipping when a NEWLINE is seen
self.last = srowcol[0]
if self.islambda: # lambdas always end at the first NEWLINE
raise EndOfBlock
+ # hitting a NEWLINE when in a decorator without args
+ # ends the decorator
+ if self.indecorator and not self.decoratorhasargs:
+ self.indecorator = False
elif self.passline:
pass
elif type == tokenize.INDENT:
@@ -822,10 +926,13 @@ def getsourcelines(object):
corresponding to the object and the line number indicates where in the
original source file the first line of code was found. An OSError is
raised if the source code cannot be retrieved."""
+ object = unwrap(object)
lines, lnum = findsource(object)
- if ismodule(object): return lines, 0
- else: return getblock(lines[lnum:]), lnum + 1
+ if ismodule(object):
+ return lines, 0
+ else:
+ return getblock(lines[lnum:]), lnum + 1
def getsource(object):
"""Return the text of the source code for an object.
@@ -919,17 +1026,18 @@ ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults')
def getargspec(func):
"""Get the names and default values of a function's arguments.
- A tuple of four things is returned: (args, varargs, varkw, defaults).
- 'args' is a list of the argument names.
- 'args' will include keyword-only argument names.
- 'varargs' and 'varkw' are the names of the * and ** arguments or None.
+ A tuple of four things is returned: (args, varargs, keywords, defaults).
+ 'args' is a list of the argument names, including keyword-only argument names.
+ 'varargs' and 'keywords' are the names of the * and ** arguments or None.
'defaults' is an n-tuple of the default values of the last n arguments.
- Use the getfullargspec() API for Python-3000 code, as annotations
+ Use the getfullargspec() API for Python 3 code, as annotations
and keyword arguments are supported. getargspec() will raise ValueError
if the func has either annotations or keyword arguments.
"""
-
+ warnings.warn("inspect.getargspec() is deprecated, "
+ "use inspect.signature() instead", DeprecationWarning,
+ stacklevel=2)
args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \
getfullargspec(func)
if kwonlyargs or ann:
@@ -953,6 +1061,8 @@ def getfullargspec(func):
'annotations' is a dictionary mapping argument names to annotations.
The first four items in the tuple correspond to getargspec().
+
+ This function is deprecated, use inspect.signature() instead.
"""
try:
@@ -972,9 +1082,10 @@ def getfullargspec(func):
# getfullargspec() historically ignored __wrapped__ attributes,
# so we ensure that remains the case in 3.3+
- sig = _signature_internal(func,
- follow_wrapper_chains=False,
- skip_bound_arg=False)
+ sig = _signature_from_callable(func,
+ follow_wrapper_chains=False,
+ skip_bound_arg=False,
+ sigcls=Signature)
except Exception as ex:
# Most of the times 'signature' will raise ValueError.
# But, it can also raise AttributeError, and, maybe something
@@ -1043,8 +1154,8 @@ def getargvalues(frame):
def formatannotation(annotation, base_module=None):
if isinstance(annotation, type):
if annotation.__module__ in ('builtins', base_module):
- return annotation.__name__
- return annotation.__module__+'.'+annotation.__name__
+ return annotation.__qualname__
+ return annotation.__module__+'.'+annotation.__qualname__
return repr(annotation)
def formatannotationrelativeto(object):
@@ -1317,6 +1428,8 @@ def getlineno(frame):
# FrameType.f_lineno is now a descriptor that grovels co_lnotab
return frame.f_lineno
+FrameInfo = namedtuple('FrameInfo', ('frame',) + Traceback._fields)
+
def getouterframes(frame, context=1):
"""Get a list of records for a frame and all higher (calling) frames.
@@ -1324,7 +1437,8 @@ def getouterframes(frame, context=1):
name, a list of lines of context, and index within the context."""
framelist = []
while frame:
- framelist.append((frame,) + getframeinfo(frame, context))
+ frameinfo = (frame,) + getframeinfo(frame, context)
+ framelist.append(FrameInfo(*frameinfo))
frame = frame.f_back
return framelist
@@ -1335,7 +1449,8 @@ def getinnerframes(tb, context=1):
name, a list of lines of context, and index within the context."""
framelist = []
while tb:
- framelist.append((tb.tb_frame,) + getframeinfo(tb, context))
+ frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)
+ framelist.append(FrameInfo(*frameinfo))
tb = tb.tb_next
return framelist
@@ -1485,6 +1600,45 @@ def getgeneratorlocals(generator):
else:
return {}
+
+# ------------------------------------------------ coroutine introspection
+
+CORO_CREATED = 'CORO_CREATED'
+CORO_RUNNING = 'CORO_RUNNING'
+CORO_SUSPENDED = 'CORO_SUSPENDED'
+CORO_CLOSED = 'CORO_CLOSED'
+
+def getcoroutinestate(coroutine):
+ """Get current state of a coroutine object.
+
+ Possible states are:
+ CORO_CREATED: Waiting to start execution.
+ CORO_RUNNING: Currently being executed by the interpreter.
+ CORO_SUSPENDED: Currently suspended at an await expression.
+ CORO_CLOSED: Execution has completed.
+ """
+ if coroutine.cr_running:
+ return CORO_RUNNING
+ if coroutine.cr_frame is None:
+ return CORO_CLOSED
+ if coroutine.cr_frame.f_lasti == -1:
+ return CORO_CREATED
+ return CORO_SUSPENDED
+
+
+def getcoroutinelocals(coroutine):
+ """
+ Get the mapping of coroutine local variables to their current values.
+
+ A dict is returned, with the keys the local variable names and values the
+ bound values."""
+ frame = getattr(coroutine, "cr_frame", None)
+ if frame is not None:
+ return frame.f_locals
+ else:
+ return {}
+
+
###############################################################################
### Function Signature Object (PEP 362)
###############################################################################
@@ -1501,6 +1655,10 @@ _NonUserDefinedCallables = (_WrapperDescriptor,
def _signature_get_user_defined_method(cls, method_name):
+ """Private helper. Checks if ``cls`` has an attribute
+ named ``method_name`` and returns it only if it is a
+ pure python function.
+ """
try:
meth = getattr(cls, method_name)
except AttributeError:
@@ -1513,9 +1671,10 @@ def _signature_get_user_defined_method(cls, method_name):
def _signature_get_partial(wrapped_sig, partial, extra_args=()):
- # Internal helper to calculate how 'wrapped_sig' signature will
- # look like after applying a 'functools.partial' object (or alike)
- # on it.
+ """Private helper to calculate how 'wrapped_sig' signature will
+ look like after applying a 'functools.partial' object (or alike)
+ on it.
+ """
old_params = wrapped_sig.parameters
new_params = OrderedDict(old_params.items())
@@ -1588,8 +1747,9 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()):
def _signature_bound_method(sig):
- # Internal helper to transform signatures for unbound
- # functions to bound methods
+ """Private helper to transform signatures for unbound
+ functions to bound methods.
+ """
params = tuple(sig.parameters.values())
@@ -1613,8 +1773,9 @@ def _signature_bound_method(sig):
def _signature_is_builtin(obj):
- # Internal helper to test if `obj` is a callable that might
- # support Argument Clinic's __text_signature__ protocol.
+ """Private helper to test if `obj` is a callable that might
+ support Argument Clinic's __text_signature__ protocol.
+ """
return (isbuiltin(obj) or
ismethoddescriptor(obj) or
isinstance(obj, _NonUserDefinedCallables) or
@@ -1624,10 +1785,11 @@ def _signature_is_builtin(obj):
def _signature_is_functionlike(obj):
- # Internal helper to test if `obj` is a duck type of FunctionType.
- # A good example of such objects are functions compiled with
- # Cython, which have all attributes that a pure Python function
- # would have, but have their code statically compiled.
+ """Private helper to test if `obj` is a duck type of FunctionType.
+ A good example of such objects are functions compiled with
+ Cython, which have all attributes that a pure Python function
+ would have, but have their code statically compiled.
+ """
if not callable(obj) or isclass(obj):
# All function-like objects are obviously callables,
@@ -1648,11 +1810,12 @@ def _signature_is_functionlike(obj):
def _signature_get_bound_param(spec):
- # Internal helper to get first parameter name from a
- # __text_signature__ of a builtin method, which should
- # be in the following format: '($param1, ...)'.
- # Assumptions are that the first argument won't have
- # a default value or an annotation.
+ """ Private helper to get first parameter name from a
+ __text_signature__ of a builtin method, which should
+ be in the following format: '($param1, ...)'.
+ Assumptions are that the first argument won't have
+ a default value or an annotation.
+ """
assert spec.startswith('($')
@@ -1671,7 +1834,9 @@ def _signature_get_bound_param(spec):
def _signature_strip_non_python_syntax(signature):
"""
- Takes a signature in Argument Clinic's extended signature format.
+ Private helper function. Takes a signature in Argument Clinic's
+ extended signature format.
+
Returns a tuple of three things:
* that signature re-rendered in standard Python syntax,
* the index of the "self" parameter (generally 0), or None if
@@ -1740,8 +1905,10 @@ def _signature_strip_non_python_syntax(signature):
def _signature_fromstr(cls, obj, s, skip_bound_arg=True):
- # Internal helper to parse content of '__text_signature__'
- # and return a Signature based on it
+ """Private helper to parse content of '__text_signature__'
+ and return a Signature based on it.
+ """
+
Parameter = cls._parameter_cls
clean_signature, self_parameter, last_positional_only = \
@@ -1879,8 +2046,10 @@ def _signature_fromstr(cls, obj, s, skip_bound_arg=True):
def _signature_from_builtin(cls, func, skip_bound_arg=True):
- # Internal helper function to get signature for
- # builtin callables
+ """Private helper function to get signature for
+ builtin callables.
+ """
+
if not _signature_is_builtin(func):
raise TypeError("{!r} is not a Python builtin "
"function".format(func))
@@ -1892,7 +2061,95 @@ def _signature_from_builtin(cls, func, skip_bound_arg=True):
return _signature_fromstr(cls, func, s, skip_bound_arg)
-def _signature_internal(obj, follow_wrapper_chains=True, skip_bound_arg=True):
+def _signature_from_function(cls, func):
+ """Private helper: constructs Signature for the given python function."""
+
+ is_duck_function = False
+ if not isfunction(func):
+ if _signature_is_functionlike(func):
+ is_duck_function = True
+ else:
+ # If it's not a pure Python function, and not a duck type
+ # of pure function:
+ raise TypeError('{!r} is not a Python function'.format(func))
+
+ Parameter = cls._parameter_cls
+
+ # Parameter information.
+ func_code = func.__code__
+ pos_count = func_code.co_argcount
+ arg_names = func_code.co_varnames
+ positional = tuple(arg_names[:pos_count])
+ keyword_only_count = func_code.co_kwonlyargcount
+ keyword_only = arg_names[pos_count:(pos_count + keyword_only_count)]
+ annotations = func.__annotations__
+ defaults = func.__defaults__
+ kwdefaults = func.__kwdefaults__
+
+ if defaults:
+ pos_default_count = len(defaults)
+ else:
+ pos_default_count = 0
+
+ parameters = []
+
+ # Non-keyword-only parameters w/o defaults.
+ non_default_count = pos_count - pos_default_count
+ for name in positional[:non_default_count]:
+ annotation = annotations.get(name, _empty)
+ parameters.append(Parameter(name, annotation=annotation,
+ kind=_POSITIONAL_OR_KEYWORD))
+
+ # ... w/ defaults.
+ for offset, name in enumerate(positional[non_default_count:]):
+ annotation = annotations.get(name, _empty)
+ parameters.append(Parameter(name, annotation=annotation,
+ kind=_POSITIONAL_OR_KEYWORD,
+ default=defaults[offset]))
+
+ # *args
+ if func_code.co_flags & CO_VARARGS:
+ name = arg_names[pos_count + keyword_only_count]
+ annotation = annotations.get(name, _empty)
+ parameters.append(Parameter(name, annotation=annotation,
+ kind=_VAR_POSITIONAL))
+
+ # Keyword-only parameters.
+ for name in keyword_only:
+ default = _empty
+ if kwdefaults is not None:
+ default = kwdefaults.get(name, _empty)
+
+ annotation = annotations.get(name, _empty)
+ parameters.append(Parameter(name, annotation=annotation,
+ kind=_KEYWORD_ONLY,
+ default=default))
+ # **kwargs
+ if func_code.co_flags & CO_VARKEYWORDS:
+ index = pos_count + keyword_only_count
+ if func_code.co_flags & CO_VARARGS:
+ index += 1
+
+ name = arg_names[index]
+ annotation = annotations.get(name, _empty)
+ parameters.append(Parameter(name, annotation=annotation,
+ kind=_VAR_KEYWORD))
+
+ # Is 'func' is a pure Python function - don't validate the
+ # parameters list (for correct order and defaults), it should be OK.
+ return cls(parameters,
+ return_annotation=annotations.get('return', _empty),
+ __validate_parameters__=is_duck_function)
+
+
+def _signature_from_callable(obj, *,
+ follow_wrapper_chains=True,
+ skip_bound_arg=True,
+ sigcls):
+
+ """Private helper function to get signature for arbitrary
+ callable objects.
+ """
if not callable(obj):
raise TypeError('{!r} is not a callable object'.format(obj))
@@ -1900,9 +2157,12 @@ def _signature_internal(obj, follow_wrapper_chains=True, skip_bound_arg=True):
if isinstance(obj, types.MethodType):
# In this case we skip the first parameter of the underlying
# function (usually `self` or `cls`).
- sig = _signature_internal(obj.__func__,
- follow_wrapper_chains,
- skip_bound_arg)
+ sig = _signature_from_callable(
+ obj.__func__,
+ follow_wrapper_chains=follow_wrapper_chains,
+ skip_bound_arg=skip_bound_arg,
+ sigcls=sigcls)
+
if skip_bound_arg:
return _signature_bound_method(sig)
else:
@@ -1915,10 +2175,11 @@ def _signature_internal(obj, follow_wrapper_chains=True, skip_bound_arg=True):
# If the unwrapped object is a *method*, we might want to
# skip its first parameter (self).
# See test_signature_wrapped_bound_method for details.
- return _signature_internal(
+ return _signature_from_callable(
obj,
follow_wrapper_chains=follow_wrapper_chains,
- skip_bound_arg=skip_bound_arg)
+ skip_bound_arg=skip_bound_arg,
+ sigcls=sigcls)
try:
sig = obj.__signature__
@@ -1945,9 +2206,12 @@ def _signature_internal(obj, follow_wrapper_chains=True, skip_bound_arg=True):
# (usually `self`, or `cls`) will not be passed
# automatically (as for boundmethods)
- wrapped_sig = _signature_internal(partialmethod.func,
- follow_wrapper_chains,
- skip_bound_arg)
+ wrapped_sig = _signature_from_callable(
+ partialmethod.func,
+ follow_wrapper_chains=follow_wrapper_chains,
+ skip_bound_arg=skip_bound_arg,
+ sigcls=sigcls)
+
sig = _signature_get_partial(wrapped_sig, partialmethod, (None,))
first_wrapped_param = tuple(wrapped_sig.parameters.values())[0]
@@ -1958,16 +2222,18 @@ def _signature_internal(obj, follow_wrapper_chains=True, skip_bound_arg=True):
if isfunction(obj) or _signature_is_functionlike(obj):
# If it's a pure Python function, or an object that is duck type
# of a Python function (Cython functions, for instance), then:
- return Signature.from_function(obj)
+ return _signature_from_function(sigcls, obj)
if _signature_is_builtin(obj):
- return _signature_from_builtin(Signature, obj,
+ return _signature_from_builtin(sigcls, obj,
skip_bound_arg=skip_bound_arg)
if isinstance(obj, functools.partial):
- wrapped_sig = _signature_internal(obj.func,
- follow_wrapper_chains,
- skip_bound_arg)
+ wrapped_sig = _signature_from_callable(
+ obj.func,
+ follow_wrapper_chains=follow_wrapper_chains,
+ skip_bound_arg=skip_bound_arg,
+ sigcls=sigcls)
return _signature_get_partial(wrapped_sig, obj)
sig = None
@@ -1978,23 +2244,29 @@ def _signature_internal(obj, follow_wrapper_chains=True, skip_bound_arg=True):
# in its metaclass
call = _signature_get_user_defined_method(type(obj), '__call__')
if call is not None:
- sig = _signature_internal(call,
- follow_wrapper_chains,
- skip_bound_arg)
+ sig = _signature_from_callable(
+ call,
+ follow_wrapper_chains=follow_wrapper_chains,
+ skip_bound_arg=skip_bound_arg,
+ sigcls=sigcls)
else:
# Now we check if the 'obj' class has a '__new__' method
new = _signature_get_user_defined_method(obj, '__new__')
if new is not None:
- sig = _signature_internal(new,
- follow_wrapper_chains,
- skip_bound_arg)
+ sig = _signature_from_callable(
+ new,
+ follow_wrapper_chains=follow_wrapper_chains,
+ skip_bound_arg=skip_bound_arg,
+ sigcls=sigcls)
else:
# Finally, we should have at least __init__ implemented
init = _signature_get_user_defined_method(obj, '__init__')
if init is not None:
- sig = _signature_internal(init,
- follow_wrapper_chains,
- skip_bound_arg)
+ sig = _signature_from_callable(
+ init,
+ follow_wrapper_chains=follow_wrapper_chains,
+ skip_bound_arg=skip_bound_arg,
+ sigcls=sigcls)
if sig is None:
# At this point we know, that `obj` is a class, with no user-
@@ -2016,7 +2288,7 @@ def _signature_internal(obj, follow_wrapper_chains=True, skip_bound_arg=True):
if text_sig:
# If 'obj' class has a __text_signature__ attribute:
# return a signature based on it
- return _signature_fromstr(Signature, obj, text_sig)
+ return _signature_fromstr(sigcls, obj, text_sig)
# No '__text_signature__' was found for the 'obj' class.
# Last option is to check if its '__init__' is
@@ -2024,9 +2296,13 @@ def _signature_internal(obj, follow_wrapper_chains=True, skip_bound_arg=True):
if type not in obj.__mro__:
# We have a class (not metaclass), but no user-defined
# __init__ or __new__ for it
- if obj.__init__ is object.__init__:
+ if (obj.__init__ is object.__init__ and
+ obj.__new__ is object.__new__):
# Return a signature of 'object' builtin.
return signature(object)
+ else:
+ raise ValueError(
+ 'no signature found for builtin type {!r}'.format(obj))
elif not isinstance(obj, _NonUserDefinedCallables):
# An object with __call__
@@ -2036,9 +2312,11 @@ def _signature_internal(obj, follow_wrapper_chains=True, skip_bound_arg=True):
call = _signature_get_user_defined_method(type(obj), '__call__')
if call is not None:
try:
- sig = _signature_internal(call,
- follow_wrapper_chains,
- skip_bound_arg)
+ sig = _signature_from_callable(
+ call,
+ follow_wrapper_chains=follow_wrapper_chains,
+ skip_bound_arg=skip_bound_arg,
+ sigcls=sigcls)
except ValueError as ex:
msg = 'no signature found for {!r}'.format(obj)
raise ValueError(msg) from ex
@@ -2058,41 +2336,35 @@ def _signature_internal(obj, follow_wrapper_chains=True, skip_bound_arg=True):
raise ValueError('callable {!r} is not supported by signature'.format(obj))
-def signature(obj):
- '''Get a signature object for the passed callable.'''
- return _signature_internal(obj)
-
class _void:
- '''A private marker - used in Parameter & Signature'''
+ """A private marker - used in Parameter & Signature."""
class _empty:
- pass
+ """Marker object for Signature.empty and Parameter.empty."""
-class _ParameterKind(int):
- def __new__(self, *args, name):
- obj = int.__new__(self, *args)
- obj._name = name
- return obj
+class _ParameterKind(enum.IntEnum):
+ POSITIONAL_ONLY = 0
+ POSITIONAL_OR_KEYWORD = 1
+ VAR_POSITIONAL = 2
+ KEYWORD_ONLY = 3
+ VAR_KEYWORD = 4
def __str__(self):
- return self._name
+ return self._name_
- def __repr__(self):
- return '<_ParameterKind: {!r}>'.format(self._name)
-
-_POSITIONAL_ONLY = _ParameterKind(0, name='POSITIONAL_ONLY')
-_POSITIONAL_OR_KEYWORD = _ParameterKind(1, name='POSITIONAL_OR_KEYWORD')
-_VAR_POSITIONAL = _ParameterKind(2, name='VAR_POSITIONAL')
-_KEYWORD_ONLY = _ParameterKind(3, name='KEYWORD_ONLY')
-_VAR_KEYWORD = _ParameterKind(4, name='VAR_KEYWORD')
+_POSITIONAL_ONLY = _ParameterKind.POSITIONAL_ONLY
+_POSITIONAL_OR_KEYWORD = _ParameterKind.POSITIONAL_OR_KEYWORD
+_VAR_POSITIONAL = _ParameterKind.VAR_POSITIONAL
+_KEYWORD_ONLY = _ParameterKind.KEYWORD_ONLY
+_VAR_KEYWORD = _ParameterKind.VAR_KEYWORD
class Parameter:
- '''Represents a parameter in a function signature.
+ """Represents a parameter in a function signature.
Has the following public attributes:
@@ -2111,7 +2383,7 @@ class Parameter:
Possible values: `Parameter.POSITIONAL_ONLY`,
`Parameter.POSITIONAL_OR_KEYWORD`, `Parameter.VAR_POSITIONAL`,
`Parameter.KEYWORD_ONLY`, `Parameter.VAR_KEYWORD`.
- '''
+ """
__slots__ = ('_name', '_kind', '_default', '_annotation')
@@ -2148,6 +2420,16 @@ class Parameter:
self._name = name
+ def __reduce__(self):
+ return (type(self),
+ (self._name, self._kind),
+ {'_default': self._default,
+ '_annotation': self._annotation})
+
+ def __setstate__(self, state):
+ self._default = state['_default']
+ self._annotation = state['_annotation']
+
@property
def name(self):
return self._name
@@ -2166,7 +2448,7 @@ class Parameter:
def replace(self, *, name=_void, kind=_void,
annotation=_void, default=_void):
- '''Creates a customized copy of the Parameter.'''
+ """Creates a customized copy of the Parameter."""
if name is _void:
name = self._name
@@ -2202,10 +2484,14 @@ class Parameter:
return formatted
def __repr__(self):
- return '<{} at {:#x} {!r}>'.format(self.__class__.__name__,
- id(self), self.name)
+ return '<{} "{}">'.format(self.__class__.__name__, self)
+
+ def __hash__(self):
+ return hash((self.name, self.kind, self.annotation, self.default))
def __eq__(self, other):
+ if self is other:
+ return True
if not isinstance(other, Parameter):
return NotImplemented
return (self._name == other._name and
@@ -2215,7 +2501,7 @@ class Parameter:
class BoundArguments:
- '''Result of `Signature.bind` call. Holds the mapping of arguments
+ """Result of `Signature.bind` call. Holds the mapping of arguments
to the function's parameters.
Has the following public attributes:
@@ -2229,7 +2515,9 @@ class BoundArguments:
Tuple of positional arguments values.
* kwargs : dict
Dict of keyword arguments values.
- '''
+ """
+
+ __slots__ = ('arguments', '_signature', '__weakref__')
def __init__(self, signature, arguments):
self.arguments = arguments
@@ -2292,15 +2580,60 @@ class BoundArguments:
return kwargs
+ def apply_defaults(self):
+ """Set default values for missing arguments.
+
+ For variable-positional arguments (*args) the default is an
+ empty tuple.
+
+ For variable-keyword arguments (**kwargs) the default is an
+ empty dict.
+ """
+ arguments = self.arguments
+ if not arguments:
+ return
+ new_arguments = []
+ for name, param in self._signature.parameters.items():
+ try:
+ new_arguments.append((name, arguments[name]))
+ except KeyError:
+ if param.default is not _empty:
+ val = param.default
+ elif param.kind is _VAR_POSITIONAL:
+ val = ()
+ elif param.kind is _VAR_KEYWORD:
+ val = {}
+ else:
+ # This BoundArguments was likely produced by
+ # Signature.bind_partial().
+ continue
+ new_arguments.append((name, val))
+ self.arguments = OrderedDict(new_arguments)
+
def __eq__(self, other):
+ if self is other:
+ return True
if not isinstance(other, BoundArguments):
return NotImplemented
return (self.signature == other.signature and
self.arguments == other.arguments)
+ def __setstate__(self, state):
+ self._signature = state['_signature']
+ self.arguments = state['arguments']
+
+ def __getstate__(self):
+ return {'_signature': self._signature, 'arguments': self.arguments}
+
+ def __repr__(self):
+ args = []
+ for arg, value in self.arguments.items():
+ args.append('{}={!r}'.format(arg, value))
+ return '<{} ({})>'.format(self.__class__.__name__, ', '.join(args))
+
class Signature:
- '''A Signature object represents the overall signature of a function.
+ """A Signature object represents the overall signature of a function.
It stores a Parameter object for each parameter accepted by the
function, as well as information specific to the function itself.
@@ -2320,7 +2653,7 @@ class Signature:
* bind_partial(*args, **kwargs) -> BoundArguments
Creates a partial mapping from positional and keyword arguments
to parameters (simulating 'functools.partial' behavior.)
- '''
+ """
__slots__ = ('_return_annotation', '_parameters')
@@ -2331,9 +2664,9 @@ class Signature:
def __init__(self, parameters=None, *, return_annotation=_empty,
__validate_parameters__=True):
- '''Constructs Signature from the given list of Parameter
+ """Constructs Signature from the given list of Parameter
objects and 'return_annotation'. All arguments are optional.
- '''
+ """
if parameters is None:
params = OrderedDict()
@@ -2382,89 +2715,28 @@ class Signature:
@classmethod
def from_function(cls, func):
- '''Constructs Signature for the given python function'''
-
- is_duck_function = False
- if not isfunction(func):
- if _signature_is_functionlike(func):
- is_duck_function = True
- else:
- # If it's not a pure Python function, and not a duck type
- # of pure function:
- raise TypeError('{!r} is not a Python function'.format(func))
-
- Parameter = cls._parameter_cls
-
- # Parameter information.
- func_code = func.__code__
- pos_count = func_code.co_argcount
- arg_names = func_code.co_varnames
- positional = tuple(arg_names[:pos_count])
- keyword_only_count = func_code.co_kwonlyargcount
- keyword_only = arg_names[pos_count:(pos_count + keyword_only_count)]
- annotations = func.__annotations__
- defaults = func.__defaults__
- kwdefaults = func.__kwdefaults__
-
- if defaults:
- pos_default_count = len(defaults)
- else:
- pos_default_count = 0
-
- parameters = []
-
- # Non-keyword-only parameters w/o defaults.
- non_default_count = pos_count - pos_default_count
- for name in positional[:non_default_count]:
- annotation = annotations.get(name, _empty)
- parameters.append(Parameter(name, annotation=annotation,
- kind=_POSITIONAL_OR_KEYWORD))
+ """Constructs Signature for the given python function."""
- # ... w/ defaults.
- for offset, name in enumerate(positional[non_default_count:]):
- annotation = annotations.get(name, _empty)
- parameters.append(Parameter(name, annotation=annotation,
- kind=_POSITIONAL_OR_KEYWORD,
- default=defaults[offset]))
-
- # *args
- if func_code.co_flags & CO_VARARGS:
- name = arg_names[pos_count + keyword_only_count]
- annotation = annotations.get(name, _empty)
- parameters.append(Parameter(name, annotation=annotation,
- kind=_VAR_POSITIONAL))
-
- # Keyword-only parameters.
- for name in keyword_only:
- default = _empty
- if kwdefaults is not None:
- default = kwdefaults.get(name, _empty)
-
- annotation = annotations.get(name, _empty)
- parameters.append(Parameter(name, annotation=annotation,
- kind=_KEYWORD_ONLY,
- default=default))
- # **kwargs
- if func_code.co_flags & CO_VARKEYWORDS:
- index = pos_count + keyword_only_count
- if func_code.co_flags & CO_VARARGS:
- index += 1
-
- name = arg_names[index]
- annotation = annotations.get(name, _empty)
- parameters.append(Parameter(name, annotation=annotation,
- kind=_VAR_KEYWORD))
-
- # Is 'func' is a pure Python function - don't validate the
- # parameters list (for correct order and defaults), it should be OK.
- return cls(parameters,
- return_annotation=annotations.get('return', _empty),
- __validate_parameters__=is_duck_function)
+ warnings.warn("inspect.Signature.from_function() is deprecated, "
+ "use Signature.from_callable()",
+ DeprecationWarning, stacklevel=2)
+ return _signature_from_function(cls, func)
@classmethod
def from_builtin(cls, func):
+ """Constructs Signature for the given builtin function."""
+
+ warnings.warn("inspect.Signature.from_builtin() is deprecated, "
+ "use Signature.from_callable()",
+ DeprecationWarning, stacklevel=2)
return _signature_from_builtin(cls, func)
+ @classmethod
+ def from_callable(cls, obj, *, follow_wrapped=True):
+ """Constructs Signature for the given callable object."""
+ return _signature_from_callable(obj, sigcls=cls,
+ follow_wrapper_chains=follow_wrapped)
+
@property
def parameters(self):
return self._parameters
@@ -2474,10 +2746,10 @@ class Signature:
return self._return_annotation
def replace(self, *, parameters=_void, return_annotation=_void):
- '''Creates a customized copy of the Signature.
+ """Creates a customized copy of the Signature.
Pass 'parameters' and/or 'return_annotation' arguments
to override them in the new copy.
- '''
+ """
if parameters is _void:
parameters = self.parameters.values()
@@ -2488,39 +2760,29 @@ class Signature:
return type(self)(parameters,
return_annotation=return_annotation)
- def __eq__(self, other):
- if not isinstance(other, Signature):
- return NotImplemented
- if (self.return_annotation != other.return_annotation or
- len(self.parameters) != len(other.parameters)):
- return False
+ def _hash_basis(self):
+ params = tuple(param for param in self.parameters.values()
+ if param.kind != _KEYWORD_ONLY)
- other_positions = {param: idx
- for idx, param in enumerate(other.parameters.keys())}
+ kwo_params = {param.name: param for param in self.parameters.values()
+ if param.kind == _KEYWORD_ONLY}
- for idx, (param_name, param) in enumerate(self.parameters.items()):
- if param.kind == _KEYWORD_ONLY:
- try:
- other_param = other.parameters[param_name]
- except KeyError:
- return False
- else:
- if param != other_param:
- return False
- else:
- try:
- other_idx = other_positions[param_name]
- except KeyError:
- return False
- else:
- if (idx != other_idx or
- param != other.parameters[param_name]):
- return False
+ return params, kwo_params, self.return_annotation
+
+ def __hash__(self):
+ params, kwo_params, return_annotation = self._hash_basis()
+ kwo_params = frozenset(kwo_params.values())
+ return hash((params, kwo_params, return_annotation))
- return True
+ def __eq__(self, other):
+ if self is other:
+ return True
+ if not isinstance(other, Signature):
+ return NotImplemented
+ return self._hash_basis() == other._hash_basis()
def _bind(self, args, kwargs, *, partial=False):
- '''Private method. Don't use directly.'''
+ """Private method. Don't use directly."""
arguments = OrderedDict()
@@ -2568,7 +2830,7 @@ class Signature:
parameters_ex = (param,)
break
else:
- msg = '{arg!r} parameter lacking default value'
+ msg = 'missing a required argument: {arg!r}'
msg = msg.format(arg=param.name)
raise TypeError(msg) from None
else:
@@ -2581,7 +2843,8 @@ class Signature:
if param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY):
# Looks like we have no parameter for this positional
# argument
- raise TypeError('too many positional arguments')
+ raise TypeError(
+ 'too many positional arguments') from None
if param.kind == _VAR_POSITIONAL:
# We have an '*args'-like argument, let's fill it with
@@ -2593,8 +2856,9 @@ class Signature:
break
if param.name in kwargs:
- raise TypeError('multiple values for argument '
- '{arg!r}'.format(arg=param.name))
+ raise TypeError(
+ 'multiple values for argument {arg!r}'.format(
+ arg=param.name)) from None
arguments[param.name] = arg_val
@@ -2623,7 +2887,7 @@ class Signature:
# arguments.
if (not partial and param.kind != _VAR_POSITIONAL and
param.default is _empty):
- raise TypeError('{arg!r} parameter lacking default value'. \
+ raise TypeError('missing a required argument: {arg!r}'. \
format(arg=param_name)) from None
else:
@@ -2642,24 +2906,37 @@ class Signature:
# Process our '**kwargs'-like parameter
arguments[kwargs_param.name] = kwargs
else:
- raise TypeError('too many keyword arguments')
+ raise TypeError(
+ 'got an unexpected keyword argument {arg!r}'.format(
+ arg=next(iter(kwargs))))
return self._bound_arguments_cls(self, arguments)
def bind(*args, **kwargs):
- '''Get a BoundArguments object, that maps the passed `args`
+ """Get a BoundArguments object, that maps the passed `args`
and `kwargs` to the function's signature. Raises `TypeError`
if the passed arguments can not be bound.
- '''
+ """
return args[0]._bind(args[1:], kwargs)
def bind_partial(*args, **kwargs):
- '''Get a BoundArguments object, that partially maps the
+ """Get a BoundArguments object, that partially maps the
passed `args` and `kwargs` to the function's signature.
Raises `TypeError` if the passed arguments can not be bound.
- '''
+ """
return args[0]._bind(args[1:], kwargs, partial=True)
+ def __reduce__(self):
+ return (type(self),
+ (tuple(self._parameters.values()),),
+ {'_return_annotation': self._return_annotation})
+
+ def __setstate__(self, state):
+ self._return_annotation = state['_return_annotation']
+
+ def __repr__(self):
+ return '<{} {}>'.format(self.__class__.__name__, self)
+
def __str__(self):
result = []
render_pos_only_separator = False
@@ -2705,6 +2982,12 @@ class Signature:
return rendered
+
+def signature(obj, *, follow_wrapped=True):
+ """Get a signature object for the passed callable."""
+ return Signature.from_callable(obj, follow_wrapped=follow_wrapped)
+
+
def _main():
""" Logic for inspecting an object given at command line """
import argparse
diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py
index ac03c36..7469a9d 100644
--- a/Lib/ipaddress.py
+++ b/Lib/ipaddress.py
@@ -135,7 +135,7 @@ def v4_int_to_packed(address):
"""
try:
return address.to_bytes(4, 'big')
- except:
+ except OverflowError:
raise ValueError("Address negative or too large for IPv4")
@@ -151,7 +151,7 @@ def v6_int_to_packed(address):
"""
try:
return address.to_bytes(16, 'big')
- except:
+ except OverflowError:
raise ValueError("Address negative or too large for IPv6")
@@ -164,22 +164,23 @@ def _split_optional_netmask(address):
def _find_address_range(addresses):
- """Find a sequence of IPv#Address.
+ """Find a sequence of sorted deduplicated IPv#Address.
Args:
addresses: a list of IPv#Address objects.
- Returns:
+ Yields:
A tuple containing the first and last IP addresses in the sequence.
"""
- first = last = addresses[0]
- for ip in addresses[1:]:
- if ip._ip == last._ip + 1:
- last = ip
- else:
- break
- return (first, last)
+ it = iter(addresses)
+ first = last = next(it)
+ for ip in it:
+ if ip._ip != last._ip + 1:
+ yield first, last
+ first = ip
+ last = ip
+ yield first, last
def _count_righthand_zero_bits(number, bits):
@@ -195,11 +196,7 @@ def _count_righthand_zero_bits(number, bits):
"""
if number == 0:
return bits
- for i in range(bits):
- if (number >> i) & 1:
- return i
- # All bits of interest were zero, even if there are more in the number
- return bits
+ return min(bits, (~number & (number-1)).bit_length())
def summarize_address_range(first, last):
@@ -250,15 +247,14 @@ def summarize_address_range(first, last):
while first_int <= last_int:
nbits = min(_count_righthand_zero_bits(first_int, ip_bits),
(last_int - first_int + 1).bit_length() - 1)
- net = ip('%s/%d' % (first, ip_bits - nbits))
+ net = ip((first_int, ip_bits - nbits))
yield net
first_int += 1 << nbits
if first_int - 1 == ip._ALL_ONES:
break
- first = first.__class__(first_int)
-def _collapse_addresses_recursive(addresses):
+def _collapse_addresses_internal(addresses):
"""Loops through the addresses, collapsing concurrent netblocks.
Example:
@@ -268,7 +264,7 @@ def _collapse_addresses_recursive(addresses):
ip3 = IPv4Network('192.0.2.128/26')
ip4 = IPv4Network('192.0.2.192/26')
- _collapse_addresses_recursive([ip1, ip2, ip3, ip4]) ->
+ _collapse_addresses_internal([ip1, ip2, ip3, ip4]) ->
[IPv4Network('192.0.2.0/24')]
This shouldn't be called directly; it is called via
@@ -282,28 +278,29 @@ def _collapse_addresses_recursive(addresses):
passed.
"""
- while True:
- last_addr = None
- ret_array = []
- optimized = False
-
- for cur_addr in addresses:
- if not ret_array:
- last_addr = cur_addr
- ret_array.append(cur_addr)
- elif (cur_addr.network_address >= last_addr.network_address and
- cur_addr.broadcast_address <= last_addr.broadcast_address):
- optimized = True
- elif cur_addr == list(last_addr.supernet().subnets())[1]:
- ret_array[-1] = last_addr = last_addr.supernet()
- optimized = True
- else:
- last_addr = cur_addr
- ret_array.append(cur_addr)
-
- addresses = ret_array
- if not optimized:
- return addresses
+ # First merge
+ to_merge = list(addresses)
+ subnets = {}
+ while to_merge:
+ net = to_merge.pop()
+ supernet = net.supernet()
+ existing = subnets.get(supernet)
+ if existing is None:
+ subnets[supernet] = net
+ elif existing != net:
+ # Merge consecutive subnets
+ del subnets[supernet]
+ to_merge.append(supernet)
+ # Then iterate over resulting networks, skipping subsumed subnets
+ last = None
+ for net in sorted(subnets.values()):
+ if last is not None:
+ # Since they are sorted, last.network_address <= net.network_address
+ # is a given.
+ if last.broadcast_address >= net.broadcast_address:
+ continue
+ yield net
+ last = net
def collapse_addresses(addresses):
@@ -324,7 +321,6 @@ def collapse_addresses(addresses):
TypeError: If passed a list of mixed version objects.
"""
- i = 0
addrs = []
ips = []
nets = []
@@ -352,15 +348,13 @@ def collapse_addresses(addresses):
# sort and dedup
ips = sorted(set(ips))
- nets = sorted(set(nets))
- while i < len(ips):
- (first, last) = _find_address_range(ips[i:])
- i = ips.index(last) + 1
- addrs.extend(summarize_address_range(first, last))
+ # find consecutive address ranges in the sorted sequence and summarize them
+ if ips:
+ for first, last in _find_address_range(ips):
+ addrs.extend(summarize_address_range(first, last))
- return iter(_collapse_addresses_recursive(sorted(
- addrs + nets, key=_BaseNetwork._get_networks_key)))
+ return _collapse_addresses_internal(addrs + nets)
def get_mixed_type_key(obj):
@@ -392,6 +386,8 @@ class _IPAddressBase:
"""The mother class."""
+ __slots__ = ()
+
@property
def exploded(self):
"""Return the longhand version of the IP address as a string."""
@@ -403,6 +399,17 @@ class _IPAddressBase:
return str(self)
@property
+ def reverse_pointer(self):
+ """The name of the reverse DNS pointer for the IP address, e.g.:
+ >>> ipaddress.ip_address("127.0.0.1").reverse_pointer
+ '1.0.0.127.in-addr.arpa'
+ >>> ipaddress.ip_address("2001:db8::1").reverse_pointer
+ '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa'
+
+ """
+ return self._reverse_pointer()
+
+ @property
def version(self):
msg = '%200s has no version specified' % (type(self),)
raise NotImplementedError(msg)
@@ -423,7 +430,8 @@ class _IPAddressBase:
raise AddressValueError(msg % (address, address_len,
expected_len, self._version))
- def _ip_int_from_prefix(self, prefixlen):
+ @classmethod
+ def _ip_int_from_prefix(cls, prefixlen):
"""Turn the prefix length into a bitwise netmask
Args:
@@ -433,9 +441,10 @@ class _IPAddressBase:
An integer.
"""
- return self._ALL_ONES ^ (self._ALL_ONES >> prefixlen)
+ return cls._ALL_ONES ^ (cls._ALL_ONES >> prefixlen)
- def _prefix_from_ip_int(self, ip_int):
+ @classmethod
+ def _prefix_from_ip_int(cls, ip_int):
"""Return prefix length from the bitwise netmask.
Args:
@@ -448,22 +457,24 @@ class _IPAddressBase:
ValueError: If the input intermingles zeroes & ones
"""
trailing_zeroes = _count_righthand_zero_bits(ip_int,
- self._max_prefixlen)
- prefixlen = self._max_prefixlen - trailing_zeroes
+ cls._max_prefixlen)
+ prefixlen = cls._max_prefixlen - trailing_zeroes
leading_ones = ip_int >> trailing_zeroes
all_ones = (1 << prefixlen) - 1
if leading_ones != all_ones:
- byteslen = self._max_prefixlen // 8
+ byteslen = cls._max_prefixlen // 8
details = ip_int.to_bytes(byteslen, 'big')
msg = 'Netmask pattern %r mixes zeroes & ones'
raise ValueError(msg % details)
return prefixlen
- def _report_invalid_netmask(self, netmask_str):
+ @classmethod
+ def _report_invalid_netmask(cls, netmask_str):
msg = '%r is not a valid netmask' % netmask_str
raise NetmaskValueError(msg) from None
- def _prefix_from_prefix_string(self, prefixlen_str):
+ @classmethod
+ def _prefix_from_prefix_string(cls, prefixlen_str):
"""Return prefix length from a numeric string
Args:
@@ -478,16 +489,17 @@ class _IPAddressBase:
# int allows a leading +/- as well as surrounding whitespace,
# so we ensure that isn't the case
if not _BaseV4._DECIMAL_DIGITS.issuperset(prefixlen_str):
- self._report_invalid_netmask(prefixlen_str)
+ cls._report_invalid_netmask(prefixlen_str)
try:
prefixlen = int(prefixlen_str)
except ValueError:
- self._report_invalid_netmask(prefixlen_str)
- if not (0 <= prefixlen <= self._max_prefixlen):
- self._report_invalid_netmask(prefixlen_str)
+ cls._report_invalid_netmask(prefixlen_str)
+ if not (0 <= prefixlen <= cls._max_prefixlen):
+ cls._report_invalid_netmask(prefixlen_str)
return prefixlen
- def _prefix_from_ip_string(self, ip_str):
+ @classmethod
+ def _prefix_from_ip_string(cls, ip_str):
"""Turn a netmask/hostmask string into a prefix length
Args:
@@ -501,24 +513,27 @@ class _IPAddressBase:
"""
# Parse the netmask/hostmask like an IP address.
try:
- ip_int = self._ip_int_from_string(ip_str)
+ ip_int = cls._ip_int_from_string(ip_str)
except AddressValueError:
- self._report_invalid_netmask(ip_str)
+ cls._report_invalid_netmask(ip_str)
# Try matching a netmask (this would be /1*0*/ as a bitwise regexp).
# Note that the two ambiguous cases (all-ones and all-zeroes) are
# treated as netmasks.
try:
- return self._prefix_from_ip_int(ip_int)
+ return cls._prefix_from_ip_int(ip_int)
except ValueError:
pass
# Invert the bits, and try matching a /0+1+/ hostmask instead.
- ip_int ^= self._ALL_ONES
+ ip_int ^= cls._ALL_ONES
try:
- return self._prefix_from_ip_int(ip_int)
+ return cls._prefix_from_ip_int(ip_int)
except ValueError:
- self._report_invalid_netmask(ip_str)
+ cls._report_invalid_netmask(ip_str)
+
+ def __reduce__(self):
+ return self.__class__, (str(self),)
@functools.total_ordering
@@ -530,10 +545,7 @@ class _BaseAddress(_IPAddressBase):
used by single IP addresses.
"""
- def __init__(self, address):
- if (not isinstance(address, bytes)
- and '/' in str(address)):
- raise AddressValueError("Unexpected '/' in %r" % address)
+ __slots__ = ()
def __int__(self):
return self._ip
@@ -579,6 +591,9 @@ class _BaseAddress(_IPAddressBase):
def _get_address_key(self):
return (self._version, self)
+ def __reduce__(self):
+ return self.__class__, (self._ip,)
+
@functools.total_ordering
class _BaseNetwork(_IPAddressBase):
@@ -765,7 +780,7 @@ class _BaseNetwork(_IPAddressBase):
other.broadcast_address <= self.broadcast_address):
raise ValueError('%s not contained in %s' % (other, self))
if other == self:
- raise StopIteration
+ return
# Make sure we're comparing the network of other.
other = other.__class__('%s/%s' % (other.network_address,
@@ -900,20 +915,11 @@ class _BaseNetwork(_IPAddressBase):
'prefix length diff %d is invalid for netblock %s' % (
new_prefixlen, self))
- first = self.__class__('%s/%s' %
- (self.network_address,
- self._prefixlen + prefixlen_diff))
-
- yield first
- current = first
- while True:
- broadcast = current.broadcast_address
- if broadcast == self.broadcast_address:
- return
- new_addr = self._address_class(int(broadcast) + 1)
- current = self.__class__('%s/%s' % (new_addr,
- new_prefixlen))
-
+ start = int(self.network_address)
+ end = int(self.broadcast_address)
+ step = (int(self.hostmask) + 1) >> prefixlen_diff
+ for new_addr in range(start, end, step):
+ current = self.__class__((new_addr, new_prefixlen))
yield current
def supernet(self, prefixlen_diff=1, new_prefix=None):
@@ -947,15 +953,15 @@ class _BaseNetwork(_IPAddressBase):
raise ValueError('cannot set prefixlen_diff and new_prefix')
prefixlen_diff = self._prefixlen - new_prefix
- if self.prefixlen - prefixlen_diff < 0:
+ new_prefixlen = self.prefixlen - prefixlen_diff
+ if new_prefixlen < 0:
raise ValueError(
'current prefixlen is %d, cannot have a prefixlen_diff of %d' %
(self.prefixlen, prefixlen_diff))
- # TODO (pmoody): optimize this.
- t = self.__class__('%s/%d' % (self.network_address,
- self.prefixlen - prefixlen_diff),
- strict=False)
- return t.__class__('%s/%d' % (t.network_address, t.prefixlen))
+ return self.__class__((
+ int(self.network_address) & (int(self.netmask) << prefixlen_diff),
+ new_prefixlen
+ ))
@property
def is_multicast(self):
@@ -1049,21 +1055,49 @@ class _BaseV4:
"""
+ __slots__ = ()
+ _version = 4
# Equivalent to 255.255.255.255 or 32 bits of 1's.
_ALL_ONES = (2**IPV4LENGTH) - 1
_DECIMAL_DIGITS = frozenset('0123456789')
# the valid octets for host and netmasks. only useful for IPv4.
- _valid_mask_octets = frozenset((255, 254, 252, 248, 240, 224, 192, 128, 0))
+ _valid_mask_octets = frozenset({255, 254, 252, 248, 240, 224, 192, 128, 0})
- def __init__(self, address):
- self._version = 4
- self._max_prefixlen = IPV4LENGTH
+ _max_prefixlen = IPV4LENGTH
+ # There are only a handful of valid v4 netmasks, so we cache them all
+ # when constructed (see _make_netmask()).
+ _netmask_cache = {}
def _explode_shorthand_ip_string(self):
return str(self)
- def _ip_int_from_string(self, ip_str):
+ @classmethod
+ def _make_netmask(cls, arg):
+ """Make a (netmask, prefix_len) tuple from the given argument.
+
+ Argument can be:
+ - an integer (the prefix length)
+ - a string representing the prefix length (e.g. "24")
+ - a string representing the prefix netmask (e.g. "255.255.255.0")
+ """
+ if arg not in cls._netmask_cache:
+ if isinstance(arg, int):
+ prefixlen = arg
+ else:
+ try:
+ # Check for a netmask in prefix length form
+ prefixlen = cls._prefix_from_prefix_string(arg)
+ except NetmaskValueError:
+ # Check for a netmask or hostmask in dotted-quad form.
+ # This may raise NetmaskValueError.
+ prefixlen = cls._prefix_from_ip_string(arg)
+ netmask = IPv4Address(cls._ip_int_from_prefix(prefixlen))
+ cls._netmask_cache[arg] = netmask, prefixlen
+ return cls._netmask_cache[arg]
+
+ @classmethod
+ def _ip_int_from_string(cls, ip_str):
"""Turn the given IP string into an integer for comparison.
Args:
@@ -1084,11 +1118,12 @@ class _BaseV4:
raise AddressValueError("Expected 4 octets in %r" % ip_str)
try:
- return int.from_bytes(map(self._parse_octet, octets), 'big')
+ return int.from_bytes(map(cls._parse_octet, octets), 'big')
except ValueError as exc:
raise AddressValueError("%s in %r" % (exc, ip_str)) from None
- def _parse_octet(self, octet_str):
+ @classmethod
+ def _parse_octet(cls, octet_str):
"""Convert a decimal octet into an integer.
Args:
@@ -1104,7 +1139,7 @@ class _BaseV4:
if not octet_str:
raise ValueError("Empty octet not permitted")
# Whitelist the characters, since int() allows a lot of bizarre stuff.
- if not self._DECIMAL_DIGITS.issuperset(octet_str):
+ if not cls._DECIMAL_DIGITS.issuperset(octet_str):
msg = "Only decimal digits permitted in %r"
raise ValueError(msg % octet_str)
# We do the length check second, since the invalid character error
@@ -1124,7 +1159,8 @@ class _BaseV4:
raise ValueError("Octet %d (> 255) not permitted" % octet_int)
return octet_int
- def _string_from_ip_int(self, ip_int):
+ @classmethod
+ def _string_from_ip_int(cls, ip_int):
"""Turns a 32-bit integer into dotted decimal notation.
Args:
@@ -1188,6 +1224,15 @@ class _BaseV4:
return True
return False
+ def _reverse_pointer(self):
+ """Return the reverse DNS pointer name for the IPv4 address.
+
+ This implements the method described in RFC1035 3.5.
+
+ """
+ reverse_octets = str(self).split('.')[::-1]
+ return '.'.join(reverse_octets) + '.in-addr.arpa'
+
@property
def max_prefixlen(self):
return self._max_prefixlen
@@ -1201,6 +1246,8 @@ class IPv4Address(_BaseV4, _BaseAddress):
"""Represent and manipulate single IPv4 Addresses."""
+ __slots__ = ('_ip', '__weakref__')
+
def __init__(self, address):
"""
@@ -1217,9 +1264,6 @@ class IPv4Address(_BaseV4, _BaseAddress):
AddressValueError: If ipaddress isn't a valid IPv4 address.
"""
- _BaseAddress.__init__(self, address)
- _BaseV4.__init__(self, address)
-
# Efficient constructor from integer.
if isinstance(address, int):
self._check_int_address(address)
@@ -1235,6 +1279,8 @@ class IPv4Address(_BaseV4, _BaseAddress):
# Assume input argument to be string or any object representation
# which converts into a formatted IP string.
addr_str = str(address)
+ if '/' in addr_str:
+ raise AddressValueError("Unexpected '/' in %r" % address)
self._ip = self._ip_int_from_string(addr_str)
@property
@@ -1251,8 +1297,7 @@ class IPv4Address(_BaseV4, _BaseAddress):
reserved IPv4 Network range.
"""
- reserved_network = IPv4Network('240.0.0.0/4')
- return self in reserved_network
+ return self in self._constants._reserved_network
@property
@functools.lru_cache()
@@ -1264,21 +1309,7 @@ class IPv4Address(_BaseV4, _BaseAddress):
iana-ipv4-special-registry.
"""
- return (self in IPv4Network('0.0.0.0/8') or
- self in IPv4Network('10.0.0.0/8') or
- self in IPv4Network('127.0.0.0/8') or
- self in IPv4Network('169.254.0.0/16') or
- self in IPv4Network('172.16.0.0/12') or
- self in IPv4Network('192.0.0.0/29') or
- self in IPv4Network('192.0.0.170/31') or
- self in IPv4Network('192.0.2.0/24') or
- self in IPv4Network('192.168.0.0/16') or
- self in IPv4Network('198.18.0.0/15') or
- self in IPv4Network('198.51.100.0/24') or
- self in IPv4Network('203.0.113.0/24') or
- self in IPv4Network('240.0.0.0/4') or
- self in IPv4Network('255.255.255.255/32'))
-
+ return any(self in net for net in self._constants._private_networks)
@property
def is_multicast(self):
@@ -1289,8 +1320,7 @@ class IPv4Address(_BaseV4, _BaseAddress):
See RFC 3171 for details.
"""
- multicast_network = IPv4Network('224.0.0.0/4')
- return self in multicast_network
+ return self in self._constants._multicast_network
@property
def is_unspecified(self):
@@ -1301,8 +1331,7 @@ class IPv4Address(_BaseV4, _BaseAddress):
RFC 5735 3.
"""
- unspecified_address = IPv4Address('0.0.0.0')
- return self == unspecified_address
+ return self == self._constants._unspecified_address
@property
def is_loopback(self):
@@ -1312,8 +1341,7 @@ class IPv4Address(_BaseV4, _BaseAddress):
A boolean, True if the address is a loopback per RFC 3330.
"""
- loopback_network = IPv4Network('127.0.0.0/8')
- return self in loopback_network
+ return self in self._constants._loopback_network
@property
def is_link_local(self):
@@ -1323,8 +1351,7 @@ class IPv4Address(_BaseV4, _BaseAddress):
A boolean, True if the address is link-local per RFC 3927.
"""
- linklocal_network = IPv4Network('169.254.0.0/16')
- return self in linklocal_network
+ return self in self._constants._linklocal_network
class IPv4Interface(IPv4Address):
@@ -1336,6 +1363,18 @@ class IPv4Interface(IPv4Address):
self._prefixlen = self._max_prefixlen
return
+ if isinstance(address, tuple):
+ IPv4Address.__init__(self, address[0])
+ if len(address) > 1:
+ self._prefixlen = int(address[1])
+ else:
+ self._prefixlen = self._max_prefixlen
+
+ self.network = IPv4Network(address, strict=False)
+ self.netmask = self.network.netmask
+ self.hostmask = self.network.hostmask
+ return
+
addr = _split_optional_netmask(address)
IPv4Address.__init__(self, addr[0])
@@ -1375,6 +1414,8 @@ class IPv4Interface(IPv4Address):
def __hash__(self):
return self._ip ^ self._prefixlen ^ int(self.network.network_address)
+ __reduce__ = _IPAddressBase.__reduce__
+
@property
def ip(self):
return IPv4Address(self._ip)
@@ -1447,24 +1488,30 @@ class IPv4Network(_BaseV4, _BaseNetwork):
supplied.
"""
-
- _BaseV4.__init__(self, address)
_BaseNetwork.__init__(self, address)
- # Constructing from a packed address
- if isinstance(address, bytes):
+ # Constructing from a packed address or integer
+ if isinstance(address, (int, bytes)):
self.network_address = IPv4Address(address)
- self._prefixlen = self._max_prefixlen
- self.netmask = IPv4Address(self._ALL_ONES)
- #fixme: address/network test here
+ self.netmask, self._prefixlen = self._make_netmask(self._max_prefixlen)
+ #fixme: address/network test here.
return
- # Efficient constructor from integer.
- if isinstance(address, int):
- self.network_address = IPv4Address(address)
- self._prefixlen = self._max_prefixlen
- self.netmask = IPv4Address(self._ALL_ONES)
- #fixme: address/network test here.
+ if isinstance(address, tuple):
+ if len(address) > 1:
+ arg = address[1]
+ else:
+ # We weren't given an address[1]
+ arg = self._max_prefixlen
+ self.network_address = IPv4Address(address[0])
+ self.netmask, self._prefixlen = self._make_netmask(arg)
+ packed = int(self.network_address)
+ if packed & int(self.netmask) != packed:
+ if strict:
+ raise ValueError('%s has host bits set' % self)
+ else:
+ self.network_address = IPv4Address(packed &
+ int(self.netmask))
return
# Assume input argument to be string or any object representation
@@ -1473,16 +1520,10 @@ class IPv4Network(_BaseV4, _BaseNetwork):
self.network_address = IPv4Address(self._ip_int_from_string(addr[0]))
if len(addr) == 2:
- try:
- # Check for a netmask in prefix length form
- self._prefixlen = self._prefix_from_prefix_string(addr[1])
- except NetmaskValueError:
- # Check for a netmask or hostmask in dotted-quad form.
- # This may raise NetmaskValueError.
- self._prefixlen = self._prefix_from_ip_string(addr[1])
+ arg = addr[1]
else:
- self._prefixlen = self._max_prefixlen
- self.netmask = IPv4Address(self._ip_int_from_prefix(self._prefixlen))
+ arg = self._max_prefixlen
+ self.netmask, self._prefixlen = self._make_netmask(arg)
if strict:
if (IPv4Address(int(self.network_address) & int(self.netmask)) !=
@@ -1509,6 +1550,37 @@ class IPv4Network(_BaseV4, _BaseNetwork):
not self.is_private)
+class _IPv4Constants:
+ _linklocal_network = IPv4Network('169.254.0.0/16')
+
+ _loopback_network = IPv4Network('127.0.0.0/8')
+
+ _multicast_network = IPv4Network('224.0.0.0/4')
+
+ _private_networks = [
+ IPv4Network('0.0.0.0/8'),
+ IPv4Network('10.0.0.0/8'),
+ IPv4Network('127.0.0.0/8'),
+ IPv4Network('169.254.0.0/16'),
+ IPv4Network('172.16.0.0/12'),
+ IPv4Network('192.0.0.0/29'),
+ IPv4Network('192.0.0.170/31'),
+ IPv4Network('192.0.2.0/24'),
+ IPv4Network('192.168.0.0/16'),
+ IPv4Network('198.18.0.0/15'),
+ IPv4Network('198.51.100.0/24'),
+ IPv4Network('203.0.113.0/24'),
+ IPv4Network('240.0.0.0/4'),
+ IPv4Network('255.255.255.255/32'),
+ ]
+
+ _reserved_network = IPv4Network('240.0.0.0/4')
+
+ _unspecified_address = IPv4Address('0.0.0.0')
+
+
+IPv4Address._constants = _IPv4Constants
+
class _BaseV6:
@@ -1519,15 +1591,37 @@ class _BaseV6:
"""
+ __slots__ = ()
+ _version = 6
_ALL_ONES = (2**IPV6LENGTH) - 1
_HEXTET_COUNT = 8
_HEX_DIGITS = frozenset('0123456789ABCDEFabcdef')
+ _max_prefixlen = IPV6LENGTH
- def __init__(self, address):
- self._version = 6
- self._max_prefixlen = IPV6LENGTH
+ # There are only a bunch of valid v6 netmasks, so we cache them all
+ # when constructed (see _make_netmask()).
+ _netmask_cache = {}
+
+ @classmethod
+ def _make_netmask(cls, arg):
+ """Make a (netmask, prefix_len) tuple from the given argument.
- def _ip_int_from_string(self, ip_str):
+ Argument can be:
+ - an integer (the prefix length)
+ - a string representing the prefix length (e.g. "24")
+ - a string representing the prefix netmask (e.g. "255.255.255.0")
+ """
+ if arg not in cls._netmask_cache:
+ if isinstance(arg, int):
+ prefixlen = arg
+ else:
+ prefixlen = cls._prefix_from_prefix_string(arg)
+ netmask = IPv6Address(cls._ip_int_from_prefix(prefixlen))
+ cls._netmask_cache[arg] = netmask, prefixlen
+ return cls._netmask_cache[arg]
+
+ @classmethod
+ def _ip_int_from_string(cls, ip_str):
"""Turn an IPv6 ip_str into an integer.
Args:
@@ -1563,7 +1657,7 @@ class _BaseV6:
# An IPv6 address can't have more than 8 colons (9 parts).
# The extra colon comes from using the "::" notation for a single
# leading or trailing zero part.
- _max_parts = self._HEXTET_COUNT + 1
+ _max_parts = cls._HEXTET_COUNT + 1
if len(parts) > _max_parts:
msg = "At most %d colons permitted in %r" % (_max_parts-1, ip_str)
raise AddressValueError(msg)
@@ -1595,17 +1689,17 @@ class _BaseV6:
if parts_lo:
msg = "Trailing ':' only permitted as part of '::' in %r"
raise AddressValueError(msg % ip_str) # :$ requires ::$
- parts_skipped = self._HEXTET_COUNT - (parts_hi + parts_lo)
+ parts_skipped = cls._HEXTET_COUNT - (parts_hi + parts_lo)
if parts_skipped < 1:
msg = "Expected at most %d other parts with '::' in %r"
- raise AddressValueError(msg % (self._HEXTET_COUNT-1, ip_str))
+ raise AddressValueError(msg % (cls._HEXTET_COUNT-1, ip_str))
else:
# Otherwise, allocate the entire address to parts_hi. The
# endpoints could still be empty, but _parse_hextet() will check
# for that.
- if len(parts) != self._HEXTET_COUNT:
+ if len(parts) != cls._HEXTET_COUNT:
msg = "Exactly %d parts expected without '::' in %r"
- raise AddressValueError(msg % (self._HEXTET_COUNT, ip_str))
+ raise AddressValueError(msg % (cls._HEXTET_COUNT, ip_str))
if not parts[0]:
msg = "Leading ':' only permitted as part of '::' in %r"
raise AddressValueError(msg % ip_str) # ^: requires ^::
@@ -1621,16 +1715,17 @@ class _BaseV6:
ip_int = 0
for i in range(parts_hi):
ip_int <<= 16
- ip_int |= self._parse_hextet(parts[i])
+ ip_int |= cls._parse_hextet(parts[i])
ip_int <<= 16 * parts_skipped
for i in range(-parts_lo, 0):
ip_int <<= 16
- ip_int |= self._parse_hextet(parts[i])
+ ip_int |= cls._parse_hextet(parts[i])
return ip_int
except ValueError as exc:
raise AddressValueError("%s in %r" % (exc, ip_str)) from None
- def _parse_hextet(self, hextet_str):
+ @classmethod
+ def _parse_hextet(cls, hextet_str):
"""Convert an IPv6 hextet string into an integer.
Args:
@@ -1645,7 +1740,7 @@ class _BaseV6:
"""
# Whitelist the characters, since int() allows a lot of bizarre stuff.
- if not self._HEX_DIGITS.issuperset(hextet_str):
+ if not cls._HEX_DIGITS.issuperset(hextet_str):
raise ValueError("Only hex digits permitted in %r" % hextet_str)
# We do the length check second, since the invalid character error
# is likely to be more informative for the user
@@ -1655,7 +1750,8 @@ class _BaseV6:
# Length check means we can skip checking the integer value
return int(hextet_str, 16)
- def _compress_hextets(self, hextets):
+ @classmethod
+ def _compress_hextets(cls, hextets):
"""Compresses a list of hextets.
Compresses a list of strings, replacing the longest continuous
@@ -1702,7 +1798,8 @@ class _BaseV6:
return hextets
- def _string_from_ip_int(self, ip_int=None):
+ @classmethod
+ def _string_from_ip_int(cls, ip_int=None):
"""Turns a 128-bit integer into hexadecimal notation.
Args:
@@ -1716,15 +1813,15 @@ class _BaseV6:
"""
if ip_int is None:
- ip_int = int(self._ip)
+ ip_int = int(cls._ip)
- if ip_int > self._ALL_ONES:
+ if ip_int > cls._ALL_ONES:
raise ValueError('IPv6 address is too large')
hex_str = '%032x' % ip_int
hextets = ['%x' % int(hex_str[x:x+4], 16) for x in range(0, 32, 4)]
- hextets = self._compress_hextets(hextets)
+ hextets = cls._compress_hextets(hextets)
return ':'.join(hextets)
def _explode_shorthand_ip_string(self):
@@ -1751,6 +1848,15 @@ class _BaseV6:
return '%s/%d' % (':'.join(parts), self._prefixlen)
return ':'.join(parts)
+ def _reverse_pointer(self):
+ """Return the reverse DNS pointer name for the IPv6 address.
+
+ This implements the method described in RFC3596 2.5.
+
+ """
+ reverse_chars = self.exploded[::-1].replace(':', '')
+ return '.'.join(reverse_chars) + '.ip6.arpa'
+
@property
def max_prefixlen(self):
return self._max_prefixlen
@@ -1764,6 +1870,8 @@ class IPv6Address(_BaseV6, _BaseAddress):
"""Represent and manipulate single IPv6 Addresses."""
+ __slots__ = ('_ip', '__weakref__')
+
def __init__(self, address):
"""Instantiate a new IPv6 address object.
@@ -1781,9 +1889,6 @@ class IPv6Address(_BaseV6, _BaseAddress):
AddressValueError: If address isn't a valid IPv6 address.
"""
- _BaseAddress.__init__(self, address)
- _BaseV6.__init__(self, address)
-
# Efficient constructor from integer.
if isinstance(address, int):
self._check_int_address(address)
@@ -1799,6 +1904,8 @@ class IPv6Address(_BaseV6, _BaseAddress):
# Assume input argument to be string or any object representation
# which converts into a formatted IP string.
addr_str = str(address)
+ if '/' in addr_str:
+ raise AddressValueError("Unexpected '/' in %r" % address)
self._ip = self._ip_int_from_string(addr_str)
@property
@@ -1815,8 +1922,7 @@ class IPv6Address(_BaseV6, _BaseAddress):
See RFC 2373 2.7 for details.
"""
- multicast_network = IPv6Network('ff00::/8')
- return self in multicast_network
+ return self in self._constants._multicast_network
@property
def is_reserved(self):
@@ -1827,16 +1933,7 @@ class IPv6Address(_BaseV6, _BaseAddress):
reserved IPv6 Network ranges.
"""
- reserved_networks = [IPv6Network('::/8'), IPv6Network('100::/8'),
- IPv6Network('200::/7'), IPv6Network('400::/6'),
- IPv6Network('800::/5'), IPv6Network('1000::/4'),
- IPv6Network('4000::/3'), IPv6Network('6000::/3'),
- IPv6Network('8000::/3'), IPv6Network('A000::/3'),
- IPv6Network('C000::/3'), IPv6Network('E000::/4'),
- IPv6Network('F000::/5'), IPv6Network('F800::/6'),
- IPv6Network('FE00::/9')]
-
- return any(self in x for x in reserved_networks)
+ return any(self in x for x in self._constants._reserved_networks)
@property
def is_link_local(self):
@@ -1846,8 +1943,7 @@ class IPv6Address(_BaseV6, _BaseAddress):
A boolean, True if the address is reserved per RFC 4291.
"""
- linklocal_network = IPv6Network('fe80::/10')
- return self in linklocal_network
+ return self in self._constants._linklocal_network
@property
def is_site_local(self):
@@ -1861,8 +1957,7 @@ class IPv6Address(_BaseV6, _BaseAddress):
A boolean, True if the address is reserved per RFC 3513 2.5.6.
"""
- sitelocal_network = IPv6Network('fec0::/10')
- return self in sitelocal_network
+ return self in self._constants._sitelocal_network
@property
@functools.lru_cache()
@@ -1874,16 +1969,7 @@ class IPv6Address(_BaseV6, _BaseAddress):
iana-ipv6-special-registry.
"""
- return (self in IPv6Network('::1/128') or
- self in IPv6Network('::/128') or
- self in IPv6Network('::ffff:0:0/96') or
- self in IPv6Network('100::/64') or
- self in IPv6Network('2001::/23') or
- self in IPv6Network('2001:2::/48') or
- self in IPv6Network('2001:db8::/32') or
- self in IPv6Network('2001:10::/28') or
- self in IPv6Network('fc00::/7') or
- self in IPv6Network('fe80::/10'))
+ return any(self in net for net in self._constants._private_networks)
@property
def is_global(self):
@@ -1968,6 +2054,16 @@ class IPv6Interface(IPv6Address):
self.network = IPv6Network(self._ip)
self._prefixlen = self._max_prefixlen
return
+ if isinstance(address, tuple):
+ IPv6Address.__init__(self, address[0])
+ if len(address) > 1:
+ self._prefixlen = int(address[1])
+ else:
+ self._prefixlen = self._max_prefixlen
+ self.network = IPv6Network(address, strict=False)
+ self.netmask = self.network.netmask
+ self.hostmask = self.network.hostmask
+ return
addr = _split_optional_netmask(address)
IPv6Address.__init__(self, addr[0])
@@ -2006,6 +2102,8 @@ class IPv6Interface(IPv6Address):
def __hash__(self):
return self._ip ^ self._prefixlen ^ int(self.network.network_address)
+ __reduce__ = _IPAddressBase.__reduce__
+
@property
def ip(self):
return IPv6Address(self._ip)
@@ -2082,21 +2180,28 @@ class IPv6Network(_BaseV6, _BaseNetwork):
supplied.
"""
- _BaseV6.__init__(self, address)
_BaseNetwork.__init__(self, address)
- # Efficient constructor from integer.
- if isinstance(address, int):
+ # Efficient constructor from integer or packed address
+ if isinstance(address, (bytes, int)):
self.network_address = IPv6Address(address)
- self._prefixlen = self._max_prefixlen
- self.netmask = IPv6Address(self._ALL_ONES)
+ self.netmask, self._prefixlen = self._make_netmask(self._max_prefixlen)
return
- # Constructing from a packed address
- if isinstance(address, bytes):
- self.network_address = IPv6Address(address)
- self._prefixlen = self._max_prefixlen
- self.netmask = IPv6Address(self._ALL_ONES)
+ if isinstance(address, tuple):
+ if len(address) > 1:
+ arg = address[1]
+ else:
+ arg = self._max_prefixlen
+ self.netmask, self._prefixlen = self._make_netmask(arg)
+ self.network_address = IPv6Address(address[0])
+ packed = int(self.network_address)
+ if packed & int(self.netmask) != packed:
+ if strict:
+ raise ValueError('%s has host bits set' % self)
+ else:
+ self.network_address = IPv6Address(packed &
+ int(self.netmask))
return
# Assume input argument to be string or any object representation
@@ -2106,12 +2211,11 @@ class IPv6Network(_BaseV6, _BaseNetwork):
self.network_address = IPv6Address(self._ip_int_from_string(addr[0]))
if len(addr) == 2:
- # This may raise NetmaskValueError
- self._prefixlen = self._prefix_from_prefix_string(addr[1])
+ arg = addr[1]
else:
- self._prefixlen = self._max_prefixlen
+ arg = self._max_prefixlen
+ self.netmask, self._prefixlen = self._make_netmask(arg)
- self.netmask = IPv6Address(self._ip_int_from_prefix(self._prefixlen))
if strict:
if (IPv6Address(int(self.network_address) & int(self.netmask)) !=
self.network_address):
@@ -2148,3 +2252,39 @@ class IPv6Network(_BaseV6, _BaseNetwork):
"""
return (self.network_address.is_site_local and
self.broadcast_address.is_site_local)
+
+
+class _IPv6Constants:
+
+ _linklocal_network = IPv6Network('fe80::/10')
+
+ _multicast_network = IPv6Network('ff00::/8')
+
+ _private_networks = [
+ IPv6Network('::1/128'),
+ IPv6Network('::/128'),
+ IPv6Network('::ffff:0:0/96'),
+ IPv6Network('100::/64'),
+ IPv6Network('2001::/23'),
+ IPv6Network('2001:2::/48'),
+ IPv6Network('2001:db8::/32'),
+ IPv6Network('2001:10::/28'),
+ IPv6Network('fc00::/7'),
+ IPv6Network('fe80::/10'),
+ ]
+
+ _reserved_networks = [
+ IPv6Network('::/8'), IPv6Network('100::/8'),
+ IPv6Network('200::/7'), IPv6Network('400::/6'),
+ IPv6Network('800::/5'), IPv6Network('1000::/4'),
+ IPv6Network('4000::/3'), IPv6Network('6000::/3'),
+ IPv6Network('8000::/3'), IPv6Network('A000::/3'),
+ IPv6Network('C000::/3'), IPv6Network('E000::/4'),
+ IPv6Network('F000::/5'), IPv6Network('F800::/6'),
+ IPv6Network('FE00::/9'),
+ ]
+
+ _sitelocal_network = IPv6Network('fec0::/10')
+
+
+IPv6Address._constants = _IPv6Constants
diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py
index 9398667..2612657 100644
--- a/Lib/json/__init__.py
+++ b/Lib/json/__init__.py
@@ -98,12 +98,12 @@ Using json.tool from the shell to validate and pretty-print::
__version__ = '2.0.9'
__all__ = [
'dump', 'dumps', 'load', 'loads',
- 'JSONDecoder', 'JSONEncoder',
+ 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder',
]
__author__ = 'Bob Ippolito <bob@redivi.com>'
-from .decoder import JSONDecoder
+from .decoder import JSONDecoder, JSONDecodeError
from .encoder import JSONEncoder
_default_encoder = JSONEncoder(
@@ -311,7 +311,8 @@ def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None,
raise TypeError('the JSON object must be str, not {!r}'.format(
s.__class__.__name__))
if s.startswith(u'\ufeff'):
- raise ValueError("Unexpected UTF-8 BOM (decode using utf-8-sig)")
+ raise JSONDecodeError("Unexpected UTF-8 BOM (decode using utf-8-sig)",
+ s, 0)
if (cls is None and object_hook is None and
parse_int is None and parse_float is None and
parse_constant is None and object_pairs_hook is None and not kw):
diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py
index 59e5f41..0f03f20 100644
--- a/Lib/json/decoder.py
+++ b/Lib/json/decoder.py
@@ -8,7 +8,7 @@ try:
except ImportError:
c_scanstring = None
-__all__ = ['JSONDecoder']
+__all__ = ['JSONDecoder', 'JSONDecodeError']
FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
@@ -17,32 +17,30 @@ PosInf = float('inf')
NegInf = float('-inf')
-def linecol(doc, pos):
- if isinstance(doc, bytes):
- newline = b'\n'
- else:
- newline = '\n'
- lineno = doc.count(newline, 0, pos) + 1
- if lineno == 1:
- colno = pos + 1
- else:
- colno = pos - doc.rindex(newline, 0, pos)
- return lineno, colno
-
-
-def errmsg(msg, doc, pos, end=None):
- # Note that this function is called from _json
- lineno, colno = linecol(doc, pos)
- if end is None:
- fmt = '{0}: line {1} column {2} (char {3})'
- return fmt.format(msg, lineno, colno, pos)
- #fmt = '%s: line %d column %d (char %d)'
- #return fmt % (msg, lineno, colno, pos)
- endlineno, endcolno = linecol(doc, end)
- fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})'
- return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end)
- #fmt = '%s: line %d column %d - line %d column %d (char %d - %d)'
- #return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end)
+class JSONDecodeError(ValueError):
+ """Subclass of ValueError with the following additional properties:
+
+ msg: The unformatted error message
+ doc: The JSON document being parsed
+ pos: The start index of doc where parsing failed
+ lineno: The line corresponding to pos
+ colno: The column corresponding to pos
+
+ """
+ # Note that this exception is used from _json
+ def __init__(self, msg, doc, pos):
+ lineno = doc.count('\n', 0, pos) + 1
+ colno = pos - doc.rfind('\n', 0, pos)
+ errmsg = '%s: line %d column %d (char %d)' % (msg, lineno, colno, pos)
+ ValueError.__init__(self, errmsg)
+ self.msg = msg
+ self.doc = doc
+ self.pos = pos
+ self.lineno = lineno
+ self.colno = colno
+
+ def __reduce__(self):
+ return self.__class__, (self.msg, self.doc, self.pos)
_CONSTANTS = {
@@ -66,7 +64,7 @@ def _decode_uXXXX(s, pos):
except ValueError:
pass
msg = "Invalid \\uXXXX escape"
- raise ValueError(errmsg(msg, s, pos))
+ raise JSONDecodeError(msg, s, pos)
def py_scanstring(s, end, strict=True,
_b=BACKSLASH, _m=STRINGCHUNK.match):
@@ -84,8 +82,7 @@ def py_scanstring(s, end, strict=True,
while 1:
chunk = _m(s, end)
if chunk is None:
- raise ValueError(
- errmsg("Unterminated string starting at", s, begin))
+ raise JSONDecodeError("Unterminated string starting at", s, begin)
end = chunk.end()
content, terminator = chunk.groups()
# Content is contains zero or more unescaped string characters
@@ -99,22 +96,21 @@ def py_scanstring(s, end, strict=True,
if strict:
#msg = "Invalid control character %r at" % (terminator,)
msg = "Invalid control character {0!r} at".format(terminator)
- raise ValueError(errmsg(msg, s, end))
+ raise JSONDecodeError(msg, s, end)
else:
_append(terminator)
continue
try:
esc = s[end]
except IndexError:
- raise ValueError(
- errmsg("Unterminated string starting at", s, begin))
+ raise JSONDecodeError("Unterminated string starting at", s, begin)
# If not a unicode escape sequence, must be in the lookup table
if esc != 'u':
try:
char = _b[esc]
except KeyError:
msg = "Invalid \\escape: {0!r}".format(esc)
- raise ValueError(errmsg(msg, s, end))
+ raise JSONDecodeError(msg, s, end)
end += 1
else:
uni = _decode_uXXXX(s, end)
@@ -163,8 +159,8 @@ def JSONObject(s_and_end, strict, scan_once, object_hook, object_pairs_hook,
pairs = object_hook(pairs)
return pairs, end + 1
elif nextchar != '"':
- raise ValueError(errmsg(
- "Expecting property name enclosed in double quotes", s, end))
+ raise JSONDecodeError(
+ "Expecting property name enclosed in double quotes", s, end)
end += 1
while True:
key, end = scanstring(s, end, strict)
@@ -174,7 +170,7 @@ def JSONObject(s_and_end, strict, scan_once, object_hook, object_pairs_hook,
if s[end:end + 1] != ':':
end = _w(s, end).end()
if s[end:end + 1] != ':':
- raise ValueError(errmsg("Expecting ':' delimiter", s, end))
+ raise JSONDecodeError("Expecting ':' delimiter", s, end)
end += 1
try:
@@ -188,7 +184,7 @@ def JSONObject(s_and_end, strict, scan_once, object_hook, object_pairs_hook,
try:
value, end = scan_once(s, end)
except StopIteration as err:
- raise ValueError(errmsg("Expecting value", s, err.value)) from None
+ raise JSONDecodeError("Expecting value", s, err.value) from None
pairs_append((key, value))
try:
nextchar = s[end]
@@ -202,13 +198,13 @@ def JSONObject(s_and_end, strict, scan_once, object_hook, object_pairs_hook,
if nextchar == '}':
break
elif nextchar != ',':
- raise ValueError(errmsg("Expecting ',' delimiter", s, end - 1))
+ raise JSONDecodeError("Expecting ',' delimiter", s, end - 1)
end = _w(s, end).end()
nextchar = s[end:end + 1]
end += 1
if nextchar != '"':
- raise ValueError(errmsg(
- "Expecting property name enclosed in double quotes", s, end - 1))
+ raise JSONDecodeError(
+ "Expecting property name enclosed in double quotes", s, end - 1)
if object_pairs_hook is not None:
result = object_pairs_hook(pairs)
return result, end
@@ -232,7 +228,7 @@ def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
try:
value, end = scan_once(s, end)
except StopIteration as err:
- raise ValueError(errmsg("Expecting value", s, err.value)) from None
+ raise JSONDecodeError("Expecting value", s, err.value) from None
_append(value)
nextchar = s[end:end + 1]
if nextchar in _ws:
@@ -242,7 +238,7 @@ def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
if nextchar == ']':
break
elif nextchar != ',':
- raise ValueError(errmsg("Expecting ',' delimiter", s, end - 1))
+ raise JSONDecodeError("Expecting ',' delimiter", s, end - 1)
try:
if s[end] in _ws:
end += 1
@@ -343,7 +339,7 @@ class JSONDecoder(object):
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
end = _w(s, end).end()
if end != len(s):
- raise ValueError(errmsg("Extra data", s, end, len(s)))
+ raise JSONDecodeError("Extra data", s, end)
return obj
def raw_decode(self, s, idx=0):
@@ -358,5 +354,5 @@ class JSONDecoder(object):
try:
obj, end = self.scan_once(s, idx)
except StopIteration as err:
- raise ValueError(errmsg("Expecting value", s, err.value)) from None
+ raise JSONDecodeError("Expecting value", s, err.value) from None
return obj, end
diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py
index 0513838..26e9eb2 100644
--- a/Lib/json/encoder.py
+++ b/Lib/json/encoder.py
@@ -7,6 +7,10 @@ try:
except ImportError:
c_encode_basestring_ascii = None
try:
+ from _json import encode_basestring as c_encode_basestring
+except ImportError:
+ c_encode_basestring = None
+try:
from _json import make_encoder as c_make_encoder
except ImportError:
c_make_encoder = None
@@ -30,7 +34,7 @@ for i in range(0x20):
INFINITY = float('inf')
FLOAT_REPR = repr
-def encode_basestring(s):
+def py_encode_basestring(s):
"""Return a JSON representation of a Python string
"""
@@ -39,6 +43,9 @@ def encode_basestring(s):
return '"' + ESCAPE.sub(replace, s) + '"'
+encode_basestring = (c_encode_basestring or py_encode_basestring)
+
+
def py_encode_basestring_ascii(s):
"""Return an ASCII-only JSON representation of a Python string
diff --git a/Lib/json/tool.py b/Lib/json/tool.py
index 7db4528..4f3182c 100644
--- a/Lib/json/tool.py
+++ b/Lib/json/tool.py
@@ -10,28 +10,39 @@ Usage::
Expecting property name enclosed in double quotes: line 1 column 3 (char 2)
"""
-import sys
+import argparse
+import collections
import json
+import sys
+
def main():
- if len(sys.argv) == 1:
- infile = sys.stdin
- outfile = sys.stdout
- elif len(sys.argv) == 2:
- infile = open(sys.argv[1], 'r')
- outfile = sys.stdout
- elif len(sys.argv) == 3:
- infile = open(sys.argv[1], 'r')
- outfile = open(sys.argv[2], 'w')
- else:
- raise SystemExit(sys.argv[0] + " [infile [outfile]]")
+ prog = 'python -m json.tool'
+ description = ('A simple command line interface for json module '
+ 'to validate and pretty-print JSON objects.')
+ parser = argparse.ArgumentParser(prog=prog, description=description)
+ parser.add_argument('infile', nargs='?', type=argparse.FileType(),
+ help='a JSON file to be validated or pretty-printed')
+ parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'),
+ help='write the output of infile to outfile')
+ parser.add_argument('--sort-keys', action='store_true', default=False,
+ help='sort the output of dictionaries alphabetically by key')
+ options = parser.parse_args()
+
+ infile = options.infile or sys.stdin
+ outfile = options.outfile or sys.stdout
+ sort_keys = options.sort_keys
with infile:
try:
- obj = json.load(infile)
+ if sort_keys:
+ obj = json.load(infile)
+ else:
+ obj = json.load(infile,
+ object_pairs_hook=collections.OrderedDict)
except ValueError as e:
raise SystemExit(e)
with outfile:
- json.dump(obj, outfile, sort_keys=True, indent=4)
+ json.dump(obj, outfile, sort_keys=sort_keys, indent=4)
outfile.write('\n')
diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt
index e667bcd..c954669 100644
--- a/Lib/lib2to3/Grammar.txt
+++ b/Lib/lib2to3/Grammar.txt
@@ -33,7 +33,8 @@ eval_input: testlist NEWLINE* ENDMARKER
decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
decorators: decorator+
-decorated: decorators (classdef | funcdef)
+decorated: decorators (classdef | funcdef | async_funcdef)
+async_funcdef: ASYNC funcdef
funcdef: 'def' NAME parameters ['->' test] ':' suite
parameters: '(' [typedargslist] ')'
typedargslist: ((tfpdef ['=' test] ',')*
@@ -82,7 +83,8 @@ global_stmt: ('global' | 'nonlocal') NAME (',' NAME)*
exec_stmt: 'exec' expr ['in' test [',' test]]
assert_stmt: 'assert' test [',' test]
-compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
+compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
+async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
while_stmt: 'while' test ':' suite ['else' ':' suite]
for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
@@ -121,7 +123,7 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)*
arith_expr: term (('+'|'-') term)*
term: factor (('*'|'@'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power
-power: atom trailer* ['**' factor]
+power: [AWAIT] atom trailer* ['**' factor]
atom: ('(' [yield_expr|testlist_gexp] ')' |
'[' [listmaker] ']' |
'{' [dictsetmaker] '}' |
@@ -142,7 +144,7 @@ dictsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
arglist: (argument ',')* (argument [',']
- |'*' test (',' argument)* [',' '**' test]
+ |'*' test (',' argument)* [',' '**' test]
|'**' test)
argument: test [comp_for] | test '=' test # Really [keyword '='] test
diff --git a/Lib/lib2to3/pgen2/token.py b/Lib/lib2to3/pgen2/token.py
index 7599396..1a67955 100755
--- a/Lib/lib2to3/pgen2/token.py
+++ b/Lib/lib2to3/pgen2/token.py
@@ -62,8 +62,10 @@ OP = 52
COMMENT = 53
NL = 54
RARROW = 55
-ERRORTOKEN = 56
-N_TOKENS = 57
+AWAIT = 56
+ASYNC = 57
+ERRORTOKEN = 58
+N_TOKENS = 59
NT_OFFSET = 256
#--end constants--
diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py
index 3dd1ee9..1ff1c61 100644
--- a/Lib/lib2to3/pgen2/tokenize.py
+++ b/Lib/lib2to3/pgen2/tokenize.py
@@ -220,7 +220,7 @@ class Untokenizer:
for tok in iterable:
toknum, tokval = tok[:2]
- if toknum in (NAME, NUMBER):
+ if toknum in (NAME, NUMBER, ASYNC, AWAIT):
tokval += ' '
if toknum == INDENT:
@@ -366,6 +366,12 @@ def generate_tokens(readline):
contline = None
indents = [0]
+ # 'stashed' and 'async_*' are used for async/await parsing
+ stashed = None
+ async_def = False
+ async_def_indent = 0
+ async_def_nl = False
+
while 1: # loop over lines in stream
try:
line = readline()
@@ -406,6 +412,10 @@ def generate_tokens(readline):
pos = pos + 1
if pos == max: break
+ if stashed:
+ yield stashed
+ stashed = None
+
if line[pos] in '#\r\n': # skip comments or blank lines
if line[pos] == '#':
comment_token = line[pos:].rstrip('\r\n')
@@ -428,8 +438,19 @@ def generate_tokens(readline):
"unindent does not match any outer indentation level",
("<tokenize>", lnum, pos, line))
indents = indents[:-1]
+
+ if async_def and async_def_indent >= indents[-1]:
+ async_def = False
+ async_def_nl = False
+ async_def_indent = 0
+
yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
+ if async_def and async_def_nl and async_def_indent >= indents[-1]:
+ async_def = False
+ async_def_nl = False
+ async_def_indent = 0
+
else: # continued statement
if not line:
raise TokenError("EOF in multi-line statement", (lnum, 0))
@@ -449,9 +470,18 @@ def generate_tokens(readline):
newline = NEWLINE
if parenlev > 0:
newline = NL
+ elif async_def:
+ async_def_nl = True
+ if stashed:
+ yield stashed
+ stashed = None
yield (newline, token, spos, epos, line)
+
elif initial == '#':
assert not token.endswith("\n")
+ if stashed:
+ yield stashed
+ stashed = None
yield (COMMENT, token, spos, epos, line)
elif token in triple_quoted:
endprog = endprogs[token]
@@ -459,6 +489,9 @@ def generate_tokens(readline):
if endmatch: # all on one line
pos = endmatch.end(0)
token = line[start:pos]
+ if stashed:
+ yield stashed
+ stashed = None
yield (STRING, token, spos, (lnum, pos), line)
else:
strstart = (lnum, start) # multiple lines
@@ -476,22 +509,63 @@ def generate_tokens(readline):
contline = line
break
else: # ordinary string
+ if stashed:
+ yield stashed
+ stashed = None
yield (STRING, token, spos, epos, line)
elif initial in namechars: # ordinary name
- yield (NAME, token, spos, epos, line)
+ if token in ('async', 'await'):
+ if async_def:
+ yield (ASYNC if token == 'async' else AWAIT,
+ token, spos, epos, line)
+ continue
+
+ tok = (NAME, token, spos, epos, line)
+ if token == 'async' and not stashed:
+ stashed = tok
+ continue
+
+ if token == 'def':
+ if (stashed
+ and stashed[0] == NAME
+ and stashed[1] == 'async'):
+
+ async_def = True
+ async_def_indent = indents[-1]
+
+ yield (ASYNC, stashed[1],
+ stashed[2], stashed[3],
+ stashed[4])
+ stashed = None
+
+ if stashed:
+ yield stashed
+ stashed = None
+
+ yield tok
elif initial == '\\': # continued stmt
# This yield is new; needed for better idempotency:
+ if stashed:
+ yield stashed
+ stashed = None
yield (NL, token, spos, (lnum, pos), line)
continued = 1
else:
if initial in '([{': parenlev = parenlev + 1
elif initial in ')]}': parenlev = parenlev - 1
+ if stashed:
+ yield stashed
+ stashed = None
yield (OP, token, spos, epos, line)
else:
yield (ERRORTOKEN, line[pos],
(lnum, pos), (lnum, pos+1), line)
pos = pos + 1
+ if stashed:
+ yield stashed
+ stashed = None
+
for indent in indents[1:]: # pop remaining indent levels
yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py
index 5bb9d2b..b533c01 100644
--- a/Lib/lib2to3/tests/test_parser.py
+++ b/Lib/lib2to3/tests/test_parser.py
@@ -55,12 +55,64 @@ class TestMatrixMultiplication(GrammarTest):
class TestYieldFrom(GrammarTest):
- def test_matrix_multiplication_operator(self):
+ def test_yield_from(self):
self.validate("yield from x")
self.validate("(yield from x) + y")
self.invalid_syntax("yield from")
+class TestAsyncAwait(GrammarTest):
+ def test_await_expr(self):
+ self.validate("""async def foo():
+ await x
+ """)
+
+ self.validate("""async def foo():
+
+ def foo(): pass
+
+ def foo(): pass
+
+ await x
+ """)
+
+ self.validate("""async def foo(): return await a""")
+
+ self.validate("""def foo():
+ def foo(): pass
+ async def foo(): await x
+ """)
+
+ self.invalid_syntax("await x")
+ self.invalid_syntax("""def foo():
+ await x""")
+
+ self.invalid_syntax("""def foo():
+ def foo(): pass
+ async def foo(): pass
+ await x
+ """)
+
+ def test_async_var(self):
+ self.validate("""async = 1""")
+ self.validate("""await = 1""")
+ self.validate("""def async(): pass""")
+
+ def test_async_with(self):
+ self.validate("""async def foo():
+ async for a in b: pass""")
+
+ self.invalid_syntax("""def foo():
+ async for a in b: pass""")
+
+ def test_async_for(self):
+ self.validate("""async def foo():
+ async with a: pass""")
+
+ self.invalid_syntax("""def foo():
+ async with a: pass""")
+
+
class TestRaiseChanges(GrammarTest):
def test_2x_style_1(self):
self.validate("raise")
diff --git a/Lib/linecache.py b/Lib/linecache.py
index 884cbf4..3afcce1 100644
--- a/Lib/linecache.py
+++ b/Lib/linecache.py
@@ -5,6 +5,7 @@ is not found, it will look down the module search path for a file by
that name.
"""
+import functools
import sys
import os
import tokenize
@@ -21,7 +22,9 @@ def getline(filename, lineno, module_globals=None):
# The cache
-cache = {} # The cache
+# The cache. Maps filenames to either a thunk which will provide source code,
+# or a tuple (size, mtime, lines, fullname) once loaded.
+cache = {}
def clearcache():
@@ -36,7 +39,9 @@ def getlines(filename, module_globals=None):
Update the cache if it doesn't contain an entry for this file already."""
if filename in cache:
- return cache[filename][2]
+ entry = cache[filename]
+ if len(entry) != 1:
+ return cache[filename][2]
try:
return updatecache(filename, module_globals)
@@ -58,7 +63,11 @@ def checkcache(filename=None):
return
for filename in filenames:
- size, mtime, lines, fullname = cache[filename]
+ entry = cache[filename]
+ if len(entry) == 1:
+ # lazy cache entry, leave it lazy.
+ continue
+ size, mtime, lines, fullname = entry
if mtime is None:
continue # no-op for files loaded via a __loader__
try:
@@ -76,7 +85,8 @@ def updatecache(filename, module_globals=None):
and return an empty list."""
if filename in cache:
- del cache[filename]
+ if len(cache[filename]) != 1:
+ del cache[filename]
if not filename or (filename.startswith('<') and filename.endswith('>')):
return []
@@ -86,27 +96,23 @@ def updatecache(filename, module_globals=None):
except OSError:
basename = filename
- # Try for a __loader__, if available
- if module_globals and '__loader__' in module_globals:
- name = module_globals.get('__name__')
- loader = module_globals['__loader__']
- get_source = getattr(loader, 'get_source', None)
-
- if name and get_source:
- try:
- data = get_source(name)
- except (ImportError, OSError):
- pass
- else:
- if data is None:
- # No luck, the PEP302 loader cannot find the source
- # for this module.
- return []
- cache[filename] = (
- len(data), None,
- [line+'\n' for line in data.splitlines()], fullname
- )
- return cache[filename][2]
+ # Realise a lazy loader based lookup if there is one
+ # otherwise try to lookup right now.
+ if lazycache(filename, module_globals):
+ try:
+ data = cache[filename][0]()
+ except (ImportError, OSError):
+ pass
+ else:
+ if data is None:
+ # No luck, the PEP302 loader cannot find the source
+ # for this module.
+ return []
+ cache[filename] = (
+ len(data), None,
+ [line+'\n' for line in data.splitlines()], fullname
+ )
+ return cache[filename][2]
# Try looking through the module search path, which is only useful
# when handling a relative filename.
@@ -136,3 +142,36 @@ def updatecache(filename, module_globals=None):
size, mtime = stat.st_size, stat.st_mtime
cache[filename] = size, mtime, lines, fullname
return lines
+
+
+def lazycache(filename, module_globals):
+ """Seed the cache for filename with module_globals.
+
+ The module loader will be asked for the source only when getlines is
+ called, not immediately.
+
+ If there is an entry in the cache already, it is not altered.
+
+ :return: True if a lazy load is registered in the cache,
+ otherwise False. To register such a load a module loader with a
+ get_source method must be found, the filename must be a cachable
+ filename, and the filename must not be already cached.
+ """
+ if filename in cache:
+ if len(cache[filename]) == 1:
+ return True
+ else:
+ return False
+ if not filename or (filename.startswith('<') and filename.endswith('>')):
+ return False
+ # Try for a __loader__, if available
+ if module_globals and '__loader__' in module_globals:
+ name = module_globals.get('__name__')
+ loader = module_globals['__loader__']
+ get_source = getattr(loader, 'get_source', None)
+
+ if name and get_source:
+ get_lines = functools.partial(get_source, name)
+ cache[filename] = (get_lines,)
+ return True
+ return False
diff --git a/Lib/locale.py b/Lib/locale.py
index 7ff4356..ceaa6d8 100644
--- a/Lib/locale.py
+++ b/Lib/locale.py
@@ -301,8 +301,8 @@ def str(val):
"""Convert float to integer, taking the locale into account."""
return format("%.12g", val)
-def atof(string, func=float):
- "Parses a string as a float according to the locale settings."
+def delocalize(string):
+ "Parses a string as a normalized number according to the locale settings."
#First, get rid of the grouping
ts = localeconv()['thousands_sep']
if ts:
@@ -311,12 +311,15 @@ def atof(string, func=float):
dd = localeconv()['decimal_point']
if dd:
string = string.replace(dd, '.')
- #finally, parse the string
- return func(string)
+ return string
+
+def atof(string, func=float):
+ "Parses a string as a float according to the locale settings."
+ return func(delocalize(string))
-def atoi(str):
+def atoi(string):
"Converts a string to an integer according to the locale settings."
- return atof(str, int)
+ return int(delocalize(string))
def _test():
setlocale(LC_ALL, "")
@@ -696,7 +699,9 @@ locale_encoding_alias = {
'euc_kr': 'eucKR',
'utf_8': 'UTF-8',
'koi8_r': 'KOI8-R',
+ 'koi8_t': 'KOI8-T',
'koi8_u': 'KOI8-U',
+ 'kz1048': 'RK1048',
'cp1251': 'CP1251',
'cp1255': 'CP1255',
'cp1256': 'CP1256',
diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py
index 67d9d2e..104b0be 100644
--- a/Lib/logging/__init__.py
+++ b/Lib/logging/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2001-2014 by Vinay Sajip. All Rights Reserved.
+# Copyright 2001-2015 by Vinay Sajip. All Rights Reserved.
#
# Permission to use, copy, modify, and distribute this software and its
# documentation for any purpose and without fee is hereby granted,
@@ -18,7 +18,7 @@
Logging package for Python. Based on PEP 282 and comments thereto in
comp.lang.python.
-Copyright (C) 2001-2014 Vinay Sajip. All Rights Reserved.
+Copyright (C) 2001-2015 Vinay Sajip. All Rights Reserved.
To use, simply 'import logging' and log away!
"""
@@ -316,6 +316,8 @@ class LogRecord(object):
return '<LogRecord: %s, %s, %s, %s, "%s">'%(self.name, self.levelno,
self.pathname, self.lineno, self.msg)
+ __repr__ = __str__
+
def getMessage(self):
"""
Return the message for this LogRecord.
@@ -1091,7 +1093,6 @@ class PlaceHolder(object):
#
# Determine which class to use when instantiating loggers.
#
-_loggerClass = None
def setLoggerClass(klass):
"""
@@ -1110,7 +1111,6 @@ def getLoggerClass():
"""
Return the class to be used when instantiating a logger.
"""
-
return _loggerClass
class Manager(object):
@@ -1307,12 +1307,11 @@ class Logger(Filterer):
if self.isEnabledFor(ERROR):
self._log(ERROR, msg, args, **kwargs)
- def exception(self, msg, *args, **kwargs):
+ def exception(self, msg, *args, exc_info=True, **kwargs):
"""
Convenience method for logging an ERROR with exception information.
"""
- kwargs['exc_info'] = True
- self.error(msg, *args, **kwargs)
+ self.error(msg, *args, exc_info=exc_info, **kwargs)
def critical(self, msg, *args, **kwargs):
"""
@@ -1407,7 +1406,9 @@ class Logger(Filterer):
else: # pragma: no cover
fn, lno, func = "(unknown file)", 0, "(unknown function)"
if exc_info:
- if not isinstance(exc_info, tuple):
+ if isinstance(exc_info, BaseException):
+ exc_info = (type(exc_info), exc_info, exc_info.__traceback__)
+ elif not isinstance(exc_info, tuple):
exc_info = sys.exc_info()
record = self.makeRecord(self.name, level, fn, lno, msg, args,
exc_info, func, extra, sinfo)
@@ -1617,12 +1618,11 @@ class LoggerAdapter(object):
"""
self.log(ERROR, msg, *args, **kwargs)
- def exception(self, msg, *args, **kwargs):
+ def exception(self, msg, *args, exc_info=True, **kwargs):
"""
Delegate an exception call to the underlying logger.
"""
- kwargs["exc_info"] = True
- self.log(ERROR, msg, *args, **kwargs)
+ self.log(ERROR, msg, *args, exc_info=exc_info, **kwargs)
def critical(self, msg, *args, **kwargs):
"""
@@ -1804,14 +1804,13 @@ def error(msg, *args, **kwargs):
basicConfig()
root.error(msg, *args, **kwargs)
-def exception(msg, *args, **kwargs):
+def exception(msg, *args, exc_info=True, **kwargs):
"""
Log a message with severity 'ERROR' on the root logger, with exception
information. If the logger has no handlers, basicConfig() is called to add
a console handler with a pre-defined format.
"""
- kwargs['exc_info'] = True
- error(msg, *args, **kwargs)
+ error(msg, *args, exc_info=exc_info, **kwargs)
def warning(msg, *args, **kwargs):
"""
diff --git a/Lib/logging/config.py b/Lib/logging/config.py
index 895fb26..8a99923 100644
--- a/Lib/logging/config.py
+++ b/Lib/logging/config.py
@@ -116,11 +116,12 @@ def _create_formatters(cp):
sectname = "formatter_%s" % form
fs = cp.get(sectname, "format", raw=True, fallback=None)
dfs = cp.get(sectname, "datefmt", raw=True, fallback=None)
+ stl = cp.get(sectname, "style", raw=True, fallback='%')
c = logging.Formatter
class_name = cp[sectname].get("class")
if class_name:
c = _resolve(class_name)
- f = c(fs, dfs)
+ f = c(fs, dfs, stl)
formatters[form] = f
return formatters
@@ -660,7 +661,12 @@ class DictConfigurator(BaseConfigurator):
fmt = config.get('format', None)
dfmt = config.get('datefmt', None)
style = config.get('style', '%')
- result = logging.Formatter(fmt, dfmt, style)
+ cname = config.get('class', None)
+ if not cname:
+ c = logging.Formatter
+ else:
+ c = _resolve(cname)
+ result = c(fmt, dfmt, style)
return result
def configure_filter(self, config):
diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py
index fda8093..02a5fc1 100644
--- a/Lib/logging/handlers.py
+++ b/Lib/logging/handlers.py